From 21ceeec0bb89be03072e32b5dc95219cdbe05fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 6 Jul 2026 01:35:15 +0200 Subject: [PATCH] ci(#9): shard the gap-suite conformance gate across 8 runners + fan-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conformance-smoke job runs the full ~258-test gap suite serially in one job — ~57 min wall-time, which is why it was never promoted into the branch-protection required set (its own comment: "Once wall-time on real PRs is confirmed acceptable, add conformance-smoke to the required contexts"). The 258 sequential Perry compiles dominate that cost and split cleanly, so: - run_parity_tests.sh gains an opt-in `--shard N/M` (1-based) that round-robins the POST-filter test set across M runners. Counting only filter-surviving tests keeps the split even for a narrow filter like `test_gap_`; the M shards partition the set with no overlap and no gaps (verified: 258 items / 8 shards -> [33,33,32,32,32,32,32,32], each index covered exactly once). Malformed specs (9/8, 0/8, abc) hard-exit so a required gate can't silently run everything or nothing. Default (no flag) is byte-identical to today for release parity, local runs, node-suite-guard. - run_gap_tests.sh forwards extra args, so CI passes `--shard N/8`. - conformance-smoke becomes an 8-way matrix (~15 min/shard, fail-fast off). New fan-in job conformance-smoke-complete needs: it and is green only when every shard succeeded — the single context branch protection will require. `if: always()` so it still reports (not skip-wedges) a required PR. Wall-time drops ~57 min -> ~15 min, clearing the bar to promote the gate to required (a follow-up branch-protection change once a real PR confirms the sharded timing). --- .github/workflows/test.yml | 49 +++++++++++++++++++++++++++++++------- run_parity_tests.sh | 40 +++++++++++++++++++++++++++++++ scripts/run_gap_tests.sh | 4 +++- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e837ac8580e..5daef078bab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -693,14 +693,20 @@ jobs: # any failure NOT already triaged in test-parity/known_failures.json # (run_gap_tests.sh's no-new-untriaged gate). # - # MEASUREMENT PHASE: deliberately not yet in the branch-protection - # required set. Once wall-time on real PRs is confirmed acceptable, add - # `conformance-smoke` to the required contexts; until then a red here is - # a loud non-blocking signal. - continue-on-error: true + # Sharded across 8 parallel runners (--shard N/8) so the ~258-test suite + # runs in ~1/8 the wall-time — the 258 sequential Perry compiles dominate + # the cost and split evenly, taking the single-job ~57 min down to ~15 min + # per shard. The fan-in job `conformance-smoke-complete` below is the ONE + # status branch protection requires; a single shard's red bubbles up + # through it. Each shard's no-new-untriaged gate (run_gap_tests.sh) covers + # only its own slice, which is exactly the right per-shard semantics. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 40 steps: - uses: actions/checkout@v7 @@ -717,16 +723,41 @@ jobs: with: node-version: '22' - - name: Run gap suite - run: ./scripts/run_gap_tests.sh + - name: Run gap suite (shard ${{ matrix.shard }}/8) + run: ./scripts/run_gap_tests.sh --shard ${{ matrix.shard }}/8 - name: Upload gap report if: always() uses: actions/upload-artifact@v7 with: - name: gap-suite-report + name: gap-suite-report-shard-${{ matrix.shard }} path: test-parity/reports/ + # --------------------------------------------------------------------------- + # Fan-in for the sharded gap suite. `needs.conformance-smoke.result` is + # "success" only when EVERY shard succeeded, so this single context is a + # faithful all-shards-green gate — the one branch protection requires. + # `if: always()` guarantees it runs and reports even when a shard fails + # (a required check that got *skipped* would otherwise wedge the PR on + # "Expected"); the event guard mirrors the shards so it no-ops off the PR + # path just like they do. + # --------------------------------------------------------------------------- + conformance-smoke-complete: + needs: conformance-smoke + if: always() && (github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require every gap-suite shard to pass + run: | + result='${{ needs.conformance-smoke.result }}' + echo "conformance-smoke (all 8 shards) => $result" + if [[ "$result" != "success" ]]; then + echo "::error::gap suite failed, was cancelled, or was skipped in one or more shards" + exit 1 + fi + echo "All gap-suite shards passed." + # --------------------------------------------------------------------------- # Parity tests (Perry output vs Node.js) # --------------------------------------------------------------------------- diff --git a/run_parity_tests.sh b/run_parity_tests.sh index 6cf7929fcd7..356f9de1e04 100755 --- a/run_parity_tests.sh +++ b/run_parity_tests.sh @@ -31,6 +31,12 @@ TEST_FILTER="" # without requiring test_parity_* names. TEST_SUITE="all" MODULE_FILTER="" +# Optional round-robin sharding (N/M, 1-based). Splits the post-filter test +# set evenly across M parallel runners so a big suite (the gap suite is the +# motivating case — conformance-smoke fans out over 8 shards) can be run in +# ~1/M the wall-time. Empty = run the whole set (default; unchanged behavior +# for release parity, local runs, and node-suite-guard). +SHARD_SPEC="" while [[ $# -gt 0 ]]; do case "$1" in --filter) TEST_FILTER="$2"; shift 2 ;; @@ -39,10 +45,30 @@ while [[ $# -gt 0 ]]; do --suite=*) TEST_SUITE="${1#--suite=}"; shift ;; --module) MODULE_FILTER="$2"; shift 2 ;; --module=*) MODULE_FILTER="${1#--module=}"; shift ;; + --shard) SHARD_SPEC="$2"; shift 2 ;; + --shard=*) SHARD_SPEC="${1#--shard=}"; shift ;; *) shift ;; esac done +# Parse/validate the optional shard spec into 1-based index + total. Kept as a +# hard input check: a malformed shard (e.g. "3/0" or "9/8") that silently ran +# the whole set — or nothing — would make a required conformance gate lie. +SHARD_INDEX=0 +SHARD_TOTAL=0 +if [[ -n "$SHARD_SPEC" ]]; then + if [[ ! "$SHARD_SPEC" =~ ^[0-9]+/[0-9]+$ ]]; then + echo -e "\033[0;31mInvalid --shard '$SHARD_SPEC' (want N/M, 1-based, e.g. 3/8)\033[0m" >&2 + exit 1 + fi + SHARD_INDEX="${SHARD_SPEC%/*}" + SHARD_TOTAL="${SHARD_SPEC#*/}" + if (( SHARD_INDEX < 1 || SHARD_TOTAL < 1 || SHARD_INDEX > SHARD_TOTAL )); then + echo -e "\033[0;31mInvalid --shard bounds '$SHARD_SPEC' (need 1 <= N <= M)\033[0m" >&2 + exit 1 + fi +fi + case "$TEST_SUITE" in all|parity|smoke|node-suite) ;; *) @@ -545,6 +571,7 @@ if [[ ${#TEST_FILES[@]} -eq 0 ]]; then fi # Run each test +SHARD_COUNTER=0 for test_file in "${TEST_FILES[@]}"; do # Skip directories (multi/ folder) [[ -d "$test_file" ]] && continue @@ -563,6 +590,19 @@ for test_file in "${TEST_FILES[@]}"; do continue fi + # Round-robin sharding over the POST-filter set (opt-in via --shard N/M). + # Counting only tests that survive the filter keeps the split even for a + # narrow filter like `test_gap_`; the counter advances for every surviving + # test so shard i keeps exactly indices where (k % M) == i-1, and the M + # shards partition the set with no overlap and no gaps. + if (( SHARD_TOTAL > 0 )); then + this_idx=$SHARD_COUNTER + SHARD_COUNTER=$((SHARD_COUNTER + 1)) + if (( this_idx % SHARD_TOTAL != SHARD_INDEX - 1 )); then + continue + fi + fi + safe_test_id="${test_id//\//__}" node_output_file="$OUTPUT_DIR/node/${safe_test_id}.txt" perry_output_file="$OUTPUT_DIR/perry/${safe_test_id}.txt" diff --git a/scripts/run_gap_tests.sh b/scripts/run_gap_tests.sh index d2edbcc7f18..eabb24a1681 100755 --- a/scripts/run_gap_tests.sh +++ b/scripts/run_gap_tests.sh @@ -39,7 +39,9 @@ echo "==> Running gap suite (test-files/test_gap_*.ts) via run_parity_tests.sh - # run_parity_tests.sh exits 1 when AGGREGATE parity < 80%. We gate on "no NEW # untriaged failures" instead (below), so don't let its aggregate exit abort us. set +e -./run_parity_tests.sh --filter test_gap_ +# Forward extra args (notably --shard N/M) so CI can fan the gap suite out +# across parallel runners; with no args this is the full serial gap suite. +./run_parity_tests.sh --filter test_gap_ "$@" set -e REPORT="test-parity/reports/latest.json"