Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 40 additions & 9 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -693,14 +693,20 @@ jobs:
# any failure NOT already triaged in test-parity/known_failures.json
# (run_gap_tests.sh's no-new-untriaged gate).
#
# MEASUREMENT PHASE: deliberately not yet in the branch-protection
# required set. Once wall-time on real PRs is confirmed acceptable, add
# `conformance-smoke` to the required contexts; until then a red here is
# a loud non-blocking signal.
continue-on-error: true
# Sharded across 8 parallel runners (--shard N/8) so the ~258-test suite
# runs in ~1/8 the wall-time — the 258 sequential Perry compiles dominate
# the cost and split evenly, taking the single-job ~57 min down to ~15 min
# per shard. The fan-in job `conformance-smoke-complete` below is the ONE
# status branch protection requires; a single shard's red bubbles up
# through it. Each shard's no-new-untriaged gate (run_gap_tests.sh) covers
# only its own slice, which is exactly the right per-shard semantics.
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5, 6, 7, 8]
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 60
timeout-minutes: 40
steps:
- uses: actions/checkout@v7

Expand All @@ -717,16 +723,41 @@ jobs:
with:
node-version: '22'

- name: Run gap suite
run: ./scripts/run_gap_tests.sh
- name: Run gap suite (shard ${{ matrix.shard }}/8)
run: ./scripts/run_gap_tests.sh --shard ${{ matrix.shard }}/8

- name: Upload gap report
if: always()
uses: actions/upload-artifact@v7
with:
name: gap-suite-report
name: gap-suite-report-shard-${{ matrix.shard }}
path: test-parity/reports/

# ---------------------------------------------------------------------------
# Fan-in for the sharded gap suite. `needs.conformance-smoke.result` is
# "success" only when EVERY shard succeeded, so this single context is a
# faithful all-shards-green gate — the one branch protection requires.
# `if: always()` guarantees it runs and reports even when a shard fails
# (a required check that got *skipped* would otherwise wedge the PR on
# "Expected"); the event guard mirrors the shards so it no-ops off the PR
# path just like they do.
# ---------------------------------------------------------------------------
conformance-smoke-complete:
needs: conformance-smoke
if: always() && (github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch')
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Require every gap-suite shard to pass
run: |
result='${{ needs.conformance-smoke.result }}'
echo "conformance-smoke (all 8 shards) => $result"
if [[ "$result" != "success" ]]; then
echo "::error::gap suite failed, was cancelled, or was skipped in one or more shards"
exit 1
fi
echo "All gap-suite shards passed."

# ---------------------------------------------------------------------------
# Parity tests (Perry output vs Node.js)
# ---------------------------------------------------------------------------
Expand Down
40 changes: 40 additions & 0 deletions run_parity_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ TEST_FILTER=""
# without requiring test_parity_* names.
TEST_SUITE="all"
MODULE_FILTER=""
# Optional round-robin sharding (N/M, 1-based). Splits the post-filter test
# set evenly across M parallel runners so a big suite (the gap suite is the
# motivating case — conformance-smoke fans out over 8 shards) can be run in
# ~1/M the wall-time. Empty = run the whole set (default; unchanged behavior
# for release parity, local runs, and node-suite-guard).
SHARD_SPEC=""
while [[ $# -gt 0 ]]; do
case "$1" in
--filter) TEST_FILTER="$2"; shift 2 ;;
Expand All @@ -39,10 +45,30 @@ while [[ $# -gt 0 ]]; do
--suite=*) TEST_SUITE="${1#--suite=}"; shift ;;
--module) MODULE_FILTER="$2"; shift 2 ;;
--module=*) MODULE_FILTER="${1#--module=}"; shift ;;
--shard) SHARD_SPEC="$2"; shift 2 ;;
--shard=*) SHARD_SPEC="${1#--shard=}"; shift ;;
*) shift ;;
esac
done

# Parse/validate the optional shard spec into 1-based index + total. Kept as a
# hard input check: a malformed shard (e.g. "3/0" or "9/8") that silently ran
# the whole set — or nothing — would make a required conformance gate lie.
SHARD_INDEX=0
SHARD_TOTAL=0
if [[ -n "$SHARD_SPEC" ]]; then
if [[ ! "$SHARD_SPEC" =~ ^[0-9]+/[0-9]+$ ]]; then
echo -e "\033[0;31mInvalid --shard '$SHARD_SPEC' (want N/M, 1-based, e.g. 3/8)\033[0m" >&2
exit 1
fi
SHARD_INDEX="${SHARD_SPEC%/*}"
SHARD_TOTAL="${SHARD_SPEC#*/}"
if (( SHARD_INDEX < 1 || SHARD_TOTAL < 1 || SHARD_INDEX > SHARD_TOTAL )); then
echo -e "\033[0;31mInvalid --shard bounds '$SHARD_SPEC' (need 1 <= N <= M)\033[0m" >&2
exit 1
fi
fi

case "$TEST_SUITE" in
all|parity|smoke|node-suite) ;;
*)
Expand Down Expand Up @@ -545,6 +571,7 @@ if [[ ${#TEST_FILES[@]} -eq 0 ]]; then
fi

# Run each test
SHARD_COUNTER=0
for test_file in "${TEST_FILES[@]}"; do
# Skip directories (multi/ folder)
[[ -d "$test_file" ]] && continue
Expand All @@ -563,6 +590,19 @@ for test_file in "${TEST_FILES[@]}"; do
continue
fi

# Round-robin sharding over the POST-filter set (opt-in via --shard N/M).
# Counting only tests that survive the filter keeps the split even for a
# narrow filter like `test_gap_`; the counter advances for every surviving
# test so shard i keeps exactly indices where (k % M) == i-1, and the M
# shards partition the set with no overlap and no gaps.
if (( SHARD_TOTAL > 0 )); then
this_idx=$SHARD_COUNTER
SHARD_COUNTER=$((SHARD_COUNTER + 1))
if (( this_idx % SHARD_TOTAL != SHARD_INDEX - 1 )); then
continue
fi
fi

safe_test_id="${test_id//\//__}"
node_output_file="$OUTPUT_DIR/node/${safe_test_id}.txt"
perry_output_file="$OUTPUT_DIR/perry/${safe_test_id}.txt"
Expand Down
4 changes: 3 additions & 1 deletion scripts/run_gap_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ echo "==> Running gap suite (test-files/test_gap_*.ts) via run_parity_tests.sh -
# run_parity_tests.sh exits 1 when AGGREGATE parity < 80%. We gate on "no NEW
# untriaged failures" instead (below), so don't let its aggregate exit abort us.
set +e
./run_parity_tests.sh --filter test_gap_
# Forward extra args (notably --shard N/M) so CI can fan the gap suite out
# across parallel runners; with no args this is the full serial gap suite.
./run_parity_tests.sh --filter test_gap_ "$@"
set -e

REPORT="test-parity/reports/latest.json"
Expand Down
Loading