diff --git a/scripts/node_suite_regression_check.py b/scripts/node_suite_regression_check.py new file mode 100755 index 00000000000..3abf00ce8b4 --- /dev/null +++ b/scripts/node_suite_regression_check.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Regression guard for the print-and-diff node-suite. + +Runs `scripts/node_suite_run.py` (pre-warm + fast/slow lanes) and compares the +per-module pass counts against a committed floor baseline. FAILS (exit 1) if any +baselined module drops below its floor. Improvements are always accepted and are +reported as `+N` so the baseline can be ratcheted up over time. + +This exists because the node-suite is NOT part of the per-PR CI gate (the parity +job is opt-in and runs node 22, while the real oracle is node 26), so a module +can silently regress and still merge green — which is exactly how node:dns once +went 83% -> 0% behind a green build. Run this in the node-26 environment (the box) +on a schedule, or before cutting a release. + +Usage: + node_suite_regression_check.py [baseline.json] + +Exit codes: 0 = no regressions (improvements ok), 1 = at least one regression, + 2 = harness error (could not run / parse). +""" +import json +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO_DEFAULT = os.path.dirname(HERE) + + +def main(): + if len(sys.argv) < 3: + print(__doc__) + return 2 + perry, root = sys.argv[1], sys.argv[2] + baseline_path = sys.argv[3] if len(sys.argv) > 3 else os.path.join( + root, "test-parity", "node_suite_baseline.json") + + if not os.path.exists(baseline_path): + print(f"ERROR: baseline not found: {baseline_path}", file=sys.stderr) + return 2 + baseline = json.load(open(baseline_path)).get("modules", {}) + + runner = os.path.join(root, "scripts", "node_suite_run.py") + proc = subprocess.run([sys.executable, runner, perry, root], + capture_output=True, text=True) + sys.stderr.write(proc.stderr) + print(proc.stdout) + # Fail closed: any non-zero runner exit means we cannot trust the table + # (crash/timeout could leave partial output), so don't risk parsing it. + if proc.returncode != 0: + print(f"ERROR: runner exited {proc.returncode}", file=sys.stderr) + return 2 + + # Parse "module pass total %" rows from the runner table. + # The header row ("module pass total %") can't match because pass/total + # are not digits, so no name-based exclusion is needed — and excluding the + # name "module" would wrongly drop the real node:module module. + current = {} + for line in proc.stdout.splitlines(): + m = re.match(r"^(\S+)\s+(\d+)\s+(\d+)\s+[\d.]+", line) + if m: + current[m.group(1)] = {"pass": int(m.group(2)), "total": int(m.group(3))} + + regressions, improvements = [], [] + for mod, floor in baseline.items(): + cur = current.get(mod) + if cur is None: + regressions.append(f"{mod}: MISSING from run (was {floor['pass']}/{floor['total']})") + continue + if cur["pass"] < floor["pass"]: + regressions.append( + f"{mod}: {cur['pass']}/{cur['total']} < floor {floor['pass']}/{floor['total']} (-{floor['pass'] - cur['pass']})") + elif cur["pass"] > floor["pass"]: + improvements.append(f"{mod}: {cur['pass']}/{cur['total']} (+{cur['pass'] - floor['pass']})") + + print("\n=== node-suite regression check ===") + if improvements: + print("improvements (ratchet the baseline up):") + for s in improvements: + print(" + " + s) + if regressions: + print("REGRESSIONS:") + for s in regressions: + print(" ! " + s) + return 1 + print("OK — no module dropped below its floor.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/node_suite_run.py b/scripts/node_suite_run.py new file mode 100755 index 00000000000..92fd3c71e11 --- /dev/null +++ b/scripts/node_suite_run.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Differential runner for the print-and-diff node-suite (test-parity/node-suite). + +For every `test-parity/node-suite//**/*.ts`, run `node ` and +`perry -o out && out`, then compare stdout (trailing whitespace ignored). +Prints a per-module pass/total table plus an overall figure. + +Two correctness measures learned the hard way (see CHANGELOG / project memory): + +1. Pre-warm pass — compile one test per module SERIALLY first so each module's + auto-optimize runtime/stdlib cache (e.g. the crypto feature) is built before + the timed run. Otherwise the first test of a crypto-feature module eats a + multi-minute cold rebuild that blows the per-test compile timeout and the + whole module is scored as `perry_err` (this once made dns look 0% and http + 47% when both are actually 100%). + +2. Low-concurrency lane — server/timing modules bind ports, spawn processes, or + assert on event-loop/timer ordering. Under the wide parallel pool they suffer + port contention and timing races, producing false `perry_err`/`diff`. They + run STRICTLY SEQUENTIALLY so their numbers are trustworthy; everything else + stays parallel. + +Usage: node_suite_run.py [comma-separated-modules] +""" +import os, subprocess, sys, tempfile +from concurrent.futures import ThreadPoolExecutor +from collections import defaultdict + +PERRY = sys.argv[1] +ROOT = sys.argv[2] +MODS = sys.argv[3].split(",") if len(sys.argv) > 3 and sys.argv[3] else None +NODE = os.environ.get("NODE_BIN", "node") + +# Modules that must run one-at-a-time (port binding / process spawn / event-loop +# or timer ordering). Parallelism corrupts their results. +SLOW_MODULES = { + "http", "http2", "https", "net", "dgram", "tls", "cluster", "dns", + "stream", "child_process", "worker_threads", "inspector", + "inspector-promises", "repl", "diagnostics_channel", "timers", "fetch", +} + +tests = [] +base = os.path.join(ROOT, "test-parity", "node-suite") +for mod in (MODS or sorted(os.listdir(base))): + md = os.path.join(base, mod) + if not os.path.isdir(md): + continue + for dp, _, files in os.walk(md): + for f in files: + if f.endswith(".ts") and not f.endswith(".d.ts"): + tests.append((mod, os.path.join(dp, f))) + + +def run_one(args): + mod, path = args + try: + n = subprocess.run([NODE, path], capture_output=True, text=True, timeout=30) + except Exception: + return (mod, "node_err") + # A non-zero node exit can be intentional (the test exercises an error path), + # so we don't bucket it as node_err; we require Perry to match BOTH stdout and + # the exit code below, which keeps genuine error-path parity counted as pass. + with tempfile.TemporaryDirectory() as td: + out = os.path.join(td, "o") + try: + c = subprocess.run([PERRY, path, "-o", out], capture_output=True, text=True, timeout=120) + if c.returncode != 0: + return (mod, "compile_fail") + p = subprocess.run([out], capture_output=True, text=True, timeout=30) + except Exception: + return (mod, "perry_err") + # Match stdout byte-for-byte (ignore only trailing-newline noise, not leading + # whitespace) AND exit code — so a Perry crash that happened to print matching + # output before dying is a diff, not a false pass. + ok = (n.stdout.rstrip("\n") == p.stdout.rstrip("\n")) and (n.returncode == p.returncode) + return (mod, "pass" if ok else "diff") + + +# --- pre-warm one test per module serially --- +seen = set() +warm = [t for t in tests if t[0] not in seen and not seen.add(t[0])] +sys.stderr.write(f"pre-warming auto-opt cache for {len(warm)} module(s)...\n") +sys.stderr.flush() +for mod, path in warm: + with tempfile.TemporaryDirectory() as td: + try: + subprocess.run([PERRY, path, "-o", os.path.join(td, "o")], capture_output=True, text=True, timeout=600) + except Exception: + pass +sys.stderr.write("pre-warm done\n") +sys.stderr.flush() + +# --- fast lane (parallel) + slow lane (sequential) --- +fast = [t for t in tests if t[0] not in SLOW_MODULES] +slow = [t for t in tests if t[0] in SLOW_MODULES] +res = defaultdict(lambda: defaultdict(int)) +sys.stderr.write(f"fast lane: {len(fast)} tests @6, slow lane: {len(slow)} tests @1\n") +sys.stderr.flush() +with ThreadPoolExecutor(max_workers=6) as ex: + for mod, outcome in ex.map(run_one, fast): + res[mod][outcome] += 1 +for t in slow: + mod, outcome = run_one(t) + res[mod][outcome] += 1 + +# --- report --- +tot_p = tot = 0 +print("%-20s %6s %6s %5s" % ("module", "pass", "total", "%")) +for mod in sorted(res): + a = res[mod] + p = a.get("pass", 0) + t = sum(a.values()) + tot_p += p + tot += t + extra = " ".join(f"{k}={v}" for k, v in a.items() if k != "pass" and v) + print("%-20s %6d %6d %5.1f %s" % (mod, p, t, 100 * p / t if t else 0, extra)) +print("-" * 44) +print("OVERALL node-suite: %d/%d (%.1f%%)" % (tot_p, tot, 100 * tot_p / tot if tot else 0)) diff --git a/test-parity/node_suite_baseline.json b/test-parity/node_suite_baseline.json new file mode 100644 index 00000000000..7efc2971fd2 --- /dev/null +++ b/test-parity/node_suite_baseline.json @@ -0,0 +1,63 @@ +{ + "_schema": { + "description": "Floor baseline for scripts/node_suite_regression_check.py. Each module's run must produce pass >= floor.pass; dropping below is a regression (exit 1). Improvements are always accepted and reported as ratchet candidates. Captured in the node-26 environment with scripts/node_suite_run.py (pre-warm + fast/slow lanes).", + "oracle": "node v26.3.0 on Linux (the box)", + "note": "Deterministic modules are floored at full pass. Timing/racy modules (http2, net, stream, diagnostics_channel, fs-promises) carry a small margin below observed pass so ordinary flake does not false-alarm; the guard still catches real regressions, which are large (e.g. dns 6->0, http 19->9)." + }, + "overall": { "pass": 2781, "total": 2863, "pct": 97.1 }, + "modules": { + "assert": { "pass": 70, "total": 70 }, + "async_hooks": { "pass": 5, "total": 5 }, + "bigint": { "pass": 3, "total": 3 }, + "buffer": { "pass": 134, "total": 134 }, + "child_process": { "pass": 26, "total": 26 }, + "cluster": { "pass": 1, "total": 1 }, + "console": { "pass": 108, "total": 119 }, + "constants": { "pass": 4, "total": 4 }, + "crypto": { "pass": 240, "total": 242 }, + "dgram": { "pass": 4, "total": 4 }, + "diagnostics_channel": { "pass": 65, "total": 69 }, + "dns": { "pass": 6, "total": 6 }, + "domain": { "pass": 3, "total": 3 }, + "events": { "pass": 65, "total": 69 }, + "fetch": { "pass": 12, "total": 12 }, + "fs": { "pass": 168, "total": 175 }, + "fs-promises": { "pass": 76, "total": 82 }, + "globals": { "pass": 107, "total": 115 }, + "http": { "pass": 19, "total": 19 }, + "http2": { "pass": 8, "total": 9 }, + "https": { "pass": 5, "total": 5 }, + "inspector": { "pass": 2, "total": 3 }, + "inspector-promises": { "pass": 1, "total": 1 }, + "module": { "pass": 28, "total": 28 }, + "net": { "pass": 12, "total": 13 }, + "node-core": { "pass": 2, "total": 2 }, + "object": { "pass": 22, "total": 23 }, + "os": { "pass": 39, "total": 39 }, + "path": { "pass": 92, "total": 92 }, + "perf_hooks": { "pass": 83, "total": 83 }, + "process": { "pass": 95, "total": 96 }, + "punycode": { "pass": 8, "total": 8 }, + "querystring": { "pass": 59, "total": 59 }, + "readline": { "pass": 12, "total": 12 }, + "repl": { "pass": 4, "total": 4 }, + "sea": { "pass": 1, "total": 1 }, + "sqlite": { "pass": 1, "total": 1 }, + "stream": { "pass": 760, "total": 801 }, + "string": { "pass": 2, "total": 2 }, + "string_decoder": { "pass": 36, "total": 36 }, + "sys": { "pass": 2, "total": 2 }, + "test": { "pass": 7, "total": 11 }, + "timers": { "pass": 90, "total": 90 }, + "tls": { "pass": 3, "total": 3 }, + "trace_events": { "pass": 6, "total": 6 }, + "tty": { "pass": 31, "total": 32 }, + "url": { "pass": 67, "total": 67 }, + "util": { "pass": 84, "total": 86 }, + "v8": { "pass": 4, "total": 4 }, + "vm": { "pass": 8, "total": 8 }, + "wasi": { "pass": 3, "total": 3 }, + "worker_threads": { "pass": 17, "total": 17 }, + "zlib": { "pass": 58, "total": 58 } + } +}