gh-109566, regrtest: Add --fast-ci and --slow-ci options

* Add --fast-ci and --slow-ci options to libregrtest:

  * --fast-ci uses a default timeout of 10 minutes and "-u all,-cpu"
    (skip slowest tests).
  * --slow-ci uses a default timeout of 20 minues and "-u all" (run
    all tests).

* Add "make fastcitest" target: similar to "make buildbottest", but
  use --fast-ci option instead of --slow-ci, and don't use
  TESTTIMEOUT.
* "make buildbottest" now uses "--slow-ci". Remove now redundant
  options.
* GitHub Actions workflow now uses "make fastcitest". Remove now
  redundant options. Just use -j0: detect the number of CPUs.

Author: vstinner

.github/workflows/build.yml

@@ -180,7 +180,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
 
   build_win_amd64:
     name: 'Windows (x64)'
@@ -199,7 +199,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
 
   build_macos:
     name: 'macOS'
@@ -235,7 +235,7 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: make fastcitest
 
   build_ubuntu:
     name: 'Ubuntu'
@@ -302,7 +302,7 @@ jobs:
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
     - name: Tests
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make fastcitest
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
@@ -518,7 +518,7 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make fastcitest
 
   all-required-green:  # This job does nothing and is only used for the branch protection
     name: All required checks pass

Doc/using/configure.rst

@@ -818,6 +818,9 @@ Main Makefile targets
 * ``make buildbottest``: Build Python and run the Python test suite, the same
   way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds)
   to change the test timeout (1200 by default: 20 minutes).
+* ``make fastcitest``: Build Python and run the Python test suite using
+  ``--fast-ci`` option.
+  test timeout (1200 by default: 20 minutes).
 * ``make install``: Build and install Python.
 * ``make regen-all``: Regenerate (almost) all generated files;
   ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately

Lib/test/libregrtest/cmdline.py

@@ -4,6 +4,8 @@
 import sys
 from test.support import os_helper
 
+from .utils import MS_WINDOWS
+
 
 USAGE = """\
 python -m test [options] [test_name1 [test_name2 ...]]
@@ -145,6 +147,7 @@
 
 class Namespace(argparse.Namespace):
     def __init__(self, **kwargs) -> None:
+        self.ci = False
         self.testdir = None
         self.verbose = 0
         self.quiet = False
@@ -209,6 +212,12 @@ def _create_parser():
     # We add help explicitly to control what argument group it renders under.
     group.add_argument('-h', '--help', action='help',
                        help='show this help message and exit')
+    group.add_argument('--fast-ci', action='store_true',
+                       help='Fast Continuous Integration (CI) mode used by '
+                            'GitHub Actions')
+    group.add_argument('--slow-ci', action='store_true',
+                       help='Slow Continuous Integration (CI) mode used by '
+                            'buildbot workers')
     group.add_argument('--timeout', metavar='TIMEOUT', type=float,
                         help='dump the traceback and exit if a test takes '
                              'more than TIMEOUT seconds; disabled if TIMEOUT '
@@ -386,6 +395,37 @@ def _parse_args(args, **kwargs):
             parser.error("unrecognized arguments: %s" % arg)
             sys.exit(1)
 
+    # Continuous Integration (CI): common options for fast/slow CI modes
+    if ns.slow_ci or ns.fast_ci:
+        # Similar to options:
+        #
+        #     -j0 --rerun -r --fail-env-changed --fail-rerun --slowest
+        #     --verbose3 --nowindows
+        if ns.use_mp is None:
+            ns.use_mp = 0
+        ns.rerun = True
+        ns.randomize = True
+        ns.fail_env_changed = True
+        ns.fail_rerun = True
+        ns.print_slow = True
+        ns.verbose3 = True
+        if MS_WINDOWS:
+            ns.nowindows = True  # Silence alerts under Windows
+
+    # When --slow-ci and --fast-ci are present, --slow-ci has the priority
+    if ns.slow_ci:
+        # Similar to: -u "all" --timeout=1200
+        if not ns.use:
+            ns.use = [['all']]
+        if ns.timeout is None:
+            ns.timeout = 1200  # 20 minutes
+    elif ns.fast_ci:
+        # Similar to: -u "all,-cpu" --timeout=600
+        if not ns.use:
+            ns.use = [['all', '-cpu']]
+        if ns.timeout is None:
+            ns.timeout = 600  # 10 minutes
+
     if ns.single and ns.fromfile:
         parser.error("-s and -f don't go together!")
     if ns.use_mp is not None and ns.trace:

Lib/test/test_regrtest.py

@@ -23,8 +23,9 @@
 from test import support
 from test.support import os_helper, TestStats, without_optimizer
 from test.libregrtest import cmdline
-from test.libregrtest import utils
+from test.libregrtest import main
 from test.libregrtest import setup
+from test.libregrtest import utils
 from test.libregrtest.utils import normalize_test_name
 
 if not support.has_subprocess_support:
@@ -366,6 +367,35 @@ def test_unknown_option(self):
         self.checkError(['--unknown-option'],
                         'unrecognized arguments: --unknown-option')
 
+    def check_ci_mode(self, args, use_resources):
+        ns = cmdline._parse_args(args)
+        if utils.MS_WINDOWS:
+            self.assertTrue(ns.nowindows)
+
+        # Check Regrtest attributes which are more reliable than Namespace
+        # which has an unclear API
+        regrtest = main.Regrtest(ns)
+        self.assertNotEqual(regrtest.num_workers, 0)
+        self.assertTrue(regrtest.want_rerun)
+        self.assertTrue(regrtest.randomize)
+        self.assertIsNone(regrtest.random_seed)
+        self.assertTrue(regrtest.fail_env_changed)
+        self.assertTrue(regrtest.fail_rerun)
+        self.assertTrue(regrtest.print_slowest)
+        self.assertTrue(regrtest.output_on_failure)
+        self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources))
+
+    def test_fast_ci(self):
+        args = ['--fast-ci']
+        use_resources = sorted(cmdline.ALL_RESOURCES)
+        use_resources.remove('cpu')
+        self.check_ci_mode(args, use_resources)
+
+    def test_slow_ci(self):
+        args = ['--slow-ci']
+        use_resources = sorted(cmdline.ALL_RESOURCES)
+        self.check_ci_mode(args, use_resources)
+
 
 @dataclasses.dataclass(slots=True)
 class Rerun:

Makefile.pre.in

@@ -1887,7 +1887,15 @@ buildbottest: all
 		-@if which pybuildbot.identify >/dev/null 2>&1; then \
 			pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
 		fi
-		$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --timeout=$(TESTTIMEOUT) $(TESTOPTS)
+		$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
+
+# Like buildbottest, but use --fast-ci option, instead of --slow-ci.
+.PHONY: fastcitest
+fastcitest: all
+		-@if which pybuildbot.identify >/dev/null 2>&1; then \
+			pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
+		fi
+		$(TESTRUNNER) --fast-ci $(TESTOPTS)
 
 # Like testall, but run Python tests with HOSTRUNNER directly.
 .PHONY: hostrunnertest

Misc/NEWS.d/next/Build/2023-09-19-13-34-17.gh-issue-109566.RQAYRN.rst

@@ -0,0 +1,3 @@
+Add ``make fastcitest`` target: similar to ``make buildbottest``, but use
+``--fast-ci`` option instead of ``--slow-ci``, and don't use ``TESTTIMEOUT``
+option. Patch by Victor Stinner.

Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst

@@ -0,0 +1,4 @@
+regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a
+default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests).
+``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all
+tests). Patch by Victor Stinner.

Tools/buildbot/test.bat

@@ -5,7 +5,8 @@ setlocal
 set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH
 set here=%~dp0
 set rt_opts=-q -d
-set regrtest_args=-j1
+set timeout=1200
+set regrtest_args=
 set arm32_ssh=
 
 :CheckOpts
@@ -23,7 +24,7 @@ if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecuti
 if "%arm32_ssh%"=="true" goto :Arm32Ssh
 
 :NativeExecution
-call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args%
+call "%here%..\..\PCbuild\rt.bat" %rt_opts% --slow-ci --timeout=%timeout% %regrtest_args%
 exit /b %ERRORLEVEL%
 
 :Arm32Ssh
@@ -35,7 +36,7 @@ if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DI
 
 set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp
 
-set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 --fail-env-changed %regrtest_args% %TEMP_ARGS%
+set rt_args=%rt_opts% --slow-ci %dashU% --timeout=%timeout% %regrtest_args% %TEMP_ARGS%
 ssh %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp& cd %REMOTE_PYTHON_DIR% & %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args%
 set ERR=%ERRORLEVEL%
 scp %SSH_SERVER%:"%REMOTE_PYTHON_DIR%test-results.xml" "%PYTHON_SOURCE%\test-results.xml"

Tools/scripts/run_tests.py

@@ -18,9 +18,6 @@ def is_multiprocess_flag(arg):
     return arg.startswith('-j') or arg.startswith('--multiprocess')
 
 
-def is_resource_use_flag(arg):
-    return arg.startswith('-u') or arg.startswith('--use')
-
 def is_python_flag(arg):
     return arg.startswith('-p') or arg.startswith('--python')
 
@@ -56,20 +53,13 @@ def main(regrtest_args):
     args.extend(test.support.args_from_interpreter_flags())
 
     args.extend(['-m', 'test',    # Run the test suite
-                 '-r',            # Randomize test order
-                 '-w',            # Re-run failed tests in verbose mode
+                 '--fast-ci',     # Fast Continuous Integration mode
                  ])
-    if sys.platform == 'win32':
-        args.append('-n')         # Silence alerts under Windows
     if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
         if cross_compile and hostrunner:
             # For now use only two cores for cross-compiled builds;
             # hostrunner can be expensive.
             args.extend(['-j', '2'])
-        else:
-            args.extend(['-j', '0'])  # Use all CPU cores
-    if not any(is_resource_use_flag(arg) for arg in regrtest_args):
-        args.extend(['-u', 'all,-largefile,-audio,-gui'])
 
     if cross_compile and hostrunner:
         # If HOSTRUNNER is set and -p/--python option is not given, then