gh-109566, regrtest: Add --fast-ci and --slow-ci options

.github/workflows/build.yml

@@ -182,7 +182,7 @@ jobs:
  - name: Display build info
  run: .\python.bat -m test.pythoninfo
  - name: Tests
- run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+ run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
 
  build_win_amd64:
  name: 'Windows (x64)'
@@ -201,7 +201,7 @@ jobs:
  - name: Display build info
  run: .\python.bat -m test.pythoninfo
  - name: Tests
- run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+ run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
 
  build_win_arm64:
  name: 'Windows (arm64)'
@@ -252,7 +252,7 @@ jobs:
  - name: Display build info
  run: make pythoninfo
  - name: Tests
- run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ run: make test
 
  build_ubuntu:
  name: 'Ubuntu'
@@ -319,7 +319,7 @@ jobs:
  run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
  - name: Tests
  working-directory: ${{ env.CPYTHON_BUILDDIR }}
- run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ run: xvfb-run make test
 
  build_ubuntu_ssltests:
  name: 'Ubuntu SSL tests with OpenSSL'
@@ -535,7 +535,7 @@ jobs:
  - name: Display build info
  run: make pythoninfo
  - name: Tests
- run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ run: xvfb-run make test
 
  all-required-green: # This job does nothing and is only used for the branch protection
  name: All required checks pass

Doc/using/configure.rst

@@ -964,9 +964,18 @@ Main Makefile targets
  You can use the configure :option:`--enable-optimizations` option to make
  this the default target of the ``make`` command (``make all`` or just
  ``make``).
-* ``make buildbottest``: Build Python and run the Python test suite, the same
- way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds)
- to change the test timeout (1200 by default: 20 minutes).
+
+* ``make test``: Build Python and run the Python test suite with ``--slow-ci``
+ option. Variables:
+
+ * ``TESTOPTS``: additional regrtest command line options.
+ * ``TESTPYTHONOPTS``: additional Python command line options.
+ * ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes).
+
+* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci``
+ option and default timeout of 20 minutes, instead of ``--fast-ci`` option
+ and a default timeout of 10 minutes.
+
 * ``make install``: Build and install Python.
 * ``make regen-all``: Regenerate (almost) all generated files;
  ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately

Lib/test/libregrtest/cmdline.py

@@ -4,6 +4,8 @@
 import sys
 from test.support import os_helper
 
+from .utils import MS_WINDOWS
+
 
 USAGE = """\
 python -m test [options] [test_name1 [test_name2 ...]]
@@ -145,6 +147,7 @@
 
 class Namespace(argparse.Namespace):
  def __init__(self, **kwargs) -> None:
+ self.ci = False
  self.testdir = None
  self.verbose = 0
  self.quiet = False
@@ -209,7 +212,13 @@ def _create_parser():
  # We add help explicitly to control what argument group it renders under.
  group.add_argument('-h', '--help', action='help',
  help='show this help message and exit')
- group.add_argument('--timeout', metavar='TIMEOUT', type=float,
+ group.add_argument('--fast-ci', action='store_true',
+ help='Fast Continuous Integration (CI) mode used by '
+ 'GitHub Actions')
+ group.add_argument('--slow-ci', action='store_true',
+ help='Slow Continuous Integration (CI) mode used by '
+ 'buildbot workers')
+ group.add_argument('--timeout', metavar='TIMEOUT',
  help='dump the traceback and exit if a test takes '
  'more than TIMEOUT seconds; disabled if TIMEOUT '
  'is negative or equals to zero')
@@ -384,7 +393,49 @@ def _parse_args(args, **kwargs):
  for arg in ns.args:
  if arg.startswith('-'):
  parser.error("unrecognized arguments: %s" % arg)
- sys.exit(1)
+
+ if ns.timeout is not None:
+ # Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT
+ # can be used by "make buildbottest" and "make test".
+ if ns.timeout != "":
+ try:
+ ns.timeout = float(ns.timeout)
+ except ValueError:
+ parser.error(f"invalid timeout value: {ns.timeout!r}")
+ else:
+ ns.timeout = None
+
+ # Continuous Integration (CI): common options for fast/slow CI modes
+ if ns.slow_ci or ns.fast_ci:
+ # Similar to options:
+ #
+ # -j0 --randomize --fail-env-changed --fail-rerun --rerun
+ # --slowest --verbose3 --nowindows
+ if ns.use_mp is None:
+ ns.use_mp = 0
+ ns.randomize = True
+ ns.fail_env_changed = True
+ ns.fail_rerun = True
+ ns.rerun = True
+ ns.print_slow = True
+ ns.verbose3 = True
+ if MS_WINDOWS:
+ ns.nowindows = True # Silence alerts under Windows
+
+ # When both --slow-ci and --fast-ci options are present,
+ # --slow-ci has the priority
+ if ns.slow_ci:
+ # Similar to: -u "all" --timeout=1200
+ if not ns.use:
+ ns.use = [['all']]
+ if ns.timeout is None:
+ ns.timeout = 1200 # 20 minutes
+ elif ns.fast_ci:
+ # Similar to: -u "all,-cpu" --timeout=600
+ if not ns.use:
+ ns.use = [['all', '-cpu']]
+ if ns.timeout is None:
+ ns.timeout = 600 # 10 minutes
 
  if ns.single and ns.fromfile:
  parser.error("-s and -f don't go together!")

Lib/test/libregrtest/main.py

@@ -425,7 +425,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
  if (self.want_header
  or not(self.pgo or self.quiet or self.single_test_run
  or tests or self.cmdline_args)):
- display_header()
+ display_header(self.use_resources)
 
  if self.randomize:
  print("Using random seed", self.random_seed)

Lib/test/libregrtest/results.py

@@ -8,11 +8,13 @@
  printlist, count, format_duration)
 
 
+# Python uses exit code 1 when an exception is not catched
+# argparse.ArgumentParser.error() uses exit code 2
 EXITCODE_BAD_TEST = 2
 EXITCODE_ENV_CHANGED = 3
 EXITCODE_NO_TESTS_RAN = 4
 EXITCODE_RERUN_FAIL = 5
-EXITCODE_INTERRUPTED = 130
+EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2
 
 
 class TestResults:

Lib/test/libregrtest/utils.py

@@ -547,7 +547,7 @@ def adjust_rlimit_nofile():
  f"{new_fd_limit}: {err}.")
 
 
-def display_header():
+def display_header(use_resources: tuple[str, ...]):
  encoding = sys.stdout.encoding
 
  # Print basic platform information
@@ -569,6 +569,13 @@ def display_header():
  print("== encodings: locale=%s, FS=%s"
  % (locale.getencoding(), sys.getfilesystemencoding()))
 
+
+ if use_resources:
+ print(f"== resources ({len(use_resources)}): "
+ f"{', '.join(sorted(use_resources))}")
+ else:
+ print(f"== resources: (all disabled, use -u option)")
+
  # This makes it easier to remember what to set in your local
  # environment when trying to reproduce a sanitizer failure.
  asan = support.check_sanitizer(address=True)

Lib/test/test_regrtest.py

@@ -23,8 +23,9 @@
 from test import support
 from test.support import os_helper, TestStats, without_optimizer
 from test.libregrtest import cmdline
-from test.libregrtest import utils
+from test.libregrtest import main
 from test.libregrtest import setup
+from test.libregrtest import utils
 from test.libregrtest.utils import normalize_test_name
 
 if not support.has_subprocess_support:
@@ -75,8 +76,15 @@ def test_help(self):
  def test_timeout(self):
  ns = self.parse_args(['--timeout', '4.2'])
  self.assertEqual(ns.timeout, 4.2)
+
+ # negative, zero and empty string are treated as "no timeout"
+ for value in ('-1', '0', ''):
+ with self.subTest(value=value):
+ ns = self.parse_args([f'--timeout={value}'])
+ self.assertEqual(ns.timeout, None)
+
  self.checkError(['--timeout'], 'expected one argument')
- self.checkError(['--timeout', 'foo'], 'invalid float value')
+ self.checkError(['--timeout', 'foo'], 'invalid timeout value:')
 
  def test_wait(self):
  ns = self.parse_args(['--wait'])
@@ -366,6 +374,44 @@ def test_unknown_option(self):
  self.checkError(['--unknown-option'],
  'unrecognized arguments: --unknown-option')
 
+ def check_ci_mode(self, args, use_resources):
+ ns = cmdline._parse_args(args)
+ if utils.MS_WINDOWS:
+ self.assertTrue(ns.nowindows)
+
+ # Check Regrtest attributes which are more reliable than Namespace
+ # which has an unclear API
+ regrtest = main.Regrtest(ns)
+ self.assertNotEqual(regrtest.num_workers, 0)
+ self.assertTrue(regrtest.want_rerun)
+ self.assertTrue(regrtest.randomize)
+ self.assertIsNone(regrtest.random_seed)
+ self.assertTrue(regrtest.fail_env_changed)
+ self.assertTrue(regrtest.fail_rerun)
+ self.assertTrue(regrtest.print_slowest)
+ self.assertTrue(regrtest.output_on_failure)
+ self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources))
+ return regrtest
+
+ def test_fast_ci(self):
+ args = ['--fast-ci']
+ use_resources = sorted(cmdline.ALL_RESOURCES)
+ use_resources.remove('cpu')
+ regrtest = self.check_ci_mode(args, use_resources)
+ self.assertEqual(regrtest.timeout, 10 * 60)
+
+ def test_fast_ci_resource(self):
+ # it should be possible to override resources
+ args = ['--fast-ci', '-u', 'network']
+ use_resources = ['network']
+ self.check_ci_mode(args, use_resources)
+
+ def test_slow_ci(self):
+ args = ['--slow-ci']
+ use_resources = sorted(cmdline.ALL_RESOURCES)
+ regrtest = self.check_ci_mode(args, use_resources)
+ self.assertEqual(regrtest.timeout, 20 * 60)
+
 
 @dataclasses.dataclass(slots=True)
 class Rerun:

Makefile.pre.in

@@ -771,7 +771,7 @@ coverage-report: regen-token regen-frozen
 	@ # build with coverage info
 	$(MAKE) coverage
 	@ # run tests, ignore failures
-	$(TESTRUNNER) $(TESTOPTS) || true
+	$(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) || true
 	@ # build lcov report
 	$(MAKE) coverage-lcov
 
@@ -1844,7 +1844,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS)
 TESTOPTS=	$(EXTRATESTOPTS)
 TESTPYTHON=	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS)
 TESTRUNNER=	$(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
-TESTTIMEOUT=	1200
+TESTTIMEOUT=
 
 # Remove "test_python_*" directories of previous failed test jobs.
 # Pass TESTOPTS options because it can contain --tempdir option.
@@ -1854,9 +1854,10 @@ cleantest: all
 
 # Run a basic set of regression tests.
 # This excludes some tests that are particularly resource-intensive.
+# Similar to buildbottest, but use --fast-ci option, instead of --slow-ci.
 .PHONY: test
 test: all
-$(TESTRUNNER) $(TESTOPTS)
+	$(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
 # Run the full test suite twice - once without .pyc files, and once with.
 # In the past, we've had problems where bugs in the marshalling or
@@ -1867,43 +1868,43 @@ test: all
 # sample data.
 .PHONY: testall
 testall: all
--find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-$(TESTPYTHON) -E $(srcdir)/Lib/compileall.py
--find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-	-$(TESTRUNNER) -u all $(TESTOPTS)
-$(TESTRUNNER) -u all $(TESTOPTS)
+	-find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true
+	$(TESTPYTHON) -E $(srcdir)/Lib/compileall.py
+	-find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true
+	$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
+	$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
 # Run the test suite for both architectures in a Universal build on OSX.
 # Must be run on an Intel box.
 .PHONY: testuniversal
 testuniversal: all
-@if [ `arch` != 'i386' ]; then \
-echo "This can only be used on OSX/i386" ;\
-exit 1 ;\
-fi
-$(TESTRUNNER) -u all $(TESTOPTS)
-$(RUNSHARED) /usr/libexec/oah/translate \
-./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS)
+	@if [ `arch` != 'i386' ]; then \
+ echo "This can only be used on OSX/i386" ;\
+ exit 1 ;\
+	fi
+	$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
+	$(RUNSHARED) /usr/libexec/oah/translate \
+ ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS)
 
 # Like testall, but with only one pass and without multiple processes.
 # Run an optional script to include information about the build environment.
 .PHONY: buildbottest
 buildbottest: all
--@if which pybuildbot.identify >/dev/null 2>&1; then \
-pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
-fi
-$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --fail-rerun --timeout=$(TESTTIMEOUT) $(TESTOPTS)
+	-@if which pybuildbot.identify >/dev/null 2>&1; then \
+ pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
+	fi
+	$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
 # Like testall, but run Python tests with HOSTRUNNER directly.
 .PHONY: hostrunnertest
 hostrunnertest: all
-	$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS)
+	$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
 .PHONY: pythoninfo
 pythoninfo: all
  $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo
 
-QUICKTESTOPTS=	$(TESTOPTS) -x test_subprocess test_io \
+QUICKTESTOPTS=	-x test_subprocess test_io \
  test_multibytecodec test_urllib2_localnet test_itertools \
  test_multiprocessing_fork test_multiprocessing_spawn \
  test_multiprocessing_forkserver \
@@ -1912,7 +1913,7 @@ QUICKTESTOPTS=	$(TESTOPTS) -x test_subprocess test_io \
 
 .PHONY: quicktest
 quicktest: all
-$(TESTRUNNER) $(QUICKTESTOPTS)
+	$(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) $(QUICKTESTOPTS)
 
 # SSL tests
 .PHONY: multisslcompile

Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst

@@ -0,0 +1,4 @@
+regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a
+default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests).
+``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all
+tests). Patch by Victor Stinner.