From 859618c8cd5de86a975e68d7e5d20c04bc5db2e5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Sep 2023 17:22:50 +0200 Subject: gh-109566, regrtest: Add --fast-ci and --slow-ci options (#109570) * Add --fast-ci and --slow-ci options to libregrtest: * --fast-ci uses a default timeout of 10 minutes and "-u all,-cpu" (skip slowest tests). * --slow-ci uses a default timeout of 20 minues and "-u all" (run all tests). * regrtest header now lists test resources. * Makefile changes: * "make test", "make hostrunnertest" and "make coverage-report" now use --fast-ci option and TESTTIMEOUT variable. * "make buildbottest" now uses "--slow-ci". Remove options which became redundant with "--slow-ci". * "make testall" and "make testuniversal" now use --slow-ci option and TESTTIMEOUT variable. * "make testall" now uses "find -exec rm ..." instead of "find ... -print|xargs rm ...", same as "make clean". * GitHub Actions workflow: * Ubuntu and Address Sanitizer jobs now use "make test". Remove options which became redundant with "--fast-ci". * Windows jobs now use --fast-ci option. * Use -j0 to detect the number of CPUs. * Set Makefile TESTTIMEOUT default to an empty string, since --slow-ci and --fast-ci use different default timeout. It's now accepted to pass "--timeout=" to regrtest: treated as not timeout. * Tools/scripts/run_tests.py now uses --fast-ci option. * Tools/buildbot/test.bat now uses --slow-ci option. Remove --timeout=1200 option, redundant with --slow-ci. --- .github/workflows/build.yml | 10 ++-- Doc/using/configure.rst | 15 ++++-- Lib/test/libregrtest/cmdline.py | 55 +++++++++++++++++++++- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/results.py | 4 +- Lib/test/libregrtest/utils.py | 9 +++- Lib/test/test_regrtest.py | 50 +++++++++++++++++++- Makefile.pre.in | 45 +++++++++--------- .../2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst | 4 ++ Tools/buildbot/test.bat | 6 +-- Tools/scripts/run_tests.py | 12 +---- 11 files changed, 161 insertions(+), 51 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f9d0f4..28ebc16 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -182,7 +182,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci build_win_amd64: name: 'Windows (x64)' @@ -201,7 +201,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci build_win_arm64: name: 'Windows (arm64)' @@ -252,7 +252,7 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: make test build_ubuntu: name: 'Ubuntu' @@ -319,7 +319,7 @@ jobs: run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: xvfb-run make test build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' @@ -535,7 +535,7 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: xvfb-run make test all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 763f977..8207475 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -964,9 +964,18 @@ Main Makefile targets You can use the configure :option:`--enable-optimizations` option to make this the default target of the ``make`` command (``make all`` or just ``make``). -* ``make buildbottest``: Build Python and run the Python test suite, the same - way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds) - to change the test timeout (1200 by default: 20 minutes). + +* ``make test``: Build Python and run the Python test suite with ``--slow-ci`` + option. Variables: + + * ``TESTOPTS``: additional regrtest command line options. + * ``TESTPYTHONOPTS``: additional Python command line options. + * ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes). + +* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci`` + option and default timeout of 20 minutes, instead of ``--fast-ci`` option + and a default timeout of 10 minutes. + * ``make install``: Build and install Python. * ``make regen-all``: Regenerate (almost) all generated files; ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 99f2815..a0a8504 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -4,6 +4,8 @@ import shlex import sys from test.support import os_helper +from .utils import MS_WINDOWS + USAGE = """\ python -m test [options] [test_name1 [test_name2 ...]] @@ -145,6 +147,7 @@ RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') class Namespace(argparse.Namespace): def __init__(self, **kwargs) -> None: + self.ci = False self.testdir = None self.verbose = 0 self.quiet = False @@ -209,7 +212,13 @@ def _create_parser(): # We add help explicitly to control what argument group it renders under. group.add_argument('-h', '--help', action='help', help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, + group.add_argument('--fast-ci', action='store_true', + help='Fast Continuous Integration (CI) mode used by ' + 'GitHub Actions') + group.add_argument('--slow-ci', action='store_true', + help='Slow Continuous Integration (CI) mode used by ' + 'buildbot workers') + group.add_argument('--timeout', metavar='TIMEOUT', help='dump the traceback and exit if a test takes ' 'more than TIMEOUT seconds; disabled if TIMEOUT ' 'is negative or equals to zero') @@ -384,7 +393,49 @@ def _parse_args(args, **kwargs): for arg in ns.args: if arg.startswith('-'): parser.error("unrecognized arguments: %s" % arg) - sys.exit(1) + + if ns.timeout is not None: + # Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT + # can be used by "make buildbottest" and "make test". + if ns.timeout != "": + try: + ns.timeout = float(ns.timeout) + except ValueError: + parser.error(f"invalid timeout value: {ns.timeout!r}") + else: + ns.timeout = None + + # Continuous Integration (CI): common options for fast/slow CI modes + if ns.slow_ci or ns.fast_ci: + # Similar to options: + # + # -j0 --randomize --fail-env-changed --fail-rerun --rerun + # --slowest --verbose3 --nowindows + if ns.use_mp is None: + ns.use_mp = 0 + ns.randomize = True + ns.fail_env_changed = True + ns.fail_rerun = True + ns.rerun = True + ns.print_slow = True + ns.verbose3 = True + if MS_WINDOWS: + ns.nowindows = True # Silence alerts under Windows + + # When both --slow-ci and --fast-ci options are present, + # --slow-ci has the priority + if ns.slow_ci: + # Similar to: -u "all" --timeout=1200 + if not ns.use: + ns.use = [['all']] + if ns.timeout is None: + ns.timeout = 1200 # 20 minutes + elif ns.fast_ci: + # Similar to: -u "all,-cpu" --timeout=600 + if not ns.use: + ns.use = [['all', '-cpu']] + if ns.timeout is None: + ns.timeout = 600 # 10 minutes if ns.single and ns.fromfile: parser.error("-s and -f don't go together!") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0ec25a0..2cd79a1 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -425,7 +425,7 @@ class Regrtest: if (self.want_header or not(self.pgo or self.quiet or self.single_test_run or tests or self.cmdline_args)): - display_header() + display_header(self.use_resources) if self.randomize: print("Using random seed", self.random_seed) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 1a8619f..35df50d 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -8,11 +8,13 @@ from .utils import ( printlist, count, format_duration) +# Python uses exit code 1 when an exception is not catched +# argparse.ArgumentParser.error() uses exit code 2 EXITCODE_BAD_TEST = 2 EXITCODE_ENV_CHANGED = 3 EXITCODE_NO_TESTS_RAN = 4 EXITCODE_RERUN_FAIL = 5 -EXITCODE_INTERRUPTED = 130 +EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 class TestResults: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 6af949c..f3f0eb5 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -547,7 +547,7 @@ def adjust_rlimit_nofile(): f"{new_fd_limit}: {err}.") -def display_header(): +def display_header(use_resources: tuple[str, ...]): encoding = sys.stdout.encoding # Print basic platform information @@ -569,6 +569,13 @@ def display_header(): print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) + + if use_resources: + print(f"== resources ({len(use_resources)}): " + f"{', '.join(sorted(use_resources))}") + else: + print(f"== resources: (all disabled, use -u option)") + # This makes it easier to remember what to set in your local # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 4b819cb..15aab60 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -23,8 +23,9 @@ import unittest from test import support from test.support import os_helper, TestStats, without_optimizer from test.libregrtest import cmdline -from test.libregrtest import utils +from test.libregrtest import main from test.libregrtest import setup +from test.libregrtest import utils from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -75,8 +76,15 @@ class ParseArgsTestCase(unittest.TestCase): def test_timeout(self): ns = self.parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) + + # negative, zero and empty string are treated as "no timeout" + for value in ('-1', '0', ''): + with self.subTest(value=value): + ns = self.parse_args([f'--timeout={value}']) + self.assertEqual(ns.timeout, None) + self.checkError(['--timeout'], 'expected one argument') - self.checkError(['--timeout', 'foo'], 'invalid float value') + self.checkError(['--timeout', 'foo'], 'invalid timeout value:') def test_wait(self): ns = self.parse_args(['--wait']) @@ -366,6 +374,44 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') + def check_ci_mode(self, args, use_resources): + ns = cmdline._parse_args(args) + if utils.MS_WINDOWS: + self.assertTrue(ns.nowindows) + + # Check Regrtest attributes which are more reliable than Namespace + # which has an unclear API + regrtest = main.Regrtest(ns) + self.assertNotEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.want_rerun) + self.assertTrue(regrtest.randomize) + self.assertIsNone(regrtest.random_seed) + self.assertTrue(regrtest.fail_env_changed) + self.assertTrue(regrtest.fail_rerun) + self.assertTrue(regrtest.print_slowest) + self.assertTrue(regrtest.output_on_failure) + self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources)) + return regrtest + + def test_fast_ci(self): + args = ['--fast-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 10 * 60) + + def test_fast_ci_resource(self): + # it should be possible to override resources + args = ['--fast-ci', '-u', 'network'] + use_resources = ['network'] + self.check_ci_mode(args, use_resources) + + def test_slow_ci(self): + args = ['--slow-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 20 * 60) + @dataclasses.dataclass(slots=True) class Rerun: diff --git a/Makefile.pre.in b/Makefile.pre.in index d123fa3..ccbacfc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -771,7 +771,7 @@ coverage-report: regen-token regen-frozen @ # build with coverage info $(MAKE) coverage @ # run tests, ignore failures - $(TESTRUNNER) $(TESTOPTS) || true + $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) || true @ # build lcov report $(MAKE) coverage-lcov @@ -1844,7 +1844,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) TESTOPTS= $(EXTRATESTOPTS) TESTPYTHON= $(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py -TESTTIMEOUT= 1200 +TESTTIMEOUT= # Remove "test_python_*" directories of previous failed test jobs. # Pass TESTOPTS options because it can contain --tempdir option. @@ -1854,9 +1854,10 @@ cleantest: all # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. +# Similar to buildbottest, but use --fast-ci option, instead of --slow-ci. .PHONY: test test: all - $(TESTRUNNER) $(TESTOPTS) + $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. # In the past, we've had problems where bugs in the marshalling or @@ -1867,43 +1868,43 @@ test: all # sample data. .PHONY: testall testall: all - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py - -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f - -$(TESTRUNNER) -u all $(TESTOPTS) - $(TESTRUNNER) -u all $(TESTOPTS) + -find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true + $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py + -find $(srcdir)/Lib -name '*.py[co]' -exec rm -f {} ';' || true + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. .PHONY: testuniversal testuniversal: all - @if [ `arch` != 'i386' ]; then \ - echo "This can only be used on OSX/i386" ;\ - exit 1 ;\ - fi - $(TESTRUNNER) -u all $(TESTOPTS) - $(RUNSHARED) /usr/libexec/oah/translate \ - ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) + @if [ `arch` != 'i386' ]; then \ + echo "This can only be used on OSX/i386" ;\ + exit 1 ;\ + fi + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) + $(RUNSHARED) /usr/libexec/oah/translate \ + ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) # Like testall, but with only one pass and without multiple processes. # Run an optional script to include information about the build environment. .PHONY: buildbottest buildbottest: all - -@if which pybuildbot.identify >/dev/null 2>&1; then \ - pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ - fi - $(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --fail-rerun --timeout=$(TESTTIMEOUT) $(TESTOPTS) + -@if which pybuildbot.identify >/dev/null 2>&1; then \ + pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ + fi + $(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Like testall, but run Python tests with HOSTRUNNER directly. .PHONY: hostrunnertest hostrunnertest: all - $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS) + $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) .PHONY: pythoninfo pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo -QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \ +QUICKTESTOPTS= -x test_subprocess test_io \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing_fork test_multiprocessing_spawn \ test_multiprocessing_forkserver \ @@ -1912,7 +1913,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \ .PHONY: quicktest quicktest: all - $(TESTRUNNER) $(QUICKTESTOPTS) + $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) $(QUICKTESTOPTS) # SSL tests .PHONY: multisslcompile diff --git a/Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst b/Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst new file mode 100644 index 0000000..10f9013 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst @@ -0,0 +1,4 @@ +regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a +default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests). +``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all +tests). Patch by Victor Stinner. diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index c1b2605..781f9a4 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -5,7 +5,7 @@ setlocal set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH set here=%~dp0 set rt_opts=-q -d -set regrtest_args=-j1 +set regrtest_args= set arm32_ssh= :CheckOpts @@ -23,7 +23,7 @@ if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecuti if "%arm32_ssh%"=="true" goto :Arm32Ssh :NativeExecution -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args% +call "%here%..\..\PCbuild\rt.bat" %rt_opts% --slow-ci %regrtest_args% exit /b %ERRORLEVEL% :Arm32Ssh @@ -35,7 +35,7 @@ if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DI set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp -set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 %regrtest_args% %TEMP_ARGS% +set rt_args=%rt_opts% --slow-ci %dashU% %regrtest_args% %TEMP_ARGS% ssh %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp& cd %REMOTE_PYTHON_DIR% & %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args% set ERR=%ERRORLEVEL% scp %SSH_SERVER%:"%REMOTE_PYTHON_DIR%test-results.xml" "%PYTHON_SOURCE%\test-results.xml" diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py index 445a34a..c62ae82 100644 --- a/Tools/scripts/run_tests.py +++ b/Tools/scripts/run_tests.py @@ -18,9 +18,6 @@ def is_multiprocess_flag(arg): return arg.startswith('-j') or arg.startswith('--multiprocess') -def is_resource_use_flag(arg): - return arg.startswith('-u') or arg.startswith('--use') - def is_python_flag(arg): return arg.startswith('-p') or arg.startswith('--python') @@ -56,20 +53,13 @@ def main(regrtest_args): args.extend(test.support.args_from_interpreter_flags()) args.extend(['-m', 'test', # Run the test suite - '-r', # Randomize test order - '-w', # Re-run failed tests in verbose mode + '--fast-ci', # Fast Continuous Integration mode ]) - if sys.platform == 'win32': - args.append('-n') # Silence alerts under Windows if not any(is_multiprocess_flag(arg) for arg in regrtest_args): if cross_compile and hostrunner: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. args.extend(['-j', '2']) - else: - args.extend(['-j', '0']) # Use all CPU cores - if not any(is_resource_use_flag(arg) for arg in regrtest_args): - args.extend(['-u', 'all,-largefile,-audio,-gui']) if cross_compile and hostrunner: # If HOSTRUNNER is set and -p/--python option is not given, then -- cgit v0.12