diff options
| -rw-r--r-- | Lib/test/libregrtest/main.py | 13 | ||||
| -rw-r--r-- | Lib/test/libregrtest/runtests.py | 44 | ||||
| -rw-r--r-- | Lib/test/libregrtest/setup.py | 4 | ||||
| -rw-r--r-- | Lib/test/test_regrtest.py | 61 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst | 6 |
5 files changed, 94 insertions, 34 deletions
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2c0a6c2..f52deac 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -112,8 +112,11 @@ class Regrtest: self.junit_filename: StrPath | None = ns.xmlpath self.memory_limit: str | None = ns.memlimit self.gc_threshold: int | None = ns.threshold - self.use_resources: list[str] = ns.use_resources - self.python_cmd: list[str] | None = ns.python + self.use_resources: tuple[str] = tuple(ns.use_resources) + if ns.python: + self.python_cmd: tuple[str] = tuple(ns.python) + else: + self.python_cmd = None self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir self.tmp_dir: StrPath | None = ns.tempdir @@ -377,8 +380,11 @@ class Regrtest: return RunTests( tests, fail_fast=self.fail_fast, + fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, ignore_tests=self.ignore_tests, + match_tests_dict=None, + rerun=None, forever=self.forever, pgo=self.pgo, pgo_extended=self.pgo_extended, @@ -393,6 +399,9 @@ class Regrtest: gc_threshold=self.gc_threshold, use_resources=self.use_resources, python_cmd=self.python_cmd, + randomize=self.randomize, + random_seed=self.random_seed, + json_fd=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 64f8f6a..656958f 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -16,29 +16,31 @@ class HuntRefleak: @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple - fail_fast: bool = False - fail_env_changed: bool = False - match_tests: FilterTuple | None = None - ignore_tests: FilterTuple | None = None - match_tests_dict: FilterDict | None = None - rerun: bool = False - forever: bool = False - pgo: bool = False - pgo_extended: bool = False - output_on_failure: bool = False - timeout: float | None = None - verbose: int = 0 - quiet: bool = False - hunt_refleak: HuntRefleak | None = None - test_dir: StrPath | None = None - use_junit: bool = False - memory_limit: str | None = None - gc_threshold: int | None = None - use_resources: list[str] = dataclasses.field(default_factory=list) - python_cmd: list[str] | None = None + fail_fast: bool + fail_env_changed: bool + match_tests: FilterTuple | None + ignore_tests: FilterTuple | None + match_tests_dict: FilterDict | None + rerun: bool + forever: bool + pgo: bool + pgo_extended: bool + output_on_failure: bool + timeout: float | None + verbose: int + quiet: bool + hunt_refleak: HuntRefleak | None + test_dir: StrPath | None + use_junit: bool + memory_limit: str | None + gc_threshold: int | None + use_resources: tuple[str] + python_cmd: tuple[str] | None + randomize: bool + random_seed: int | None # On Unix, it's a file descriptor. # On Windows, it's a handle. - json_fd: int | None = None + json_fd: int | None def copy(self, **override): state = dataclasses.asdict(self) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 353a0f7..1c40b7c 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,5 +1,6 @@ import faulthandler import os +import random import signal import sys import unittest @@ -127,3 +128,6 @@ def setup_tests(runtests: RunTests): if runtests.gc_threshold is not None: gc.set_threshold(runtests.gc_threshold) + + if runtests.randomize: + random.seed(runtests.random_seed) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 466b6f6..7cf3d05 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -11,6 +11,7 @@ import io import locale import os.path import platform +import random import re import subprocess import sys @@ -504,7 +505,7 @@ class BaseTestCase(unittest.TestCase): if rerun is not None: regex = list_regex('%s re-run test%s', [rerun.name]) self.check_line(output, regex) - regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode" + regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode" self.check_line(output, regex) regex = fr"Re-running {rerun.name} in verbose mode" if rerun.match: @@ -1019,13 +1020,13 @@ class ArgsTestCase(BaseTestCase): forever=True) @without_optimizer - def check_leak(self, code, what, *, multiprocessing=False): + def check_leak(self, code, what, *, run_workers=False): test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) cmd = ['--huntrleaks', '3:3:'] - if multiprocessing: + if run_workers: cmd.append('-j1') cmd.append(test) output = self.run_tests(*cmd, @@ -1044,7 +1045,7 @@ class ArgsTestCase(BaseTestCase): self.assertIn(line2, reflog) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') - def check_huntrleaks(self, *, multiprocessing: bool): + def check_huntrleaks(self, *, run_workers: bool): # test --huntrleaks code = textwrap.dedent(""" import unittest @@ -1055,13 +1056,13 @@ class ArgsTestCase(BaseTestCase): def test_leak(self): GLOBAL_LIST.append(object()) """) - self.check_leak(code, 'references', multiprocessing=multiprocessing) + self.check_leak(code, 'references', run_workers=run_workers) def test_huntrleaks(self): - self.check_huntrleaks(multiprocessing=False) + self.check_huntrleaks(run_workers=False) def test_huntrleaks_mp(self): - self.check_huntrleaks(multiprocessing=True) + self.check_huntrleaks(run_workers=True) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): @@ -1139,8 +1140,6 @@ class ArgsTestCase(BaseTestCase): def test_method4(self): pass """) - all_methods = ['test_method1', 'test_method2', - 'test_method3', 'test_method4'] testname = self.create_test(code=code) # only run a subset @@ -1762,7 +1761,7 @@ class ArgsTestCase(BaseTestCase): if encoding is None: encoding = sys.__stdout__.encoding if encoding is None: - self.skipTest(f"cannot get regrtest worker encoding") + self.skipTest("cannot get regrtest worker encoding") nonascii = b"byte:\xa0\xa9\xff\n" try: @@ -1789,7 +1788,7 @@ class ArgsTestCase(BaseTestCase): stats=0) def test_doctest(self): - code = textwrap.dedent(fr''' + code = textwrap.dedent(r''' import doctest import sys from test import support @@ -1827,6 +1826,46 @@ class ArgsTestCase(BaseTestCase): randomize=True, stats=TestStats(1, 1, 0)) + def _check_random_seed(self, run_workers: bool): + # gh-109276: When -r/--randomize is used, random.seed() is called + # with the same random seed before running each test file. + code = textwrap.dedent(r''' + import random + import unittest + + class RandomSeedTest(unittest.TestCase): + def test_randint(self): + numbers = [random.randint(0, 1000) for _ in range(10)] + print(f"Random numbers: {numbers}") + ''') + tests = [self.create_test(name=f'test_random{i}', code=code) + for i in range(1, 3+1)] + + random_seed = 856_656_202 + cmd = ["--randomize", f"--randseed={random_seed}"] + if run_workers: + # run as many worker processes than the number of tests + cmd.append(f'-j{len(tests)}') + cmd.extend(tests) + output = self.run_tests(*cmd) + + random.seed(random_seed) + # Make the assumption that nothing consume entropy between libregrest + # setup_tests() which calls random.seed() and RandomSeedTest calling + # random.randint(). + numbers = [random.randint(0, 1000) for _ in range(10)] + expected = f"Random numbers: {numbers}" + + regex = r'^Random numbers: .*$' + matches = re.findall(regex, output, flags=re.MULTILINE) + self.assertEqual(matches, [expected] * len(tests)) + + def test_random_seed(self): + self._check_random_seed(run_workers=False) + + def test_random_seed_workers(self): + self._check_random_seed(run_workers=True) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst b/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst new file mode 100644 index 0000000..cf4074b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst @@ -0,0 +1,6 @@ +libregrtest now calls :func:`random.seed()` before running each test file +when ``-r/--randomize`` command line option is used. Moreover, it's also +called in worker processes. It should help to make tests more +deterministic. Previously, it was only called once in the main process before +running all test files and it was not called in worker processes. Patch by +Victor Stinner. |