gh-109276: libregrtest calls random.seed() before each test (#109279)

libregrtest now calls random.seed() before running each test file
when -r/--randomize command line option is used. Moreover, it's also
called in worker processes. It should help to make tests more
deterministic. Previously, it was only called once in the main
process before running all test files and it was not called in worker
processes.

* Convert some f-strings to regular strings in test_regrtest when
  f-string is not needed.
* Remove unused all_methods variable from test_regrtest.
* Add RunTests members are now mandatory.

4 files changed, 88 insertions, 34 deletions

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 2c0a6c2..f52deac 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -112,8 +112,11 @@ class Regrtest:
         self.junit_filename: StrPath | None = ns.xmlpath
         self.memory_limit: str | None = ns.memlimit
         self.gc_threshold: int | None = ns.threshold
-        self.use_resources: list[str] = ns.use_resources
-        self.python_cmd: list[str] | None = ns.python
+        self.use_resources: tuple[str] = tuple(ns.use_resources)
+        if ns.python:
+            self.python_cmd: tuple[str] = tuple(ns.python)
+        else:
+            self.python_cmd = None
         self.coverage: bool = ns.trace
         self.coverage_dir: StrPath | None = ns.coverdir
         self.tmp_dir: StrPath | None = ns.tempdir
@@ -377,8 +380,11 @@ class Regrtest:
         return RunTests(
             tests,
             fail_fast=self.fail_fast,
+            fail_env_changed=self.fail_env_changed,
             match_tests=self.match_tests,
             ignore_tests=self.ignore_tests,
+            match_tests_dict=None,
+            rerun=None,
             forever=self.forever,
             pgo=self.pgo,
             pgo_extended=self.pgo_extended,
@@ -393,6 +399,9 @@ class Regrtest:
             gc_threshold=self.gc_threshold,
             use_resources=self.use_resources,
             python_cmd=self.python_cmd,
+            randomize=self.randomize,
+            random_seed=self.random_seed,
+            json_fd=None,
         )
 
     def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py
index 64f8f6a..656958f 100644
--- a/Lib/test/libregrtest/runtests.py
+++ b/Lib/test/libregrtest/runtests.py
@@ -16,29 +16,31 @@ class HuntRefleak:
 @dataclasses.dataclass(slots=True, frozen=True)
 class RunTests:
     tests: TestTuple
-    fail_fast: bool = False
-    fail_env_changed: bool = False
-    match_tests: FilterTuple | None = None
-    ignore_tests: FilterTuple | None = None
-    match_tests_dict: FilterDict | None = None
-    rerun: bool = False
-    forever: bool = False
-    pgo: bool = False
-    pgo_extended: bool = False
-    output_on_failure: bool = False
-    timeout: float | None = None
-    verbose: int = 0
-    quiet: bool = False
-    hunt_refleak: HuntRefleak | None = None
-    test_dir: StrPath | None = None
-    use_junit: bool = False
-    memory_limit: str | None = None
-    gc_threshold: int | None = None
-    use_resources: list[str] = dataclasses.field(default_factory=list)
-    python_cmd: list[str] | None = None
+    fail_fast: bool
+    fail_env_changed: bool
+    match_tests: FilterTuple | None
+    ignore_tests: FilterTuple | None
+    match_tests_dict: FilterDict | None
+    rerun: bool
+    forever: bool
+    pgo: bool
+    pgo_extended: bool
+    output_on_failure: bool
+    timeout: float | None
+    verbose: int
+    quiet: bool
+    hunt_refleak: HuntRefleak | None
+    test_dir: StrPath | None
+    use_junit: bool
+    memory_limit: str | None
+    gc_threshold: int | None
+    use_resources: tuple[str]
+    python_cmd: tuple[str] | None
+    randomize: bool
+    random_seed: int | None
     # On Unix, it's a file descriptor.
     # On Windows, it's a handle.
-    json_fd: int | None = None
+    json_fd: int | None
 
     def copy(self, **override):
         state = dataclasses.asdict(self)
diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py
index 353a0f7..1c40b7c 100644
--- a/Lib/test/libregrtest/setup.py
+++ b/Lib/test/libregrtest/setup.py
@@ -1,5 +1,6 @@
 import faulthandler
 import os
+import random
 import signal
 import sys
 import unittest
@@ -127,3 +128,6 @@ def setup_tests(runtests: RunTests):
 
     if runtests.gc_threshold is not None:
         gc.set_threshold(runtests.gc_threshold)
+
+    if runtests.randomize:
+        random.seed(runtests.random_seed)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 466b6f6..7cf3d05 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -11,6 +11,7 @@ import io
 import locale
 import os.path
 import platform
+import random
 import re
 import subprocess
 import sys
@@ -504,7 +505,7 @@ class BaseTestCase(unittest.TestCase):
         if rerun is not None:
             regex = list_regex('%s re-run test%s', [rerun.name])
             self.check_line(output, regex)
-            regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode"
+            regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode"
             self.check_line(output, regex)
             regex = fr"Re-running {rerun.name} in verbose mode"
             if rerun.match:
@@ -1019,13 +1020,13 @@ class ArgsTestCase(BaseTestCase):
                                   forever=True)
 
     @without_optimizer
-    def check_leak(self, code, what, *, multiprocessing=False):
+    def check_leak(self, code, what, *, run_workers=False):
         test = self.create_test('huntrleaks', code=code)
 
         filename = 'reflog.txt'
         self.addCleanup(os_helper.unlink, filename)
         cmd = ['--huntrleaks', '3:3:']
-        if multiprocessing:
+        if run_workers:
             cmd.append('-j1')
         cmd.append(test)
         output = self.run_tests(*cmd,
@@ -1044,7 +1045,7 @@ class ArgsTestCase(BaseTestCase):
             self.assertIn(line2, reflog)
 
     @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
-    def check_huntrleaks(self, *, multiprocessing: bool):
+    def check_huntrleaks(self, *, run_workers: bool):
         # test --huntrleaks
         code = textwrap.dedent("""
             import unittest
@@ -1055,13 +1056,13 @@ class ArgsTestCase(BaseTestCase):
                 def test_leak(self):
                     GLOBAL_LIST.append(object())
         """)
-        self.check_leak(code, 'references', multiprocessing=multiprocessing)
+        self.check_leak(code, 'references', run_workers=run_workers)
 
     def test_huntrleaks(self):
-        self.check_huntrleaks(multiprocessing=False)
+        self.check_huntrleaks(run_workers=False)
 
     def test_huntrleaks_mp(self):
-        self.check_huntrleaks(multiprocessing=True)
+        self.check_huntrleaks(run_workers=True)
 
     @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
     def test_huntrleaks_fd_leak(self):
@@ -1139,8 +1140,6 @@ class ArgsTestCase(BaseTestCase):
                 def test_method4(self):
                     pass
         """)
-        all_methods = ['test_method1', 'test_method2',
-                       'test_method3', 'test_method4']
         testname = self.create_test(code=code)
 
         # only run a subset
@@ -1762,7 +1761,7 @@ class ArgsTestCase(BaseTestCase):
             if encoding is None:
                 encoding = sys.__stdout__.encoding
                 if encoding is None:
-                    self.skipTest(f"cannot get regrtest worker encoding")
+                    self.skipTest("cannot get regrtest worker encoding")
 
         nonascii = b"byte:\xa0\xa9\xff\n"
         try:
@@ -1789,7 +1788,7 @@ class ArgsTestCase(BaseTestCase):
                                   stats=0)
 
     def test_doctest(self):
-        code = textwrap.dedent(fr'''
+        code = textwrap.dedent(r'''
             import doctest
             import sys
             from test import support
@@ -1827,6 +1826,46 @@ class ArgsTestCase(BaseTestCase):
                                   randomize=True,
                                   stats=TestStats(1, 1, 0))
 
+    def _check_random_seed(self, run_workers: bool):
+        # gh-109276: When -r/--randomize is used, random.seed() is called
+        # with the same random seed before running each test file.
+        code = textwrap.dedent(r'''
+            import random
+            import unittest
+
+            class RandomSeedTest(unittest.TestCase):
+                def test_randint(self):
+                    numbers = [random.randint(0, 1000) for _ in range(10)]
+                    print(f"Random numbers: {numbers}")
+        ''')
+        tests = [self.create_test(name=f'test_random{i}', code=code)
+                 for i in range(1, 3+1)]
+
+        random_seed = 856_656_202
+        cmd = ["--randomize", f"--randseed={random_seed}"]
+        if run_workers:
+            # run as many worker processes than the number of tests
+            cmd.append(f'-j{len(tests)}')
+        cmd.extend(tests)
+        output = self.run_tests(*cmd)
+
+        random.seed(random_seed)
+        # Make the assumption that nothing consume entropy between libregrest
+        # setup_tests() which calls random.seed() and RandomSeedTest calling
+        # random.randint().
+        numbers = [random.randint(0, 1000) for _ in range(10)]
+        expected = f"Random numbers: {numbers}"
+
+        regex = r'^Random numbers: .*$'
+        matches = re.findall(regex, output, flags=re.MULTILINE)
+        self.assertEqual(matches, [expected] * len(tests))
+
+    def test_random_seed(self):
+        self._check_random_seed(run_workers=False)
+
+    def test_random_seed_workers(self):
+        self._check_random_seed(run_workers=True)
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):