Issue #25220: Add functional tests to test_regrtest

* test all available ways to run the Python test suite
* test many regrtest options: --slow, --coverage, -r, -u,  etc.

Note: python -m test --coverage doesn't work on Windows.

1 files changed, 291 insertions, 2 deletions

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index f744127..c0f1e7b 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -1,18 +1,33 @@
 """
 Tests of regrtest.py.
+
+Note: test_regrtest cannot be run twice in parallel.
 """
 
 import argparse
 import faulthandler
 import getopt
 import os.path
+import platform
+import re
+import subprocess
+import sys
+import textwrap
 import unittest
 from test import libregrtest
 from test import support
+from test.support import script_helper
 
-class ParseArgsTestCase(unittest.TestCase):
 
-    """Test regrtest's argument parsing."""
+Py_DEBUG = hasattr(sys, 'getobjects')
+ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
+ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
+
+
+class ParseArgsTestCase(unittest.TestCase):
+    """
+    Test regrtest's argument parsing, function _parse_args().
+    """
 
     def checkError(self, args, msg):
         with support.captured_stderr() as err, self.assertRaises(SystemExit):
@@ -272,5 +287,279 @@ class ParseArgsTestCase(unittest.TestCase):
         self.assertEqual(ns.args, ['foo'])
 
 
+class BaseTestCase(unittest.TestCase):
+    TEST_UNIQUE_ID = 1
+    TESTNAME_PREFIX = 'test_regrtest_'
+    TESTNAME_REGEX = r'test_[a-z0-9_]+'
+
+    def setUp(self):
+        self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test')
+
+        # When test_regrtest is interrupted by CTRL+c, it can leave
+        # temporary test files
+        remove = [entry.path
+                  for entry in os.scandir(self.testdir)
+                  if (entry.name.startswith(self.TESTNAME_PREFIX)
+                      and entry.name.endswith(".py"))]
+        for path in remove:
+            print("WARNING: test_regrtest: remove %s" % path)
+            support.unlink(path)
+
+    def create_test(self, name=None, code=''):
+        if not name:
+            name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID
+            BaseTestCase.TEST_UNIQUE_ID += 1
+
+        # test_regrtest cannot be run twice in parallel because
+        # of setUp() and create_test()
+        name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name)
+        path = os.path.join(self.testdir, name + '.py')
+
+        self.addCleanup(support.unlink, path)
+        # Use 'x' mode to ensure that we do not override existing tests
+        with open(path, 'x', encoding='utf-8') as fp:
+            fp.write(code)
+        return name
+
+    def regex_search(self, regex, output):
+        match = re.search(regex, output, re.MULTILINE)
+        if not match:
+            self.fail("%r not found in %r" % (regex, output))
+        return match
+
+    def check_line(self, output, regex):
+        regex = re.compile(r'^' + regex, re.MULTILINE)
+        self.assertRegex(output, regex)
+
+    def parse_executed_tests(self, output):
+        parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX,
+                             output,
+                             re.MULTILINE)
+        return set(match.group(1) for match in parser)
+
+    def check_executed_tests(self, output, tests, skipped=None):
+        if isinstance(tests, str):
+            tests = [tests]
+        executed = self.parse_executed_tests(output)
+        self.assertEqual(executed, set(tests), output)
+        ntest = len(tests)
+        if skipped:
+            if isinstance(skipped, str):
+                skipped = [skipped]
+            nskipped = len(skipped)
+
+            plural = 's' if nskipped != 1 else ''
+            names = ' '.join(sorted(skipped))
+            expected = (r'%s test%s skipped:\n    %s$'
+                        % (nskipped, plural, names))
+            self.check_line(output, expected)
+
+            ok = ntest - nskipped
+            if ok:
+                self.check_line(output, r'%s test OK\.$' % ok)
+        else:
+            self.check_line(output, r'All %s tests OK\.$' % ntest)
+
+    def parse_random_seed(self, output):
+        match = self.regex_search(r'Using random seed ([0-9]+)', output)
+        randseed = int(match.group(1))
+        self.assertTrue(0 <= randseed <= 10000000, randseed)
+        return randseed
+
+
+class ProgramsTestCase(BaseTestCase):
+    """
+    Test various ways to run the Python test suite. Use options close
+    to options used on the buildbot.
+    """
+
+    NTEST = 4
+
+    def setUp(self):
+        super().setUp()
+
+        # Create NTEST tests doing nothing
+        self.tests = [self.create_test() for index in range(self.NTEST)]
+
+        self.python_args = ['-Wd', '-E', '-bb']
+        self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4']
+        if sys.platform == 'win32':
+            self.regrtest_args.append('-n')
+
+    def check_output(self, output):
+        self.parse_random_seed(output)
+        self.check_executed_tests(output, self.tests)
+
+    def run_tests(self, args):
+        res = script_helper.assert_python_ok(*args)
+        output = os.fsdecode(res.out)
+        self.check_output(output)
+
+    def test_script_regrtest(self):
+        # Lib/test/regrtest.py
+        script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py')
+
+        args = [*self.python_args, script, *self.regrtest_args, *self.tests]
+        self.run_tests(args)
+
+    def test_module_test(self):
+        # -m test
+        args = [*self.python_args, '-m', 'test',
+                *self.regrtest_args, *self.tests]
+        self.run_tests(args)
+
+    def test_module_regrtest(self):
+        # -m test.regrtest
+        args = [*self.python_args, '-m', 'test.regrtest',
+                *self.regrtest_args, *self.tests]
+        self.run_tests(args)
+
+    def test_module_autotest(self):
+        # -m test.autotest
+        args = [*self.python_args, '-m', 'test.autotest',
+                *self.regrtest_args, *self.tests]
+        self.run_tests(args)
+
+    def test_module_from_test_autotest(self):
+        # from test import autotest
+        code = 'from test import autotest'
+        args = [*self.python_args, '-c', code,
+                *self.regrtest_args, *self.tests]
+        self.run_tests(args)
+
+    def test_script_autotest(self):
+        # Lib/test/autotest.py
+        script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py')
+        args = [*self.python_args, script, *self.regrtest_args, *self.tests]
+        self.run_tests(args)
+
+    def test_tools_script_run_tests(self):
+        # Tools/scripts/run_tests.py
+        script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
+        self.run_tests([script, *self.tests])
+
+    def run_rt_bat(self, script, *args):
+        rt_args = []
+        if platform.architecture()[0] == '64bit':
+            rt_args.append('-x64')   # 64-bit build
+        if Py_DEBUG:
+            rt_args.append('-d')   # Debug build
+
+        args = [script, *rt_args, *args]
+        proc = subprocess.run(args,
+                              check=True, universal_newlines=True,
+                              input='',
+                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.check_output(proc.stdout)
+
+    @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
+    def test_tools_buildbot_test(self):
+        # Tools\buildbot\test.bat
+        script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat')
+        self.run_rt_bat(script, *self.tests)
+
+    @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
+    def test_pcbuild_rt(self):
+        # PCbuild\rt.bat
+        script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat')
+        # -q: quick, don't run tests twice
+        rt_args = ["-q"]
+        self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests)
+
+
+class ArgsTestCase(BaseTestCase):
+    """
+    Test arguments of the Python test suite.
+    """
+
+    def run_tests(self, *args):
+        args = ['-m', 'test', *args]
+        res = script_helper.assert_python_ok(*args)
+        return os.fsdecode(res.out)
+
+    def test_resources(self):
+        # test -u command line option
+        tests = {}
+        for resource in ('audio', 'network'):
+            code = 'from test import support\nsupport.requires(%r)' % resource
+            tests[resource] = self.create_test(resource, code)
+        test_names = sorted(tests.values())
+
+        # -u all: 2 resources enabled
+        output = self.run_tests('-u', 'all', *test_names)
+        self.check_executed_tests(output, test_names)
+
+        # -u audio: 1 resource enabled
+        output = self.run_tests('-uaudio', *test_names)
+        self.check_executed_tests(output, test_names,
+                                  skipped=tests['network'])
+
+        # no option: 0 resources enabled
+        output = self.run_tests(*test_names)
+        self.check_executed_tests(output, test_names,
+                                  skipped=test_names)
+
+    def test_random(self):
+        # test -r and --randseed command line option
+        code = textwrap.dedent("""
+            import random
+            print("TESTRANDOM: %s" % random.randint(1, 1000))
+        """)
+        test = self.create_test('random', code)
+
+        # first run to get the output with the random seed
+        output = self.run_tests('-r', test)
+        randseed = self.parse_random_seed(output)
+        match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
+        test_random = int(match.group(1))
+
+        # try to reproduce with the random seed
+        output = self.run_tests('-r', '--randseed=%s' % randseed, test)
+        randseed2 = self.parse_random_seed(output)
+        self.assertEqual(randseed2, randseed)
+
+        match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
+        test_random2 = int(match.group(1))
+        self.assertEqual(test_random2, test_random)
+
+    def test_fromfile(self):
+        # test --fromfile
+        tests = [self.create_test() for index in range(5)]
+
+        # Write the list of files using a format similar to regrtest output:
+        # [1/2] test_1
+        # [2/2] test_2
+        filename = support.TESTFN
+        self.addCleanup(support.unlink, filename)
+        with open(filename, "w") as fp:
+            for index, name in enumerate(tests, 1):
+                print("[%s/%s] %s" % (index, len(tests), name), file=fp)
+
+        output = self.run_tests('--fromfile', filename)
+        self.check_executed_tests(output, tests)
+
+    def test_slow(self):
+        # test --slow
+        tests = [self.create_test() for index in range(3)]
+        output = self.run_tests("--slow", *tests)
+        self.check_executed_tests(output, tests)
+        regex = ('10 slowest tests:\n'
+                 '(?:%s: [0-9]+\.[0-9]+s\n){%s}'
+                 % (self.TESTNAME_REGEX, len(tests)))
+        self.check_line(output, regex)
+
+    @unittest.skipIf(sys.platform == 'win32',
+                     "FIXME: coverage doesn't work on Windows")
+    def test_coverage(self):
+        # test --coverage
+        test = self.create_test()
+        output = self.run_tests("--coverage", test)
+        executed = self.parse_executed_tests(output)
+        self.assertEqual(executed, {test}, output)
+        regex = ('lines +cov% +module +\(path\)\n'
+                 '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+')
+        self.check_line(output, regex)
+
+
 if __name__ == '__main__':
     unittest.main()