From 9a14214aee22512f833a8d5cf8627ca45627d330 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 13:51:17 +0200 Subject: Issue #25220: Fix "-m test --forever" * Fix "-m test --forever": replace _test_forever() with self._test_forever() * Add unit test for --forever * Add unit test for a failing test * Fix also some pyflakes warnings in libregrtest --- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/refleak.py | 6 +- Lib/test/libregrtest/runtest_mp.py | 3 +- Lib/test/test_regrtest.py | 133 ++++++++++++++++++++++++++----------- 4 files changed, 100 insertions(+), 44 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fdb925d..e1a99fb 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -319,7 +319,7 @@ class Regrtest: def run_tests(self): if self.ns.forever: - self.tests = _test_forever(list(self.selected)) + self.tests = self._test_forever(list(self.selected)) self.test_count = '' self.test_count_width = 3 else: diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index db8a445..9be0dec 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -46,6 +46,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) + # initialize variables to make pyflakes quiet + rc_before = alloc_before = 0 for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) @@ -158,6 +160,6 @@ def warm_caches(): for i in range(256): s[i:i+1] # unicode cache - x = [chr(i) for i in range(256)] + [chr(i) for i in range(256)] # int cache - x = list(range(-5, 257)) + list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index b31b51e..a732d70 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -5,7 +5,6 @@ import sys import time import traceback import types -import unittest from test import support try: import threading @@ -173,7 +172,7 @@ def run_tests_multiprocess(regrtest): try: while finished < regrtest.ns.use_mp: try: - item = output.get(timeout=PROGRESS_UPDATE) + item = output.get(timeout=timeout) except queue.Empty: running = get_running(workers) if running: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c277e10..897598d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -330,33 +330,52 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX, - output, - re.MULTILINE) - return set(match.group(1) for match in parser) + regex = r'^\[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + parser = re.finditer(regex, output, re.MULTILINE) + return list(match.group(1) for match in parser) - def check_executed_tests(self, output, tests, skipped=None): + def check_executed_tests(self, output, tests, skipped=(), failed=(), + randomize=False): if isinstance(tests, str): tests = [tests] - executed = self.parse_executed_tests(output) - self.assertEqual(executed, set(tests), output) + if isinstance(skipped, str): + skipped = [skipped] + if isinstance(failed, str): + failed = [failed] ntest = len(tests) - if skipped: - if isinstance(skipped, str): - skipped = [skipped] - nskipped = len(skipped) - - plural = 's' if nskipped != 1 else '' - names = ' '.join(sorted(skipped)) - expected = (r'%s test%s skipped:\n %s$' - % (nskipped, plural, names)) - self.check_line(output, expected) - - ok = ntest - nskipped - if ok: - self.check_line(output, r'%s test OK\.$' % ok) + nskipped = len(skipped) + nfailed = len(failed) + + executed = self.parse_executed_tests(output) + if randomize: + self.assertEqual(set(executed), set(tests), output) else: - self.check_line(output, r'All %s tests OK\.$' % ntest) + self.assertEqual(executed, tests, output) + + def plural(count): + return 's' if count != 1 else '' + + def list_regex(line_format, tests): + count = len(tests) + names = ' '.join(sorted(tests)) + regex = line_format % (count, plural(count)) + regex = r'%s:\n %s$' % (regex, names) + return regex + + if skipped: + regex = list_regex('%s test%s skipped', skipped) + self.check_line(output, regex) + + if failed: + regex = list_regex('%s test%s failed', failed) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed + if good: + regex = r'%s test%s OK\.$' % (good, plural(good)) + if not skipped and not failed and good > 1: + regex = 'All %s' % regex + self.check_line(output, regex) def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) @@ -364,24 +383,28 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed - def run_command(self, args, input=None): + def run_command(self, args, input=None, exitcode=0): if not input: input = '' - try: - return subprocess.run(args, - check=True, universal_newlines=True, - input=input, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except subprocess.CalledProcessError as exc: - self.fail("%s\n" + proc = subprocess.run(args, + universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if proc.returncode != exitcode: + self.fail("Command %s failed with exit code %s\n" "\n" "stdout:\n" + "---\n" "%s\n" + "---\n" "\n" "stderr:\n" + "---\n" "%s" - % (str(exc), exc.stdout, exc.stderr)) + "---\n" + % (str(args), proc.returncode, proc.stdout, proc.stderr)) + return proc def run_python(self, args, **kw): @@ -411,11 +434,11 @@ class ProgramsTestCase(BaseTestCase): def check_output(self, output): self.parse_random_seed(output) - self.check_executed_tests(output, self.tests) + self.check_executed_tests(output, self.tests, randomize=True) def run_tests(self, args): - stdout = self.run_python(args) - self.check_output(stdout) + output = self.run_python(args) + self.check_output(output) def test_script_regrtest(self): # Lib/test/regrtest.py @@ -492,8 +515,24 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args, input=None): - return self.run_python(['-m', 'test', *args], input=input) + def run_tests(self, *args, **kw): + return self.run_python(['-m', 'test', *args], **kw) + + def test_failing_test(self): + # test a failing test + code = textwrap.dedent(""" + import unittest + + class FailingTest(unittest.TestCase): + def test_failing(self): + self.fail("bug") + """) + test_ok = self.create_test() + test_failing = self.create_test(code=code) + tests = [test_ok, test_failing] + + output = self.run_tests(*tests, exitcode=1) + self.check_executed_tests(output, tests, failed=test_failing) def test_resources(self): # test -u command line option @@ -572,8 +611,7 @@ class ArgsTestCase(BaseTestCase): # test --coverage test = self.create_test() output = self.run_tests("--coverage", test) - executed = self.parse_executed_tests(output) - self.assertEqual(executed, {test}, output) + self.check_executed_tests(output, [test]) regex = ('lines +cov% +module +\(path\)\n' '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) @@ -584,6 +622,23 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests("--wait", test, input='key') self.check_line(output, 'Press any key to continue') + def test_forever(self): + # test --forever + code = textwrap.dedent(""" + import unittest + + class ForeverTester(unittest.TestCase): + RUN = 1 + + def test_run(self): + ForeverTester.RUN += 1 + if ForeverTester.RUN > 3: + self.fail("fail at the 3rd runs") + """) + test = self.create_test(code=code) + output = self.run_tests('--forever', test, exitcode=1) + self.check_executed_tests(output, [test]*3, failed=test) + if __name__ == '__main__': unittest.main() -- cgit v0.12