From 9a1fe09622cd0f1e24c2ba5335c94c5d70306fd0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 21 Oct 2023 17:44:46 +0300 Subject: gh-110918: regrtest: allow to intermix --match and --ignore options (GH-110919) Test case matching patterns specified by options --match, --ignore, --matchfile and --ignorefile are now tested in the order of specification, and the last match determines whether the test case be run or ignored. --- Lib/test/libregrtest/cmdline.py | 40 +++++++----- Lib/test/libregrtest/findtests.py | 7 +- Lib/test/libregrtest/main.py | 15 +---- Lib/test/libregrtest/run_workers.py | 2 +- Lib/test/libregrtest/runtests.py | 5 +- Lib/test/libregrtest/setup.py | 2 +- Lib/test/libregrtest/utils.py | 1 + Lib/test/libregrtest/worker.py | 6 +- Lib/test/support/__init__.py | 76 +++++++++------------- Lib/test/test_regrtest.py | 42 ++++++------ Lib/test/test_support.py | 67 ++++++++++--------- .../2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst | 4 ++ 12 files changed, 126 insertions(+), 141 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 152b558..87b926d 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -161,8 +161,7 @@ class Namespace(argparse.Namespace): self.forever = False self.header = False self.failfast = False - self.match_tests = None - self.ignore_tests = None + self.match_tests = [] self.pgo = False self.pgo_extended = False self.worker_json = None @@ -183,6 +182,20 @@ class _ArgParser(argparse.ArgumentParser): super().error(message + "\nPass -h or --help for complete help.") +class FilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + items.append((value, self.const)) + + +class FromFileFilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + with open(value, encoding='utf-8') as fp: + for line in fp: + items.append((line.strip(), self.const)) + + def _create_parser(): # Set prog to prevent the uninformative "__main__.py" from displaying in # error messages when using "python -m test ...". @@ -192,6 +205,7 @@ def _create_parser(): epilog=EPILOG, add_help=False, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.set_defaults(match_tests=[]) # Arguments with this clause added to its help are described further in # the epilog's "Additional option details" section. @@ -251,17 +265,19 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', action='append', + dest='match_tests', action=FilterAction, const=True, help='match test cases and methods with glob pattern PAT') group.add_argument('-i', '--ignore', metavar='PAT', - dest='ignore_tests', action='append', + dest='match_tests', action=FilterAction, const=False, help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', - dest='match_filename', + dest='match_tests', + action=FromFileFilterAction, const=True, help='similar to --match but get patterns from a ' 'text file, one pattern per line') group.add_argument('--ignorefile', metavar='FILENAME', - dest='ignore_filename', + dest='match_tests', + action=FromFileFilterAction, const=False, help='similar to --matchfile but it receives patterns ' 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', @@ -482,18 +498,6 @@ def _parse_args(args, **kwargs): print("WARNING: Disable --verbose3 because it's incompatible with " "--huntrleaks: see http://bugs.python.org/issue27103", file=sys.stderr) - if ns.match_filename: - if ns.match_tests is None: - ns.match_tests = [] - with open(ns.match_filename) as fp: - for line in fp: - ns.match_tests.append(line.strip()) - if ns.ignore_filename: - if ns.ignore_tests is None: - ns.ignore_tests = [] - with open(ns.ignore_filename) as fp: - for line in fp: - ns.ignore_tests.append(line.strip()) if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index caf2872..f3ff362 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -5,7 +5,7 @@ import unittest from test import support from .utils import ( - StrPath, TestName, TestTuple, TestList, FilterTuple, + StrPath, TestName, TestTuple, TestList, TestFilter, abs_module_name, count, printlist) @@ -83,11 +83,10 @@ def _list_cases(suite): print(test.id()) def list_cases(tests: TestTuple, *, - match_tests: FilterTuple | None = None, - ignore_tests: FilterTuple | None = None, + match_tests: TestFilter | None = None, test_dir: StrPath | None = None): support.verbose = False - support.set_match_tests(match_tests, ignore_tests) + support.set_match_tests(match_tests) skipped = [] for test_name in tests: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 02f3f84..9b86548 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -19,7 +19,7 @@ from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME from .utils import ( - StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, + StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, @@ -78,14 +78,7 @@ class Regrtest: and ns._add_python_opts) # Select tests - if ns.match_tests: - self.match_tests: FilterTuple | None = tuple(ns.match_tests) - else: - self.match_tests = None - if ns.ignore_tests: - self.ignore_tests: FilterTuple | None = tuple(ns.ignore_tests) - else: - self.ignore_tests = None + self.match_tests: TestFilter = ns.match_tests self.exclude: bool = ns.exclude self.fromfile: StrPath | None = ns.fromfile self.starting_test: TestName | None = ns.start @@ -389,7 +382,7 @@ class Regrtest: def display_summary(self): duration = time.perf_counter() - self.logger.start_time - filtered = bool(self.match_tests) or bool(self.ignore_tests) + filtered = bool(self.match_tests) # Total duration print() @@ -407,7 +400,6 @@ class Regrtest: fail_fast=self.fail_fast, fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, match_tests_dict=None, rerun=False, forever=self.forever, @@ -660,7 +652,6 @@ class Regrtest: elif self.want_list_cases: list_cases(selected, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, test_dir=self.test_dir) else: exitcode = self.run_tests(selected, tests) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 16f8331..ab03cb5 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -261,7 +261,7 @@ class WorkerThread(threading.Thread): kwargs = {} if match_tests: - kwargs['match_tests'] = match_tests + kwargs['match_tests'] = [(test, True) for test in match_tests] if self.runtests.output_on_failure: kwargs['verbose'] = True kwargs['output_on_failure'] = False diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 893b311..bfed1b4 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -8,7 +8,7 @@ from typing import Any from test import support from .utils import ( - StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) + StrPath, StrJSON, TestTuple, TestFilter, FilterTuple, FilterDict) class JsonFileType: @@ -72,8 +72,7 @@ class RunTests: tests: TestTuple fail_fast: bool fail_env_changed: bool - match_tests: FilterTuple | None - ignore_tests: FilterTuple | None + match_tests: TestFilter match_tests_dict: FilterDict | None rerun: bool forever: bool diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 793347f..6a96b05 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -92,7 +92,7 @@ def setup_tests(runtests: RunTests): support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended - support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + support.set_match_tests(runtests.match_tests) if runtests.use_junit: support.junit_xml_list = [] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index aac8395..bd4dce3 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -52,6 +52,7 @@ TestTuple = tuple[TestName, ...] TestList = list[TestName] # --match and --ignore options: list of patterns # ('*' joker character can be used) +TestFilter = list[tuple[TestName, bool]] FilterTuple = tuple[TestName, ...] FilterDict = dict[TestName, FilterTuple] diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index a9c8be0..2eccfab 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -10,7 +10,7 @@ from .setup import setup_process, setup_test_dir from .runtests import RunTests, JsonFile, JsonFileType from .single import run_single_test from .utils import ( - StrPath, StrJSON, FilterTuple, + StrPath, StrJSON, TestFilter, get_temp_dir, get_work_dir, exit_timeout) @@ -73,7 +73,7 @@ def create_worker_process(runtests: RunTests, output_fd: int, def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] - match_tests: FilterTuple | None = runtests.match_tests + match_tests: TestFilter = runtests.match_tests json_file: JsonFile = runtests.json_file setup_test_dir(runtests.test_dir) @@ -81,7 +81,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn: if runtests.rerun: if match_tests: - matching = "matching: " + ", ".join(match_tests) + matching = "matching: " + ", ".join(pattern for pattern, result in match_tests if result) print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) else: print(f"Re-running {test_name} in verbose mode", flush=True) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 21e8770..695ffd0 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -6,8 +6,10 @@ if __name__ != 'test.support': import contextlib import dataclasses import functools +import itertools import getpass import _opcode +import operator import os import re import stat @@ -1183,18 +1185,17 @@ def _run_suite(suite): # By default, don't filter tests -_match_test_func = None - -_accept_test_patterns = None -_ignore_test_patterns = None +_test_matchers = () +_test_patterns = () def match_test(test): # Function used by support.run_unittest() and regrtest --list-cases - if _match_test_func is None: - return True - else: - return _match_test_func(test.id()) + result = False + for matcher, result in reversed(_test_matchers): + if matcher(test.id()): + return result + return not result def _is_full_match_test(pattern): @@ -1207,47 +1208,30 @@ def _is_full_match_test(pattern): return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) -def set_match_tests(accept_patterns=None, ignore_patterns=None): - global _match_test_func, _accept_test_patterns, _ignore_test_patterns - - if accept_patterns is None: - accept_patterns = () - if ignore_patterns is None: - ignore_patterns = () - - accept_func = ignore_func = None - - if accept_patterns != _accept_test_patterns: - accept_patterns, accept_func = _compile_match_function(accept_patterns) - if ignore_patterns != _ignore_test_patterns: - ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) - - # Create a copy since patterns can be mutable and so modified later - _accept_test_patterns = tuple(accept_patterns) - _ignore_test_patterns = tuple(ignore_patterns) +def set_match_tests(patterns): + global _test_matchers, _test_patterns - if accept_func is not None or ignore_func is not None: - def match_function(test_id): - accept = True - ignore = False - if accept_func: - accept = accept_func(test_id) - if ignore_func: - ignore = ignore_func(test_id) - return accept and not ignore - - _match_test_func = match_function + if not patterns: + _test_matchers = () + _test_patterns = () + else: + itemgetter = operator.itemgetter + patterns = tuple(patterns) + if patterns != _test_patterns: + _test_matchers = [ + (_compile_match_function(map(itemgetter(0), it)), result) + for result, it in itertools.groupby(patterns, itemgetter(1)) + ] + _test_patterns = patterns def _compile_match_function(patterns): - if not patterns: - func = None - # set_match_tests(None) behaves as set_match_tests(()) - patterns = () - elif all(map(_is_full_match_test, patterns)): + patterns = list(patterns) + + if all(map(_is_full_match_test, patterns)): # Simple case: all patterns are full test identifier. # The test.bisect_cmd utility only uses such full test identifiers. - func = set(patterns).__contains__ + return set(patterns).__contains__ else: import fnmatch regex = '|'.join(map(fnmatch.translate, patterns)) @@ -1255,7 +1239,7 @@ def _compile_match_function(patterns): # don't use flags=re.IGNORECASE regex_match = re.compile(regex).match - def match_test_regex(test_id): + def match_test_regex(test_id, regex_match=regex_match): if regex_match(test_id): # The regex matches the whole identifier, for example # 'test.test_os.FileTests.test_access'. @@ -1266,9 +1250,7 @@ def _compile_match_function(patterns): # into: 'test', 'test_os', 'FileTests' and 'test_access'. return any(map(regex_match, test_id.split("."))) - func = match_test_regex - - return patterns, func + return match_test_regex def run_unittest(*classes): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 03c180e..22b38ac 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -192,34 +192,27 @@ class ParseArgsTestCase(unittest.TestCase): self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") - def test_ignore(self): - for opt in '-i', '--ignore': + def test_match(self): + for opt in '-m', '--match': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.ignore_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', True)]) self.checkError([opt], 'expected one argument') - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "w") as fp: - print('matchfile1', file=fp) - print('matchfile2', file=fp) - - filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--ignorefile', filename]) - self.assertEqual(ns.ignore_tests, - ['matchfile1', 'matchfile2']) - - def test_match(self): - for opt in '-m', '--match': + for opt in '-i', '--ignore': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', False)]) self.checkError([opt], 'expected one argument') - ns = self.parse_args(['-m', 'pattern1', - '-m', 'pattern2']) - self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)]) + + ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)]) + + ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)]) self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, "w") as fp: @@ -227,10 +220,13 @@ class ParseArgsTestCase(unittest.TestCase): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--matchfile', filename]) + ns = self.parse_args(['-m', 'match', '--matchfile', filename]) + self.assertEqual(ns.match_tests, + [('match', True), ('matchfile1', True), ('matchfile2', True)]) + + ns = self.parse_args(['-i', 'match', '--ignorefile', filename]) self.assertEqual(ns.match_tests, - ['match', 'matchfile1', 'matchfile2']) + [('match', False), ('matchfile1', False), ('matchfile2', False)]) def test_failfast(self): for opt in '-G', '--failfast': diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 97de816..41fcc9d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -557,100 +557,109 @@ class TestSupport(unittest.TestCase): test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + test_copy = Test('test.test_shutil.TestCopy.test_copy') # Test acceptance - with support.swap_attr(support, '_match_test_func', None): + with support.swap_attr(support, '_test_matchers', ()): # match all support.set_match_tests([]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match all using None - support.set_match_tests(None, None) + support.set_match_tests(None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match the full test identifier - support.set_match_tests([test_access.id()], None) + support.set_match_tests([(test_access.id(), True)]) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # match the module name - support.set_match_tests(['test_os'], None) + support.set_match_tests([('test_os', True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) # Test '*' pattern - support.set_match_tests(['test_*'], None) + support.set_match_tests([('test_*', True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test case sensitivity - support.set_match_tests(['filetests'], None) + support.set_match_tests([('filetests', True)]) self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests'], None) + support.set_match_tests([('FileTests', True)]) self.assertTrue(support.match_test(test_access)) # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*'], None) + support.set_match_tests([('*test_os.*.test_*', True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()], None) + support.set_match_tests([(test_access.id(), True), (test_chdir.id(), True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) - support.set_match_tests(['test_access', 'DONTMATCH'], None) + support.set_match_tests([('test_access', True), ('DONTMATCH', True)]) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # Test rejection - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests(ignore_patterns=[]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - + with support.swap_attr(support, '_test_matchers', ()): # match the full test identifier - support.set_match_tests(None, [test_access.id()]) + support.set_match_tests([(test_access.id(), False)]) self.assertFalse(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match the module name - support.set_match_tests(None, ['test_os']) + support.set_match_tests([('test_os', False)]) self.assertFalse(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) # Test '*' pattern - support.set_match_tests(None, ['test_*']) + support.set_match_tests([('test_*', False)]) self.assertFalse(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # Test case sensitivity - support.set_match_tests(None, ['filetests']) + support.set_match_tests([('filetests', False)]) self.assertTrue(support.match_test(test_access)) - support.set_match_tests(None, ['FileTests']) + support.set_match_tests([('FileTests', False)]) self.assertFalse(support.match_test(test_access)) # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(None, ['*test_os.*.test_*']) + support.set_match_tests([('*test_os.*.test_*', False)]) self.assertFalse(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) # Multiple patterns - support.set_match_tests(None, [test_access.id(), test_chdir.id()]) + support.set_match_tests([(test_access.id(), False), (test_chdir.id(), False)]) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) + + support.set_match_tests([('test_access', False), ('DONTMATCH', False)]) self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test mixed filters + with support.swap_attr(support, '_test_matchers', ()): + support.set_match_tests([('*test_os', False), ('test_access', True)]) + self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) - support.set_match_tests(None, ['test_access', 'DONTMATCH']) + support.set_match_tests([('*test_os', True), ('test_access', False)]) self.assertFalse(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") diff --git a/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst b/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst new file mode 100644 index 0000000..7cb79c0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst @@ -0,0 +1,4 @@ +Test case matching patterns specified by options ``--match``, ``--ignore``, +``--matchfile`` and ``--ignorefile`` are now tested in the order of +specification, and the last match determines whether the test case be run or +ignored. -- cgit v0.12