diff options
author | Victor Stinner <vstinner@python.org> | 2023-09-09 01:37:48 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-09-09 01:37:48 (GMT) |
commit | e9e2ca7a7b4b4320009cdf85c84ec5bd6c4923c3 (patch) | |
tree | acbd759edcaf2a286a378d7678c910a3963e2d37 | |
parent | 057bc7249073066ed8087b548ee06f0eabfa9e7c (diff) | |
download | cpython-e9e2ca7a7b4b4320009cdf85c84ec5bd6c4923c3.zip cpython-e9e2ca7a7b4b4320009cdf85c84ec5bd6c4923c3.tar.gz cpython-e9e2ca7a7b4b4320009cdf85c84ec5bd6c4923c3.tar.bz2 |
gh-109162: Refactor libregrtest.runtest (#109172)
* Rename runtest() to run_single_test().
* Pass runtests to run_single_test().
* Add type annotation to Regrtest attributes. Add missing attributes
to Namespace.
* Add attributes to Regrtest and RunTests:
* fail_fast
* ignore_tests
* match_tests
* output_on_failure
* pgo
* pgo_extended
* timeout
* Get pgo from 'runtests', rather than from 'ns'.
* Remove WorkerJob.match_tests.
* setup_support() now gets pgo_extended from runtests.
* save_env(): change parameter order, pass test_name first.
* Add setup_test_dir() function.
* Pass runtests to setup_tests().
-rw-r--r-- | Lib/test/libregrtest/cmdline.py | 6 | ||||
-rw-r--r-- | Lib/test/libregrtest/main.py | 91 | ||||
-rw-r--r-- | Lib/test/libregrtest/runtest.py | 60 | ||||
-rw-r--r-- | Lib/test/libregrtest/runtest_mp.py | 52 | ||||
-rw-r--r-- | Lib/test/libregrtest/setup.py | 29 |
5 files changed, 134 insertions, 104 deletions
diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index c08b3be..71e7396 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -149,6 +149,10 @@ class Namespace(argparse.Namespace): self.verbose = 0 self.quiet = False self.exclude = False + self.cleanup = False + self.wait = False + self.list_cases = False + self.list_tests = False self.single = False self.randomize = False self.fromfile = None @@ -171,6 +175,8 @@ class Namespace(argparse.Namespace): self.pgo = False self.pgo_extended = False self.worker_json = None + self.start = None + self.timeout = None super().__init__(**kwargs) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cd053c5..f7d28a8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,10 +11,10 @@ import time import unittest from test.libregrtest.cmdline import _parse_args, Namespace from test.libregrtest.runtest import ( - findtests, split_test_packages, runtest, abs_module_name, + findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, TestResult, FilterTuple, FilterDict, TestList) -from test.libregrtest.setup import setup_tests +from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import (strip_py_suffix, count, format_duration, printlist, get_build_info) @@ -64,11 +64,11 @@ class Regrtest: self.ns: Namespace = ns # Actions - self.want_header = ns.header - self.want_list_tests = ns.list_tests - self.want_list_cases = ns.list_cases - self.want_wait = ns.wait - self.want_cleanup = ns.cleanup + self.want_header: bool = ns.header + self.want_list_tests: bool = ns.list_tests + self.want_list_cases: bool = ns.list_cases + self.want_wait: bool = ns.wait + self.want_cleanup: bool = ns.cleanup # Select tests if ns.match_tests: @@ -79,14 +79,19 @@ class Regrtest: self.ignore_tests: FilterTuple = tuple(ns.ignore_tests) else: self.ignore_tests = None - self.exclude = ns.exclude - self.fromfile = ns.fromfile - self.starting_test = ns.start + self.exclude: bool = ns.exclude + self.fromfile: str | None = ns.fromfile + self.starting_test: str | None = ns.start # Options to run tests - self.forever = ns.forever - self.randomize = ns.randomize - self.random_seed = ns.random_seed + self.fail_fast: bool = ns.failfast + self.forever: bool = ns.forever + self.randomize: bool = ns.randomize + self.random_seed: int | None = ns.random_seed + self.pgo: bool = ns.pgo + self.pgo_extended: bool = ns.pgo_extended + self.output_on_failure: bool = ns.verbose3 + self.timeout: float | None = ns.timeout # tests self.tests = [] @@ -196,21 +201,19 @@ class Regrtest: def display_progress(self, test_index, text): quiet = self.ns.quiet - pgo = self.ns.pgo if quiet: return # "[ 51/405/1] test_tcl passed" line = f"{test_index:{self.test_count_width}}{self.test_count_text}" fails = len(self.bad) + len(self.environment_changed) - if fails and not pgo: + if fails and not self.pgo: line = f"{line}/{fails}" self.log(f"[{line}] {text}") def find_tests(self): ns = self.ns single = ns.single - pgo = ns.pgo test_dir = ns.testdir if single: @@ -237,7 +240,7 @@ class Regrtest: strip_py_suffix(self.tests) - if pgo: + if self.pgo: # add default PGO tests if no tests are specified setup_pgo_tests(ns) @@ -329,8 +332,6 @@ class Regrtest: # Configure the runner to re-run tests ns = self.ns ns.verbose = True - ns.failfast = False - ns.verbose3 = False if ns.use_mp is None: ns.use_mp = 1 @@ -345,12 +346,16 @@ class Regrtest: # Re-run failed tests self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") - runtests = runtests.copy(tests=tuple(tests), - match_tests_dict=match_tests_dict, - rerun=True, - forever=False) + runtests = runtests.copy( + tests=tuple(tests), + rerun=True, + forever=False, + fail_fast=False, + match_tests_dict=match_tests_dict, + output_on_failure=False) self.set_tests(runtests) self._run_tests_mp(runtests) + return runtests def rerun_failed_tests(self, need_rerun, runtests: RunTests): if self.ns.python: @@ -364,16 +369,16 @@ class Regrtest: self.first_state = self.get_tests_state() print() - self._rerun_failed_tests(need_rerun, runtests) + rerun_runtests = self._rerun_failed_tests(need_rerun, runtests) if self.bad: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - self.display_result() + self.display_result(rerun_runtests) - def display_result(self): - pgo = self.ns.pgo + def display_result(self, runtests): + pgo = runtests.pgo quiet = self.ns.quiet print_slow = self.ns.print_slow @@ -444,12 +449,12 @@ class Regrtest: if tracer is not None: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name)') + cmd = ('result = run_single_test(test_name, runtests, self.ns)') ns = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: - result = runtest(self.ns, test_name) + result = run_single_test(test_name, runtests, self.ns) self.accumulate_result(result) @@ -458,9 +463,7 @@ class Regrtest: def run_tests_sequentially(self, runtests): ns = self.ns coverage = ns.trace - fail_fast = ns.failfast fail_env_changed = ns.fail_env_changed - timeout = ns.timeout if coverage: import trace @@ -471,8 +474,8 @@ class Regrtest: save_modules = sys.modules.keys() msg = "Run tests sequentially" - if timeout: - msg += " (timeout: %s)" % format_duration(timeout) + if runtests.timeout: + msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) previous_test = None @@ -492,7 +495,7 @@ class Regrtest: if module not in save_modules and module.startswith("test."): support.unload(module) - if result.must_stop(fail_fast, fail_env_changed): + if result.must_stop(self.fail_fast, fail_env_changed): break previous_test = str(result) @@ -850,16 +853,28 @@ class Regrtest: # For a partial run, we do not need to clutter the output. if (self.want_header - or not(self.ns.pgo or self.ns.quiet or self.ns.single + or not(self.pgo or self.ns.quiet or self.ns.single or self.tests or self.ns.args)): self.display_header() if self.randomize: print("Using random seed", self.random_seed) - runtests = RunTests(tuple(self.selected), forever=self.forever) + runtests = RunTests( + tuple(self.selected), + fail_fast=self.fail_fast, + match_tests=self.match_tests, + ignore_tests=self.ignore_tests, + forever=self.forever, + pgo=self.pgo, + pgo_extended=self.pgo_extended, + output_on_failure=self.output_on_failure, + timeout=self.timeout) + + setup_tests(runtests, self.ns) + tracer = self.run_tests(runtests) - self.display_result() + self.display_result(runtests) need_rerun = self.need_rerun if self.ns.rerun and need_rerun: @@ -877,7 +892,7 @@ class Regrtest: if self.want_wait: input("Press any key to continue...") - setup_tests(self.ns) + setup_test_dir(self.ns.testdir) self.find_tests() exitcode = 0 diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ea5d056..bfb0718 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -208,9 +208,16 @@ class TestResult: @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple + fail_fast: bool = False + match_tests: FilterTuple | None = None + ignore_tests: FilterTuple | None = None match_tests_dict: FilterDict | None = None rerun: bool = False forever: bool = False + pgo: bool = False + pgo_extended: bool = False + output_on_failure: bool = False + timeout: float | None = None def copy(self, **override): state = dataclasses.asdict(self) @@ -295,11 +302,11 @@ def abs_module_name(test_name: str, test_dir: str | None) -> str: return 'test.' + test_name -def setup_support(ns: Namespace): - support.PGO = ns.pgo - support.PGO_EXTENDED = ns.pgo_extended - support.set_match_tests(ns.match_tests, ns.ignore_tests) - support.failfast = ns.failfast +def setup_support(runtests: RunTests, ns: Namespace): + support.PGO = runtests.pgo + support.PGO_EXTENDED = runtests.pgo_extended + support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + support.failfast = runtests.fail_fast support.verbose = ns.verbose if ns.xmlpath: support.junit_xml_list = [] @@ -307,12 +314,12 @@ def setup_support(ns: Namespace): support.junit_xml_list = None -def _runtest(result: TestResult, ns: Namespace) -> None: +def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Capture stdout and stderr, set faulthandler timeout, # and create JUnit XML report. verbose = ns.verbose - output_on_failure = ns.verbose3 - timeout = ns.timeout + output_on_failure = runtests.output_on_failure + timeout = runtests.timeout use_timeout = ( timeout is not None and threading_helper.can_start_thread @@ -321,7 +328,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: faulthandler.dump_traceback_later(timeout, exit=True) try: - setup_support(ns) + setup_support(runtests, ns) if output_on_failure: support.verbose = True @@ -341,7 +348,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: # warnings will be written to sys.stderr below. print_warning.orig_stderr = stream - _runtest_env_changed_exc(result, ns, display_failure=False) + _runtest_env_changed_exc(result, runtests, ns, display_failure=False) # Ignore output if the test passed successfully if result.state != State.PASSED: output = stream.getvalue() @@ -356,7 +363,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: else: # Tell tests to be moderately quiet support.verbose = verbose - _runtest_env_changed_exc(result, ns, display_failure=not verbose) + _runtest_env_changed_exc(result, runtests, ns, display_failure=not verbose) xml_list = support.junit_xml_list if xml_list: @@ -369,7 +376,7 @@ def _runtest(result: TestResult, ns: Namespace) -> None: support.junit_xml_list = None -def runtest(ns: Namespace, test_name: str) -> TestResult: +def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestResult: """Run a single test. ns -- regrtest namespace of options @@ -382,10 +389,11 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: """ start_time = time.perf_counter() result = TestResult(test_name) + pgo = runtests.pgo try: - _runtest(result, ns) + _runtest(result, runtests, ns) except: - if not ns.pgo: + if not pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) @@ -404,8 +412,8 @@ def run_unittest(test_mod): return support.run_unittest(tests) -def save_env(ns: Namespace, test_name: str): - return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo) +def save_env(test_name: str, runtests: RunTests, ns: Namespace): + return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=runtests.pgo) def regrtest_runner(result, test_func, ns) -> None: @@ -442,7 +450,7 @@ def regrtest_runner(result, test_func, ns) -> None: FOUND_GARBAGE = [] -def _load_run_test(result: TestResult, ns: Namespace) -> None: +def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None: # Load the test function, run the test function. module_name = abs_module_name(result.test_name, ns.testdir) @@ -458,7 +466,7 @@ def _load_run_test(result: TestResult, ns: Namespace) -> None: return run_unittest(test_mod) try: - with save_env(ns, result.test_name): + with save_env(result.test_name, runtests, ns): regrtest_runner(result, test_func, ns) finally: # First kill any dangling references to open files etc. @@ -482,7 +490,8 @@ def _load_run_test(result: TestResult, ns: Namespace) -> None: support.reap_children() -def _runtest_env_changed_exc(result: TestResult, ns: Namespace, +def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, + ns: Namespace, display_failure: bool = True) -> None: # Detect environment changes, handle exceptions. @@ -490,7 +499,8 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, # the environment support.environment_altered = False - if ns.pgo: + pgo = runtests.pgo + if pgo: display_failure = False test_name = result.test_name @@ -498,15 +508,15 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, clear_caches() support.gc_collect() - with save_env(ns, test_name): - _load_run_test(result, ns) + with save_env(test_name, runtests, ns): + _load_run_test(result, runtests, ns) except support.ResourceDenied as msg: - if not ns.quiet and not ns.pgo: + if not ns.quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as msg: - if not ns.quiet and not ns.pgo: + if not ns.quiet and not pgo: print(f"{test_name} skipped -- {msg}", flush=True) result.state = State.SKIPPED return @@ -536,7 +546,7 @@ def _runtest_env_changed_exc(result: TestResult, ns: Namespace, result.state = State.INTERRUPTED return except: - if not ns.pgo: + if not pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 45db24f..ecdde3a 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -10,7 +10,7 @@ import tempfile import threading import time import traceback -from typing import NamedTuple, NoReturn, Literal, Any, TextIO +from typing import NoReturn, Literal, Any, TextIO from test import support from test.support import os_helper @@ -19,9 +19,9 @@ from test.support import TestStats from test.libregrtest.cmdline import Namespace from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( - runtest, TestResult, State, PROGRESS_MIN_TIME, + run_single_test, TestResult, State, PROGRESS_MIN_TIME, FilterTuple, RunTests) -from test.libregrtest.setup import setup_tests +from test.libregrtest.setup import setup_tests, setup_test_dir from test.libregrtest.utils import format_duration, print_warning if sys.platform == 'win32': @@ -48,7 +48,6 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) class WorkerJob: runtests: RunTests namespace: Namespace - match_tests: FilterTuple | None = None class _EncodeWorkerJob(json.JSONEncoder): @@ -126,9 +125,10 @@ def worker_process(worker_json: str) -> NoReturn: runtests = worker_job.runtests ns = worker_job.namespace test_name = runtests.tests[0] - match_tests: FilterTuple | None = worker_job.match_tests + match_tests: FilterTuple | None = runtests.match_tests - setup_tests(ns) + setup_test_dir(ns.testdir) + setup_tests(runtests, ns) if runtests.rerun: if match_tests: @@ -138,10 +138,7 @@ def worker_process(worker_json: str) -> NoReturn: print(f"Re-running {test_name} in verbose mode", flush=True) ns.verbose = True - if match_tests is not None: - ns.match_tests = match_tests - - result = runtest(ns, test_name) + result = run_single_test(test_name, runtests, ns) print() # Force a newline (just in case) # Serialize TestResult as dict in JSON @@ -330,11 +327,13 @@ class TestWorkerProcess(threading.Thread): match_tests = self.runtests.get_match_tests(test_name) else: match_tests = None - worker_runtests = self.runtests.copy(tests=tests) + kwargs = {} + if match_tests: + kwargs['match_tests'] = match_tests + worker_runtests = self.runtests.copy(tests=tests, **kwargs) worker_job = WorkerJob( worker_runtests, - namespace=self.ns, - match_tests=match_tests) + namespace=self.ns) # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. @@ -401,7 +400,7 @@ class TestWorkerProcess(threading.Thread): return MultiprocessResult(result, stdout) def run(self) -> None: - fail_fast = self.ns.failfast + fail_fast = self.runtests.fail_fast fail_env_changed = self.ns.fail_env_changed while not self._stopped: try: @@ -473,7 +472,6 @@ def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: class MultiprocessTestRunner: def __init__(self, regrtest: Regrtest, runtests: RunTests) -> None: ns = regrtest.ns - timeout = ns.timeout self.regrtest = regrtest self.runtests = runtests @@ -483,24 +481,24 @@ class MultiprocessTestRunner: self.output: queue.Queue[QueueOutput] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) - if timeout is not None: + self.timeout = runtests.timeout + if self.timeout is not None: # Rely on faulthandler to kill a worker process. This timouet is # when faulthandler fails to kill a worker process. Give a maximum # of 5 minutes to faulthandler to kill the worker. - self.worker_timeout = min(timeout * 1.5, timeout + 5 * 60) + self.worker_timeout = min(self.timeout * 1.5, self.timeout + 5 * 60) else: self.worker_timeout = None self.workers = None def start_workers(self) -> None: use_mp = self.ns.use_mp - timeout = self.ns.timeout self.workers = [TestWorkerProcess(index, self) for index in range(1, use_mp + 1)] msg = f"Run tests in parallel using {len(self.workers)} child processes" - if timeout: + if self.timeout: msg += (" (timeout: %s, worker timeout: %s)" - % (format_duration(timeout), + % (format_duration(self.timeout), format_duration(self.worker_timeout))) self.log(msg) for worker in self.workers: @@ -514,9 +512,8 @@ class MultiprocessTestRunner: worker.wait_stopped(start_time) def _get_result(self) -> QueueOutput | None: - pgo = self.ns.pgo - use_faulthandler = (self.ns.timeout is not None) - timeout = PROGRESS_UPDATE + pgo = self.runtests.pgo + use_faulthandler = (self.timeout is not None) # bpo-46205: check the status of workers every iteration to avoid # waiting forever on an empty queue. @@ -527,7 +524,7 @@ class MultiprocessTestRunner: # wait for a thread try: - return self.output.get(timeout=timeout) + return self.output.get(timeout=PROGRESS_UPDATE) except queue.Empty: pass @@ -544,7 +541,7 @@ class MultiprocessTestRunner: def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result - pgo = self.ns.pgo + pgo = self.runtests.pgo text = str(result) if mp_result.err_msg: @@ -580,9 +577,8 @@ class MultiprocessTestRunner: return result def run_tests(self) -> None: - fail_fast = self.ns.failfast + fail_fast = self.runtests.fail_fast fail_env_changed = self.ns.fail_env_changed - timeout = self.ns.timeout self.start_workers() @@ -600,7 +596,7 @@ class MultiprocessTestRunner: print() self.regrtest.interrupted = True finally: - if timeout is not None: + if self.timeout is not None: faulthandler.cancel_dump_traceback_later() # Always ensure that all worker processes are no longer diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index b76bece..f640362 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -18,7 +18,14 @@ from test.libregrtest.utils import (setup_unraisable_hook, UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" -def setup_tests(ns): +def setup_test_dir(testdir): + if testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(testdir)) + + +def setup_tests(runtests, ns): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -44,11 +51,6 @@ def setup_tests(ns): replace_stdout() support.record_original_stdout(sys.stdout) - if ns.testdir: - # Prepend test directory to sys.path, so runtest() will be able - # to locate tests - sys.path.insert(0, os.path.abspath(ns.testdir)) - # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). @@ -88,16 +90,17 @@ def setup_tests(ns): setup_unraisable_hook() setup_threading_excepthook() - if ns.timeout is not None: + timeout = runtests.timeout + if timeout is not None: # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT - support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, ns.timeout / 40) - support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, ns.timeout / 4) + support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, timeout / 40) + support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, timeout / 4) # If --timeout is short: reduce timeouts - support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, ns.timeout) - support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, ns.timeout) - support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, ns.timeout) - support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, ns.timeout) + support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, timeout) + support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, timeout) + support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) + support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) if ns.xmlpath: from test.support.testresult import RegressionTestResult |