diff options
Diffstat (limited to 'Lib/test/libregrtest/main.py')
| -rw-r--r-- | Lib/test/libregrtest/main.py | 195 |
1 files changed, 133 insertions, 62 deletions
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0125227..6e6423e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -11,14 +11,14 @@ import time import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, runtest, get_abs_module, is_failed, - STDTESTS, NOTTESTS, PROGRESS_MIN_TIME, - Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted, - ChildError, DidNotRun) + findtests, split_test_packages, runtest, get_abs_module, + PROGRESS_MIN_TIME, State) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests -from test.libregrtest.utils import removepy, count, format_duration, printlist +from test.libregrtest.utils import (removepy, count, format_duration, + printlist, get_build_info) from test import support +from test.support import TestStats from test.support import os_helper from test.support import threading_helper @@ -77,13 +77,14 @@ class Regrtest: self.good = [] self.bad = [] self.skipped = [] - self.resource_denieds = [] + self.resource_denied = [] self.environment_changed = [] self.run_no_tests = [] self.need_rerun = [] self.rerun = [] self.first_result = None self.interrupted = False + self.stats_dict: dict[str, TestStats] = {} # used by --slow self.test_times = [] @@ -92,7 +93,7 @@ class Regrtest: self.tracer = None # used to display the progress bar "[ 3/100]" - self.start_time = time.monotonic() + self.start_time = time.perf_counter() self.test_count = '' self.test_count_width = 1 @@ -110,36 +111,41 @@ class Regrtest: def get_executed(self): return (set(self.good) | set(self.bad) | set(self.skipped) - | set(self.resource_denieds) | set(self.environment_changed) + | set(self.resource_denied) | set(self.environment_changed) | set(self.run_no_tests)) def accumulate_result(self, result, rerun=False): - test_name = result.name - - if not isinstance(result, (ChildError, Interrupted)) and not rerun: - self.test_times.append((result.duration_sec, test_name)) - - if isinstance(result, Passed): - self.good.append(test_name) - elif isinstance(result, ResourceDenied): - self.skipped.append(test_name) - self.resource_denieds.append(test_name) - elif isinstance(result, Skipped): - self.skipped.append(test_name) - elif isinstance(result, EnvChanged): - self.environment_changed.append(test_name) - elif isinstance(result, Failed): - if not rerun: - self.bad.append(test_name) - self.need_rerun.append(result) - elif isinstance(result, DidNotRun): - self.run_no_tests.append(test_name) - elif isinstance(result, Interrupted): - self.interrupted = True - else: - raise ValueError("invalid test result: %r" % result) + test_name = result.test_name + + if result.has_meaningful_duration() and not rerun: + self.test_times.append((result.duration, test_name)) - if rerun and not isinstance(result, (Failed, Interrupted)): + match result.state: + case State.PASSED: + self.good.append(test_name) + case State.ENV_CHANGED: + self.environment_changed.append(test_name) + case State.SKIPPED: + self.skipped.append(test_name) + case State.RESOURCE_DENIED: + self.skipped.append(test_name) + self.resource_denied.append(test_name) + case State.INTERRUPTED: + self.interrupted = True + case State.DID_NOT_RUN: + self.run_no_tests.append(test_name) + case _: + if result.is_failed(self.ns.fail_env_changed): + if not rerun: + self.bad.append(test_name) + self.need_rerun.append(result) + else: + raise ValueError(f"invalid test state: {state!r}") + + if result.stats is not None: + self.stats_dict[result.test_name] = result.stats + + if rerun and not(result.is_failed(False) or result.state == State.INTERRUPTED): self.bad.remove(test_name) xml_data = result.xml_data @@ -161,7 +167,7 @@ class Regrtest: line = f"load avg: {load_avg:.2f} {line}" # add the timestamp prefix: "0:01:05 " - test_time = time.monotonic() - self.start_time + test_time = time.perf_counter() - self.start_time mins, secs = divmod(int(test_time), 60) hours, mins = divmod(mins, 60) @@ -245,26 +251,23 @@ class Regrtest: # add default PGO tests if no tests are specified setup_pgo_tests(self.ns) - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() + exclude = set() if self.ns.exclude: for arg in self.ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) + exclude.add(arg) self.ns.args = [] - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if self.ns.testdir: - alltests = findtests(self.ns.testdir, list(), set()) - else: - alltests = findtests(self.ns.testdir, stdtests, nottests) + alltests = findtests(testdir=self.ns.testdir, exclude=exclude) if not self.ns.fromfile: - self.selected = self.tests or self.ns.args or alltests + self.selected = self.tests or self.ns.args + if self.selected: + self.selected = split_test_packages(self.selected) + else: + self.selected = alltests else: self.selected = self.tests + if self.ns.single: self.selected = self.selected[:1] try: @@ -339,7 +342,7 @@ class Regrtest: rerun_list = list(self.need_rerun) self.need_rerun.clear() for result in rerun_list: - test_name = result.name + test_name = result.test_name self.rerun.append(test_name) errors = result.errors or [] @@ -366,7 +369,7 @@ class Regrtest: self.accumulate_result(result, rerun=True) - if isinstance(result, Interrupted): + if result.state == State.INTERRUPTED: break if self.bad: @@ -463,7 +466,7 @@ class Regrtest: previous_test = None for test_index, test_name in enumerate(self.tests, 1): - start_time = time.monotonic() + start_time = time.perf_counter() text = test_name if previous_test: @@ -482,14 +485,14 @@ class Regrtest: result = runtest(self.ns, test_name) self.accumulate_result(result) - if isinstance(result, Interrupted): + if result.state == State.INTERRUPTED: break previous_test = str(result) - test_time = time.monotonic() - start_time + test_time = time.perf_counter() - start_time if test_time >= PROGRESS_MIN_TIME: previous_test = "%s in %s" % (previous_test, format_duration(test_time)) - elif isinstance(result, Passed): + elif result.state == State.PASSED: # be quiet: say nothing if the test passed shortly previous_test = None @@ -498,7 +501,7 @@ class Regrtest: if module not in save_modules and module.startswith("test."): support.unload(module) - if self.ns.failfast and is_failed(result, self.ns): + if self.ns.failfast and result.is_failed(self.ns.fail_env_changed): break if previous_test: @@ -518,12 +521,44 @@ class Regrtest: print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), "%s-endian" % sys.byteorder) + print("== Python build:", ' '.join(get_build_info())) print("== cwd:", os.getcwd()) cpu_count = os.cpu_count() if cpu_count: print("== CPU count:", cpu_count) print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) + self.display_sanitizers() + + def display_sanitizers(self): + # This makes it easier to remember what to set in your local + # environment when trying to reproduce a sanitizer failure. + asan = support.check_sanitizer(address=True) + msan = support.check_sanitizer(memory=True) + ubsan = support.check_sanitizer(ub=True) + sanitizers = [] + if asan: + sanitizers.append("address") + if msan: + sanitizers.append("memory") + if ubsan: + sanitizers.append("undefined behavior") + if not sanitizers: + return + + print(f"== sanitizers: {', '.join(sanitizers)}") + for sanitizer, env_var in ( + (asan, "ASAN_OPTIONS"), + (msan, "MSAN_OPTIONS"), + (ubsan, "UBSAN_OPTIONS"), + ): + options= os.environ.get(env_var) + if sanitizer and options is not None: + print(f"== {env_var}={options!r}") + + def no_tests_run(self): + return not any((self.good, self.bad, self.skipped, self.interrupted, + self.environment_changed)) def get_tests_result(self): result = [] @@ -531,9 +566,8 @@ class Regrtest: result.append("FAILURE") elif self.ns.fail_env_changed and self.environment_changed: result.append("ENV CHANGED") - elif not any((self.good, self.bad, self.skipped, self.interrupted, - self.environment_changed)): - result.append("NO TEST RUN") + elif self.no_tests_run(): + result.append("NO TESTS RAN") if self.interrupted: result.append("INTERRUPTED") @@ -609,13 +643,48 @@ class Regrtest: coverdir=self.ns.coverdir) print() - duration = time.monotonic() - self.start_time - print("Total duration: %s" % format_duration(duration)) - print("Tests result: %s" % self.get_tests_result()) + self.display_summary() if self.ns.runleaks: os.system("leaks %d" % os.getpid()) + def display_summary(self): + duration = time.perf_counter() - self.start_time + + # Total duration + print("Total duration: %s" % format_duration(duration)) + + # Total tests + total = TestStats() + for stats in self.stats_dict.values(): + total.accumulate(stats) + stats = [f'run={total.tests_run:,}'] + if total.failures: + stats.append(f'failures={total.failures:,}') + if total.skipped: + stats.append(f'skipped={total.skipped:,}') + print(f"Total tests: {' '.join(stats)}") + + # Total test files + report = [f'success={len(self.good)}'] + if self.bad: + report.append(f'failed={len(self.bad)}') + if self.environment_changed: + report.append(f'env_changed={len(self.environment_changed)}') + if self.skipped: + report.append(f'skipped={len(self.skipped)}') + if self.resource_denied: + report.append(f'resource_denied={len(self.resource_denied)}') + if self.rerun: + report.append(f'rerun={len(self.rerun)}') + if self.run_no_tests: + report.append(f'run_no_tests={len(self.run_no_tests)}') + print(f"Total test files: {' '.join(report)}") + + # Result + result = self.get_tests_result() + print(f"Result: {result}") + def save_xml_result(self): if not self.ns.xmlpath and not self.testsuite_xml: return @@ -782,11 +851,13 @@ class Regrtest: self.save_xml_result() if self.bad: - sys.exit(2) + sys.exit(EXITCODE_BAD_TEST) if self.interrupted: - sys.exit(130) + sys.exit(EXITCODE_INTERRUPTED) if self.ns.fail_env_changed and self.environment_changed: - sys.exit(3) + sys.exit(EXITCODE_ENV_CHANGED) + if self.no_tests_run(): + sys.exit(EXITCODE_NO_TESTS_RAN) sys.exit(0) |