diff options
author | Victor Stinner <vstinner@python.org> | 2023-09-10 03:04:26 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-09-10 03:04:26 (GMT) |
commit | 0eab2427b149cd46e0dee3efbb6b2cfca2a4f723 (patch) | |
tree | 61d639554eeb6741d9e5ace97f8d39fc2f99d727 | |
parent | db5bfe91f822bb06b4c8fe7c58e649694c854bf2 (diff) | |
download | cpython-0eab2427b149cd46e0dee3efbb6b2cfca2a4f723.zip cpython-0eab2427b149cd46e0dee3efbb6b2cfca2a4f723.tar.gz cpython-0eab2427b149cd46e0dee3efbb6b2cfca2a4f723.tar.bz2 |
gh-109162: libregrtest: add Logger class (#109212)
* Add Logger class in a new logger.py file.
* Move Regrtest attributes to Logger:
* start_time
* test_count_text
* test_count_width
* win_load_tracker
* Move Regrtest method to Logger:
* log()
* getloadavg(): rename to get_load_avg()
* set_tests()
* Add methods to the Logger class:
* start_load_tracker()
* stop_load_tracker()
-rw-r--r-- | Lib/test/libregrtest/logger.py | 69 | ||||
-rw-r--r-- | Lib/test/libregrtest/main.py | 148 | ||||
-rw-r--r-- | Lib/test/libregrtest/runtest_mp.py | 2 | ||||
-rw-r--r-- | Lib/test/libregrtest/utils.py | 3 |
4 files changed, 123 insertions, 99 deletions
diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py new file mode 100644 index 0000000..c4498a4 --- /dev/null +++ b/Lib/test/libregrtest/logger.py @@ -0,0 +1,69 @@ +import os +import time + +from test.libregrtest.runtest import RunTests +from test.libregrtest.utils import print_warning, MS_WINDOWS + +if MS_WINDOWS: + from test.libregrtest.win_utils import WindowsLoadTracker + + +class Logger: + def __init__(self): + self.start_time = time.perf_counter() + self.test_count_text = '' + self.test_count_width = 3 + self.win_load_tracker = None + + def log(self, line: str = '') -> None: + empty = not line + + # add the system load prefix: "load avg: 1.80 " + load_avg = self.get_load_avg() + if load_avg is not None: + line = f"load avg: {load_avg:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " + test_time = time.perf_counter() - self.start_time + + mins, secs = divmod(int(test_time), 60) + hours, mins = divmod(mins, 60) + test_time = "%d:%02d:%02d" % (hours, mins, secs) + + line = f"{test_time} {line}" + if empty: + line = line[:-1] + + print(line, flush=True) + + def get_load_avg(self) -> float | None: + if hasattr(os, 'getloadavg'): + return os.getloadavg()[0] + if self.win_load_tracker is not None: + return self.win_load_tracker.getloadavg() + return None + + def set_tests(self, runtests: RunTests) -> None: + if runtests.forever: + self.test_count_text = '' + self.test_count_width = 3 + else: + self.test_count_text = '/{}'.format(len(runtests.tests)) + self.test_count_width = len(self.test_count_text) - 1 + + def start_load_tracker(self) -> None: + if not MS_WINDOWS: + return + + try: + self.win_load_tracker = WindowsLoadTracker() + except PermissionError as error: + # Standard accounts may not have access to the performance + # counters. + print_warning(f'Failed to create WindowsLoadTracker: {error}') + + def stop_load_tracker(self) -> None: + if self.win_load_tracker is None: + return + self.win_load_tracker.close() + self.win_load_tracker = None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 75c3d0e..74ef69b 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,6 +10,7 @@ import tempfile import time import unittest from test.libregrtest.cmdline import _parse_args, Namespace +from test.libregrtest.logger import Logger from test.libregrtest.runtest import ( findtests, split_test_packages, run_single_test, abs_module_name, PROGRESS_MIN_TIME, State, RunTests, HuntRefleak, @@ -54,6 +55,8 @@ class Regrtest: on the command line. """ def __init__(self, ns: Namespace): + self.logger = Logger() + # Actions self.want_header: bool = ns.header self.want_list_tests: bool = ns.list_tests @@ -137,29 +140,8 @@ class Regrtest: self.next_single_test: TestName | None = None self.next_single_filename: StrPath | None = None - # misc - self.win_load_tracker = None - def log(self, line=''): - empty = not line - - # add the system load prefix: "load avg: 1.80 " - load_avg = self.getloadavg() - if load_avg is not None: - line = f"load avg: {load_avg:.2f} {line}" - - # add the timestamp prefix: "0:01:05 " - test_time = time.perf_counter() - self.start_time - - mins, secs = divmod(int(test_time), 60) - hours, mins = divmod(mins, 60) - test_time = "%d:%02d:%02d" % (hours, mins, secs) - - line = f"{test_time} {line}" - if empty: - line = line[:-1] - - print(line, flush=True) + self.logger.log(line) def display_progress(self, test_index, text): if self.quiet: @@ -293,7 +275,7 @@ class Regrtest: fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False) - self.set_tests(runtests) + self.logger.set_tests(runtests) self._run_tests_mp(runtests, self.num_workers) return runtests @@ -437,44 +419,7 @@ class Regrtest: def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from test.libregrtest.runtest_mp import RunWorkers - - # If we're on windows and this is the parent runner (not a worker), - # track the load average. - if sys.platform == 'win32': - from test.libregrtest.win_utils import WindowsLoadTracker - - try: - self.win_load_tracker = WindowsLoadTracker() - except PermissionError as error: - # Standard accounts may not have access to the performance - # counters. - print(f'Failed to create WindowsLoadTracker: {error}') - - try: - RunWorkers(self, runtests, num_workers).run() - finally: - if self.win_load_tracker is not None: - self.win_load_tracker.close() - self.win_load_tracker = None - - def set_tests(self, runtests: RunTests): - self.tests = runtests.tests - if runtests.forever: - self.test_count_text = '' - self.test_count_width = 3 - else: - self.test_count_text = '/{}'.format(len(self.tests)) - self.test_count_width = len(self.test_count_text) - 1 - - def run_tests(self, runtests: RunTests): - self.first_runtests = runtests - self.set_tests(runtests) - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - tracer = None - else: - tracer = self.run_tests_sequentially(runtests) - return tracer + RunWorkers(self, runtests, num_workers).run() def finalize_tests(self, tracer): if self.next_single_filename: @@ -496,7 +441,7 @@ class Regrtest: self.results.write_junit(self.junit_filename) def display_summary(self): - duration = time.perf_counter() - self.start_time + duration = time.perf_counter() - self.logger.start_time filtered = bool(self.match_tests) or bool(self.ignore_tests) # Total duration @@ -619,35 +564,8 @@ class Regrtest: sys.exit(exc.code) - def getloadavg(self): - if self.win_load_tracker is not None: - return self.win_load_tracker.getloadavg() - - if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] - - return None - - def action_run_tests(self): - if self.hunt_refleak and self.hunt_refleak.warmups < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and " - "less than 3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) - - # For a partial run, we do not need to clutter the output. - if (self.want_header - or not(self.pgo or self.quiet or self.single_test_run - or self.tests or self.cmdline_args)): - self.display_header() - - if self.randomize: - print("Using random seed", self.random_seed) - - if self.num_workers < 0: - # Use all cores + extras for tests that like to sleep - self.num_workers = 2 + (os.cpu_count() or 1) - - runtests = RunTests( + def create_run_tests(self): + return RunTests( tuple(self.selected), fail_fast=self.fail_fast, match_tests=self.match_tests, @@ -668,17 +586,53 @@ class Regrtest: python_cmd=self.python_cmd, ) + def run_tests(self) -> int: + if self.hunt_refleak and self.hunt_refleak.warmups < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and " + "less than 3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) + + if self.num_workers < 0: + # Use all CPUs + 2 extra worker processes for tests + # that like to sleep + self.num_workers = (os.cpu_count() or 1) + 2 + + # For a partial run, we do not need to clutter the output. + if (self.want_header + or not(self.pgo or self.quiet or self.single_test_run + or self.tests or self.cmdline_args)): + self.display_header() + + if self.randomize: + print("Using random seed", self.random_seed) + + runtests = self.create_run_tests() + self.first_runtests = runtests + self.logger.set_tests(runtests) + setup_tests(runtests) - tracer = self.run_tests(runtests) - self.display_result(runtests) + self.logger.start_load_tracker() + try: + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + tracer = None + else: + tracer = self.run_tests_sequentially(runtests) - if self.want_rerun and self.results.need_rerun(): - self.rerun_failed_tests(runtests) + self.display_result(runtests) + + if self.want_rerun and self.results.need_rerun(): + self.rerun_failed_tests(runtests) + finally: + self.logger.stop_load_tracker() self.display_summary() self.finalize_tests(tracer) + return self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + def _main(self): if self.is_worker(): from test.libregrtest.runtest_mp import worker_process @@ -697,9 +651,7 @@ class Regrtest: elif self.want_list_cases: self.list_cases() else: - self.action_run_tests() - exitcode = self.results.get_exitcode(self.fail_env_changed, - self.fail_rerun) + exitcode = self.run_tests() sys.exit(exitcode) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 179b610..c4cffff 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -433,7 +433,7 @@ def get_running(workers: list[WorkerThread]) -> list[str]: class RunWorkers: def __init__(self, regrtest: Regrtest, runtests: RunTests, num_workers: int) -> None: self.results: TestResults = regrtest.results - self.log = regrtest.log + self.log = regrtest.logger.log self.display_progress = regrtest.display_progress self.num_workers = num_workers self.runtests = runtests diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 9a60a3d..57d8543 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -6,6 +6,9 @@ import textwrap from test import support +MS_WINDOWS = (sys.platform == 'win32') + + def format_duration(seconds): ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) |