diff options
Diffstat (limited to 'Lib/test')
| -rw-r--r-- | Lib/test/eintrdata/eintr_tester.py | 3 | ||||
| -rw-r--r-- | Lib/test/libregrtest/cmdline.py | 4 | ||||
| -rw-r--r-- | Lib/test/libregrtest/main.py | 47 | ||||
| -rw-r--r-- | Lib/test/libregrtest/runtest.py | 30 | ||||
| -rw-r--r-- | Lib/test/libregrtest/runtest_mp.py | 7 | ||||
| -rw-r--r-- | Lib/test/support/__init__.py | 18 | ||||
| -rw-r--r-- | Lib/test/support/testresult.py | 201 | ||||
| -rw-r--r-- | Lib/test/test_argparse.py | 18 |
8 files changed, 298 insertions, 30 deletions
diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index bc308fe..1caeafe 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -52,7 +52,8 @@ class EINTRBaseTest(unittest.TestCase): # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD if hasattr(faulthandler, 'dump_traceback_later'): - faulthandler.dump_traceback_later(10 * 60, exit=True) + faulthandler.dump_traceback_later(10 * 60, exit=True, + file=sys.__stderr__) @classmethod def stop_alarm(cls): diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index bd126b3..538ff05 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -268,6 +268,10 @@ def _create_parser(): help='if a test file alters the environment, mark ' 'the test as failed') + group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', + help='writes JUnit-style XML results to the specified ' + 'file') + return parser diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index a176db5..1fd4132 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -100,8 +100,11 @@ class Regrtest: self.next_single_test = None self.next_single_filename = None + # used by --junit-xml + self.testsuite_xml = None + def accumulate_result(self, test, result): - ok, test_time = result + ok, test_time, xml_data = result if ok not in (CHILD_ERROR, INTERRUPTED): self.test_times.append((test_time, test)) if ok == PASSED: @@ -118,6 +121,15 @@ class Regrtest: elif ok != INTERRUPTED: raise ValueError("invalid test result: %r" % ok) + if xml_data: + import xml.etree.ElementTree as ET + for e in xml_data: + try: + self.testsuite_xml.append(ET.fromstring(e)) + except ET.ParseError: + print(xml_data, file=sys.__stderr__) + raise + def display_progress(self, test_index, test): if self.ns.quiet: return @@ -164,6 +176,9 @@ class Regrtest: file=sys.stderr) ns.findleaks = False + if ns.xmlpath: + support.junit_xml_list = self.testsuite_xml = [] + # Strip .py extensions. removepy(ns.args) @@ -384,7 +399,7 @@ class Regrtest: result = runtest(self.ns, test) except KeyboardInterrupt: self.interrupted = True - self.accumulate_result(test, (INTERRUPTED, None)) + self.accumulate_result(test, (INTERRUPTED, None, None)) break else: self.accumulate_result(test, result) @@ -508,6 +523,31 @@ class Regrtest: if self.ns.runleaks: os.system("leaks %d" % os.getpid()) + def save_xml_result(self): + if not self.ns.xmlpath and not self.testsuite_xml: + return + + import xml.etree.ElementTree as ET + root = ET.Element("testsuites") + + # Manually count the totals for the overall summary + totals = {'tests': 0, 'errors': 0, 'failures': 0} + for suite in self.testsuite_xml: + root.append(suite) + for k in totals: + try: + totals[k] += int(suite.get(k, 0)) + except ValueError: + pass + + for k, v in totals.items(): + root.set(k, str(v)) + + xmlpath = os.path.join(support.SAVEDCWD, self.ns.xmlpath) + with open(xmlpath, 'wb') as f: + for s in ET.tostringlist(root): + f.write(s) + def main(self, tests=None, **kwargs): global TEMPDIR @@ -570,6 +610,9 @@ class Regrtest: self.rerun_failed_tests() self.finalize() + + self.save_xml_result() + if self.bad: sys.exit(2) if self.interrupted: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 3e1afd4..4f41080 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -85,8 +85,8 @@ def runtest(ns, test): ns -- regrtest namespace of options test -- the name of the test - Returns the tuple (result, test_time), where result is one of the - constants: + Returns the tuple (result, test_time, xml_data), where result is one + of the constants: INTERRUPTED KeyboardInterrupt when run under -j RESOURCE_DENIED test skipped because resource denied @@ -94,6 +94,9 @@ def runtest(ns, test): ENV_CHANGED test failed because it changed the execution environment FAILED test failed PASSED test passed + + If ns.xmlpath is not None, xml_data is a list containing each + generated testsuite element. """ output_on_failure = ns.verbose3 @@ -106,22 +109,13 @@ def runtest(ns, test): # reset the environment_altered flag to detect if a test altered # the environment support.environment_altered = False + support.junit_xml_list = xml_list = [] if ns.xmlpath else None if ns.failfast: support.failfast = True if output_on_failure: support.verbose = True - # Reuse the same instance to all calls to runtest(). Some - # tests keep a reference to sys.stdout or sys.stderr - # (eg. test_argparse). - if runtest.stringio is None: - stream = io.StringIO() - runtest.stringio = stream - else: - stream = runtest.stringio - stream.seek(0) - stream.truncate() - + stream = io.StringIO() orig_stdout = sys.stdout orig_stderr = sys.stderr try: @@ -138,12 +132,18 @@ def runtest(ns, test): else: support.verbose = ns.verbose # Tell tests to be moderately quiet result = runtest_inner(ns, test, display_failure=not ns.verbose) - return result + + if xml_list: + import xml.etree.ElementTree as ET + xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list] + else: + xml_data = None + return result + (xml_data,) finally: if use_timeout: faulthandler.cancel_dump_traceback_later() cleanup_test_droppings(test, ns.verbose) -runtest.stringio = None + support.junit_xml_list = None def post_test_cleanup(): diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 1f07cfb..6190574 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -67,7 +67,7 @@ def run_tests_worker(worker_args): try: result = runtest(ns, testname) except KeyboardInterrupt: - result = INTERRUPTED, '' + result = INTERRUPTED, '', None except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) @@ -122,7 +122,7 @@ class MultiprocessThread(threading.Thread): self.current_test = None if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) + result = (CHILD_ERROR, "Exit code %s" % retcode, None) self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) return False @@ -133,6 +133,7 @@ class MultiprocessThread(threading.Thread): return True result = json.loads(result) + assert len(result) == 3, f"Invalid result tuple: {result!r}" self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) return False @@ -195,7 +196,7 @@ def run_tests_multiprocess(regrtest): regrtest.accumulate_result(test, result) # Display progress - ok, test_time = result + ok, test_time, xml_data = result text = format_test_result(test, ok) if (ok not in (CHILD_ERROR, INTERRUPTED) and test_time >= PROGRESS_MIN_TIME diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index de997b2..19701cf 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -6,6 +6,7 @@ if __name__ != 'test.support': import asyncio.events import collections.abc import contextlib +import datetime import errno import faulthandler import fnmatch @@ -13,6 +14,7 @@ import functools import gc import importlib import importlib.util +import io import logging.handlers import nntplib import os @@ -34,6 +36,8 @@ import unittest import urllib.error import warnings +from .testresult import get_test_runner + try: import multiprocessing.process except ImportError: @@ -295,6 +299,7 @@ use_resources = None # Flag set to [] by regrtest.py max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 +junit_xml_list = None # list of testsuite XML elements failfast = False # _original_stdout is meant to hold stdout at the time regrtest began. @@ -1891,13 +1896,16 @@ def _filter_suite(suite, pred): def _run_suite(suite): """Run tests from a unittest.TestSuite-derived class.""" - if verbose: - runner = unittest.TextTestRunner(sys.stdout, verbosity=2, - failfast=failfast) - else: - runner = BasicTestRunner() + runner = get_test_runner(sys.stdout, verbosity=verbose) + + # TODO: Remove this before merging (here for easy comparison with old impl) + #runner = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=failfast) result = runner.run(suite) + + if junit_xml_list is not None: + junit_xml_list.append(result.get_xml_element()) + if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: err = result.errors[0][1] diff --git a/Lib/test/support/testresult.py b/Lib/test/support/testresult.py new file mode 100644 index 0000000..8988d3d --- /dev/null +++ b/Lib/test/support/testresult.py @@ -0,0 +1,201 @@ +'''Test runner and result class for the regression test suite. + +''' + +import functools +import io +import sys +import time +import traceback +import unittest + +import xml.etree.ElementTree as ET + +from datetime import datetime + +class RegressionTestResult(unittest.TextTestResult): + separator1 = '=' * 70 + '\n' + separator2 = '-' * 70 + '\n' + + def __init__(self, stream, descriptions, verbosity): + super().__init__(stream=stream, descriptions=descriptions, verbosity=0) + self.buffer = True + self.__suite = ET.Element('testsuite') + self.__suite.set('start', datetime.utcnow().isoformat(' ')) + + self.__e = None + self.__start_time = None + self.__results = [] + self.__verbose = bool(verbosity) + + @classmethod + def __getId(cls, test): + try: + test_id = test.id + except AttributeError: + return str(test) + try: + return test_id() + except TypeError: + return str(test_id) + return repr(test) + + def startTest(self, test): + super().startTest(test) + self.__e = e = ET.SubElement(self.__suite, 'testcase') + self.__start_time = time.perf_counter() + if self.__verbose: + self.stream.write(f'{self.getDescription(test)} ... ') + self.stream.flush() + + def _add_result(self, test, capture=False, **args): + e = self.__e + self.__e = None + if e is None: + return + e.set('name', args.pop('name', self.__getId(test))) + e.set('status', args.pop('status', 'run')) + e.set('result', args.pop('result', 'completed')) + if self.__start_time: + e.set('time', f'{time.perf_counter() - self.__start_time:0.6f}') + + if capture: + stdout = self._stdout_buffer.getvalue().rstrip() + ET.SubElement(e, 'system-out').text = stdout + stderr = self._stderr_buffer.getvalue().rstrip() + ET.SubElement(e, 'system-err').text = stderr + + for k, v in args.items(): + if not k or not v: + continue + e2 = ET.SubElement(e, k) + if hasattr(v, 'items'): + for k2, v2 in v.items(): + if k2: + e2.set(k2, str(v2)) + else: + e2.text = str(v2) + else: + e2.text = str(v) + + def __write(self, c, word): + if self.__verbose: + self.stream.write(f'{word}\n') + + @classmethod + def __makeErrorDict(cls, err_type, err_value, err_tb): + if isinstance(err_type, type): + if err_type.__module__ == 'builtins': + typename = err_type.__name__ + else: + typename = f'{err_type.__module__}.{err_type.__name__}' + else: + typename = repr(err_type) + + msg = traceback.format_exception(err_type, err_value, None) + tb = traceback.format_exception(err_type, err_value, err_tb) + + return { + 'type': typename, + 'message': ''.join(msg), + '': ''.join(tb), + } + + def addError(self, test, err): + self._add_result(test, True, error=self.__makeErrorDict(*err)) + super().addError(test, err) + self.__write('E', 'ERROR') + + def addExpectedFailure(self, test, err): + self._add_result(test, True, output=self.__makeErrorDict(*err)) + super().addExpectedFailure(test, err) + self.__write('x', 'expected failure') + + def addFailure(self, test, err): + self._add_result(test, True, failure=self.__makeErrorDict(*err)) + super().addFailure(test, err) + self.__write('F', 'FAIL') + + def addSkip(self, test, reason): + self._add_result(test, skipped=reason) + super().addSkip(test, reason) + self.__write('S', f'skipped {reason!r}') + + def addSuccess(self, test): + self._add_result(test) + super().addSuccess(test) + self.__write('.', 'ok') + + def addUnexpectedSuccess(self, test): + self._add_result(test, outcome='UNEXPECTED_SUCCESS') + super().addUnexpectedSuccess(test) + self.__write('u', 'unexpected success') + + def printErrors(self): + if self.__verbose: + self.stream.write('\n') + self.printErrorList('ERROR', self.errors) + self.printErrorList('FAIL', self.failures) + + def printErrorList(self, flavor, errors): + for test, err in errors: + self.stream.write(self.separator1) + self.stream.write(f'{flavor}: {self.getDescription(test)}\n') + self.stream.write(self.separator2) + self.stream.write('%s\n' % err) + + def get_xml_element(self): + e = self.__suite + e.set('tests', str(self.testsRun)) + e.set('errors', str(len(self.errors))) + e.set('failures', str(len(self.failures))) + return e + +class QuietRegressionTestRunner: + def __init__(self, stream): + self.result = RegressionTestResult(stream, None, 0) + + def run(self, test): + test(self.result) + return self.result + +def get_test_runner_class(verbosity): + if verbosity: + return functools.partial(unittest.TextTestRunner, + resultclass=RegressionTestResult, + buffer=True, + verbosity=verbosity) + return QuietRegressionTestRunner + +def get_test_runner(stream, verbosity): + return get_test_runner_class(verbosity)(stream) + +if __name__ == '__main__': + class TestTests(unittest.TestCase): + def test_pass(self): + pass + + def test_pass_slow(self): + time.sleep(1.0) + + def test_fail(self): + print('stdout', file=sys.stdout) + print('stderr', file=sys.stderr) + self.fail('failure message') + + def test_error(self): + print('stdout', file=sys.stdout) + print('stderr', file=sys.stderr) + raise RuntimeError('error message') + + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestTests)) + stream = io.StringIO() + runner_cls = get_test_runner_class(sum(a == '-v' for a in sys.argv)) + runner = runner_cls(sys.stdout) + result = runner.run(suite) + print('Output:', stream.getvalue()) + print('XML: ', end='') + for s in ET.tostringlist(result.get_xml_element()): + print(s.decode(), end='') + print() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index f0802a5..c0c7cb0 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1459,6 +1459,16 @@ class TestFileTypeRepr(TestCase): type = argparse.FileType('r', 1, errors='replace') self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) +class StdStreamComparer: + def __init__(self, attr): + self.attr = attr + + def __eq__(self, other): + return other == getattr(sys, self.attr) + +eq_stdin = StdStreamComparer('stdin') +eq_stdout = StdStreamComparer('stdout') +eq_stderr = StdStreamComparer('stderr') class RFile(object): seen = {} @@ -1497,7 +1507,7 @@ class TestFileTypeR(TempDirMixin, ParserTestCase): ('foo', NS(x=None, spam=RFile('foo'))), ('-x foo bar', NS(x=RFile('foo'), spam=RFile('bar'))), ('bar -x foo', NS(x=RFile('foo'), spam=RFile('bar'))), - ('-x - -', NS(x=sys.stdin, spam=sys.stdin)), + ('-x - -', NS(x=eq_stdin, spam=eq_stdin)), ('readonly', NS(x=None, spam=RFile('readonly'))), ] @@ -1537,7 +1547,7 @@ class TestFileTypeRB(TempDirMixin, ParserTestCase): ('foo', NS(x=None, spam=RFile('foo'))), ('-x foo bar', NS(x=RFile('foo'), spam=RFile('bar'))), ('bar -x foo', NS(x=RFile('foo'), spam=RFile('bar'))), - ('-x - -', NS(x=sys.stdin, spam=sys.stdin)), + ('-x - -', NS(x=eq_stdin, spam=eq_stdin)), ] @@ -1576,7 +1586,7 @@ class TestFileTypeW(TempDirMixin, ParserTestCase): ('foo', NS(x=None, spam=WFile('foo'))), ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), ('bar -x foo', NS(x=WFile('foo'), spam=WFile('bar'))), - ('-x - -', NS(x=sys.stdout, spam=sys.stdout)), + ('-x - -', NS(x=eq_stdout, spam=eq_stdout)), ] @@ -1591,7 +1601,7 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): ('foo', NS(x=None, spam=WFile('foo'))), ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), ('bar -x foo', NS(x=WFile('foo'), spam=WFile('bar'))), - ('-x - -', NS(x=sys.stdout, spam=sys.stdout)), + ('-x - -', NS(x=eq_stdout, spam=eq_stdout)), ] |