bpo-34582: Adds JUnit XML output for regression tests (GH-9210)

4 files changed, 68 insertions, 20 deletions

diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
index bd126b3..538ff05 100644
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -268,6 +268,10 @@ def _create_parser():
                        help='if a test file alters the environment, mark '
                             'the test as failed')
 
+    group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
+                       help='writes JUnit-style XML results to the specified '
+                            'file')
+
     return parser
 
 
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index a176db5..1fd4132 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -100,8 +100,11 @@ class Regrtest:
         self.next_single_test = None
         self.next_single_filename = None
 
+        # used by --junit-xml
+        self.testsuite_xml = None
+
     def accumulate_result(self, test, result):
-        ok, test_time = result
+        ok, test_time, xml_data = result
         if ok not in (CHILD_ERROR, INTERRUPTED):
             self.test_times.append((test_time, test))
         if ok == PASSED:
@@ -118,6 +121,15 @@ class Regrtest:
         elif ok != INTERRUPTED:
             raise ValueError("invalid test result: %r" % ok)
 
+        if xml_data:
+            import xml.etree.ElementTree as ET
+            for e in xml_data:
+                try:
+                    self.testsuite_xml.append(ET.fromstring(e))
+                except ET.ParseError:
+                    print(xml_data, file=sys.__stderr__)
+                    raise
+
     def display_progress(self, test_index, test):
         if self.ns.quiet:
             return
@@ -164,6 +176,9 @@ class Regrtest:
                       file=sys.stderr)
                 ns.findleaks = False
 
+        if ns.xmlpath:
+            support.junit_xml_list = self.testsuite_xml = []
+
         # Strip .py extensions.
         removepy(ns.args)
 
@@ -384,7 +399,7 @@ class Regrtest:
                     result = runtest(self.ns, test)
                 except KeyboardInterrupt:
                     self.interrupted = True
-                    self.accumulate_result(test, (INTERRUPTED, None))
+                    self.accumulate_result(test, (INTERRUPTED, None, None))
                     break
                 else:
                     self.accumulate_result(test, result)
@@ -508,6 +523,31 @@ class Regrtest:
         if self.ns.runleaks:
             os.system("leaks %d" % os.getpid())
 
+    def save_xml_result(self):
+        if not self.ns.xmlpath and not self.testsuite_xml:
+            return
+
+        import xml.etree.ElementTree as ET
+        root = ET.Element("testsuites")
+
+        # Manually count the totals for the overall summary
+        totals = {'tests': 0, 'errors': 0, 'failures': 0}
+        for suite in self.testsuite_xml:
+            root.append(suite)
+            for k in totals:
+                try:
+                    totals[k] += int(suite.get(k, 0))
+                except ValueError:
+                    pass
+
+        for k, v in totals.items():
+            root.set(k, str(v))
+
+        xmlpath = os.path.join(support.SAVEDCWD, self.ns.xmlpath)
+        with open(xmlpath, 'wb') as f:
+            for s in ET.tostringlist(root):
+                f.write(s)
+
     def main(self, tests=None, **kwargs):
         global TEMPDIR
 
@@ -570,6 +610,9 @@ class Regrtest:
             self.rerun_failed_tests()
 
         self.finalize()
+
+        self.save_xml_result()
+
         if self.bad:
             sys.exit(2)
         if self.interrupted:
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
index 3e1afd4..4f41080 100644
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -85,8 +85,8 @@ def runtest(ns, test):
     ns -- regrtest namespace of options
     test -- the name of the test
 
-    Returns the tuple (result, test_time), where result is one of the
-    constants:
+    Returns the tuple (result, test_time, xml_data), where result is one
+    of the constants:
 
         INTERRUPTED      KeyboardInterrupt when run under -j
         RESOURCE_DENIED  test skipped because resource denied
@@ -94,6 +94,9 @@ def runtest(ns, test):
         ENV_CHANGED      test failed because it changed the execution environment
         FAILED           test failed
         PASSED           test passed
+
+    If ns.xmlpath is not None, xml_data is a list containing each
+    generated testsuite element.
     """
 
     output_on_failure = ns.verbose3
@@ -106,22 +109,13 @@ def runtest(ns, test):
         # reset the environment_altered flag to detect if a test altered
         # the environment
         support.environment_altered = False
+        support.junit_xml_list = xml_list = [] if ns.xmlpath else None
         if ns.failfast:
             support.failfast = True
         if output_on_failure:
             support.verbose = True
 
-            # Reuse the same instance to all calls to runtest(). Some
-            # tests keep a reference to sys.stdout or sys.stderr
-            # (eg. test_argparse).
-            if runtest.stringio is None:
-                stream = io.StringIO()
-                runtest.stringio = stream
-            else:
-                stream = runtest.stringio
-                stream.seek(0)
-                stream.truncate()
-
+            stream = io.StringIO()
             orig_stdout = sys.stdout
             orig_stderr = sys.stderr
             try:
@@ -138,12 +132,18 @@ def runtest(ns, test):
         else:
             support.verbose = ns.verbose  # Tell tests to be moderately quiet
             result = runtest_inner(ns, test, display_failure=not ns.verbose)
-        return result
+
+        if xml_list:
+            import xml.etree.ElementTree as ET
+            xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list]
+        else:
+            xml_data = None
+        return result + (xml_data,)
     finally:
         if use_timeout:
             faulthandler.cancel_dump_traceback_later()
         cleanup_test_droppings(test, ns.verbose)
-runtest.stringio = None
+        support.junit_xml_list = None
 
 
 def post_test_cleanup():
diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index 1f07cfb..6190574 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -67,7 +67,7 @@ def run_tests_worker(worker_args):
     try:
         result = runtest(ns, testname)
     except KeyboardInterrupt:
-        result = INTERRUPTED, ''
+        result = INTERRUPTED, '', None
     except BaseException as e:
         traceback.print_exc()
         result = CHILD_ERROR, str(e)
@@ -122,7 +122,7 @@ class MultiprocessThread(threading.Thread):
             self.current_test = None
 
         if retcode != 0:
-            result = (CHILD_ERROR, "Exit code %s" % retcode)
+            result = (CHILD_ERROR, "Exit code %s" % retcode, None)
             self.output.put((test, stdout.rstrip(), stderr.rstrip(),
                              result))
             return False
@@ -133,6 +133,7 @@ class MultiprocessThread(threading.Thread):
             return True
 
         result = json.loads(result)
+        assert len(result) == 3, f"Invalid result tuple: {result!r}"
         self.output.put((test, stdout.rstrip(), stderr.rstrip(),
                          result))
         return False
@@ -195,7 +196,7 @@ def run_tests_multiprocess(regrtest):
             regrtest.accumulate_result(test, result)
 
             # Display progress
-            ok, test_time = result
+            ok, test_time, xml_data = result
             text = format_test_result(test, ok)
             if (ok not in (CHILD_ERROR, INTERRUPTED)
                 and test_time >= PROGRESS_MIN_TIME