Added REPORT_ONLY_FIRST_FAILURE flag, which supresses output after the

first failing example in each test.

3 files changed, 116 insertions, 12 deletions

diff --git a/Doc/lib/libdoctest.tex b/Doc/lib/libdoctest.tex
index dca79f7..99fbeb4 100644
--- a/Doc/lib/libdoctest.tex
+++ b/Doc/lib/libdoctest.tex
@@ -364,6 +364,17 @@ can also be used in doctest directives (see below).
     positions.
 \end{datadesc}
 
+\begin{datadesc}{REPORT_ONLY_FIRST_FAILURE}
+  When specified, display the first failing example in each doctest,
+  but suppress output for all remaining examples.  This will prevent
+  doctest from reporting correct examples that break because of
+  earlier failures; but it might also hide incorrect examples that
+  fail independently of the first failure.  When
+  \constant{REPORT_ONLY_FIRST_FAILURE} is specified, the remaining
+  examples are still run, and still count towards the total number of
+  failures reported; only the output is suppressed.
+\end{datadesc}
+
 A "doctest directive" is a trailing Python comment on a line of a doctest
 example:
 
@@ -421,8 +432,8 @@ can be useful.
 
 \versionchanged[Constants \constant{DONT_ACCEPT_BLANKLINE},
     \constant{NORMALIZE_WHITESPACE}, \constant{ELLIPSIS},
-    \constant{REPORT_UDIFF}, \constant{REPORT_CDIFF}, and
-    \constant{REPORT_NDIFF}
+    \constant{REPORT_UDIFF}, \constant{REPORT_CDIFF},
+    \constant{REPORT_NDIFF}, and \constant{REPORT_ONLY_FIRST_FAILURE}
     were added; by default \code{<BLANKLINE>} in expected output
     matches an empty line in actual output; and doctest directives
     were added]{2.4}
diff --git a/Lib/doctest.py b/Lib/doctest.py
index 5e83c18..d7978fd 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -260,6 +260,7 @@ ELLIPSIS = register_optionflag('ELLIPSIS')
 REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
 REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
 REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
+REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
 
 # Special string markers for use in `want` strings:
 BLANKLINE_MARKER = '<BLANKLINE>'
@@ -1280,7 +1281,6 @@ class DocTestRunner:
         """
         Report that the given example failed.
         """
-        # Print an error message.
         out(self._failure_header(test, example) +
             self._checker.output_difference(example.want, got,
                                             self.optionflags))
@@ -1331,6 +1331,11 @@ class DocTestRunner:
 
         # Process each example.
         for example in test.examples:
+            # If REPORT_ONLY_FIRST_FAILURE is set, then supress
+            # reporting after the first failure.
+            quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
+                     failures > 0)
+
             # Merge in the example's options.
             self.optionflags = original_optionflags
             if example.options:
@@ -1342,7 +1347,8 @@ class DocTestRunner:
 
             # Record that we started this example.
             tries += 1
-            self.report_start(out, test, example)
+            if not quiet:
+                self.report_start(out, test, example)
 
             # Run the example in the given context (globs), and record
             # any exception that gets raised.  (But don't intercept
@@ -1365,9 +1371,11 @@ class DocTestRunner:
             if exception is None:
                 if self._checker.check_output(example.want, got,
                                               self.optionflags):
-                    self.report_success(out, test, example, got)
+                    if not quiet:
+                        self.report_success(out, test, example, got)
                 else:
-                    self.report_failure(out, test, example, got)
+                    if not quiet:
+                        self.report_failure(out, test, example, got)
                     failures += 1
 
             # If the example raised an exception, then check if it was
@@ -1379,19 +1387,22 @@ class DocTestRunner:
                 # If `example.exc_msg` is None, then we weren't
                 # expecting an exception.
                 if example.exc_msg is None:
-                    self.report_unexpected_exception(out, test, example,
-                                                     exc_info)
+                    if not quiet:
+                        self.report_unexpected_exception(out, test, example,
+                                                         exc_info)
                     failures += 1
                 # If `example.exc_msg` matches the actual exception
                 # message (`exc_msg`), then the example succeeds.
                 elif (self._checker.check_output(example.exc_msg, exc_msg,
                                                  self.optionflags)):
-                    self.report_success(out, test, example,
-                                        got + _exception_traceback(exc_info))
+                    if not quiet:
+                        got += _exception_traceback(exc_info)
+                        self.report_success(out, test, example, got)
                 # Otherwise, the example fails.
                 else:
-                    self.report_failure(out, test, example,
-                                        got + _exception_traceback(exc_info))
+                    if not quiet:
+                        got += _exception_traceback(exc_info)
+                        self.report_failure(out, test, example, got)
                     failures += 1
 
         # Restore the option flags (in case they were modified)
@@ -1842,6 +1853,7 @@ def testmod(m=None, name=None, globs=None, verbose=None, isprivate=None,
         REPORT_UDIFF
         REPORT_CDIFF
         REPORT_NDIFF
+        REPORT_ONLY_FIRST_FAILURE
 
     Optional keyword arg "raise_on_error" raises an exception on the
     first unexpected exception or failure. This allows failures to be
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index d683051..2464b23 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -1042,6 +1042,87 @@ marking, as well as interline differences.
         ?     +              ++    ^
     <BLANKLINE>
     (1, 1)
+
+The REPORT_ONLY_FIRST_FAILURE supresses result output after the first
+failing example:
+
+    >>> def f(x):
+    ...     r'''
+    ...     >>> print 1 # first success
+    ...     1
+    ...     >>> print 2 # first failure
+    ...     200
+    ...     >>> print 3 # second failure
+    ...     300
+    ...     >>> print 4 # second success
+    ...     4
+    ...     >>> print 5 # third failure
+    ...     500
+    ...     '''
+    >>> test = doctest.DocTestFinder().find(f)[0]
+    >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE
+    >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+    **********************************************************************
+    Line 4, in f
+    Failed example:
+        print 2 # first failure
+    Expected:
+        200
+    Got:
+        2
+    (3, 5)
+
+However, output from `report_start` is not supressed:
+
+    >>> doctest.DocTestRunner(verbose=True, optionflags=flags).run(test)
+    Trying:
+        print 1 # first success
+    Expecting:
+        1
+    ok
+    Trying:
+        print 2 # first failure
+    Expecting:
+        200
+    **********************************************************************
+    Line 4, in f
+    Failed example:
+        print 2 # first failure
+    Expected:
+        200
+    Got:
+        2
+    (3, 5)
+
+For the purposes of REPORT_ONLY_FIRST_FAILURE, unexpected exceptions
+count as failures:
+
+    >>> def f(x):
+    ...     r'''
+    ...     >>> print 1 # first success
+    ...     1
+    ...     >>> raise ValueError(2) # first failure
+    ...     200
+    ...     >>> print 3 # second failure
+    ...     300
+    ...     >>> print 4 # second success
+    ...     4
+    ...     >>> print 5 # third failure
+    ...     500
+    ...     '''
+    >>> test = doctest.DocTestFinder().find(f)[0]
+    >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE
+    >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+    ... # doctest: +ELLIPSIS
+    **********************************************************************
+    Line 4, in f
+    Failed example:
+        raise ValueError(2) # first failure
+    Exception raised:
+        ...
+        ValueError: 2
+    (3, 5)
+
     """
 
     def option_directives(): r"""