#16522: Add FAIL_FAST flag to doctest.

Patch by me, most of the work (doc and tests) by Daniel Urban.

5 files changed, 69 insertions, 4 deletions

diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst
index 222c719..8822ef3 100644
--- a/Doc/library/doctest.rst
+++ b/Doc/library/doctest.rst
@@ -633,6 +633,16 @@ The second group of options controls how test failures are reported:
    the output is suppressed.
 
 
+.. data:: FAIL_FAST
+
+   When specified, exit after the first failing example and don't attempt to run
+   the remaining examples. Thus, the number of failures reported will always be 1.
+   This flag may be useful during debugging, since examples after the first
+   failure won't even produce debugging output.
+
+   .. versionadded:: 3.4
+
+
 .. data:: REPORTING_FLAGS
 
    A bitmask or'ing together all the reporting flags above.
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
index 4bed2ed..ab0b163 100644
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -150,7 +150,11 @@ New Modules
 Improved Modules
 ================
 
-* None yet.
+doctest
+-------
+
+Added ``FAIL_FAST`` flag to halt test running as soon as the first failure is
+detected.  (Contributed by R. David Murray and Daniel Urban in :issue:`16522`.)
 
 
 Optimizations
diff --git a/Lib/doctest.py b/Lib/doctest.py
index 3af05fb..16a732d 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -62,6 +62,7 @@ __all__ = [
     'REPORT_NDIFF',
     'REPORT_ONLY_FIRST_FAILURE',
     'REPORTING_FLAGS',
+    'FAIL_FAST',
     # 1. Utility Functions
     # 2. Example & DocTest
     'Example',
@@ -150,11 +151,13 @@ REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
 REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
 REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
 REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
+FAIL_FAST = register_optionflag('FAIL_FAST')
 
 REPORTING_FLAGS = (REPORT_UDIFF |
                    REPORT_CDIFF |
                    REPORT_NDIFF |
-                   REPORT_ONLY_FIRST_FAILURE)
+                   REPORT_ONLY_FIRST_FAILURE |
+                   FAIL_FAST)
 
 # Special string markers for use in `want` strings:
 BLANKLINE_MARKER = '<BLANKLINE>'
@@ -1342,6 +1345,9 @@ class DocTestRunner:
             else:
                 assert False, ("unknown outcome", outcome)
 
+            if failures and self.optionflags & FAIL_FAST:
+                break
+
         # Restore the option flags (in case they were modified)
         self.optionflags = original_optionflags
 
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index 8f8c7c7..3e36f19 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -1409,8 +1409,40 @@ However, output from `report_start` is not suppressed:
         2
     TestResults(failed=3, attempted=5)
 
-For the purposes of REPORT_ONLY_FIRST_FAILURE, unexpected exceptions
-count as failures:
+The FAIL_FAST flag causes the runner to exit after the first failing example,
+so subsequent examples are not even attempted:
+
+    >>> flags = doctest.FAIL_FAST
+    >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+    ... # doctest: +ELLIPSIS
+    **********************************************************************
+    File ..., line 5, in f
+    Failed example:
+        print(2) # first failure
+    Expected:
+        200
+    Got:
+        2
+    TestResults(failed=1, attempted=2)
+
+Specifying both FAIL_FAST and REPORT_ONLY_FIRST_FAILURE is equivalent to
+FAIL_FAST only:
+
+    >>> flags = doctest.FAIL_FAST | doctest.REPORT_ONLY_FIRST_FAILURE
+    >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+    ... # doctest: +ELLIPSIS
+    **********************************************************************
+    File ..., line 5, in f
+    Failed example:
+        print(2) # first failure
+    Expected:
+        200
+    Got:
+        2
+    TestResults(failed=1, attempted=2)
+
+For the purposes of both REPORT_ONLY_FIRST_FAILURE and FAIL_FAST, unexpected
+exceptions count as failures:
 
     >>> def f(x):
     ...     r'''
@@ -1437,6 +1469,17 @@ count as failures:
         ...
         ValueError: 2
     TestResults(failed=3, attempted=5)
+    >>> flags = doctest.FAIL_FAST
+    >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+    ... # doctest: +ELLIPSIS
+    **********************************************************************
+    File ..., line 5, in f
+    Failed example:
+        raise ValueError(2) # first failure
+    Exception raised:
+        ...
+        ValueError: 2
+    TestResults(failed=1, attempted=2)
 
 New option flags can also be registered, via register_optionflag().  Here
 we reach into doctest's internals a bit.
diff --git a/Misc/NEWS b/Misc/NEWS
index a56194c..10a07cd 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -138,6 +138,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #16522: added FAIL_FAST flag to doctest.
+
 - Issue #15627: Add the importlib.abc.SourceLoader.compile_source() method.
 
 - Issue #16408: Fix file descriptors not being closed in error conditions