- Removed redundant call to expandtabs in DocTestParesr.

- Improvements to interactive debugging support:
  - Changed the replacement pdb.set_trace to redirect stdout to the
    real stdout *only* during interactive debugging; stdout from code
    continues to go to the fake stdout.
  - When the interactive debugger gets to the end of an example,
    automatically continue.
  - Use a replacement linecache.getlines that will return source lines
    from doctest examples; this makes the source available to the
    debugger for interactive debugging.
- In test_doctest, use a specialized _FakeOutput class instead of a
  temporary file to fake stdin for the interactive interpreter.

2 files changed, 175 insertions, 67 deletions

diff --git a/Lib/doctest.py b/Lib/doctest.py
index 0eced35..127e119 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -441,6 +441,28 @@ def _comment_line(line):
     else:
         return '#'
 
+class _OutputRedirectingPdb(pdb.Pdb):
+    """
+    A specialized version of the python debugger that redirects stdout
+    to a given stream when interacting with the user.  Stdout is *not*
+    redirected when traced code is executed.
+    """
+    def __init__(self, out):
+        self.__out = out
+        pdb.Pdb.__init__(self)
+
+    def trace_dispatch(self, *args):
+        # Redirect stdout to the given stream.
+        save_stdout = sys.stdout
+        sys.stdout = self.__out
+        # Call Pdb's trace dispatch method.
+        pdb.Pdb.trace_dispatch(self, *args)
+        # Restore stdout.
+        sys.stdout = save_stdout
+
+    def resume(self):
+        self._resume = 1
+
 ######################################################################
 ## 2. Example & DocTest
 ######################################################################
@@ -631,7 +653,7 @@ class DocTestParser:
         output = []
         charno, lineno = 0, 0
         # Find all doctest examples in the string:
-        for m in self._EXAMPLE_RE.finditer(string.expandtabs()):
+        for m in self._EXAMPLE_RE.finditer(string):
             # Add the pre-example text to `output`.
             output.append(string[charno:m.start()])
             # Update lineno (lines before this example)
@@ -1260,7 +1282,8 @@ class DocTestRunner:
         original_optionflags = self.optionflags
 
         # Process each example.
-        for example in test.examples:
+        for examplenum, example in enumerate(test.examples):
+
             # If REPORT_ONLY_FIRST_FAILURE is set, then supress
             # reporting after the first failure.
             quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
@@ -1280,18 +1303,25 @@ class DocTestRunner:
             if not quiet:
                 self.report_start(out, test, example)
 
+            # Use a special filename for compile(), so we can retrieve
+            # the source code during interactive debugging (see
+            # __patched_linecache_getlines).
+            filename = '<doctest %s[%d]>' % (test.name, examplenum)
+
             # Run the example in the given context (globs), and record
             # any exception that gets raised.  (But don't intercept
             # keyboard interrupts.)
             try:
                 # Don't blink!  This is where the user's code gets run.
-                exec compile(example.source, "<string>", "single",
+                exec compile(example.source, filename, "single",
                              compileflags, 1) in test.globs
+                self.debugger.set_continue() # ==== Example Finished ====
                 exception = None
             except KeyboardInterrupt:
                 raise
             except:
                 exception = sys.exc_info()
+                self.debugger.set_continue() # ==== Example Finished ====
 
             got = self._fakeout.getvalue()  # the actual output
             self._fakeout.truncate(0)
@@ -1352,6 +1382,17 @@ class DocTestRunner:
         self.failures += f
         self.tries += t
 
+    __LINECACHE_FILENAME_RE = re.compile(r'<doctest '
+                                         r'(?P<name>[\w\.]+)'
+                                         r'\[(?P<examplenum>\d+)\]>$')
+    def __patched_linecache_getlines(self, filename):
+        m = self.__LINECACHE_FILENAME_RE.match(filename)
+        if m and m.group('name') == self.test.name:
+            example = self.test.examples[int(m.group('examplenum'))]
+            return example.source.splitlines(True)
+        else:
+            return self.save_linecache_getlines(filename)
+
     def run(self, test, compileflags=None, out=None, clear_globs=True):
         """
         Run the examples in `test`, and display the results using the
@@ -1372,6 +1413,8 @@ class DocTestRunner:
         `DocTestRunner.check_output`, and the results are formatted by
         the `DocTestRunner.report_*` methods.
         """
+        self.test = test
+
         if compileflags is None:
             compileflags = _extract_future_flags(test.globs)
 
@@ -1380,25 +1423,27 @@ class DocTestRunner:
             out = save_stdout.write
         sys.stdout = self._fakeout
 
-        # Patch pdb.set_trace to restore sys.stdout, so that interactive
-        # debugging output is visible (not still redirected to self._fakeout).
-        # Note that we run "the real" pdb.set_trace (captured at doctest
-        # import time) in our replacement.  Because the current run() may
-        # run another doctest (and so on), the current pdb.set_trace may be
-        # our set_trace function, which changes sys.stdout.  If we called
-        # a chain of those, we wouldn't be left with the save_stdout
-        # *this* run() invocation wants.
-        def set_trace():
-            sys.stdout = save_stdout
-            real_pdb_set_trace()
-
+        # Patch pdb.set_trace to restore sys.stdout during interactive
+        # debugging (so it's not still redirected to self._fakeout).
+        # Note that the interactive output will go to *our*
+        # save_stdout, even if that's not the real sys.stdout; this
+        # allows us to write test cases for the set_trace behavior.
         save_set_trace = pdb.set_trace
-        pdb.set_trace = set_trace
+        self.debugger = _OutputRedirectingPdb(save_stdout)
+        self.debugger.reset()
+        pdb.set_trace = self.debugger.set_trace
+
+        # Patch linecache.getlines, so we can see the example's source
+        # when we're inside the debugger.
+        self.save_linecache_getlines = linecache.getlines
+        linecache.getlines = self.__patched_linecache_getlines
+
         try:
             return self.__run(test, compileflags, out)
         finally:
             sys.stdout = save_stdout
             pdb.set_trace = save_set_trace
+            linecache.getlines = self.save_linecache_getlines
             if clear_globs:
                 test.globs.clear()
 
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index 7ce3e3b..6f71577 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -117,6 +117,25 @@ class SampleNewStyleClass(object):
         return self.val
 
 ######################################################################
+## Fake stdin (for testing interactive debugging)
+######################################################################
+
+class _FakeInput:
+    """
+    A fake input stream for pdb's interactive debugger.  Whenever a
+    line is read, print it (to simulate the user typing it), and then
+    return it.  The set of lines to return is specified in the
+    constructor; they should not have trailing newlines.
+    """
+    def __init__(self, lines):
+        self.lines = lines
+
+    def readline(self):
+        line = self.lines.pop(0)
+        print line
+        return line+'\n'
+
+######################################################################
 ## Test Cases
 ######################################################################
 
@@ -1436,31 +1455,28 @@ Create a docstring that we want to debug:
 Create some fake stdin input, to feed to the debugger:
 
     >>> import tempfile
-    >>> fake_stdin = tempfile.TemporaryFile(mode='w+')
-    >>> fake_stdin.write('\n'.join(['next', 'print x', 'continue', '']))
-    >>> fake_stdin.seek(0)
     >>> real_stdin = sys.stdin
-    >>> sys.stdin = fake_stdin
+    >>> sys.stdin = _FakeInput(['next', 'print x', 'continue'])
 
 Run the debugger on the docstring, and then restore sys.stdin.
 
-    >>> try:
-    ...     doctest.debug_src(s)
-    ... finally:
-    ...      sys.stdin = real_stdin
-    ...      fake_stdin.close()
-    ... # doctest: +NORMALIZE_WHITESPACE
+    >>> try: doctest.debug_src(s)
+    ... finally: sys.stdin = real_stdin
     > <string>(1)?()
-    (Pdb) 12
+    (Pdb) next
+    12
     --Return--
     > <string>(1)?()->None
-    (Pdb) 12
-    (Pdb)
+    (Pdb) print x
+    12
+    (Pdb) continue
 
 """
 
 def test_pdb_set_trace():
-    r"""Using pdb.set_trace from a doctest
+    # Note: this should *not* be an r'...' string, because we need
+    # to use '\t' for the output of ...
+    """Using pdb.set_trace from a doctest
 
     You can use pdb.set_trace from a doctest.  To do so, you must
     retrieve the set_trace function from the pdb module at the time
@@ -1481,29 +1497,21 @@ def test_pdb_set_trace():
     captures our debugger input:
 
       >>> import tempfile
-      >>> fake_stdin = tempfile.TemporaryFile(mode='w+')
-      >>> fake_stdin.write('\n'.join([
-      ...    'up',       # up out of pdb.set_trace
-      ...    'up',       # up again to get out of our wrapper
+      >>> real_stdin = sys.stdin
+      >>> sys.stdin = _FakeInput([
       ...    'print x',  # print data defined by the example
       ...    'continue', # stop debugging
-      ...    '']))
-      >>> fake_stdin.seek(0)
-      >>> real_stdin = sys.stdin
-      >>> sys.stdin = fake_stdin
+      ...    ''])
 
-      >>> runner.run(test) # doctest: +ELLIPSIS
+      >>> try: runner.run(test)
+      ... finally: sys.stdin = real_stdin
       --Return--
-      > ...set_trace()->None
-      -> Pdb().set_trace()
-      (Pdb) > ...set_trace()
-      -> real_pdb_set_trace()
-      (Pdb) > <string>(1)?()
-      (Pdb) 42
-      (Pdb) (0, 2)
-
-      >>> sys.stdin = real_stdin
-      >>> fake_stdin.close()
+      > <doctest foo[1]>(1)?()->None
+      -> import pdb; pdb.set_trace()
+      (Pdb) print x
+      42
+      (Pdb) continue
+      (0, 2)
 
       You can also put pdb.set_trace in a function called from a test:
 
@@ -1516,30 +1524,85 @@ def test_pdb_set_trace():
       ... >>> calls_set_trace()
       ... '''
       >>> test = parser.get_doctest(doc, globals(), "foo", "foo.py", 0)
-      >>> fake_stdin = tempfile.TemporaryFile(mode='w+')
-      >>> fake_stdin.write('\n'.join([
-      ...    'up',       # up out of pdb.set_trace
-      ...    'up',       # up again to get out of our wrapper
+      >>> real_stdin = sys.stdin
+      >>> sys.stdin = _FakeInput([
       ...    'print y',  # print data defined in the function
       ...    'up',       # out of function
       ...    'print x',  # print data defined by the example
       ...    'continue', # stop debugging
-      ...    '']))
-      >>> fake_stdin.seek(0)
-      >>> real_stdin = sys.stdin
-      >>> sys.stdin = fake_stdin
+      ...    ''])
+
+      >>> try: runner.run(test)
+      ... finally: sys.stdin = real_stdin
+      --Return--
+      > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None
+      -> import pdb; pdb.set_trace()
+      (Pdb) print y
+      2
+      (Pdb) up
+      > <doctest foo[1]>(1)?()
+      -> calls_set_trace()
+      (Pdb) print x
+      1
+      (Pdb) continue
+      (0, 2)
+
+    During interactive debugging, source code is shown, even for
+    doctest examples:
 
-      >>> runner.run(test) # doctest: +ELLIPSIS
+      >>> doc = '''
+      ... >>> def f(x):
+      ... ...     g(x*2)
+      ... >>> def g(x):
+      ... ...     print x+3
+      ... ...     import pdb; pdb.set_trace()
+      ... >>> f(3)
+      ... '''
+      >>> test = parser.get_doctest(doc, globals(), "foo", "foo.py", 0)
+      >>> real_stdin = sys.stdin
+      >>> sys.stdin = _FakeInput([
+      ...    'list',     # list source from example 2
+      ...    'next',     # return from g()
+      ...    'list',     # list source from example 1
+      ...    'next',     # return from f()
+      ...    'list',     # list source from example 3
+      ...    'continue', # stop debugging
+      ...    ''])
+      >>> try: runner.run(test)
+      ... finally: sys.stdin = real_stdin
+      ... # doctest: +NORMALIZE_WHITESPACE
+      --Return--
+      > <doctest foo[1]>(3)g()->None
+      -> import pdb; pdb.set_trace()
+      (Pdb) list
+        1     def g(x):
+        2         print x+3
+        3  ->     import pdb; pdb.set_trace()
+      [EOF]
+      (Pdb) next
+      --Return--
+      > <doctest foo[0]>(2)f()->None
+      -> g(x*2)
+      (Pdb) list
+        1     def f(x):
+        2  ->     g(x*2)
+      [EOF]
+      (Pdb) next
       --Return--
-      > ...set_trace()->None
-      -> Pdb().set_trace()
-      (Pdb) ...set_trace()
-      -> real_pdb_set_trace()
-      (Pdb) > <string>(3)calls_set_trace()
-      (Pdb) 2
-      (Pdb) > <string>(1)?()
-      (Pdb) 1
-      (Pdb) (0, 2)
+      > <doctest foo[2]>(1)?()->None
+      -> f(3)
+      (Pdb) list
+        1  -> f(3)
+      [EOF]
+      (Pdb) continue
+      **********************************************************************
+      File "foo.py", line 7, in foo
+      Failed example:
+          f(3)
+      Expected nothing
+      Got:
+          9
+      (1, 3)
       """
 
 def test_DocTestSuite():