Correction for issue1265 (pdb bug with "with" statement).

When an unfinished generator-iterator is garbage collected, PyEval_EvalFrameEx
is called with a GeneratorExit exception set.  This leads to funny results
if the sys.settrace function itself makes use of generators.
A visible effect is that the settrace function is reset to None.
Another is that the eventual "finally" block of the generator is not called.

It is necessary to save/restore the exception around the call to the trace
function.

This happens a lot with py3k: isinstance() of an ABCMeta instance runs
    def __instancecheck__(cls, instance):
        """Override for isinstance(instance, cls)."""
        return any(cls.__subclasscheck__(c)
                   for c in {instance.__class__, type(instance)})
which lets an opened generator expression each time it returns True.

Seems a backport candidate, even if the case is less frequent in 2.5.

2 files changed, 59 insertions, 9 deletions

diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py
index f67da77..3ec00d2 100644
--- a/Lib/test/test_trace.py
+++ b/Lib/test/test_trace.py
@@ -204,12 +204,44 @@ tighterloop_example.events = [(0, 'call'),
                             (6, 'line'),
                             (6, 'return')]
 
+def generator_function():
+    try:
+        yield True
+        "continued"
+    finally:
+        "finally"
+def generator_example():
+    # any() will leave the generator before its end
+    x = any(generator_function())
+
+    # the following lines were not traced
+    for x in range(10):
+        y = x
+
+generator_example.events = ([(0, 'call'),
+                             (2, 'line'),
+                             (-6, 'call'),
+                             (-5, 'line'),
+                             (-4, 'line'),
+                             (-4, 'return'),
+                             (-4, 'call'),
+                             (-4, 'exception'),
+                             (-1, 'line'),
+                             (-1, 'return')] +
+                            [(5, 'line'), (6, 'line')] * 10 +
+                            [(5, 'line'), (5, 'return')])
+
+
 class Tracer:
     def __init__(self):
         self.events = []
     def trace(self, frame, event, arg):
         self.events.append((frame.f_lineno, event))
         return self.trace
+    def traceWithGenexp(self, frame, event, arg):
+        (o for o in [1])
+        self.events.append((frame.f_lineno, event))
+        return self.trace
 
 class TraceTestCase(unittest.TestCase):
     def compare_events(self, line_offset, events, expected_events):
@@ -217,8 +249,8 @@ class TraceTestCase(unittest.TestCase):
         if events != expected_events:
             self.fail(
                 "events did not match expectation:\n" +
-                "\n".join(difflib.ndiff(map(str, expected_events),
-                                        map(str, events))))
+                "\n".join(difflib.ndiff([str(x) for x in expected_events],
+                                        [str(x) for x in events])))
 
 
     def run_test(self, func):
@@ -262,6 +294,19 @@ class TraceTestCase(unittest.TestCase):
     def test_12_tighterloop(self):
         self.run_test(tighterloop_example)
 
+    def test_13_genexp(self):
+        self.run_test(generator_example)
+        # issue1265: if the trace function contains a generator,
+        # and if the traced function contains another generator
+        # that is not completely exhausted, the trace stopped.
+        # Worse: the 'finally' clause was not invoked.
+        tracer = Tracer()
+        sys.settrace(tracer.traceWithGenexp)
+        generator_example()
+        sys.settrace(None)
+        self.compare_events(generator_example.__code__.co_firstlineno,
+                            tracer.events, generator_example.events)
+
 class RaisingTraceFuncTestCase(unittest.TestCase):
     def trace(self, frame, event, arg):
         """A trace function that raises an exception in response to a
diff --git a/Python/ceval.c b/Python/ceval.c
index b42444d..70086e1 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -107,7 +107,7 @@ static int prtrace(PyObject *, char *);
 #endif
 static int call_trace(Py_tracefunc, PyObject *, PyFrameObject *,
 		      int, PyObject *);
-static void call_trace_protected(Py_tracefunc, PyObject *,
+static int call_trace_protected(Py_tracefunc, PyObject *,
 				 PyFrameObject *, int, PyObject *);
 static void call_exc_trace(Py_tracefunc, PyObject *, PyFrameObject *);
 static int maybe_call_line_trace(Py_tracefunc, PyObject *,
@@ -717,8 +717,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			   an argument which depends on the situation.
 			   The global trace function is also called
 			   whenever an exception is detected. */
-			if (call_trace(tstate->c_tracefunc, tstate->c_traceobj,
-				       f, PyTrace_CALL, Py_None)) {
+			if (call_trace_protected(tstate->c_tracefunc, 
+						 tstate->c_traceobj,
+						 f, PyTrace_CALL, Py_None)) {
 				/* Trace function raised an error */
 				goto exit_eval_frame;
 			}
@@ -726,9 +727,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 		if (tstate->c_profilefunc != NULL) {
 			/* Similar for c_profilefunc, except it needn't
 			   return itself and isn't called for "line" events */
-			if (call_trace(tstate->c_profilefunc,
-				       tstate->c_profileobj,
-				       f, PyTrace_CALL, Py_None)) {
+			if (call_trace_protected(tstate->c_profilefunc,
+						 tstate->c_profileobj,
+						 f, PyTrace_CALL, Py_None)) {
 				/* Profile function raised an error */
 				goto exit_eval_frame;
 			}
@@ -3127,7 +3128,7 @@ call_exc_trace(Py_tracefunc func, PyObject *self, PyFrameObject *f)
 	}
 }
 
-static void
+static int
 call_trace_protected(Py_tracefunc func, PyObject *obj, PyFrameObject *frame,
 		     int what, PyObject *arg)
 {
@@ -3136,11 +3137,15 @@ call_trace_protected(Py_tracefunc func, PyObject *obj, PyFrameObject *frame,
 	PyErr_Fetch(&type, &value, &traceback);
 	err = call_trace(func, obj, frame, what, arg);
 	if (err == 0)
+	{
 		PyErr_Restore(type, value, traceback);
+		return 0;
+	}
 	else {
 		Py_XDECREF(type);
 		Py_XDECREF(value);
 		Py_XDECREF(traceback);
+		return -1;
 	}
 }