bpo-31344: Per-frame control of trace events (GH-3417)

f_trace_lines: enable/disable line trace events
f_trace_opcodes: enable/disable opcode trace events

These are intended primarily for testing of the interpreter
itself, as they make it much easier to emulate signals
arriving at unfortunate times.

11 files changed, 126 insertions, 17 deletions

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 8b94929..85f3136 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1068,7 +1068,7 @@ always available.
    Trace functions should have three arguments: *frame*, *event*, and
    *arg*. *frame* is the current stack frame.  *event* is a string: ``'call'``,
    ``'line'``, ``'return'``, ``'exception'``, ``'c_call'``, ``'c_return'``, or
-   ``'c_exception'``. *arg* depends on the event type.
+   ``'c_exception'``, ``'opcode'``. *arg* depends on the event type.
 
    The trace function is invoked (with *event* set to ``'call'``) whenever a new
    local scope is entered; it should return a reference to a local trace
@@ -1091,6 +1091,8 @@ always available.
       ``None``; the return value specifies the new local trace function.  See
       :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this
       works.
+      Per-line events may be disabled for a frame by setting
+      :attr:`f_trace_lines` to :const:`False` on that frame.
 
    ``'return'``
       A function (or other code block) is about to return.  The local trace
@@ -1113,6 +1115,14 @@ always available.
    ``'c_exception'``
       A C function has raised an exception.  *arg* is the C function object.
 
+   ``'opcode'``
+      The interpreter is about to execute a new opcode (see :mod:`dis` for
+      opcode details).  The local trace function is called; *arg* is
+      ``None``; the return value specifies the new local trace function.
+      Per-opcode events are not emitted by default: they must be explicitly
+      requested by setting :attr:`f_trace_opcodes` to :const:`True` on the
+      frame.
+
    Note that as an exception is propagated down the chain of callers, an
    ``'exception'`` event is generated at each level.
 
@@ -1125,6 +1135,11 @@ always available.
       implementation platform, rather than part of the language definition, and
       thus may not be available in all Python implementations.
 
+   .. versionchanged:: 3.7
+
+      ``'opcode'`` event type added; :attr:`f_trace_lines` and
+      :attr:`f_trace_opcodes` attributes added to frames
+
 .. function:: set_asyncgen_hooks(firstiter, finalizer)
 
    Accepts two optional keyword arguments which are callables that accept an
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 24a2618..5f932ae 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -970,10 +970,20 @@ Internal types
 
       .. index::
          single: f_trace (frame attribute)
+         single: f_trace_lines (frame attribute)
+         single: f_trace_opcodes (frame attribute)
          single: f_lineno (frame attribute)
 
       Special writable attributes: :attr:`f_trace`, if not ``None``, is a function
-      called at the start of each source code line (this is used by the debugger);
+      called for various events during code execution (this is used by the debugger).
+      Normally an event is triggered for each new source line - this can be
+      disabled by setting :attr:`f_trace_lines` to :const:`False`.
+
+      Implementations *may* allow per-opcode events to be requested by setting
+      :attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to
+      undefined interpreter behaviour if exceptions raised by the trace
+      function escape to the function being traced.
+
       :attr:`f_lineno` is the current line number of the frame --- writing to this
       from within a trace function jumps to the given line (only for the bottom-most
       frame).  A debugger can implement a Jump command (aka Set Next Statement)
diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst
index 14c2182..5ee41dc 100644
--- a/Doc/whatsnew/3.7.rst
+++ b/Doc/whatsnew/3.7.rst
@@ -348,6 +348,18 @@ Build and C API Changes
   (Contributed by Antoine Pitrou in :issue:`31370`.).
 
 
+Other CPython Implementation Changes
+====================================
+
+* Trace hooks may now opt out of receiving ``line`` events from the interpreter
+  by setting the new ``f_trace_lines`` attribute to :const:`False` on the frame
+  being traced. (Contributed by Nick Coghlan in :issue:`31344`.)
+
+* Trace hooks may now opt in to receiving ``opcode`` events from the interpreter
+  by setting the new ``f_trace_opcodes`` attribute to :const:`True` on the frame
+  being traced. (Contributed by Nick Coghlan in :issue:`31344`.)
+
+
 Deprecated
 ==========
 
diff --git a/Include/frameobject.h b/Include/frameobject.h
index 616c611..dbe0a84 100644
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -27,6 +27,8 @@ typedef struct _frame {
        to the current stack top. */
     PyObject **f_stacktop;
     PyObject *f_trace;          /* Trace function */
+    char f_trace_lines;         /* Emit per-line trace events? */
+    char f_trace_opcodes;       /* Emit per-opcode trace events? */
 
     /* In a generator, we need to be able to swap between the exception
        state inside the generator and the exception state of the calling
diff --git a/Include/pystate.h b/Include/pystate.h
index 6229236..f957d53 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -92,7 +92,11 @@ typedef struct _is {
 /* Py_tracefunc return -1 when raising an exception, or 0 for success. */
 typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);
 
-/* The following values are used for 'what' for tracefunc functions: */
+/* The following values are used for 'what' for tracefunc functions
+ *
+ * To add a new kind of trace event, also update "trace_init" in
+ * Python/sysmodule.c to define the Python level event name
+ */
 #define PyTrace_CALL 0
 #define PyTrace_EXCEPTION 1
 #define PyTrace_LINE 2
@@ -100,6 +104,7 @@ typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);
 #define PyTrace_C_CALL 4
 #define PyTrace_C_EXCEPTION 5
 #define PyTrace_C_RETURN 6
+#define PyTrace_OPCODE 7
 #endif
 
 #ifdef Py_LIMITED_API
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 04550e5..fd45abe 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -971,7 +971,7 @@ class SizeofTest(unittest.TestCase):
         nfrees = len(x.f_code.co_freevars)
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                   ncells + nfrees - 1
-        check(x, vsize('12P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+        check(x, vsize('8P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
         # function
         def func(): pass
         check(func, size('12P'))
diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index 25c5835..ed9e6d4 100644
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -234,16 +234,29 @@ generator_example.events = ([(0, 'call'),
 
 
 class Tracer:
-    def __init__(self):
+    def __init__(self, trace_line_events=None, trace_opcode_events=None):
+        self.trace_line_events = trace_line_events
+        self.trace_opcode_events = trace_opcode_events
         self.events = []
+
+    def _reconfigure_frame(self, frame):
+        if self.trace_line_events is not None:
+            frame.f_trace_lines = self.trace_line_events
+        if self.trace_opcode_events is not None:
+            frame.f_trace_opcodes = self.trace_opcode_events
+
     def trace(self, frame, event, arg):
+        self._reconfigure_frame(frame)
         self.events.append((frame.f_lineno, event))
         return self.trace
+
     def traceWithGenexp(self, frame, event, arg):
+        self._reconfigure_frame(frame)
         (o for o in [1])
         self.events.append((frame.f_lineno, event))
         return self.trace
 
+
 class TraceTestCase(unittest.TestCase):
 
     # Disable gc collection when tracing, otherwise the
@@ -257,6 +270,11 @@ class TraceTestCase(unittest.TestCase):
         if self.using_gc:
             gc.enable()
 
+    @staticmethod
+    def make_tracer():
+        """Helper to allow test subclasses to configure tracers differently"""
+        return Tracer()
+
     def compare_events(self, line_offset, events, expected_events):
         events = [(l - line_offset, e) for (l, e) in events]
         if events != expected_events:
@@ -266,7 +284,7 @@ class TraceTestCase(unittest.TestCase):
                                         [str(x) for x in events])))
 
     def run_and_compare(self, func, events):
-        tracer = Tracer()
+        tracer = self.make_tracer()
         sys.settrace(tracer.trace)
         func()
         sys.settrace(None)
@@ -277,7 +295,7 @@ class TraceTestCase(unittest.TestCase):
         self.run_and_compare(func, func.events)
 
     def run_test2(self, func):
-        tracer = Tracer()
+        tracer = self.make_tracer()
         func(tracer.trace)
         sys.settrace(None)
         self.compare_events(func.__code__.co_firstlineno,
@@ -329,7 +347,7 @@ class TraceTestCase(unittest.TestCase):
         # and if the traced function contains another generator
         # that is not completely exhausted, the trace stopped.
         # Worse: the 'finally' clause was not invoked.
-        tracer = Tracer()
+        tracer = self.make_tracer()
         sys.settrace(tracer.traceWithGenexp)
         generator_example()
         sys.settrace(None)
@@ -398,6 +416,34 @@ class TraceTestCase(unittest.TestCase):
              (1, 'line')])
 
 
+class SkipLineEventsTraceTestCase(TraceTestCase):
+    """Repeat the trace tests, but with per-line events skipped"""
+
+    def compare_events(self, line_offset, events, expected_events):
+        skip_line_events = [e for e in expected_events if e[1] != 'line']
+        super().compare_events(line_offset, events, skip_line_events)
+
+    @staticmethod
+    def make_tracer():
+        return Tracer(trace_line_events=False)
+
+
+@support.cpython_only
+class TraceOpcodesTestCase(TraceTestCase):
+    """Repeat the trace tests, but with per-opcodes events enabled"""
+
+    def compare_events(self, line_offset, events, expected_events):
+        skip_opcode_events = [e for e in events if e[1] != 'opcode']
+        if len(events) > 1:
+            self.assertLess(len(skip_opcode_events), len(events),
+                            msg="No 'opcode' events received by the tracer")
+        super().compare_events(line_offset, skip_opcode_events, expected_events)
+
+    @staticmethod
+    def make_tracer():
+        return Tracer(trace_opcode_events=True)
+
+
 class RaisingTraceFuncTestCase(unittest.TestCase):
     def setUp(self):
         self.addCleanup(sys.settrace, sys.gettrace())
@@ -846,6 +892,8 @@ output.append(4)
 def test_main():
     support.run_unittest(
         TraceTestCase,
+        SkipLineEventsTraceTestCase,
+        TraceOpcodesTestCase,
         RaisingTraceFuncTestCase,
         JumpTestCase
     )
diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-06-20-25-47.bpo-31344.XpFs-q.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-06-20-25-47.bpo-31344.XpFs-q.rst
new file mode 100644
index 0000000..2b3a386
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-06-20-25-47.bpo-31344.XpFs-q.rst
@@ -0,0 +1,5 @@
+For finer control of tracing behaviour when testing the interpreter, two new
+frame attributes have been added to control the emission of particular trace
+events: ``f_trace_lines`` (``True`` by default) to turn off per-line trace
+events; and ``f_trace_opcodes`` (``False`` by default) to turn on per-opcode
+trace events.
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index 8448319..b3f0b65 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -15,6 +15,8 @@ static PyMemberDef frame_memberlist[] = {
     {"f_builtins",      T_OBJECT,       OFF(f_builtins),  READONLY},
     {"f_globals",       T_OBJECT,       OFF(f_globals),   READONLY},
     {"f_lasti",         T_INT,          OFF(f_lasti),     READONLY},
+    {"f_trace_lines",   T_BOOL,         OFF(f_trace_lines), 0},
+    {"f_trace_opcodes", T_BOOL,         OFF(f_trace_opcodes), 0},
     {NULL}      /* Sentinel */
 };
 
@@ -728,6 +730,8 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code,
     f->f_iblock = 0;
     f->f_executing = 0;
     f->f_gen = NULL;
+    f->f_trace_opcodes = 0;
+    f->f_trace_lines = 1;
 
     return f;
 }
diff --git a/Python/ceval.c b/Python/ceval.c
index 8fc65cd..a072a5f 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4458,12 +4458,19 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
         *instr_lb = bounds.ap_lower;
         *instr_ub = bounds.ap_upper;
     }
-    /* If the last instruction falls at the start of a line or if
-       it represents a jump backwards, update the frame's line
-       number and call the trace function. */
-    if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) {
+    /* Always emit an opcode event if we're tracing all opcodes. */
+    if (frame->f_trace_opcodes) {
+        result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None);
+    }
+    /* If the last instruction falls at the start of a line or if it
+       represents a jump backwards, update the frame's line number and
+       then call the trace function if we're tracing source lines.
+    */
+    if ((frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev)) {
         frame->f_lineno = line;
-        result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
+        if (frame->f_trace_lines) {
+            result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
+        }
     }
     *instr_prev = frame->f_lasti;
     return result;
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index fba7220..021b95d 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -349,18 +349,19 @@ same value.");
  * Cached interned string objects used for calling the profile and
  * trace functions.  Initialized by trace_init().
  */
-static PyObject *whatstrings[7] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL};
+static PyObject *whatstrings[8] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL};
 
 static int
 trace_init(void)
 {
-    static const char * const whatnames[7] = {
+    static const char * const whatnames[8] = {
         "call", "exception", "line", "return",
-        "c_call", "c_exception", "c_return"
+        "c_call", "c_exception", "c_return",
+        "opcode"
     };
     PyObject *name;
     int i;
-    for (i = 0; i < 7; ++i) {
+    for (i = 0; i < 8; ++i) {
         if (whatstrings[i] == NULL) {
             name = PyUnicode_InternFromString(whatnames[i]);
             if (name == NULL)