bpo-31344: Per-frame control of trace events (GH-3417)

f_trace_lines: enable/disable line trace events
f_trace_opcodes: enable/disable opcode trace events

These are intended primarily for testing of the interpreter
itself, as they make it much easier to emulate signals
arriving at unfortunate times.

Author: ncoghlan

Doc/library/sys.rst

@@ -1068,7 +1068,7 @@ always available.
    Trace functions should have three arguments: *frame*, *event*, and
    *arg*. *frame* is the current stack frame.  *event* is a string: ``'call'``,
    ``'line'``, ``'return'``, ``'exception'``, ``'c_call'``, ``'c_return'``, or
-   ``'c_exception'``. *arg* depends on the event type.
+   ``'c_exception'``, ``'opcode'``. *arg* depends on the event type.
 
    The trace function is invoked (with *event* set to ``'call'``) whenever a new
    local scope is entered; it should return a reference to a local trace
@@ -1091,6 +1091,8 @@ always available.
       ``None``; the return value specifies the new local trace function.  See
       :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this
       works.
+      Per-line events may be disabled for a frame by setting
+      :attr:`f_trace_lines` to :const:`False` on that frame.
 
    ``'return'``
       A function (or other code block) is about to return.  The local trace
@@ -1113,6 +1115,14 @@ always available.
    ``'c_exception'``
       A C function has raised an exception.  *arg* is the C function object.
 
+   ``'opcode'``
+      The interpreter is about to execute a new opcode (see :mod:`dis` for
+      opcode details).  The local trace function is called; *arg* is
+      ``None``; the return value specifies the new local trace function.
+      Per-opcode events are not emitted by default: they must be explicitly
+      requested by setting :attr:`f_trace_opcodes` to :const:`True` on the
+      frame.
+
    Note that as an exception is propagated down the chain of callers, an
    ``'exception'`` event is generated at each level.
 
@@ -1125,6 +1135,11 @@ always available.
       implementation platform, rather than part of the language definition, and
       thus may not be available in all Python implementations.
 
+   .. versionchanged:: 3.7
+
+      ``'opcode'`` event type added; :attr:`f_trace_lines` and
+      :attr:`f_trace_opcodes` attributes added to frames
+
 .. function:: set_asyncgen_hooks(firstiter, finalizer)
 
    Accepts two optional keyword arguments which are callables that accept an

Doc/reference/datamodel.rst

@@ -970,10 +970,20 @@ Internal types
 
       .. index::
          single: f_trace (frame attribute)
+         single: f_trace_lines (frame attribute)
+         single: f_trace_opcodes (frame attribute)
          single: f_lineno (frame attribute)
 
       Special writable attributes: :attr:`f_trace`, if not ``None``, is a function
-      called at the start of each source code line (this is used by the debugger);
+      called for various events during code execution (this is used by the debugger).
+      Normally an event is triggered for each new source line - this can be
+      disabled by setting :attr:`f_trace_lines` to :const:`False`.
+
+      Implementations *may* allow per-opcode events to be requested by setting
+      :attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to
+      undefined interpreter behaviour if exceptions raised by the trace
+      function escape to the function being traced.
+
       :attr:`f_lineno` is the current line number of the frame --- writing to this
       from within a trace function jumps to the given line (only for the bottom-most
       frame).  A debugger can implement a Jump command (aka Set Next Statement)

Doc/whatsnew/3.7.rst

@@ -348,6 +348,18 @@ Build and C API Changes
   (Contributed by Antoine Pitrou in :issue:`31370`.).
 
 
+Other CPython Implementation Changes
+====================================
+
+* Trace hooks may now opt out of receiving ``line`` events from the interpreter
+  by setting the new ``f_trace_lines`` attribute to :const:`False` on the frame
+  being traced. (Contributed by Nick Coghlan in :issue:`31344`.)
+
+* Trace hooks may now opt in to receiving ``opcode`` events from the interpreter
+  by setting the new ``f_trace_opcodes`` attribute to :const:`True` on the frame
+  being traced. (Contributed by Nick Coghlan in :issue:`31344`.)
+
+
 Deprecated
 ==========
 

Include/frameobject.h

@@ -27,6 +27,8 @@ typedef struct _frame {
        to the current stack top. */
     PyObject **f_stacktop;
     PyObject *f_trace;          /* Trace function */
+    char f_trace_lines;         /* Emit per-line trace events? */
+    char f_trace_opcodes;       /* Emit per-opcode trace events? */
 
     /* In a generator, we need to be able to swap between the exception
        state inside the generator and the exception state of the calling

Include/pystate.h

@@ -92,14 +92,19 @@ typedef struct _is {
 /* Py_tracefunc return -1 when raising an exception, or 0 for success. */
 typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);
 
-/* The following values are used for 'what' for tracefunc functions: */
+/* The following values are used for 'what' for tracefunc functions
+ *
+ * To add a new kind of trace event, also update "trace_init" in
+ * Python/sysmodule.c to define the Python level event name
+ */
 #define PyTrace_CALL 0
 #define PyTrace_EXCEPTION 1
 #define PyTrace_LINE 2
 #define PyTrace_RETURN 3
 #define PyTrace_C_CALL 4
 #define PyTrace_C_EXCEPTION 5
 #define PyTrace_C_RETURN 6
+#define PyTrace_OPCODE 7
 #endif
 
 #ifdef Py_LIMITED_API

Lib/test/test_sys.py

@@ -971,7 +971,7 @@ class C(object): pass
         nfrees = len(x.f_code.co_freevars)
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                   ncells + nfrees - 1
-        check(x, vsize('12P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+        check(x, vsize('8P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
         # function
         def func(): pass
         check(func, size('12P'))

Lib/test/test_sys_settrace.py

@@ -234,16 +234,29 @@ def generator_example():
 
 
 class Tracer:
-    def __init__(self):
+    def __init__(self, trace_line_events=None, trace_opcode_events=None):
+        self.trace_line_events = trace_line_events
+        self.trace_opcode_events = trace_opcode_events
         self.events = []
+
+    def _reconfigure_frame(self, frame):
+        if self.trace_line_events is not None:
+            frame.f_trace_lines = self.trace_line_events
+        if self.trace_opcode_events is not None:
+            frame.f_trace_opcodes = self.trace_opcode_events
+
     def trace(self, frame, event, arg):
+        self._reconfigure_frame(frame)
         self.events.append((frame.f_lineno, event))
         return self.trace
+
     def traceWithGenexp(self, frame, event, arg):
+        self._reconfigure_frame(frame)
         (o for o in [1])
         self.events.append((frame.f_lineno, event))
         return self.trace
 
+
 class TraceTestCase(unittest.TestCase):
 
     # Disable gc collection when tracing, otherwise the
@@ -257,6 +270,11 @@ def tearDown(self):
         if self.using_gc:
             gc.enable()
 
+    @staticmethod
+    def make_tracer():
+        """Helper to allow test subclasses to configure tracers differently"""
+        return Tracer()
+
     def compare_events(self, line_offset, events, expected_events):
         events = [(l - line_offset, e) for (l, e) in events]
         if events != expected_events:
@@ -266,7 +284,7 @@ def compare_events(self, line_offset, events, expected_events):
                                         [str(x) for x in events])))
 
     def run_and_compare(self, func, events):
-        tracer = Tracer()
+        tracer = self.make_tracer()
         sys.settrace(tracer.trace)
         func()
         sys.settrace(None)
@@ -277,7 +295,7 @@ def run_test(self, func):
         self.run_and_compare(func, func.events)
 
     def run_test2(self, func):
-        tracer = Tracer()
+        tracer = self.make_tracer()
         func(tracer.trace)
         sys.settrace(None)
         self.compare_events(func.__code__.co_firstlineno,
@@ -329,7 +347,7 @@ def test_13_genexp(self):
         # and if the traced function contains another generator
         # that is not completely exhausted, the trace stopped.
         # Worse: the 'finally' clause was not invoked.
-        tracer = Tracer()
+        tracer = self.make_tracer()
         sys.settrace(tracer.traceWithGenexp)
         generator_example()
         sys.settrace(None)
@@ -398,6 +416,34 @@ def func():
              (1, 'line')])
 
 
+class SkipLineEventsTraceTestCase(TraceTestCase):
+    """Repeat the trace tests, but with per-line events skipped"""
+
+    def compare_events(self, line_offset, events, expected_events):
+        skip_line_events = [e for e in expected_events if e[1] != 'line']
+        super().compare_events(line_offset, events, skip_line_events)
+
+    @staticmethod
+    def make_tracer():
+        return Tracer(trace_line_events=False)
+
+
+@support.cpython_only
+class TraceOpcodesTestCase(TraceTestCase):
+    """Repeat the trace tests, but with per-opcodes events enabled"""
+
+    def compare_events(self, line_offset, events, expected_events):
+        skip_opcode_events = [e for e in events if e[1] != 'opcode']
+        if len(events) > 1:
+            self.assertLess(len(skip_opcode_events), len(events),
+                            msg="No 'opcode' events received by the tracer")
+        super().compare_events(line_offset, skip_opcode_events, expected_events)
+
+    @staticmethod
+    def make_tracer():
+        return Tracer(trace_opcode_events=True)
+
+
 class RaisingTraceFuncTestCase(unittest.TestCase):
     def setUp(self):
         self.addCleanup(sys.settrace, sys.gettrace())
@@ -846,6 +892,8 @@ class fake_function:
 def test_main():
     support.run_unittest(
         TraceTestCase,
+        SkipLineEventsTraceTestCase,
+        TraceOpcodesTestCase,
         RaisingTraceFuncTestCase,
         JumpTestCase
     )

Misc/NEWS.d/next/Core and Builtins/2017-09-06-20-25-47.bpo-31344.XpFs-q.rst

@@ -0,0 +1,5 @@
+For finer control of tracing behaviour when testing the interpreter, two new
+frame attributes have been added to control the emission of particular trace
+events: ``f_trace_lines`` (``True`` by default) to turn off per-line trace
+events; and ``f_trace_opcodes`` (``False`` by default) to turn on per-opcode
+trace events.

Objects/frameobject.c

@@ -15,6 +15,8 @@ static PyMemberDef frame_memberlist[] = {
     {"f_builtins",      T_OBJECT,       OFF(f_builtins),  READONLY},
     {"f_globals",       T_OBJECT,       OFF(f_globals),   READONLY},
     {"f_lasti",         T_INT,          OFF(f_lasti),     READONLY},
+    {"f_trace_lines",   T_BOOL,         OFF(f_trace_lines), 0},
+    {"f_trace_opcodes", T_BOOL,         OFF(f_trace_opcodes), 0},
     {NULL}      /* Sentinel */
 };
 
@@ -728,6 +730,8 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code,
     f->f_iblock = 0;
     f->f_executing = 0;
     f->f_gen = NULL;
+    f->f_trace_opcodes = 0;
+    f->f_trace_lines = 1;
 
     return f;
 }

Python/ceval.c

@@ -4458,12 +4458,19 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
         *instr_lb = bounds.ap_lower;
         *instr_ub = bounds.ap_upper;
     }
-    /* If the last instruction falls at the start of a line or if
-       it represents a jump backwards, update the frame's line
-       number and call the trace function. */
-    if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) {
+    /* Always emit an opcode event if we're tracing all opcodes. */
+    if (frame->f_trace_opcodes) {
+        result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None);
+    }
+    /* If the last instruction falls at the start of a line or if it
+       represents a jump backwards, update the frame's line number and
+       then call the trace function if we're tracing source lines.
+    */
+    if ((frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev)) {
         frame->f_lineno = line;
-        result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
+        if (frame->f_trace_lines) {
+            result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
+        }
     }
     *instr_prev = frame->f_lasti;
     return result;

Python/sysmodule.c

@@ -349,18 +349,19 @@ same value.");
  * Cached interned string objects used for calling the profile and
  * trace functions.  Initialized by trace_init().
  */
-static PyObject *whatstrings[7] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL};
+static PyObject *whatstrings[8] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL};
 
 static int
 trace_init(void)
 {
-    static const char * const whatnames[7] = {
+    static const char * const whatnames[8] = {
         "call", "exception", "line", "return",
-        "c_call", "c_exception", "c_return"
+        "c_call", "c_exception", "c_return",
+        "opcode"
     };
     PyObject *name;
     int i;
-    for (i = 0; i < 7; ++i) {
+    for (i = 0; i < 8; ++i) {
         if (whatstrings[i] == NULL) {
             name = PyUnicode_InternFromString(whatnames[i]);
             if (name == NULL)