From ec8b6b6165d1d6bd4a6d0b7bb6cdd55fc0513e99 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Oct 2021 13:37:27 +0200 Subject: [PATCH] bpo-39947: Add PyThreadState_SetTrace() function Add new PyThreadState_SetProfile() and PyThreadState_SetTrace() functions to set the profile and the trace function of a Python thread state. * Rename _PyEval_SetProfile() to PyThreadState_SetProfile(). * Rename _PyEval_SetTrace() to PyThreadState_SetTrace(). * Rename is_tstate_valid() to _PyThreadState_CheckConsistency(). --- Doc/c-api/init.rst | 86 ++++++++++++++---- Doc/whatsnew/3.11.rst | 5 ++ Include/cpython/ceval.h | 2 - Include/cpython/pystate.h | 8 ++ Include/internal/pycore_pystate.h | 10 +++ .../2021-10-21-15-00-43.bpo-39947.KOqrdL.rst | 3 + Modules/_lsprof.c | 6 +- Python/ceval.c | 89 ++----------------- Python/ceval_gil.h | 8 +- Python/pystate.c | 76 +++++++++++++++- Python/sysmodule.c | 12 +-- 11 files changed, 189 insertions(+), 116 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2021-10-21-15-00-43.bpo-39947.KOqrdL.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 09dfc68fee57dc..d06ccd1fbaba47 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1187,8 +1187,68 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Resume tracing and profiling in the Python thread state *tstate* suspended by the :c:func:`PyThreadState_EnterTracing` function. - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. + See also :c:func:`PyThreadState_SetTrace` and + :c:func:`PyThreadState_SetProfile` functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyThreadState_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) + + Set the profile function of *tstate*. + + The *arg* parameter is passed to the function as its first parameter, and + may be any Python object, or ``NULL``. + + If the profile function needs to maintain state, using a different value for + *arg* for each thread provides a convenient and thread-safe place to store + it. + + The profile function is called for all monitored events except + :const:`PyTrace_LINE` :const:`PyTrace_OPCODE` and + :const:`PyTrace_EXCEPTION`. + + Calling ``PyThreadState_SetProfile(tstate, NULL, NULL)`` removes the profile + function. + + Return 0 on success. Raise an exception and return -1 on error. + + The caller must hold the :term:`GIL`. *func* and *arg* can be *NULL*. + + See also functions: + + * :func:`sys.setprofile` + * :c:func:`PyEval_SetProfile` + * :c:func:`PyThreadState_EnterTracing` + * :c:func:`PyThreadState_SetTrace` + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) + + Set the tracing function of *tstate*. + + This is similar to :c:func:`PyThreadState_SetProfile`, except the tracing function + does receive line-number events and per-opcode events, but does not receive + any event related to C function objects being called. Any trace function + registered using :c:func:`PyThreadState_SetTrace` will not receive + :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or + :const:`PyTrace_C_RETURN` as a value for the *what* parameter. + + Calling ``PyThreadState_SetTrace(tstate, NULL, NULL)`` removes the trace + function. + + Return 0 on success. Raise an exception and return -1 on error. + + The caller must hold the :term:`GIL`. *func* and *arg* can be *NULL*. + + See also functions: + + * :func:`sys.settrace` + * :c:func:`PyEval_SetTrace` + * :c:func:`PyThreadState_EnterTracing` + * :c:func:`PyThreadState_SetProfile` .. versionadded:: 3.11 @@ -1545,7 +1605,7 @@ Python-level trace functions in previous versions. .. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - The type of the trace function registered using :c:func:`PyEval_SetProfile` and + The type of the trace function registered using :c:func:`PyThreadState_SetProfile` and :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the registration function as *obj*, *frame* is the frame object to which the event pertains, *what* is one of the constants :const:`PyTrace_CALL`, @@ -1636,28 +1696,20 @@ Python-level trace functions in previous versions. .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or ``NULL``. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :const:`PyTrace_LINE` - :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`. + Set the trace function of the current thread using the + :c:func:`PyThreadState_SetProfile` function. - See also the :func:`sys.setprofile` function. + On error, log the unraisable exception with :data:`sys.unraisablehook`. The caller must hold the :term:`GIL`. .. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and per-opcode events, but does not receive any event related to C function - objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or - :const:`PyTrace_C_RETURN` as a value for the *what* parameter. + Set the trace function of the current thread using the + :c:func:`PyThreadState_SetTrace` function. - See also the :func:`sys.settrace` function. + On error, log the unraisable exception with :data:`sys.unraisablehook`. The caller must hold the :term:`GIL`. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 13c1e72306653e..1bdd477fc40681 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -482,6 +482,11 @@ New Features suspend and resume tracing and profiling. (Contributed by Victor Stinner in :issue:`43760`.) +* Add new :c:func:`PyThreadState_SetProfile` and + :c:func:`PyThreadState_SetTrace` functions to set the profile and the trace + function of a Python thread state. + (Contributed by Victor Stinner in :issue:`39947`.) + Porting to Python 3.11 ---------------------- diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index caf64401307c07..5cbfe70e42ea74 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -5,9 +5,7 @@ PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args); PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); -PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); -PyAPI_FUNC(int) _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(int) _PyEval_GetCoroutineOriginTrackingDepth(void); PyAPI_FUNC(int) _PyEval_SetAsyncGenFirstiter(PyObject *); PyAPI_FUNC(PyObject *) _PyEval_GetAsyncGenFirstiter(void); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 528d2a2998c997..3707b11a765d96 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -192,6 +192,14 @@ PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); // function is set, otherwise disable them. PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); +PyAPI_FUNC(int) PyThreadState_SetProfile(PyThreadState *tstate, + Py_tracefunc func, + PyObject *arg); + +PyAPI_FUNC(int) PyThreadState_SetTrace(PyThreadState *tstate, + Py_tracefunc func, + PyObject *arg); + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 9a570b08bc5839..746dfc593de49e 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -147,6 +147,16 @@ _PyThreadState_ResumeTracing(PyThreadState *tstate) tstate->cframe->use_tracing = (use_tracing ? 255 : 0); } +#ifndef NDEBUG +// Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and +// PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen +// when a thread continues to run after Python finalization, especially +// daemon threads. +// +// Usage: assert(_PyThreadState_CheckConsistency(tstate)); +extern int _PyThreadState_CheckConsistency(PyThreadState *tstate); +#endif + /* Other */ diff --git a/Misc/NEWS.d/next/C API/2021-10-21-15-00-43.bpo-39947.KOqrdL.rst b/Misc/NEWS.d/next/C API/2021-10-21-15-00-43.bpo-39947.KOqrdL.rst new file mode 100644 index 00000000000000..c910c8bc8a28d9 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-10-21-15-00-43.bpo-39947.KOqrdL.rst @@ -0,0 +1,3 @@ +Add new :c:func:`PyThreadState_SetProfile` and +:c:func:`PyThreadState_SetTrace` functions to set the profile and the trace +function of a Python thread state. Patch by Victor Stinner. diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 2e27afcea1b799..cd898c89e40ec2 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -674,7 +674,7 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) } PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { + if (PyThreadState_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { return NULL; } @@ -708,7 +708,7 @@ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { + if (PyThreadState_SetProfile(tstate, NULL, NULL) < 0) { return NULL; } self->flags &= ~POF_ENABLED; @@ -745,7 +745,7 @@ profiler_dealloc(ProfilerObject *op) { if (op->flags & POF_ENABLED) { PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { + if (PyThreadState_SetProfile(tstate, NULL, NULL) < 0) { PyErr_WriteUnraisable((PyObject *)op); } } diff --git a/Python/ceval.c b/Python/ceval.c index 76325903149cd5..94936f3ff07b9f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -123,20 +123,6 @@ static long dxp[256]; #endif #endif -#ifndef NDEBUG -/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and - PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen - when a thread continues to run after Python finalization, especially - daemon threads. */ -static int -is_tstate_valid(PyThreadState *tstate) -{ - assert(!_PyMem_IsPtrFreed(tstate)); - assert(!_PyMem_IsPtrFreed(tstate->interp)); - return 1; -} -#endif - /* This can set eval_breaker to 0 even though gil_drop_request became 1. We believe this is all right because the eval loop will release @@ -403,7 +389,7 @@ PyEval_AcquireThread(PyThreadState *tstate) void PyEval_ReleaseThread(PyThreadState *tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); _PyRuntimeState *runtime = tstate->interp->runtime; PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); @@ -623,7 +609,7 @@ Py_AddPendingCall(int (*func)(void *), void *arg) static int handle_signals(PyThreadState *tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } @@ -691,7 +677,7 @@ void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); struct _pending_calls *pending = &tstate->interp->ceval.pending; @@ -716,7 +702,7 @@ Py_MakePendingCalls(void) assert(PyGILState_Check()); PyThreadState *tstate = _PyThreadState_GET(); - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); /* Python signal handler doesn't really queue a callback: it only signals that a signal was received, see _PyEval_SignalReceived(). */ @@ -5841,7 +5827,7 @@ make_coro_frame(PyThreadState *tstate, PyObject *const *args, Py_ssize_t argcount, PyObject *kwnames) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); assert(con->fc_defaults == NULL || PyTuple_CheckExact(con->fc_defaults)); PyCodeObject *code = (PyCodeObject *)con->fc_code; int size = code->co_nlocalsplus+code->co_stacksize + FRAME_SPECIALS_SIZE; @@ -6381,84 +6367,23 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, return result; } -int -_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { - return -1; - } - - PyObject *profileobj = tstate->c_profileobj; - - tstate->c_profilefunc = NULL; - tstate->c_profileobj = NULL; - /* Must make sure that tracing is not ignored if 'profileobj' is freed */ - _PyThreadState_ResumeTracing(tstate); - Py_XDECREF(profileobj); - - Py_XINCREF(arg); - tstate->c_profileobj = arg; - tstate->c_profilefunc = func; - - /* Flag that tracing or profiling is turned on */ - _PyThreadState_ResumeTracing(tstate); - return 0; -} void PyEval_SetProfile(Py_tracefunc func, PyObject *arg) { PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, func, arg) < 0) { + if (PyThreadState_SetProfile(tstate, func, arg) < 0) { /* Log _PySys_Audit() error */ _PyErr_WriteUnraisableMsg("in PyEval_SetProfile", NULL); } } -int -_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { - return -1; - } - - PyObject *traceobj = tstate->c_traceobj; - - tstate->c_tracefunc = NULL; - tstate->c_traceobj = NULL; - /* Must make sure that profiling is not ignored if 'traceobj' is freed */ - _PyThreadState_ResumeTracing(tstate); - Py_XDECREF(traceobj); - - Py_XINCREF(arg); - tstate->c_traceobj = arg; - tstate->c_tracefunc = func; - - /* Flag that tracing or profiling is turned on */ - _PyThreadState_ResumeTracing(tstate); - - return 0; -} void PyEval_SetTrace(Py_tracefunc func, PyObject *arg) { PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetTrace(tstate, func, arg) < 0) { + if (PyThreadState_SetTrace(tstate, func, arg) < 0) { /* Log _PySys_Audit() error */ _PyErr_WriteUnraisableMsg("in PyEval_SetTrace", NULL); } diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h index 9b8b43253f04d2..9a4d3968973c1e 100644 --- a/Python/ceval_gil.h +++ b/Python/ceval_gil.h @@ -173,7 +173,7 @@ drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) { - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); RESET_GIL_DROP_REQUEST(tstate->interp); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take @@ -228,7 +228,7 @@ take_gil(PyThreadState *tstate) PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; struct _ceval_runtime_state *ceval = &interp->runtime->ceval; struct _ceval_state *ceval2 = &interp->ceval; @@ -264,7 +264,7 @@ take_gil(PyThreadState *tstate) MUTEX_UNLOCK(gil->mutex); PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); SET_GIL_DROP_REQUEST(interp); } @@ -302,7 +302,7 @@ take_gil(PyThreadState *tstate) drop_gil(ceval, ceval2, tstate); PyThread_exit_thread(); } - assert(is_tstate_valid(tstate)); + assert(_PyThreadState_CheckConsistency(tstate)); if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) { RESET_GIL_DROP_REQUEST(interp); diff --git a/Python/pystate.c b/Python/pystate.c index 7804e17a064e15..fa46deaef9e7b5 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -608,6 +608,17 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) return interp->dict; } + +#ifndef NDEBUG +int +_PyThreadState_CheckConsistency(PyThreadState *tstate) +{ + assert(!_PyMem_IsPtrFreed(tstate)); + assert(!_PyMem_IsPtrFreed(tstate->interp)); + return 1; +} +#endif + /* Minimum size of data stack chunk */ #define DATA_STACK_CHUNK_SIZE (16*1024) @@ -1141,8 +1152,7 @@ PyThreadState_GetFrame(PyThreadState *tstate) if (frame == NULL) { PyErr_Clear(); } - Py_XINCREF(frame); - return frame; + return (PyFrameObject*)Py_XNewRef(frame); } @@ -2110,6 +2120,68 @@ _PyThreadState_PopFrame(PyThreadState *tstate, InterpreterFrame * frame) } +int +PyThreadState_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(_PyThreadState_CheckConsistency(tstate)); + // The caller must hold the GIL + assert(PyGILState_Check()); + + // Call _PySys_Audit() in the context of the current thread state, + // even if tstate is not the current thread state. + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { + return -1; + } + + PyObject *profileobj = tstate->c_profileobj; + + tstate->c_profilefunc = NULL; + tstate->c_profileobj = NULL; + // Must make sure that tracing is not ignored if 'profileobj' is freed + _PyThreadState_ResumeTracing(tstate); + Py_XDECREF(profileobj); + + tstate->c_profileobj = Py_XNewRef(arg); + tstate->c_profilefunc = func; + + // Flag that tracing or profiling is turned on + _PyThreadState_ResumeTracing(tstate); + return 0; +} + + +int +PyThreadState_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(_PyThreadState_CheckConsistency(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + + PyObject *traceobj = tstate->c_traceobj; + + tstate->c_tracefunc = NULL; + tstate->c_traceobj = NULL; + /* Must make sure that profiling is not ignored if 'traceobj' is freed */ + _PyThreadState_ResumeTracing(tstate); + Py_XDECREF(traceobj); + + tstate->c_traceobj = Py_XNewRef(arg); + tstate->c_tracefunc = func; + + /* Flag that tracing or profiling is turned on */ + _PyThreadState_ResumeTracing(tstate); + + return 0; +} + #ifdef __cplusplus } #endif diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5e663c17c79b85..894ba179bcbfcb 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -980,7 +980,7 @@ profile_trampoline(PyObject *self, PyFrameObject *frame, PyThreadState *tstate = _PyThreadState_GET(); PyObject *result = call_trampoline(tstate, self, frame, what, arg); if (result == NULL) { - _PyEval_SetProfile(tstate, NULL, NULL); + PyThreadState_SetProfile(tstate, NULL, NULL); return -1; } @@ -1006,7 +1006,7 @@ trace_trampoline(PyObject *self, PyFrameObject *frame, PyThreadState *tstate = _PyThreadState_GET(); PyObject *result = call_trampoline(tstate, callback, frame, what, arg); if (result == NULL) { - _PyEval_SetTrace(tstate, NULL, NULL); + PyThreadState_SetTrace(tstate, NULL, NULL); Py_CLEAR(frame->f_trace); return -1; } @@ -1029,12 +1029,12 @@ sys_settrace(PyObject *self, PyObject *args) PyThreadState *tstate = _PyThreadState_GET(); if (args == Py_None) { - if (_PyEval_SetTrace(tstate, NULL, NULL) < 0) { + if (PyThreadState_SetTrace(tstate, NULL, NULL) < 0) { return NULL; } } else { - if (_PyEval_SetTrace(tstate, trace_trampoline, args) < 0) { + if (PyThreadState_SetTrace(tstate, trace_trampoline, args) < 0) { return NULL; } } @@ -1078,12 +1078,12 @@ sys_setprofile(PyObject *self, PyObject *args) PyThreadState *tstate = _PyThreadState_GET(); if (args == Py_None) { - if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { + if (PyThreadState_SetProfile(tstate, NULL, NULL) < 0) { return NULL; } } else { - if (_PyEval_SetProfile(tstate, profile_trampoline, args) < 0) { + if (PyThreadState_SetProfile(tstate, profile_trampoline, args) < 0) { return NULL; } }