From ab2aa3ba491f5d69dd703c0150a12fcf5a1d9f3d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 9 Sep 2023 14:23:58 -0600 Subject: [PATCH 01/19] Export _PyInterpreterState_LookUpID(). --- Include/cpython/pystate.h | 2 ++ Include/internal/pycore_interp.h | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7e4c57efc7c00c..a9a2ffff16c1d4 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -236,6 +236,8 @@ PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *); PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *); PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); + /* Frame evaluation API */ typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ebf02281a7a2a6..864b29ad9ca04a 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -254,8 +254,6 @@ struct _xidregitem { crossinterpdatafunc getdata; }; -extern PyInterpreterState* _PyInterpreterState_LookUpID(int64_t); - extern int _PyInterpreterState_IDInitref(PyInterpreterState *); extern int _PyInterpreterState_IDIncref(PyInterpreterState *); extern void _PyInterpreterState_IDDecref(PyInterpreterState *); From 4d103dd4e5bbad54d3a34b349070248da1b39438 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 12:36:00 -0600 Subject: [PATCH 02/19] Optionally free the pending call's data. --- Include/internal/pycore_ceval.h | 6 +++++- Include/internal/pycore_ceval_state.h | 1 + Modules/signalmodule.c | 4 ++-- Python/ceval_gil.c | 31 ++++++++++++++++----------- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 23d0fa399d7e6f..48861da2f19765 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -41,12 +41,16 @@ extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); extern void _PyEval_SignalReceived(PyInterpreterState *interp); +// bitwise flags: +#define _Py_PENDING_MAINTHREADONLY 1 +#define _Py_PENDING_RAWFREE 2 + // Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyEval_AddPendingCall( PyInterpreterState *interp, _Py_pending_call_func func, void *arg, - int mainthreadonly); + int flags); extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index d0af5b542233e0..7befc82ea7349c 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -26,6 +26,7 @@ struct _pending_calls { struct _pending_call { _Py_pending_call_func func; void *arg; + int flags; } calls[NPENDINGCALLS]; int first; int last; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 8d6556727b3a5a..673b30cff55c06 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -315,7 +315,7 @@ trip_signal(int sig_num) _PyEval_AddPendingCall(interp, report_wakeup_send_error, (void *)(intptr_t) last_error, - 1); + _Py_PENDING_MAINTHREADONLY); } } } @@ -335,7 +335,7 @@ trip_signal(int sig_num) _PyEval_AddPendingCall(interp, report_wakeup_write_error, (void *)(intptr_t)errno, - 1); + _Py_PENDING_MAINTHREADONLY); } } } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index ba16f5eb9bfe74..b45f93bd2acab8 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -763,7 +763,7 @@ _PyEval_SignalReceived(PyInterpreterState *interp) /* Push one item onto the queue while holding the lock. */ static int _push_pending_call(struct _pending_calls *pending, - _Py_pending_call_func func, void *arg) + _Py_pending_call_func func, void *arg, int flags) { int i = pending->last; int j = (i + 1) % NPENDINGCALLS; @@ -772,13 +772,14 @@ _push_pending_call(struct _pending_calls *pending, } pending->calls[i].func = func; pending->calls[i].arg = arg; + pending->calls[i].flags = flags; pending->last = j; return 0; } static int _next_pending_call(struct _pending_calls *pending, - int (**func)(void *), void **arg) + int (**func)(void *), void **arg, int *flags) { int i = pending->first; if (i == pending->last) { @@ -788,15 +789,16 @@ _next_pending_call(struct _pending_calls *pending, } *func = pending->calls[i].func; *arg = pending->calls[i].arg; + *flags = pending->calls[i].flags; return i; } /* Pop one item off the queue while holding the lock. */ static void _pop_pending_call(struct _pending_calls *pending, - int (**func)(void *), void **arg) + int (**func)(void *), void **arg, int *flags) { - int i = _next_pending_call(pending, func, arg); + int i = _next_pending_call(pending, func, arg, flags); if (i >= 0) { pending->calls[i] = (struct _pending_call){0}; pending->first = (i + 1) % NPENDINGCALLS; @@ -810,12 +812,12 @@ _pop_pending_call(struct _pending_calls *pending, int _PyEval_AddPendingCall(PyInterpreterState *interp, - _Py_pending_call_func func, void *arg, - int mainthreadonly) + _Py_pending_call_func func, void *arg, int flags) { - assert(!mainthreadonly || _Py_IsMainInterpreter(interp)); + assert(!(flags & _Py_PENDING_MAINTHREADONLY) + || _Py_IsMainInterpreter(interp)); struct _pending_calls *pending = &interp->ceval.pending; - if (mainthreadonly) { + if (flags & _Py_PENDING_MAINTHREADONLY) { /* The main thread only exists in the main interpreter. */ assert(_Py_IsMainInterpreter(interp)); pending = &_PyRuntime.ceval.pending_mainthread; @@ -825,7 +827,7 @@ _PyEval_AddPendingCall(PyInterpreterState *interp, assert(pending->lock != NULL); PyThread_acquire_lock(pending->lock, WAIT_LOCK); - int result = _push_pending_call(pending, func, arg); + int result = _push_pending_call(pending, func, arg, flags); PyThread_release_lock(pending->lock); /* signal main loop */ @@ -839,7 +841,7 @@ Py_AddPendingCall(_Py_pending_call_func func, void *arg) /* Legacy users of this API will continue to target the main thread (of the main interpreter). */ PyInterpreterState *interp = _PyInterpreterState_Main(); - return _PyEval_AddPendingCall(interp, func, arg, 1); + return _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_MAINTHREADONLY); } static int @@ -880,17 +882,22 @@ _make_pending_calls(struct _pending_calls *pending) for (int i=0; ilock, WAIT_LOCK); - _pop_pending_call(pending, &func, &arg); + _pop_pending_call(pending, &func, &arg, &flags); PyThread_release_lock(pending->lock); /* having released the lock, perform the callback */ if (func == NULL) { break; } - if (func(arg) != 0) { + int res = func(arg); + if ((flags & _Py_PENDING_RAWFREE) && arg != NULL) { + PyMem_RawFree(arg); + } + if (res != 0) { return -1; } } From faf77787b172c9e24e781ed8784a83c35b1803b0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 12:37:44 -0600 Subject: [PATCH 03/19] Add _Py_CallInInterpreter() and _Py_CallInInterpreterAndRawFree(). --- Include/internal/pycore_ceval.h | 10 ++++++++ Python/pystate.c | 43 ++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 48861da2f19765..3bf48cd9c34ba8 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -52,6 +52,16 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall( void *arg, int flags); +typedef int (*_Py_simple_func)(void *); +extern int _Py_CallInInterpreter( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); +extern int _Py_CallInInterpreterAndRawFree( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); + extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); diff --git a/Python/pystate.c b/Python/pystate.c index fe056bf4687026..532caa816613b0 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2492,18 +2492,36 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } -static int -_release_xidata_pending(void *data) +int +_Py_CallInInterpreter(PyInterpreterState *interp, + _Py_simple_func func, void *arg) { - _xidata_clear((_PyCrossInterpreterData *)data); + if (interp == current_fast_get(interp->runtime)->interp) { + return func(arg); + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); + return 0; +} + +int +_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == current_fast_get(interp->runtime)->interp) { + int res = func(arg); + PyMem_RawFree(arg); + return res; + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); return 0; } static int -_xidata_release_and_rawfree_pending(void *data) +_call_clear_xidata(void *data) { _xidata_clear((_PyCrossInterpreterData *)data); - PyMem_RawFree(data); return 0; } @@ -2535,21 +2553,12 @@ _xidata_release(_PyCrossInterpreterData *data, int rawfree) } // "Release" the data and/or the object. - if (interp == current_fast_get(interp->runtime)->interp) { - _xidata_clear(data); - if (rawfree) { - PyMem_RawFree(data); - } + if (rawfree) { + return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, data); } else { - _Py_pending_call_func func = _release_xidata_pending; - if (rawfree) { - func = _xidata_release_and_rawfree_pending; - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, func, data, 0); + return _Py_CallInInterpreter(interp, _call_clear_xidata, data); } - return 0; } int From 5c5662dd9f2dd78ad80cd760c33406d98bd168d7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 12:38:39 -0600 Subject: [PATCH 04/19] Add PyUnstable_Buffer_ReleaseInInterpreter() and PyUnstable_Buffer_ReleaseInInterpreterAndRawFree(). --- Include/pybuffer.h | 4 ++++ Objects/abstract.c | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/Include/pybuffer.h b/Include/pybuffer.h index ca1c6058d9052c..ea09b9c1b31717 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -100,6 +100,10 @@ PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf, /* Releases a Py_buffer obtained from getbuffer ParseTuple's "s*". */ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); +PyAPI_FUNC(int) PyUnstable_Buffer_ReleaseInInterpreter( + PyInterpreterState *interp, Py_buffer *view); +PyAPI_FUNC(int) PyUnstable_Buffer_ReleaseInInterpreterAndRawFree( + PyInterpreterState *interp, Py_buffer *view); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 diff --git a/Objects/abstract.c b/Objects/abstract.c index 55d3b3ada296be..b8b4464e320362 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -806,6 +806,27 @@ PyBuffer_Release(Py_buffer *view) Py_DECREF(obj); } +static int +_buffer_release_call(void *arg) +{ + PyBuffer_Release((Py_buffer *)arg); + return 0; +} + +int +PyUnstable_Buffer_ReleaseInInterpreter(PyInterpreterState *interp, + Py_buffer *view) +{ + return _Py_CallInInterpreter(interp, _buffer_release_call, view); +} + +int +PyUnstable_Buffer_ReleaseInInterpreterAndRawFree(PyInterpreterState *interp, + Py_buffer *view) +{ + return _Py_CallInInterpreterAndRawFree(interp, _buffer_release_call, view); +} + PyObject * PyObject_Format(PyObject *obj, PyObject *format_spec) { From 573abef2ae87f2eaf8dc0b4faf9532f98e8694f9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 15:06:49 -0600 Subject: [PATCH 05/19] Add CrossInterpreterBufferView. --- Modules/_xxinterpchannelsmodule.c | 70 +++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index d5be76f1f0e38e..5b82528d425b01 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -195,6 +195,68 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) } +/* Cross-interpreter Buffer Views *******************************************/ + +// XXX Release when the original interpreter is destroyed. + +typedef struct { + PyObject_HEAD + Py_buffer *view; + int64_t interp; +} XIBufferViewObject; + +static void +xibufferview_dealloc(XIBufferViewObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interp); + /* If the interpreter is no longer alive then we have problems, + since other objects may be using the buffer still. */ + assert(interp != NULL); + + if (PyUnstable_Buffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { + // XXX Emit a warning? + PyErr_Clear(); + } + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + /* "Instances of heap-allocated types hold a reference to their type." + * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol + * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse + */ + // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, + // like we do for _abc._abc_data? + Py_DECREF(tp); +} + +static int +xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) +{ + /* Only PyMemoryView_FromObject() should ever call this, + via _memoryview_from_xid() below. */ + *view = *self->view; + view->obj = (PyObject *)self; + // XXX Should we leave it alone? + view->internal = NULL; + return 0; +} + +static PyType_Slot XIBufferViewType_slots[] = { + {Py_tp_dealloc, (destructor)xibufferview_dealloc}, + {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, + // We don't bother with Py_bf_releasebuffer since we don't need it. + {0, NULL}, +}; + +static PyType_Spec XIBufferViewType_spec = { + .name = MODULE_NAME ".CrossInterpreterBufferView", + .basicsize = sizeof(XIBufferViewObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .slots = XIBufferViewType_slots, +}; + + /* module state *************************************************************/ typedef struct { @@ -203,6 +265,7 @@ typedef struct { /* heap types */ PyTypeObject *ChannelIDType; + PyTypeObject *XIBufferViewType; /* exceptions */ PyObject *ChannelError; @@ -241,6 +304,7 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) { /* heap types */ Py_VISIT(state->ChannelIDType); + Py_VISIT(state->XIBufferViewType); /* exceptions */ Py_VISIT(state->ChannelError); @@ -263,6 +327,7 @@ clear_module_state(module_state *state) (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); } Py_CLEAR(state->ChannelIDType); + Py_CLEAR(state->XIBufferViewType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -2555,6 +2620,11 @@ module_exec(PyObject *mod) goto error; } + state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL); + if (state->XIBufferViewType == NULL) { + goto error; + } + // Make sure chnnels drop objects owned by this interpreter PyInterpreterState *interp = _get_current_interp(); PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); From be7e9279b56499887f9ae21aa5242f2274b541d7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 15:08:13 -0600 Subject: [PATCH 06/19] Add _get_current_xibufferview_type(). --- Modules/_xxinterpchannelsmodule.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 5b82528d425b01..89f0d7110de7bd 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -340,6 +340,24 @@ clear_module_state(module_state *state) } +static PyTypeObject * +_get_current_xibufferview_type(void) +{ + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME " module not imported yet"); + return NULL; + } + module_state *state = get_module_state(mod); + if (state == NULL) { + return NULL; + } + return state->XIBufferViewType; +} + + /* channel-specific code ****************************************************/ #define CHANNEL_SEND 1 From 955989db18b56c9917523e3fd00036a523e04091 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 15:08:45 -0600 Subject: [PATCH 07/19] Add xibufferview_from_xid(). --- Modules/_xxinterpchannelsmodule.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 89f0d7110de7bd..eae023e0f0c134 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -205,6 +205,22 @@ typedef struct { int64_t interp; } XIBufferViewObject; +static PyObject * +xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) +{ + assert(data->data != NULL); + assert(data->obj == NULL); + assert(data->interp >= 0); + XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); + if (self == NULL) { + return NULL; + } + PyObject_Init((PyObject *)self, cls); + self->view = (Py_buffer *)data->data; + self->interp = data->interp; + return (PyObject *)self; +} + static void xibufferview_dealloc(XIBufferViewObject *self) { From 05d13f51e11f5e9091feefc828a7953ffdbaea81 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 15:10:07 -0600 Subject: [PATCH 08/19] Register memoryview for cross-interpreter use. --- Modules/_xxinterpchannelsmodule.c | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index eae023e0f0c134..6eae9c85eb518c 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -273,6 +273,58 @@ static PyType_Spec XIBufferViewType_spec = { }; +/* extra XID types **********************************************************/ + +static PyTypeObject * _get_current_xibufferview_type(void); + +static PyObject * +_memoryview_from_xid(_PyCrossInterpreterData *data) +{ + PyTypeObject *cls = _get_current_xibufferview_type(); + if (cls == NULL) { + return NULL; + } + PyObject *obj = xibufferview_from_xid(cls, data); + if (obj == NULL) { + return NULL; + } + return PyMemoryView_FromObject(obj); +} + +static int +_memoryview_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); + if (view == NULL) { + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { + PyMem_RawFree(view); + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, + _memoryview_from_xid); + return 0; +} + +static int +register_xid_types(void) +{ + PyTypeObject *cls; + crossinterpdatafunc func; + + // builtin memoryview + cls = &PyMemoryView_Type; + func = _memoryview_shared; + if (_PyCrossInterpreterData_RegisterClass(cls, func)) { + return -1; + } + + return 0; +} + + /* module state *************************************************************/ typedef struct { @@ -2659,6 +2711,10 @@ module_exec(PyObject *mod) goto error; } + if (register_xid_types() < 0) { + goto error; + } + // Make sure chnnels drop objects owned by this interpreter PyInterpreterState *interp = _get_current_interp(); PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); From 3e4e97799e6c45a93f039a697b07c725e483a401 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Sep 2023 15:10:26 -0600 Subject: [PATCH 09/19] Add _channels.send_buffer(). --- Modules/_xxinterpchannelsmodule.c | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 6eae9c85eb518c..c529b9684ee6bb 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2472,6 +2472,39 @@ PyDoc_STRVAR(channel_send_doc, \n\ Add the object's data to the channel's queue."); +static PyObject * +channel_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", "obj", NULL}; + int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; + PyObject *obj; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:channel_send", kwlist, + channel_id_converter, &cid_data, &obj)) { + return NULL; + } + cid = cid_data.cid; + + PyObject *tempobj = PyMemoryView_FromObject(obj); + if (tempobj == NULL) { + return NULL; + } + + int err = _channel_send(&_globals.channels, cid, tempobj); + Py_DECREF(tempobj); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_send_buffer_doc, +"channel_send_buffer(cid, obj)\n\ +\n\ +Add the object's buffer to the channel's queue."); + static PyObject * channel_recv(PyObject *self, PyObject *args, PyObject *kwds) { @@ -2660,6 +2693,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channel_list_interpreters_doc}, {"send", _PyCFunction_CAST(channel_send), METH_VARARGS | METH_KEYWORDS, channel_send_doc}, + {"send_buffer", _PyCFunction_CAST(channel_send_buffer), + METH_VARARGS | METH_KEYWORDS, channel_send_buffer_doc}, {"recv", _PyCFunction_CAST(channel_recv), METH_VARARGS | METH_KEYWORDS, channel_recv_doc}, {"close", _PyCFunction_CAST(channel_close), From 16994950ecb04f88d1881e34061f35eaff859642 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 18 Sep 2023 16:38:18 -0600 Subject: [PATCH 10/19] Use _get_current_module_state(). --- Modules/_xxinterpchannelsmodule.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index c529b9684ee6bb..ffcc3f12787f1d 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -411,14 +411,7 @@ clear_module_state(module_state *state) static PyTypeObject * _get_current_xibufferview_type(void) { - PyObject *mod = _get_current_module(); - if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME " module not imported yet"); - return NULL; - } - module_state *state = get_module_state(mod); + module_state *state = _get_current_module_state(); if (state == NULL) { return NULL; } From 0c216ff82efb431bd17241b675c567d5239a3af7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 19 Sep 2023 11:43:37 -0600 Subject: [PATCH 11/19] Add SendChannel.send_buffer(). --- Lib/test/support/interpreters.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index d2beba31e80283..034e046bdbe97c 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -211,6 +211,21 @@ def send_nowait(self, obj): # See bpo-32604 and gh-19829. return _channels.send(self._id, obj) + def send_buffer(self, obj, timeout=None): + """Send the object's buffer to the channel's receiving end. + + This blocks until the object is received. + """ + _channels.send_buffer(self._id, obj, timeout=timeout) + + def send_buffer_nowait(self, obj): + """Send the object's buffer to the channel's receiving end. + + If the object is immediately received then return True + (else False). Otherwise this is the same as send(). + """ + return _channels.send_buffer(self._id, obj, blocking=False) + def close(self): _channels.close(self._id, send=True) From 0ad6f4190330e84018d7df74ef7e17b2cb90cf9e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 16:21:57 -0600 Subject: [PATCH 12/19] Keep some C-API functions private for now. --- Include/cpython/pystate.h | 2 -- Include/internal/pycore_interp.h | 3 +++ Include/internal/pycore_pybuffer.h | 21 +++++++++++++++++++++ Include/pybuffer.h | 4 ---- Makefile.pre.in | 1 + Modules/_xxinterpchannelsmodule.c | 8 +++++++- Objects/abstract.c | 9 +++++---- 7 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 Include/internal/pycore_pybuffer.h diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index a9a2ffff16c1d4..7e4c57efc7c00c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -236,8 +236,6 @@ PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *); PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *); PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void); -PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); - /* Frame evaluation API */ typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 864b29ad9ca04a..10e103aceccab9 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -254,6 +254,9 @@ struct _xidregitem { crossinterpdatafunc getdata; }; +// Export for the _xxinterpchannels module. +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); + extern int _PyInterpreterState_IDInitref(PyInterpreterState *); extern int _PyInterpreterState_IDIncref(PyInterpreterState *); extern void _PyInterpreterState_IDDecref(PyInterpreterState *); diff --git a/Include/internal/pycore_pybuffer.h b/Include/internal/pycore_pybuffer.h new file mode 100644 index 00000000000000..3cbc290b2ea3ee --- /dev/null +++ b/Include/internal/pycore_pybuffer.h @@ -0,0 +1,21 @@ +#ifndef Py_INTERNAL_PYBUFFER_H +#define Py_INTERNAL_PYBUFFER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +// Exported for the _xxinterpchannels module. +PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreter( + PyInterpreterState *interp, Py_buffer *view); +PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreterAndRawFree( + PyInterpreterState *interp, Py_buffer *view); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYBUFFER_H */ diff --git a/Include/pybuffer.h b/Include/pybuffer.h index ea09b9c1b31717..ca1c6058d9052c 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -100,10 +100,6 @@ PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf, /* Releases a Py_buffer obtained from getbuffer ParseTuple's "s*". */ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); -PyAPI_FUNC(int) PyUnstable_Buffer_ReleaseInInterpreter( - PyInterpreterState *interp, Py_buffer *view); -PyAPI_FUNC(int) PyUnstable_Buffer_ReleaseInInterpreterAndRawFree( - PyInterpreterState *interp, Py_buffer *view); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 diff --git a/Makefile.pre.in b/Makefile.pre.in index cf03c86f18b3c3..477eac0b6b49dc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1790,6 +1790,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_parking_lot.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ + $(srcdir)/Include/internal/pycore_pybuffer.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ $(srcdir)/Include/internal/pycore_pyhash.h \ $(srcdir)/Include/internal/pycore_pylifecycle.h \ diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index ffcc3f12787f1d..3a7ef2e4f61538 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -1,8 +1,14 @@ /* interpreters module */ /* low-level access to interpreter primitives */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "interpreteridobject.h" +#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() +#include "pycore_interp.h" // _PyInterpreterState_LookUpID() /* @@ -229,7 +235,7 @@ xibufferview_dealloc(XIBufferViewObject *self) since other objects may be using the buffer still. */ assert(interp != NULL); - if (PyUnstable_Buffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { + if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { // XXX Emit a warning? PyErr_Clear(); } diff --git a/Objects/abstract.c b/Objects/abstract.c index b8b4464e320362..806ca6584bda95 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_pybuffer.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_object.h" // _Py_CheckSlotResult() @@ -814,15 +815,15 @@ _buffer_release_call(void *arg) } int -PyUnstable_Buffer_ReleaseInInterpreter(PyInterpreterState *interp, - Py_buffer *view) +_PyBuffer_ReleaseInInterpreter(PyInterpreterState *interp, + Py_buffer *view) { return _Py_CallInInterpreter(interp, _buffer_release_call, view); } int -PyUnstable_Buffer_ReleaseInInterpreterAndRawFree(PyInterpreterState *interp, - Py_buffer *view) +_PyBuffer_ReleaseInInterpreterAndRawFree(PyInterpreterState *interp, + Py_buffer *view) { return _Py_CallInInterpreterAndRawFree(interp, _buffer_release_call, view); } From 56e3774fb9db88e618b76c17014aaf505794eed4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 16:43:52 -0600 Subject: [PATCH 13/19] Add a test. --- Lib/test/test__xxinterpchannels.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 750cd99b85e7a6..cb69f73c4348d4 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -703,6 +703,21 @@ def test_recv_sending_interp_destroyed(self): channels.recv(cid2) del cid2 + def test_send_buffer(self): + buf = bytearray(b'spamspamspam') + cid = channels.create() + channels.send_buffer(cid, buf) + obj = channels.recv(cid) + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + def test_allowed_types(self): cid = channels.create() objects = [ From 272488a8683b3639c3704f76671e03e708d4e7ad Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 16:48:19 -0600 Subject: [PATCH 14/19] Fix SendChannel.send_buffer_nowait(). --- Lib/test/support/interpreters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 034e046bdbe97c..9a34684a1ae532 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -224,7 +224,8 @@ def send_buffer_nowait(self, obj): If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - return _channels.send_buffer(self._id, obj, blocking=False) + return _channels.send_buffer(self._id, obj) + #return _channels.send_buffer(self._id, obj, blocking=False) def close(self): _channels.close(self._id, send=True) From ceb3dae6ad1e1df025f891369d4220f1f9fcabe7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 16:48:38 -0600 Subject: [PATCH 15/19] Fix a typo. --- Modules/_xxinterpchannelsmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 3a7ef2e4f61538..af35374f62120b 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2480,7 +2480,8 @@ channel_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) .module = self, }; PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:channel_send", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&O:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj)) { return NULL; } From 873eec23c996d0d23a09d6d3109430b2e996635c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 16:48:47 -0600 Subject: [PATCH 16/19] Add tests. --- Lib/test/test_interpreters.py | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index ffdd8a12769397..ad241856b39f5e 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1055,3 +1055,46 @@ def test_recv_nowait_default(self): self.assertEqual(obj4, b'spam') self.assertEqual(obj5, b'eggs') self.assertIs(obj6, default) + + def test_send_buffer(self): + buf = bytearray(b'spamspamspam') + obj = None + rch, sch = interpreters.create_channel() + + def f(): + nonlocal obj + while True: + try: + obj = rch.recv() + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + t = threading.Thread(target=f) + t.start() + + sch.send_buffer(buf) + t.join() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + def test_send_buffer_nowait(self): + buf = bytearray(b'spamspamspam') + rch, sch = interpreters.create_channel() + sch.send_buffer_nowait(buf) + obj = rch.recv() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) From da9779fe12fba29d89e5ec62ab16adbd89a4c0a8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 17:09:50 -0600 Subject: [PATCH 17/19] Fix SendChannel.send_buffer(). --- Lib/test/support/interpreters.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 9a34684a1ae532..04e7dbfee9971e 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -211,12 +211,12 @@ def send_nowait(self, obj): # See bpo-32604 and gh-19829. return _channels.send(self._id, obj) - def send_buffer(self, obj, timeout=None): + def send_buffer(self, obj): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - _channels.send_buffer(self._id, obj, timeout=timeout) + _channels.send_buffer(self._id, obj) def send_buffer_nowait(self, obj): """Send the object's buffer to the channel's receiving end. @@ -225,7 +225,6 @@ def send_buffer_nowait(self, obj): (else False). Otherwise this is the same as send(). """ return _channels.send_buffer(self._id, obj) - #return _channels.send_buffer(self._id, obj, blocking=False) def close(self): _channels.close(self._id, send=True) From dea173a1c9c4b3ee118d5e8115be153322f4f9b2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Oct 2023 10:15:54 -0600 Subject: [PATCH 18/19] Clear the XID classes during module fini. --- Modules/_xxinterpchannelsmodule.c | 136 ++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 43 deletions(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index af35374f62120b..01aa9f5eae64ea 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -82,6 +82,74 @@ API.. The module does not create any objects that are shared globally. PyMem_RawFree(VAR) +struct xid_class_registry { + size_t count; +#define MAX_XID_CLASSES 5 + struct { + PyTypeObject *cls; + } added[MAX_XID_CLASSES]; +}; + +static int +register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, + struct xid_class_registry *classes) +{ + int res = _PyCrossInterpreterData_RegisterClass(cls, shared); + if (res == 0) { + assert(classes->count < MAX_XID_CLASSES); + Py_INCREF(cls); + classes->added[classes->count].cls = cls; + classes->count += 1; + } + return res; +} + +static void +clear_xid_class_registry(struct xid_class_registry *classes) +{ + while (classes->count > 0) { + classes->count -= 1; + PyTypeObject *cls = classes->added[classes->count].cls; + _PyCrossInterpreterData_UnregisterClass(cls); + Py_DECREF(cls); + } +} + +#define XID_IGNORE_EXC 1 +#define XID_FREE 2 + +static int +_release_xid_data(_PyCrossInterpreterData *data, int flags) +{ + int ignoreexc = flags & XID_IGNORE_EXC; + PyObject *exc; + if (ignoreexc) { + exc = PyErr_GetRaisedException(); + } + int res; + if (flags & XID_FREE) { + res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + } + else { + res = _PyCrossInterpreterData_Release(data); + } + if (res < 0) { + /* The owning interpreter is already destroyed. */ + if (ignoreexc) { + // XXX Emit a warning? + PyErr_Clear(); + } + } + if (flags & XID_FREE) { + /* Either way, we free the data. */ + } + if (ignoreexc) { + PyErr_SetRaisedException(exc); + } + return res; +} + + static PyInterpreterState * _get_current_interp(void) { @@ -146,7 +214,8 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) static PyTypeObject * -add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared) +add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, + struct xid_class_registry *classes) { PyTypeObject *cls = (PyTypeObject *)PyType_FromMetaclass( NULL, mod, spec, NULL); @@ -158,7 +227,7 @@ add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared) return NULL; } if (shared != NULL) { - if (_PyCrossInterpreterData_RegisterClass(cls, shared)) { + if (register_xid_class(cls, shared, classes)) { Py_DECREF(cls); return NULL; } @@ -166,40 +235,6 @@ add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared) return cls; } -#define XID_IGNORE_EXC 1 -#define XID_FREE 2 - -static int -_release_xid_data(_PyCrossInterpreterData *data, int flags) -{ - int ignoreexc = flags & XID_IGNORE_EXC; - PyObject *exc; - if (ignoreexc) { - exc = PyErr_GetRaisedException(); - } - int res; - if (flags & XID_FREE) { - res = _PyCrossInterpreterData_ReleaseAndRawFree(data); - } - else { - res = _PyCrossInterpreterData_Release(data); - } - if (res < 0) { - /* The owning interpreter is already destroyed. */ - if (ignoreexc) { - // XXX Emit a warning? - PyErr_Clear(); - } - } - if (flags & XID_FREE) { - /* Either way, we free the data. */ - } - if (ignoreexc) { - PyErr_SetRaisedException(exc); - } - return res; -} - /* Cross-interpreter Buffer Views *******************************************/ @@ -315,7 +350,7 @@ _memoryview_shared(PyThreadState *tstate, PyObject *obj, } static int -register_xid_types(void) +register_builtin_xid_types(struct xid_class_registry *classes) { PyTypeObject *cls; crossinterpdatafunc func; @@ -323,7 +358,7 @@ register_xid_types(void) // builtin memoryview cls = &PyMemoryView_Type; func = _memoryview_shared; - if (_PyCrossInterpreterData_RegisterClass(cls, func)) { + if (register_xid_class(cls, func, classes)) { return -1; } @@ -334,6 +369,9 @@ register_xid_types(void) /* module state *************************************************************/ typedef struct { + struct xid_class_registry xid_classes; + + /* Added at runtime by interpreters module. */ PyTypeObject *send_channel_type; PyTypeObject *recv_channel_type; @@ -2191,6 +2229,7 @@ set_channel_end_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) if (state == NULL) { return -1; } + struct xid_class_registry *xid_classes = &state->xid_classes; if (state->send_channel_type != NULL || state->recv_channel_type != NULL) @@ -2201,10 +2240,10 @@ set_channel_end_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - if (_PyCrossInterpreterData_RegisterClass(send, _channel_end_shared)) { + if (register_xid_class(send, _channel_end_shared, xid_classes)) { return -1; } - if (_PyCrossInterpreterData_RegisterClass(recv, _channel_end_shared)) { + if (register_xid_class(recv, _channel_end_shared, xid_classes)) { return -1; } @@ -2722,6 +2761,7 @@ module_exec(PyObject *mod) if (_globals_init() != 0) { return -1; } + struct xid_class_registry *xid_classes = NULL; /* Add exception types */ if (exceptions_init(mod) != 0) { @@ -2733,20 +2773,22 @@ module_exec(PyObject *mod) if (state == NULL) { goto error; } + xid_classes = &state->xid_classes; // ChannelID state->ChannelIDType = add_new_type( - mod, &ChannelIDType_spec, _channelid_shared); + mod, &ChannelIDType_spec, _channelid_shared, xid_classes); if (state->ChannelIDType == NULL) { goto error; } - state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL); + state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL, + xid_classes); if (state->XIBufferViewType == NULL) { goto error; } - if (register_xid_types() < 0) { + if (register_builtin_xid_types(xid_classes) < 0) { goto error; } @@ -2757,6 +2799,9 @@ module_exec(PyObject *mod) return 0; error: + if (xid_classes != NULL) { + clear_xid_class_registry(xid_classes); + } _globals_fini(); return -1; } @@ -2781,6 +2826,11 @@ module_clear(PyObject *mod) { module_state *state = get_module_state(mod); assert(state != NULL); + + // Before clearing anything, we unregister the various XID types. */ + clear_xid_class_registry(&state->xid_classes); + + // Now we clear the module state. clear_module_state(state); return 0; } From fc15f77681f819b5c195673f9ced76fa8fae7fc9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 9 Oct 2023 05:55:48 -0600 Subject: [PATCH 19/19] Fix ref leaks. --- Modules/_xxinterpchannelsmodule.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 99ae433788f3cf..a1531c5c3db34d 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -97,7 +97,7 @@ register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, int res = _PyCrossInterpreterData_RegisterClass(cls, shared); if (res == 0) { assert(classes->count < MAX_XID_CLASSES); - Py_INCREF(cls); + // The class has refs elsewhere, so we need to incref here. classes->added[classes->count].cls = cls; classes->count += 1; } @@ -111,7 +111,6 @@ clear_xid_class_registry(struct xid_class_registry *classes) classes->count -= 1; PyTypeObject *cls = classes->added[classes->count].cls; _PyCrossInterpreterData_UnregisterClass(cls); - Py_DECREF(cls); } } @@ -2770,17 +2769,18 @@ module_exec(PyObject *mod) } struct xid_class_registry *xid_classes = NULL; + module_state *state = get_module_state(mod); + if (state == NULL) { + goto error; + } + xid_classes = &state->xid_classes; + /* Add exception types */ if (exceptions_init(mod) != 0) { goto error; } /* Add other types */ - module_state *state = get_module_state(mod); - if (state == NULL) { - goto error; - } - xid_classes = &state->xid_classes; // ChannelID state->ChannelIDType = add_new_type( @@ -2799,7 +2799,7 @@ module_exec(PyObject *mod) goto error; } - // Make sure chnnels drop objects owned by this interpreter + /* Make sure chnnels drop objects owned by this interpreter. */ PyInterpreterState *interp = _get_current_interp(); PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); @@ -2847,7 +2847,13 @@ module_free(void *mod) { module_state *state = get_module_state(mod); assert(state != NULL); + + // Before clearing anything, we unregister the various XID types. */ + clear_xid_class_registry(&state->xid_classes); + + // Now we clear the module state. clear_module_state(state); + _globals_fini(); }