From 530ccbf3eaae9ff67d53db2b2419b241ea5d673c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 9 Jun 2020 01:53:16 +0200 Subject: [PATCH] [WIP] bpo-1635741: Py_Finalize() finalizes builtin static types Clear the following PyTypeObject members of builtin static types in Py_Finalize(): * tp_bases * tp_cache * tp_dict * tp_mro * tp_subclasses Finalize static types initialized by _PyStaticTypes_Init() and builtin exceptions initialized by _PyExc_Init(). Once a static type is finalized, it must no longer be used. Use assert(!_PyStaticType_IsFinalized(type)); to ensure that static type is not finalized. With this change, Valgrind log: possibly lost: 612,326 bytes in 5,503 blocks becomes: possibly lost: 369,346 bytes in 2,805 blocks" Changes: * Rename _PyTypes_Init() to _PyStaticTypes_Init(). * Move _PyExc_Fini() call inside finalize_interp_types(). --- Include/internal/pycore_object.h | 13 +++++ Include/internal/pycore_pylifecycle.h | 3 +- Objects/exceptions.c | 66 ++++++++++++++++++++++ Objects/object.c | 80 ++++++++++++++++++++++++++- Objects/typeobject.c | 50 +++++++++++++++++ Python/pylifecycle.c | 9 ++- 6 files changed, 217 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 32e86d06db5b43..92250c444dbcbc 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -15,6 +15,19 @@ extern "C" { PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); +// Finalize a static type: it must not be used after this call. +// Once a static type is finalized, it must no longer be used. +// Use assert(!_PyStaticType_IsFinalized(type)); to ensure that +// a static type is not finalized. +extern void _PyStaticType_Fini(PyTypeObject *type); + +#ifndef NDEBUG +// Test if _PyStaticType_Fini() was called on a static type. +// Always return 0 for heap types. +// Usage: assert(!_PyStaticType_IsFinalized(type)); +extern int _PyStaticType_IsFinalized(PyTypeObject *type); +#endif + /* Tell the GC to track this object. * * NB: While the object is tracked by the collector, it must be safe to call the diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 3e3657339a4a49..884bb5f37dee2a 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -50,7 +50,8 @@ extern PyStatus _PyImportHooks_Init(PyThreadState *tstate); extern int _PyFloat_Init(void); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); -extern PyStatus _PyTypes_Init(void); +extern PyStatus _PyStaticTypes_Init(void); +extern void _PyStaticTypes_Fini(void); extern PyStatus _PyTypes_InitSlotDefs(void); extern PyStatus _PyImportZip_Init(PyThreadState *tstate); extern PyStatus _PyGC_Init(PyThreadState *tstate); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index db5e3da12b00f3..76f117362d086f 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2740,11 +2740,77 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod) #undef INIT_ALIAS } +// Finalize exceptions initialized by _PyExc_Init(). void _PyExc_Fini(void) { free_preallocated_memerrors(); Py_CLEAR(errnomap); + + _PyStaticType_Fini(&_PyExc_BaseException); + _PyStaticType_Fini(&_PyExc_Exception); + _PyStaticType_Fini(&_PyExc_TypeError); + _PyStaticType_Fini(&_PyExc_StopAsyncIteration); + _PyStaticType_Fini(&_PyExc_StopIteration); + _PyStaticType_Fini(&_PyExc_GeneratorExit); + _PyStaticType_Fini(&_PyExc_SystemExit); + _PyStaticType_Fini(&_PyExc_KeyboardInterrupt); + _PyStaticType_Fini(&_PyExc_ImportError); + _PyStaticType_Fini(&_PyExc_ModuleNotFoundError); + _PyStaticType_Fini(&_PyExc_OSError); + _PyStaticType_Fini(&_PyExc_EOFError); + _PyStaticType_Fini(&_PyExc_RuntimeError); + _PyStaticType_Fini(&_PyExc_RecursionError); + _PyStaticType_Fini(&_PyExc_NotImplementedError); + _PyStaticType_Fini(&_PyExc_NameError); + _PyStaticType_Fini(&_PyExc_UnboundLocalError); + _PyStaticType_Fini(&_PyExc_AttributeError); + _PyStaticType_Fini(&_PyExc_SyntaxError); + _PyStaticType_Fini(&_PyExc_IndentationError); + _PyStaticType_Fini(&_PyExc_TabError); + _PyStaticType_Fini(&_PyExc_LookupError); + _PyStaticType_Fini(&_PyExc_IndexError); + _PyStaticType_Fini(&_PyExc_KeyError); + _PyStaticType_Fini(&_PyExc_ValueError); + _PyStaticType_Fini(&_PyExc_UnicodeError); + _PyStaticType_Fini(&_PyExc_UnicodeEncodeError); + _PyStaticType_Fini(&_PyExc_UnicodeDecodeError); + _PyStaticType_Fini(&_PyExc_UnicodeTranslateError); + _PyStaticType_Fini(&_PyExc_AssertionError); + _PyStaticType_Fini(&_PyExc_ArithmeticError); + _PyStaticType_Fini(&_PyExc_FloatingPointError); + _PyStaticType_Fini(&_PyExc_OverflowError); + _PyStaticType_Fini(&_PyExc_ZeroDivisionError); + _PyStaticType_Fini(&_PyExc_SystemError); + _PyStaticType_Fini(&_PyExc_ReferenceError); + _PyStaticType_Fini(&_PyExc_MemoryError); + _PyStaticType_Fini(&_PyExc_BufferError); + _PyStaticType_Fini(&_PyExc_Warning); + _PyStaticType_Fini(&_PyExc_UserWarning); + _PyStaticType_Fini(&_PyExc_DeprecationWarning); + _PyStaticType_Fini(&_PyExc_PendingDeprecationWarning); + _PyStaticType_Fini(&_PyExc_SyntaxWarning); + _PyStaticType_Fini(&_PyExc_RuntimeWarning); + _PyStaticType_Fini(&_PyExc_FutureWarning); + _PyStaticType_Fini(&_PyExc_ImportWarning); + _PyStaticType_Fini(&_PyExc_UnicodeWarning); + _PyStaticType_Fini(&_PyExc_BytesWarning); + _PyStaticType_Fini(&_PyExc_ResourceWarning); + _PyStaticType_Fini(&_PyExc_ConnectionError); + _PyStaticType_Fini(&_PyExc_BlockingIOError); + _PyStaticType_Fini(&_PyExc_BrokenPipeError); + _PyStaticType_Fini(&_PyExc_ChildProcessError); + _PyStaticType_Fini(&_PyExc_ConnectionAbortedError); + _PyStaticType_Fini(&_PyExc_ConnectionRefusedError); + _PyStaticType_Fini(&_PyExc_ConnectionResetError); + _PyStaticType_Fini(&_PyExc_FileExistsError); + _PyStaticType_Fini(&_PyExc_FileNotFoundError); + _PyStaticType_Fini(&_PyExc_IsADirectoryError); + _PyStaticType_Fini(&_PyExc_NotADirectoryError); + _PyStaticType_Fini(&_PyExc_InterruptedError); + _PyStaticType_Fini(&_PyExc_PermissionError); + _PyStaticType_Fini(&_PyExc_ProcessLookupError); + _PyStaticType_Fini(&_PyExc_TimeoutError); } /* Helper to do the equivalent of "raise X from Y" in C, but always using diff --git a/Objects/object.c b/Objects/object.c index 623ee52eb1e22d..9d5676ed4082c7 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1731,7 +1731,7 @@ PyObject _Py_NotImplementedStruct = { }; PyStatus -_PyTypes_Init(void) +_PyStaticTypes_Init(void) { PyStatus status = _PyTypes_InitSlotDefs(); if (_PyStatus_EXCEPTION(status)) { @@ -1818,6 +1818,84 @@ _PyTypes_Init(void) } +// Finalize static types initialized by _PyStaticTypes_Init(). +void +_PyStaticTypes_Fini(void) +{ + _PyStaticType_Fini(&_PyWeakref_RefType); + _PyStaticType_Fini(&_PyWeakref_CallableProxyType); + _PyStaticType_Fini(&_PyWeakref_ProxyType); + _PyStaticType_Fini(&PyLong_Type); + _PyStaticType_Fini(&PyBool_Type); + _PyStaticType_Fini(&PyByteArray_Type); + _PyStaticType_Fini(&PyBytes_Type); + _PyStaticType_Fini(&PyList_Type); + _PyStaticType_Fini(&_PyNone_Type); + _PyStaticType_Fini(&_PyNotImplemented_Type); + _PyStaticType_Fini(&PyTraceBack_Type); + _PyStaticType_Fini(&PySuper_Type); + _PyStaticType_Fini(&PyRange_Type); + _PyStaticType_Fini(&PyDict_Type); + _PyStaticType_Fini(&PyDictKeys_Type); + _PyStaticType_Fini(&PyDictValues_Type); + _PyStaticType_Fini(&PyDictItems_Type); + _PyStaticType_Fini(&PyDictRevIterKey_Type); + _PyStaticType_Fini(&PyDictRevIterValue_Type); + _PyStaticType_Fini(&PyDictRevIterItem_Type); + _PyStaticType_Fini(&PyODict_Type); + _PyStaticType_Fini(&PyODictKeys_Type); + _PyStaticType_Fini(&PyODictItems_Type); + _PyStaticType_Fini(&PyODictValues_Type); + _PyStaticType_Fini(&PyODictIter_Type); + _PyStaticType_Fini(&PySet_Type); + _PyStaticType_Fini(&PyUnicode_Type); + _PyStaticType_Fini(&PySlice_Type); + _PyStaticType_Fini(&PyStaticMethod_Type); + _PyStaticType_Fini(&PyComplex_Type); + _PyStaticType_Fini(&PyFloat_Type); + _PyStaticType_Fini(&PyFrozenSet_Type); + _PyStaticType_Fini(&PyProperty_Type); + _PyStaticType_Fini(&_PyManagedBuffer_Type); + _PyStaticType_Fini(&PyMemoryView_Type); + _PyStaticType_Fini(&PyTuple_Type); + _PyStaticType_Fini(&PyEnum_Type); + _PyStaticType_Fini(&PyReversed_Type); + _PyStaticType_Fini(&PyStdPrinter_Type); + _PyStaticType_Fini(&PyCode_Type); + _PyStaticType_Fini(&PyFrame_Type); + _PyStaticType_Fini(&PyCFunction_Type); + _PyStaticType_Fini(&PyCMethod_Type); + _PyStaticType_Fini(&PyMethod_Type); + _PyStaticType_Fini(&PyFunction_Type); + _PyStaticType_Fini(&PyDictProxy_Type); + _PyStaticType_Fini(&PyGen_Type); + _PyStaticType_Fini(&PyGetSetDescr_Type); + _PyStaticType_Fini(&PyWrapperDescr_Type); + _PyStaticType_Fini(&_PyMethodWrapper_Type); + _PyStaticType_Fini(&PyEllipsis_Type); + _PyStaticType_Fini(&PyMemberDescr_Type); + _PyStaticType_Fini(&_PyNamespace_Type); + _PyStaticType_Fini(&PyCapsule_Type); + _PyStaticType_Fini(&PyLongRangeIter_Type); + _PyStaticType_Fini(&PyCell_Type); + _PyStaticType_Fini(&PyInstanceMethod_Type); + _PyStaticType_Fini(&PyClassMethodDescr_Type); + _PyStaticType_Fini(&PyMethodDescr_Type); + _PyStaticType_Fini(&PyCallIter_Type); + _PyStaticType_Fini(&PySeqIter_Type); + _PyStaticType_Fini(&PyPickleBuffer_Type); + _PyStaticType_Fini(&PyCoro_Type); + _PyStaticType_Fini(&_PyCoroWrapper_Type); + _PyStaticType_Fini(&_PyInterpreterID_Type); + + // Finish by core types: object and type. + _PyStaticType_Fini(&PyType_Type); + _PyStaticType_Fini(&PyBaseObject_Type); + + PyType_ClearCache(); +} + + void _Py_NewReference(PyObject *op) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1d556e96be5f26..fa9e89aad25353 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -173,6 +173,29 @@ _PyType_CheckConsistency(PyTypeObject *type) #undef CHECK } + +#ifndef NDEBUG +int +_PyStaticType_IsFinalized(PyTypeObject *type) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + if (_PyRuntimeState_GetFinalizing(&_PyRuntime)) { + // _PyStaticType_Fini() sets tp_dict to NULL + return (type->tp_dict == NULL); + } + else { + return 0; + } + } + else { + // Assume that a heap type is not finalized if it's still accessible + return 0; + } + +} +#endif + + static const char * _PyType_DocWithoutSignature(const char *name, const char *internal_doc) { @@ -983,6 +1006,7 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) caller loses its exception */ assert(!_PyErr_Occurred(tstate)); #endif + assert(!_PyStaticType_IsFinalized(type)); /* Special case: type(x) should return Py_TYPE(x) */ /* We only want type itself to accept the one-argument form (#27157) */ @@ -5362,6 +5386,8 @@ static int add_operators(PyTypeObject *); int PyType_Ready(PyTypeObject *type) { + assert(!_PyStaticType_IsFinalized(type)); + PyObject *dict, *bases; PyTypeObject *base; Py_ssize_t i, n; @@ -5593,6 +5619,30 @@ PyType_Ready(PyTypeObject *type) return -1; } + +void +_PyStaticType_Fini(PyTypeObject *type) +{ + assert(!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)); + +#ifdef Py_TRACE_REFS + _Py_ForgetReference((PyObject *)type); +#endif + + Py_CLEAR(type->tp_dict); + Py_CLEAR(type->tp_bases); + Py_CLEAR(type->tp_mro); + Py_CLEAR(type->tp_cache); + Py_CLEAR(type->tp_subclasses); + + // Clear Py_TPFLAGS_READY flag + type->tp_flags &= ~Py_TPFLAGS_READY; + + // Test that _PyStaticType_IsFinalized() works as expected + assert(_PyStaticType_IsFinalized(type)); +} + + static int add_subclass(PyTypeObject *base, PyTypeObject *type) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f2f7d585c8000d..c2f2b08f3f9f2e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -586,7 +586,7 @@ pycore_init_types(PyThreadState *tstate) } if (is_main_interp) { - status = _PyTypes_Init(); + status = _PyStaticTypes_Init(); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -1253,6 +1253,12 @@ flush_std_files(void) static void finalize_interp_types(PyThreadState *tstate, int is_main_interp) { + if (is_main_interp) { + _PyExc_Fini(); + _PyStaticTypes_Fini(); + // after this point, builtin static types must no longer be used + } + _PyFrame_Fini(tstate); _PyAsyncGen_Fini(tstate); _PyContext_Fini(tstate); @@ -1302,7 +1308,6 @@ finalize_interp_clear(PyThreadState *tstate) if (is_main_interp) { PyGrammar_RemoveAccelerators(&_PyParser_Grammar); - _PyExc_Fini(); } finalize_interp_types(tstate, is_main_interp);