-
-
Notifications
You must be signed in to change notification settings - Fork 31.9k
[WIP, DO NOT MERGE] bpo-41188: Prepare CPython for opague PyObject structure. #21262
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
3d5826b
ff05339
8aa000e
8774b02
da3b98a
686d657
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -263,6 +263,10 @@ struct _typeobject { | |
|
||
destructor tp_finalize; | ||
vectorcallfunc tp_vectorcall; | ||
|
||
/* INTERNAL USE ONLY! MODIFYING THIS CAN CRASH PYTHON! */ | ||
const Py_ssize_t tp_obj_offset; /* Offset from "PyObject *" pointer */ | ||
const Py_ssize_t tp_obj_size; /* Total memory allocation size */ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't see the point of adding a new member. You can just check for Py_TPFLAGS_USES_OPAQUE_OBJECT flag in functions like PyType_GenericAlloc(), _PyObject_SIZE() and _PyObject_VAR_SIZE(). If the flag is set, add sizeof(PyObject). There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. See previous comment. |
||
}; | ||
|
||
/* The *real* layout of a type object when allocated on the heap */ | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -102,26 +102,62 @@ typedef struct _typeobject PyTypeObject; | |
* by hand. Similarly every pointer to a variable-size Python object can, | ||
* in addition, be cast to PyVarObject*. | ||
*/ | ||
typedef struct _object { | ||
|
||
typedef struct _object PyObject; | ||
typedef struct _varobject PyVarObject; | ||
|
||
// TODO: Send "_object" and "_varobject" to CPython internal only. | ||
struct _object { | ||
_PyObject_HEAD_EXTRA | ||
Py_ssize_t ob_refcnt; | ||
PyTypeObject *ob_type; | ||
} PyObject; | ||
}; | ||
|
||
struct _varobject { | ||
PyObject ob_base; | ||
Py_ssize_t ob_size; /* Number of items in variable part */ | ||
}; | ||
|
||
/* Cast argument to PyObject* type. */ | ||
#define _PyObject_CAST(op) ((PyObject*)(op)) | ||
#define _PyObject_CAST_CONST(op) ((const PyObject*)(op)) | ||
|
||
typedef struct { | ||
PyObject ob_base; | ||
Py_ssize_t ob_size; /* Number of items in variable part */ | ||
} PyVarObject; | ||
|
||
/* Cast argument to PyVarObject* type. */ | ||
#define _PyVarObject_CAST(op) ((PyVarObject*)(op)) | ||
#define _PyVarObject_CAST_CONST(op) ((const PyVarObject*)(op)) | ||
|
||
Py_SLIB_LOCAL(Py_ssize_t) PyObject_GetRefCount(const PyObject *ob); | ||
Py_SLIB_LOCAL(PyTypeObject *) PyObject_GetType(const PyObject *ob); | ||
|
||
Py_SLIB_LOCAL(void) PyObject_SetRefCount(PyObject *ob, const Py_ssize_t refcnt); | ||
Py_SLIB_LOCAL(void) PyObject_SetType(PyObject *ob, const PyTypeObject *type); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Python 3.9 already have Py_SET_REFCNT() and Py_SET_TYPE() functions. What is the advantage of adding new functions? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These aren't essentially "new functions", they are bridges into CPython's internals WITHOUT leaking implementation details. As mentioned before, the existing functions WILL FAIL when PyObject goes full opaque. |
||
|
||
|
||
Py_SLIB_LOCAL(Py_ssize_t) PyVarObject_GetSize(const PyVarObject *ob); | ||
|
||
Py_SLIB_LOCAL(void) PyVarObject_SetSize(PyVarObject *ob, const Py_ssize_t size); | ||
|
||
|
||
Py_SLIB_LOCAL(int) PyObject_IsType(const PyObject *ob, const PyTypeObject *type); | ||
|
||
|
||
Py_SLIB_LOCAL(void *) PyObject_GetStructure(const PyObject *ob, const PyTypeObject *type); | ||
#define Py_GET_STRUCTURE(ob, type) PyObject_GetStructure(_PyObject_CAST_CONST(ob), (const PyTypeObject *)type) | ||
|
||
#ifdef Py_USE_SLIB | ||
#define Py_REFCNT(ob) PyObject_GetRefCount(_PyObject_CAST_CONST(ob)) | ||
#define Py_TYPE(ob) PyObject_GetType(_PyObject_CAST_CONST(ob)) | ||
|
||
#define Py_SET_REFCNT(ob, refcnt) PyObject_SetRefCount(_PyObject_CAST(ob), refcnt) | ||
#define Py_SET_TYPE(ob, type) PyObject_SetType(_PyObject_CAST(ob), type) | ||
|
||
|
||
#define Py_SIZE(ob) PyVarObject_GetSize(_PyVarObject_CAST_CONST(ob)) | ||
|
||
#define Py_SET_SIZE(ob, size) PyVarObject_SetSize(_PyVarObject_CAST(ob), size) | ||
|
||
#define Py_IS_TYPE(ob, type) PyObject_IsType(_PyObject_CAST_CONST(ob), type) | ||
#else | ||
static inline Py_ssize_t _Py_REFCNT(const PyObject *ob) { | ||
return ob->ob_refcnt; | ||
} | ||
|
@@ -162,7 +198,7 @@ static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { | |
ob->ob_size = size; | ||
} | ||
#define Py_SET_SIZE(ob, size) _Py_SET_SIZE(_PyVarObject_CAST(ob), size) | ||
|
||
#endif | ||
|
||
/* | ||
Type objects contain a string containing the type name (to help somewhat | ||
|
@@ -317,6 +353,9 @@ Code can use PyType_HasFeature(type_ob, flag_value) to test whether the | |
given type object has a specified feature. | ||
*/ | ||
|
||
/* Set if the type object's tp_basicsize is set for opague object */ | ||
#define Py_TPFLAGS_OMIT_PYOBJECT_SIZE (1UL << 8) | ||
|
||
/* Set if the type object is dynamically allocated */ | ||
#define Py_TPFLAGS_HEAPTYPE (1UL << 9) | ||
|
||
|
@@ -419,14 +458,30 @@ PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, | |
|
||
PyAPI_FUNC(void) _Py_Dealloc(PyObject *); | ||
|
||
Py_SLIB_LOCAL(void) PyObject_IncRef(PyObject *op); | ||
|
||
Py_SLIB_LOCAL(void) PyObject_DecRef( | ||
#ifdef Py_REF_DEBUG | ||
const char *filename, int lineno, | ||
#endif | ||
PyObject *op); | ||
|
||
#ifdef Py_USE_SLIB | ||
# define Py_INCREF(op) PyObject_IncRef(_PyObject_CAST(op)) | ||
|
||
# ifdef Py_REF_DEBUG | ||
# define Py_DECREF(op) PyObject_DecRef(__FILE__, __LINE__, _PyObject_CAST(op)) | ||
# else | ||
# define Py_DECREF(op) PyObject_DecRef(_PyObject_CAST(op)) | ||
# endif | ||
#else | ||
static inline void _Py_INCREF(PyObject *op) | ||
{ | ||
#ifdef Py_REF_DEBUG | ||
_Py_RefTotal++; | ||
#endif | ||
op->ob_refcnt++; | ||
} | ||
|
||
#define Py_INCREF(op) _Py_INCREF(_PyObject_CAST(op)) | ||
|
||
static inline void _Py_DECREF( | ||
|
@@ -449,13 +504,12 @@ static inline void _Py_DECREF( | |
_Py_Dealloc(op); | ||
} | ||
} | ||
|
||
#ifdef Py_REF_DEBUG | ||
# define Py_DECREF(op) _Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) | ||
#else | ||
# define Py_DECREF(op) _Py_DECREF(_PyObject_CAST(op)) | ||
#endif | ||
|
||
#endif | ||
|
||
/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear | ||
* and tp_dealloc implementations. | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't think that this is needed. When I get a "PyObject *ob", the offset to the PyObject structure is 0: "PyObject copy = *ob;" in Python internals is valid.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's not required if everything inherited ONLY PyObject, that will never be true.
The offset is calculated from the immediate base type of the type, which can be for example: list, tuple, dict, type, custom types.
Same thing applies to "tp_obj_size", without these members, heap corruption can be very likely.