From 158010e1f2de24fba72ac97a31c09e15b016ec5a Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 8 Dec 2021 14:20:52 +0000
Subject: [PATCH 01/21] Spacialize calls to normal Python classes.

---
 Include/internal/pycore_ceval.h      |   2 +
 Include/internal/pycore_code.h       |   2 +-
 Include/internal/pycore_object.h     |   2 +
 Include/opcode.h                     |  94 ++++++++++----------
 Lib/importlib/_bootstrap_external.py |   3 +-
 Lib/opcode.py                        |   2 +
 Objects/dictobject.c                 |   8 +-
 Objects/typeobject.c                 |  20 +++++
 Python/ceval.c                       |  90 +++++++++++++++++++
 Python/compile.c                     |   3 +
 Python/opcode_targets.h              |  32 +++----
 Python/specialize.c                  | 127 +++++++++++++++++++++++++++
 12 files changed, 317 insertions(+), 68 deletions(-)

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 20508d4a687475..cedbd59952bcaf 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -115,6 +115,8 @@ struct _interpreter_frame *_PyEval_GetFrame(void);
 
 PyObject *_Py_MakeCoro(PyFunctionObject *func);
 
+extern PyFunctionObject *_Py_InitCleanupFunc;
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 496d52f580f1f3..43b3cd2d5c39b2 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -288,7 +288,7 @@ void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
 #define COLLECT_SPECIALIZATION_STATS_DETAILED PRINT_SPECIALIZATION_STATS_DETAILED
 #endif
 
-#define SPECIALIZATION_FAILURE_KINDS 20
+#define SPECIALIZATION_FAILURE_KINDS 30
 
 #if COLLECT_SPECIALIZATION_STATS
 
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index 9041a4dc8a3ce5..c5981a24de7ef2 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -188,8 +188,10 @@ extern int _Py_CheckSlotResult(
 #define _PyType_IsReady(type) ((type)->tp_dict != NULL)
 
 extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems);
+PyObject *_PyType_NewManagedObject(PyTypeObject *type);
 
 extern int _PyObject_InitializeDict(PyObject *obj);
+int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
 extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
                                           PyObject *name, PyObject *value);
 PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
diff --git a/Include/opcode.h b/Include/opcode.h
index f22f7e94f6190c..40d05af9760abe 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -18,6 +18,7 @@ extern "C" {
 #define UNARY_NEGATIVE                   11
 #define UNARY_NOT                        12
 #define UNARY_INVERT                     15
+#define EXIT_INIT_CHECK                  16
 #define BINARY_SUBSCR                    25
 #define GET_LEN                          30
 #define MATCH_MAPPING                    31
@@ -116,52 +117,53 @@ extern "C" {
 #define BINARY_OP_ADD_INT                 8
 #define BINARY_OP_ADD_FLOAT              13
 #define BINARY_OP_ADD_UNICODE            14
-#define BINARY_OP_INPLACE_ADD_UNICODE    16
-#define BINARY_OP_MULTIPLY_INT           17
-#define BINARY_OP_MULTIPLY_FLOAT         18
-#define BINARY_OP_SUBTRACT_INT           19
-#define BINARY_OP_SUBTRACT_FLOAT         20
-#define COMPARE_OP_ADAPTIVE              21
-#define COMPARE_OP_FLOAT_JUMP            22
-#define COMPARE_OP_INT_JUMP              23
-#define COMPARE_OP_STR_JUMP              24
-#define BINARY_SUBSCR_ADAPTIVE           26
-#define BINARY_SUBSCR_GETITEM            27
-#define BINARY_SUBSCR_LIST_INT           28
-#define BINARY_SUBSCR_TUPLE_INT          29
-#define BINARY_SUBSCR_DICT               34
-#define STORE_SUBSCR_ADAPTIVE            36
-#define STORE_SUBSCR_LIST_INT            38
-#define STORE_SUBSCR_DICT                39
-#define CALL_FUNCTION_ADAPTIVE           40
-#define CALL_FUNCTION_BUILTIN_O          41
-#define CALL_FUNCTION_BUILTIN_FAST       42
-#define CALL_FUNCTION_LEN                43
-#define CALL_FUNCTION_ISINSTANCE         44
-#define CALL_FUNCTION_PY_SIMPLE          45
-#define JUMP_ABSOLUTE_QUICK              46
-#define LOAD_ATTR_ADAPTIVE               47
-#define LOAD_ATTR_INSTANCE_VALUE         48
-#define LOAD_ATTR_WITH_HINT              55
-#define LOAD_ATTR_SLOT                   56
-#define LOAD_ATTR_MODULE                 57
-#define LOAD_GLOBAL_ADAPTIVE             58
-#define LOAD_GLOBAL_MODULE               59
-#define LOAD_GLOBAL_BUILTIN              62
-#define LOAD_METHOD_ADAPTIVE             63
-#define LOAD_METHOD_CACHED               64
-#define LOAD_METHOD_CLASS                65
-#define LOAD_METHOD_MODULE               66
-#define LOAD_METHOD_NO_DICT              67
-#define STORE_ATTR_ADAPTIVE              75
-#define STORE_ATTR_INSTANCE_VALUE        76
-#define STORE_ATTR_SLOT                  77
-#define STORE_ATTR_WITH_HINT             78
-#define LOAD_FAST__LOAD_FAST             79
-#define STORE_FAST__LOAD_FAST            80
-#define LOAD_FAST__LOAD_CONST            81
-#define LOAD_CONST__LOAD_FAST            87
-#define STORE_FAST__STORE_FAST           88
+#define BINARY_OP_INPLACE_ADD_UNICODE    17
+#define BINARY_OP_MULTIPLY_INT           18
+#define BINARY_OP_MULTIPLY_FLOAT         19
+#define BINARY_OP_SUBTRACT_INT           20
+#define BINARY_OP_SUBTRACT_FLOAT         21
+#define COMPARE_OP_ADAPTIVE              22
+#define COMPARE_OP_FLOAT_JUMP            23
+#define COMPARE_OP_INT_JUMP              24
+#define COMPARE_OP_STR_JUMP              26
+#define BINARY_SUBSCR_ADAPTIVE           27
+#define BINARY_SUBSCR_GETITEM            28
+#define BINARY_SUBSCR_LIST_INT           29
+#define BINARY_SUBSCR_TUPLE_INT          34
+#define BINARY_SUBSCR_DICT               36
+#define STORE_SUBSCR_ADAPTIVE            38
+#define STORE_SUBSCR_LIST_INT            39
+#define STORE_SUBSCR_DICT                40
+#define CALL_FUNCTION_ADAPTIVE           41
+#define CALL_FUNCTION_BUILTIN_O          42
+#define CALL_FUNCTION_BUILTIN_FAST       43
+#define CALL_FUNCTION_LEN                44
+#define CALL_FUNCTION_ISINSTANCE         45
+#define CALL_FUNCTION_PY_SIMPLE          46
+#define CALL_FUNCTION_ALLOC_AND_ENTER_INIT  47
+#define JUMP_ABSOLUTE_QUICK              48
+#define LOAD_ATTR_ADAPTIVE               55
+#define LOAD_ATTR_INSTANCE_VALUE         56
+#define LOAD_ATTR_WITH_HINT              57
+#define LOAD_ATTR_SLOT                   58
+#define LOAD_ATTR_MODULE                 59
+#define LOAD_GLOBAL_ADAPTIVE             62
+#define LOAD_GLOBAL_MODULE               63
+#define LOAD_GLOBAL_BUILTIN              64
+#define LOAD_METHOD_ADAPTIVE             65
+#define LOAD_METHOD_CACHED               66
+#define LOAD_METHOD_CLASS                67
+#define LOAD_METHOD_MODULE               75
+#define LOAD_METHOD_NO_DICT              76
+#define STORE_ATTR_ADAPTIVE              77
+#define STORE_ATTR_INSTANCE_VALUE        78
+#define STORE_ATTR_SLOT                  79
+#define STORE_ATTR_WITH_HINT             80
+#define LOAD_FAST__LOAD_FAST             81
+#define STORE_FAST__LOAD_FAST            87
+#define LOAD_FAST__LOAD_CONST            88
+#define LOAD_CONST__LOAD_FAST           123
+#define STORE_FAST__STORE_FAST          127
 #define DO_TRACING                      255
 #ifdef NEED_OPCODE_JUMP_TABLES
 static uint32_t _PyOpcode_RelativeJump[8] = {
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 6970e9f0a94d49..790c82d716dc53 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -371,6 +371,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into
 #                         BINARY_OP)
 #     Python 3.11a3 3465 (Add COPY_FREE_VARS opcode)
+#     Python 3.11a3 3469 (Add EXIT_INIT_CHECK opcode)
 
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -380,7 +381,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3465).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3469).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'
diff --git a/Lib/opcode.py b/Lib/opcode.py
index e5889bca4c161c..7b6d6994de5729 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -67,6 +67,7 @@ def jabs_op(name, op):
 def_op('UNARY_NOT', 12)
 
 def_op('UNARY_INVERT', 15)
+def_op('EXIT_INIT_CHECK', 16)
 
 def_op('BINARY_SUBSCR', 25)
 
@@ -252,6 +253,7 @@ def jabs_op(name, op):
     "CALL_FUNCTION_LEN",
     "CALL_FUNCTION_ISINSTANCE",
     "CALL_FUNCTION_PY_SIMPLE",
+    "CALL_FUNCTION_ALLOC_AND_ENTER_INIT",
     "JUMP_ABSOLUTE_QUICK",
     "LOAD_ATTR_ADAPTIVE",
     "LOAD_ATTR_INSTANCE_VALUE",
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 7ce4b9069f77ef..a2d16df94bcfdd 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -4957,11 +4957,10 @@ _PyDict_NewKeysForClass(void)
 
 #define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
 
-static int
-init_inline_values(PyObject *obj, PyTypeObject *tp)
+int
+_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
 {
     assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
-    // assert(type->tp_dictoffset > 0);  -- TO DO Update this assert.
     assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictKeysObject *keys = CACHED_KEYS(tp);
     assert(keys != NULL);
@@ -4972,6 +4971,7 @@ init_inline_values(PyObject *obj, PyTypeObject *tp)
     assert(size > 0);
     PyDictValues *values = new_values(size);
     if (values == NULL) {
+        *_PyObject_ValuesPointer(obj) = NULL;
         PyErr_NoMemory();
         return -1;
     }
@@ -4991,7 +4991,7 @@ _PyObject_InitializeDict(PyObject *obj)
         return 0;
     }
     if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        return init_inline_values(obj, tp);
+        return _PyObject_InitInlineValues(obj, tp);
     }
     PyObject *dict;
     if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 2fd93b61c0b2b0..1a1972bfc7a8c9 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1139,6 +1139,26 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds)
     return obj;
 }
 
+PyObject *
+_PyType_NewManagedObject(PyTypeObject *type)
+{
+    assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    assert(_PyType_IS_GC(type));
+    assert(type->tp_new == PyBaseObject_Type.tp_new);
+    assert(type->tp_alloc == PyType_GenericAlloc);
+    assert(type->tp_itemsize == 0);
+    PyObject *obj = PyType_GenericAlloc(type, 0);
+    if (obj == NULL) {
+        return PyErr_NoMemory();
+    }
+    *_PyObject_ManagedDictPointer(obj) = NULL;
+    if (_PyObject_InitInlineValues(obj, type)) {
+        Py_DECREF(obj);
+        return NULL;
+    }
+    return obj;
+}
+
 PyObject *
 _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
 {
diff --git a/Python/ceval.c b/Python/ceval.c
index 4f5ccf51e9cfe7..ffa987cf1c3bcb 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -102,6 +102,8 @@ static InterpreterFrame *
 _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
                         PyObject *locals, PyObject* const* args,
                         size_t argcount, PyObject *kwnames);
+static InterpreterFrame *
+_PyEvalFramePush(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals);
 static void
 _PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame *frame);
 
@@ -2720,6 +2722,20 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             DISPATCH();
         }
 
+        TARGET(EXIT_INIT_CHECK) {
+            assert(STACK_LEVEL() == 2);
+            PyObject *should_be_none = TOP();
+            if (should_be_none != Py_None) {
+                PyErr_Format(PyExc_TypeError,
+                    "__init__() should return None, not '%.200s'",
+                    Py_TYPE(should_be_none)->tp_name);
+                goto error;
+            }
+            Py_DECREF(Py_None);
+            STACK_SHRINK(1);
+            DISPATCH();
+        }
+
         TARGET(POP_EXCEPT) {
             PyObject *type, *value, *traceback;
             _PyErr_StackItem *exc_info;
@@ -4741,6 +4757,63 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             goto start_frame;
         }
 
+        TARGET(CALL_FUNCTION_ALLOC_AND_ENTER_INIT) {
+            SpecializedCacheEntry *caches = GET_CACHE();
+            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
+            int argcount = cache0->original_oparg;
+            _PyObjectCache *cache1 = &caches[-1].obj;
+            PyObject *callable = PEEK(argcount+1);
+            DEOPT_IF(!PyType_Check(callable), CALL_FUNCTION);
+            PyTypeObject *tp = (PyTypeObject *)callable;
+            DEOPT_IF(tp->tp_version_tag != cache0->version, CALL_FUNCTION);
+            PyFunctionObject *init = (PyFunctionObject *)cache1->obj;
+            PyCodeObject *code = (PyCodeObject *)init->func_code;
+            DEOPT_IF(code->co_argcount != argcount+1, CALL_FUNCTION);
+            PyObject *self = _PyType_NewManagedObject(tp);
+            if (self == NULL) {
+                goto error;
+            }
+            PEEK(argcount+1) = self;
+            Py_DECREF(tp);
+            assert(_Py_InitCleanupFunc != NULL);
+            InterpreterFrame *shim = _PyEvalFramePush(tstate, _Py_InitCleanupFunc, NULL);
+            if (shim == NULL) {
+                goto error;
+            }
+            shim->previous = frame;
+            shim->depth = frame->depth + 1;
+            shim->f_lasti = 1;
+            if (_Py_EnterRecursiveCall(tstate, "")) {
+                tstate->recursion_remaining--;
+                goto exit_unwind;
+            }
+            /* Push self onto stack of shim */
+            Py_INCREF(self);
+            shim->stacktop = 1;
+            shim->localsplus[0] = self;
+            size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
+            InterpreterFrame *init_frame = _PyThreadState_BumpFramePointer(tstate, size);
+            if (init_frame == NULL) {
+                _PyEvalFrameClearAndPop(tstate, shim);
+                goto error;
+            }
+            _PyFrame_InitializeSpecials(init_frame, init,
+                                        NULL, code->co_nlocalsplus);
+            /* Copy self followed by args to __init__ frame */
+            STACK_SHRINK(argcount+1);
+            _PyFrame_SetStackPointer(frame, stack_pointer);
+            for (int i = 0; i < argcount+1; i++) {
+                init_frame->localsplus[i] = stack_pointer[i];
+            }
+            for (int i = argcount+1; i < code->co_nlocalsplus; i++) {
+                init_frame->localsplus[i] = NULL;
+            }
+            init_frame->previous = shim;
+            init_frame->depth = shim->depth + 1;
+            frame = cframe.current_frame = init_frame;
+            goto start_frame;
+        }
+
         TARGET(CALL_FUNCTION_BUILTIN_O) {
             assert(cframe.use_tracing == 0);
             /* Builtin METH_O functions */
@@ -5858,6 +5931,23 @@ make_coro(PyThreadState *tstate, PyFunctionObject *func,
     return gen;
 }
 
+static InterpreterFrame *
+_PyEvalFramePush(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals)
+{
+    PyCodeObject * code = (PyCodeObject *)func->func_code;
+    size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
+    InterpreterFrame *frame = _PyThreadState_BumpFramePointer(tstate, size);
+    if (frame == NULL) {
+        return NULL;
+    }
+    _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus);
+    PyObject **localsarray = &frame->localsplus[0];
+    for (int i = 0; i < code->co_nlocalsplus; i++) {
+        localsarray[i] = NULL;
+    }
+    return frame;
+}
+
 /* Consumes all the references to the args */
 static InterpreterFrame *
 _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
diff --git a/Python/compile.c b/Python/compile.c
index 6138031833ac93..e370c5f232253d 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -1115,6 +1115,9 @@ stack_effect(int opcode, int oparg, int jump)
         case LOAD_GLOBAL:
             return 1;
 
+        case EXIT_INIT_CHECK:
+            return -1;
+
         /* Exception handling pseudo-instructions */
         case SETUP_FINALLY:
             /* 0 in the normal flow.
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 872a6883119926..8c016460986948 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -15,6 +15,7 @@ static void *opcode_targets[256] = {
     &&TARGET_BINARY_OP_ADD_FLOAT,
     &&TARGET_BINARY_OP_ADD_UNICODE,
     &&TARGET_UNARY_INVERT,
+    &&TARGET_EXIT_INIT_CHECK,
     &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
     &&TARGET_BINARY_OP_MULTIPLY_INT,
     &&TARGET_BINARY_OP_MULTIPLY_FLOAT,
@@ -23,20 +24,20 @@ static void *opcode_targets[256] = {
     &&TARGET_COMPARE_OP_ADAPTIVE,
     &&TARGET_COMPARE_OP_FLOAT_JUMP,
     &&TARGET_COMPARE_OP_INT_JUMP,
-    &&TARGET_COMPARE_OP_STR_JUMP,
     &&TARGET_BINARY_SUBSCR,
+    &&TARGET_COMPARE_OP_STR_JUMP,
     &&TARGET_BINARY_SUBSCR_ADAPTIVE,
     &&TARGET_BINARY_SUBSCR_GETITEM,
     &&TARGET_BINARY_SUBSCR_LIST_INT,
-    &&TARGET_BINARY_SUBSCR_TUPLE_INT,
     &&TARGET_GET_LEN,
     &&TARGET_MATCH_MAPPING,
     &&TARGET_MATCH_SEQUENCE,
     &&TARGET_MATCH_KEYS,
-    &&TARGET_BINARY_SUBSCR_DICT,
+    &&TARGET_BINARY_SUBSCR_TUPLE_INT,
     &&TARGET_PUSH_EXC_INFO,
-    &&TARGET_STORE_SUBSCR_ADAPTIVE,
+    &&TARGET_BINARY_SUBSCR_DICT,
     &&TARGET_POP_EXCEPT_AND_RERAISE,
+    &&TARGET_STORE_SUBSCR_ADAPTIVE,
     &&TARGET_STORE_SUBSCR_LIST_INT,
     &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_CALL_FUNCTION_ADAPTIVE,
@@ -45,28 +46,27 @@ static void *opcode_targets[256] = {
     &&TARGET_CALL_FUNCTION_LEN,
     &&TARGET_CALL_FUNCTION_ISINSTANCE,
     &&TARGET_CALL_FUNCTION_PY_SIMPLE,
+    &&TARGET_CALL_FUNCTION_ALLOC_AND_ENTER_INIT,
     &&TARGET_JUMP_ABSOLUTE_QUICK,
-    &&TARGET_LOAD_ATTR_ADAPTIVE,
-    &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
     &&TARGET_WITH_EXCEPT_START,
     &&TARGET_GET_AITER,
     &&TARGET_GET_ANEXT,
     &&TARGET_BEFORE_ASYNC_WITH,
     &&TARGET_BEFORE_WITH,
     &&TARGET_END_ASYNC_FOR,
+    &&TARGET_LOAD_ATTR_ADAPTIVE,
+    &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
     &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LOAD_ATTR_SLOT,
     &&TARGET_LOAD_ATTR_MODULE,
-    &&TARGET_LOAD_GLOBAL_ADAPTIVE,
-    &&TARGET_LOAD_GLOBAL_MODULE,
     &&TARGET_STORE_SUBSCR,
     &&TARGET_DELETE_SUBSCR,
+    &&TARGET_LOAD_GLOBAL_ADAPTIVE,
+    &&TARGET_LOAD_GLOBAL_MODULE,
     &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LOAD_METHOD_ADAPTIVE,
     &&TARGET_LOAD_METHOD_CACHED,
     &&TARGET_LOAD_METHOD_CLASS,
-    &&TARGET_LOAD_METHOD_MODULE,
-    &&TARGET_LOAD_METHOD_NO_DICT,
     &&TARGET_GET_ITER,
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
@@ -74,20 +74,20 @@ static void *opcode_targets[256] = {
     &&TARGET_YIELD_FROM,
     &&TARGET_GET_AWAITABLE,
     &&TARGET_LOAD_ASSERTION_ERROR,
+    &&TARGET_LOAD_METHOD_MODULE,
+    &&TARGET_LOAD_METHOD_NO_DICT,
     &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
     &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_LOAD_FAST__LOAD_FAST,
-    &&TARGET_STORE_FAST__LOAD_FAST,
-    &&TARGET_LOAD_FAST__LOAD_CONST,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
     &&TARGET_YIELD_VALUE,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
-    &&TARGET_STORE_FAST__STORE_FAST,
+    &&TARGET_STORE_FAST__LOAD_FAST,
+    &&TARGET_LOAD_FAST__LOAD_CONST,
     &&TARGET_POP_EXCEPT,
     &&TARGET_STORE_NAME,
     &&TARGET_DELETE_NAME,
@@ -122,11 +122,11 @@ static void *opcode_targets[256] = {
     &&TARGET_COPY,
     &&TARGET_JUMP_IF_NOT_EXC_MATCH,
     &&TARGET_BINARY_OP,
-    &&_unknown_opcode,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_LOAD_FAST,
     &&TARGET_STORE_FAST,
     &&TARGET_DELETE_FAST,
-    &&_unknown_opcode,
+    &&TARGET_STORE_FAST__STORE_FAST,
     &&_unknown_opcode,
     &&TARGET_GEN_START,
     &&TARGET_RAISE_VARARGS,
diff --git a/Python/specialize.c b/Python/specialize.c
index cdc535396fa762..1c92fd1192f572 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -6,6 +6,8 @@
 #include "pycore_object.h"
 #include "opcode.h"
 #include "structmember.h"         // struct PyMemberDef, T_OFFSET_EX
+#include "pycore_function.h"      // _PyFunction_FromConstructor()
+
 
 #include <stdlib.h> // rand()
 
@@ -491,6 +493,7 @@ initial_counter_value(void) {
 #define SPEC_FAIL_PYCFUNCTION_NOARGS 16
 #define SPEC_FAIL_BAD_CALL_FLAGS 17
 #define SPEC_FAIL_CLASS 18
+#define SPEC_FAIL_INIT_NOT_SIMPLE 25
 
 /* COMPARE_OP */
 #define SPEC_FAIL_STRING_COMPARE 13
@@ -1256,11 +1259,56 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins
     return 0;
 }
 
+
+static PyFunctionObject *
+get_init_for_simple_managed_python_class(PyTypeObject *tp)
+{
+    _Py_IDENTIFIER(__init__);
+    if (tp->tp_new != PyBaseObject_Type.tp_new) {
+        return NULL;
+    }
+    if (tp->tp_alloc != PyType_GenericAlloc) {
+        return NULL;
+    }
+    if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+        return NULL;
+    }
+    PyObject *init = _PyType_LookupId(tp, &PyId___init__);
+    if (init == NULL || !PyFunction_Check(init)) {
+        return NULL;
+    }
+    int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init));
+    if (kind != SIMPLE_FUNCTION) {
+        return NULL;
+    }
+    return (PyFunctionObject *)init;
+}
+
+static int
+setup_init_cleanup_func(void);
+
 static int
 specialize_class_call(
     PyObject *callable, _Py_CODEUNIT *instr,
     int nargs, SpecializedCacheEntry *cache)
 {
+    if (setup_init_cleanup_func()) {
+        return -1;
+    }
+    PyTypeObject *tp = (PyTypeObject *)callable;
+    PyFunctionObject *init = get_init_for_simple_managed_python_class(tp);
+    if (init) {
+        if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) {
+            SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+            return -1;
+        }
+        _PyAdaptiveEntry *cache0 = &cache[0].adaptive;
+        _PyObjectCache *cache1 = &cache[-1].obj;
+        cache0->version = tp->tp_version_tag;
+        cache1->obj = (PyObject *)init; /* borrowed */
+        *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_ALLOC_AND_ENTER_INIT, _Py_OPARG(*instr));
+        return 0;
+    }
     SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_CLASS);
     return -1;
 }
@@ -1587,3 +1635,82 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
     STAT_INC(COMPARE_OP, specialization_success);
     adaptive->counter = initial_counter_value();
 }
+
+PyFunctionObject *_Py_InitCleanupFunc = NULL;
+
+char INIT_CLEANUP_CODE[8] = {
+    LOAD_ASSERTION_ERROR, 0,
+    RAISE_VARARGS, 1,
+    EXIT_INIT_CHECK, 0,
+    RETURN_VALUE, 0
+};
+
+static int
+setup_init_cleanup_func(void) {
+    if (_Py_InitCleanupFunc != NULL) {
+        return 0;
+    }
+    PyObject *empty_bytes = PyBytes_FromStringAndSize(NULL, 0);
+    PyObject *empty_tuple = PyTuple_New(0);
+    PyObject *empty_str = _PyUnicode_FromASCII("", 0);
+    PyObject *name = _PyUnicode_FromASCII("type.__call__", strlen("type.__call__"));
+    PyObject *code = PyBytes_FromStringAndSize(INIT_CLEANUP_CODE, 8);
+    if (empty_bytes == NULL || empty_str == NULL || name == NULL || code == NULL) {
+        goto cleanup;
+    }
+    struct _PyCodeConstructor con = {
+        .filename = empty_str,
+        .name = name,
+        .qualname = name,
+        .flags = CO_NEWLOCALS | CO_OPTIMIZED,
+
+        .code = code,
+        .firstlineno = 1,
+        .linetable = empty_bytes,
+        .endlinetable = empty_bytes,
+        .columntable = empty_bytes,
+
+        .consts = empty_tuple,
+        .names = empty_tuple,
+
+        .localsplusnames = empty_tuple,
+        .localspluskinds = empty_bytes,
+
+        .argcount = 0,
+        .posonlyargcount = 0,
+        .kwonlyargcount = 0,
+
+        .stacksize = 2,
+
+        .exceptiontable = empty_bytes,
+    };
+
+    PyCodeObject *codeobj = _PyCode_New(&con);
+    if (codeobj == NULL) {
+        goto cleanup;
+    }
+    PyObject *globals = PyDict_New();
+    if (globals == NULL) {
+        Py_DECREF(codeobj);
+        goto cleanup;
+    }
+    PyFrameConstructor desc = {
+        .fc_globals = globals,
+        .fc_builtins = globals,
+        .fc_name = codeobj->co_name,
+        .fc_qualname = codeobj->co_name,
+        .fc_code = (PyObject *)codeobj,
+        .fc_defaults = NULL,
+        .fc_kwdefaults = NULL,
+        .fc_closure = NULL
+    };
+    _Py_InitCleanupFunc = _PyFunction_FromConstructor(&desc);
+cleanup:
+    Py_XDECREF(empty_bytes);
+    Py_XDECREF(empty_tuple);
+    Py_XDECREF(empty_str);
+    Py_XDECREF(name);
+    Py_XDECREF(code);
+    PyErr_Clear();
+    return _Py_InitCleanupFunc == NULL ? -1 : 0;
+}

From c607dfe563707fcce00a5665fb8bae446db6bf7a Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 5 Jan 2022 10:56:38 +0000
Subject: [PATCH 02/21] Don't change magic number.

---
 Lib/importlib/_bootstrap_external.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 5ead6caf9f3c75..29324664cea864 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -375,6 +375,8 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.11a4 3467 (Change CALL_xxx opcodes)
 #     Python 3.11a4 3468 (Add SEND opcode)
 #     Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info)
+#     Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti)
+#     Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE)
 
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -384,7 +386,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3469).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3471).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

From e6002c4e4265761292a94772ff36b335c04c7770 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 5 Jan 2022 11:12:06 +0000
Subject: [PATCH 03/21] Add news

---
 .../2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst          | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst

diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst
new file mode 100644
index 00000000000000..5633097f4a3fdd
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst	
@@ -0,0 +1,11 @@
+Specializes calls to most Python classes. Specifically, any class that
+inherits from ``object``, or another Python class, and does not override
+``__new__``.
+
+The specialized instruction does the following:
+
+1. Creates the object (by calling ``object.__new__``)
+2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
+3. Pushes the frame for ``__init__`` to the frame stack
+
+Speeds up the instantiation of most Python classes.

From 697bc4fae11920b9b9c0bd04f40b2d9d56e4bba3 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 25 May 2022 11:06:00 +0100
Subject: [PATCH 04/21] Fix line table for init cleanup function.

---
 Python/specialize.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Python/specialize.c b/Python/specialize.c
index bc30b79fde8356..35ad6fa2b5267f 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -2027,8 +2027,11 @@ setup_init_cleanup_func(void) {
     PyObject *empty_str = _PyUnicode_FromASCII("", 0);
     PyObject *name = _PyUnicode_FromASCII("type.__call__", strlen("type.__call__"));
     PyObject *code = PyBytes_FromStringAndSize(INIT_CLEANUP_CODE, 8);
+    const unsigned char loc[2] = { 0x80 | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) | 3, 0 };
+    PyObject *lines = PyBytes_FromStringAndSize((const char *)&loc, 2);
     if (empty_bytes == NULL || empty_str == NULL || name == NULL || code == NULL) {
         goto cleanup;
+
     }
     struct _PyCodeConstructor con = {
         .filename = empty_str,
@@ -2038,7 +2041,7 @@ setup_init_cleanup_func(void) {
 
         .code = code,
         .firstlineno = 1,
-        .linetable = empty_bytes,
+        .linetable = lines,
 
         .consts = empty_tuple,
         .names = empty_tuple,
@@ -2081,6 +2084,7 @@ setup_init_cleanup_func(void) {
     Py_XDECREF(empty_str);
     Py_XDECREF(name);
     Py_XDECREF(code);
+    Py_XDECREF(lines);
     PyErr_Clear();
     return _Py_InitCleanupFunc == NULL ? -1 : 0;
 }

From c6f8f8e83e1fb4507c8e15f88e2a1638e8d9f3d8 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 25 May 2022 11:35:30 +0100
Subject: [PATCH 05/21] Fix refleaks

---
 Python/ceval.c      |  1 +
 Python/specialize.c | 50 +++++++++++++++++++++++++--------------------
 2 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/Python/ceval.c b/Python/ceval.c
index 0cc34e0b791f09..c1423c198dce35 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -446,6 +446,7 @@ PyEval_InitThreads(void)
 void
 _PyEval_Fini(void)
 {
+    Py_CLEAR(_Py_InitCleanupFunc);
 #ifdef Py_STATS
     _Py_PrintSpecializationStats(1);
 #endif
diff --git a/Python/specialize.c b/Python/specialize.c
index 35ad6fa2b5267f..2dda1aa8a4b991 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -2022,19 +2022,27 @@ setup_init_cleanup_func(void) {
     if (_Py_InitCleanupFunc != NULL) {
         return 0;
     }
-    PyObject *empty_bytes = PyBytes_FromStringAndSize(NULL, 0);
-    PyObject *empty_tuple = PyTuple_New(0);
-    PyObject *empty_str = _PyUnicode_FromASCII("", 0);
-    PyObject *name = _PyUnicode_FromASCII("type.__call__", strlen("type.__call__"));
-    PyObject *code = PyBytes_FromStringAndSize(INIT_CLEANUP_CODE, 8);
-    const unsigned char loc[2] = { 0x80 | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) | 3, 0 };
-    PyObject *lines = PyBytes_FromStringAndSize((const char *)&loc, 2);
-    if (empty_bytes == NULL || empty_str == NULL || name == NULL || code == NULL) {
+    PyObject *name = NULL;
+    PyObject *code = NULL;
+    PyObject *lines = NULL;
+    PyCodeObject *codeobj = NULL;
+    PyObject *globals = NULL;
+
+    name = _PyUnicode_FromASCII("type.__call__", strlen("type.__call__"));
+    if (name == NULL) {
+        goto cleanup;
+    }
+    code = PyBytes_FromStringAndSize(INIT_CLEANUP_CODE, 8);
+    if (code == NULL) {
+        goto cleanup;
+    }
+    const char loc[2] = { 0x80 | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) | 3, 0 };
+    lines = PyBytes_FromStringAndSize(loc, 2);
+    if (lines == NULL) {
         goto cleanup;
-
     }
     struct _PyCodeConstructor con = {
-        .filename = empty_str,
+        .filename = &_Py_STR(empty),
         .name = name,
         .qualname = name,
         .flags = CO_NEWLOCALS | CO_OPTIMIZED,
@@ -2043,11 +2051,11 @@ setup_init_cleanup_func(void) {
         .firstlineno = 1,
         .linetable = lines,
 
-        .consts = empty_tuple,
-        .names = empty_tuple,
+        .consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
+        .names = (PyObject *)&_Py_SINGLETON(tuple_empty),
 
-        .localsplusnames = empty_tuple,
-        .localspluskinds = empty_bytes,
+        .localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
+        .localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
 
         .argcount = 0,
         .posonlyargcount = 0,
@@ -2055,16 +2063,15 @@ setup_init_cleanup_func(void) {
 
         .stacksize = 2,
 
-        .exceptiontable = empty_bytes,
+        .exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
     };
 
-    PyCodeObject *codeobj = _PyCode_New(&con);
+    codeobj = _PyCode_New(&con);
     if (codeobj == NULL) {
         goto cleanup;
     }
-    PyObject *globals = PyDict_New();
+    globals = PyDict_New();
     if (globals == NULL) {
-        Py_DECREF(codeobj);
         goto cleanup;
     }
     PyFrameConstructor desc = {
@@ -2079,13 +2086,12 @@ setup_init_cleanup_func(void) {
     };
     _Py_InitCleanupFunc = _PyFunction_FromConstructor(&desc);
 cleanup:
-    Py_XDECREF(empty_bytes);
-    Py_XDECREF(empty_tuple);
-    Py_XDECREF(empty_str);
+    PyErr_Clear();
+    Py_XDECREF(codeobj);
+    Py_DECREF(globals);
     Py_XDECREF(name);
     Py_XDECREF(code);
     Py_XDECREF(lines);
-    PyErr_Clear();
     return _Py_InitCleanupFunc == NULL ? -1 : 0;
 }
 

From 5edb42383830d0f459134c9d48145fb169f1e0f7 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 25 May 2022 11:59:37 +0100
Subject: [PATCH 06/21] Add news

---
 .../2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst              | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst

diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst
new file mode 100644
index 00000000000000..af55673c5064e9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst	
@@ -0,0 +1,3 @@
+Specialize calls to Python class by inserting a shim frame to cleanup after
+``__init__`` method, allocating the object, and jumping directly to the
+``__init__`` method.

From 7166184b5165a4acb4d91cce5f01681526b5a32f Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 25 May 2022 12:36:09 +0100
Subject: [PATCH 07/21] Update summarize stats script.

---
 Tools/scripts/summarize_stats.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 3d7479f261b4af..c0886a4e39c9b8 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -68,7 +68,7 @@ def print_specialization_stats(name, family_stats, defines):
             rows.append((label, val, f"{100*val/total_attempts:0.1f}%"))
         emit_table(("", "Count:", "Ratio:"), rows)
         total_failures = family_stats.get("specialization.failure", 0)
-        failure_kinds = [ 0 ] * 30
+        failure_kinds = [ 0 ] * 32
         for key in family_stats:
             if not key.startswith("specialization.failure_kind"):
                 continue

From 3260c42060598b5980191945b38eb0d7111dc2a6 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 25 May 2022 13:59:02 +0100
Subject: [PATCH 08/21] Fix stats for inlined py calls

---
 Python/ceval.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Python/ceval.c b/Python/ceval.c
index c1423c198dce35..6e6f968792e6da 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4985,6 +4985,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             if (shim == NULL) {
                 goto error;
             }
+            CALL_STAT_INC(inlined_py_calls);
             shim->previous = frame;
             shim->prev_instr = _PyCode_CODE(shim->f_code) + 1;
             assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK);

From 67492425fd31ae4835ac88f83cfbd50c157de5bf Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 25 May 2022 16:25:11 +0100
Subject: [PATCH 09/21] Remove duplicate news item.

---
 .../2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst              | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst

diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst
deleted file mode 100644
index af55673c5064e9..00000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2022-05-25-11-58-04.gh-issue-91095.tOu58a.rst	
+++ /dev/null
@@ -1,3 +0,0 @@
-Specialize calls to Python class by inserting a shim frame to cleanup after
-``__init__`` method, allocating the object, and jumping directly to the
-``__init__`` method.

From 426b5b3f647874b3eb8ec2814ec9796ea7c36575 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 25 May 2022 17:56:18 +0100
Subject: [PATCH 10/21] Tidy up

---
 Lib/opcode.py       |  2 +-
 Python/specialize.c | 11 +----------
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/Lib/opcode.py b/Lib/opcode.py
index 7079c6a5b6d9e1..9258f730e277c4 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -262,7 +262,7 @@ def jabs_op(name, op):
         "CALL_NO_KW_BUILTIN_O",
         "CALL_NO_KW_ISINSTANCE",
         "CALL_NO_KW_LEN",
-    "CALL_NO_KW_ALLOC_AND_ENTER_INIT",
+        "CALL_NO_KW_ALLOC_AND_ENTER_INIT",
         "CALL_NO_KW_LIST_APPEND",
         "CALL_NO_KW_METHOD_DESCRIPTOR_FAST",
         "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
diff --git a/Python/specialize.c b/Python/specialize.c
index 5ce64250baea69..63432623dd3e23 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -907,7 +907,7 @@ typedef enum {
     MANAGED_DICT = 2,
     OFFSET_DICT = 3,
     NO_DICT = 4,
-    LAZY_DICT = 5,    
+    LAZY_DICT = 5,
 } ObjectDictKind;
 
 // Please collect stats carefully before and after modifying. A subtle change
@@ -2015,15 +2015,6 @@ char INIT_CLEANUP_CODE[8] = {
     RETURN_VALUE, 0
 };
 
-/* Long form
-    LOAD_ASSERTION_ERROR, 0,
-    RAISE_VARARGS, 1,
-    POP_JUMP_FORWARD_IF_NOT_NONE, 1,
-    RETURN_VALUE, 0
-    LOAD_CONSTANT TypeError
-    LOAD_CONSTANT "__init__() should return None, not
-*/
-
 PyFunctionObject *_Py_InitCleanupFunc = NULL;
 
 static int

From e70711a15567c87389b9fdc42c6d5c8d390c9cbc Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Fri, 27 May 2022 11:44:03 +0100
Subject: [PATCH 11/21] Fix decrefs

---
 Python/ceval.c      | 2 +-
 Python/specialize.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Python/ceval.c b/Python/ceval.c
index a55bae24c8c0ea..691ec39cddb9f9 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5054,7 +5054,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
                 goto error;
             }
             STACK_SHRINK(1);
-            Py_DECREF(Py_None);
+            _Py_DECREF_NO_DEALLOC(Py_None);
             NOTRACE_DISPATCH();
         }
 
diff --git a/Python/specialize.c b/Python/specialize.c
index 63432623dd3e23..bf9ddfca3fe1af 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -2088,7 +2088,7 @@ setup_init_cleanup_func(void) {
 cleanup:
     PyErr_Clear();
     Py_XDECREF(codeobj);
-    Py_DECREF(globals);
+    Py_XDECREF(globals);
     Py_XDECREF(name);
     Py_XDECREF(code);
     Py_XDECREF(lines);

From 4a58e6815cc2f2c50a75b8ea74876dc1743645bd Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Fri, 27 May 2022 14:34:28 +0100
Subject: [PATCH 12/21] Make it explicit that shim frame is artificial and
 should not show up in tracebacks or sys._getframe()

---
 Include/internal/pycore_frame.h | 2 ++
 Python/ceval.c                  | 3 ++-
 Python/frame.c                  | 6 +++++-
 Python/sysmodule.c              | 4 +++-
 4 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
index 405afd67d2274a..b8a7e65dc323ec 100644
--- a/Include/internal/pycore_frame.h
+++ b/Include/internal/pycore_frame.h
@@ -61,6 +61,7 @@ typedef struct _PyInterpreterFrame {
     _Py_CODEUNIT *prev_instr;
     int stacktop;     /* Offset of TOS from localsplus  */
     bool is_entry;  // Whether this is the "root" frame for the current _PyCFrame.
+    bool is_artificial; // Hide this frame in backtraces.
     char owner;
     /* Locals and stack */
     PyObject *localsplus[1];
@@ -109,6 +110,7 @@ _PyFrame_InitializeSpecials(
     frame->frame_obj = NULL;
     frame->prev_instr = _PyCode_CODE(frame->f_code) - 1;
     frame->is_entry = false;
+    frame->is_artificial = false;
     frame->owner = FRAME_OWNED_BY_THREAD;
 }
 
diff --git a/Python/ceval.c b/Python/ceval.c
index 691ec39cddb9f9..65c1f7294f3ffa 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1574,7 +1574,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
     Py_INCREF(res);
 
 #define TRACE_FUNCTION_EXIT() \
-    if (cframe.use_tracing) { \
+    if (cframe.use_tracing && !frame->is_artificial) { \
         if (trace_function_exit(tstate, frame, retval)) { \
             Py_DECREF(retval); \
             goto exit_unwind; \
@@ -5010,6 +5010,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             if (shim == NULL) {
                 goto error;
             }
+            shim->is_artificial = true;
             CALL_STAT_INC(inlined_py_calls);
             shim->previous = frame;
             shim->prev_instr = _PyCode_CODE(shim->f_code) + 1;
diff --git a/Python/frame.c b/Python/frame.c
index 3573f54ad63e9b..4b46139cb3f346 100644
--- a/Python/frame.c
+++ b/Python/frame.c
@@ -70,7 +70,11 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame)
     assert(f->f_back == NULL);
     if (frame->previous != NULL) {
         /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */
-        PyFrameObject *back = _PyFrame_GetFrameObject(frame->previous);
+        _PyInterpreterFrame *prev = frame->previous;
+        while (prev->is_artificial) {
+            prev = prev->previous;
+        }
+        PyFrameObject *back = _PyFrame_GetFrameObject(prev);
         if (back == NULL) {
             /* Memory error here. */
             assert(PyErr_ExceptionMatches(PyExc_MemoryError));
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index b9cae15568d0f5..5b640495eca6c3 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1778,7 +1778,9 @@ sys__getframe_impl(PyObject *module, int depth)
 
     while (depth > 0 && frame != NULL) {
         frame = frame->previous;
-        --depth;
+        if (!frame->is_artificial) {
+            --depth;
+        }
     }
     if (frame == NULL) {
         _PyErr_SetString(tstate, PyExc_ValueError,

From 7c0876ca269de779309f6c09a4e4cb9dbcfff8ff Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Fri, 27 May 2022 14:57:37 +0100
Subject: [PATCH 13/21] Fixup sys._getframe

---
 Python/sysmodule.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 5b640495eca6c3..23e366774ff035 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1776,10 +1776,15 @@ sys__getframe_impl(PyObject *module, int depth)
         return NULL;
     }
 
-    while (depth > 0 && frame != NULL) {
-        frame = frame->previous;
-        if (!frame->is_artificial) {
-            --depth;
+    if (frame != NULL) {
+        while (depth > 0) {
+            frame = frame->previous;
+            if (frame == NULL) {
+                break;
+            }
+            if (!frame->is_artificial) {
+                --depth;
+            }
         }
     }
     if (frame == NULL) {

From 365fceb5005701239ea255c1f4ea8041b9a8fb03 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Tue, 7 Jun 2022 11:45:58 +0100
Subject: [PATCH 14/21] Add explanatory comment to
 CALL_NO_KW_ALLOC_AND_ENTER_INIT.

---
 Python/ceval.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Python/ceval.c b/Python/ceval.c
index 65c1f7294f3ffa..fd9a0f16a7596f 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4988,6 +4988,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
         }
 
         TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) {
+            /* This instruction does the following:
+             * 1. Creates the object (by calling ``object.__new__``)
+             * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
+             * 3. Pushes the frame for ``__init__`` to the frame stack
+             * */
             assert(call_shape.kwnames == NULL);
             _PyCallCache *cache = (_PyCallCache *)next_instr;
             DEOPT_IF(is_method(stack_pointer, oparg), CALL);

From 1ee2f1c8abdde1cb6707dfa8972c6daef84230fe Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Tue, 7 Jun 2022 11:55:55 +0100
Subject: [PATCH 15/21] Add test for tracing when quickening calls to Python
 classes.

---
 Lib/test/test_sys_settrace.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index 162fd8328582ca..a71471538488d0 100644
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -1570,6 +1570,28 @@ def func():
 
         self.run_and_compare(func, EXPECTED_EVENTS)
 
+    def test_correct_tracing_quickened_call_class_init(self):
+
+        class C:
+            def __init__(self):
+                self
+
+        def func():
+            C()
+
+        EXPECTED_EVENTS = [
+            (0, 'call'),
+            (1, 'line'),
+            (-3, 'call'),
+            (-2, 'line'),
+            (-2, 'return'),
+            (1, 'return')]
+
+        self.run_and_compare(func, EXPECTED_EVENTS)
+        # Quicken
+        for _ in range(100):
+            func()
+        self.run_and_compare(func, EXPECTED_EVENTS)
 
 EVENT_NAMES = [
     'call',

From b5cf97dbee3f4804ddbd725d87f6706fcdc04e82 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 29 Jun 2022 11:48:51 +0100
Subject: [PATCH 16/21] Remove is_artificial and use _co_firsttraceable
 instead.

---
 Include/internal/pycore_frame.h |  2 -
 Python/ceval.c                  |  3 +-
 Python/frame.c                  |  6 +--
 Python/specialize.c             |  7 +--
 Python/sysmodule.c              | 79 ++++++++++++++++++++++++++++-----
 5 files changed, 75 insertions(+), 22 deletions(-)

diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
index 35f1ef8d9cc934..eed26fbb06218a 100644
--- a/Include/internal/pycore_frame.h
+++ b/Include/internal/pycore_frame.h
@@ -62,7 +62,6 @@ typedef struct _PyInterpreterFrame {
     _Py_CODEUNIT *prev_instr;
     int stacktop;     /* Offset of TOS from localsplus  */
     bool is_entry;  // Whether this is the "root" frame for the current _PyCFrame.
-    bool is_artificial; // Hide this frame in backtraces.
     char owner;
     /* Locals and stack */
     PyObject *localsplus[1];
@@ -111,7 +110,6 @@ _PyFrame_InitializeSpecials(
     frame->frame_obj = NULL;
     frame->prev_instr = _PyCode_CODE(code) - 1;
     frame->is_entry = false;
-    frame->is_artificial = false;
     frame->owner = FRAME_OWNED_BY_THREAD;
 }
 
diff --git a/Python/ceval.c b/Python/ceval.c
index 7d6c62d10ea924..a66eda0e645f03 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1504,7 +1504,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
 /* Shared opcode macros */
 
 #define TRACE_FUNCTION_EXIT() \
-    if (cframe.use_tracing && !frame->is_artificial) { \
+    if (cframe.use_tracing && INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) { \
         if (trace_function_exit(tstate, frame, retval)) { \
             Py_DECREF(retval); \
             goto exit_unwind; \
@@ -5002,7 +5002,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             Py_DECREF(tp);
             Py_INCREF(_Py_InitCleanupFunc);
             _PyInterpreterFrame *shim = _PyFrame_PushUnchecked(tstate, _Py_InitCleanupFunc);
-            shim->is_artificial = true;
             CALL_STAT_INC(inlined_py_calls);
             shim->previous = frame;
             shim->prev_instr = _PyCode_CODE(shim->f_code) + 1;
diff --git a/Python/frame.c b/Python/frame.c
index f2880f3d8b2e50..206ecab44c1b87 100644
--- a/Python/frame.c
+++ b/Python/frame.c
@@ -70,11 +70,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame)
     assert(f->f_back == NULL);
     if (frame->previous != NULL) {
         /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */
-        _PyInterpreterFrame *prev = frame->previous;
-        while (prev->is_artificial) {
-            prev = prev->previous;
-        }
-        PyFrameObject *back = _PyFrame_GetFrameObject(prev);
+        PyFrameObject *back = _PyFrame_GetFrameObject(frame->previous);
         if (back == NULL) {
             /* Memory error here. */
             assert(PyErr_ExceptionMatches(PyExc_MemoryError));
diff --git a/Python/specialize.c b/Python/specialize.c
index ef4d62cc8c0ba5..73ad6f46966632 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -2023,11 +2023,12 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
     cache->counter = miss_counter_start();
 }
 
-char INIT_CLEANUP_CODE[8] = {
+char INIT_CLEANUP_CODE[10] = {
     LOAD_ASSERTION_ERROR, 0,
     RAISE_VARARGS, 1,
     EXIT_INIT_CHECK, 0,
-    RETURN_VALUE, 0
+    RETURN_VALUE, 0,
+    RESUME, 0
 };
 
 PyFunctionObject *_Py_InitCleanupFunc = NULL;
@@ -2047,7 +2048,7 @@ setup_init_cleanup_func(void) {
     if (name == NULL) {
         goto cleanup;
     }
-    code = PyBytes_FromStringAndSize(INIT_CLEANUP_CODE, 8);
+    code = PyBytes_FromStringAndSize(INIT_CLEANUP_CODE, sizeof(INIT_CLEANUP_CODE));
     if (code == NULL) {
         goto cleanup;
     }
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index ba2f13836b6566..444042f82d9473 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1776,16 +1776,9 @@ sys__getframe_impl(PyObject *module, int depth)
         return NULL;
     }
 
-    if (frame != NULL) {
-        while (depth > 0) {
-            frame = frame->previous;
-            if (frame == NULL) {
-                break;
-            }
-            if (!frame->is_artificial) {
-                --depth;
-            }
-        }
+    while (depth > 0 && frame != NULL) {
+        frame = frame->previous;
+        --depth;
     }
     if (frame == NULL) {
         _PyErr_SetString(tstate, PyExc_ValueError,
@@ -1916,6 +1909,66 @@ sys_is_finalizing_impl(PyObject *module)
     return PyBool_FromLong(_Py_IsFinalizing());
 }
 
+#ifdef Py_STATS
+/*[clinic input]
+sys._stats_on
+
+Turns on stats gathering (stats gathering is on by default).
+[clinic start generated code]*/
+
+static PyObject *
+sys__stats_on_impl(PyObject *module)
+/*[clinic end generated code: output=aca53eafcbb4d9fe input=8ddc6df94e484f3a]*/
+{
+    _py_stats = &_py_stats_struct;
+    Py_RETURN_NONE;
+}
+
+/*[clinic input]
+sys._stats_off
+
+Turns off stats gathering (stats gathering is on by default).
+[clinic start generated code]*/
+
+static PyObject *
+sys__stats_off_impl(PyObject *module)
+/*[clinic end generated code: output=1534c1ee63812214 input=b3e50e71ecf29f66]*/
+{
+    _py_stats = NULL;
+    Py_RETURN_NONE;
+}
+
+/*[clinic input]
+sys._stats_clear
+
+Clears the stats.
+[clinic start generated code]*/
+
+static PyObject *
+sys__stats_clear_impl(PyObject *module)
+/*[clinic end generated code: output=fb65a2525ee50604 input=3e03f2654f44da96]*/
+{
+    _Py_StatsClear();
+    Py_RETURN_NONE;
+}
+
+/*[clinic input]
+sys._stats_dump
+
+Dump stats to file, and clears the stats.
+[clinic start generated code]*/
+
+static PyObject *
+sys__stats_dump_impl(PyObject *module)
+/*[clinic end generated code: output=79f796fb2b4ddf05 input=92346f16d64f6f95]*/
+{
+    _Py_PrintSpecializationStats(1);
+    _Py_StatsClear();
+    Py_RETURN_NONE;
+}
+
+#endif
+
 #ifdef ANDROID_API_LEVEL
 /*[clinic input]
 sys.getandroidapilevel
@@ -1985,6 +2038,12 @@ static PyMethodDef sys_methods[] = {
     SYS_GET_ASYNCGEN_HOOKS_METHODDEF
     SYS_GETANDROIDAPILEVEL_METHODDEF
     SYS_UNRAISABLEHOOK_METHODDEF
+#ifdef Py_STATS
+    SYS__STATS_ON_METHODDEF
+    SYS__STATS_OFF_METHODDEF
+    SYS__STATS_CLEAR_METHODDEF
+    SYS__STATS_DUMP_METHODDEF
+#endif
     {NULL, NULL}  // sentinel
 };
 

From 8a5c7e62c127d4808e02f1bcc38807a2bc7d5cb4 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 29 Jun 2022 12:25:10 +0100
Subject: [PATCH 17/21] Explain unusual structure of _Py_InitCleanupFunc code.

---
 Python/specialize.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Python/specialize.c b/Python/specialize.c
index 4c926b724fff10..d7ca1c124a619e 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -2038,6 +2038,14 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
     cache->counter = miss_counter_start();
 }
 
+/* Code for init cleanup function.
+ * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up
+ * the frame to execute the EXIT_INIT_CHECK
+ * instruction.
+ * Starts with an assertion error, in case it is called
+ * directly.
+ * Ends with a RESUME so that it is not traced.
+ */
 char INIT_CLEANUP_CODE[10] = {
     LOAD_ASSERTION_ERROR, 0,
     RAISE_VARARGS, 1,

From cd413081c0b5daef5fb6420a7ed81b452094d7d1 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Tue, 19 Jul 2022 15:46:59 +0100
Subject: [PATCH 18/21] Do not include incomplete frames in tracebacks.

---
 Python/ceval.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Python/ceval.c b/Python/ceval.c
index ce42135687ced2..1919ba95c4d7c1 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5854,9 +5854,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #endif
 
         /* Log traceback info. */
-        PyFrameObject *f = _PyFrame_GetFrameObject(frame);
-        if (f != NULL) {
-            PyTraceBack_Here(f);
+        if (!_PyFrame_IsIncomplete(frame)) {
+            PyFrameObject *f = _PyFrame_GetFrameObject(frame);
+            if (f != NULL) {
+                PyTraceBack_Here(f);
+            }
         }
 
         if (tstate->c_tracefunc != NULL) {

From 9ffd10d930da2404a6131821c2c236a646bbb5af Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Mon, 25 Jul 2022 12:18:42 +0100
Subject: [PATCH 19/21] Make init-cleanup function per-interpreter, rather than
 global.

---
 Include/internal/pycore_ceval.h | 2 --
 Include/internal/pycore_code.h  | 2 ++
 Python/ceval.c                  | 7 ++++---
 Python/pylifecycle.c            | 1 +
 Python/pystate.c                | 1 +
 Python/specialize.c             | 9 ++++-----
 6 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 633dc29e59685b..1b999301938c59 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -133,8 +133,6 @@ extern struct _PyInterpreterFrame* _PyEval_GetFrame(void);
 
 extern PyObject* _Py_MakeCoro(PyFunctionObject *func);
 
-extern PyFunctionObject *_Py_InitCleanupFunc;
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 61e40eb8a94c24..f577dfb14826c1 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -118,6 +118,8 @@ struct callable_cache {
     PyObject *isinstance;
     PyObject *len;
     PyObject *list_append;
+    /* This is a strong reference */
+    PyFunctionObject *init_cleanup;
 };
 
 /* "Locals plus" for a code object is the set of locals + cell vars +
diff --git a/Python/ceval.c b/Python/ceval.c
index 1919ba95c4d7c1..b3516ea4ab84db 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -418,7 +418,6 @@ PyEval_InitThreads(void)
 void
 _PyEval_Fini(void)
 {
-    Py_CLEAR(_Py_InitCleanupFunc);
 #ifdef Py_STATS
     _Py_PrintSpecializationStats(1);
 #endif
@@ -5041,8 +5040,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             }
             PEEK(oparg+1) = self;
             Py_DECREF(tp);
-            Py_INCREF(_Py_InitCleanupFunc);
-            _PyInterpreterFrame *shim = _PyFrame_PushUnchecked(tstate, _Py_InitCleanupFunc);
+            PyInterpreterState *interp = _PyInterpreterState_GET();
+            PyFunctionObject *init_cleanup = interp->callable_cache.init_cleanup;
+            Py_INCREF(init_cleanup);
+            _PyInterpreterFrame *shim = _PyFrame_PushUnchecked(tstate, init_cleanup);
             CALL_STAT_INC(inlined_py_calls);
             shim->previous = frame;
             shim->prev_instr = _PyCode_CODE(shim->f_code) + 1;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 65e7f23e963b5f..c228ff9efc9560 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -787,6 +787,7 @@ pycore_init_builtins(PyThreadState *tstate)
     PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append));
     assert(list_append);
     interp->callable_cache.list_append = list_append;
+    interp->callable_cache.init_cleanup = NULL;
 
     if (_PyBuiltins_AddExceptions(bimod) < 0) {
         return _PyStatus_ERR("failed to add exceptions to builtins");
diff --git a/Python/pystate.c b/Python/pystate.c
index 11cc122185f781..ddec108c0efa63 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -435,6 +435,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
     PyDict_Clear(interp->builtins);
     Py_CLEAR(interp->sysdict);
     Py_CLEAR(interp->builtins);
+    Py_CLEAR(interp->callable_cache.init_cleanup);
 
     // XXX Once we have one allocator per interpreter (i.e.
     // per-interpreter GC) we must ensure that all of the interpreter's
diff --git a/Python/specialize.c b/Python/specialize.c
index fa86994fd057e4..84d7709d3ea611 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -2053,11 +2053,10 @@ char INIT_CLEANUP_CODE[10] = {
     RESUME, 0
 };
 
-PyFunctionObject *_Py_InitCleanupFunc = NULL;
-
 static int
 setup_init_cleanup_func(void) {
-    if (_Py_InitCleanupFunc != NULL) {
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    if (interp->callable_cache.init_cleanup != NULL) {
         return 0;
     }
     PyObject *name = NULL;
@@ -2122,7 +2121,7 @@ setup_init_cleanup_func(void) {
         .fc_kwdefaults = NULL,
         .fc_closure = NULL
     };
-    _Py_InitCleanupFunc = _PyFunction_FromConstructor(&desc);
+    interp->callable_cache.init_cleanup = _PyFunction_FromConstructor(&desc);
 cleanup:
     PyErr_Clear();
     Py_XDECREF(codeobj);
@@ -2130,7 +2129,7 @@ setup_init_cleanup_func(void) {
     Py_XDECREF(name);
     Py_XDECREF(code);
     Py_XDECREF(lines);
-    return _Py_InitCleanupFunc == NULL ? -1 : 0;
+    return interp->callable_cache.init_cleanup == NULL ? -1 : 0;
 }
 
 #ifdef Py_STATS

From 255e1c19a01f27d77d1629cc423c8848e69c5e03 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Mon, 25 Jul 2022 14:16:59 +0100
Subject: [PATCH 20/21] Fix whitespace

---
 Lib/test/test_sys_settrace.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index f8e66c51340569..67bdd585927dc8 100644
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -1593,6 +1593,7 @@ def func():
         for _ in range(100):
             func()
         self.run_and_compare(func, EXPECTED_EVENTS)
+
     def test_very_large_function(self):
         # There is a separate code path when the number of lines > (1 << 15).
         d = {}

From ff3ffefa58806eab6d9487f7deefd0e421ce5007 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Thu, 25 Aug 2022 12:21:36 +0100
Subject: [PATCH 21/21] Use code object instead of function as trampoline.

---
 Include/internal/pycore_code.h  |  2 +-
 Include/internal/pycore_frame.h | 35 +++++++++++++++++++++++++++++++--
 Objects/frameobject.c           |  2 +-
 Python/ceval.c                  | 25 +++++++++++------------
 Python/specialize.c             | 31 ++++++++---------------------
 5 files changed, 55 insertions(+), 40 deletions(-)

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index b94fb730c686ad..46b7e2c3864f49 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -120,7 +120,7 @@ struct callable_cache {
     PyObject *list_append;
     PyObject *object__getattribute__;
     /* This is a strong reference */
-    PyFunctionObject *init_cleanup;
+    PyCodeObject *init_cleanup;
 };
 
 /* "Locals plus" for a code object is the set of locals + cell vars +
diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
index decaafd141e9e1..0d523cb5bf3349 100644
--- a/Include/internal/pycore_frame.h
+++ b/Include/internal/pycore_frame.h
@@ -95,9 +95,27 @@ static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) {
 
 void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest);
 
+static inline void
+_PyFrame_InitializeSpecialsTrampoline(
+    _PyInterpreterFrame *frame, PyCodeObject *code, int stackdepth, int start)
+{
+    frame->f_funcobj = Py_NewRef(Py_None);
+    frame->f_code = (PyCodeObject *)Py_NewRef(code);
+#ifdef Py_DEBUG
+    frame->f_builtins = NULL;
+    frame->f_globals = NULL;
+#endif
+    frame->f_locals = NULL;
+    frame->stacktop = code->co_nlocalsplus + stackdepth;
+    frame->frame_obj = NULL;
+    frame->prev_instr = _PyCode_CODE(code) + start;
+    frame->is_entry = false;
+    frame->owner = FRAME_OWNED_BY_THREAD;
+}
+
 /* Consumes reference to func and locals */
 static inline void
-_PyFrame_InitializeSpecials(
+_PyFrame_InitializeSpecialsFromFunction(
     _PyInterpreterFrame *frame, PyFunctionObject *func,
     PyObject *locals, PyCodeObject *code)
 {
@@ -213,7 +231,20 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func)
     _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top;
     tstate->datastack_top += code->co_framesize;
     assert(tstate->datastack_top < tstate->datastack_limit);
-    _PyFrame_InitializeSpecials(new_frame, func, NULL, code);
+    _PyFrame_InitializeSpecialsFromFunction(new_frame, func, NULL, code);
+    return new_frame;
+}
+
+/* Pushes a trampoline frame without checking for space.
+ * Must be guarded by _PyThreadState_HasStackSpace() */
+static inline _PyInterpreterFrame *
+_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int start)
+{
+    CALL_STAT_INC(frames_pushed);
+    _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top;
+    tstate->datastack_top += code->co_framesize;
+    assert(tstate->datastack_top < tstate->datastack_limit);
+    _PyFrame_InitializeSpecialsTrampoline(new_frame, code, stackdepth, start);
     return new_frame;
 }
 
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index d2647bd122888c..a0cb0bb77975db 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -1052,7 +1052,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals)
     Py_INCREF(func);
     Py_XINCREF(locals);
     PyCodeObject *code = (PyCodeObject *)func->func_code;
-    _PyFrame_InitializeSpecials(frame, func, locals, code);
+    _PyFrame_InitializeSpecialsFromFunction(frame, func, locals, code);
     for (Py_ssize_t i = 0; i < code->co_nlocalsplus; i++) {
         frame->localsplus[i] = NULL;
     }
diff --git a/Python/ceval.c b/Python/ceval.c
index fbc5c03adb8857..d912e486976c56 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1088,11 +1088,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 
 #ifdef LLTRACE
     {
-        int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__));
-        if (r < 0) {
-            goto exit_unwind;
+        if (PyFunction_Check(frame->f_funcobj)) {
+            int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__));
+            if (r < 0) {
+                goto exit_unwind;
+            }
+            lltrace = r;
         }
-        lltrace = r;
     }
     if (lltrace) {
         lltrace_resume_frame(frame);
@@ -4473,22 +4475,19 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             }
             PEEK(oparg+1) = self;
             Py_DECREF(tp);
-            PyInterpreterState *interp = _PyInterpreterState_GET();
-            PyFunctionObject *init_cleanup = interp->callable_cache.init_cleanup;
-            Py_INCREF(init_cleanup);
-            _PyInterpreterFrame *shim = _PyFrame_PushUnchecked(tstate, init_cleanup);
             CALL_STAT_INC(inlined_py_calls);
-            shim->previous = frame;
-            shim->prev_instr = _PyCode_CODE(shim->f_code) + 1;
-            assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK);
             if (_Py_EnterRecursiveCallTstate(tstate, "")) {
                 tstate->recursion_remaining--;
                 goto exit_unwind;
             }
+            PyInterpreterState *interp = _PyInterpreterState_GET();
+            _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
+                tstate, interp->callable_cache.init_cleanup, 1, 1);
+            assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK);
             /* Push self onto stack of shim */
             Py_INCREF(self);
-            shim->stacktop = 1;
             shim->localsplus[0] = self;
+            shim->previous = frame;
             Py_INCREF(init);
             _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init);
             /* Copy self followed by args to __init__ frame */
@@ -5907,7 +5906,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
     if (frame == NULL) {
         goto fail;
     }
-    _PyFrame_InitializeSpecials(frame, func, locals, code);
+    _PyFrame_InitializeSpecialsFromFunction(frame, func, locals, code);
     PyObject **localsarray = &frame->localsplus[0];
     for (int i = 0; i < code->co_nlocalsplus; i++) {
         localsarray[i] = NULL;
diff --git a/Python/specialize.c b/Python/specialize.c
index bccc145dfe07fe..0e8631f192c849 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -1503,12 +1503,13 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp)
         SPECIALIZATION_FAIL(CALL, SPEC_FAIL_INIT_NOT_SIMPLE);
         return NULL;
     }
+    Py_CLEAR(((PyHeapTypeObject *)tp)->_spec_cache.init);
     ((PyHeapTypeObject *)tp)->_spec_cache.init = init;
     return (PyFunctionObject *)init;
 }
 
 static int
-setup_init_cleanup_func(void);
+setup_init_cleanup_code(void);
 
 static int
 specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr,
@@ -1516,7 +1517,7 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr,
 {
     assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
     _PyCallCache *cache = (_PyCallCache *)(instr + 1);
-    if (setup_init_cleanup_func()) {
+    if (setup_init_cleanup_code()) {
         return -1;
     }
     assert(PyType_Check(callable));
@@ -2121,13 +2122,15 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
     cache->counter = miss_counter_start();
 }
 
-/* Code for init cleanup function.
+/* Code init cleanup.
  * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up
  * the frame to execute the EXIT_INIT_CHECK
  * instruction.
  * Starts with an assertion error, in case it is called
  * directly.
  * Ends with a RESUME so that it is not traced.
+ * This is used as a plain code object, not a function,
+ * so must not access globals or builtins.
  */
 char INIT_CLEANUP_CODE[10] = {
     LOAD_ASSERTION_ERROR, 0,
@@ -2138,7 +2141,7 @@ char INIT_CLEANUP_CODE[10] = {
 };
 
 static int
-setup_init_cleanup_func(void) {
+setup_init_cleanup_code(void) {
     PyInterpreterState *interp = _PyInterpreterState_GET();
     if (interp->callable_cache.init_cleanup != NULL) {
         return 0;
@@ -2188,27 +2191,9 @@ setup_init_cleanup_func(void) {
     };
 
     codeobj = _PyCode_New(&con);
-    if (codeobj == NULL) {
-        goto cleanup;
-    }
-    globals = PyDict_New();
-    if (globals == NULL) {
-        goto cleanup;
-    }
-    PyFrameConstructor desc = {
-        .fc_globals = globals,
-        .fc_builtins = globals,
-        .fc_name = codeobj->co_name,
-        .fc_qualname = codeobj->co_name,
-        .fc_code = (PyObject *)codeobj,
-        .fc_defaults = NULL,
-        .fc_kwdefaults = NULL,
-        .fc_closure = NULL
-    };
-    interp->callable_cache.init_cleanup = _PyFunction_FromConstructor(&desc);
+    interp->callable_cache.init_cleanup = codeobj;
 cleanup:
     PyErr_Clear();
-    Py_XDECREF(codeobj);
     Py_XDECREF(globals);
     Py_XDECREF(name);
     Py_XDECREF(code);