diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index f0ad1e47cb0d86..41b1dd62590bf7 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -81,6 +81,9 @@ typedef struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + unsigned int _tp_cache_used; + PyObject *_tp_cache[_TP_CACHE_SIZE]; + /* bitset of which type-watchers care about this type */ char tp_watched; } PyTypeObject; diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 859ffb91e223dc..9870e7d8436b1b 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -141,6 +141,8 @@ typedef struct { * backwards-compatibility */ typedef Py_ssize_t printfunc; +#define _TP_CACHE_SIZE (1 << 4) + // If this structure is modified, Doc/includes/typestruct.h should be updated // as well. struct _typeobject { @@ -226,6 +228,9 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + unsigned int _tp_cache_used; + PyObject *_tp_cache[_TP_CACHE_SIZE]; + /* bitset of which type-watchers care about this type */ char tp_watched; }; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 3359dfd8a499e0..461402764294cb 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -63,7 +63,7 @@ typedef struct { uint16_t counter; uint16_t type_version[2]; uint16_t keys_version[2]; - uint16_t descr[4]; + uint16_t index; } _PyLoadMethodCache; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 4a0b27a13ae96c..b1d2d00951d025 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -46,7 +46,7 @@ const uint8_t _PyOpcode_Caches[256] = { [UNPACK_SEQUENCE] = 1, [FOR_ITER] = 1, [STORE_ATTR] = 4, - [LOAD_ATTR] = 9, + [LOAD_ATTR] = 6, [COMPARE_OP] = 1, [LOAD_GLOBAL] = 4, [BINARY_OP] = 1, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 28e55fdc8b7d10..a8a157a0ebabc2 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -435,7 +435,9 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a6 3519 (Modify SEND instruction) # Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) # Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) +# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) # Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) +# Python 3.12a7 3524 (Shrink the LOAD_ATTR caches) # Python 3.13 will start with 3550 @@ -452,7 +454,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3523).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3524).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 60670f571fdc4d..1c086185edf67e 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -402,7 +402,7 @@ def pseudo_op(name, op, real_ops): "counter": 1, "version": 2, "keys_version": 2, - "descr": 4, + "index": 1, }, "STORE_ATTR": { "counter": 1, diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ed66b362b08080..c58bf5e6639544 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -354,7 +354,7 @@ def bug42562(): %3d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to 80) + POP_JUMP_IF_FALSE 20 (to 74) STORE_FAST 0 (e) %3d LOAD_FAST 0 (e) @@ -1128,7 +1128,7 @@ def test_load_attr_specialize(self): 1 2 LOAD_CONST 0 ('a') 4 LOAD_ATTR_SLOT 0 (__class__) - 24 RETURN_VALUE + 18 RETURN_VALUE """ co = compile("'a'.__class__", "", "eval") self.code_quicken(lambda: exec(co, {}, {})) @@ -1193,8 +1193,8 @@ def test_show_caches(self): caches = list(self.get_cached_values(quickened, adaptive)) for cache in caches: self.assertRegex(cache, pattern) - total_caches = 22 - empty_caches = 7 + total_caches = 19 + empty_caches = 4 self.assertEqual(caches.count(""), empty_caches) self.assertEqual(len(caches), total_caches) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index fb578c5ae6e5d5..88611790268c75 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1545,7 +1545,7 @@ def delx(self): del self.__x check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn12PIPc' + fmt = 'P2nPI13Pl4Pn9Pn12PIPI16Pc' s = vsize('2P' + fmt) check(int, s) # class diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-23-22-31-15.gh-issue-93533.RyrB_g.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-23-22-31-15.gh-issue-93533.RyrB_g.rst new file mode 100644 index 00000000000000..2e07ae75752599 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-23-22-31-15.gh-issue-93533.RyrB_g.rst @@ -0,0 +1,3 @@ +Reduce the number of inline :opcode:`CACHE` entries for :opcode:`LOAD_ATTR` +instructions by adding a shared array of cached methods to the +:c:type:`PyTypeObject` struct. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 69e84743f13aac..9dc3ace419f88b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -510,6 +510,7 @@ PyType_Modified(PyTypeObject *type) type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; /* 0 is not a valid version tag */ + type->_tp_cache_used = 0; } static void @@ -563,6 +564,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; /* 0 is not a valid version tag */ + type->_tp_cache_used = 0; } static int @@ -586,6 +588,7 @@ assign_version_tag(PyTypeObject *type) } type->tp_version_tag = next_version_tag++; assert (type->tp_version_tag != 0); + assert(type->_tp_cache_used == 0); PyObject *bases = type->tp_bases; Py_ssize_t n = PyTuple_GET_SIZE(bases); @@ -4493,6 +4496,7 @@ _PyStaticType_Dealloc(PyTypeObject *type) static_builtin_state_clear(type); /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } + type->_tp_cache_used = 0; } diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 8e5055bd7bceb1..205f4e605df1f6 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,39 +1,38 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,182,0,0,0,151,0,100,0,100,1, + 0,0,0,0,0,243,170,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, 100,2,171,1,0,0,0,0,0,0,0,0,1,0,2,0, 101,2,100,3,101,0,106,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,2,0,0,0,0, - 0,0,0,0,1,0,2,0,101,1,106,8,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,171,0, - 0,0,0,0,0,0,0,0,100,4,25,0,0,0,0,0, - 0,0,0,0,90,5,100,5,68,0,93,23,0,0,90,6, - 2,0,101,2,100,6,101,6,155,0,100,7,101,5,101,6, - 25,0,0,0,0,0,0,0,0,0,155,0,157,4,171,1, - 0,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, - 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, - 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, - 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, - 218,12,112,114,111,103,114,97,109,95,110,97,109,101,218,10, - 101,120,101,99,117,116,97,98,108,101,218,15,117,115,101,95, - 101,110,118,105,114,111,110,109,101,110,116,218,17,99,111,110, - 102,105,103,117,114,101,95,99,95,115,116,100,105,111,218,14, - 98,117,102,102,101,114,101,100,95,115,116,100,105,111,122,7, - 99,111,110,102,105,103,32,122,2,58,32,41,7,218,3,115, - 121,115,218,17,95,116,101,115,116,105,110,116,101,114,110,97, - 108,99,97,112,105,218,5,112,114,105,110,116,218,4,97,114, - 103,118,218,11,103,101,116,95,99,111,110,102,105,103,115,114, - 3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0, - 250,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, - 110,46,112,121,250,8,60,109,111,100,117,108,101,62,114,18, - 0,0,0,1,0,0,0,115,100,0,0,0,240,3,1,1, - 1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,213, - 0,27,217,0,5,128,106,144,35,151,40,145,40,213,0,27, - 216,9,38,208,9,26,215,9,38,209,9,38,212,9,40,168, - 24,212,9,50,128,6,240,2,6,12,2,242,0,7,1,42, - 128,67,241,14,0,5,10,208,10,40,144,67,209,10,40,152, - 54,160,35,156,59,209,10,40,214,4,41,241,15,7,1,42, - 114,16,0,0,0, + 0,0,0,0,171,2,0,0,0,0,0,0,0,0,1,0, + 2,0,101,1,106,8,0,0,0,0,0,0,0,0,0,0, + 0,0,171,0,0,0,0,0,0,0,0,0,100,4,25,0, + 0,0,0,0,0,0,0,0,90,5,100,5,68,0,93,23, + 0,0,90,6,2,0,101,2,100,6,101,6,155,0,100,7, + 101,5,101,6,25,0,0,0,0,0,0,0,0,0,155,0, + 157,4,171,1,0,0,0,0,0,0,0,0,1,0,140,25, + 4,0,121,1,41,8,233,0,0,0,0,78,122,18,70,114, + 111,122,101,110,32,72,101,108,108,111,32,87,111,114,108,100, + 122,8,115,121,115,46,97,114,103,118,218,6,99,111,110,102, + 105,103,41,5,218,12,112,114,111,103,114,97,109,95,110,97, + 109,101,218,10,101,120,101,99,117,116,97,98,108,101,218,15, + 117,115,101,95,101,110,118,105,114,111,110,109,101,110,116,218, + 17,99,111,110,102,105,103,117,114,101,95,99,95,115,116,100, + 105,111,218,14,98,117,102,102,101,114,101,100,95,115,116,100, + 105,111,122,7,99,111,110,102,105,103,32,122,2,58,32,41, + 7,218,3,115,121,115,218,17,95,116,101,115,116,105,110,116, + 101,114,110,97,108,99,97,112,105,218,5,112,114,105,110,116, + 218,4,97,114,103,118,218,11,103,101,116,95,99,111,110,102, + 105,103,115,114,3,0,0,0,218,3,107,101,121,169,0,243, + 0,0,0,0,250,18,116,101,115,116,95,102,114,111,122,101, + 110,109,97,105,110,46,112,121,250,8,60,109,111,100,117,108, + 101,62,114,18,0,0,0,1,0,0,0,115,95,0,0,0, + 240,3,1,1,1,243,8,0,1,11,219,0,24,225,0,5, + 208,6,26,213,0,27,217,0,5,128,106,144,35,150,40,213, + 0,27,216,9,38,208,9,26,214,9,38,212,9,40,168,24, + 212,9,50,128,6,240,2,6,12,2,242,0,7,1,42,128, + 67,241,14,0,5,10,208,10,40,144,67,209,10,40,152,54, + 160,35,156,59,209,10,40,214,4,41,241,15,7,1,42,114, + 16,0,0,0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2fe85dfeedf47f..c851d9d5244e5a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1413,7 +1413,7 @@ dummy_func( LOAD_ATTR_METHOD_LAZY_DICT, }; - inst(LOAD_ATTR, (unused/9, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR, (unused/6, owner -- res2 if (oparg & 1), res)) { #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1462,7 +1462,7 @@ dummy_func( } } - inst(LOAD_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, unused/2, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1479,7 +1479,7 @@ dummy_func( DECREF_INPUTS(); } - inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/2, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; @@ -1496,7 +1496,7 @@ dummy_func( DECREF_INPUTS(); } - inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/2, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1527,7 +1527,7 @@ dummy_func( DECREF_INPUTS(); } - inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/2, owner -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1541,7 +1541,7 @@ dummy_func( DECREF_INPUTS(); } - inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, index/1, cls -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); @@ -1551,19 +1551,20 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); res2 = NULL; - res = descr; + res = ((PyTypeObject *)cls)->_tp_cache[index]; assert(res != NULL); Py_INCREF(res); DECREF_INPUTS(); } - inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) { + inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, index/1, owner -- unused if (oparg & 1), unused)) { assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); + PyObject *fget = cls->_tp_cache[index]; assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; assert(func_version != 0); @@ -1583,12 +1584,13 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused if (oparg & 1), unused)) { + inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, index/1, owner -- unused if (oparg & 1), unused)) { assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); + PyObject *getattribute = cls->_tp_cache[index]; assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -2214,7 +2216,7 @@ dummy_func( exc_info->exc_value = Py_NewRef(new_exc); } - inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, index/1, self -- res2 if (oparg & 1), res)) { /* Cached method object */ assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); @@ -2227,6 +2229,7 @@ dummy_func( DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); + PyObject *descr = self_cls->_tp_cache[index]; assert(descr != NULL); res2 = Py_NewRef(descr); assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -2234,12 +2237,13 @@ dummy_func( assert(oparg & 1); } - inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, index/1, self -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); + PyObject *descr = self_cls->_tp_cache[index]; assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); res2 = Py_NewRef(descr); @@ -2247,7 +2251,7 @@ dummy_func( assert(oparg & 1); } - inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, index/1, self -- res2 if (oparg & 1), res)) { assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2257,6 +2261,7 @@ dummy_func( /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); + PyObject *descr = self_cls->_tp_cache[index]; assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); res2 = Py_NewRef(descr); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d793c1e23bc48e..6207a72ff1bfc5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2032,7 +2032,7 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 6, "incorrect cache size"); PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; @@ -2091,7 +2091,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -2120,7 +2120,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -2149,7 +2149,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -2192,7 +2192,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -2218,7 +2218,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -2227,7 +2227,7 @@ PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); + uint16_t index = read_u16(&next_instr[5].cache); #line 1545 "Python/bytecodes.c" assert(cframe.use_tracing == 0); @@ -2238,7 +2238,7 @@ STAT_INC(LOAD_ATTR, hit); res2 = NULL; - res = descr; + res = ((PyTypeObject *)cls)->_tp_cache[index]; assert(res != NULL); Py_INCREF(res); #line 2245 "Python/generated_cases.c.h" @@ -2246,7 +2246,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -2254,7 +2254,7 @@ PyObject *owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *fget = read_obj(&next_instr[5].cache); + uint16_t index = read_u16(&next_instr[5].cache); #line 1561 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); @@ -2262,6 +2262,7 @@ PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); + PyObject *fget = cls->_tp_cache[index]; assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; assert(func_version != 0); @@ -2279,20 +2280,21 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2283 "Python/generated_cases.c.h" + #line 2284 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { PyObject *owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1587 "Python/bytecodes.c" + uint16_t index = read_u16(&next_instr[5].cache); + #line 1588 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); + PyObject *getattribute = cls->_tp_cache[index]; assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -2313,7 +2315,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2317 "Python/generated_cases.c.h" + #line 2319 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2321,7 +2323,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1615 "Python/bytecodes.c" + #line 1617 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2340,7 +2342,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2344 "Python/generated_cases.c.h" + #line 2346 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2351,7 +2353,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1636 "Python/bytecodes.c" + #line 1638 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2391,7 +2393,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2395 "Python/generated_cases.c.h" + #line 2397 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2402,7 +2404,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1678 "Python/bytecodes.c" + #line 1680 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2413,7 +2415,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2417 "Python/generated_cases.c.h" + #line 2419 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2425,7 +2427,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1698 "Python/bytecodes.c" + #line 1700 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2439,12 +2441,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2443 "Python/generated_cases.c.h" + #line 2445 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1712 "Python/bytecodes.c" + #line 1714 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2448 "Python/generated_cases.c.h" + #line 2450 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2455,7 +2457,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1716 "Python/bytecodes.c" + #line 1718 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); @@ -2468,7 +2470,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2472 "Python/generated_cases.c.h" + #line 2474 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2479,7 +2481,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1732 "Python/bytecodes.c" + #line 1734 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); @@ -2496,7 +2498,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2500 "Python/generated_cases.c.h" + #line 2502 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2507,7 +2509,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1752 "Python/bytecodes.c" + #line 1754 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); @@ -2521,7 +2523,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2525 "Python/generated_cases.c.h" + #line 2527 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2532,14 +2534,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1768 "Python/bytecodes.c" + #line 1770 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2538 "Python/generated_cases.c.h" + #line 2540 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1770 "Python/bytecodes.c" + #line 1772 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2543 "Python/generated_cases.c.h" + #line 2545 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2549,15 +2551,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1774 "Python/bytecodes.c" + #line 1776 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2555 "Python/generated_cases.c.h" + #line 2557 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1776 "Python/bytecodes.c" + #line 1778 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2561 "Python/generated_cases.c.h" + #line 2563 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2568,12 +2570,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1781 "Python/bytecodes.c" + #line 1783 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2574 "Python/generated_cases.c.h" + #line 2576 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1783 "Python/bytecodes.c" + #line 1785 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2581,10 +2583,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2585 "Python/generated_cases.c.h" + #line 2587 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1791 "Python/bytecodes.c" + #line 1793 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2593,7 +2595,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2597 "Python/generated_cases.c.h" + #line 2599 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2603,21 +2605,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1802 "Python/bytecodes.c" + #line 1804 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2610 "Python/generated_cases.c.h" + #line 2612 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1805 "Python/bytecodes.c" + #line 1807 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2617 "Python/generated_cases.c.h" + #line 2619 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1810 "Python/bytecodes.c" + #line 1812 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2621 "Python/generated_cases.c.h" + #line 2623 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2626,15 +2628,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1814 "Python/bytecodes.c" + #line 1816 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2633 "Python/generated_cases.c.h" + #line 2635 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1817 "Python/bytecodes.c" + #line 1819 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2638 "Python/generated_cases.c.h" + #line 2640 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2643,29 +2645,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1821 "Python/bytecodes.c" + #line 1823 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2651 "Python/generated_cases.c.h" + #line 2653 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1827 "Python/bytecodes.c" + #line 1829 "Python/bytecodes.c" JUMPBY(oparg); - #line 2660 "Python/generated_cases.c.h" + #line 2662 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1831 "Python/bytecodes.c" + #line 1833 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2669 "Python/generated_cases.c.h" + #line 2671 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2673,7 +2675,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1837 "Python/bytecodes.c" + #line 1839 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2683,9 +2685,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2687 "Python/generated_cases.c.h" + #line 2689 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1847 "Python/bytecodes.c" + #line 1849 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2693,14 +2695,14 @@ if (err < 0) goto pop_1_error; } } - #line 2697 "Python/generated_cases.c.h" + #line 2699 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1857 "Python/bytecodes.c" + #line 1859 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2710,9 +2712,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2714 "Python/generated_cases.c.h" + #line 2716 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1867 "Python/bytecodes.c" + #line 1869 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2720,67 +2722,67 @@ if (err < 0) goto pop_1_error; } } - #line 2724 "Python/generated_cases.c.h" + #line 2726 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1877 "Python/bytecodes.c" + #line 1879 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2733 "Python/generated_cases.c.h" + #line 2735 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1879 "Python/bytecodes.c" + #line 1881 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2741 "Python/generated_cases.c.h" + #line 2743 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1887 "Python/bytecodes.c" + #line 1889 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2754 "Python/generated_cases.c.h" + #line 2756 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1893 "Python/bytecodes.c" + #line 1895 "Python/bytecodes.c" } - #line 2758 "Python/generated_cases.c.h" + #line 2760 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1897 "Python/bytecodes.c" + #line 1899 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2771 "Python/generated_cases.c.h" + #line 2773 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1906 "Python/bytecodes.c" + #line 1908 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2784 "Python/generated_cases.c.h" + #line 2786 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2791,16 +2793,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 1914 "Python/bytecodes.c" + #line 1916 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2800 "Python/generated_cases.c.h" + #line 2802 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 1919 "Python/bytecodes.c" + #line 1921 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2808,7 +2810,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2812 "Python/generated_cases.c.h" + #line 2814 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2817,10 +2819,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 1929 "Python/bytecodes.c" + #line 1931 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2824 "Python/generated_cases.c.h" + #line 2826 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2830,10 +2832,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 1935 "Python/bytecodes.c" + #line 1937 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2837 "Python/generated_cases.c.h" + #line 2839 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2844,11 +2846,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 1941 "Python/bytecodes.c" + #line 1943 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2852 "Python/generated_cases.c.h" + #line 2854 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2857,14 +2859,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 1947 "Python/bytecodes.c" + #line 1949 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2864 "Python/generated_cases.c.h" + #line 2866 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1950 "Python/bytecodes.c" + #line 1952 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2868 "Python/generated_cases.c.h" + #line 2870 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2872,7 +2874,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 1954 "Python/bytecodes.c" + #line 1956 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2895,11 +2897,11 @@ if (iter == NULL) { goto error; } - #line 2899 "Python/generated_cases.c.h" + #line 2901 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1977 "Python/bytecodes.c" + #line 1979 "Python/bytecodes.c" } - #line 2903 "Python/generated_cases.c.h" + #line 2905 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -2910,7 +2912,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 1996 "Python/bytecodes.c" + #line 1998 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2943,7 +2945,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 2947 "Python/generated_cases.c.h" + #line 2949 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2953,7 +2955,7 @@ TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2031 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; @@ -2974,7 +2976,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 2978 "Python/generated_cases.c.h" + #line 2980 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2984,7 +2986,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2054 "Python/bytecodes.c" + #line 2056 "Python/bytecodes.c" assert(cframe.use_tracing == 0); _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); @@ -3005,7 +3007,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3009 "Python/generated_cases.c.h" + #line 3011 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3015,7 +3017,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2077 "Python/bytecodes.c" + #line 2079 "Python/bytecodes.c" assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); @@ -3034,7 +3036,7 @@ if (next == NULL) { goto error; } - #line 3038 "Python/generated_cases.c.h" + #line 3040 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3043,7 +3045,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2098 "Python/bytecodes.c" + #line 2100 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3058,14 +3060,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); assert(next_instr->op.code == END_FOR); DISPATCH_INLINED(gen_frame); - #line 3062 "Python/generated_cases.c.h" + #line 3064 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2115 "Python/bytecodes.c" + #line 2117 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3088,16 +3090,16 @@ Py_DECREF(enter); goto error; } - #line 3092 "Python/generated_cases.c.h" + #line 3094 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2138 "Python/bytecodes.c" + #line 2140 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3101 "Python/generated_cases.c.h" + #line 3103 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3109,7 +3111,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2148 "Python/bytecodes.c" + #line 2150 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3135,16 +3137,16 @@ Py_DECREF(enter); goto error; } - #line 3139 "Python/generated_cases.c.h" + #line 3141 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2174 "Python/bytecodes.c" + #line 2176 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3148 "Python/generated_cases.c.h" + #line 3150 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3156,7 +3158,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2183 "Python/bytecodes.c" + #line 2185 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3177,7 +3179,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3181 "Python/generated_cases.c.h" + #line 3183 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3186,7 +3188,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2206 "Python/bytecodes.c" + #line 2208 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3196,7 +3198,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3200 "Python/generated_cases.c.h" + #line 3202 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3209,8 +3211,8 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - #line 2218 "Python/bytecodes.c" + uint16_t index = read_u16(&next_instr[5].cache); + #line 2220 "Python/bytecodes.c" /* Cached method object */ assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); @@ -3223,16 +3225,17 @@ DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); + PyObject *descr = self_cls->_tp_cache[index]; assert(descr != NULL); res2 = Py_NewRef(descr); assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3232 "Python/generated_cases.c.h" + #line 3235 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -3241,23 +3244,24 @@ PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - #line 2238 "Python/bytecodes.c" + uint16_t index = read_u16(&next_instr[5].cache); + #line 2241 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); + PyObject *descr = self_cls->_tp_cache[index]; assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3257 "Python/generated_cases.c.h" + #line 3261 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } @@ -3266,8 +3270,8 @@ PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); - PyObject *descr = read_obj(&next_instr[5].cache); - #line 2251 "Python/bytecodes.c" + uint16_t index = read_u16(&next_instr[5].cache); + #line 2255 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); @@ -3277,25 +3281,26 @@ /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); + PyObject *descr = self_cls->_tp_cache[index]; assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3286 "Python/generated_cases.c.h" + #line 3291 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } - next_instr += 9; + next_instr += 6; DISPATCH(); } TARGET(KW_NAMES) { - #line 2268 "Python/bytecodes.c" + #line 2273 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3299 "Python/generated_cases.c.h" + #line 3304 "Python/generated_cases.c.h" DISPATCH(); } @@ -3306,7 +3311,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2304 "Python/bytecodes.c" + #line 2309 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3378,7 +3383,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3382 "Python/generated_cases.c.h" + #line 3387 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3390,7 +3395,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2382 "Python/bytecodes.c" + #line 2387 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3400,7 +3405,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3404 "Python/generated_cases.c.h" + #line 3409 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3409,7 +3414,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2394 "Python/bytecodes.c" + #line 2399 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3434,7 +3439,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3438 "Python/generated_cases.c.h" + #line 3443 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3443,7 +3448,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2421 "Python/bytecodes.c" + #line 2426 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3473,7 +3478,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3477 "Python/generated_cases.c.h" + #line 3482 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3481,7 +3486,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2453 "Python/bytecodes.c" + #line 2458 "Python/bytecodes.c" assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -3492,7 +3497,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3496 "Python/generated_cases.c.h" + #line 3501 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3505,7 +3510,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2466 "Python/bytecodes.c" + #line 2471 "Python/bytecodes.c" assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -3517,7 +3522,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3521 "Python/generated_cases.c.h" + #line 3526 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3531,7 +3536,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2481 "Python/bytecodes.c" + #line 2486 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3542,7 +3547,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3546 "Python/generated_cases.c.h" + #line 3551 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3556,7 +3561,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2495 "Python/bytecodes.c" + #line 2500 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3578,7 +3583,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3582 "Python/generated_cases.c.h" + #line 3587 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3592,7 +3597,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2520 "Python/bytecodes.c" + #line 2525 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); @@ -3621,7 +3626,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3625 "Python/generated_cases.c.h" + #line 3630 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3635,7 +3640,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2552 "Python/bytecodes.c" + #line 2557 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); @@ -3668,7 +3673,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3672 "Python/generated_cases.c.h" + #line 3677 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3682,7 +3687,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2588 "Python/bytecodes.c" + #line 2593 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; @@ -3715,7 +3720,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3719 "Python/generated_cases.c.h" + #line 3724 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3729,7 +3734,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2624 "Python/bytecodes.c" + #line 2629 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ @@ -3755,7 +3760,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3759 "Python/generated_cases.c.h" + #line 3764 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3768,7 +3773,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2652 "Python/bytecodes.c" + #line 2657 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ @@ -3796,7 +3801,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3800 "Python/generated_cases.c.h" + #line 3805 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3808,7 +3813,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2683 "Python/bytecodes.c" + #line 2688 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); @@ -3827,14 +3832,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3831 "Python/generated_cases.c.h" + #line 3836 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2704 "Python/bytecodes.c" + #line 2709 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3865,7 +3870,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3869 "Python/generated_cases.c.h" + #line 3874 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3878,7 +3883,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2738 "Python/bytecodes.c" + #line 2743 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3907,7 +3912,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3911 "Python/generated_cases.c.h" + #line 3916 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3920,7 +3925,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2770 "Python/bytecodes.c" + #line 2775 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -3949,7 +3954,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3953 "Python/generated_cases.c.h" + #line 3958 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3962,7 +3967,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2802 "Python/bytecodes.c" + #line 2807 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3990,7 +3995,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3994 "Python/generated_cases.c.h" + #line 3999 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4005,7 +4010,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2833 "Python/bytecodes.c" + #line 2838 "Python/bytecodes.c" if (oparg & 1) { // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. @@ -4024,15 +4029,15 @@ assert(PyTuple_CheckExact(callargs)); result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - #line 4028 "Python/generated_cases.c.h" + #line 4033 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 2852 "Python/bytecodes.c" + #line 2857 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4036 "Python/generated_cases.c.h" + #line 4041 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4047,7 +4052,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 2863 "Python/bytecodes.c" + #line 2868 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4076,14 +4081,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4080 "Python/generated_cases.c.h" + #line 4085 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 2894 "Python/bytecodes.c" + #line 2899 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4104,7 +4109,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4108 "Python/generated_cases.c.h" + #line 4113 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4112,15 +4117,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 2917 "Python/bytecodes.c" + #line 2922 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4118 "Python/generated_cases.c.h" + #line 4123 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 2919 "Python/bytecodes.c" + #line 2924 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4124 "Python/generated_cases.c.h" + #line 4129 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4131,7 +4136,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 2923 "Python/bytecodes.c" + #line 2928 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4166,7 +4171,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4170 "Python/generated_cases.c.h" + #line 4175 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4175,10 +4180,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 2960 "Python/bytecodes.c" + #line 2965 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4182 "Python/generated_cases.c.h" + #line 4187 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4190,7 +4195,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 2965 "Python/bytecodes.c" + #line 2970 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4206,12 +4211,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4210 "Python/generated_cases.c.h" + #line 4215 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 2981 "Python/bytecodes.c" + #line 2986 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4215 "Python/generated_cases.c.h" + #line 4220 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4221,27 +4226,27 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 2986 "Python/bytecodes.c" + #line 2991 "Python/bytecodes.c" assert(oparg >= 2); - #line 4227 "Python/generated_cases.c.h" + #line 4232 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 2990 "Python/bytecodes.c" + #line 2995 "Python/bytecodes.c" assert(oparg); assert(cframe.use_tracing == 0); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4241 "Python/generated_cases.c.h" + #line 4246 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 2999 "Python/bytecodes.c" + #line 3004 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4247 "Python/generated_cases.c.h" + #line 4252 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 347a84dad46351..cfa014317e405d 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -695,7 +695,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBC00000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; struct opcode_metadata { bool valid_entry; enum InstructionFormat instr_format; @@ -796,14 +796,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [DICT_UPDATE] = { true, INSTR_FMT_IB }, [DICT_MERGE] = { true, INSTR_FMT_IB }, [MAP_ADD] = { true, INSTR_FMT_IB }, - [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000 }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000 }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000 }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000 }, @@ -840,9 +840,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [BEFORE_WITH] = { true, INSTR_FMT_IX }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000 }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000 }, [KW_NAMES] = { true, INSTR_FMT_IB }, [CALL] = { true, INSTR_FMT_IBC000 }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC000 }, diff --git a/Python/specialize.c b/Python/specialize.c index dd5b22dd2346c5..ae9f4aa1968954 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -457,6 +457,21 @@ static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); +static int +insert_into_cache(PyTypeObject *type, PyObject *descr) +{ + for (unsigned int i = 0; i < type->_tp_cache_used; i++) { + if (type->_tp_cache[i] == descr) { + return i; + } + } + if (type->_tp_cache_used < _TP_CACHE_SIZE) { + type->_tp_cache[type->_tp_cache_used] = descr; + return type->_tp_cache_used++; + } + return -1; +} + static int specialize_module_load_attr( PyObject *owner, _Py_CODEUNIT *instr, PyObject *name @@ -751,7 +766,13 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(type->tp_version_tag != 0); write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ - write_obj(lm_cache->descr, fget); + int index = insert_into_cache(type, fget); + if (index < 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } + assert(index <= UINT16_MAX); + lm_cache->index = index; instr->op.code = LOAD_ATTR_PROPERTY; goto success; } @@ -811,7 +832,13 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) } write_u32(lm_cache->keys_version, version); /* borrowed */ - write_obj(lm_cache->descr, descr); + int index = insert_into_cache(type, descr); + if (index < 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } + assert(index <= UINT16_MAX); + lm_cache->index = index; write_u32(lm_cache->type_version, type->tp_version_tag); instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; goto success; @@ -1003,7 +1030,13 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, case METHOD: case NON_DESCRIPTOR: write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); - write_obj(cache->descr, descr); + int index = insert_into_cache((PyTypeObject *)owner, descr); + if (index < 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return -1; + } + assert(index <= UINT16_MAX); + cache->index = index; instr->op.code = LOAD_ATTR_CLASS; return 0; #ifdef Py_STATS @@ -1084,7 +1117,13 @@ PyObject *descr, DescriptorClassification kind) * working since Python 2.6 and it's battle-tested. */ write_u32(cache->type_version, owner_cls->tp_version_tag); - write_obj(cache->descr, descr); + int index = insert_into_cache(owner_cls, descr); + if (index < 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + assert(index <= UINT16_MAX); + cache->index = index; return 1; }