From 1ffbb6b1f1dafc983783eb218fd43038340e90c9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 20 Apr 2025 00:17:43 +0800 Subject: [PATCH 1/6] Automatically constant evaluate pure operations --- Include/internal/pycore_opcode_metadata.h | 34 ++-- Include/internal/pycore_optimizer.h | 1 + Include/internal/pycore_uop_metadata.h | 82 ++++---- Python/bytecodes.c | 40 ++-- Python/optimizer_analysis.c | 6 + Python/optimizer_bytecodes.c | 81 ++------ Python/optimizer_cases.c.h | 199 +++++++++++-------- Python/optimizer_symbols.c | 9 + Tools/cases_generator/generators_common.py | 44 ++-- Tools/cases_generator/optimizer_generator.py | 168 +++++++++++++++- 10 files changed, 422 insertions(+), 242 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 521f7a92cf08c4..32635cefb5f79e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1123,7 +1123,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1135,7 +1135,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1204,9 +1204,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_CONST_IMMORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [LOAD_CONST_MORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1229,17 +1229,17 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [NOP] = { true, INSTR_FMT_IX, 0 }, + [NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, @@ -1267,7 +1267,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, @@ -1275,8 +1275,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1289,11 +1289,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, - [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, - [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [POP_BLOCK] = { true, -1, 0 }, + [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, }; #endif diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 4af1fa63ac1f1a..6fa2337a15f9f8 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -256,6 +256,7 @@ extern bool _Py_uop_sym_is_null(JitOptSymbol *sym); extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym); extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym); +extern JitOptSymbol *_Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val); extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); extern JitOptSymbol *_Py_uop_sym_new_type( diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 874756770c1871..6492855fde9414 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,29 +19,29 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = HAS_PURE_FLAG, + [_NOP] = 0, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_0] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_1] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_2] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_3] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_4] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_5] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_6] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW_7] = HAS_LOCAL_FLAG, + [_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, @@ -63,11 +63,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_PUSH_NULL] = HAS_PURE_FLAG, + [_POP_TOP] = HAS_ESCAPES_FLAG, + [_PUSH_NULL] = 0, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, - [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_END_SEND] = HAS_ESCAPES_FLAG, + [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, @@ -80,7 +80,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, [_TO_BOOL_STR] = HAS_ESCAPES_FLAG, [_REPLACE_WITH_TRUE] = HAS_ESCAPES_FLAG, - [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -88,13 +88,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -233,12 +233,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = 0, + [_INIT_CALL_PY_EXACT_ARGS_1] = 0, + [_INIT_CALL_PY_EXACT_ARGS_2] = 0, + [_INIT_CALL_PY_EXACT_ARGS_3] = 0, + [_INIT_CALL_PY_EXACT_ARGS_4] = 0, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG, [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -272,9 +272,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_COPY] = HAS_ARG_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_SWAP] = HAS_ARG_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, @@ -285,11 +285,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_LOAD_CONST_INLINE] = 0, + [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG, + [_LOAD_CONST_INLINE_BORROW] = 0, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_START_EXECUTOR] = HAS_ESCAPES_FLAG, [_MAKE_WARM] = 0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2796c3f2e85732..4bedb3a753d729 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -142,7 +142,7 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - pure inst(NOP, (--)) { + inst(NOP, (--)) { } family(RESUME, 0) = { @@ -265,12 +265,12 @@ dummy_func( value = PyStackRef_DUP(value_s); } - replicate(8) pure inst(LOAD_FAST, (-- value)) { + replicate(8) inst(LOAD_FAST, (-- value)) { assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); } - replicate(8) pure inst (LOAD_FAST_BORROW, (-- value)) { + replicate(8) inst (LOAD_FAST_BORROW, (-- value)) { assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); } @@ -388,11 +388,11 @@ dummy_func( PyStackRef_XCLOSE(tmp); } - pure inst(POP_TOP, (value --)) { + inst(POP_TOP, (value --)) { PyStackRef_CLOSE(value); } - pure inst(PUSH_NULL, (-- res)) { + inst(PUSH_NULL, (-- res)) { res = PyStackRef_NULL; } @@ -424,7 +424,7 @@ dummy_func( PyStackRef_CLOSE(iter); } - pure inst(END_SEND, (receiver, value -- val)) { + inst(END_SEND, (receiver, value -- val)) { val = value; DEAD(value); PyStackRef_CLOSE(receiver); @@ -443,7 +443,7 @@ dummy_func( PyStackRef_CLOSE(receiver); } - inst(UNARY_NEGATIVE, (value -- res)) { + pure inst(UNARY_NEGATIVE, (value -- res)) { PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); @@ -573,7 +573,7 @@ dummy_func( _GUARD_TYPE_VERSION + _REPLACE_WITH_TRUE; - inst(UNARY_INVERT, (value -- res)) { + pure inst(UNARY_INVERT, (value -- res)) { PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); @@ -669,7 +669,7 @@ dummy_func( EXIT_IF(!PyFloat_CheckExact(value_o)); } - pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -684,7 +684,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res), error); } - pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -699,7 +699,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res), error); } - pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -799,7 +799,7 @@ dummy_func( DEOPT_IF(!res); } - pure op(_BINARY_OP_EXTEND, (descr/4, left, right -- res)) { + op(_BINARY_OP_EXTEND, (descr/4, left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); @@ -3906,7 +3906,7 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); @@ -4916,7 +4916,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); top = PyStackRef_DUP(bottom); } @@ -4950,7 +4950,7 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + unused/4 + _BINARY_OP; - pure inst(SWAP, (bottom, unused[oparg-2], top -- + inst(SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { _PyStackRef temp = bottom; bottom = top; @@ -5187,25 +5187,25 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } - tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + tier2 op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectNew(ptr); } - tier2 pure op (_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { + tier2 op (_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { PyStackRef_CLOSE(pop); value = PyStackRef_FromPyObjectNew(ptr); } - tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + tier2 op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectImmortal(ptr); } - tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + tier2 op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { PyStackRef_CLOSE(pop); value = PyStackRef_FromPyObjectImmortal(ptr); } - tier2 pure op(_POP_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, pop1, pop2 -- value)) { + tier2 op(_POP_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, pop1, pop2 -- value)) { PyStackRef_CLOSE(pop2); PyStackRef_CLOSE(pop1); value = PyStackRef_FromPyObjectImmortal(ptr); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index ab28fae94a96a9..569436b05b1bf1 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -29,6 +29,7 @@ #include "pycore_uop_metadata.h" #include "pycore_uop_ids.h" #include "pycore_range.h" +#include "pycore_unicodeobject.h" #include #include @@ -321,6 +322,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const #define sym_get_const _Py_uop_sym_get_const +#define sym_new_const_steal _Py_uop_sym_new_const_steal #define sym_new_unknown _Py_uop_sym_new_unknown #define sym_new_not_null _Py_uop_sym_new_not_null #define sym_new_type _Py_uop_sym_new_type @@ -346,6 +348,8 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_is_immortal _Py_uop_sym_is_immortal #define sym_new_truthiness _Py_uop_sym_new_truthiness +#define JUMP_TO_LABEL(label) goto label; + static int optimize_to_bool( _PyUOpInstruction *this_instr, @@ -515,6 +519,8 @@ optimize_uops( } return trace_len; +pop_2_error: +pop_1_error: error: DPRINTF(3, "\n"); DPRINTF(1, "Encountered error in abstract interpreter\n"); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index c5d8b536bc6341..e2d58210f2ecc3 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -210,60 +210,27 @@ dummy_func(void) { } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + // We need to tell the cases generator that it's being used by the constant generator. + // We should fix this in the cases generator. + (void)(left); + (void)(right); + res = sym_new_type(ctx, &PyLong_Type); } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + // We need to tell the cases generator that it's being used by the constant generator. + // We should fix this in the cases generator. + (void)(left); + (void)(right); + res = sym_new_type(ctx, &PyLong_Type); } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + // We need to tell the cases generator that it's being used by the constant generator. + // We should fix this in the cases generator. + (void)(left); + (void)(right); + res = sym_new_type(ctx, &PyLong_Type); } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { @@ -327,19 +294,11 @@ dummy_func(void) { } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyUnicode_Type); - } + // We need to tell the cases generator that it's being used here. + // We should fix this in the cases generator. + (void)(left); + (void)(right); + res = sym_new_type(ctx, &PyUnicode_Type); } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 828f0943a8db86..a74f7df9aa56a2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -141,8 +141,21 @@ } case _UNARY_NEGATIVE: { + JitOptSymbol *value; JitOptSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if ( + sym_is_const(ctx, value) && + 1) { + PyObject *res_o = PyNumber_Negative( sym_get_const(ctx, value)); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + res = sym_new_const_steal(ctx, res_o); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-1] = res; break; } @@ -151,8 +164,15 @@ JitOptSymbol *value; JitOptSymbol *res; value = stack_pointer[-1]; - sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, false); + if ( + sym_is_const(ctx, value) && + 1) { + res =(sym_is_const(ctx, value) && Py_IsFalse(sym_get_const(ctx, value)))?sym_new_const(ctx, Py_True):sym_new_const(ctx, Py_False); + } + else { + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); + } stack_pointer[-1] = res; break; } @@ -280,8 +300,21 @@ } case _UNARY_INVERT: { + JitOptSymbol *value; JitOptSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if ( + sym_is_const(ctx, value) && + 1) { + PyObject *res_o = PyNumber_Invert( sym_get_const(ctx, value)); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_1_error); + } + res = sym_new_const_steal(ctx, res_o); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-1] = res; break; } @@ -312,25 +345,27 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; + if ( + sym_is_const(ctx, left) && + sym_is_const(ctx, right) && + 1) { + PyObject *left_o = sym_get_const(ctx, left); + PyObject *right_o = sym_get_const(ctx, right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + res = sym_new_const_steal(ctx, res_o); } else { + (void)(left); + (void)(right); res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -340,25 +375,27 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; + if ( + sym_is_const(ctx, left) && + sym_is_const(ctx, right) && + 1) { + PyObject *left_o = sym_get_const(ctx, left); + PyObject *right_o = sym_get_const(ctx, right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + res = sym_new_const_steal(ctx, res_o); } else { + (void)(left); + (void)(right); res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -368,25 +405,27 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; + if ( + sym_is_const(ctx, left) && + sym_is_const(ctx, right) && + 1) { + PyObject *left_o = sym_get_const(ctx, left); + PyObject *right_o = sym_get_const(ctx, right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + res = sym_new_const_steal(ctx, res_o); } else { + (void)(left); + (void)(right); res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -426,16 +465,14 @@ goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { res = sym_new_type(ctx, &PyFloat_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -455,16 +492,14 @@ goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { res = sym_new_type(ctx, &PyFloat_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -484,16 +519,14 @@ goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { res = sym_new_type(ctx, &PyFloat_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -503,24 +536,27 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; + if ( + sym_is_const(ctx, left) && + sym_is_const(ctx, right) && + 1) { + PyObject *left_o = sym_get_const(ctx, left); + PyObject *right_o = sym_get_const(ctx, right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + res = sym_new_const_steal(ctx, res_o); } else { + (void)(left); + (void)(right); res = sym_new_type(ctx, &PyUnicode_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1177,7 +1213,6 @@ PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; - stack_pointer[-1] = attr; uint64_t watched_mutations = get_mutations(dict); if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); @@ -1301,16 +1336,14 @@ assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); res = sym_new_const(ctx, tmp); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); Py_DECREF(tmp); } else { res = sym_new_type(ctx, &PyBool_Type); - stack_pointer += -1; } - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2079,13 +2112,13 @@ assert(framesize > 0); assert(framesize <= curr_space); curr_space -= framesize; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); co = get_code(this_instr); if (co == NULL) { ctx->done = true; } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e8a4f87031b76a..c0fe70a667830a 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -381,6 +381,15 @@ _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) return res; } +JitOptSymbol * +_Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val) +{ + assert(const_val != NULL); + JitOptSymbol *res = _Py_uop_sym_new_const(ctx, const_val); + Py_DECREF(const_val); + return res; +} + JitOptSymbol * _Py_uop_sym_new_null(JitOptContext *ctx) { diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9ba0767cba35a0..5e8f64f1e70943 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -75,6 +75,18 @@ def write_header( """ ) +def skip_to(tkn_iter: TokenIterator, end: str) -> Token: + tkn = None + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return tkn + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + assert tkn is not None + return tkn def emit_to(out: CWriter, tkn_iter: TokenIterator, end: str) -> Token: parens = 0 @@ -463,12 +475,13 @@ def _emit_stmt( uop: CodeSection, storage: Storage, inst: Instruction | None, + is_abstract: bool, ) -> tuple[bool, Token | None, Storage]: method_name = "emit_" + stmt.__class__.__name__ method = getattr(self, method_name, None) if method is None: raise NotImplementedError - return method(stmt, uop, storage, inst) # type: ignore[no-any-return] + return method(stmt, uop, storage, inst, is_abstract) # type: ignore[no-any-return] def emit_SimpleStmt( self, @@ -476,12 +489,13 @@ def emit_SimpleStmt( uop: CodeSection, storage: Storage, inst: Instruction | None, + is_abstract: bool, ) -> tuple[bool, Token | None, Storage]: local_stores = set(uop.local_stores) reachable = True tkn = stmt.contents[-1] try: - if stmt in uop.properties.escaping_calls: + if stmt in uop.properties.escaping_calls and not is_abstract: escape = uop.properties.escaping_calls[stmt] if escape.kills is not None: self.stackref_kill(escape.kills, storage, True) @@ -513,7 +527,7 @@ def emit_SimpleStmt( self.out.emit(tkn) else: self.out.emit(tkn) - if stmt in uop.properties.escaping_calls: + if stmt in uop.properties.escaping_calls and not is_abstract: self.emit_reload(storage) return reachable, None, storage except StackError as ex: @@ -526,6 +540,7 @@ def emit_MacroIfStmt( uop: CodeSection, storage: Storage, inst: Instruction | None, + is_abstract: bool ) -> tuple[bool, Token | None, Storage]: self.out.emit(stmt.condition) branch = stmt.else_ is not None @@ -533,7 +548,7 @@ def emit_MacroIfStmt( if branch: else_storage = storage.copy() for s in stmt.body: - r, tkn, storage = self._emit_stmt(s, uop, storage, inst) + r, tkn, storage = self._emit_stmt(s, uop, storage, inst, is_abstract) if tkn is not None: self.out.emit(tkn) if not r: @@ -543,7 +558,7 @@ def emit_MacroIfStmt( self.out.emit(stmt.else_) assert stmt.else_body is not None for s in stmt.else_body: - r, tkn, else_storage = self._emit_stmt(s, uop, else_storage, inst) + r, tkn, else_storage = self._emit_stmt(s, uop, else_storage, inst, is_abstract) if tkn is not None: self.out.emit(tkn) if not r: @@ -560,6 +575,7 @@ def emit_IfStmt( uop: CodeSection, storage: Storage, inst: Instruction | None, + is_abstract: bool ) -> tuple[bool, Token | None, Storage]: self.out.emit(stmt.if_) for tkn in stmt.condition: @@ -567,13 +583,13 @@ def emit_IfStmt( if_storage = storage.copy() rbrace: Token | None = stmt.if_ try: - reachable, rbrace, if_storage = self._emit_stmt(stmt.body, uop, if_storage, inst) + reachable, rbrace, if_storage = self._emit_stmt(stmt.body, uop, if_storage, inst, is_abstract) if stmt.else_ is not None: assert rbrace is not None self.out.emit(rbrace) self.out.emit(stmt.else_) if stmt.else_body is not None: - else_reachable, rbrace, else_storage = self._emit_stmt(stmt.else_body, uop, storage, inst) + else_reachable, rbrace, else_storage = self._emit_stmt(stmt.else_body, uop, storage, inst, is_abstract) if not reachable: reachable, storage = else_reachable, else_storage elif not else_reachable: @@ -601,6 +617,7 @@ def emit_BlockStmt( uop: CodeSection, storage: Storage, inst: Instruction | None, + is_abstract: bool, emit_braces: bool = True, ) -> tuple[bool, Token | None, Storage]: """ Returns (reachable?, closing '}', stack).""" @@ -610,7 +627,7 @@ def emit_BlockStmt( self.out.emit(stmt.open) reachable = True for s in stmt.body: - reachable, tkn, storage = self._emit_stmt(s, uop, storage, inst) + reachable, tkn, storage = self._emit_stmt(s, uop, storage, inst, is_abstract) if tkn is not None: self.out.emit(tkn) if not reachable: @@ -627,12 +644,13 @@ def emit_ForStmt( uop: CodeSection, storage: Storage, inst: Instruction | None, + is_abstract: bool, ) -> tuple[bool, Token | None, Storage]: """ Returns (reachable?, closing '}', stack).""" self.out.emit(stmt.for_) for tkn in stmt.header: self.out.emit(tkn) - return self._emit_stmt(stmt.body, uop, storage, inst) + return self._emit_stmt(stmt.body, uop, storage, inst, is_abstract) def emit_WhileStmt( self, @@ -640,12 +658,13 @@ def emit_WhileStmt( uop: CodeSection, storage: Storage, inst: Instruction | None, + is_abstract: bool, ) -> tuple[bool, Token | None, Storage]: """ Returns (reachable?, closing '}', stack).""" self.out.emit(stmt.while_) for tkn in stmt.condition: self.out.emit(tkn) - return self._emit_stmt(stmt.body, uop, storage, inst) + return self._emit_stmt(stmt.body, uop, storage, inst, is_abstract) def emit_tokens( @@ -653,10 +672,11 @@ def emit_tokens( code: CodeSection, storage: Storage, inst: Instruction | None, - emit_braces: bool = True + emit_braces: bool = True, + is_abstract: bool = False, ) -> tuple[bool, Storage]: self.out.start_line() - reachable, tkn, storage = self.emit_BlockStmt(code.body, code, storage, inst, emit_braces) + reachable, tkn, storage = self.emit_BlockStmt(code.body, code, storage, inst, is_abstract, emit_braces) assert tkn is not None try: if reachable: diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 7a32275347e896..4ef1def139715f 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -12,6 +12,8 @@ analyze_files, StackItem, analysis_error, + CodeSection, + Label, ) from generators_common import ( DEFAULT_INPUT, @@ -19,6 +21,8 @@ write_header, Emitter, TokenIterator, + emit_to, + skip_to, ) from cwriter import CWriter from typing import TextIO @@ -111,19 +115,151 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: self.out.emit(goto) self.out.emit(label) +class OptimizerConstantEmitter(OptimizerEmitter): + def __init__(self, out: CWriter, labels: dict[str, Label]): + super().__init__(out, labels) + overrides = { + "PyStackRef_AsPyObjectBorrow": self.emit_stackref_borrow, + "PyStackRef_CLOSE_SPECIALIZED": self.emit_nothing, + "PyStackRef_CLOSE": self.emit_nothing, + "PyStackRef_FromPyObjectSteal": self.emit_stackref_steal, + "PyStackRef_IsNull": self.emit_stackref_null, + "PyStackRef_IsFalse": self.emit_stackref_isfalse, + "PyStackRef_IsTrue": self.emit_stackref_istrue, + "PyStackRef_False": self.emit_stackref_false, + "PyStackRef_True": self.emit_stackref_true, + "assert": self.emit_nothing, + } + self._replacers = {**self._replacers, **overrides} + + def emit_stackref_borrow( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + next(tkn_iter) + self.out.emit(" sym_get_const(ctx, ") + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + return True + + def emit_stackref_steal( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + next(tkn_iter) + self.out.emit(" sym_new_const_steal(ctx, ") + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + return True + + def emit_nothing( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + while (tkn := next(tkn_iter)).kind != "SEMI": + pass + return True + + def emit_stackref_null( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + next(tkn_iter) + self.out.emit(" sym_is_null(") + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + return True + + def emit_stackref_isfalse( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + next(tkn_iter) + name = next(tkn_iter) + assert name.kind == "IDENTIFIER", \ + "PyStackRef_IsFalse(target), target must be a simple identifier" + self.out.emit(f"(sym_is_const(ctx, {name.text}) && " + f"Py_IsFalse(sym_get_const(ctx, {name.text})))") + next(tkn_iter) + return True + + def emit_stackref_istrue( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + next(tkn_iter) + name = next(tkn_iter) + assert name.kind == "IDENTIFIER", \ + "PyStackRef_IsTrue(target), target must be a simple identifier" + self.out.emit(f"(sym_is_const(ctx, {name.text}_o) && " + f"Py_IsTrue(sym_get_const(ctx, {name.text}_o)))") + next(tkn_iter) + return True + + def emit_stackref_false( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + name = tkn + assert name.kind == "IDENTIFIER", \ + "PyStackRef_False must be a simple identifier" + self.out.emit(f"sym_new_const(ctx, Py_False)") + return True + + def emit_stackref_true( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ): + name = tkn + assert name.kind == "IDENTIFIER", \ + "PyStackRef_True must be a simple identifier" + self.out.emit(f"sym_new_const(ctx, Py_True)") + return True + def write_uop( override: Uop | None, uop: Uop, out: CWriter, stack: Stack, debug: bool, - skip_inputs: bool, ) -> None: locals: dict[str, Local] = {} prototype = override if override else uop try: out.start_line() - if override: + if override or uop.properties.pure: storage = Storage.for_uop(stack, prototype, out, check_liveness=False) if debug: args = [] @@ -140,13 +276,29 @@ def write_uop( type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" out.emit(f"{type}{cache.name} = ({cast})this_instr->operand0;\n") - if override: - emitter = OptimizerEmitter(out, {}) + if override or uop.properties.pure: # No reference management of inputs needed. for var in storage.inputs: # type: ignore[possibly-undefined] var.in_local = False - _, storage = emitter.emit_tokens(override, storage, None, False) + if uop.properties.pure: + emitter = OptimizerConstantEmitter(out, {}) + emitter.emit("if (\n") + for inp in uop.stack.inputs: + emitter.emit(f"sym_is_const(ctx, {inp.name}) &&\n") + emitter.emit("1) {\n") + _, storage = emitter.emit_tokens(uop, storage, None, False, is_abstract=True) + out.start_line() + emitter.emit("}\n") + emitter.emit("else {\n") + out.start_line() + if override: + emitter = OptimizerEmitter(out, {}) + _, storage = emitter.emit_tokens(override, storage, None, False, is_abstract=True) + else: + emit_default(out, uop, stack) out.start_line() + if uop.properties.pure: + emitter.emit("}\n") storage.flush(out) else: emit_default(out, uop, stack) @@ -193,9 +345,9 @@ def generate_abstract_interpreter( if override: declare_variables(override, out, skip_inputs=False) else: - declare_variables(uop, out, skip_inputs=True) + declare_variables(uop, out, skip_inputs=not uop.properties.pure) stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") - write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) + write_uop(override, uop, out, stack, debug) out.start_line() out.emit("break;\n") out.emit("}") @@ -207,7 +359,7 @@ def generate_tier2_abstract_from_files( ) -> None: assert len(filenames) == 2, "Need a base file and an abstract cases file." base = analyze_files([filenames[0]]) - abstract = analyze_files([filenames[1]]) + abstract = analyze_files([filenames[1]], abstract=True) with open(outfilename, "w") as outfile: generate_abstract_interpreter(filenames, abstract, base, outfile, debug) From 691084db029fbcafd51d030446e402af45805c13 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 19 Apr 2025 16:22:52 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-04-19-16-22-47.gh-issue-132732.jgqhlF.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-16-22-47.gh-issue-132732.jgqhlF.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-16-22-47.gh-issue-132732.jgqhlF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-16-22-47.gh-issue-132732.jgqhlF.rst new file mode 100644 index 00000000000000..aadaf2169fd01a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-16-22-47.gh-issue-132732.jgqhlF.rst @@ -0,0 +1 @@ +Automatically constant evaluate bytecode operations marked as pure in the JIT optimizer. From b89e4dc0478fce5a9a36349458c265e681e2fdf8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 20 Apr 2025 00:41:44 +0800 Subject: [PATCH 3/6] Fix tests --- Lib/test/test_generated_cases.py | 12 ++++++------ Tools/cases_generator/optimizer_generator.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 5b120f28131d51..9c29f859caa78b 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1972,12 +1972,12 @@ def run_cases_test(self, input: str, input2: str, expected: str): def test_overridden_abstract(self): input = """ - pure op(OP, (--)) { + op(OP, (--)) { SPAM(); } """ input2 = """ - pure op(OP, (--)) { + op(OP, (--)) { eggs(); } """ @@ -1991,7 +1991,7 @@ def test_overridden_abstract(self): def test_overridden_abstract_args(self): input = """ - pure op(OP, (arg1 -- out)) { + op(OP, (arg1 -- out)) { out = SPAM(arg1); } op(OP2, (arg1 -- out)) { @@ -2024,16 +2024,16 @@ def test_overridden_abstract_args(self): def test_no_overridden_case(self): input = """ - pure op(OP, (arg1 -- out)) { + op(OP, (arg1 -- out)) { out = SPAM(arg1); } - pure op(OP2, (arg1 -- out)) { + op(OP2, (arg1 -- out)) { } """ input2 = """ - pure op(OP2, (arg1 -- out)) { + op(OP2, (arg1 -- out)) { out = NULL; } """ diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 4ef1def139715f..55d10f12366372 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -359,7 +359,7 @@ def generate_tier2_abstract_from_files( ) -> None: assert len(filenames) == 2, "Need a base file and an abstract cases file." base = analyze_files([filenames[0]]) - abstract = analyze_files([filenames[1]], abstract=True) + abstract = analyze_files([filenames[1]]) with open(outfilename, "w") as outfile: generate_abstract_interpreter(filenames, abstract, base, outfile, debug) From d5b2208a531424b22d53b79230eebe77159df506 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 25 Apr 2025 09:05:11 +0800 Subject: [PATCH 4/6] Apply review suggestions --- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_optimizer.h | 2 + Include/internal/pycore_stackref.h | 26 ++ Include/internal/pycore_uop_metadata.h | 10 +- Python/bytecodes.c | 10 +- Python/optimizer_analysis.c | 2 + Python/optimizer_bytecodes.c | 66 ++--- Python/optimizer_cases.c.h | 267 ++++++++++++------- Python/optimizer_symbols.c | 12 + Tools/cases_generator/generators_common.py | 28 +- Tools/cases_generator/optimizer_generator.py | 199 +++++--------- Tools/cases_generator/stack.py | 1 - 12 files changed, 336 insertions(+), 291 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7d45f16966a019..88fcf5d801fcf5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1275,8 +1275,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 6fa2337a15f9f8..864213b2ac9379 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_typedefs.h" // _PyInterpreterFrame #include "pycore_uop_ids.h" +#include "pycore_stackref.h" // _PyStackRef #include @@ -257,6 +258,7 @@ extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym); extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val); +extern _PyStackRef _Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); extern JitOptSymbol *_Py_uop_sym_new_type( diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 5683b98470d3ea..99581f6e14fc2c 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -132,6 +132,15 @@ _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenu } #define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__) +static inline _PyStackRef +_PyStackRef_FromPyObjectImmortalUnchecked(PyObject *obj, const char *filename, int linenumber) +{ + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectImmortalUnchecked(obj) _PyStackRef_FromPyObjectImmortalUnchecked(_PyObject_CAST(obj), __FILE__, __LINE__) + + + static inline void _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) { @@ -324,6 +333,17 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj) } #define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) +static inline _PyStackRef +PyStackRef_FromPyObjectImmortalUnchecked(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; +} +#define PyStackRef_FromPyObjectImmortalUnchecked(obj) PyStackRef_FromPyObjectImmortalUnchecked(_PyObject_CAST(obj)) + + #define PyStackRef_CLOSE(REF) \ do { \ _PyStackRef _close_tmp = (REF); \ @@ -535,6 +555,12 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj) return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } +static inline _PyStackRef +PyStackRef_FromPyObjectImmortalUnchecked(PyObject *obj) +{ + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; +} + /* WARNING: This macro evaluates its argument more than once */ #ifdef _WIN32 #define PyStackRef_DUP(REF) \ diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 9bc83ac4caf510..8b36103cc4888d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -67,7 +67,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_PUSH_NULL] = 0, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG, - [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, @@ -80,7 +80,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, [_TO_BOOL_STR] = HAS_ESCAPES_FLAG, [_REPLACE_WITH_TRUE] = HAS_ESCAPES_FLAG, - [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -88,9 +88,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3ff1835ba2e3fe..6c025d2f9d39ac 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -443,7 +443,7 @@ dummy_func( PyStackRef_CLOSE(receiver); } - pure inst(UNARY_NEGATIVE, (value -- res)) { + inst(UNARY_NEGATIVE, (value -- res)) { PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); @@ -573,7 +573,7 @@ dummy_func( _GUARD_TYPE_VERSION + _REPLACE_WITH_TRUE; - pure inst(UNARY_INVERT, (value -- res)) { + inst(UNARY_INVERT, (value -- res)) { PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); @@ -669,7 +669,7 @@ dummy_func( EXIT_IF(!PyFloat_CheckExact(value_o)); } - op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -684,7 +684,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res), error); } - op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -699,7 +699,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res), error); } - op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index cf56dd806c6f79..43e99d125009da 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -30,6 +30,7 @@ #include "pycore_uop_ids.h" #include "pycore_range.h" #include "pycore_unicodeobject.h" +#include "pycore_ceval.h" #include #include @@ -323,6 +324,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_is_const _Py_uop_sym_is_const #define sym_get_const _Py_uop_sym_get_const #define sym_new_const_steal _Py_uop_sym_new_const_steal +#define sym_get_const_as_stackref _Py_uop_sym_get_const_as_stackref #define sym_new_unknown _Py_uop_sym_new_unknown #define sym_new_not_null _Py_uop_sym_new_not_null #define sym_new_type _Py_uop_sym_new_type diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index acad877e0d2466..3a2cdb35efc5a4 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -234,63 +234,27 @@ dummy_func(void) { } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); - } + // We need to tell the cases generator that it's being used by the constant generator. + // We should fix this in the cases generator. + (void)(left); + (void)(right); + res = sym_new_type(ctx, &PyFloat_Type); } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); - } + // We need to tell the cases generator that it's being used by the constant generator. + // We should fix this in the cases generator. + (void)(left); + (void)(right); + res = sym_new_type(ctx, &PyFloat_Type); } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); - } + // We need to tell the cases generator that it's being used by the constant generator. + // We should fix this in the cases generator. + (void)(left); + (void)(right); + res = sym_new_type(ctx, &PyFloat_Type); } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5f677106401f75..24bdbb476195bf 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -141,21 +141,8 @@ } case _UNARY_NEGATIVE: { - JitOptSymbol *value; JitOptSymbol *res; - value = stack_pointer[-1]; - if ( - sym_is_const(ctx, value) && - 1) { - PyObject *res_o = PyNumber_Negative( sym_get_const(ctx, value)); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); - } - res = sym_new_const_steal(ctx, res_o); - } - else { - res = sym_new_not_null(ctx); - } + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } @@ -167,13 +154,20 @@ if ( sym_is_const(ctx, value) && 1) { - res =(sym_is_const(ctx, value) && Py_IsFalse(sym_get_const(ctx, value)))?sym_new_const(ctx, Py_True):sym_new_const(ctx, Py_False); + JitOptSymbol *value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + assert(PyStackRef_BoolCheck(value)); + res_stackref= PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-1] = res; } else { sym_set_type(value, &PyBool_Type); res = sym_new_truthiness(ctx, value, false); + stack_pointer[-1] = res; } - stack_pointer[-1] = res; break; } @@ -301,21 +295,8 @@ } case _UNARY_INVERT: { - JitOptSymbol *value; JitOptSymbol *res; - value = stack_pointer[-1]; - if ( - sym_is_const(ctx, value) && - 1) { - PyObject *res_o = PyNumber_Invert( sym_get_const(ctx, value)); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); - } - res = sym_new_const_steal(ctx, res_o); - } - else { - res = sym_new_not_null(ctx); - } + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } @@ -350,23 +331,36 @@ sym_is_const(ctx, left) && sym_is_const(ctx, right) && 1) { - PyObject *left_o = sym_get_const(ctx, left); - PyObject *right_o = sym_get_const(ctx, right); + JitOptSymbol *left_sym = left; + JitOptSymbol *right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const_steal(ctx, res_o); + res_stackref= PyStackRef_FromPyObjectSteal(res_o); + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } else { (void)(left); (void)(right); res = sym_new_type(ctx, &PyLong_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -380,23 +374,36 @@ sym_is_const(ctx, left) && sym_is_const(ctx, right) && 1) { - PyObject *left_o = sym_get_const(ctx, left); - PyObject *right_o = sym_get_const(ctx, right); + JitOptSymbol *left_sym = left; + JitOptSymbol *right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const_steal(ctx, res_o); + res_stackref= PyStackRef_FromPyObjectSteal(res_o); + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } else { (void)(left); (void)(right); res = sym_new_type(ctx, &PyLong_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -410,23 +417,36 @@ sym_is_const(ctx, left) && sym_is_const(ctx, right) && 1) { - PyObject *left_o = sym_get_const(ctx, left); - PyObject *right_o = sym_get_const(ctx, right); + JitOptSymbol *left_sym = left; + JitOptSymbol *right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const_steal(ctx, res_o); + res_stackref= PyStackRef_FromPyObjectSteal(res_o); + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } else { (void)(left); (void)(right); res = sym_new_type(ctx, &PyLong_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -456,24 +476,40 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; + if ( + sym_is_const(ctx, left) && + sym_is_const(ctx, right) && + 1) { + JitOptSymbol *left_sym = left; + JitOptSymbol *right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res_stackref= _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res_stackref)) { + JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } else { + (void)(left); + (void)(right); res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -483,24 +519,40 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; + if ( + sym_is_const(ctx, left) && + sym_is_const(ctx, right) && + 1) { + JitOptSymbol *left_sym = left; + JitOptSymbol *right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res_stackref= _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res_stackref)) { + JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } else { + (void)(left); + (void)(right); res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -510,24 +562,40 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; + if ( + sym_is_const(ctx, left) && + sym_is_const(ctx, right) && + 1) { + JitOptSymbol *left_sym = left; + JitOptSymbol *right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res_stackref= _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res_stackref)) { + JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } else { + (void)(left); + (void)(right); res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -541,23 +609,36 @@ sym_is_const(ctx, left) && sym_is_const(ctx, right) && 1) { - PyObject *left_o = sym_get_const(ctx, left); - PyObject *right_o = sym_get_const(ctx, right); + JitOptSymbol *left_sym = left; + JitOptSymbol *right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { JUMP_TO_LABEL(pop_2_error); } - res = sym_new_const_steal(ctx, res_o); + res_stackref= PyStackRef_FromPyObjectSteal(res_o); + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } else { (void)(left); (void)(right); res = sym_new_type(ctx, &PyUnicode_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index c0fe70a667830a..ccebd00688c72a 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -147,6 +147,18 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) return NULL; } +_PyStackRef +_Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptSymbol *sym) +{ + PyObject *const_val = _Py_uop_sym_get_const(ctx, sym); + if (const_val == NULL) { + return PyStackRef_NULL; + } + // This is actually more like a borrow, but it doesn't matter here. + // Eventually we discard the stackref anyways. + return PyStackRef_FromPyObjectImmortalUnchecked(const_val); +} + void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) { diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 5e8f64f1e70943..f1c6613557d57e 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -142,6 +142,30 @@ def __init__(self, out: CWriter, labels: dict[str, Label]): self.out = out self.labels = labels + def emit_to_with_replacement( + self, + out: CWriter, + tkn_iter: TokenIterator, + end: str, + uop: CodeSection, + storage: Storage, + inst: Instruction | None + ) -> Token: + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return tkn + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + if tkn.text in self._replacers: + self._replacers[tkn.text](tkn, tkn_iter, uop, storage, inst) + else: + out.emit(tkn) + raise analysis_error(f"Expecting {end}. Reached end of file", tkn) + + def dispatch( self, tkn: Token, @@ -168,7 +192,7 @@ def deopt_if( lparen = next(tkn_iter) assert lparen.kind == "LPAREN" first_tkn = tkn_iter.peek() - emit_to(self.out, tkn_iter, "RPAREN") + self.emit_to_with_replacement(self.out, tkn_iter, "RPAREN", uop, storage, inst) self.emit(") {\n") next(tkn_iter) # Semi colon assert inst is not None @@ -210,7 +234,7 @@ def error_if( else: self.out.emit_at("if ", tkn) self.emit(lparen) - emit_to(self.out, tkn_iter, "COMMA") + self.emit_to_with_replacement(self.out, tkn_iter, "COMMA", uop, storage, inst) self.out.emit(") {\n") label = next(tkn_iter).text next(tkn_iter) # RPAREN diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 55d10f12366372..d7a41902e82177 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -4,6 +4,7 @@ """ import argparse +import copy from analyzer import ( Analysis, @@ -25,7 +26,7 @@ skip_to, ) from cwriter import CWriter -from typing import TextIO +from typing import TextIO, Callable from lexer import Token from stack import Local, Stack, StackError, Storage @@ -45,6 +46,12 @@ def type_name(var: StackItem) -> str: return var.type return f"JitOptSymbol *" +def stackref_type_name(var: StackItem) -> str: + if var.is_array(): + assert False, "Unsafe to convert a symbol to an array-like StackRef." + if var.type: + return var.type + return f"_PyStackRef " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: variables = {"unused"} @@ -115,148 +122,85 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: self.out.emit(goto) self.out.emit(label) + class OptimizerConstantEmitter(OptimizerEmitter): - def __init__(self, out: CWriter, labels: dict[str, Label]): + def __init__(self, out: CWriter, labels: dict[str, Label], uop: Uop): super().__init__(out, labels) + # Replace all outputs to point to their stackref versions. overrides = { - "PyStackRef_AsPyObjectBorrow": self.emit_stackref_borrow, - "PyStackRef_CLOSE_SPECIALIZED": self.emit_nothing, - "PyStackRef_CLOSE": self.emit_nothing, - "PyStackRef_FromPyObjectSteal": self.emit_stackref_steal, - "PyStackRef_IsNull": self.emit_stackref_null, - "PyStackRef_IsFalse": self.emit_stackref_isfalse, - "PyStackRef_IsTrue": self.emit_stackref_istrue, - "PyStackRef_False": self.emit_stackref_false, - "PyStackRef_True": self.emit_stackref_true, - "assert": self.emit_nothing, + outp.name: self.emit_stackref_override for outp in uop.stack.outputs } self._replacers = {**self._replacers, **overrides} - def emit_stackref_borrow( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ): - next(tkn_iter) - self.out.emit(" sym_get_const(ctx, ") - rparen = emit_to(self.out, tkn_iter, "RPAREN") - self.emit(rparen) - return True - - def emit_stackref_steal( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ): - next(tkn_iter) - self.out.emit(" sym_new_const_steal(ctx, ") - rparen = emit_to(self.out, tkn_iter, "RPAREN") - self.emit(rparen) - return True - - def emit_nothing( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ): - while (tkn := next(tkn_iter)).kind != "SEMI": - pass - return True - - def emit_stackref_null( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ): - next(tkn_iter) - self.out.emit(" sym_is_null(") - rparen = emit_to(self.out, tkn_iter, "RPAREN") - self.emit(rparen) - return True - - def emit_stackref_isfalse( + def emit_stackref_override( self, tkn: Token, tkn_iter: TokenIterator, uop: CodeSection, storage: Storage, inst: Instruction | None, - ): - next(tkn_iter) - name = next(tkn_iter) - assert name.kind == "IDENTIFIER", \ - "PyStackRef_IsFalse(target), target must be a simple identifier" - self.out.emit(f"(sym_is_const(ctx, {name.text}) && " - f"Py_IsFalse(sym_get_const(ctx, {name.text})))") - next(tkn_iter) + ) -> bool: + self.out.emit(tkn) + self.out.emit("_stackref") return True - def emit_stackref_istrue( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ): - next(tkn_iter) - name = next(tkn_iter) - assert name.kind == "IDENTIFIER", \ - "PyStackRef_IsTrue(target), target must be a simple identifier" - self.out.emit(f"(sym_is_const(ctx, {name.text}_o) && " - f"Py_IsTrue(sym_get_const(ctx, {name.text}_o)))") - next(tkn_iter) - return True - - def emit_stackref_false( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ): - name = tkn - assert name.kind == "IDENTIFIER", \ - "PyStackRef_False must be a simple identifier" - self.out.emit(f"sym_new_const(ctx, Py_False)") - return True +def write_uop_pure_evaluation_region_header( + uop: Uop, + out: CWriter, + stack: Stack, +) -> None: + emitter = OptimizerConstantEmitter(out, {}, uop) + emitter.emit("if (\n") + assert len(uop.stack.inputs) > 0, "Pure operations must have at least 1 input" + for inp in uop.stack.inputs: + emitter.emit(f"sym_is_const(ctx, {inp.name}) &&\n") + emitter.emit("1) {\n") + # Declare variables, before they are shadowed. + for inp in uop.stack.inputs: + if inp.used: + emitter.emit(f"{type_name(inp)}{inp.name}_sym = {inp.name};\n") + # Shadow the symbolic variables with stackrefs. + for inp in uop.stack.inputs: + if inp.used: + emitter.emit(f"{stackref_type_name(inp)}{inp.name} = sym_get_const_as_stackref(ctx, {inp.name}_sym);\n") + # Rename all output variables to stackref variant. + for outp in uop.stack.outputs: + assert not outp.is_array(), "Array output StackRefs not supported for pure ops." + emitter.emit(f"_PyStackRef {outp.name}_stackref;\n") + stack = copy.deepcopy(stack) + + storage = Storage.for_uop(stack, uop, CWriter.null(), check_liveness=False) + # No reference management of outputs needed. + for var in storage.outputs: + var.in_local = True + emitter.emit_tokens(uop, storage, None, False, is_abstract=True) + out.start_line() + # Finally, assign back the output stackrefs to symbolics. + for outp in uop.stack.outputs: + # All new stackrefs are created from new references. + # That's how the stackref contract works. + if not outp.peek: + out.emit(f"{outp.name} = sym_new_const_steal(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") + else: + out.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") + storage.flush(out) + emitter.emit("}\n") + emitter.emit("else {\n") - def emit_stackref_true( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ): - name = tkn - assert name.kind == "IDENTIFIER", \ - "PyStackRef_True must be a simple identifier" - self.out.emit(f"sym_new_const(ctx, Py_True)") - return True +def write_uop_pure_evaluation_region_footer( + out: CWriter, +) -> None: + out.emit("}\n") def write_uop( override: Uop | None, uop: Uop, out: CWriter, - stack: Stack, debug: bool, ) -> None: locals: dict[str, Local] = {} prototype = override if override else uop + stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") try: out.start_line() if override or uop.properties.pure: @@ -281,25 +225,17 @@ def write_uop( for var in storage.inputs: # type: ignore[possibly-undefined] var.in_local = False if uop.properties.pure: - emitter = OptimizerConstantEmitter(out, {}) - emitter.emit("if (\n") - for inp in uop.stack.inputs: - emitter.emit(f"sym_is_const(ctx, {inp.name}) &&\n") - emitter.emit("1) {\n") - _, storage = emitter.emit_tokens(uop, storage, None, False, is_abstract=True) - out.start_line() - emitter.emit("}\n") - emitter.emit("else {\n") + write_uop_pure_evaluation_region_header(uop, out, stack) out.start_line() if override: emitter = OptimizerEmitter(out, {}) _, storage = emitter.emit_tokens(override, storage, None, False, is_abstract=True) + storage.flush(out) else: emit_default(out, uop, stack) out.start_line() if uop.properties.pure: - emitter.emit("}\n") - storage.flush(out) + write_uop_pure_evaluation_region_footer(out) else: emit_default(out, uop, stack) out.start_line() @@ -346,8 +282,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=not uop.properties.pure) - stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") - write_uop(override, uop, out, stack, debug) + write_uop(override, uop, out, debug) out.start_line() out.emit("break;\n") out.emit("}") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 6b681775f48c81..f4b623d153fd9d 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -557,7 +557,6 @@ def for_uop(stack: Stack, uop: Uop, out: CWriter, check_liveness: bool = True) - stack.push(var) outputs = peeks + [ Local.undefined(var) for var in uop.stack.outputs if not var.peek ] return Storage(stack, inputs, outputs, len(peeks), check_liveness) - @staticmethod def copy_list(arg: list[Local]) -> list[Local]: return [ l.copy() for l in arg ] From 71ced861d851b42c1e747f51a0fb5fc16e470014 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 25 Apr 2025 09:09:06 +0800 Subject: [PATCH 5/6] reduce diff --- Tools/cases_generator/stack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index f4b623d153fd9d..6b681775f48c81 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -557,6 +557,7 @@ def for_uop(stack: Stack, uop: Uop, out: CWriter, check_liveness: bool = True) - stack.push(var) outputs = peeks + [ Local.undefined(var) for var in uop.stack.outputs if not var.peek ] return Storage(stack, inputs, outputs, len(peeks), check_liveness) + @staticmethod def copy_list(arg: list[Local]) -> list[Local]: return [ l.copy() for l in arg ] From d22f1650203a50f6a68dc25577a5263e9339470b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 7 May 2025 06:48:20 +0800 Subject: [PATCH 6/6] Update pycore_opcode_metadata.h --- Include/internal/pycore_opcode_metadata.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index fae075895367ef..28ee42cae59bfe 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1305,7 +1305,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, + [ANNOTATIONS_PLACEHOLDER] = { true, -1, 0 }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },