From 2f475e14663b640f02eed8bcfc91239e1bf69513 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Fri, 3 Jan 2025 22:18:47 +0100 Subject: [PATCH 01/17] Disable tuple folding in the AST optimizer --- Python/ast_opt.c | 12 +++++++----- Python/codegen.c | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 01e208b88eca8b..0b83d413006b1f 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -558,13 +558,15 @@ make_const_tuple(asdl_expr_seq *elts) static int fold_tuple(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { - PyObject *newval; + return 1; + // Disable tuple folding for now + // PyObject *newval; - if (node->v.Tuple.ctx != Load) - return 1; + // if (node->v.Tuple.ctx != Load) + // return 1; - newval = make_const_tuple(node->v.Tuple.elts); - return make_const(node, newval, arena); + // newval = make_const_tuple(node->v.Tuple.elts); + // return make_const(node, newval, arena); } static int diff --git a/Python/codegen.c b/Python/codegen.c index 61707ba677097c..88557bbd0a20ea 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1696,11 +1696,26 @@ codegen_typealias(compiler *c, stmt_ty s) return SUCCESS; } +static bool +is_const_tuple(asdl_expr_seq *elts) +{ + for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) { + expr_ty e = (expr_ty)asdl_seq_GET(elts, i); + if (e->kind != Constant_kind) { + return false; + } + } + return true; +} + /* Return false if the expression is a constant value except named singletons. Return true otherwise. */ static bool check_is_arg(expr_ty e) { + if (e->kind == Tuple_kind) { + return !is_const_tuple(e->v.Tuple.elts); + } if (e->kind != Constant_kind) { return true; } From 7a96d471e6a3471c0489aa8ec4c23651d1461162 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Fri, 3 Jan 2025 22:19:03 +0100 Subject: [PATCH 02/17] Provisionally fix tests --- Lib/test/test_ast/test_ast.py | 13 +++++++----- Lib/test/test_compile.py | 9 ++++++-- Lib/test/test_opcache.py | 39 ++++++++++++++++++----------------- Lib/test/test_peepholer.py | 19 ++++++++++------- 4 files changed, 46 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index c268a1f00f938e..08f7dbf93019d5 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3153,12 +3153,13 @@ def test_folding_binop(self): ): self.assert_ast(result_code, non_optimized_target, optimized_target) + # Tuple folding is currently disabled in the AST optimizer # Multiplication of constant tuples must be folded - code = "(1,) * 3" - non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) - optimized_target = self.wrap_expr(ast.Constant(eval(code))) + # code = "(1,) * 3" + # non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) + # optimized_target = self.wrap_expr(ast.Constant(eval(code))) - self.assert_ast(code, non_optimized_target, optimized_target) + # self.assert_ast(code, non_optimized_target, optimized_target) def test_folding_unaryop(self): code = "%s1" @@ -3179,6 +3180,7 @@ def create_unaryop(operand): ): self.assert_ast(result_code, non_optimized_target, optimized_target) + @unittest.skip("Tuple folding is currently disabled in the AST optimizer") def test_folding_not(self): code = "not (1 %s (1,))" operators = { @@ -3230,7 +3232,7 @@ def test_folding_format(self): self.assert_ast(code, non_optimized_target, optimized_target) - + @unittest.skip("Tuple folding is currently disabled in the AST optimizer") def test_folding_tuple(self): code = "(1,)" @@ -3279,6 +3281,7 @@ def test_folding_iter(self): self.assert_ast(code % (left, right), non_optimized_target, optimized_target) + @unittest.skip("Tuple folding is currently disabled in the AST optimizer") def test_folding_subscript(self): code = "(1,)[0]" diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index b5cf2ad18fe60b..956ea590052811 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -796,10 +796,15 @@ def check_same_constant(const): # Merge constants in tuple or frozenset f1, f2 = lambda: "not a name", lambda: ("not a name",) f3 = lambda x: x in {("not a name",)} + # TODO: I'm not sure if this is right.. self.assertIs(f1.__code__.co_consts[0], - f2.__code__.co_consts[0][0]) - self.assertIs(next(iter(f3.__code__.co_consts[0])), f2.__code__.co_consts[0]) + self.assertIs(f1.__code__.co_consts[0], + f2.__code__.co_consts[1][0]) + self.assertIs(f1.__code__.co_consts[0], + f3.__code__.co_consts[0]) + self.assertIs(f1.__code__.co_consts[0], + f3.__code__.co_consts[1][0]) # {0} is converted to a constant frozenset({0}) by the peephole # optimizer diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index c7cd4c2e8a3146..d46570bb775a5f 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1508,25 +1508,26 @@ def to_bool_str(): @cpython_only @requires_specialization_ft def test_unpack_sequence(self): - def unpack_sequence_two_tuple(): - for _ in range(100): - a, b = 1, 2 - self.assertEqual(a, 1) - self.assertEqual(b, 2) - - unpack_sequence_two_tuple() - self.assert_specialized(unpack_sequence_two_tuple, - "UNPACK_SEQUENCE_TWO_TUPLE") - self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") - - def unpack_sequence_tuple(): - for _ in range(100): - a, = 1, - self.assertEqual(a, 1) - - unpack_sequence_tuple() - self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") - self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") + # Tuple folding is currently disabled in the AST optimizer + # def unpack_sequence_two_tuple(): + # for _ in range(100): + # a, b = 1, 2 + # self.assertEqual(a, 1) + # self.assertEqual(b, 2) + + # unpack_sequence_two_tuple() + # self.assert_specialized(unpack_sequence_two_tuple, + # "UNPACK_SEQUENCE_TWO_TUPLE") + # self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") + + # def unpack_sequence_tuple(): + # for _ in range(100): + # a, = 1, + # self.assertEqual(a, 1) + + # unpack_sequence_tuple() + # self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") + # self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") def unpack_sequence_list(): for _ in range(100): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index b5b2b350e77a3b..4a16df1b82e0ae 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -203,7 +203,8 @@ def test_folding_of_sets_of_constants(self): ('a in {1,2,3}', frozenset({1, 2, 3})), ('a not in {"a","b","c"}', frozenset({'a', 'c', 'b'})), ('a in {None, 1, None}', frozenset({1, None})), - ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), + # Tuple folding is currently disabled in the AST optimizer + # ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), ('a in {1, 2, 3, 3, 2, 1}', frozenset({1, 2, 3})), ): with self.subTest(line=line): @@ -239,7 +240,8 @@ def test_folding_of_binops_on_constants(self): ('a = 14%4', 2), # binary modulo ('a = 2+3', 5), # binary add ('a = 13-4', 9), # binary subtract - ('a = (12,13)[1]', 13), # binary subscr + # Tuple folding is currently disabled in the AST optimizer + # ('a = (12,13)[1]', 13), # binary subscr ('a = 13 << 2', 52), # binary lshift ('a = 13 >> 2', 3), # binary rshift ('a = 13 & 7', 5), # binary and @@ -457,12 +459,13 @@ def test_constant_folding(self): '3 * -5', '-3 * 5', '2 * (3 * 4)', - '(2 * 3) * 4', - '(-1, 2, 3)', - '(1, -2, 3)', - '(1, 2, -3)', - '(1, 2, -3) * 6', - 'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}', + # Tuple folding is currently disabled in the AST optimizer + # '(2 * 3) * 4', + # '(-1, 2, 3)', + # '(1, -2, 3)', + # '(1, 2, -3)', + # '(1, 2, -3) * 6', + # 'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}', ] for e in exprs: with self.subTest(e=e): From 6d93343499a5228af338bb20ea1a51f7e7c81182 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 16 Jan 2025 14:57:13 +0200 Subject: [PATCH 03/17] Tweak tests --- Lib/test/test_ast/test_ast.py | 60 +++-------------------------------- Lib/test/test_compile.py | 13 -------- Lib/test/test_opcache.py | 28 +++++----------- Python/ast_opt.c | 15 --------- 4 files changed, 13 insertions(+), 103 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 08f7dbf93019d5..5c6339ce4e7bf9 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3153,14 +3153,6 @@ def test_folding_binop(self): ): self.assert_ast(result_code, non_optimized_target, optimized_target) - # Tuple folding is currently disabled in the AST optimizer - # Multiplication of constant tuples must be folded - # code = "(1,) * 3" - # non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) - # optimized_target = self.wrap_expr(ast.Constant(eval(code))) - - # self.assert_ast(code, non_optimized_target, optimized_target) - def test_folding_unaryop(self): code = "%s1" operators = self.unaryop.keys() @@ -3180,39 +3172,6 @@ def create_unaryop(operand): ): self.assert_ast(result_code, non_optimized_target, optimized_target) - @unittest.skip("Tuple folding is currently disabled in the AST optimizer") - def test_folding_not(self): - code = "not (1 %s (1,))" - operators = { - "in": ast.In(), - "is": ast.Is(), - } - opt_operators = { - "is": ast.IsNot(), - "in": ast.NotIn(), - } - - def create_notop(operand): - return ast.UnaryOp(op=ast.Not(), operand=ast.Compare( - left=ast.Constant(value=1), - ops=[operators[operand]], - comparators=[ast.Tuple(elts=[ast.Constant(value=1)])] - )) - - for op in operators.keys(): - result_code = code % op - non_optimized_target = self.wrap_expr(create_notop(op)) - optimized_target = self.wrap_expr( - ast.Compare(left=ast.Constant(1), ops=[opt_operators[op]], comparators=[ast.Constant(value=(1,))]) - ) - - with self.subTest( - result_code=result_code, - non_optimized_target=non_optimized_target, - optimized_target=optimized_target - ): - self.assert_ast(result_code, non_optimized_target, optimized_target) - def test_folding_format(self): code = "'%s' % (a,)" @@ -3232,15 +3191,6 @@ def test_folding_format(self): self.assert_ast(code, non_optimized_target, optimized_target) - @unittest.skip("Tuple folding is currently disabled in the AST optimizer") - def test_folding_tuple(self): - code = "(1,)" - - non_optimized_target = self.wrap_expr(ast.Tuple(elts=[ast.Constant(1)])) - optimized_target = self.wrap_expr(ast.Constant(value=(1,))) - - self.assert_ast(code, non_optimized_target, optimized_target) - def test_folding_comparator(self): code = "1 %s %s1%s" operators = [("in", ast.In()), ("not in", ast.NotIn())] @@ -3281,15 +3231,15 @@ def test_folding_iter(self): self.assert_ast(code % (left, right), non_optimized_target, optimized_target) - @unittest.skip("Tuple folding is currently disabled in the AST optimizer") def test_folding_subscript(self): - code = "(1,)[0]" + code = "'abcd'[0]" non_optimized_target = self.wrap_expr( - ast.Subscript(value=ast.Tuple(elts=[ast.Constant(value=1)]), slice=ast.Constant(value=0)) + ast.Subscript(value=ast.Constant(value='abcd'), slice=ast.Constant(value=0)) + ) + optimized_target = self.wrap_expr( + ast.JoinedStr(values=[ast.Constant(value='a')]) ) - optimized_target = self.wrap_expr(ast.Constant(value=1)) - self.assert_ast(code, non_optimized_target, optimized_target) def test_folding_type_param_in_function_def(self): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 956ea590052811..fe1880a2b52463 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -793,19 +793,6 @@ def check_same_constant(const): self.check_constant(f1, Ellipsis) self.assertEqual(repr(f1()), repr(Ellipsis)) - # Merge constants in tuple or frozenset - f1, f2 = lambda: "not a name", lambda: ("not a name",) - f3 = lambda x: x in {("not a name",)} - # TODO: I'm not sure if this is right.. - self.assertIs(f1.__code__.co_consts[0], - f2.__code__.co_consts[0]) - self.assertIs(f1.__code__.co_consts[0], - f2.__code__.co_consts[1][0]) - self.assertIs(f1.__code__.co_consts[0], - f3.__code__.co_consts[0]) - self.assertIs(f1.__code__.co_consts[0], - f3.__code__.co_consts[1][0]) - # {0} is converted to a constant frozenset({0}) by the peephole # optimizer f1, f2 = lambda x: x in {0}, lambda x: x in {0} diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index d46570bb775a5f..041dc8d7b17b97 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1508,26 +1508,14 @@ def to_bool_str(): @cpython_only @requires_specialization_ft def test_unpack_sequence(self): - # Tuple folding is currently disabled in the AST optimizer - # def unpack_sequence_two_tuple(): - # for _ in range(100): - # a, b = 1, 2 - # self.assertEqual(a, 1) - # self.assertEqual(b, 2) - - # unpack_sequence_two_tuple() - # self.assert_specialized(unpack_sequence_two_tuple, - # "UNPACK_SEQUENCE_TWO_TUPLE") - # self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") - - # def unpack_sequence_tuple(): - # for _ in range(100): - # a, = 1, - # self.assertEqual(a, 1) - - # unpack_sequence_tuple() - # self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") - # self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") + def unpack_sequence_tuple(): + for _ in range(100): + a, b, c, d = 1, 2, 3, 4 + self.assertEqual((a, b, c, d), (1, 2, 3, 4)) + + unpack_sequence_tuple() + self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") + self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") def unpack_sequence_list(): for _ in range(100): diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 0b83d413006b1f..a51cbad12fcb49 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -555,20 +555,6 @@ make_const_tuple(asdl_expr_seq *elts) return newval; } -static int -fold_tuple(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) -{ - return 1; - // Disable tuple folding for now - // PyObject *newval; - - // if (node->v.Tuple.ctx != Load) - // return 1; - - // newval = make_const_tuple(node->v.Tuple.elts); - // return make_const(node, newval, arena); -} - static int fold_subscr(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { @@ -839,7 +825,6 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) break; case Tuple_kind: CALL_SEQ(astfold_expr, expr, node_->v.Tuple.elts); - CALL(fold_tuple, expr_ty, node_); break; case Name_kind: if (node_->v.Name.ctx == Load && From 5c05d69401f23c5f108f917b4c1ae6b0e6943a57 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 21 Jan 2025 14:40:40 +0200 Subject: [PATCH 04/17] Restore UNPACK_SEQUENCE_TWO_TUPLE --- Lib/test/test_opcache.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index e988fb81a4a232..1ddceb4321a6bf 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1603,6 +1603,18 @@ def to_bool_str(): @cpython_only @requires_specialization_ft def test_unpack_sequence(self): + def unpack_sequence_two_tuple(): + t = 1, 2 + for _ in range(100): + a, b = t + self.assertEqual(a, 1) + self.assertEqual(b, 2) + + unpack_sequence_two_tuple() + self.assert_specialized(unpack_sequence_two_tuple, + "UNPACK_SEQUENCE_TWO_TUPLE") + self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") + def unpack_sequence_tuple(): for _ in range(100): a, b, c, d = 1, 2, 3, 4 From 05f3e627761644e23377ce346b31d66312eafce2 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 21 Jan 2025 16:22:09 +0200 Subject: [PATCH 05/17] Update Lib/test/test_peepholer.py --- Lib/test/test_peepholer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 4a16df1b82e0ae..046fa1edd1bc5b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -459,8 +459,8 @@ def test_constant_folding(self): '3 * -5', '-3 * 5', '2 * (3 * 4)', - # Tuple folding is currently disabled in the AST optimizer - # '(2 * 3) * 4', + '(2 * 3) * 4', + # Tuple folding is currently disabled in the AST optimizer # '(-1, 2, 3)', # '(1, -2, 3)', # '(1, 2, -3)', From 9f1feb4c8fb337ae53d3f425be29ef436a7fdb70 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 27 Jan 2025 17:35:47 +0200 Subject: [PATCH 06/17] Restore test_folding_op by changing const from tuple to a string --- Lib/test/test_ast/test_ast.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 5c6339ce4e7bf9..8b6584c55a1fec 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3172,6 +3172,38 @@ def create_unaryop(operand): ): self.assert_ast(result_code, non_optimized_target, optimized_target) + def test_folding_not(self): + code = "not ('a' %s 'ab')" + operators = { + "in": ast.In(), + "is": ast.Is(), + } + opt_operators = { + "is": ast.IsNot(), + "in": ast.NotIn(), + } + + def create_notop(operand): + return ast.UnaryOp(op=ast.Not(), operand=ast.Compare( + left=ast.Constant(value="a"), + ops=[operators[operand]], + comparators=[ast.Constant(value="ab")] + )) + + for op in operators.keys(): + result_code = code % op + non_optimized_target = self.wrap_expr(create_notop(op)) + optimized_target = self.wrap_expr( + ast.Compare(left=ast.Constant(value="a"), ops=[opt_operators[op]], comparators=[ast.Constant(value="ab")]) + ) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + def test_folding_format(self): code = "'%s' % (a,)" From ac50aadaffc746ac91a9ddac522a8372978389ad Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 27 Jan 2025 20:30:10 +0200 Subject: [PATCH 07/17] Fold set into frozenset in CFG --- Lib/test/test_ast/test_ast.py | 2 - Python/ast_opt.c | 12 +----- Python/flowgraph.c | 75 +++++++++++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 8b6584c55a1fec..18d2ec96164cc1 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3228,7 +3228,6 @@ def test_folding_comparator(self): operators = [("in", ast.In()), ("not in", ast.NotIn())] braces = [ ("[", "]", ast.List, (1,)), - ("{", "}", ast.Set, frozenset({1})), ] for left, right, non_optimized_comparator, optimized_comparator in braces: for op, node in operators: @@ -3246,7 +3245,6 @@ def test_folding_iter(self): code = "for _ in %s1%s: pass" braces = [ ("[", "]", ast.List, (1,)), - ("{", "}", ast.Set, frozenset({1})), ] for left, right, ast_cls, optimized_iter in braces: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index a51cbad12fcb49..7babc88c17c371 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -575,8 +575,7 @@ fold_subscr(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) } /* Change literal list or set of constants into constant - tuple or frozenset respectively. Change literal list of - non-constants into tuple. + tuple. Change literal list of non-constants into tuple. Used for right operand of "in" and "not in" tests and for iterable in "for" loop and comprehensions. */ @@ -597,12 +596,6 @@ fold_iter(expr_ty arg, PyArena *arena, _PyASTOptimizeState *state) /* Try to create a constant tuple. */ newval = make_const_tuple(elts); } - else if (arg->kind == Set_kind) { - newval = make_const_tuple(arg->v.Set.elts); - if (newval) { - Py_SETREF(newval, PyFrozenSet_New(newval)); - } - } else { return 1; } @@ -618,8 +611,7 @@ fold_compare(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) ops = node->v.Compare.ops; args = node->v.Compare.comparators; - /* Change literal list or set in 'in' or 'not in' into - tuple or frozenset respectively. */ + /* Change literal list or set in 'in' or 'not in' into tuple. */ i = asdl_seq_LEN(ops) - 1; int op = asdl_seq_GET(ops, i); if (op == In || op == NotIn) { diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 3f8d9db166ff98..1a7ba5ce23c357 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1336,6 +1336,18 @@ add_const(PyObject *newconst, PyObject *consts, PyObject *const_cache) return (int)index; } +static int +is_sequence_constant(cfg_instr *inst, int n) +{ + for (int i = 0; i < n; i++) { + if (!loads_const(inst[i].i_opcode)) { + return 0; + } + } + return 1; +} + + /* Replace LOAD_CONST c1, LOAD_CONST c2 ... LOAD_CONST cn, BUILD_TUPLE n with LOAD_CONST (c1, c2, ... cn). The consts table must still be in list form so that the @@ -1353,17 +1365,60 @@ fold_tuple_on_constants(PyObject *const_cache, assert(inst[n].i_opcode == BUILD_TUPLE); assert(inst[n].i_oparg == n); + if (!is_sequence_constant(inst, n)) { + return SUCCESS; + } + + /* Buildup new tuple of constants */ + PyObject *newconst = PyTuple_New(n); + if (newconst == NULL) { + return ERROR; + } for (int i = 0; i < n; i++) { - if (!loads_const(inst[i].i_opcode)) { - return SUCCESS; + int op = inst[i].i_opcode; + int arg = inst[i].i_oparg; + PyObject *constant = get_const_value(op, arg, consts); + if (constant == NULL) { + return ERROR; } + PyTuple_SET_ITEM(newconst, i, constant); + } + int index = add_const(newconst, consts, const_cache); + if (index < 0) { + return ERROR; + } + for (int i = 0; i < n; i++) { + INSTR_SET_OP0(&inst[i], NOP); + } + INSTR_SET_OP1(&inst[n], LOAD_CONST, index); + return SUCCESS; +} + + +// Replaces const set with a frozenset. +// This should be used only in situations where we 100% sure that +// this set cannot be changed: where's constant set is a rhs in `for` loop +// or it's a rhs in `in` operation. +static int +fold_set_on_constants(PyObject *const_cache, + cfg_instr *inst, + int n, PyObject *consts) +{ + /* Pre-conditions */ + assert(PyDict_CheckExact(const_cache)); + assert(PyList_CheckExact(consts)); + assert(inst[n].i_opcode == BUILD_SET); + assert(inst[n].i_oparg == n); + + if (!is_sequence_constant(inst, n)) { + return SUCCESS; } - /* Buildup new tuple of constants */ PyObject *newconst = PyTuple_New(n); if (newconst == NULL) { return ERROR; } + for (int i = 0; i < n; i++) { int op = inst[i].i_opcode; int arg = inst[i].i_oparg; @@ -1373,6 +1428,13 @@ fold_tuple_on_constants(PyObject *const_cache, } PyTuple_SET_ITEM(newconst, i, constant); } + + PyObject *frozenset = PyFrozenSet_New(newconst); + if (frozenset == NULL) { + return ERROR; + } + Py_SETREF(newconst, frozenset); + int index = add_const(newconst, consts, const_cache); if (index < 0) { return ERROR; @@ -1751,6 +1813,13 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) } } break; + case BUILD_SET: + if (nextop == CONTAINS_OP || nextop == GET_ITER) { + if (fold_set_on_constants(const_cache, inst-oparg, oparg, consts)) { + goto error; + } + } + break; case POP_JUMP_IF_NOT_NONE: case POP_JUMP_IF_NONE: switch (target->i_opcode) { From 17dffaa2f36adaf3492a8bf6587994b054811c18 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 1 Feb 2025 14:41:32 +0200 Subject: [PATCH 08/17] `for` target optimization (set/list -> frozenset/tuple) --- Lib/test/test_ast/test_ast.py | 38 ------------------ Python/ast_opt.c | 75 ----------------------------------- Python/flowgraph.c | 30 ++++++++------ 3 files changed, 17 insertions(+), 126 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 18d2ec96164cc1..1a472dcdbe6fcf 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3223,44 +3223,6 @@ def test_folding_format(self): self.assert_ast(code, non_optimized_target, optimized_target) - def test_folding_comparator(self): - code = "1 %s %s1%s" - operators = [("in", ast.In()), ("not in", ast.NotIn())] - braces = [ - ("[", "]", ast.List, (1,)), - ] - for left, right, non_optimized_comparator, optimized_comparator in braces: - for op, node in operators: - non_optimized_target = self.wrap_expr(ast.Compare( - left=ast.Constant(1), ops=[node], - comparators=[non_optimized_comparator(elts=[ast.Constant(1)])] - )) - optimized_target = self.wrap_expr(ast.Compare( - left=ast.Constant(1), ops=[node], - comparators=[ast.Constant(value=optimized_comparator)] - )) - self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target) - - def test_folding_iter(self): - code = "for _ in %s1%s: pass" - braces = [ - ("[", "]", ast.List, (1,)), - ] - - for left, right, ast_cls, optimized_iter in braces: - non_optimized_target = self.wrap_statement(ast.For( - target=ast.Name(id="_", ctx=ast.Store()), - iter=ast_cls(elts=[ast.Constant(1)]), - body=[ast.Pass()] - )) - optimized_target = self.wrap_statement(ast.For( - target=ast.Name(id="_", ctx=ast.Store()), - iter=ast.Constant(value=optimized_iter), - body=[ast.Pass()] - )) - - self.assert_ast(code % (left, right), non_optimized_target, optimized_target) - def test_folding_subscript(self): code = "'abcd'[0]" diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 7babc88c17c371..758f2fc88c9115 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -532,29 +532,6 @@ fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) return make_const(node, newval, arena); } -static PyObject* -make_const_tuple(asdl_expr_seq *elts) -{ - for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) { - expr_ty e = (expr_ty)asdl_seq_GET(elts, i); - if (e->kind != Constant_kind) { - return NULL; - } - } - - PyObject *newval = PyTuple_New(asdl_seq_LEN(elts)); - if (newval == NULL) { - return NULL; - } - - for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) { - expr_ty e = (expr_ty)asdl_seq_GET(elts, i); - PyObject *v = e->v.Constant.value; - PyTuple_SET_ITEM(newval, i, Py_NewRef(v)); - } - return newval; -} - static int fold_subscr(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { @@ -574,54 +551,6 @@ fold_subscr(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) return make_const(node, newval, arena); } -/* Change literal list or set of constants into constant - tuple. Change literal list of non-constants into tuple. - Used for right operand of "in" and "not in" tests and for iterable - in "for" loop and comprehensions. -*/ -static int -fold_iter(expr_ty arg, PyArena *arena, _PyASTOptimizeState *state) -{ - PyObject *newval; - if (arg->kind == List_kind) { - /* First change a list into tuple. */ - asdl_expr_seq *elts = arg->v.List.elts; - if (has_starred(elts)) { - return 1; - } - expr_context_ty ctx = arg->v.List.ctx; - arg->kind = Tuple_kind; - arg->v.Tuple.elts = elts; - arg->v.Tuple.ctx = ctx; - /* Try to create a constant tuple. */ - newval = make_const_tuple(elts); - } - else { - return 1; - } - return make_const(arg, newval, arena); -} - -static int -fold_compare(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) -{ - asdl_int_seq *ops; - asdl_expr_seq *args; - Py_ssize_t i; - - ops = node->v.Compare.ops; - args = node->v.Compare.comparators; - /* Change literal list or set in 'in' or 'not in' into tuple. */ - i = asdl_seq_LEN(ops) - 1; - int op = asdl_seq_GET(ops, i); - if (op == In || op == NotIn) { - if (!fold_iter((expr_ty)asdl_seq_GET(args, i), arena, state)) { - return 0; - } - } - return 1; -} - static int astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); @@ -782,7 +711,6 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case Compare_kind: CALL(astfold_expr, expr_ty, node_->v.Compare.left); CALL_SEQ(astfold_expr, expr, node_->v.Compare.comparators); - CALL(fold_compare, expr_ty, node_); break; case Call_kind: CALL(astfold_expr, expr_ty, node_->v.Call.func); @@ -852,7 +780,6 @@ astfold_comprehension(comprehension_ty node_, PyArena *ctx_, _PyASTOptimizeState CALL(astfold_expr, expr_ty, node_->iter); CALL_SEQ(astfold_expr, expr, node_->ifs); - CALL(fold_iter, expr_ty, node_->iter); return 1; } @@ -939,8 +866,6 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL(astfold_expr, expr_ty, node_->v.For.iter); CALL_SEQ(astfold_stmt, stmt, node_->v.For.body); CALL_SEQ(astfold_stmt, stmt, node_->v.For.orelse); - - CALL(fold_iter, expr_ty, node_->v.For.iter); break; case AsyncFor_kind: CALL(astfold_expr, expr_ty, node_->v.AsyncFor.target); diff --git a/Python/flowgraph.c b/Python/flowgraph.c index e33cc0e340f15e..771a5226aa0080 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1395,21 +1395,24 @@ fold_tuple_on_constants(PyObject *const_cache, } -// Replaces const set with a frozenset. -// This should be used only in situations where we 100% sure that -// this set cannot be changed: where's constant set is a rhs in `for` loop -// or it's a rhs in `in` operation. +/* Replaces const set/list with a frozenset/tuple. + This should be used only in situations where we 100% sure that + this set cannot be changed: where's constant set/list is a rhs in `for` loop + or it's a rhs in `in` operation. +*/ static int -fold_set_on_constants(PyObject *const_cache, +fold_if_const_list_or_set(PyObject *const_cache, cfg_instr *inst, int n, PyObject *consts) { /* Pre-conditions */ assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); - assert(inst[n].i_opcode == BUILD_SET); assert(inst[n].i_oparg == n); + int build = inst[n].i_opcode; + assert(build == BUILD_LIST || build == BUILD_SET); + if (!is_constant_sequence(inst, n)) { return SUCCESS; } @@ -1428,13 +1431,13 @@ fold_set_on_constants(PyObject *const_cache, } PyTuple_SET_ITEM(newconst, i, constant); } - - PyObject *frozenset = PyFrozenSet_New(newconst); - if (frozenset == NULL) { - return ERROR; + if (build == BUILD_SET) { + PyObject *frozenset = PyFrozenSet_New(newconst); + if (frozenset == NULL) { + return ERROR; + } + Py_SETREF(newconst, frozenset); } - Py_SETREF(newconst, frozenset); - int index = add_const(newconst, consts, const_cache); if (index < 0) { return ERROR; @@ -1866,9 +1869,10 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) case BUILD_LIST: case BUILD_SET: if (nextop == CONTAINS_OP || nextop == GET_ITER) { - if (fold_set_on_constants(const_cache, inst-oparg, oparg, consts) < 0) { + if (fold_if_const_list_or_set(const_cache, inst-oparg, oparg, consts) < 0) { goto error; } + break; } if (i >= oparg) { if (optimize_if_const_list_or_set(const_cache, inst-oparg, oparg, consts) < 0) { From 46845f0eb1a0b8466d68d6fb6bbf219976d3363a Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 1 Feb 2025 17:36:48 +0200 Subject: [PATCH 09/17] Fold list into a tuple as rhs in `for`/`in` operators --- Python/flowgraph.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 771a5226aa0080..4e06e9dceca4ee 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1867,6 +1867,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) } break; case BUILD_LIST: + if (!is_constant_sequence(inst-oparg, oparg) + && (nextop == CONTAINS_OP || nextop == GET_ITER)) { + INSTR_SET_OP1(inst, BUILD_TUPLE, oparg); + break; + } + _Py_FALLTHROUGH; case BUILD_SET: if (nextop == CONTAINS_OP || nextop == GET_ITER) { if (fold_if_const_list_or_set(const_cache, inst-oparg, oparg, consts) < 0) { From e9631d8a287d082dc9cc7759f42479a3e842f6cd Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 1 Feb 2025 18:22:57 +0200 Subject: [PATCH 10/17] Fix tests --- Lib/test/test_peepholer.py | 7 ++++--- Python/flowgraph.c | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 046fa1edd1bc5b..ca1d9eb04d9d0e 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -146,7 +146,7 @@ def test_folding_of_tuples_of_constants(self): for line, elem in ( ('a = 1,2,3', (1, 2, 3)), ('("a","b","c")', ('a', 'b', 'c')), - ('a,b,c = 1,2,3', (1, 2, 3)), + ('a,b,c,d = 1,2,3,4', (1, 2, 3, 4)), ('(None, 1, None)', (None, 1, None)), ('((1, 2), 3, 4)', ((1, 2), 3, 4)), ): @@ -156,8 +156,9 @@ def test_folding_of_tuples_of_constants(self): self.assertNotInBytecode(code, 'BUILD_TUPLE') self.check_lnotab(code) - # Long tuples should be folded too. - code = compile(repr(tuple(range(10000))),'','single') + # Long tuples should be folded too, but their length should not + # exceed the `STACK_USE_GUIDELINE` + code = compile(repr(tuple(range(30))),'','single') self.assertNotInBytecode(code, 'BUILD_TUPLE') # One LOAD_CONST for the tuple, one for the None return value load_consts = [instr for instr in dis.get_instructions(code) diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 4e06e9dceca4ee..4d5a3b4439cd86 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -124,6 +124,16 @@ is_jump(cfg_instr *i) _instr__ptr_->i_oparg = 0; \ } while (0); +/* No args, reset lineno*/ +#define INSTR_SET_OP0_RESET_LINENO(I, OP) \ + do { \ + assert(!OPCODE_HAS_ARG(OP)); \ + cfg_instr *_instr__ptr_ = (I); \ + _instr__ptr_->i_opcode = (OP); \ + _instr__ptr_->i_oparg = 0; \ + _instr__ptr_->i_loc.lineno = -1; \ + } while (0); + /***** Blocks *****/ /* Returns the offset of the next instruction in the current block's @@ -1388,7 +1398,7 @@ fold_tuple_on_constants(PyObject *const_cache, return ERROR; } for (int i = 0; i < n; i++) { - INSTR_SET_OP0(&inst[i], NOP); + INSTR_SET_OP0_RESET_LINENO(&inst[i], NOP); } INSTR_SET_OP1(&inst[n], LOAD_CONST, index); return SUCCESS; @@ -1443,7 +1453,7 @@ fold_if_const_list_or_set(PyObject *const_cache, return ERROR; } for (int i = 0; i < n; i++) { - INSTR_SET_OP0(&inst[i], NOP); + INSTR_SET_OP0_RESET_LINENO(&inst[i], NOP); } INSTR_SET_OP1(&inst[n], LOAD_CONST, index); return SUCCESS; @@ -1492,7 +1502,7 @@ optimize_if_const_list_or_set(PyObject *const_cache, cfg_instr* inst, int n, PyO RETURN_IF_ERROR(index); INSTR_SET_OP1(&inst[0], build, 0); for (int i = 1; i < n - 1; i++) { - INSTR_SET_OP0(&inst[i], NOP); + INSTR_SET_OP0_RESET_LINENO(&inst[i], NOP); } INSTR_SET_OP1(&inst[n-1], LOAD_CONST, index); INSTR_SET_OP1(&inst[n], extend, 1); @@ -1867,14 +1877,14 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) } break; case BUILD_LIST: - if (!is_constant_sequence(inst-oparg, oparg) + if (i >= oparg && !is_constant_sequence(inst-oparg, oparg) && (nextop == CONTAINS_OP || nextop == GET_ITER)) { INSTR_SET_OP1(inst, BUILD_TUPLE, oparg); break; } _Py_FALLTHROUGH; case BUILD_SET: - if (nextop == CONTAINS_OP || nextop == GET_ITER) { + if ((i >= oparg) && (nextop == CONTAINS_OP || nextop == GET_ITER)) { if (fold_if_const_list_or_set(const_cache, inst-oparg, oparg, consts) < 0) { goto error; } From 477c7843e17e8f0872b8a6bb9abad4e4f357d922 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 14 Feb 2025 10:25:26 +0200 Subject: [PATCH 11/17] Fix merge artifacts --- Python/flowgraph.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Python/flowgraph.c b/Python/flowgraph.c index aae711eaada602..d0b392371babcf 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1348,8 +1348,6 @@ add_const(PyObject *newconst, PyObject *consts, PyObject *const_cache) return (int)index; } -ant_sequence(cfg_instr *inst, int n) -======= /* Walk basic block backwards starting from "start" trying to collect "size" number of subsequent constants from instructions loading constants into new tuple ignoring NOP's in between. @@ -1435,7 +1433,7 @@ fold_tuple_of_constants(basicblock *bb, int n, PyObject *consts, PyObject *const int index = add_const(newconst, consts, const_cache); RETURN_IF_ERROR(index); nop_out(bb, n-1, seq_size); - INSTR_SET_OP1(&bb->b_instr[n], LOAD_CONST, index) + INSTR_SET_OP1(&bb->b_instr[n], LOAD_CONST, index); return SUCCESS; } @@ -1950,12 +1948,6 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) RETURN_IF_ERROR(fold_tuple_of_constants(bb, i, consts, const_cache)); break; case BUILD_LIST: - if (i >= oparg && !is_constant_sequence(inst-oparg, oparg) - && (nextop == CONTAINS_OP || nextop == GET_ITER)) { - INSTR_SET_OP1(inst, BUILD_TUPLE, oparg); - break; - } - _Py_FALLTHROUGH; case BUILD_SET: RETURN_IF_ERROR(optimize_lists_and_sets(bb, i, nextop, consts, const_cache)); break; From aad9fb3a25cc07ba792d3d73a14d4fbb4af0d807 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 22 Feb 2025 12:50:01 +0200 Subject: [PATCH 12/17] Remove failing tests --- Lib/test/test_ast/test_ast.py | 102 ---------------------------------- Lib/test/test_builtin.py | 5 -- 2 files changed, 107 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 179e4fe07eb47d..b952c0c984b1f1 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -153,22 +153,6 @@ def test_optimization_levels__debug__(self): self.assertIsInstance(res.body[0].value, ast.Name) self.assertEqual(res.body[0].value.id, expected) - def test_optimization_levels_const_folding(self): - folded = ('Expr', (1, 0, 1, 6), ('Constant', (1, 0, 1, 6), (1, 2), None)) - not_folded = ('Expr', (1, 0, 1, 6), - ('Tuple', (1, 0, 1, 6), - [('Constant', (1, 1, 1, 2), 1, None), - ('Constant', (1, 4, 1, 5), 2, None)], ('Load',))) - - cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] - for (optval, expected) in cases: - with self.subTest(optval=optval): - tree1 = ast.parse("(1, 2)", optimize=optval) - tree2 = ast.parse(ast.parse("(1, 2)"), optimize=optval) - for tree in [tree1, tree2]: - res = to_tuple(tree.body[0]) - self.assertEqual(res, expected) - def test_invalid_position_information(self): invalid_linenos = [ (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) @@ -3138,92 +3122,6 @@ def test_folding_format(self): self.assert_ast(code, non_optimized_target, optimized_target) - def test_folding_type_param_in_function_def(self): - code = "def foo[%s = (1, 2)](): pass" - - unoptimized_tuple = ast.Tuple(elts=[ast.Constant(1), ast.Constant(2)]) - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] - - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.FunctionDef( - name='foo', - args=ast.arguments(), - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=ast.Constant((1, 2)))] - ) - ) - non_optimized_target = self.wrap_statement( - ast.FunctionDef( - name='foo', - args=ast.arguments(), - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=unoptimized_tuple)] - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) - - def test_folding_type_param_in_class_def(self): - code = "class foo[%s = (1, 2)]: pass" - - unoptimized_tuple = ast.Tuple(elts=[ast.Constant(1), ast.Constant(2)]) - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] - - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.ClassDef( - name='foo', - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=ast.Constant((1, 2)))] - ) - ) - non_optimized_target = self.wrap_statement( - ast.ClassDef( - name='foo', - body=[ast.Pass()], - type_params=[type_param(name=name, default_value=unoptimized_tuple)] - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) - - def test_folding_type_param_in_type_alias(self): - code = "type foo[%s = (1, 2)] = 1" - - unoptimized_tuple = ast.Tuple(elts=[ast.Constant(1), ast.Constant(2)]) - unoptimized_type_params = [ - ("T", "T", ast.TypeVar), - ("**P", "P", ast.ParamSpec), - ("*Ts", "Ts", ast.TypeVarTuple), - ] - - for type, name, type_param in unoptimized_type_params: - result_code = code % type - optimized_target = self.wrap_statement( - ast.TypeAlias( - name=ast.Name(id='foo', ctx=ast.Store()), - type_params=[type_param(name=name, default_value=ast.Constant((1, 2)))], - value=ast.Constant(value=1), - ) - ) - non_optimized_target = self.wrap_statement( - ast.TypeAlias( - name=ast.Name(id='foo', ctx=ast.Store()), - type_params=[type_param(name=name, default_value=unoptimized_tuple)], - value=ast.Constant(value=1), - ) - ) - self.assert_ast(result_code, non_optimized_target, optimized_target) - def test_folding_match_case_allowed_expressions(self): def get_match_case_values(node): result = [] diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index d15964fe9dd88b..f87ab611bbc129 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -570,11 +570,6 @@ def test_compile_ast(self): self.assertIsInstance(raw_right, ast.Tuple) self.assertListEqual([elt.value for elt in raw_right.elts], [1, 2]) - for opt in [opt1, opt2]: - opt_right = opt.value.right # expect Constant((1,2)) - self.assertIsInstance(opt_right, ast.Constant) - self.assertEqual(opt_right.value, (1, 2)) - def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') From be40093134af92e41c68aed19d36acacffa8ea7a Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 22 Feb 2025 12:53:21 +0200 Subject: [PATCH 13/17] Restore test_peepholer tests --- Lib/test/test_peepholer.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index af80d70ec3b3a8..8fd4528cca9b35 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -249,8 +249,7 @@ def test_folding_of_binops_on_constants(self): ('a = 14%4', 2), # binary modulo ('a = 2+3', 5), # binary add ('a = 13-4', 9), # binary subtract - # Tuple folding is currently disabled in the AST optimizer - # ('a = (12,13)[1]', 13), # binary subscr + ('a = (12,13)[1]', 13), # binary subscr ('a = 13 << 2', 52), # binary lshift ('a = 13 >> 2', 3), # binary rshift ('a = 13 & 7', 5), # binary and @@ -469,12 +468,11 @@ def test_constant_folding(self): '-3 * 5', '2 * (3 * 4)', '(2 * 3) * 4', - # Tuple folding is currently disabled in the AST optimizer - # '(-1, 2, 3)', - # '(1, -2, 3)', - # '(1, 2, -3)', - # '(1, 2, -3) * 6', - # 'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}', + '(-1, 2, 3)', + '(1, -2, 3)', + '(1, 2, -3)', + '(1, 2, -3) * 6', + 'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}', ] for e in exprs: with self.subTest(e=e): From 5aec965a9ea1721cb7decd8caa91e1d8e20c2e89 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 22 Feb 2025 15:28:07 +0200 Subject: [PATCH 14/17] Add a few tests --- Lib/test/test_peepholer.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 8fd4528cca9b35..a00a0f09f1f12f 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -347,6 +347,28 @@ def negzero(): self.assertInBytecode(code, opname) self.check_lnotab(code) + def test_folding_of_tuples_on_constants(self): + tests =[ + ('()', True, 0), + ('(1, 2, 3)', True, 3), + ('("a", "b", "c")', True, 3), + ('(1, a)', False, 2), + ('(a, b, c)', False, 3), + ('(1, (2, 3))', True, 2), + ('(a, (b, c))', False, 2), + ('(1, [], {})', False, 3), + (repr(tuple(range(30))), True, 30), + ('(1, (2, (3, (4, (5)))))', True, 2) + ] + for expr, is_const, length in tests: + with self.subTest(expr=expr, is_const=is_const, length=length): + code = compile(expr, '', 'eval') + if is_const: + self.assertNotInBytecode(code, 'BUILD_TUPLE', length) + self.assertInBytecode(code, 'LOAD_CONST', eval(expr)) + else: + self.assertInBytecode(code, 'BUILD_TUPLE', length) + def test_elim_extra_return(self): # RETURN LOAD_CONST None RETURN --> RETURN def f(x): From ee69f0ff788d61f5e0852f12e3717cf37fc7d51b Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 22 Feb 2025 15:29:25 +0200 Subject: [PATCH 15/17] Restore test --- Lib/test/test_peepholer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index a00a0f09f1f12f..1394b539b121dd 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -212,8 +212,7 @@ def test_folding_of_sets_of_constants(self): ('a in {1,2,3}', frozenset({1, 2, 3})), ('a not in {"a","b","c"}', frozenset({'a', 'c', 'b'})), ('a in {None, 1, None}', frozenset({1, None})), - # Tuple folding is currently disabled in the AST optimizer - # ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), + ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), ('a in {1, 2, 3, 3, 2, 1}', frozenset({1, 2, 3})), ): with self.subTest(line=line): From 080146385d598028362afdf4144915d8c2765f7c Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 22 Feb 2025 16:13:45 +0200 Subject: [PATCH 16/17] Regenerate some files --- Programs/test_frozenmain.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 0fe8d3d3f7d8c6..18537311174dcf 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -9,19 +9,19 @@ unsigned char M_test_frozenmain[] = { 31,0,89,1,78,8,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,33,0,51,0,0,0,0,0, 0,0,80,3,44,26,0,0,0,0,0,0,0,0,0,0, - 112,5,80,4,16,0,68,24,0,0,112,6,89,2,33,0, - 80,5,89,6,12,0,80,6,89,5,89,6,44,26,0,0, + 112,5,80,6,16,0,68,24,0,0,112,6,89,2,33,0, + 80,4,89,6,12,0,80,5,89,5,89,6,44,26,0,0, 0,0,0,0,0,0,0,0,12,0,49,4,51,1,0,0, 0,0,0,0,31,0,73,26,0,0,9,0,30,0,80,0, 35,0,41,7,78,122,18,70,114,111,122,101,110,32,72,101, 108,108,111,32,87,111,114,108,100,122,8,115,121,115,46,97, - 114,103,118,218,6,99,111,110,102,105,103,41,5,218,12,112, - 114,111,103,114,97,109,95,110,97,109,101,218,10,101,120,101, - 99,117,116,97,98,108,101,218,15,117,115,101,95,101,110,118, - 105,114,111,110,109,101,110,116,218,17,99,111,110,102,105,103, - 117,114,101,95,99,95,115,116,100,105,111,218,14,98,117,102, - 102,101,114,101,100,95,115,116,100,105,111,122,7,99,111,110, - 102,105,103,32,122,2,58,32,41,7,218,3,115,121,115,218, + 114,103,118,218,6,99,111,110,102,105,103,122,7,99,111,110, + 102,105,103,32,122,2,58,32,41,5,218,12,112,114,111,103, + 114,97,109,95,110,97,109,101,218,10,101,120,101,99,117,116, + 97,98,108,101,218,15,117,115,101,95,101,110,118,105,114,111, + 110,109,101,110,116,218,17,99,111,110,102,105,103,117,114,101, + 95,99,95,115,116,100,105,111,218,14,98,117,102,102,101,114, + 101,100,95,115,116,100,105,111,41,7,218,3,115,121,115,218, 17,95,116,101,115,116,105,110,116,101,114,110,97,108,99,97, 112,105,218,5,112,114,105,110,116,218,4,97,114,103,118,218, 11,103,101,116,95,99,111,110,102,105,103,115,114,2,0,0, From a268315f2d1503c88d25369a4701b883f25005cb Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sun, 23 Feb 2025 18:53:09 +0200 Subject: [PATCH 17/17] Address review --- Lib/test/test_builtin.py | 12 ++++++++---- Lib/test/test_compile.py | 8 ++++++++ Lib/test/test_peepholer.py | 36 ++++++++++++++---------------------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index f87ab611bbc129..c6c0d844ab960d 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -555,7 +555,7 @@ def test_compile_async_generator(self): self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) def test_compile_ast(self): - args = ("a*(1,2)", "f.py", "exec") + args = ("a*__debug__", "f.py", "exec") raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] opt1 = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] opt2 = compile(ast.parse(args[0]), *args[1:], flags = ast.PyCF_OPTIMIZED_AST).body[0] @@ -566,9 +566,13 @@ def test_compile_ast(self): self.assertIsInstance(tree.value.left, ast.Name) self.assertEqual(tree.value.left.id, 'a') - raw_right = raw.value.right # expect Tuple((1, 2)) - self.assertIsInstance(raw_right, ast.Tuple) - self.assertListEqual([elt.value for elt in raw_right.elts], [1, 2]) + raw_right = raw.value.right + self.assertIsInstance(raw_right, ast.Name) + + for opt in [opt1, opt2]: + opt_right = opt.value.right + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, __debug__) def test_delattr(self): sys.spam = 1 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 0730a3882f6326..bed91337581fa9 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -793,6 +793,14 @@ def check_same_constant(const): self.check_constant(f1, Ellipsis) self.assertEqual(repr(f1()), repr(Ellipsis)) + # Merge constants in tuple or frozenset + f1, f2 = lambda: "not a name", lambda: ("not a name",) + f3 = lambda x: x in {("not a name",)} + self.assertIs(f1.__code__.co_consts[0], + f2.__code__.co_consts[1][0]) + self.assertIs(next(iter(f3.__code__.co_consts[1])), + f2.__code__.co_consts[1]) + # {0} is converted to a constant frozenset({0}) by the peephole # optimizer f1, f2 = lambda x: x in {0}, lambda x: x in {0} diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 1394b539b121dd..7ee8aa4de51c5d 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -157,6 +157,9 @@ def test_folding_of_tuples_of_constants(self): ('a,b,c,d = 1,2,3,4', (1, 2, 3, 4)), ('(None, 1, None)', (None, 1, None)), ('((1, 2), 3, 4)', ((1, 2), 3, 4)), + ('(1, 2, (3, 4))', (1, 2, (3, 4))), + ('()', ()), + ('(1, (2, (3, (4, (5,)))))', (1, (2, (3, (4, (5,)))))), ): with self.subTest(line=line): code = compile(line,'','single') @@ -164,6 +167,17 @@ def test_folding_of_tuples_of_constants(self): self.assertNotInBytecode(code, 'BUILD_TUPLE') self.check_lnotab(code) + for expr, length in ( + ('(1, a)', 2), + ('(a, b, c)', 3), + ('(a, (b, c))', 2), + ('(1, [], {})', 3), + ): + with self.subTest(expr=expr, length=length): + code = compile(expr, '', 'single') + self.assertInBytecode(code, 'BUILD_TUPLE', length) + self.check_lnotab(code) + # Long tuples should be folded too, but their length should not # exceed the `STACK_USE_GUIDELINE` code = compile(repr(tuple(range(30))),'','single') @@ -346,28 +360,6 @@ def negzero(): self.assertInBytecode(code, opname) self.check_lnotab(code) - def test_folding_of_tuples_on_constants(self): - tests =[ - ('()', True, 0), - ('(1, 2, 3)', True, 3), - ('("a", "b", "c")', True, 3), - ('(1, a)', False, 2), - ('(a, b, c)', False, 3), - ('(1, (2, 3))', True, 2), - ('(a, (b, c))', False, 2), - ('(1, [], {})', False, 3), - (repr(tuple(range(30))), True, 30), - ('(1, (2, (3, (4, (5)))))', True, 2) - ] - for expr, is_const, length in tests: - with self.subTest(expr=expr, is_const=is_const, length=length): - code = compile(expr, '', 'eval') - if is_const: - self.assertNotInBytecode(code, 'BUILD_TUPLE', length) - self.assertInBytecode(code, 'LOAD_CONST', eval(expr)) - else: - self.assertInBytecode(code, 'BUILD_TUPLE', length) - def test_elim_extra_return(self): # RETURN LOAD_CONST None RETURN --> RETURN def f(x):