bpo-17611. Move unwinding of stack for "pseudo exceptions" from interpreter to compiler.

Include/frameobject.h

@@ -9,8 +9,7 @@ extern "C" {
 #endif
 
 typedef struct {
-    int b_type;                 /* what kind of block this is */
-    int b_handler;              /* where to jump to find handler */
+    int b_handler;              /* where to jump to find handler, or -1 if in exception handler. */
     int b_level;                /* value stack level to pop to */
 } PyTryBlock;
 
@@ -65,7 +64,7 @@ PyFrameObject* _PyFrame_New_NoTrack(PyThreadState *, PyCodeObject *,
 
 /* Block management functions */
 
-PyAPI_FUNC(void) PyFrame_BlockSetup(PyFrameObject *, int, int, int);
+PyAPI_FUNC(void) PyFrame_BlockSetup(PyFrameObject *, int, int);
 PyAPI_FUNC(PyTryBlock *) PyFrame_BlockPop(PyFrameObject *);
 
 /* Extend the value stack */

Include/opcode.h

@@ -12,6 +12,7 @@ extern "C" {
 #define ROT_THREE                 3
 #define DUP_TOP                   4
 #define DUP_TOP_TWO               5
+#define ROT_FOUR                  6
 #define NOP                       9
 #define UNARY_POSITIVE           10
 #define UNARY_NEGATIVE           11
@@ -29,9 +30,11 @@ extern "C" {
 #define BINARY_TRUE_DIVIDE       27
 #define INPLACE_FLOOR_DIVIDE     28
 #define INPLACE_TRUE_DIVIDE      29
+#define WITH_EXCEPT_START        42
 #define GET_AITER                50
 #define GET_ANEXT                51
 #define BEFORE_ASYNC_WITH        52
+#define BEFORE_WITH              53
 #define INPLACE_ADD              55
 #define INPLACE_SUBTRACT         56
 #define INPLACE_MULTIPLY         57
@@ -50,20 +53,18 @@ extern "C" {
 #define LOAD_BUILD_CLASS         71
 #define YIELD_FROM               72
 #define GET_AWAITABLE            73
+#define END_FINALLY              74
 #define INPLACE_LSHIFT           75
 #define INPLACE_RSHIFT           76
 #define INPLACE_AND              77
 #define INPLACE_XOR              78
 #define INPLACE_OR               79
-#define BREAK_LOOP               80
-#define WITH_CLEANUP_START       81
-#define WITH_CLEANUP_FINISH      82
+#define RERAISE                  80
 #define RETURN_VALUE             83
 #define IMPORT_STAR              84
 #define SETUP_ANNOTATIONS        85
 #define YIELD_VALUE              86
 #define POP_BLOCK                87
-#define END_FINALLY              88
 #define POP_EXCEPT               89
 #define HAVE_ARGUMENT            90
 #define STORE_NAME               90
@@ -92,10 +93,9 @@ extern "C" {
 #define POP_JUMP_IF_FALSE       114
 #define POP_JUMP_IF_TRUE        115
 #define LOAD_GLOBAL             116
-#define CONTINUE_LOOP           119
-#define SETUP_LOOP              120
+#define END_ITER                117
 #define SETUP_EXCEPT            121
-#define SETUP_FINALLY           122
+#define SETUP_WITH              123
 #define LOAD_FAST               124
 #define STORE_FAST              125
 #define DELETE_FAST             126
@@ -110,7 +110,6 @@ extern "C" {
 #define DELETE_DEREF            138
 #define CALL_FUNCTION_KW        141
 #define CALL_FUNCTION_EX        142
-#define SETUP_WITH              143
 #define EXTENDED_ARG            144
 #define LIST_APPEND             145
 #define SET_ADD                 146
@@ -121,13 +120,14 @@ extern "C" {
 #define BUILD_MAP_UNPACK_WITH_CALL 151
 #define BUILD_TUPLE_UNPACK      152
 #define BUILD_SET_UNPACK        153
-#define SETUP_ASYNC_WITH        154
 #define FORMAT_VALUE            155
 #define BUILD_CONST_KEY_MAP     156
 #define BUILD_STRING            157
 #define BUILD_TUPLE_UNPACK_WITH_CALL 158
 #define LOAD_METHOD             160
 #define CALL_METHOD             161
+#define JUMP_FINALLY            162
+#define LOAD_ADDR               163
 
 /* EXCEPT_HANDLER is a special, implicit block type which is created when
    entering an except handler. It is not an opcode but we define it here

Lib/importlib/_bootstrap_external.py

@@ -242,7 +242,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.6rc1 3379 (more thorough __class__ validation #23722)
 #     Python 3.7a0  3390 (add LOAD_METHOD and CALL_METHOD opcodes)
 #     Python 3.7a0  3391 (update GET_AITER #31709)
-#     Python 3.7a0  3392 (PEP 552: Deterministic pycs)
+#     Python 3.7a4  3393 (bpo-17611: move frame block handling to compiler)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually

Lib/opcode.py

@@ -60,6 +60,7 @@ def jabs_op(name, op):
 def_op('ROT_THREE', 3)
 def_op('DUP_TOP', 4)
 def_op('DUP_TOP_TWO', 5)
+def_op('ROT_FOUR', 6)
 
 def_op('NOP', 9)
 def_op('UNARY_POSITIVE', 10)
@@ -83,9 +84,12 @@ def jabs_op(name, op):
 def_op('INPLACE_FLOOR_DIVIDE', 28)
 def_op('INPLACE_TRUE_DIVIDE', 29)
 
+def_op('WITH_EXCEPT_START', 42)
+
 def_op('GET_AITER', 50)
 def_op('GET_ANEXT', 51)
 def_op('BEFORE_ASYNC_WITH', 52)
+def_op('BEFORE_WITH', 53)
 
 def_op('INPLACE_ADD', 55)
 def_op('INPLACE_SUBTRACT', 56)
@@ -107,22 +111,20 @@ def jabs_op(name, op):
 def_op('LOAD_BUILD_CLASS', 71)
 def_op('YIELD_FROM', 72)
 def_op('GET_AWAITABLE', 73)
+def_op('END_FINALLY', 74)
 
 def_op('INPLACE_LSHIFT', 75)
 def_op('INPLACE_RSHIFT', 76)
 def_op('INPLACE_AND', 77)
 def_op('INPLACE_XOR', 78)
 def_op('INPLACE_OR', 79)
-def_op('BREAK_LOOP', 80)
-def_op('WITH_CLEANUP_START', 81)
-def_op('WITH_CLEANUP_FINISH', 82)
+def_op('RERAISE', 80)
 
 def_op('RETURN_VALUE', 83)
 def_op('IMPORT_STAR', 84)
 def_op('SETUP_ANNOTATIONS', 85)
 def_op('YIELD_VALUE', 86)
 def_op('POP_BLOCK', 87)
-def_op('END_FINALLY', 88)
 def_op('POP_EXCEPT', 89)
 
 HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
@@ -158,10 +160,10 @@ def jabs_op(name, op):
 
 name_op('LOAD_GLOBAL', 116)     # Index in name list
 
-jabs_op('CONTINUE_LOOP', 119)   # Target address
-jrel_op('SETUP_LOOP', 120)      # Distance to target address
-jrel_op('SETUP_EXCEPT', 121)    # ""
-jrel_op('SETUP_FINALLY', 122)   # ""
+jabs_op('END_ITER', 117)        # Target byte offset from beginning of code
+
+jrel_op('SETUP_EXCEPT', 121)    # Distance to target address
+jrel_op('SETUP_WITH', 123)   # ""
 
 def_op('LOAD_FAST', 124)        # Local variable number
 haslocal.append(124)
@@ -187,8 +189,6 @@ def jabs_op(name, op):
 def_op('CALL_FUNCTION_KW', 141)  # #args + #kwargs
 def_op('CALL_FUNCTION_EX', 142)  # Flags
 
-jrel_op('SETUP_WITH', 143)
-
 def_op('LIST_APPEND', 145)
 def_op('SET_ADD', 146)
 def_op('MAP_ADD', 147)
@@ -205,8 +205,6 @@ def jabs_op(name, op):
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
-jrel_op('SETUP_ASYNC_WITH', 154)
-
 def_op('FORMAT_VALUE', 155)
 def_op('BUILD_CONST_KEY_MAP', 156)
 def_op('BUILD_STRING', 157)
@@ -215,4 +213,7 @@ def jabs_op(name, op):
 name_op('LOAD_METHOD', 160)
 def_op('CALL_METHOD', 161)
 
+jrel_op('JUMP_FINALLY', 162)
+jrel_op('LOAD_ADDR', 163)
+
 del def_op, name_op, jrel_op, jabs_op

Lib/test/test_compile.py

@@ -671,6 +671,18 @@ def __fspath__(self):
 
         compile("42", PathLike("test_compile_pathlike"), "single")
 
+    def test_unwinding_in_nested_finallys(self):
+        code = """
+def foo():
+    try:
+        a
+    finally:
+        try:
+            b
+        finally:
+            return
+"""
+        compile(code, "<test>", "exec")
 
 class TestStackSize(unittest.TestCase):
     # These tests check that the computed stack size for a code object

Lib/test/test_peepholer.py

@@ -268,7 +268,7 @@ def f(cond1, cond2):
         self.assertNotInBytecode(f, 'JUMP_ABSOLUTE')
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
-        self.assertEqual(len(returns), 6)
+        self.assertLessEqual(len(returns), 6)
 
     def test_elim_jump_after_return2(self):
         # Eliminate dead code: jumps immediately after returns can't be reached
@@ -282,7 +282,7 @@ def f(cond1, cond2):
         self.assertEqual(len(returns), 1)
         returns = [instr for instr in dis.get_instructions(f)
                           if instr.opname == 'RETURN_VALUE']
-        self.assertEqual(len(returns), 2)
+        self.assertLessEqual(len(returns), 2)
 
     def test_make_function_doesnt_bail(self):
         def f():

Lib/test/test_sys.py

@@ -1015,7 +1015,7 @@ class C(object): pass
         nfrees = len(x.f_code.co_freevars)
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                   ncells + nfrees - 1
-        check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+        check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'2i' + 'P' + extras*'P'))
         # function
         def func(): pass
         check(func, size('12P'))

Lib/test/test_sys_settrace.py

@@ -187,7 +187,6 @@ def tightloop_example():
                             (1, 'line'),
                             (2, 'line'),
                             (3, 'line'),
-                            (4, 'line'),
                             (5, 'line'),
                             (5, 'line'),
                             (5, 'line'),
@@ -897,16 +896,16 @@ def test_no_jump_backwards_into_for_block(output):
             output.append(2)
         output.append(3)
 
-    @jump_test(2, 4, [], (ValueError, 'into'))
-    def test_no_jump_forwards_into_while_block(output):
+    @jump_test(2, 4, [4, 4])
+    def test_jump_forwards_into_while_block(output):
         i = 1
         output.append(2)
         while i <= 2:
             output.append(4)
             i += 1
 
-    @jump_test(5, 3, [3, 3], (ValueError, 'into'))
-    def test_no_jump_backwards_into_while_block(output):
+    @jump_test(5, 3, [3, 3, 3, 5])
+    def test_jump_backwards_into_while_block(output):
         i = 1
         while i <= 2:
             output.append(3)

Lib/test/test_with.py

@@ -602,6 +602,14 @@ def testWithRaise(self):
         else:
             self.fail("Didn't raise RuntimeError")
 
+    def testWithRaiseInExpr(self):
+        l = []
+        try:
+            m = mock_contextmanager_generator()
+            with m as l[0]:
+                pass
+        except IndexError:
+            self.assertTrue(m.exit_called)
 
 class AssignmentTargetTestCase(unittest.TestCase):