Produce cleaner bytecode for 'with' and 'async with' by generating separate code for normal and exceptional paths.

Remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY and POP_FINALLY bytecodes. Implement finally blocks by code duplication.
Reimplement frame.lineno setter using line numbers rather than bytecode offsets.

Author: markshannon

Doc/library/dis.rst

@@ -695,50 +695,21 @@ iterations of the loop.
    popped values are used to restore the exception state.
 
 
-.. opcode:: POP_FINALLY (preserve_tos)
+.. opcode:: RERAISE
 
-   Cleans up the value stack and the block stack.  If *preserve_tos* is not
-   ``0`` TOS first is popped from the stack and pushed on the stack after
-   perfoming other stack operations:
+    Re-raises the exception currently on top of the stack.
 
-   * If TOS is ``NULL`` or an integer (pushed by :opcode:`BEGIN_FINALLY`
-     or :opcode:`CALL_FINALLY`) it is popped from the stack.
-   * If TOS is an exception type (pushed when an exception has been raised)
-     6 values are popped from the stack, the last three popped values are
-     used to restore the exception state.  An exception handler block is
-     removed from the block stack.
+    .. versionadded:: 3.8
 
-   It is similar to :opcode:`END_FINALLY`, but doesn't change the bytecode
-   counter nor raise an exception.  Used for implementing :keyword:`break`,
-   :keyword:`continue` and :keyword:`return` in the :keyword:`finally` block.
 
-   .. versionadded:: 3.8
-
-
-.. opcode:: BEGIN_FINALLY
-
-   Pushes ``NULL`` onto the stack for using it in :opcode:`END_FINALLY`,
-   :opcode:`POP_FINALLY`, :opcode:`WITH_CLEANUP_START` and
-   :opcode:`WITH_CLEANUP_FINISH`.  Starts the :keyword:`finally` block.
-
-   .. versionadded:: 3.8
+.. opcode:: WITH_EXCEPT_START
 
+    Calls the function in position 7 on the stack with the top three
+    items on the stack as arguments.
+    Used to implement the call ``exit(*exc_info())`` when an exception
+    has occurred in a :keyword:`with` statement.
 
-.. opcode:: END_FINALLY
-
-   Terminates a :keyword:`finally` clause.  The interpreter recalls whether the
-   exception has to be re-raised or execution has to be continued depending on
-   the value of TOS.
-
-   * If TOS is ``NULL`` (pushed by :opcode:`BEGIN_FINALLY`) continue from
-     the next instruction. TOS is popped.
-   * If TOS is an integer (pushed by :opcode:`CALL_FINALLY`), sets the
-     bytecode counter to TOS.  TOS is popped.
-   * If TOS is an exception type (pushed when an exception has been raised)
-     6 values are popped from the stack, the first three popped values are
-     used to re-raise the exception and the last three popped values are used
-     to restore the exception state.  An exception handler block is removed
-     from the block stack.
+    .. versionadded:: 3.8
 
 
 .. opcode:: LOAD_BUILD_CLASS
@@ -761,35 +732,6 @@ iterations of the loop.
    .. versionadded:: 3.2
 
 
-.. opcode:: WITH_CLEANUP_START
-
-   Starts cleaning up the stack when a :keyword:`with` statement block exits.
-
-   At the top of the stack are either ``NULL`` (pushed by
-   :opcode:`BEGIN_FINALLY`) or 6 values pushed if an exception has been
-   raised in the with block.  Below is the context manager's
-   :meth:`~object.__exit__` or :meth:`~object.__aexit__` bound method.
-
-   If TOS is ``NULL``, calls ``SECOND(None, None, None)``,
-   removes the function from the stack, leaving TOS, and pushes ``None``
-   to the stack.  Otherwise calls ``SEVENTH(TOP, SECOND, THIRD)``,
-   shifts the bottom 3 values of the stack down, replaces the empty spot
-   with ``NULL`` and pushes TOS.  Finally pushes the result of the call.
-
-
-.. opcode:: WITH_CLEANUP_FINISH
-
-   Finishes cleaning up the stack when a :keyword:`with` statement block exits.
-
-   TOS is result of ``__exit__()`` or ``__aexit__()`` function call pushed
-   by :opcode:`WITH_CLEANUP_START`.  SECOND is ``None`` or an exception type
-   (pushed when an exception has been raised).
-
-   Pops two values from the stack.  If SECOND is not None and TOS is true
-   unwinds the EXCEPT_HANDLER block which was created when the exception
-   was caught and pushes ``NULL`` to the stack.
-
-
 All of the following opcodes use their arguments.
 
 .. opcode:: STORE_NAME (namei)
@@ -1041,15 +983,6 @@ All of the following opcodes use their arguments.
    stack.  *delta* points to the finally block or the first except block.
 
 
-.. opcode:: CALL_FINALLY (delta)
-
-   Pushes the address of the next instruction onto the stack and increments
-   bytecode counter by *delta*.  Used for calling the finally block as a
-   "subroutine".
-
-   .. versionadded:: 3.8
-
-
 .. opcode:: LOAD_FAST (var_num)
 
    Pushes a reference to the local ``co_varnames[var_num]`` onto the stack.

Include/opcode.h

@@ -259,6 +259,8 @@ def _write_atomic(path, data, mode=0o666):
 #                         this might affected the first line number #32911)
 #     Python 3.8a1  3400 (move frame block handling to compiler #17611)
 #     Python 3.8a1  3401 (add END_ASYNC_FOR #33041)
+#     Python 3.8a1  3402 (simplified bytecode for with blocks #32949)
+#     Python 3.8a1  3403 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -267,7 +269,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3401).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3403).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/importlib/_bootstrap_external.py

@@ -84,10 +84,12 @@ def jabs_op(name, op):
 def_op('INPLACE_FLOOR_DIVIDE', 28)
 def_op('INPLACE_TRUE_DIVIDE', 29)
 
+def_op('RERAISE', 48)
+def_op('WITH_EXCEPT_START', 49)
 def_op('GET_AITER', 50)
 def_op('GET_ANEXT', 51)
 def_op('BEFORE_ASYNC_WITH', 52)
-def_op('BEGIN_FINALLY', 53)
+
 def_op('END_ASYNC_FOR', 54)
 def_op('INPLACE_ADD', 55)
 def_op('INPLACE_SUBTRACT', 56)
@@ -115,14 +117,13 @@ def jabs_op(name, op):
 def_op('INPLACE_AND', 77)
 def_op('INPLACE_XOR', 78)
 def_op('INPLACE_OR', 79)
-def_op('WITH_CLEANUP_START', 81)
-def_op('WITH_CLEANUP_FINISH', 82)
+
 def_op('RETURN_VALUE', 83)
 def_op('IMPORT_STAR', 84)
 def_op('SETUP_ANNOTATIONS', 85)
 def_op('YIELD_VALUE', 86)
 def_op('POP_BLOCK', 87)
-def_op('END_FINALLY', 88)
+
 def_op('POP_EXCEPT', 89)
 
 HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
@@ -210,7 +211,5 @@ def jabs_op(name, op):
 
 name_op('LOAD_METHOD', 160)
 def_op('CALL_METHOD', 161)
-jrel_op('CALL_FINALLY', 162)
-def_op('POP_FINALLY', 163)
 
 del def_op, name_op, jrel_op, jabs_op

Lib/opcode.py

@@ -274,32 +274,34 @@ def bug1333982(x=[]):
     -->       6 BINARY_TRUE_DIVIDE
               8 POP_TOP
              10 POP_BLOCK
-             12 JUMP_FORWARD            40 (to 54)
+             12 JUMP_FORWARD            44 (to 58)
 
 %3d     >>   14 DUP_TOP
              16 LOAD_GLOBAL              0 (Exception)
              18 COMPARE_OP              10 (exception match)
-             20 POP_JUMP_IF_FALSE       52
+             20 POP_JUMP_IF_FALSE       56
              22 POP_TOP
              24 STORE_FAST               0 (e)
              26 POP_TOP
-             28 SETUP_FINALLY           10 (to 40)
+             28 SETUP_FINALLY           18 (to 48)
 
 %3d          30 LOAD_FAST                0 (e)
              32 LOAD_ATTR                1 (__traceback__)
              34 STORE_FAST               1 (tb)
              36 POP_BLOCK
-             38 BEGIN_FINALLY
-        >>   40 LOAD_CONST               0 (None)
+             38 POP_EXCEPT
+             40 LOAD_CONST               0 (None)
              42 STORE_FAST               0 (e)
              44 DELETE_FAST              0 (e)
-             46 END_FINALLY
-             48 POP_EXCEPT
-             50 JUMP_FORWARD             2 (to 54)
-        >>   52 END_FINALLY
-
-%3d     >>   54 LOAD_FAST                1 (tb)
-             56 RETURN_VALUE
+             46 JUMP_FORWARD            10 (to 58)
+        >>   48 LOAD_CONST               0 (None)
+             50 STORE_FAST               0 (e)
+             52 DELETE_FAST              0 (e)
+             54 RERAISE
+        >>   56 RERAISE
+
+%3d     >>   58 LOAD_FAST                1 (tb)
+             60 RETURN_VALUE
 """ % (TRACEBACK_CODE.co_firstlineno + 1,
        TRACEBACK_CODE.co_firstlineno + 2,
        TRACEBACK_CODE.co_firstlineno + 3,
@@ -740,7 +742,7 @@ async def async_def():
 Argument count:    0
 Kw-only arguments: 0
 Number of locals:  2
-Stack size:        10
+Stack size:        9
 Flags:             OPTIMIZED, NEWLOCALS, NOFREE, COROUTINE
 Constants:
    0: None
@@ -965,7 +967,7 @@ def jumpy():
   Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=96, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=98, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=100, starts_line=None, is_jump_target=False),
-  Instruction(opname='SETUP_FINALLY', opcode=122, arg=70, argval=174, argrepr='to 174', offset=102, starts_line=20, is_jump_target=True),
+  Instruction(opname='SETUP_FINALLY', opcode=122, arg=98, argval=202, argrepr='to 202', offset=102, starts_line=20, is_jump_target=True),
   Instruction(opname='SETUP_FINALLY', opcode=122, arg=12, argval=118, argrepr='to 118', offset=104, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=106, starts_line=21, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=108, starts_line=None, is_jump_target=False),
@@ -985,29 +987,43 @@ def jumpy():
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=136, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=138, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=140, starts_line=None, is_jump_target=False),
-  Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=170, argrepr='to 170', offset=142, starts_line=None, is_jump_target=False),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=144, starts_line=None, is_jump_target=True),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=46, argval=190, argrepr='to 190', offset=142, starts_line=None, is_jump_target=False),
+  Instruction(opname='RERAISE', opcode=48, arg=None, argval=None, argrepr='', offset=144, starts_line=None, is_jump_target=True),
   Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=146, starts_line=25, is_jump_target=True),
-  Instruction(opname='SETUP_WITH', opcode=143, arg=14, argval=164, argrepr='to 164', offset=148, starts_line=None, is_jump_target=False),
+  Instruction(opname='SETUP_WITH', opcode=143, arg=24, argval=174, argrepr='to 174', offset=148, starts_line=None, is_jump_target=False),
   Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=150, starts_line=None, is_jump_target=False),
   Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=152, starts_line=26, is_jump_target=False),
   Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=154, starts_line=None, is_jump_target=False),
   Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=156, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=158, starts_line=None, is_jump_target=False),
   Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=160, starts_line=None, is_jump_target=False),
-  Instruction(opname='BEGIN_FINALLY', opcode=53, arg=None, argval=None, argrepr='', offset=162, starts_line=None, is_jump_target=False),
-  Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=164, starts_line=None, is_jump_target=True),
-  Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=168, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=170, starts_line=None, is_jump_target=True),
-  Instruction(opname='BEGIN_FINALLY', opcode=53, arg=None, argval=None, argrepr='', offset=172, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=174, starts_line=28, is_jump_target=True),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=176, starts_line=None, is_jump_target=False),
-  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=178, starts_line=None, is_jump_target=False),
-  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=180, starts_line=None, is_jump_target=False),
-  Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
-  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=184, starts_line=None, is_jump_target=False),
-  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=186, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=162, starts_line=None, is_jump_target=False),
+  Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=164, starts_line=None, is_jump_target=False),
+  Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=3, argval=3, argrepr='', offset=168, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=170, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=16, argval=190, argrepr='to 190', offset=172, starts_line=None, is_jump_target=False),
+  Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=174, starts_line=None, is_jump_target=True),
+  Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=180, argval=180, argrepr='', offset=176, starts_line=None, is_jump_target=False),
+  Instruction(opname='RERAISE', opcode=48, arg=None, argval=None, argrepr='', offset=178, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=180, starts_line=None, is_jump_target=True),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=184, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=186, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=188, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=190, starts_line=None, is_jump_target=True),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=192, starts_line=28, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=194, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=196, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=198, starts_line=None, is_jump_target=False),
+  Instruction(opname='JUMP_FORWARD', opcode=110, arg=10, argval=212, argrepr='to 212', offset=200, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=202, starts_line=None, is_jump_target=True),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=204, starts_line=None, is_jump_target=False),
+  Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='', offset=206, starts_line=None, is_jump_target=False),
+  Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=208, starts_line=None, is_jump_target=False),
+  Instruction(opname='RERAISE', opcode=48, arg=None, argval=None, argrepr='', offset=210, starts_line=None, is_jump_target=False),
+  Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=212, starts_line=None, is_jump_target=True),
+  Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=214, starts_line=None, is_jump_target=False)
 ]
 
 # One last piece of inspect fodder to check the default line number handling
@@ -1020,6 +1036,10 @@ def simple(): pass
 
 class InstructionTests(BytecodeTestCase):
 
+    def __init__(self, *args):
+        super().__init__(*args)
+        self.maxDiff = None
+
     def test_default_first_line(self):
         actual = dis.get_instructions(simple)
         self.assertEqual(list(actual), expected_opinfo_simple)
@@ -1051,6 +1071,7 @@ def test_jumpy(self):
 # get_instructions has its own tests above, so can rely on it to validate
 # the object oriented API
 class BytecodeTests(unittest.TestCase):
+
     def test_instantiation(self):
         # Test with function, method, code string and code object
         for obj in [_f, _C(1).__init__, "a=1", _f.__code__]:

Lib/test/test_dis.py

@@ -0,0 +1,5 @@
+Removed WITH_CLEANUP_START, WITH_CLEANUP_FINISH, BEGIN_FINALLY, END_FINALLY, CALL_FINALLY and POP_FINALLY bytecodes.
+Replaced with RERAISE and WITH_EXCEPT_FINISH bytecodes.
+The compiler now generates different code for exceptional and non-exceptional branches for 'with' and 'try-except'
+statements. For 'try-finally' statements the 'finally' block is replicated for each exit from the 'try' body.
+