Merge branch 'main' into sqlite-expose-error-code

Author: Erlend E. Aasland

Include/internal/pycore_frame.h

@@ -29,10 +29,9 @@ typedef struct _interpreter_frame {
     PyObject *generator;
     struct _interpreter_frame *previous;
     int f_lasti;       /* Last instruction if called */
-    int stackdepth;  /* Depth of value stack */
-    int nlocalsplus;
-    PyFrameState f_state;       /* What state the frame is in */
-    PyObject *stack[1];
+    int stacktop;     /* Offset of TOS from localsplus  */
+    PyFrameState f_state;  /* What state the frame is in */
+    PyObject *localsplus[1];
 } InterpreterFrame;
 
 static inline int _PyFrame_IsRunnable(InterpreterFrame *f) {
@@ -47,6 +46,26 @@ static inline int _PyFrameHasCompleted(InterpreterFrame *f) {
     return f->f_state > FRAME_EXECUTING;
 }
 
+static inline PyObject **_PyFrame_Stackbase(InterpreterFrame *f) {
+    return f->localsplus + f->f_code->co_nlocalsplus;
+}
+
+static inline PyObject *_PyFrame_StackPeek(InterpreterFrame *f) {
+    assert(f->stacktop > f->f_code->co_nlocalsplus);
+    return f->localsplus[f->stacktop-1];
+}
+
+static inline PyObject *_PyFrame_StackPop(InterpreterFrame *f) {
+    assert(f->stacktop > f->f_code->co_nlocalsplus);
+    f->stacktop--;
+    return f->localsplus[f->stacktop];
+}
+
+static inline void _PyFrame_StackPush(InterpreterFrame *f, PyObject *value) {
+    f->localsplus[f->stacktop] = value;
+    f->stacktop++;
+}
+
 #define FRAME_SPECIALS_SIZE ((sizeof(InterpreterFrame)-1)/sizeof(PyObject *))
 
 InterpreterFrame *
@@ -61,8 +80,7 @@ _PyFrame_InitializeSpecials(
     frame->f_builtins = Py_NewRef(con->fc_builtins);
     frame->f_globals = Py_NewRef(con->fc_globals);
     frame->f_locals = Py_XNewRef(locals);
-    frame->nlocalsplus = nlocalsplus;
-    frame->stackdepth = 0;
+    frame->stacktop = nlocalsplus;
     frame->frame_obj = NULL;
     frame->generator = NULL;
     frame->f_lasti = -1;
@@ -75,7 +93,19 @@ _PyFrame_InitializeSpecials(
 static inline PyObject**
 _PyFrame_GetLocalsArray(InterpreterFrame *frame)
 {
-    return ((PyObject **)frame) - frame->nlocalsplus;
+    return frame->localsplus;
+}
+
+static inline PyObject**
+_PyFrame_GetStackPointer(InterpreterFrame *frame)
+{
+    return frame->localsplus+frame->stacktop;
+}
+
+static inline void
+_PyFrame_SetStackPointer(InterpreterFrame *frame, PyObject **stack_pointer)
+{
+    frame->stacktop = (int)(stack_pointer - frame->localsplus);
 }
 
 /* For use by _PyFrame_GetFrameObject

Lib/enum.py

@@ -1390,17 +1390,28 @@ def _power_of_two(value):
     return value == 2 ** _high_bit(value)
 
 def global_enum_repr(self):
-    return '%s.%s' % (self.__class__.__module__, self._name_)
+    """
+    use module.enum_name instead of class.enum_name
+
+    the module is the last module in case of a multi-module name
+    """
+    module = self.__class__.__module__.split('.')[-1]
+    return '%s.%s' % (module, self._name_)
 
 def global_flag_repr(self):
-    module = self.__class__.__module__
+    """
+    use module.flag_name instead of class.flag_name
+
+    the module is the last module in case of a multi-module name
+    """
+    module = self.__class__.__module__.split('.')[-1]
     cls_name = self.__class__.__name__
     if self._name_ is None:
-        return "%x" % (module, cls_name, self._value_)
+        return "%s.%s(0x%x)" % (module, cls_name, self._value_)
     if _is_single_bit(self):
         return '%s.%s' % (module, self._name_)
     if self._boundary_ is not FlagBoundary.KEEP:
-        return module + module.join(self.name.split('|'))
+        return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')])
     else:
         name = []
         for n in self._name_.split('|'):

Lib/sqlite3/test/dbapi.py

@@ -22,11 +22,17 @@
 
 import contextlib
 import sqlite3 as sqlite
+import subprocess
 import sys
 import threading
 import unittest
 
-from test.support import check_disallow_instantiation, threading_helper, bigmemtest
+from test.support import (
+    SHORT_TIMEOUT,
+    bigmemtest,
+    check_disallow_instantiation,
+    threading_helper,
+)
 from test.support.os_helper import TESTFN, unlink, temp_dir
 
 
@@ -1069,6 +1075,77 @@ def test_on_conflict_replace(self):
         self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')])
 
 
+class MultiprocessTests(unittest.TestCase):
+    CONNECTION_TIMEOUT = SHORT_TIMEOUT / 1000.  # Defaults to 30 ms
+
+    def tearDown(self):
+        unlink(TESTFN)
+
+    def test_ctx_mgr_rollback_if_commit_failed(self):
+        # bpo-27334: ctx manager does not rollback if commit fails
+        SCRIPT = f"""if 1:
+            import sqlite3
+            def wait():
+                print("started")
+                assert "database is locked" in input()
+
+            cx = sqlite3.connect("{TESTFN}", timeout={self.CONNECTION_TIMEOUT})
+            cx.create_function("wait", 0, wait)
+            with cx:
+                cx.execute("create table t(t)")
+            try:
+                # execute two transactions; both will try to lock the db
+                cx.executescript('''
+                    -- start a transaction and wait for parent
+                    begin transaction;
+                    select * from t;
+                    select wait();
+                    rollback;
+
+                    -- start a new transaction; would fail if parent holds lock
+                    begin transaction;
+                    select * from t;
+                    rollback;
+                ''')
+            finally:
+                cx.close()
+        """
+
+        # spawn child process
+        proc = subprocess.Popen(
+            [sys.executable, "-c", SCRIPT],
+            encoding="utf-8",
+            bufsize=0,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        self.addCleanup(proc.communicate)
+
+        # wait for child process to start
+        self.assertEqual("started", proc.stdout.readline().strip())
+
+        cx = sqlite.connect(TESTFN, timeout=self.CONNECTION_TIMEOUT)
+        try:  # context manager should correctly release the db lock
+            with cx:
+                cx.execute("insert into t values('test')")
+        except sqlite.OperationalError as exc:
+            proc.stdin.write(str(exc))
+        else:
+            proc.stdin.write("no error")
+        finally:
+            cx.close()
+
+        # terminate child process
+        self.assertIsNone(proc.returncode)
+        try:
+            proc.communicate(input="end", timeout=SHORT_TIMEOUT)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.communicate()
+            raise
+        self.assertEqual(proc.returncode, 0)
+
+
 def suite():
     tests = [
         ClosedConTests,
@@ -1078,6 +1155,7 @@ def suite():
         CursorTests,
         ExtensionTests,
         ModuleTests,
+        MultiprocessTests,
         OpenTests,
         SqliteOnConflictTests,
         ThreadTests,

Lib/test/test_enum.py

@@ -28,6 +28,9 @@ def load_tests(loader, tests, ignore):
                 ))
     return tests
 
+MODULE = ('test.test_enum', '__main__')[__name__=='__main__']
+SHORT_MODULE = MODULE.split('.')[-1]
+
 # for pickle tests
 try:
     class Stooges(Enum):
@@ -143,6 +146,23 @@ def __init__(self, fget=None, fset=None, fdel=None, doc=None):
     def __get__(self, instance, ownerclass):
         return self.fget(ownerclass)
 
+# for global repr tests
+
+@enum.global_enum
+class HeadlightsK(IntFlag, boundary=enum.KEEP):
+    OFF_K = 0
+    LOW_BEAM_K = auto()
+    HIGH_BEAM_K = auto()
+    FOG_K = auto()
+
+
+@enum.global_enum
+class HeadlightsC(IntFlag, boundary=enum.CONFORM):
+    OFF_C = 0
+    LOW_BEAM_C = auto()
+    HIGH_BEAM_C = auto()
+    FOG_C = auto()
+
 
 # tests
 
@@ -3224,6 +3244,34 @@ def test_repr(self):
         self.assertEqual(repr(~(Open.WO | Open.CE)), 'Open.RW')
         self.assertEqual(repr(Open(~4)), '-5')
 
+    def test_global_repr_keep(self):
+        self.assertEqual(
+                repr(HeadlightsK(0)),
+                '%s.OFF_K' % SHORT_MODULE,
+                )
+        self.assertEqual(
+                repr(HeadlightsK(2**0 + 2**2 + 2**3)),
+                '%(m)s.LOW_BEAM_K|%(m)s.FOG_K|0x8' % {'m': SHORT_MODULE},
+                )
+        self.assertEqual(
+                repr(HeadlightsK(2**3)),
+                '%(m)s.HeadlightsK(0x8)' % {'m': SHORT_MODULE},
+                )
+
+    def test_global_repr_conform1(self):
+        self.assertEqual(
+                repr(HeadlightsC(0)),
+                '%s.OFF_C' % SHORT_MODULE,
+                )
+        self.assertEqual(
+                repr(HeadlightsC(2**0 + 2**2 + 2**3)),
+                '%(m)s.LOW_BEAM_C|%(m)s.FOG_C' % {'m': SHORT_MODULE},
+                )
+        self.assertEqual(
+                repr(HeadlightsC(2**3)),
+                '%(m)s.OFF_C' % {'m': SHORT_MODULE},
+                )
+
     def test_format(self):
         Perm = self.Perm
         self.assertEqual(format(Perm.R, ''), '4')
@@ -4085,7 +4133,7 @@ def setUp(self):
     def test_convert_value_lookup_priority(self):
         test_type = enum.IntEnum._convert_(
                 'UnittestConvert',
-                ('test.test_enum', '__main__')[__name__=='__main__'],
+                MODULE,
                 filter=lambda x: x.startswith('CONVERT_TEST_'))
         # We don't want the reverse lookup value to vary when there are
         # multiple possible names for a given value.  It should always
@@ -4095,7 +4143,7 @@ def test_convert_value_lookup_priority(self):
     def test_convert(self):
         test_type = enum.IntEnum._convert_(
                 'UnittestConvert',
-                ('test.test_enum', '__main__')[__name__=='__main__'],
+                MODULE,
                 filter=lambda x: x.startswith('CONVERT_TEST_'))
         # Ensure that test_type has all of the desired names and values.
         self.assertEqual(test_type.CONVERT_TEST_NAME_F,
@@ -4115,7 +4163,7 @@ def test_convert_warn(self):
         with self.assertWarns(DeprecationWarning):
             enum.IntEnum._convert(
                 'UnittestConvert',
-                ('test.test_enum', '__main__')[__name__=='__main__'],
+                MODULE,
                 filter=lambda x: x.startswith('CONVERT_TEST_'))
 
     @unittest.skipUnless(python_version >= (3, 9),
@@ -4124,16 +4172,15 @@ def test_convert_raise(self):
         with self.assertRaises(AttributeError):
             enum.IntEnum._convert(
                 'UnittestConvert',
-                ('test.test_enum', '__main__')[__name__=='__main__'],
+                MODULE,
                 filter=lambda x: x.startswith('CONVERT_TEST_'))
 
     def test_convert_repr_and_str(self):
-        module = ('test.test_enum', '__main__')[__name__=='__main__']
         test_type = enum.IntEnum._convert_(
                 'UnittestConvert',
-                module,
+                MODULE,
                 filter=lambda x: x.startswith('CONVERT_STRING_TEST_'))
-        self.assertEqual(repr(test_type.CONVERT_STRING_TEST_NAME_A), '%s.CONVERT_STRING_TEST_NAME_A' % module)
+        self.assertEqual(repr(test_type.CONVERT_STRING_TEST_NAME_A), '%s.CONVERT_STRING_TEST_NAME_A' % SHORT_MODULE)
         self.assertEqual(str(test_type.CONVERT_STRING_TEST_NAME_A), 'CONVERT_STRING_TEST_NAME_A')
         self.assertEqual(format(test_type.CONVERT_STRING_TEST_NAME_A), '5')
 
@@ -4151,7 +4198,7 @@ def setUp(self):
     def test_convert(self):
         test_type = enum.StrEnum._convert_(
                 'UnittestConvert',
-                ('test.test_enum', '__main__')[__name__=='__main__'],
+                MODULE,
                 filter=lambda x: x.startswith('CONVERT_STR_'))
         # Ensure that test_type has all of the desired names and values.
         self.assertEqual(test_type.CONVERT_STR_TEST_1, 'hello')
@@ -4162,12 +4209,11 @@ def test_convert(self):
                          [], msg='Names other than CONVERT_STR_* found.')
 
     def test_convert_repr_and_str(self):
-        module = ('test.test_enum', '__main__')[__name__=='__main__']
         test_type = enum.StrEnum._convert_(
                 'UnittestConvert',
-                module,
+                MODULE,
                 filter=lambda x: x.startswith('CONVERT_STR_'))
-        self.assertEqual(repr(test_type.CONVERT_STR_TEST_1), '%s.CONVERT_STR_TEST_1' % module)
+        self.assertEqual(repr(test_type.CONVERT_STR_TEST_1), '%s.CONVERT_STR_TEST_1' % SHORT_MODULE)
         self.assertEqual(str(test_type.CONVERT_STR_TEST_2), 'goodbye')
         self.assertEqual(format(test_type.CONVERT_STR_TEST_1), 'hello')
 

Lib/test/test_marshal.py

@@ -344,6 +344,31 @@ def test_eof(self):
         for i in range(len(data)):
             self.assertRaises(EOFError, marshal.loads, data[0: i])
 
+    def test_deterministic_sets(self):
+        # bpo-37596: To support reproducible builds, sets and frozensets need to
+        # have their elements serialized in a consistent order (even when they
+        # have been scrambled by hash randomization):
+        for kind in ("set", "frozenset"):
+            for elements in (
+                "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'",
+                # Also test for bad interactions with backreferencing:
+                "('string', 1), ('string', 2), ('string', 3)",
+            ):
+                s = f"{kind}([{elements}])"
+                with self.subTest(s):
+                    # First, make sure that our test case still has different
+                    # orders under hash seeds 0 and 1. If this check fails, we
+                    # need to update this test with different elements:
+                    args = ["-c", f"print({s})"]
+                    _, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
+                    _, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
+                    self.assertNotEqual(repr_0, repr_1)
+                    # Then, perform the actual test:
+                    args = ["-c", f"import marshal; print(marshal.dumps({s}))"]
+                    _, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
+                    _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
+                    self.assertEqual(dump_0, dump_1)
+
 LARGE_SIZE = 2**31
 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
 

Misc/NEWS.d/next/Core and Builtins/2021-08-16-23-16-17.bpo-44929.qpMEky.rst

@@ -0,0 +1,2 @@
+Fix some edge cases of ``enum.Flag`` string representation in the REPL.
+Patch by Pablo Galindo.

Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst

@@ -0,0 +1,2 @@
+The :mod:`sqlite3` context manager now performs a rollback (thus releasing the
+database lock) if commit failed.  Patch by Luca Citi and Erlend E. Aasland.

Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst

@@ -0,0 +1,2 @@
+Ensure that :class:`set` and :class:`frozenset` objects are always
+:mod:`marshalled <marshal>` reproducibly.