bpo-43916: Add _PyType_DisabledNew to prevent new heap types from being created uninitialised

Include/internal/pycore_object.h

@@ -29,6 +29,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
 
 extern void _PyType_InitCache(PyInterpreterState *interp);
 
+PyAPI_FUNC(PyObject *) _PyType_DisabledNew(PyTypeObject *type, PyObject *args,
+                                           PyObject *kwds);
+
 
 /* Inline functions trading binary compatibility for speed:
    _PyObject_Init() is the fast version of PyObject_Init(), and

Lib/test/test_array.py

@@ -40,6 +40,12 @@ def test_bad_constructor(self):
         self.assertRaises(TypeError, array.array, 'xx')
         self.assertRaises(ValueError, array.array, 'x')
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        tp = type(iter(array.array('I')))
+        self.assertRaises(TypeError, tp)
+
     @support.cpython_only
     def test_immutable(self):
         # bpo-43908: check that array.array is immutable

Lib/test/test_curses.py

@@ -6,7 +6,7 @@
 import tempfile
 import unittest
 
-from test.support import requires, verbose, SaveSignals
+from test.support import requires, verbose, SaveSignals, cpython_only
 from test.support.import_helper import import_module
 
 # Optionally test curses module.  This currently requires that the
@@ -1046,6 +1046,7 @@ def __del__(self):
         panel.set_userptr(A())
         panel.set_userptr(None)
 
+    @cpython_only
     @requires_curses_func('panel')
     def test_new_curses_panel(self):
         w = curses.newwin(10, 10)

Lib/test/test_dbm_gnu.py

@@ -1,5 +1,5 @@
 from test import support
-from test.support import import_helper
+from test.support import import_helper, cpython_only
 gdbm = import_helper.import_module("dbm.gnu") #skip if not supported
 import unittest
 import os
@@ -27,6 +27,13 @@ def tearDown(self):
             self.g.close()
         unlink(filename)
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.g = gdbm.open(filename, 'c')
+        tp = type(self.g)
+        self.assertRaises(TypeError, tp)
+
     def test_key_methods(self):
         self.g = gdbm.open(filename, 'c')
         self.assertEqual(self.g.keys(), [])

Lib/test/test_functools.py

@@ -948,6 +948,12 @@ class TestCmpToKeyC(TestCmpToKey, unittest.TestCase):
     if c_functools:
         cmp_to_key = c_functools.cmp_to_key
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        tp = type(c_functools.cmp_to_key(None))
+        self.assertRaises(TypeError, tp)
+
 
 class TestCmpToKeyPy(TestCmpToKey, unittest.TestCase):
     cmp_to_key = staticmethod(py_functools.cmp_to_key)

Lib/test/test_posix.py

@@ -58,6 +58,13 @@ def tearDown(self):
             os_helper.unlink(teardown_file)
         self._warnings_manager.__exit__(None, None, None)
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.assertRaises(TypeError, posix.DirEntry)
+        tp = type(posix.scandir())
+        self.assertRaises(TypeError, tp)
+
     def testNoArgFunctions(self):
         # test posix functions which take no arguments and have
         # no side-effects which we need to cleanup (e.g., fork, wait, abort)

Lib/test/test_re.py

@@ -2215,6 +2215,15 @@ def test_signedness(self):
         self.assertGreaterEqual(sre_compile.MAXREPEAT, 0)
         self.assertGreaterEqual(sre_compile.MAXGROUPS, 0)
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.assertRaises(TypeError, re.Match)
+        self.assertRaises(TypeError, re.Pattern)
+        pat = re.compile("")
+        tp = type(pat.scanner(""))
+        self.assertRaises(TypeError, tp)
+
 
 class ExternalTests(unittest.TestCase):
 

Lib/test/test_tcl.py

@@ -736,8 +736,11 @@ def check(value):
         check('{\n')
         check('}\n')
 
+    @support.cpython_only
     def test_new_tcl_obj(self):
         self.assertRaises(TypeError, _tkinter.Tcl_Obj)
+        self.assertRaises(TypeError, _tkinter.TkttType)
+        self.assertRaises(TypeError, _tkinter.TkappType)
 
 class BigmemTclTest(unittest.TestCase):
 

Lib/test/test_threading.py

@@ -119,6 +119,13 @@ def func(): pass
             thread = threading.Thread(target=func)
             self.assertEqual(thread.name, "Thread-5 (func)")
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        lock = threading.Lock()
+        tp = type(lock)
+        self.assertRaises(TypeError, tp)
+
     # Create a bunch of threads, let each do some work, wait until all are
     # done.
     def test_various_ops(self):

Lib/test/test_unicodedata.py

@@ -11,7 +11,8 @@
 import sys
 import unicodedata
 import unittest
-from test.support import open_urlresource, requires_resource, script_helper
+from test.support import (open_urlresource, requires_resource, script_helper,
+                          cpython_only)
 
 
 class UnicodeMethodsTest(unittest.TestCase):
@@ -225,6 +226,11 @@ def test_east_asian_width_9_0_changes(self):
 
 class UnicodeMiscTest(UnicodeDatabaseTest):
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.assertRaises(TypeError, unicodedata.UCD)
+
     def test_failed_import_during_compiling(self):
         # Issue 4367
         # Decoding \N escapes requires the unicodedata module. If it can't be

Lib/test/test_zlib.py

@@ -129,6 +129,14 @@ def test_overflow(self):
         with self.assertRaisesRegex(OverflowError, 'int too large'):
             zlib.decompressobj().flush(sys.maxsize + 1)
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        comp_type = type(zlib.compressobj())
+        decomp_type = type(zlib.decompressobj())
+        self.assertRaises(TypeError, comp_type)
+        self.assertRaises(TypeError, decomp_type)
+
 
 class BaseCompressTestCase(object):
     def check_big_compress_buffer(self, size, compress_func):

Misc/NEWS.d/next/Core and Builtins/2021-04-28-10-09-16.bpo-43916.ecedNo.rst

@@ -0,0 +1,7 @@
+Fix regressions for extension modules that have been converted to :ref:`heap
+types <heap-types>`. Inheritance differs for *static types* and *heap
+types*: A *static type* with its :c:member:`~PyTypeObject.tp_new` set to
+``NULL`` does not have a public contructor, but *heap types* inherit
+contructors from their base classes, resulting in it being possible to
+create instances without proper initialisation, possibly resulting in a
+crash.

Modules/_curses_panel.c

@@ -13,6 +13,7 @@ static const char PyCursesVersion[] = "2.1";
 #include "Python.h"
 
 #include "py_curses.h"
+#include "pycore_object.h"        // _PyType_DisabledNew()
 
 #include <panel.h>
 
@@ -514,6 +515,7 @@ static PyMethodDef PyCursesPanel_Methods[] = {
 static PyType_Slot PyCursesPanel_Type_slots[] = {
     {Py_tp_dealloc, PyCursesPanel_Dealloc},
     {Py_tp_methods, PyCursesPanel_Methods},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0},
 };
 
@@ -656,7 +658,6 @@ _curses_panel_exec(PyObject *mod)
     if (state->PyCursesPanel_Type == NULL) {
         return -1;
     }
-    ((PyTypeObject *)state->PyCursesPanel_Type)->tp_new = NULL;
 
     if (PyModule_AddType(mod, state->PyCursesPanel_Type) < 0) {
         return -1;

Modules/_dbmmodule.c

@@ -4,6 +4,7 @@
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
+#include "pycore_object.h"        // _PyType_DisabledNew()
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -397,6 +398,7 @@ static PyMethodDef dbm_methods[] = {
 };
 
 static PyType_Slot dbmtype_spec_slots[] = {
+    {Py_tp_new, _PyType_DisabledNew},
     {Py_tp_dealloc, dbm_dealloc},
     {Py_tp_methods, dbm_methods},
     {Py_sq_contains, dbm_contains},

Modules/_functoolsmodule.c

@@ -540,6 +540,7 @@ static PyType_Slot keyobject_type_slots[] = {
     {Py_tp_clear, keyobject_clear},
     {Py_tp_richcompare, keyobject_richcompare},
     {Py_tp_members, keyobject_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 
@@ -760,6 +761,7 @@ lru_list_elem_dealloc(lru_list_elem *link)
 
 static PyType_Slot lru_list_elem_type_slots[] = {
     {Py_tp_dealloc, lru_list_elem_dealloc},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 

Modules/_gdbmmodule.c

@@ -5,6 +5,7 @@
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
+#include "pycore_object.h"        // _PyType_DisabledNew()
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -560,6 +561,7 @@ static PyType_Slot gdbmtype_spec_slots[] = {
     {Py_mp_subscript, gdbm_subscript},
     {Py_mp_ass_subscript, gdbm_ass_sub},
     {Py_tp_doc, (char*)gdbm_object__doc__},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 

Modules/_sre.c

@@ -43,6 +43,7 @@ static const char copyright[] =
 #include "Python.h"
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
+#include "pycore_object.h"        // _PyType_DisabledNew()
 #include "structmember.h"         // PyMemberDef
 
 #include "sre.h"
@@ -2683,6 +2684,7 @@ static PyType_Slot pattern_slots[] = {
     {Py_tp_methods, pattern_methods},
     {Py_tp_members, pattern_members},
     {Py_tp_getset, pattern_getset},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, NULL},
 };
 
@@ -2740,6 +2742,7 @@ static PyType_Slot match_slots[] = {
     {Py_tp_methods, match_methods},
     {Py_tp_members, match_members},
     {Py_tp_getset, match_getset},
+    {Py_tp_new, _PyType_DisabledNew},
 
     /* As mapping.
      *
@@ -2775,6 +2778,7 @@ static PyType_Slot scanner_slots[] = {
     {Py_tp_dealloc, scanner_dealloc},
     {Py_tp_methods, scanner_methods},
     {Py_tp_members, scanner_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, NULL},
 };
 

Modules/_threadmodule.c

@@ -5,6 +5,7 @@
 #include "Python.h"
 #include "pycore_interp.h"        // _PyInterpreterState.num_threads
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
+#include "pycore_object.h"        // _PyType_DisabledNew()
 #include "pycore_pylifecycle.h"
 #include "pycore_pystate.h"       // _PyThreadState_Init()
 #include <stddef.h>               // offsetof()
@@ -306,6 +307,7 @@ static PyType_Slot lock_type_slots[] = {
     {Py_tp_methods, lock_methods},
     {Py_tp_traverse, lock_traverse},
     {Py_tp_members, lock_type_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 
@@ -677,6 +679,7 @@ static PyType_Slot local_dummy_type_slots[] = {
     {Py_tp_dealloc, (destructor)localdummy_dealloc},
     {Py_tp_doc, "Thread-local dummy"},
     {Py_tp_members, local_dummy_type_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 

Modules/_tkinter.c

@@ -24,6 +24,7 @@ Copyright (C) 1994 Steen Lumholt.
 #define PY_SSIZE_T_CLEAN
 
 #include "Python.h"
+#include "pycore_object.h"        // _PyType_DisabledNew()
 #include <ctype.h>
 
 #ifdef MS_WINDOWS
@@ -995,6 +996,7 @@ static PyType_Slot PyTclObject_Type_slots[] = {
     {Py_tp_getattro, PyObject_GenericGetAttr},
     {Py_tp_richcompare, PyTclObject_richcompare},
     {Py_tp_getset, PyTclObject_getsetlist},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 
@@ -3287,6 +3289,7 @@ static PyType_Slot Tktt_Type_slots[] = {
     {Py_tp_dealloc, Tktt_Dealloc},
     {Py_tp_repr, Tktt_Repr},
     {Py_tp_methods, Tktt_methods},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 
@@ -3341,6 +3344,7 @@ static PyMethodDef Tkapp_methods[] =
 static PyType_Slot Tkapp_Type_slots[] = {
     {Py_tp_dealloc, Tkapp_Dealloc},
     {Py_tp_methods, Tkapp_methods},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 
@@ -3537,7 +3541,6 @@ PyInit__tkinter(void)
         Py_DECREF(m);
         return NULL;
     }
-    ((PyTypeObject *)o)->tp_new = NULL;
     if (PyModule_AddObject(m, "TkappType", o)) {
         Py_DECREF(o);
         Py_DECREF(m);
@@ -3550,7 +3553,6 @@ PyInit__tkinter(void)
         Py_DECREF(m);
         return NULL;
     }
-    ((PyTypeObject *)o)->tp_new = NULL;
     if (PyModule_AddObject(m, "TkttType", o)) {
         Py_DECREF(o);
         Py_DECREF(m);
@@ -3563,7 +3565,6 @@ PyInit__tkinter(void)
         Py_DECREF(m);
         return NULL;
     }
-    ((PyTypeObject *)o)->tp_new = NULL;
     if (PyModule_AddObject(m, "Tcl_Obj", o)) {
         Py_DECREF(o);
         Py_DECREF(m);

Modules/_winapi.c

@@ -44,6 +44,9 @@
 #include <crtdbg.h>
 #include "winreparse.h"
 
+// Include pycore_object.h to avoid conflict with windows.h
+#include "pycore_object.h"        // _PyType_DisabledNew()
+
 #if defined(MS_WIN32) && !defined(MS_WIN64)
 #define HANDLE_TO_PYNUM(handle) \
     PyLong_FromUnsignedLong((unsigned long) handle)
@@ -325,6 +328,7 @@ static PyType_Slot winapi_overlapped_type_slots[] = {
     {Py_tp_doc, "OVERLAPPED structure wrapper"},
     {Py_tp_methods, overlapped_methods},
     {Py_tp_members, overlapped_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0,0}
 };
 

Modules/arraymodule.c

@@ -6,6 +6,7 @@
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
+#include "pycore_object.h"        // _PyType_DisabledNew()
 #include "structmember.h"         // PyMemberDef
 #include <stddef.h>               // offsetof()
 
@@ -2980,6 +2981,7 @@ static PyType_Slot arrayiter_slots[] = {
     {Py_tp_iter, PyObject_SelfIter},
     {Py_tp_iternext, arrayiter_next},
     {Py_tp_methods, arrayiter_methods},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, NULL},
 };