bpo-43916: Add _PyType_DisabledNew to prevent new heap types from being created uninitialised

Include/internal/pycore_object.h

@@ -29,6 +29,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
 
 extern void _PyType_InitCache(PyInterpreterState *interp);
 
+PyAPI_FUNC(PyObject *) _PyType_DisabledNew(PyTypeObject *type, PyObject *args,
+                                           PyObject *kwds);
+
 
 /* Inline functions trading binary compatibility for speed:
    _PyObject_Init() is the fast version of PyObject_Init(), and

Lib/test/test_array.py

@@ -40,6 +40,12 @@ def test_bad_constructor(self):
         self.assertRaises(TypeError, array.array, 'xx')
         self.assertRaises(ValueError, array.array, 'x')
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        tp = type(iter(array.array('I')))
+        self.assertRaises(TypeError, tp)
+
     @support.cpython_only
     def test_immutable(self):
         # bpo-43908: check that array.array is immutable

Lib/test/test_curses.py

@@ -6,7 +6,7 @@
 import tempfile
 import unittest
 
-from test.support import requires, verbose, SaveSignals
+from test.support import requires, verbose, SaveSignals, cpython_only
 from test.support.import_helper import import_module
 
 # Optionally test curses module.  This currently requires that the
@@ -1046,6 +1046,7 @@ def __del__(self):
         panel.set_userptr(A())
         panel.set_userptr(None)
 
+    @cpython_only
     @requires_curses_func('panel')
     def test_new_curses_panel(self):
         w = curses.newwin(10, 10)

Lib/test/test_dbm_gnu.py

@@ -1,5 +1,5 @@
 from test import support
-from test.support import import_helper
+from test.support import import_helper, cpython_only
 gdbm = import_helper.import_module("dbm.gnu") #skip if not supported
 import unittest
 import os
@@ -27,6 +27,13 @@ def tearDown(self):
             self.g.close()
         unlink(filename)
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.g = gdbm.open(filename, 'c')
+        tp = type(self.g)
+        self.assertRaises(TypeError, tp)
+
     def test_key_methods(self):
         self.g = gdbm.open(filename, 'c')
         self.assertEqual(self.g.keys(), [])

Lib/test/test_functools.py

@@ -948,6 +948,12 @@ class TestCmpToKeyC(TestCmpToKey, unittest.TestCase):
     if c_functools:
         cmp_to_key = c_functools.cmp_to_key
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        tp = type(c_functools.cmp_to_key(None))
+        self.assertRaises(TypeError, tp)
+
 
 class TestCmpToKeyPy(TestCmpToKey, unittest.TestCase):
     cmp_to_key = staticmethod(py_functools.cmp_to_key)

Lib/test/test_posix.py

@@ -58,6 +58,13 @@ def tearDown(self):
             os_helper.unlink(teardown_file)
         self._warnings_manager.__exit__(None, None, None)
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.assertRaises(TypeError, posix.DirEntry)
+        tp = type(posix.scandir())
+        self.assertRaises(TypeError, tp)
+
     def testNoArgFunctions(self):
         # test posix functions which take no arguments and have
         # no side-effects which we need to cleanup (e.g., fork, wait, abort)

Lib/test/test_re.py

@@ -2215,6 +2215,15 @@ def test_signedness(self):
         self.assertGreaterEqual(sre_compile.MAXREPEAT, 0)
         self.assertGreaterEqual(sre_compile.MAXGROUPS, 0)
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.assertRaises(TypeError, re.Match)
+        self.assertRaises(TypeError, re.Pattern)
+        pat = re.compile("")
+        tp = type(pat.scanner(""))
+        self.assertRaises(TypeError, tp)
+
 
 class ExternalTests(unittest.TestCase):
 

Lib/test/test_tcl.py

@@ -736,8 +736,11 @@ def check(value):
         check('{\n')
         check('}\n')
 
+    @support.cpython_only
     def test_new_tcl_obj(self):
         self.assertRaises(TypeError, _tkinter.Tcl_Obj)
+        self.assertRaises(TypeError, _tkinter.TkttType)
+        self.assertRaises(TypeError, _tkinter.TkappType)
 
 class BigmemTclTest(unittest.TestCase):
 

Lib/test/test_threading.py

@@ -119,6 +119,13 @@ def func(): pass
             thread = threading.Thread(target=func)
             self.assertEqual(thread.name, "Thread-5 (func)")
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        lock = threading.Lock()
+        tp = type(lock)
+        self.assertRaises(TypeError, tp)
+
     # Create a bunch of threads, let each do some work, wait until all are
     # done.
     def test_various_ops(self):

Lib/test/test_unicodedata.py

@@ -11,7 +11,8 @@
 import sys
 import unicodedata
 import unittest
-from test.support import open_urlresource, requires_resource, script_helper
+from test.support import (open_urlresource, requires_resource, script_helper,
+                          cpython_only)
 
 
 class UnicodeMethodsTest(unittest.TestCase):
@@ -225,6 +226,11 @@ def test_east_asian_width_9_0_changes(self):
 
 class UnicodeMiscTest(UnicodeDatabaseTest):
 
+    @cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        self.assertRaises(TypeError, unicodedata.UCD)
+
     def test_failed_import_during_compiling(self):
         # Issue 4367
         # Decoding \N escapes requires the unicodedata module. If it can't be

Lib/test/test_zlib.py

@@ -129,6 +129,14 @@ def test_overflow(self):
         with self.assertRaisesRegex(OverflowError, 'int too large'):
             zlib.decompressobj().flush(sys.maxsize + 1)
 
+    @support.cpython_only
+    def test_uninitialised_new(self):
+        # Prevent heap types from being created uninitialised (bpo-43916)
+        comp_type = type(zlib.compressobj())
+        decomp_type = type(zlib.decompressobj())
+        self.assertRaises(TypeError, comp_type)
+        self.assertRaises(TypeError, decomp_type)
+
 
 class BaseCompressTestCase(object):
     def check_big_compress_buffer(self, size, compress_func):

Misc/NEWS.d/next/Core and Builtins/2021-04-28-10-09-16.bpo-43916.ecedNo.rst

@@ -0,0 +1,7 @@
+Fix regressions for extension modules that have been converted to :ref:`heap
+types <heap-types>`. Inheritance differs for *static types* and *heap
+types*: A *static type* with its :c:member:`~PyTypeObject.tp_new` set to
+``NULL`` does not have a public contructor, but *heap types* inherit
+contructors from their base classes, resulting in it being possible to
+create instances without proper initialisation, possibly resulting in a
+crash.

Modules/Setup

@@ -166,7 +166,7 @@ _symtable symtablemodule.c
 
 # Modules that should always be present (non UNIX dependent):
 
-#array -DPy_BUILD_CORE_MODULE arraymodule.c	# array objects
+#array -DPy_BUILD_CORE_BUILTIN arraymodule.c	# array objects
 #cmath cmathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # complex math library functions
 #math mathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # math library functions, e.g. sin()
 #_contextvars _contextvarsmodule.c  # Context Variables
@@ -279,7 +279,7 @@ _symtable symtablemodule.c
 # every system.
 
 # *** Always uncomment this (leave the leading underscore in!):
-# _tkinter _tkinter.c tkappinit.c -DWITH_APPINIT \
+# _tkinter _tkinter.c tkappinit.c -DWITH_APPINIT -DPy_BUILD_CORE_BUILTIN \
 # *** Uncomment and edit to reflect where your Tcl/Tk libraries are:
 #	-L/usr/local/lib \
 # *** Uncomment and edit to reflect where your Tcl/Tk headers are:
@@ -320,7 +320,7 @@ _symtable symtablemodule.c
 
 #_curses _cursesmodule.c -lcurses -ltermcap -DPy_BUILD_CORE_MODULE
 # Wrapper for the panel library that's part of ncurses and SYSV curses.
-#_curses_panel _curses_panel.c -lpanel -lncurses
+#_curses_panel _curses_panel.c -lpanel -lncurses -DPy_BUILD_CORE_BUILTIN
 
 
 # Modules that provide persistent dictionary-like semantics.  You will
@@ -330,11 +330,11 @@ _symtable symtablemodule.c
 # implementation independent wrapper for these; dbm/dumb.py provides
 # similar functionality (but slower of course) implemented in Python.
 
-#_dbm _dbmmodule.c 	# dbm(3) may require -lndbm or similar
+#_dbm _dbmmodule.c -DPy_BUILD_CORE_BUILTIN	# dbm(3) may require -lndbm or similar
 
 # Anthony Baxter's gdbm module.  GNU dbm(3) will require -lgdbm:
 
-#_gdbm _gdbmmodule.c -I/usr/local/include -L/usr/local/lib -lgdbm
+#_gdbm _gdbmmodule.c -I/usr/local/include -L/usr/local/lib -lgdbm -DPy_BUILD_CORE_BUILTIN
 
 
 # Helper module for various ascii-encoders
@@ -343,17 +343,17 @@ _symtable symtablemodule.c
 # Andrew Kuchling's zlib module.
 # This require zlib 1.1.3 (or later).
 # See http://www.gzip.org/zlib/
-#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz
+#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz -DPy_BUILD_CORE_BUILTIN
 
 # Interface to the Expat XML parser
 # More information on Expat can be found at www.libexpat.org.
 #
-#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI
+#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI -DPy_BUILD_CORE_BUILTIN
 
 # Hye-Shik Chang's CJKCodecs
 
 # multibytecodec is required for all the other CJK codec modules
-#_multibytecodec cjkcodecs/multibytecodec.c
+#_multibytecodec cjkcodecs/multibytecodec.c -DPy_BUILD_CORE_BUILTIN
 
 #_codecs_cn cjkcodecs/_codecs_cn.c
 #_codecs_hk cjkcodecs/_codecs_hk.c

Modules/_curses_panel.c

@@ -13,6 +13,7 @@ static const char PyCursesVersion[] = "2.1";
 #include "Python.h"
 
 #include "py_curses.h"
+#include "pycore_object.h"        // _PyType_DisabledNew()
 
 #include <panel.h>
 
@@ -514,6 +515,7 @@ static PyMethodDef PyCursesPanel_Methods[] = {
 static PyType_Slot PyCursesPanel_Type_slots[] = {
     {Py_tp_dealloc, PyCursesPanel_Dealloc},
     {Py_tp_methods, PyCursesPanel_Methods},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0},
 };
 
@@ -656,7 +658,6 @@ _curses_panel_exec(PyObject *mod)
     if (state->PyCursesPanel_Type == NULL) {
         return -1;
     }
-    ((PyTypeObject *)state->PyCursesPanel_Type)->tp_new = NULL;
 
     if (PyModule_AddType(mod, state->PyCursesPanel_Type) < 0) {
         return -1;

Modules/_dbmmodule.c

@@ -4,6 +4,7 @@
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
+#include "pycore_object.h"        // _PyType_DisabledNew()
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -397,6 +398,7 @@ static PyMethodDef dbm_methods[] = {
 };
 
 static PyType_Slot dbmtype_spec_slots[] = {
+    {Py_tp_new, _PyType_DisabledNew},
     {Py_tp_dealloc, dbm_dealloc},
     {Py_tp_methods, dbm_methods},
     {Py_sq_contains, dbm_contains},

Modules/_functoolsmodule.c

@@ -540,6 +540,7 @@ static PyType_Slot keyobject_type_slots[] = {
     {Py_tp_clear, keyobject_clear},
     {Py_tp_richcompare, keyobject_richcompare},
     {Py_tp_members, keyobject_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 
@@ -760,6 +761,7 @@ lru_list_elem_dealloc(lru_list_elem *link)
 
 static PyType_Slot lru_list_elem_type_slots[] = {
     {Py_tp_dealloc, lru_list_elem_dealloc},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 

Modules/_gdbmmodule.c

@@ -5,6 +5,7 @@
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
+#include "pycore_object.h"        // _PyType_DisabledNew()
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -560,6 +561,7 @@ static PyType_Slot gdbmtype_spec_slots[] = {
     {Py_mp_subscript, gdbm_subscript},
     {Py_mp_ass_subscript, gdbm_ass_sub},
     {Py_tp_doc, (char*)gdbm_object__doc__},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 

Modules/_sre.c

@@ -43,6 +43,7 @@ static const char copyright[] =
 #include "Python.h"
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
+#include "pycore_object.h"        // _PyType_DisabledNew()
 #include "structmember.h"         // PyMemberDef
 
 #include "sre.h"
@@ -2683,6 +2684,7 @@ static PyType_Slot pattern_slots[] = {
     {Py_tp_methods, pattern_methods},
     {Py_tp_members, pattern_members},
     {Py_tp_getset, pattern_getset},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, NULL},
 };
 
@@ -2740,6 +2742,7 @@ static PyType_Slot match_slots[] = {
     {Py_tp_methods, match_methods},
     {Py_tp_members, match_members},
     {Py_tp_getset, match_getset},
+    {Py_tp_new, _PyType_DisabledNew},
 
     /* As mapping.
      *
@@ -2775,6 +2778,7 @@ static PyType_Slot scanner_slots[] = {
     {Py_tp_dealloc, scanner_dealloc},
     {Py_tp_methods, scanner_methods},
     {Py_tp_members, scanner_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, NULL},
 };
 

Modules/_threadmodule.c

@@ -5,6 +5,7 @@
 #include "Python.h"
 #include "pycore_interp.h"        // _PyInterpreterState.num_threads
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
+#include "pycore_object.h"        // _PyType_DisabledNew()
 #include "pycore_pylifecycle.h"
 #include "pycore_pystate.h"       // _PyThreadState_Init()
 #include <stddef.h>               // offsetof()
@@ -306,6 +307,7 @@ static PyType_Slot lock_type_slots[] = {
     {Py_tp_methods, lock_methods},
     {Py_tp_traverse, lock_traverse},
     {Py_tp_members, lock_type_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };
 
@@ -677,6 +679,7 @@ static PyType_Slot local_dummy_type_slots[] = {
     {Py_tp_dealloc, (destructor)localdummy_dealloc},
     {Py_tp_doc, "Thread-local dummy"},
     {Py_tp_members, local_dummy_type_members},
+    {Py_tp_new, _PyType_DisabledNew},
     {0, 0}
 };