From 36150d62c820877e5f5d9cf8d77cd1882336fc3c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 26 Apr 2021 22:32:30 +0200 Subject: [PATCH 01/17] bpo-43916: Apply tp_new = 0 hack to affected types --- Lib/test/test_array.py | 6 ++++++ Lib/test/test_unicodedata.py | 4 ++++ Modules/_dbmmodule.c | 1 + Modules/_functoolsmodule.c | 2 ++ Modules/_gdbmmodule.c | 1 + Modules/_sre.c | 5 +++++ Modules/_threadmodule.c | 2 ++ Modules/_winapi.c | 1 + Modules/arraymodule.c | 1 + Modules/cjkcodecs/multibytecodec.c | 1 + Modules/posixmodule.c | 2 ++ Modules/pyexpat.c | 1 + Modules/unicodedata.c | 1 + Modules/zlibmodule.c | 2 ++ 14 files changed, 30 insertions(+) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 11184add6d399b..e7e1b04656f94d 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -40,6 +40,12 @@ def test_bad_constructor(self): self.assertRaises(TypeError, array.array, 'xx') self.assertRaises(ValueError, array.array, 'x') + @support.cpython_only + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + tp = type(iter(array.array('I'))) + self.assertRaises(TypeError, tp) + @support.cpython_only def test_immutable(self): # bpo-43908: check that array.array is immutable diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index edfd860fd5f12f..2acf387d00440a 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -309,6 +309,10 @@ def test_linebreak_7643(self): self.assertEqual(len(lines), 1, r"\u%.4x should not be a linebreak" % i) + def test_no_uninit_new(self): + # See bpo-43916 + self.assertRaises(TypeError, unicodedata.UCD) + class NormalizationTest(unittest.TestCase): @staticmethod def check_version(testfile): diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 97772a04d08fe0..40cc8e7461745a 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -499,6 +499,7 @@ _dbm_exec(PyObject *module) if (state->dbm_type == NULL) { return -1; } + state->dbm_type->tp_new = 0; // See bpo-43916 state->dbm_error = PyErr_NewException("_dbm.error", PyExc_OSError, NULL); if (state->dbm_error == NULL) { return -1; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index eea542e18c92d2..c70e094fc96872 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1451,6 +1451,7 @@ _functools_exec(PyObject *module) if (state->keyobject_type == NULL) { return -1; } + state->keyobject_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(module, state->keyobject_type) < 0) { return -1; } @@ -1460,6 +1461,7 @@ _functools_exec(PyObject *module) if (state->lru_list_elem_type == NULL) { return -1; } + state->lru_list_elem_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(module, state->lru_list_elem_type) < 0) { return -1; } diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 9e843acbaa6ba5..2041674c12a5b9 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -702,6 +702,7 @@ _gdbm_exec(PyObject *module) if (state->gdbm_type == NULL) { return -1; } + state->gdbm_type->tp_new = 0; // See bpo-43916 state->gdbm_error = PyErr_NewException("_gdbm.error", PyExc_OSError, NULL); if (state->gdbm_error == NULL) { return -1; diff --git a/Modules/_sre.c b/Modules/_sre.c index 59f7551a227cd3..ca6b4f707d8bb3 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -2856,6 +2856,11 @@ sre_exec(PyObject *m) CREATE_TYPE(m, state->Match_Type, &match_spec); CREATE_TYPE(m, state->Scanner_Type, &scanner_spec); + // See bpo-43916 + state->Pattern_Type->tp_new = 0; + state->Match_Type->tp_new = 0; + state->Scanner_Type->tp_new = 0; + if (PyModule_AddIntConstant(m, "MAGIC", SRE_MAGIC) < 0) { goto error; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 7feb0b8a1f1f4d..a84008ac590467 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1580,6 +1580,7 @@ thread_module_exec(PyObject *module) if (state->lock_type == NULL) { return -1; } + state->lock_type->tp_new = 0; // See bpo-43916 if (PyDict_SetItemString(d, "LockType", (PyObject *)state->lock_type) < 0) { return -1; } @@ -1600,6 +1601,7 @@ thread_module_exec(PyObject *module) if (state->local_dummy_type == NULL) { return -1; } + state->local_dummy_type->tp_new = 0; // See bpo-43916 // Local state->local_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &local_type_spec, NULL); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 9d5a45adac59d0..0df4d569fd8f67 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1931,6 +1931,7 @@ static int winapi_exec(PyObject *m) if (st->overlapped_type == NULL) { return -1; } + st->overlapped_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(m, st->overlapped_type) < 0) { return -1; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 367621fd03b882..eb01a1da50d607 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3041,6 +3041,7 @@ array_modexec(PyObject *m) CREATE_TYPE(m, state->ArrayType, &array_spec); CREATE_TYPE(m, state->ArrayIterType, &arrayiter_spec); + state->ArrayIterType->tp_new = 0; // See bpo-43916 Py_SET_TYPE(state->ArrayIterType, &PyType_Type); Py_INCREF((PyObject *)state->ArrayType); diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 5070c983d402c3..7623dd849838b1 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -2012,6 +2012,7 @@ _multibytecodec_exec(PyObject *mod) { _multibytecodec_state *state = _multibytecodec_get_state(mod); CREATE_TYPE(mod, state->multibytecodec_type, &multibytecodec_spec); + state->multibytecodec_type->tp_new = 0; // See bpo-43916 CREATE_TYPE(mod, state->encoder_type, &encoder_spec); CREATE_TYPE(mod, state->decoder_type, &decoder_spec); CREATE_TYPE(mod, state->reader_type, &reader_spec); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ecd210e4babf54..0548b6505d2b06 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15730,12 +15730,14 @@ posixmodule_exec(PyObject *m) if (ScandirIteratorType == NULL) { return -1; } + ((PyTypeObject *)ScandirIteratorType)->tp_new = 0; // See bpo-43916 state->ScandirIteratorType = ScandirIteratorType; PyObject *DirEntryType = PyType_FromModuleAndSpec(m, &DirEntryType_spec, NULL); if (DirEntryType == NULL) { return -1; } + ((PyTypeObject *)DirEntryType)->tp_new = 0; // See bpo-43916 Py_INCREF(DirEntryType); PyModule_AddObject(m, "DirEntry", DirEntryType); state->DirEntryType = DirEntryType; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index a13d340a3ea0f0..e0ae4d54f0728d 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1862,6 +1862,7 @@ pyexpat_exec(PyObject *mod) if (state->xml_parse_type == NULL) { return -1; } + state->xml_parse_type->tp_new = 0; // See bpo-43916 if (init_handler_descrs(state) < 0) { return -1; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index aebae7da576561..476662c7390c76 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1478,6 +1478,7 @@ unicodedata_exec(PyObject *module) if (ucd_type == NULL) { return -1; } + ucd_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(module, ucd_type) < 0) { Py_DECREF(ucd_type); diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 8a20dfcbf9e27a..1dfb4e3114ce0e 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1459,12 +1459,14 @@ zlib_exec(PyObject *mod) if (state->Comptype == NULL) { return -1; } + state->Comptype->tp_new = 0; // See bpo-43916 state->Decomptype = (PyTypeObject *)PyType_FromModuleAndSpec( mod, &Decomptype_spec, NULL); if (state->Decomptype == NULL) { return -1; } + state->Decomptype->tp_new = 0; // See bpo-43916 state->ZlibError = PyErr_NewException("zlib.error", NULL, NULL); if (state->ZlibError == NULL) { From dbebde5886c75808be302eb0a9331076e8c3cfdc Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 27 Apr 2021 10:10:37 +0200 Subject: [PATCH 02/17] Add _PyType_DisabledNew --- Include/cpython/object.h | 3 +++ Objects/typeobject.c | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 58e4d2b11b93f9..5444358054e867 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -4,6 +4,9 @@ PyAPI_FUNC(void) _Py_NewReference(PyObject *op); +PyAPI_FUNC(PyObject *) _PyType_DisabledNew(PyTypeObject *, PyObject *, + PyObject *); + #ifdef Py_TRACE_REFS /* Py_TRACE_REFS is such major surgery that we call external routines. */ PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e1c8be4b815452..25c00db9e2349f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1004,6 +1004,14 @@ type_repr(PyTypeObject *type) return rtn; } +PyObject * +_PyType_DisabledNew(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyErr_Format(PyExc_TypeError, "cannot create '%.100s' instances", + type->tp_name); + return NULL; +} + static PyObject * type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1041,10 +1049,7 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) } if (type->tp_new == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "cannot create '%.100s' instances", - type->tp_name); - return NULL; + return _PyType_DisabledNew(type, args, kwds); } obj = type->tp_new(type, args, kwds); From 54646a511c44ff53ec1903064c232ea0bbba4877 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 27 Apr 2021 10:24:20 +0200 Subject: [PATCH 03/17] Use _PyType_DisabledNew in affected types --- Modules/_dbmmodule.c | 2 +- Modules/_functoolsmodule.c | 4 ++-- Modules/_gdbmmodule.c | 2 +- Modules/_sre.c | 8 +++----- Modules/_threadmodule.c | 4 ++-- Modules/_winapi.c | 2 +- Modules/arraymodule.c | 2 +- Modules/cjkcodecs/multibytecodec.c | 2 +- Modules/posixmodule.c | 4 ++-- Modules/pyexpat.c | 2 +- Modules/unicodedata.c | 2 +- Modules/zlibmodule.c | 4 ++-- 12 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 40cc8e7461745a..247d50ef39ca35 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -397,6 +397,7 @@ static PyMethodDef dbm_methods[] = { }; static PyType_Slot dbmtype_spec_slots[] = { + {Py_tp_new, _PyType_DisabledNew}, {Py_tp_dealloc, dbm_dealloc}, {Py_tp_methods, dbm_methods}, {Py_sq_contains, dbm_contains}, @@ -499,7 +500,6 @@ _dbm_exec(PyObject *module) if (state->dbm_type == NULL) { return -1; } - state->dbm_type->tp_new = 0; // See bpo-43916 state->dbm_error = PyErr_NewException("_dbm.error", PyExc_OSError, NULL); if (state->dbm_error == NULL) { return -1; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index c70e094fc96872..3cb6a73c5077f8 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -540,6 +540,7 @@ static PyType_Slot keyobject_type_slots[] = { {Py_tp_clear, keyobject_clear}, {Py_tp_richcompare, keyobject_richcompare}, {Py_tp_members, keyobject_members}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -760,6 +761,7 @@ lru_list_elem_dealloc(lru_list_elem *link) static PyType_Slot lru_list_elem_type_slots[] = { {Py_tp_dealloc, lru_list_elem_dealloc}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -1451,7 +1453,6 @@ _functools_exec(PyObject *module) if (state->keyobject_type == NULL) { return -1; } - state->keyobject_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(module, state->keyobject_type) < 0) { return -1; } @@ -1461,7 +1462,6 @@ _functools_exec(PyObject *module) if (state->lru_list_elem_type == NULL) { return -1; } - state->lru_list_elem_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(module, state->lru_list_elem_type) < 0) { return -1; } diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 2041674c12a5b9..7f752ebfca5692 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -560,6 +560,7 @@ static PyType_Slot gdbmtype_spec_slots[] = { {Py_mp_subscript, gdbm_subscript}, {Py_mp_ass_subscript, gdbm_ass_sub}, {Py_tp_doc, (char*)gdbm_object__doc__}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -702,7 +703,6 @@ _gdbm_exec(PyObject *module) if (state->gdbm_type == NULL) { return -1; } - state->gdbm_type->tp_new = 0; // See bpo-43916 state->gdbm_error = PyErr_NewException("_gdbm.error", PyExc_OSError, NULL); if (state->gdbm_error == NULL) { return -1; diff --git a/Modules/_sre.c b/Modules/_sre.c index ca6b4f707d8bb3..0838936d3413b7 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -2683,6 +2683,7 @@ static PyType_Slot pattern_slots[] = { {Py_tp_methods, pattern_methods}, {Py_tp_members, pattern_members}, {Py_tp_getset, pattern_getset}, + {Py_tp_new, _PyType_DisabledNew}, {0, NULL}, }; @@ -2740,6 +2741,7 @@ static PyType_Slot match_slots[] = { {Py_tp_methods, match_methods}, {Py_tp_members, match_members}, {Py_tp_getset, match_getset}, + {Py_tp_new, _PyType_DisabledNew}, /* As mapping. * @@ -2775,6 +2777,7 @@ static PyType_Slot scanner_slots[] = { {Py_tp_dealloc, scanner_dealloc}, {Py_tp_methods, scanner_methods}, {Py_tp_members, scanner_members}, + {Py_tp_new, _PyType_DisabledNew}, {0, NULL}, }; @@ -2856,11 +2859,6 @@ sre_exec(PyObject *m) CREATE_TYPE(m, state->Match_Type, &match_spec); CREATE_TYPE(m, state->Scanner_Type, &scanner_spec); - // See bpo-43916 - state->Pattern_Type->tp_new = 0; - state->Match_Type->tp_new = 0; - state->Scanner_Type->tp_new = 0; - if (PyModule_AddIntConstant(m, "MAGIC", SRE_MAGIC) < 0) { goto error; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index a84008ac590467..fc867891e3481b 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -306,6 +306,7 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_methods, lock_methods}, {Py_tp_traverse, lock_traverse}, {Py_tp_members, lock_type_members}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -677,6 +678,7 @@ static PyType_Slot local_dummy_type_slots[] = { {Py_tp_dealloc, (destructor)localdummy_dealloc}, {Py_tp_doc, "Thread-local dummy"}, {Py_tp_members, local_dummy_type_members}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -1580,7 +1582,6 @@ thread_module_exec(PyObject *module) if (state->lock_type == NULL) { return -1; } - state->lock_type->tp_new = 0; // See bpo-43916 if (PyDict_SetItemString(d, "LockType", (PyObject *)state->lock_type) < 0) { return -1; } @@ -1601,7 +1602,6 @@ thread_module_exec(PyObject *module) if (state->local_dummy_type == NULL) { return -1; } - state->local_dummy_type->tp_new = 0; // See bpo-43916 // Local state->local_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &local_type_spec, NULL); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 0df4d569fd8f67..f6bf5f10531ce2 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -325,6 +325,7 @@ static PyType_Slot winapi_overlapped_type_slots[] = { {Py_tp_doc, "OVERLAPPED structure wrapper"}, {Py_tp_methods, overlapped_methods}, {Py_tp_members, overlapped_members}, + {Py_tp_new, _PyType_DisabledNew}, {0,0} }; @@ -1931,7 +1932,6 @@ static int winapi_exec(PyObject *m) if (st->overlapped_type == NULL) { return -1; } - st->overlapped_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(m, st->overlapped_type) < 0) { return -1; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index eb01a1da50d607..2c91f45a30758b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2980,6 +2980,7 @@ static PyType_Slot arrayiter_slots[] = { {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, arrayiter_next}, {Py_tp_methods, arrayiter_methods}, + {Py_tp_new, _PyType_DisabledNew}, {0, NULL}, }; @@ -3041,7 +3042,6 @@ array_modexec(PyObject *m) CREATE_TYPE(m, state->ArrayType, &array_spec); CREATE_TYPE(m, state->ArrayIterType, &arrayiter_spec); - state->ArrayIterType->tp_new = 0; // See bpo-43916 Py_SET_TYPE(state->ArrayIterType, &PyType_Type); Py_INCREF((PyObject *)state->ArrayType); diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 7623dd849838b1..3d81b557193904 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -742,6 +742,7 @@ static PyType_Slot multibytecodec_slots[] = { {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_methods, multibytecodec_methods}, {Py_tp_traverse, multibytecodec_traverse}, + {Py_tp_new, _PyType_DisabledNew}, {0, NULL}, }; @@ -2012,7 +2013,6 @@ _multibytecodec_exec(PyObject *mod) { _multibytecodec_state *state = _multibytecodec_get_state(mod); CREATE_TYPE(mod, state->multibytecodec_type, &multibytecodec_spec); - state->multibytecodec_type->tp_new = 0; // See bpo-43916 CREATE_TYPE(mod, state->encoder_type, &encoder_spec); CREATE_TYPE(mod, state->decoder_type, &decoder_spec); CREATE_TYPE(mod, state->reader_type, &reader_spec); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0548b6505d2b06..2ffbe813866e17 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13786,6 +13786,7 @@ static PyType_Slot DirEntryType_slots[] = { {Py_tp_repr, DirEntry_repr}, {Py_tp_methods, DirEntry_methods}, {Py_tp_members, DirEntry_members}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0}, }; @@ -14219,6 +14220,7 @@ static PyType_Slot ScandirIteratorType_slots[] = { {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, ScandirIterator_iternext}, {Py_tp_methods, ScandirIterator_methods}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0}, }; @@ -15730,14 +15732,12 @@ posixmodule_exec(PyObject *m) if (ScandirIteratorType == NULL) { return -1; } - ((PyTypeObject *)ScandirIteratorType)->tp_new = 0; // See bpo-43916 state->ScandirIteratorType = ScandirIteratorType; PyObject *DirEntryType = PyType_FromModuleAndSpec(m, &DirEntryType_spec, NULL); if (DirEntryType == NULL) { return -1; } - ((PyTypeObject *)DirEntryType)->tp_new = 0; // See bpo-43916 Py_INCREF(DirEntryType); PyModule_AddObject(m, "DirEntry", DirEntryType); state->DirEntryType = DirEntryType; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index e0ae4d54f0728d..0cd9810bbdff92 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1500,6 +1500,7 @@ static PyType_Slot _xml_parse_type_spec_slots[] = { {Py_tp_methods, xmlparse_methods}, {Py_tp_members, xmlparse_members}, {Py_tp_getset, xmlparse_getsetlist}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -1862,7 +1863,6 @@ pyexpat_exec(PyObject *mod) if (state->xml_parse_type == NULL) { return -1; } - state->xml_parse_type->tp_new = 0; // See bpo-43916 if (init_handler_descrs(state) < 0) { return -1; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 476662c7390c76..0db05f99c10835 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1448,6 +1448,7 @@ static PyType_Slot ucd_type_slots[] = { {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_methods, unicodedata_functions}, {Py_tp_members, DB_members}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -1478,7 +1479,6 @@ unicodedata_exec(PyObject *module) if (ucd_type == NULL) { return -1; } - ucd_type->tp_new = 0; // See bpo-43916 if (PyModule_AddType(module, ucd_type) < 0) { Py_DECREF(ucd_type); diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 1dfb4e3114ce0e..698f569e58c98d 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1382,6 +1382,7 @@ static PyMethodDef zlib_methods[] = static PyType_Slot Comptype_slots[] = { {Py_tp_dealloc, Comp_dealloc}, {Py_tp_methods, comp_methods}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0}, }; @@ -1397,6 +1398,7 @@ static PyType_Slot Decomptype_slots[] = { {Py_tp_dealloc, Decomp_dealloc}, {Py_tp_methods, Decomp_methods}, {Py_tp_members, Decomp_members}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0}, }; @@ -1459,14 +1461,12 @@ zlib_exec(PyObject *mod) if (state->Comptype == NULL) { return -1; } - state->Comptype->tp_new = 0; // See bpo-43916 state->Decomptype = (PyTypeObject *)PyType_FromModuleAndSpec( mod, &Decomptype_spec, NULL); if (state->Decomptype == NULL) { return -1; } - state->Decomptype->tp_new = 0; // See bpo-43916 state->ZlibError = PyErr_NewException("zlib.error", NULL, NULL); if (state->ZlibError == NULL) { From 02829da758e49691839eeaf76b2616665e7978c1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 27 Apr 2021 11:13:35 +0200 Subject: [PATCH 04/17] Address review: improve comments in Lib/test --- Lib/test/test_unicodedata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 2acf387d00440a..111f367e47c04d 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -310,7 +310,7 @@ def test_linebreak_7643(self): r"\u%.4x should not be a linebreak" % i) def test_no_uninit_new(self): - # See bpo-43916 + # Prevent heap types from being created uninitialised (bpo-43916) self.assertRaises(TypeError, unicodedata.UCD) class NormalizationTest(unittest.TestCase): From 0c3872e7850306ce0d6c8756aed0a79502f642ec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 27 Apr 2021 13:06:06 +0200 Subject: [PATCH 05/17] Add re, functools, threading, zlib, posix, and unicodedata tests --- Lib/test/test_functools.py | 4 ++++ Lib/test/test_posix.py | 5 +++++ Lib/test/test_re.py | 8 ++++++++ Lib/test/test_threading.py | 5 +++++ Lib/test/test_unicodedata.py | 8 ++++---- Lib/test/test_zlib.py | 7 +++++++ 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index fba9281deb3763..e1523a2709f30c 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -948,6 +948,10 @@ class TestCmpToKeyC(TestCmpToKey, unittest.TestCase): if c_functools: cmp_to_key = c_functools.cmp_to_key + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + self.assertRaises(TypeError, type(c_functools.cmp_to_key(None))) + class TestCmpToKeyPy(TestCmpToKey, unittest.TestCase): cmp_to_key = staticmethod(py_functools.cmp_to_key) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 93611f04515ae9..696cec2ce45b17 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -58,6 +58,11 @@ def tearDown(self): os_helper.unlink(teardown_file) self._warnings_manager.__exit__(None, None, None) + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + self.assertRaises(TypeError, posix.DirEntry) + self.assertRaises(TypeError, type(posix.scandir())) + def testNoArgFunctions(self): # test posix functions which take no arguments and have # no side-effects which we need to cleanup (e.g., fork, wait, abort) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 96d0cdb17a7ee3..4daaac6ed30865 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2215,6 +2215,14 @@ def test_signedness(self): self.assertGreaterEqual(sre_compile.MAXREPEAT, 0) self.assertGreaterEqual(sre_compile.MAXGROUPS, 0) + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + self.assertRaises(TypeError, re.Match) + self.assertRaises(TypeError, re.Pattern) + pat = re.compile("") + scn = pat.scanner("") + self.assertRaises(TypeError, type(scn)) + class ExternalTests(unittest.TestCase): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index f44f17f2978f7b..29ad3be30c26b2 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -119,6 +119,11 @@ def func(): pass thread = threading.Thread(target=func) self.assertEqual(thread.name, "Thread-5 (func)") + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + lock = threading.lock() + self.assertRaises(TypeError, type(lock)) + # Create a bunch of threads, let each do some work, wait until all are # done. def test_various_ops(self): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 111f367e47c04d..89eed58953c00b 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -225,6 +225,10 @@ def test_east_asian_width_9_0_changes(self): class UnicodeMiscTest(UnicodeDatabaseTest): + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + self.assertRaises(TypeError, unicodedata.UCD) + def test_failed_import_during_compiling(self): # Issue 4367 # Decoding \N escapes requires the unicodedata module. If it can't be @@ -309,10 +313,6 @@ def test_linebreak_7643(self): self.assertEqual(len(lines), 1, r"\u%.4x should not be a linebreak" % i) - def test_no_uninit_new(self): - # Prevent heap types from being created uninitialised (bpo-43916) - self.assertRaises(TypeError, unicodedata.UCD) - class NormalizationTest(unittest.TestCase): @staticmethod def check_version(testfile): diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 7f30cac64f71b9..a3aea28cdd47c1 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -129,6 +129,13 @@ def test_overflow(self): with self.assertRaisesRegex(OverflowError, 'int too large'): zlib.decompressobj().flush(sys.maxsize + 1) + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + comp = zlib.compressobj() + decomp = zlib.decompressobj() + self.assertRaises(TypeError, type(comp)) + self.assertRaises(TypeError, type(decomp)) + class BaseCompressTestCase(object): def check_big_compress_buffer(self, size, compress_func): From 45169ffe5ed58382d7c649b11ad119cb7c392f45 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 27 Apr 2021 14:20:39 +0200 Subject: [PATCH 06/17] Fix threading test --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 29ad3be30c26b2..2ec723cab35642 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -121,7 +121,7 @@ def func(): pass def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) - lock = threading.lock() + lock = threading.Lock() self.assertRaises(TypeError, type(lock)) # Create a bunch of threads, let each do some work, wait until all are From 69b19bc990e4a1ef3ecba07501101ce5f79bb0cf Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Apr 2021 09:28:18 +0200 Subject: [PATCH 07/17] Decorate tests with @cpython_only --- Lib/test/test_functools.py | 1 + Lib/test/test_posix.py | 1 + Lib/test/test_re.py | 1 + Lib/test/test_threading.py | 1 + Lib/test/test_unicodedata.py | 4 +++- Lib/test/test_zlib.py | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index e1523a2709f30c..db5e18856df1dc 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -948,6 +948,7 @@ class TestCmpToKeyC(TestCmpToKey, unittest.TestCase): if c_functools: cmp_to_key = c_functools.cmp_to_key + @support.cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) self.assertRaises(TypeError, type(c_functools.cmp_to_key(None))) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 696cec2ce45b17..6e0794a7de0081 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -58,6 +58,7 @@ def tearDown(self): os_helper.unlink(teardown_file) self._warnings_manager.__exit__(None, None, None) + @support.cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) self.assertRaises(TypeError, posix.DirEntry) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 4daaac6ed30865..abf95a286c677c 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2215,6 +2215,7 @@ def test_signedness(self): self.assertGreaterEqual(sre_compile.MAXREPEAT, 0) self.assertGreaterEqual(sre_compile.MAXGROUPS, 0) + @cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) self.assertRaises(TypeError, re.Match) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 2ec723cab35642..e18a003cd3792b 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -119,6 +119,7 @@ def func(): pass thread = threading.Thread(target=func) self.assertEqual(thread.name, "Thread-5 (func)") + @cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) lock = threading.Lock() diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 89eed58953c00b..64a75c502748bc 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -11,7 +11,8 @@ import sys import unicodedata import unittest -from test.support import open_urlresource, requires_resource, script_helper +from test.support import (open_urlresource, requires_resource, script_helper, + cpython_only) class UnicodeMethodsTest(unittest.TestCase): @@ -225,6 +226,7 @@ def test_east_asian_width_9_0_changes(self): class UnicodeMiscTest(UnicodeDatabaseTest): + @support.cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) self.assertRaises(TypeError, unicodedata.UCD) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index a3aea28cdd47c1..83b33d73ff9864 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -129,6 +129,7 @@ def test_overflow(self): with self.assertRaisesRegex(OverflowError, 'int too large'): zlib.decompressobj().flush(sys.maxsize + 1) + @support.cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) comp = zlib.compressobj() From 2cc4dd1f880e52b740659de8f28272f33504982c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Apr 2021 09:34:51 +0200 Subject: [PATCH 08/17] Adapt _curses_panel and tkinter workarounds/tests --- Lib/test/test_curses.py | 3 ++- Lib/test/test_tcl.py | 3 +++ Lib/test/test_unicodedata.py | 2 +- Modules/_curses_panel.c | 2 +- Modules/_tkinter.c | 6 +++--- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 280a9dc0eedfce..5ad787775796d5 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -6,7 +6,7 @@ import tempfile import unittest -from test.support import requires, verbose, SaveSignals +from test.support import requires, verbose, SaveSignals, cpython_only from test.support.import_helper import import_module # Optionally test curses module. This currently requires that the @@ -1046,6 +1046,7 @@ def __del__(self): panel.set_userptr(A()) panel.set_userptr(None) + @cpython_only @requires_curses_func('panel') def test_new_curses_panel(self): w = curses.newwin(10, 10) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index ee7344c48ed0be..cd3aacf6f8844c 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -736,8 +736,11 @@ def check(value): check('{\n') check('}\n') + @support.cpython_only def test_new_tcl_obj(self): self.assertRaises(TypeError, _tkinter.Tcl_Obj) + self.assertRaises(TypeError, _tkinter.TkttType) + self.assertRaises(TypeError, _tkinter.TkappType) class BigmemTclTest(unittest.TestCase): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 64a75c502748bc..40b6da3bf59156 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -226,7 +226,7 @@ def test_east_asian_width_9_0_changes(self): class UnicodeMiscTest(UnicodeDatabaseTest): - @support.cpython_only + @cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) self.assertRaises(TypeError, unicodedata.UCD) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 94caf8c93bc8c3..84868e715dd1c7 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -514,6 +514,7 @@ static PyMethodDef PyCursesPanel_Methods[] = { static PyType_Slot PyCursesPanel_Type_slots[] = { {Py_tp_dealloc, PyCursesPanel_Dealloc}, {Py_tp_methods, PyCursesPanel_Methods}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0}, }; @@ -656,7 +657,6 @@ _curses_panel_exec(PyObject *mod) if (state->PyCursesPanel_Type == NULL) { return -1; } - ((PyTypeObject *)state->PyCursesPanel_Type)->tp_new = NULL; if (PyModule_AddType(mod, state->PyCursesPanel_Type) < 0) { return -1; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 46d6a6e0954f51..2762ee652a6005 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -995,6 +995,7 @@ static PyType_Slot PyTclObject_Type_slots[] = { {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_richcompare, PyTclObject_richcompare}, {Py_tp_getset, PyTclObject_getsetlist}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -3287,6 +3288,7 @@ static PyType_Slot Tktt_Type_slots[] = { {Py_tp_dealloc, Tktt_Dealloc}, {Py_tp_repr, Tktt_Repr}, {Py_tp_methods, Tktt_methods}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -3341,6 +3343,7 @@ static PyMethodDef Tkapp_methods[] = static PyType_Slot Tkapp_Type_slots[] = { {Py_tp_dealloc, Tkapp_Dealloc}, {Py_tp_methods, Tkapp_methods}, + {Py_tp_new, _PyType_DisabledNew}, {0, 0} }; @@ -3537,7 +3540,6 @@ PyInit__tkinter(void) Py_DECREF(m); return NULL; } - ((PyTypeObject *)o)->tp_new = NULL; if (PyModule_AddObject(m, "TkappType", o)) { Py_DECREF(o); Py_DECREF(m); @@ -3550,7 +3552,6 @@ PyInit__tkinter(void) Py_DECREF(m); return NULL; } - ((PyTypeObject *)o)->tp_new = NULL; if (PyModule_AddObject(m, "TkttType", o)) { Py_DECREF(o); Py_DECREF(m); @@ -3563,7 +3564,6 @@ PyInit__tkinter(void) Py_DECREF(m); return NULL; } - ((PyTypeObject *)o)->tp_new = NULL; if (PyModule_AddObject(m, "Tcl_Obj", o)) { Py_DECREF(o); Py_DECREF(m); From d41b799a5a84bdb4461bfcd23c76bc03eb48f064 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Apr 2021 10:09:47 +0200 Subject: [PATCH 09/17] Add news (DRAFT) --- .../2021-04-28-10-09-16.bpo-43916.ecedNo.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-04-28-10-09-16.bpo-43916.ecedNo.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-28-10-09-16.bpo-43916.ecedNo.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-28-10-09-16.bpo-43916.ecedNo.rst new file mode 100644 index 00000000000000..cf25431e12b1d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-04-28-10-09-16.bpo-43916.ecedNo.rst @@ -0,0 +1,7 @@ +Fix regressions for extension modules that have been converted to :ref:`heap +types `. Inheritance differs for *static types* and *heap +types*: A *static type* with its :c:member:`~PyTypeObject.tp_new` set to +``NULL`` does not have a public contructor, but *heap types* inherit +contructors from their base classes, resulting in it being possible to +create instances without proper initialisation, possibly resulting in a +crash. From 3f34237ef9c143a364ea1ecc5a0bed9eff3a330e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Apr 2021 11:42:10 +0200 Subject: [PATCH 10/17] Move _PyType_DisabledNew to Include/internal and build affected extension modules with Py_BUILD_CORE_* --- Include/cpython/object.h | 3 --- Include/internal/pycore_object.h | 3 +++ Modules/Setup | 16 ++++++++-------- Modules/_curses_panel.c | 1 + Modules/_dbmmodule.c | 1 + Modules/_gdbmmodule.c | 1 + Modules/_sre.c | 1 + Modules/_threadmodule.c | 1 + Modules/_tkinter.c | 1 + Modules/arraymodule.c | 1 + Modules/cjkcodecs/multibytecodec.c | 1 + Modules/posixmodule.c | 1 + Modules/pyexpat.c | 1 + Modules/unicodedata.c | 1 + Modules/zlibmodule.c | 1 + setup.py | 15 +++++++++++++-- 16 files changed, 36 insertions(+), 13 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 5444358054e867..58e4d2b11b93f9 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -4,9 +4,6 @@ PyAPI_FUNC(void) _Py_NewReference(PyObject *op); -PyAPI_FUNC(PyObject *) _PyType_DisabledNew(PyTypeObject *, PyObject *, - PyObject *); - #ifdef Py_TRACE_REFS /* Py_TRACE_REFS is such major surgery that we call external routines. */ PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 9dfc8c62babadb..990478152f84a6 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -29,6 +29,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); +PyAPI_FUNC(PyObject *) _PyType_DisabledNew(PyTypeObject *type, PyObject *args, + PyObject *kwds); + /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and diff --git a/Modules/Setup b/Modules/Setup index 87c6a152f86eac..234cdba2a14adf 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -166,7 +166,7 @@ _symtable symtablemodule.c # Modules that should always be present (non UNIX dependent): -#array -DPy_BUILD_CORE_MODULE arraymodule.c # array objects +#array -DPy_BUILD_CORE_BUILTIN arraymodule.c # array objects #cmath cmathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # complex math library functions #math mathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # math library functions, e.g. sin() #_contextvars _contextvarsmodule.c # Context Variables @@ -279,7 +279,7 @@ _symtable symtablemodule.c # every system. # *** Always uncomment this (leave the leading underscore in!): -# _tkinter _tkinter.c tkappinit.c -DWITH_APPINIT \ +# _tkinter _tkinter.c tkappinit.c -DWITH_APPINIT -DPy_BUILD_CORE_BUILTIN \ # *** Uncomment and edit to reflect where your Tcl/Tk libraries are: # -L/usr/local/lib \ # *** Uncomment and edit to reflect where your Tcl/Tk headers are: @@ -320,7 +320,7 @@ _symtable symtablemodule.c #_curses _cursesmodule.c -lcurses -ltermcap -DPy_BUILD_CORE_MODULE # Wrapper for the panel library that's part of ncurses and SYSV curses. -#_curses_panel _curses_panel.c -lpanel -lncurses +#_curses_panel _curses_panel.c -lpanel -lncurses -DPy_BUILD_CORE_BUILTIN # Modules that provide persistent dictionary-like semantics. You will @@ -330,11 +330,11 @@ _symtable symtablemodule.c # implementation independent wrapper for these; dbm/dumb.py provides # similar functionality (but slower of course) implemented in Python. -#_dbm _dbmmodule.c # dbm(3) may require -lndbm or similar +#_dbm _dbmmodule.c -DPy_BUILD_CORE_BUILTIN # dbm(3) may require -lndbm or similar # Anthony Baxter's gdbm module. GNU dbm(3) will require -lgdbm: -#_gdbm _gdbmmodule.c -I/usr/local/include -L/usr/local/lib -lgdbm +#_gdbm _gdbmmodule.c -I/usr/local/include -L/usr/local/lib -lgdbm -DPy_BUILD_CORE_BUILTIN # Helper module for various ascii-encoders @@ -343,17 +343,17 @@ _symtable symtablemodule.c # Andrew Kuchling's zlib module. # This require zlib 1.1.3 (or later). # See http://www.gzip.org/zlib/ -#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz +#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz -DPy_BUILD_CORE_BUILTIN # Interface to the Expat XML parser # More information on Expat can be found at www.libexpat.org. # -#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI +#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI -DPy_BUILD_CORE_BUILTIN # Hye-Shik Chang's CJKCodecs # multibytecodec is required for all the other CJK codec modules -#_multibytecodec cjkcodecs/multibytecodec.c +#_multibytecodec cjkcodecs/multibytecodec.c -DPy_BUILD_CORE_BUILTIN #_codecs_cn cjkcodecs/_codecs_cn.c #_codecs_hk cjkcodecs/_codecs_hk.c diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 84868e715dd1c7..92e59e7bd6c1cc 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -13,6 +13,7 @@ static const char PyCursesVersion[] = "2.1"; #include "Python.h" #include "py_curses.h" +#include "pycore_object.h" // _PyType_DisabledNew() #include diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 247d50ef39ca35..7153df6f39a0fc 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -4,6 +4,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_object.h" // _PyType_DisabledNew() #include #include diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 7f752ebfca5692..f60bb19622567c 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -5,6 +5,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_object.h" // _PyType_DisabledNew() #include #include diff --git a/Modules/_sre.c b/Modules/_sre.c index 0838936d3413b7..ed8e515d59a456 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -43,6 +43,7 @@ static const char copyright[] = #include "Python.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyType_DisabledNew() #include "structmember.h" // PyMemberDef #include "sre.h" diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index fc867891e3481b..f1646540889adf 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -5,6 +5,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.num_threads #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyType_DisabledNew() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_Init() #include // offsetof() diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 2762ee652a6005..2a539099863f20 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -24,6 +24,7 @@ Copyright (C) 1994 Steen Lumholt. #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_object.h" // _PyType_DisabledNew() #include #ifdef MS_WINDOWS diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2c91f45a30758b..65d8dab4fd933f 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -6,6 +6,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyType_DisabledNew() #include "structmember.h" // PyMemberDef #include // offsetof() diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 3d81b557193904..2206c09126e51e 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -6,6 +6,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_object.h" // _PyType_DisabledNew() #include "structmember.h" // PyMemberDef #include "multibytecodec.h" #include "clinic/multibytecodec.c.h" diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2ffbe813866e17..af95faa13fb858 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -29,6 +29,7 @@ #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_object.h" // _PyType_DisabledNew() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "structmember.h" // PyMemberDef #ifndef MS_WINDOWS diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0cd9810bbdff92..01b850429ba489 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1,6 +1,7 @@ #include "Python.h" #include +#include "pycore_object.h" // _PyType_DisabledNew() #include "structmember.h" // PyMemberDef #include "frameobject.h" #include "expat.h" diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 0db05f99c10835..5ac58951df0bbe 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -16,6 +16,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_object.h" // _PyType_DisabledNew() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "structmember.h" // PyMemberDef diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 698f569e58c98d..810214a2404d67 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -6,6 +6,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_object.h" // _PyType_DisabledNew() #include "structmember.h" // PyMemberDef #include "zlib.h" diff --git a/setup.py b/setup.py index ca0ed8363ef358..c9469944e58cbe 100644 --- a/setup.py +++ b/setup.py @@ -1190,6 +1190,7 @@ def detect_readline_curses(self): self.add(Extension('_curses_panel', ['_curses_panel.c'], include_dirs=curses_includes, define_macros=curses_defines, + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], libraries=[panel_library, *curses_libs])) elif not skip_curses_panel: self.missing.append('_curses_panel') @@ -1440,6 +1441,7 @@ class db_found(Exception): pass else: dbm_order = "ndbm:gdbm:bdb".split(":") dbmext = None + dbm_compile_args = ['-DPy_BUILD_CORE_MODULE'] for cand in dbm_order: if cand == "ndbm": if find_file("ndbm.h", self.inc_dirs, []) is not None: @@ -1458,6 +1460,7 @@ class db_found(Exception): pass define_macros=[ ('HAVE_NDBM_H',None), ], + extra_compile_args=dbm_compile_args, libraries=ndbm_libs) break @@ -1474,6 +1477,7 @@ class db_found(Exception): pass define_macros=[ ('HAVE_GDBM_NDBM_H', None), ], + extra_compile_args=dbm_compile_args, libraries = gdbm_libs) break if find_file("gdbm-ndbm.h", self.inc_dirs, []) is not None: @@ -1483,6 +1487,7 @@ class db_found(Exception): pass define_macros=[ ('HAVE_GDBM_DASH_NDBM_H', None), ], + extra_compile_args=dbm_compile_args, libraries = gdbm_libs) break elif cand == "bdb": @@ -1496,6 +1501,7 @@ class db_found(Exception): pass ('HAVE_BERKDB_H', None), ('DB_DBM_HSEARCH', None), ], + extra_compile_args=dbm_compile_args, libraries=dblibs) break if dbmext is not None: @@ -1507,6 +1513,7 @@ class db_found(Exception): pass if ('gdbm' in dbm_order and self.compiler.find_library_file(self.lib_dirs, 'gdbm')): self.add(Extension('_gdbm', ['_gdbmmodule.c'], + extra_compile_args=dbm_compile_args, libraries=['gdbm'])) else: self.missing.append('_gdbm') @@ -1682,6 +1689,7 @@ def detect_compress_exts(self): zlib_extra_link_args = () self.add(Extension('zlib', ['zlibmodule.c'], libraries=['z'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], extra_link_args=zlib_extra_link_args)) have_zlib = True else: @@ -1752,7 +1760,7 @@ def detect_expat_elementtree(self): # call XML_SetHashSalt(), expat entropy sources are not needed ('XML_POOR_ENTROPY', '1'), ] - extra_compile_args = [] + extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] expat_lib = [] expat_sources = ['expat/xmlparse.c', 'expat/xmlrole.c', @@ -1802,7 +1810,8 @@ def detect_expat_elementtree(self): def detect_multibytecodecs(self): # Hye-Shik Chang's CJKCodecs modules. self.add(Extension('_multibytecodec', - ['cjkcodecs/multibytecodec.c'])) + ['cjkcodecs/multibytecodec.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) for loc in ('kr', 'jp', 'cn', 'tw', 'hk', 'iso2022'): self.add(Extension('_codecs_%s' % loc, ['cjkcodecs/_codecs_%s.c' % loc])) @@ -1925,6 +1934,7 @@ def detect_tkinter_fromenv(self): return False extra_compile_args = tcltk_includes.split() + extra_compile_args.append('-DPy_BUILD_CORE_MODULE') extra_link_args = tcltk_libs.split() self.add(Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], define_macros=[('WITH_APPINIT', 1)], @@ -2056,6 +2066,7 @@ def detect_tkinter_darwin(self): if '-Wstrict-prototypes' in cflags.split(): compile_args.append('-Wno-strict-prototypes') + compile_args.append('-DPy_BUILD_CORE_MODULE') self.add(Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], define_macros=[('WITH_APPINIT', 1)], include_dirs=include_dirs, From eee3044337d6e0174aa22216991eb84743d8472c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Apr 2021 18:26:21 +0200 Subject: [PATCH 11/17] Address review: don't truncate type names --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 25c00db9e2349f..45a9a73f687910 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1007,7 +1007,7 @@ type_repr(PyTypeObject *type) PyObject * _PyType_DisabledNew(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyErr_Format(PyExc_TypeError, "cannot create '%.100s' instances", + PyErr_Format(PyExc_TypeError, "cannot create '%s' instances", type->tp_name); return NULL; } From 5c7f047ac7989f3b5a80e579d1298e5bdf7eb82d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Apr 2021 18:32:02 +0200 Subject: [PATCH 12/17] Address review: bring the types outside the assertRaises() --- Lib/test/test_functools.py | 3 ++- Lib/test/test_posix.py | 3 ++- Lib/test/test_re.py | 4 ++-- Lib/test/test_threading.py | 3 ++- Lib/test/test_zlib.py | 8 ++++---- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index db5e18856df1dc..e3f7ddc18b242d 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -951,7 +951,8 @@ class TestCmpToKeyC(TestCmpToKey, unittest.TestCase): @support.cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) - self.assertRaises(TypeError, type(c_functools.cmp_to_key(None))) + tp = type(c_functools.cmp_to_key(None)) + self.assertRaises(TypeError, tp) class TestCmpToKeyPy(TestCmpToKey, unittest.TestCase): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 6e0794a7de0081..daed5a0e8fd695 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -62,7 +62,8 @@ def tearDown(self): def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) self.assertRaises(TypeError, posix.DirEntry) - self.assertRaises(TypeError, type(posix.scandir())) + tp = type(posix.scandir()) + self.assertRaises(TypeError, tp) def testNoArgFunctions(self): # test posix functions which take no arguments and have diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index abf95a286c677c..c131407a40786d 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2221,8 +2221,8 @@ def test_uninitialised_new(self): self.assertRaises(TypeError, re.Match) self.assertRaises(TypeError, re.Pattern) pat = re.compile("") - scn = pat.scanner("") - self.assertRaises(TypeError, type(scn)) + tp = type(pat.scanner("")) + self.assertRaises(TypeError, tp) class ExternalTests(unittest.TestCase): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index e18a003cd3792b..a1519c042f1adb 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -123,7 +123,8 @@ def func(): pass def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) lock = threading.Lock() - self.assertRaises(TypeError, type(lock)) + tp = type(lock) + self.assertRaises(TypeError, tp) # Create a bunch of threads, let each do some work, wait until all are # done. diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 83b33d73ff9864..274d51a24e20de 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -132,10 +132,10 @@ def test_overflow(self): @support.cpython_only def test_uninitialised_new(self): # Prevent heap types from being created uninitialised (bpo-43916) - comp = zlib.compressobj() - decomp = zlib.decompressobj() - self.assertRaises(TypeError, type(comp)) - self.assertRaises(TypeError, type(decomp)) + comp_type = type(zlib.compressobj()) + decomp_type = type(zlib.decompressobj()) + self.assertRaises(TypeError, comp_type) + self.assertRaises(TypeError, decomp_type) class BaseCompressTestCase(object): From 70dcae3b625f36f84e0182d7915648d4a302a2bd Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Apr 2021 01:25:34 +0200 Subject: [PATCH 13/17] Add missing include in _winapi.c --- Modules/_winapi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index f6bf5f10531ce2..5a9d462d052248 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -36,6 +36,7 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyType_DisabledNew() #include "structmember.h" // PyMemberDef From 7fb93a9c79cd5364ae77f45399a93ba47f3c7454 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Apr 2021 08:30:01 +0200 Subject: [PATCH 14/17] Include pycore_object late to avoid conflict with windows.h --- Modules/_winapi.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 5a9d462d052248..1cbde5b15a3fa2 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -36,7 +36,6 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_object.h" // _PyType_DisabledNew() #include "structmember.h" // PyMemberDef @@ -45,6 +44,9 @@ #include #include "winreparse.h" +// Include pycore_object.h to avoid conflict with windows.h +#include "pycore_object.h" // _PyType_DisabledNew() + #if defined(MS_WIN32) && !defined(MS_WIN64) #define HANDLE_TO_PYNUM(handle) \ PyLong_FromUnsignedLong((unsigned long) handle) From 66552f01b17016b2ea6b0d1761237316a2936614 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Apr 2021 10:05:08 +0200 Subject: [PATCH 15/17] Add gdbm test --- Lib/test/test_dbm_gnu.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 017d0ffa658bd6..e22440b8a45295 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -1,5 +1,5 @@ from test import support -from test.support import import_helper +from test.support import import_helper, cpython_only gdbm = import_helper.import_module("dbm.gnu") #skip if not supported import unittest import os @@ -27,6 +27,13 @@ def tearDown(self): self.g.close() unlink(filename) + @cpython_only + def test_uninitialised_new(self): + # Prevent heap types from being created uninitialised (bpo-43916) + self.g = gdbm.open(filename, 'c') + tp = type(self.g) + self.assertRaises(TypeError, tp) + def test_key_methods(self): self.g = gdbm.open(filename, 'c') self.assertEqual(self.g.keys(), []) From ce16da51678f2569b415bce45005983a7532dc41 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Apr 2021 10:55:11 +0200 Subject: [PATCH 16/17] Always build tkinter with Py_BUILD_CORE set --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index c9469944e58cbe..afea4404cfd725 100644 --- a/setup.py +++ b/setup.py @@ -2191,10 +2191,12 @@ def detect_tkinter(self): # *** Uncomment these for TOGL extension only: # -lGL -lGLU -lXext -lXmu \ + extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] self.add(Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], define_macros=[('WITH_APPINIT', 1)] + defs, include_dirs=include_dirs, libraries=libs, + extra_compile_args=extra_compile_args, library_dirs=added_lib_dirs)) return True From 14504853fd62506f03c8cb1ef7d5e695cac2b31f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Apr 2021 11:16:57 +0200 Subject: [PATCH 17/17] Revert changes to Modules/Setup --- Modules/Setup | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Modules/Setup b/Modules/Setup index 234cdba2a14adf..87c6a152f86eac 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -166,7 +166,7 @@ _symtable symtablemodule.c # Modules that should always be present (non UNIX dependent): -#array -DPy_BUILD_CORE_BUILTIN arraymodule.c # array objects +#array -DPy_BUILD_CORE_MODULE arraymodule.c # array objects #cmath cmathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # complex math library functions #math mathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # math library functions, e.g. sin() #_contextvars _contextvarsmodule.c # Context Variables @@ -279,7 +279,7 @@ _symtable symtablemodule.c # every system. # *** Always uncomment this (leave the leading underscore in!): -# _tkinter _tkinter.c tkappinit.c -DWITH_APPINIT -DPy_BUILD_CORE_BUILTIN \ +# _tkinter _tkinter.c tkappinit.c -DWITH_APPINIT \ # *** Uncomment and edit to reflect where your Tcl/Tk libraries are: # -L/usr/local/lib \ # *** Uncomment and edit to reflect where your Tcl/Tk headers are: @@ -320,7 +320,7 @@ _symtable symtablemodule.c #_curses _cursesmodule.c -lcurses -ltermcap -DPy_BUILD_CORE_MODULE # Wrapper for the panel library that's part of ncurses and SYSV curses. -#_curses_panel _curses_panel.c -lpanel -lncurses -DPy_BUILD_CORE_BUILTIN +#_curses_panel _curses_panel.c -lpanel -lncurses # Modules that provide persistent dictionary-like semantics. You will @@ -330,11 +330,11 @@ _symtable symtablemodule.c # implementation independent wrapper for these; dbm/dumb.py provides # similar functionality (but slower of course) implemented in Python. -#_dbm _dbmmodule.c -DPy_BUILD_CORE_BUILTIN # dbm(3) may require -lndbm or similar +#_dbm _dbmmodule.c # dbm(3) may require -lndbm or similar # Anthony Baxter's gdbm module. GNU dbm(3) will require -lgdbm: -#_gdbm _gdbmmodule.c -I/usr/local/include -L/usr/local/lib -lgdbm -DPy_BUILD_CORE_BUILTIN +#_gdbm _gdbmmodule.c -I/usr/local/include -L/usr/local/lib -lgdbm # Helper module for various ascii-encoders @@ -343,17 +343,17 @@ _symtable symtablemodule.c # Andrew Kuchling's zlib module. # This require zlib 1.1.3 (or later). # See http://www.gzip.org/zlib/ -#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz -DPy_BUILD_CORE_BUILTIN +#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz # Interface to the Expat XML parser # More information on Expat can be found at www.libexpat.org. # -#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI -DPy_BUILD_CORE_BUILTIN +#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI # Hye-Shik Chang's CJKCodecs # multibytecodec is required for all the other CJK codec modules -#_multibytecodec cjkcodecs/multibytecodec.c -DPy_BUILD_CORE_BUILTIN +#_multibytecodec cjkcodecs/multibytecodec.c #_codecs_cn cjkcodecs/_codecs_cn.c #_codecs_hk cjkcodecs/_codecs_hk.c