From d557a64a4cc82230a8a0368404a3862996effc3e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 09:00:07 -0800 Subject: [PATCH 01/72] PEP 585 step 2: Write a Py_GenericAlias() function that returns origin --- Include/descrobject.h | 4 ++++ Objects/descrobject.c | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/Include/descrobject.h b/Include/descrobject.h index ead269d1d2f796..04c509e4193c07 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -101,6 +101,10 @@ PyAPI_FUNC(PyObject *) PyWrapper_New(PyObject *, PyObject *); PyAPI_DATA(PyTypeObject) PyProperty_Type; + +// Experimental code to implement PEP 585 (list[int] etc.) +PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *); + #ifdef __cplusplus } #endif diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 342b993e090390..fdedfff2149dc4 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1790,3 +1790,22 @@ PyTypeObject PyProperty_Type = { PyType_GenericNew, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; + + +// Experimental code to implement PEP 585 (list[int] etc.) + +PyObject * +Py_GenericAlias(PyObject *origin, PyObject *parameters) +{ + // Step 1: just return `origin` + Py_INCREF(origin); + /* Debug + fprintf(stderr, "origin="); + PyObject_Print(origin, stderr, 0); + fprintf(stderr, "\n"); + fprintf(stderr, "parameters="); + PyObject_Print(parameters, stderr, 0); + fprintf(stderr, "\n"); + */ + return origin; +} From da332673eccb81a3e5f173f02e33871930ad7d64 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 10:26:10 -0800 Subject: [PATCH 02/72] PEP 585 step 3: Add Py_GenericAlias() as __class_getitem__ to tuple, list, dict, set, frozenset --- Objects/dictobject.c | 1 + Objects/listobject.c | 1 + Objects/setobject.c | 2 ++ Objects/tupleobject.c | 1 + 4 files changed, 5 insertions(+) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 87f88abbe53bd9..ddcbff4fbc45fe 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3212,6 +3212,7 @@ static PyMethodDef mapp_methods[] = { {"copy", (PyCFunction)dict_copy, METH_NOARGS, copy__doc__}, DICT___REVERSED___METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/listobject.c b/Objects/listobject.c index 2c07ceb0d412ce..e8b72885c59cd4 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2788,6 +2788,7 @@ static PyMethodDef list_methods[] = { LIST_COUNT_METHODDEF LIST_REVERSE_METHODDEF LIST_SORT_METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/setobject.c b/Objects/setobject.c index 924885d7505a0b..e0d13c847a5ab2 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2079,6 +2079,7 @@ static PyMethodDef set_methods[] = { union_doc}, {"update", (PyCFunction)set_update, METH_VARARGS, update_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2190,6 +2191,7 @@ static PyMethodDef frozenset_methods[] = { symmetric_difference_doc}, {"union", (PyCFunction)set_union, METH_VARARGS, union_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 08f7022fda25a6..8a549c8e86567f 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -854,6 +854,7 @@ static PyMethodDef tuple_methods[] = { TUPLE___GETNEWARGS___METHODDEF TUPLE_INDEX_METHODDEF TUPLE_COUNT_METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; From 447fbe9308de63b15d99c2c3e6c6c5b960eeb801 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 10:27:04 -0800 Subject: [PATCH 03/72] PEP 585 step 4: Write some tests --- Lib/test/test_genericalias.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 Lib/test/test_genericalias.py diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py new file mode 100644 index 00000000000000..da5206f79e809b --- /dev/null +++ b/Lib/test/test_genericalias.py @@ -0,0 +1,24 @@ +"""Tests for C-implemented GenericAlias.""" + +import unittest + + +class BaseTest(unittest.TestCase): + """Test basics.""" + + def test_subscriptable(self): + for t in tuple, list, dict, set, frozenset: + tname = t.__name__ + with self.subTest(f"Testing {tname}"): + self.assertEqual(t[int], t) + + def test_unsubscriptable(self): + for t in int, str, float: + tname = t.__name__ + with self.subTest(f"Testing {tname}"): + with self.assertRaises(TypeError): + t[int] + + +if __name__ == "__main__": + unittest.main() From 83e2ba25481768d867d0534d2aa7a66fc45b37a7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 11:49:06 -0800 Subject: [PATCH 04/72] Define a GenericAlias (proxy) object (with lots of TODOs still) --- Lib/test/test_genericalias.py | 3 +- Objects/descrobject.c | 145 +++++++++++++++++++++++++++++++++- 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index da5206f79e809b..aa07d057120b88 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -10,7 +10,8 @@ def test_subscriptable(self): for t in tuple, list, dict, set, frozenset: tname = t.__name__ with self.subTest(f"Testing {tname}"): - self.assertEqual(t[int], t) + t[int] + ## self.assertEqual(t[int], t) def test_unsubscriptable(self): for t in int, str, float: diff --git a/Objects/descrobject.c b/Objects/descrobject.c index fdedfff2149dc4..4bf2c55cd99546 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1793,12 +1793,130 @@ PyTypeObject PyProperty_Type = { // Experimental code to implement PEP 585 (list[int] etc.) +// TODO: Does this belong in this file? + +typedef struct { + PyObject_HEAD + PyObject *origin; + PyObject *parameters; +} gaobject; + +static void +ga_dealloc(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(alias->origin); + Py_XDECREF(alias->parameters); + self->ob_type->tp_free(self); +} + +static int +ga_traverse(PyObject *self, visitproc visit, void *arg) +{ + gaobject *alias = (gaobject *)self; + Py_VISIT(alias->origin); + Py_VISIT(alias->parameters); + return 0; +} + +// TODO: nicely format origin and each parameter +static PyObject * +ga_repr(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + return PyUnicode_FromFormat("%R[%R]", alias->origin, alias->parameters); +} + +static PyObject * +ga_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + gaobject *alias = (gaobject *)self; + return PyObject_Call(alias->origin, args, kwds); +} + +static const char* attr_exceptions[] = { + "__origin__", + "__parameters__", + NULL, +}; + +static PyObject * +ga_getattro(PyObject *self, PyObject *name) +{ + gaobject *alias = (gaobject *)self; + // TODO: Use ga_members instead? + if (alias->origin != NULL && PyUnicode_Check(name)) { + for (const char **p = attr_exceptions; ; p++) { + if (*p == NULL) { + return PyObject_GenericGetAttr(alias->origin, name); + } + if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { + break; + } + } + } + return PyObject_GenericGetAttr(self, name); +} + +static PyMemberDef ga_members[] = { + {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, + {"__parameters__", T_OBJECT, offsetof(gaobject, parameters), READONLY}, + {0} +}; + +// TODO: Maybe move this to tp_new(), so we won't ever have origin==NULL? +static int +ga_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + gaobject *alias = (gaobject *)self; + if (alias->origin != NULL || alias->parameters != NULL) { + PyErr_Format(PyExc_TypeError, "This GenericAlias is alreay initialized"); + return -1; + } + if (kwds != NULL) { + PyErr_Format(PyExc_TypeError, "GenericAlias does not support keyword arguments"); + return -1; + } + if (!PyTuple_Check(args) || PyTuple_Size(args) != 2) { + PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); + return -1; + } + PyObject *origin = PyTuple_GET_ITEM(args, 0); + PyObject *parameters = PyTuple_GET_ITEM(args, 1); + Py_INCREF(origin); + Py_INCREF(parameters); + alias->origin = origin; + alias->parameters = parameters; + return 0; +} + +// TODO: +// - __mro_entries__, to support class C(list[int]): ... +// - argument clinic +// - __doc__ +// - cache +PyTypeObject Py_GenericAliasType = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "GenericAlias", + .tp_basicsize = sizeof(gaobject), + .tp_dealloc = ga_dealloc, + .tp_repr = ga_repr, + .tp_call = ga_call, + .tp_getattro = ga_getattro, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = ga_traverse, + .tp_members = ga_members, + .tp_init = ga_init, + .tp_alloc = PyType_GenericAlloc, + .tp_new = PyType_GenericNew, + .tp_free = PyObject_GC_Del, +}; PyObject * Py_GenericAlias(PyObject *origin, PyObject *parameters) { - // Step 1: just return `origin` - Py_INCREF(origin); /* Debug fprintf(stderr, "origin="); PyObject_Print(origin, stderr, 0); @@ -1807,5 +1925,26 @@ Py_GenericAlias(PyObject *origin, PyObject *parameters) PyObject_Print(parameters, stderr, 0); fprintf(stderr, "\n"); */ - return origin; + PyObject *wrapper = NULL; + if (!PyTuple_Check(parameters)) { + wrapper = PyTuple_New(1); + if (wrapper == NULL) + return NULL; + Py_INCREF(parameters); + PyTuple_SetItem(wrapper, 0, parameters); + parameters = wrapper; + } + + gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); + if (alias == NULL) { + Py_XDECREF(wrapper); + return NULL; + } + + Py_INCREF(origin); + Py_INCREF(parameters); + alias->origin = origin; + alias->parameters = parameters; + _PyObject_GC_TRACK(alias); + return (PyObject *)alias; } From fb3e8d389394660b35673d084c056b58fde2e0bf Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 16:51:58 -0800 Subject: [PATCH 05/72] More tests --- Lib/test/test_genericalias.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index aa07d057120b88..bd741ac6fd4b8c 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -10,8 +10,9 @@ def test_subscriptable(self): for t in tuple, list, dict, set, frozenset: tname = t.__name__ with self.subTest(f"Testing {tname}"): - t[int] - ## self.assertEqual(t[int], t) + alias = t[int] + self.assertIs(alias.__origin__, t) + self.assertEqual(alias.__parameters__, (int,)) def test_unsubscriptable(self): for t in int, str, float: @@ -20,6 +21,27 @@ def test_unsubscriptable(self): with self.assertRaises(TypeError): t[int] + def test_instantiate(self): + for t in tuple, list, dict, set, frozenset: + tname = t.__name__ + with self.subTest(f"Testing {tname}"): + alias = t[int] + self.assertEqual(alias(), t()) + if t is dict: + self.assertEqual(alias(iter([('a', 1), ('b', 2)])), dict(a=1, b=2)) + self.assertEqual(alias(a=1, b=2), dict(a=1, b=2)) + else: + self.assertEqual(alias(iter((1, 2, 3))), t((1, 2, 3))) + + def test_unbound_methods(self): + t = list[int] + a = t() + t.append(a, 'foo') + self.assertEqual(a, ['foo']) + x = t.__getitem__(a, 0) + self.assertEqual(x, 'foo') + self.assertEqual(t.__len__(a), 1) + if __name__ == "__main__": unittest.main() From 235180b767f5f455873c8638f1b39c377aa2254e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 17:04:14 -0800 Subject: [PATCH 06/72] PEP 585 steps 5-6: Finish basic GenericAlias object and test --- Lib/test/test_genericalias.py | 6 ++++++ Objects/descrobject.c | 22 +++++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index bd741ac6fd4b8c..a3d28e96ab9ce5 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -42,6 +42,12 @@ def test_unbound_methods(self): self.assertEqual(x, 'foo') self.assertEqual(t.__len__(a), 1) + def test_subclassing(self): + class C(list[int]): + pass + self.assertEqual(C.__bases__, (list,)) + self.assertEqual(C.__class__, type) + if __name__ == "__main__": unittest.main() diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 4bf2c55cd99546..f04d350809e7e5 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1839,6 +1839,7 @@ ga_call(PyObject *self, PyObject *args, PyObject *kwds) static const char* attr_exceptions[] = { "__origin__", "__parameters__", + "__mro_entries__", NULL, }; @@ -1846,7 +1847,6 @@ static PyObject * ga_getattro(PyObject *self, PyObject *name) { gaobject *alias = (gaobject *)self; - // TODO: Use ga_members instead? if (alias->origin != NULL && PyUnicode_Check(name)) { for (const char **p = attr_exceptions; ; p++) { if (*p == NULL) { @@ -1860,6 +1860,18 @@ ga_getattro(PyObject *self, PyObject *name) return PyObject_GenericGetAttr(self, name); } +static PyObject * +ga_mro_entries(PyObject *self, PyObject *args) +{ + gaobject *alias = (gaobject *)self; + return Py_BuildValue("(O)", alias->origin); +} + +static PyMethodDef ga_methods[] = { + {"__mro_entries__", ga_mro_entries, METH_O}, + {0} +}; + static PyMemberDef ga_members[] = { {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, {"__parameters__", T_OBJECT, offsetof(gaobject, parameters), READONLY}, @@ -1893,10 +1905,9 @@ ga_init(PyObject *self, PyObject *args, PyObject *kwds) } // TODO: -// - __mro_entries__, to support class C(list[int]): ... -// - argument clinic -// - __doc__ -// - cache +// - argument clinic? +// - __doc__? +// - cache? PyTypeObject Py_GenericAliasType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "GenericAlias", @@ -1907,6 +1918,7 @@ PyTypeObject Py_GenericAliasType = { .tp_getattro = ga_getattro, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_traverse = ga_traverse, + .tp_methods = ga_methods, .tp_members = ga_members, .tp_init = ga_init, .tp_alloc = PyType_GenericAlloc, From 70f3abc1609305f8d37fe47638d6bbd9da38a9ce Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 17:58:39 -0800 Subject: [PATCH 07/72] Fix bug with class methods --- Lib/test/test_genericalias.py | 5 +++++ Objects/descrobject.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index a3d28e96ab9ce5..5baf59e641bf37 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -48,6 +48,11 @@ class C(list[int]): self.assertEqual(C.__bases__, (list,)) self.assertEqual(C.__class__, type) + def test_class_methods(self): + t = dict[int, None] + self.assertEqual(dict.fromkeys(range(2)), {0: None, 1: None}) # This works + self.assertEqual(t.fromkeys(range(2)), {0: None, 1: None}) # Should be equivalent + if __name__ == "__main__": unittest.main() diff --git a/Objects/descrobject.c b/Objects/descrobject.c index f04d350809e7e5..649502f98b0ecb 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1850,7 +1850,7 @@ ga_getattro(PyObject *self, PyObject *name) if (alias->origin != NULL && PyUnicode_Check(name)) { for (const char **p = attr_exceptions; ; p++) { if (*p == NULL) { - return PyObject_GenericGetAttr(alias->origin, name); + return PyObject_GetAttr(alias->origin, name); } if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { break; From e4d1356edfd758387fc838a87f03628c201c2aae Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 21:25:19 -0800 Subject: [PATCH 08/72] Define tp_new instead of tp_init --- Objects/descrobject.c | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 649502f98b0ecb..818bfe2a044e77 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1878,30 +1878,18 @@ static PyMemberDef ga_members[] = { {0} }; -// TODO: Maybe move this to tp_new(), so we won't ever have origin==NULL? -static int -ga_init(PyObject *self, PyObject *args, PyObject *kwds) +static PyObject * +ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - gaobject *alias = (gaobject *)self; - if (alias->origin != NULL || alias->parameters != NULL) { - PyErr_Format(PyExc_TypeError, "This GenericAlias is alreay initialized"); - return -1; - } if (kwds != NULL) { - PyErr_Format(PyExc_TypeError, "GenericAlias does not support keyword arguments"); - return -1; + return PyErr_Format(PyExc_TypeError, "GenericAlias does not support keyword arguments"); } if (!PyTuple_Check(args) || PyTuple_Size(args) != 2) { - PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); - return -1; + return PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); } PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *parameters = PyTuple_GET_ITEM(args, 1); - Py_INCREF(origin); - Py_INCREF(parameters); - alias->origin = origin; - alias->parameters = parameters; - return 0; + return Py_GenericAlias(origin, parameters); } // TODO: @@ -1920,23 +1908,14 @@ PyTypeObject Py_GenericAliasType = { .tp_traverse = ga_traverse, .tp_methods = ga_methods, .tp_members = ga_members, - .tp_init = ga_init, .tp_alloc = PyType_GenericAlloc, - .tp_new = PyType_GenericNew, + .tp_new = ga_new, .tp_free = PyObject_GC_Del, }; PyObject * Py_GenericAlias(PyObject *origin, PyObject *parameters) { - /* Debug - fprintf(stderr, "origin="); - PyObject_Print(origin, stderr, 0); - fprintf(stderr, "\n"); - fprintf(stderr, "parameters="); - PyObject_Print(parameters, stderr, 0); - fprintf(stderr, "\n"); - */ PyObject *wrapper = NULL; if (!PyTuple_Check(parameters)) { wrapper = PyTuple_New(1); From 8f7a55e31f97d0bc7e82394a5d17244f59e31bf3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 21:47:56 -0800 Subject: [PATCH 09/72] Test that list[int][int] fails --- Lib/test/test_genericalias.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 5baf59e641bf37..c0ff1fbe5a0b4d 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -53,6 +53,11 @@ def test_class_methods(self): self.assertEqual(dict.fromkeys(range(2)), {0: None, 1: None}) # This works self.assertEqual(t.fromkeys(range(2)), {0: None, 1: None}) # Should be equivalent + def test_no_chaining(self): + t = list[int] + with self.assertRaises(TypeError): + t[int] + if __name__ == "__main__": unittest.main() From f2ba2e53c09be858736e866c05b2acd671fa6176 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 Jan 2020 22:00:58 -0800 Subject: [PATCH 10/72] Test that class L(list): pass; L[int] works --- Lib/test/test_genericalias.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index c0ff1fbe5a0b4d..0c892d957b093d 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -58,6 +58,13 @@ def test_no_chaining(self): with self.assertRaises(TypeError): t[int] + def test_generic_subclass(self): + class MyList(list): + pass + t = MyList[int] + self.assertIs(t.__origin__, MyList) + self.assertEqual(t.__parameters__, (int,)) + if __name__ == "__main__": unittest.main() From 0af74d105b66621ee666b8dcaf2a518fb7ea2d30 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 28 Jan 2020 07:27:27 -0800 Subject: [PATCH 11/72] Implement PEP 585 for collections.deque and ... (#2) ... collections.defaultdict --- Lib/test/test_genericalias.py | 12 +++++++++--- Modules/_collectionsmodule.c | 4 ++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 0c892d957b093d..b38fe803ca0a92 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -1,13 +1,13 @@ """Tests for C-implemented GenericAlias.""" import unittest - +from collections import defaultdict, deque class BaseTest(unittest.TestCase): """Test basics.""" def test_subscriptable(self): - for t in tuple, list, dict, set, frozenset: + for t in tuple, list, dict, set, frozenset, defaultdict, deque: tname = t.__name__ with self.subTest(f"Testing {tname}"): alias = t[int] @@ -22,7 +22,7 @@ def test_unsubscriptable(self): t[int] def test_instantiate(self): - for t in tuple, list, dict, set, frozenset: + for t in tuple, list, dict, set, frozenset, defaultdict, deque: tname = t.__name__ with self.subTest(f"Testing {tname}"): alias = t[int] @@ -30,6 +30,12 @@ def test_instantiate(self): if t is dict: self.assertEqual(alias(iter([('a', 1), ('b', 2)])), dict(a=1, b=2)) self.assertEqual(alias(a=1, b=2), dict(a=1, b=2)) + elif t is defaultdict: + def default(): + return 'value' + a = alias(default) + d = defaultdict(default) + self.assertEqual(a['test'], d['test']) else: self.assertEqual(alias(iter((1, 2, 3))), t((1, 2, 3))) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 1d23973fd05661..d670b5a465cbb2 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1605,6 +1605,8 @@ static PyMethodDef deque_methods[] = { METH_FASTCALL, rotate_doc}, {"__sizeof__", (PyCFunction)deque_sizeof, METH_NOARGS, sizeof_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ }; @@ -2067,6 +2069,8 @@ static PyMethodDef defdict_methods[] = { defdict_copy_doc}, {"__reduce__", (PyCFunction)defdict_reduce, METH_NOARGS, reduce_doc}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {NULL} }; From b2642948cd0de776cd2315119b366847c3ad5735 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 28 Jan 2020 09:15:03 -0800 Subject: [PATCH 12/72] Respond to Serhiy's code review --- Objects/descrobject.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 818bfe2a044e77..3cf8b56aaee4f6 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1836,7 +1836,7 @@ ga_call(PyObject *self, PyObject *args, PyObject *kwds) return PyObject_Call(alias->origin, args, kwds); } -static const char* attr_exceptions[] = { +static const char* const attr_exceptions[] = { "__origin__", "__parameters__", "__mro_entries__", @@ -1847,8 +1847,8 @@ static PyObject * ga_getattro(PyObject *self, PyObject *name) { gaobject *alias = (gaobject *)self; - if (alias->origin != NULL && PyUnicode_Check(name)) { - for (const char **p = attr_exceptions; ; p++) { + if (PyUnicode_Check(name)) { + for (const char * const *p = attr_exceptions; ; p++) { if (*p == NULL) { return PyObject_GetAttr(alias->origin, name); } @@ -1864,7 +1864,7 @@ static PyObject * ga_mro_entries(PyObject *self, PyObject *args) { gaobject *alias = (gaobject *)self; - return Py_BuildValue("(O)", alias->origin); + return PyTuple_Pack(1, alias->origin); } static PyMethodDef ga_methods[] = { @@ -1881,10 +1881,10 @@ static PyMemberDef ga_members[] = { static PyObject * ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (kwds != NULL) { + if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { return PyErr_Format(PyExc_TypeError, "GenericAlias does not support keyword arguments"); } - if (!PyTuple_Check(args) || PyTuple_Size(args) != 2) { + if (PyTuple_Size(args) != 2) { return PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); } PyObject *origin = PyTuple_GET_ITEM(args, 0); @@ -1916,24 +1916,23 @@ PyTypeObject Py_GenericAliasType = { PyObject * Py_GenericAlias(PyObject *origin, PyObject *parameters) { - PyObject *wrapper = NULL; if (!PyTuple_Check(parameters)) { - wrapper = PyTuple_New(1); - if (wrapper == NULL) + parameters = PyTuple_Pack(1, parameters); + if (parameters == NULL) { return NULL; + } + } + else { Py_INCREF(parameters); - PyTuple_SetItem(wrapper, 0, parameters); - parameters = wrapper; } gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); if (alias == NULL) { - Py_XDECREF(wrapper); + Py_DECREF(parameters); return NULL; } Py_INCREF(origin); - Py_INCREF(parameters); alias->origin = origin; alias->parameters = parameters; _PyObject_GC_TRACK(alias); From 0ddea5628af6f799e98a02886fbe3c46ce43c4a7 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2020 17:19:25 +0000 Subject: [PATCH 13/72] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst new file mode 100644 index 00000000000000..9652a3fccd7107 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-28-17-19-18.bpo-39481.rqSeGl.rst @@ -0,0 +1 @@ +Implement PEP 585. This supports list[int], tuple[str, ...] etc. \ No newline at end of file From 91edf1c6b22f2ff94ad7b6fe7490068a619b19e0 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 28 Jan 2020 16:03:05 -0800 Subject: [PATCH 14/72] PEP 585 steps 7-8: Implement repr for GenericAlias (#1) By Ethan Smith. --- Lib/test/test_genericalias.py | 9 ++++ Objects/descrobject.c | 81 ++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index b38fe803ca0a92..0131f33ab536d7 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -71,6 +71,15 @@ class MyList(list): self.assertIs(t.__origin__, MyList) self.assertEqual(t.__parameters__, (int,)) + def test_repr(self): + class MyList(list): + pass + self.assertEqual(repr(list[str]), 'list[str]') + self.assertEqual(repr(list[()]), 'list[()]') + self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]') + self.assertTrue(repr(MyList[int]).endswith('BaseTest.test_repr..MyList[int]')) + self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr + if __name__ == "__main__": unittest.main() diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 3cf8b56aaee4f6..2f8ea3284a7d2e 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1821,12 +1821,89 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) return 0; } -// TODO: nicely format origin and each parameter +int +ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) +{ + PyObject *qualname = PyObject_GetAttrString(p, "__qualname__"); + PyObject *module = PyObject_GetAttrString(p, "__module__"); + PyObject *r = NULL; + int err; + if (PyObject_HasAttrString(p, "__origin__") && + PyObject_HasAttrString(p, "__parameters__")) + { + // It looks like a GenericAlias + r = PyObject_Repr(p); + } + else if (p == Py_Ellipsis) { + // The Ellipsis object + r = PyUnicode_FromString("..."); + } + else if (qualname != NULL && module != NULL) { + // Looks like a class + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + // builtins don't need a module name + Py_INCREF(qualname); + r = qualname; + } + else { + r = PyUnicode_FromFormat("%U.%U", module, qualname); + } + } + else { + // fallback + r = PyObject_Repr(p); + } + if (r == NULL) { + // error if any of the above PyObject_Repr/PyUnicode_From* fail + err = -1; + } else { + err = _PyUnicodeWriter_WriteStr(writer, r); + } + Py_XDECREF(qualname); + Py_XDECREF(module); + Py_XDECREF(r); + return err; +} + static PyObject * ga_repr(PyObject *self) { gaobject *alias = (gaobject *)self; - return PyUnicode_FromFormat("%R[%R]", alias->origin, alias->parameters); + Py_ssize_t len = PyTuple_Size(alias->parameters); + + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + + if (ga_repr_item(&writer, alias->origin) < 0) { + goto error; + } + if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) { + goto error; + } + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + goto error; + } + } + PyObject *p = PyTuple_GET_ITEM(alias->parameters, i); + if (ga_repr_item(&writer, p) < 0) { + goto error; + } + } + if (len == 0) { + // for something like tuple[()] we should print a "()" + if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) { + goto error; + } + } + if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) { + goto error; + } + return _PyUnicodeWriter_Finish(&writer); +error: + _PyUnicodeWriter_Dealloc(&writer); + return NULL; } static PyObject * From 6b1a218162e45e83de7be8e6bfaa4984b3963e66 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 28 Jan 2020 16:52:06 -0800 Subject: [PATCH 15/72] Add static keyword to ga_repr_item (#3) --- Objects/descrobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 2f8ea3284a7d2e..4ff9204401a951 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1821,7 +1821,7 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) return 0; } -int +static int ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) { PyObject *qualname = PyObject_GetAttrString(p, "__qualname__"); From ce21a96d2c32718550af45f8279bbb47013cf40a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 28 Jan 2020 18:16:53 -0800 Subject: [PATCH 16/72] Add GenericAlias to types.py --- Lib/types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/types.py b/Lib/types.py index ea3c0b29d5d4dd..ad2020ec69b637 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -293,4 +293,7 @@ def wrapped(*args, **kwargs): return wrapped +GenericAlias = type(list[int]) + + __all__ = [n for n in globals() if n[:1] != '_'] From dbe67d0b995adbafffc3f11f2bbbb5522bba6899 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 28 Jan 2020 18:20:35 -0800 Subject: [PATCH 17/72] Add test for types.py Also strengthen test for repr(MyList[int]) --- Lib/test/test_genericalias.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 0131f33ab536d7..cedb62fc39b242 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -77,9 +77,16 @@ class MyList(list): self.assertEqual(repr(list[str]), 'list[str]') self.assertEqual(repr(list[()]), 'list[()]') self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]') - self.assertTrue(repr(MyList[int]).endswith('BaseTest.test_repr..MyList[int]')) + self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr + def test_exposed_type(self): + import types + a = types.GenericAlias(list, int) + self.assertEqual(str(a), 'list[int]') + self.assertIs(a.__origin__, list) + self.assertEqual(a.__parameters__, (int,)) + if __name__ == "__main__": unittest.main() From 15697f09c9674763d493b683c3f193334c1b3a73 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 28 Jan 2020 19:42:26 -0800 Subject: [PATCH 18/72] Fix tests that were sensitive to the presence of __class_getitem__ Note that UserDict and UserList are now generic --- Lib/collections/__init__.py | 7 +++++++ Lib/test/test_descrtut.py | 1 + 2 files changed, 8 insertions(+) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index cec6c9781a15e0..6c6f8b3fe2bd1e 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -25,6 +25,7 @@ from _weakref import proxy as _proxy from itertools import repeat as _repeat, chain as _chain, starmap as _starmap from reprlib import recursive_repr as _recursive_repr +from types import GenericAlias as _GenericAlias try: from _collections import deque @@ -961,6 +962,10 @@ def __init__(self, dict=None, /, **kwargs): if kwargs: self.update(kwargs) + # It's a generic class, just like dict. + def __class_getitem__(cls, item): + return _GenericAlias(cls, item) + def __len__(self): return len(self.data) def __getitem__(self, key): if key in self.data: @@ -1024,6 +1029,8 @@ def __init__(self, initlist=None): self.data[:] = initlist.data[:] else: self.data = list(initlist) + def __class_getitem__(cls, item): + return _GenericAlias(cls, item) def __repr__(self): return repr(self.data) def __lt__(self, other): return self.data < self.__cast(other) def __le__(self, other): return self.data <= self.__cast(other) diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index b84d6447850a06..8e25f58d7aa20e 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -167,6 +167,7 @@ def merge(self, other): >>> pprint.pprint(dir(list)) # like list.__dict__.keys(), but sorted ['__add__', '__class__', + '__class_getitem__', '__contains__', '__delattr__', '__delitem__', From da15d43335abb002b6c36332881c3af4b25bb3d8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 28 Jan 2020 20:31:11 -0800 Subject: [PATCH 19/72] Fix crash in repr(tuple[...]) -- call PyErr_Clear() --- Objects/descrobject.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 4ff9204401a951..901cf2a91b3d55 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1825,7 +1825,9 @@ static int ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) { PyObject *qualname = PyObject_GetAttrString(p, "__qualname__"); + PyErr_Clear(); PyObject *module = PyObject_GetAttrString(p, "__module__"); + PyErr_Clear(); PyObject *r = NULL; int err; if (PyObject_HasAttrString(p, "__origin__") && From 647cda8fab6cc18a7dbbec1ffc51513231d5a427 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 29 Jan 2020 07:07:15 -0800 Subject: [PATCH 20/72] Make re.Match, re.Pattern, and io.IOBase generic (#4) --- Lib/test/test_genericalias.py | 4 +++- Modules/_io/iobase.c | 2 ++ Modules/_sre.c | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index cedb62fc39b242..853440e6c8ef56 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -2,12 +2,14 @@ import unittest from collections import defaultdict, deque +from io import IOBase +from re import Pattern, Match class BaseTest(unittest.TestCase): """Test basics.""" def test_subscriptable(self): - for t in tuple, list, dict, set, frozenset, defaultdict, deque: + for t in tuple, list, dict, set, frozenset, defaultdict, deque, IOBase, Pattern, Match: tname = t.__name__ with self.subTest(f"Testing {tname}"): alias = t[int] diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index d51fc944e1a629..e72a24f480e75a 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -824,6 +824,8 @@ static PyMethodDef iobase_methods[] = { _IO__IOBASE_READLINES_METHODDEF _IO__IOBASE_WRITELINES_METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {NULL, NULL} }; diff --git a/Modules/_sre.c b/Modules/_sre.c index 6518e98f04e409..d47da3d664f5db 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -2568,6 +2568,8 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN_SCANNER_METHODDEF _SRE_SRE_PATTERN___COPY___METHODDEF _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {NULL, NULL} }; @@ -2638,6 +2640,8 @@ static PyMethodDef match_methods[] = { _SRE_SRE_MATCH_EXPAND_METHODDEF _SRE_SRE_MATCH___COPY___METHODDEF _SRE_SRE_MATCH___DEEPCOPY___METHODDEF + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {NULL, NULL} }; From 382495476ffe1efd03f790c043c4d2adb6b4aa03 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 09:25:22 -0800 Subject: [PATCH 21/72] Restore parity between _io._IOBase and _pyio.IOBase --- Lib/_pyio.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 4c2414672ed56c..a07610a9b4e250 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -8,6 +8,7 @@ import errno import stat import sys +import types # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock if sys.platform in {'win32', 'cygwin'}: @@ -340,6 +341,10 @@ def _unsupported(self, name): raise UnsupportedOperation("%s.%s() not supported" % (self.__class__.__name__, name)) + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return types.GenericAlias(cls, item) + ### Positioning ### def seek(self, pos, whence=0): From 8aec6ae15063388bd4902c45500042522da4744d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 10:42:57 -0800 Subject: [PATCH 22/72] Make contextlib.Abstract*ContextManager generic, with tests --- Lib/contextlib.py | 8 +++++++- Lib/test/test_genericalias.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 69c272831a55c2..7601a8f7c20775 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -4,7 +4,7 @@ import _collections_abc from collections import deque from functools import wraps -from types import MethodType +from types import MethodType, GenericAlias __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", @@ -16,6 +16,9 @@ class AbstractContextManager(abc.ABC): """An abstract base class for context managers.""" + def __class_getitem__(cls, parameters): + return GenericAlias(cls, parameters) + def __enter__(self): """Return `self` upon entering the runtime context.""" return self @@ -36,6 +39,9 @@ class AbstractAsyncContextManager(abc.ABC): """An abstract base class for asynchronous context managers.""" + def __class_getitem__(cls, parameters): + return GenericAlias(cls, parameters) + async def __aenter__(self): """Return `self` upon entering the runtime context.""" return self diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 853440e6c8ef56..3289e0bf90d75a 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -2,6 +2,7 @@ import unittest from collections import defaultdict, deque +from contextlib import AbstractContextManager, AbstractAsyncContextManager from io import IOBase from re import Pattern, Match @@ -9,7 +10,12 @@ class BaseTest(unittest.TestCase): """Test basics.""" def test_subscriptable(self): - for t in tuple, list, dict, set, frozenset, defaultdict, deque, IOBase, Pattern, Match: + for t in (tuple, list, dict, set, frozenset, + defaultdict, deque, + IOBase, + Pattern, Match, + AbstractContextManager, AbstractAsyncContextManager, + ): tname = t.__name__ with self.subTest(f"Testing {tname}"): alias = t[int] From 58af1834d51c27b21132173509f07714887822a9 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 15:09:16 -0800 Subject: [PATCH 23/72] WIP: Tweak test_typing.py to pass with the new definition of __orig_class__ --- Lib/test/test_typing.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 5b4916f9c3260b..eb703faf570b59 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1817,18 +1817,21 @@ def test_type_erasure_special(self): # this is the only test that checks type caching self.clear_caches() class MyTup(Tuple[T, T]): ... - self.assertIs(MyTup[int]().__class__, MyTup) - self.assertIs(MyTup[int]().__orig_class__, MyTup[int]) + tt = MyTup[int] + self.assertIs(tt().__class__, MyTup) + self.assertIs(tt().__orig_class__, tt) class MyCall(Callable[..., T]): def __call__(self): return None self.assertIs(MyCall[T]().__class__, MyCall) self.assertIs(MyCall[T]().__orig_class__, MyCall[T]) class MyDict(typing.Dict[T, T]): ... - self.assertIs(MyDict[int]().__class__, MyDict) - self.assertIs(MyDict[int]().__orig_class__, MyDict[int]) + dd = MyDict[int] + self.assertIs(dd().__class__, MyDict) + self.assertIs(dd().__orig_class__, dd) class MyDef(typing.DefaultDict[str, T]): ... - self.assertIs(MyDef[int]().__class__, MyDef) - self.assertIs(MyDef[int]().__orig_class__, MyDef[int]) + df = MyDef[int] + self.assertIs(df().__class__, MyDef) + self.assertIs(df().__orig_class__, df) # ChainMap was added in 3.3 if sys.version_info >= (3, 3): class MyChain(typing.ChainMap[str, T]): ... From 8da0a089f5a3561438bed0fff5dbac6b87ae262c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 14:02:48 -0800 Subject: [PATCH 24/72] Rename __parameters__ to __args__ --- Lib/test/test_genericalias.py | 6 +++--- Objects/descrobject.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 3289e0bf90d75a..0e7b5a0810a6be 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -20,7 +20,7 @@ def test_subscriptable(self): with self.subTest(f"Testing {tname}"): alias = t[int] self.assertIs(alias.__origin__, t) - self.assertEqual(alias.__parameters__, (int,)) + self.assertEqual(alias.__args__, (int,)) def test_unsubscriptable(self): for t in int, str, float: @@ -77,7 +77,7 @@ class MyList(list): pass t = MyList[int] self.assertIs(t.__origin__, MyList) - self.assertEqual(t.__parameters__, (int,)) + self.assertEqual(t.__args__, (int,)) def test_repr(self): class MyList(list): @@ -93,7 +93,7 @@ def test_exposed_type(self): a = types.GenericAlias(list, int) self.assertEqual(str(a), 'list[int]') self.assertIs(a.__origin__, list) - self.assertEqual(a.__parameters__, (int,)) + self.assertEqual(a.__args__, (int,)) if __name__ == "__main__": diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 901cf2a91b3d55..000f2e0b0eb848 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1831,7 +1831,7 @@ ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) PyObject *r = NULL; int err; if (PyObject_HasAttrString(p, "__origin__") && - PyObject_HasAttrString(p, "__parameters__")) + PyObject_HasAttrString(p, "__args__")) { // It looks like a GenericAlias r = PyObject_Repr(p); @@ -1917,7 +1917,7 @@ ga_call(PyObject *self, PyObject *args, PyObject *kwds) static const char* const attr_exceptions[] = { "__origin__", - "__parameters__", + "__args__", "__mro_entries__", NULL, }; @@ -1953,7 +1953,7 @@ static PyMethodDef ga_methods[] = { static PyMemberDef ga_members[] = { {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, - {"__parameters__", T_OBJECT, offsetof(gaobject, parameters), READONLY}, + {"__args__", T_OBJECT, offsetof(gaobject, parameters), READONLY}, {0} }; From eaecacb060805df0223b2123d04d9f91b8b3a3b3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 14:10:00 -0800 Subject: [PATCH 25/72] Rename parameters to args Keep not-yet-functional parameters, and add __parameters__ to match needed. --- Lib/test/test_typing.py | 12 ++++++------ Objects/descrobject.c | 30 ++++++++++++++++++------------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index eb703faf570b59..1b7edfb1d1997f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1653,8 +1653,8 @@ class C(List[T][U][V]): ... class D(C, List[T][U][V]): ... self.assertEqual(C.__parameters__, (V,)) self.assertEqual(D.__parameters__, (V,)) - self.assertEqual(C[int].__parameters__, ()) - self.assertEqual(D[int].__parameters__, ()) + ## self.assertEqual(C[int].__parameters__, ()) + ## self.assertEqual(D[int].__parameters__, ()) self.assertEqual(C[int].__args__, (int,)) self.assertEqual(D[int].__args__, (int,)) self.assertEqual(C.__bases__, (list, Generic)) @@ -1776,10 +1776,10 @@ def __call__(self): self.assertEqual(T1[int, T].__origin__, T1) self.assertEqual(T2.__parameters__, (T,)) - with self.assertRaises(TypeError): - T1[int] - with self.assertRaises(TypeError): - T2[int, str] + ## with self.assertRaises(TypeError): + ## T1[int] + ## with self.assertRaises(TypeError): + ## T2[int, str] self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') self.assertEqual(C2.__parameters__, ()) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 000f2e0b0eb848..0adb9ebfbe2ea9 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1798,6 +1798,7 @@ PyTypeObject PyProperty_Type = { typedef struct { PyObject_HEAD PyObject *origin; + PyObject *args; PyObject *parameters; } gaobject; @@ -1808,6 +1809,7 @@ ga_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); Py_XDECREF(alias->origin); + Py_XDECREF(alias->args); Py_XDECREF(alias->parameters); self->ob_type->tp_free(self); } @@ -1817,6 +1819,7 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) { gaobject *alias = (gaobject *)self; Py_VISIT(alias->origin); + Py_VISIT(alias->args); Py_VISIT(alias->parameters); return 0; } @@ -1871,7 +1874,7 @@ static PyObject * ga_repr(PyObject *self) { gaobject *alias = (gaobject *)self; - Py_ssize_t len = PyTuple_Size(alias->parameters); + Py_ssize_t len = PyTuple_Size(alias->args); _PyUnicodeWriter writer; _PyUnicodeWriter_Init(&writer); @@ -1888,7 +1891,7 @@ ga_repr(PyObject *self) goto error; } } - PyObject *p = PyTuple_GET_ITEM(alias->parameters, i); + PyObject *p = PyTuple_GET_ITEM(alias->args, i); if (ga_repr_item(&writer, p) < 0) { goto error; } @@ -1918,6 +1921,7 @@ ga_call(PyObject *self, PyObject *args, PyObject *kwds) static const char* const attr_exceptions[] = { "__origin__", "__args__", + "__parameters__", "__mro_entries__", NULL, }; @@ -1953,7 +1957,8 @@ static PyMethodDef ga_methods[] = { static PyMemberDef ga_members[] = { {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, - {"__args__", T_OBJECT, offsetof(gaobject, parameters), READONLY}, + {"__args__", T_OBJECT, offsetof(gaobject, args), READONLY}, + {"__parameters__", T_OBJECT_EX, offsetof(gaobject, parameters), READONLY}, {0} }; @@ -1967,8 +1972,8 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); } PyObject *origin = PyTuple_GET_ITEM(args, 0); - PyObject *parameters = PyTuple_GET_ITEM(args, 1); - return Py_GenericAlias(origin, parameters); + PyObject *arguments = PyTuple_GET_ITEM(args, 1); + return Py_GenericAlias(origin, arguments); } // TODO: @@ -1993,27 +1998,28 @@ PyTypeObject Py_GenericAliasType = { }; PyObject * -Py_GenericAlias(PyObject *origin, PyObject *parameters) +Py_GenericAlias(PyObject *origin, PyObject *args) { - if (!PyTuple_Check(parameters)) { - parameters = PyTuple_Pack(1, parameters); - if (parameters == NULL) { + if (!PyTuple_Check(args)) { + args = PyTuple_Pack(1, args); + if (args == NULL) { return NULL; } } else { - Py_INCREF(parameters); + Py_INCREF(args); } gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); if (alias == NULL) { - Py_DECREF(parameters); + Py_DECREF(args); return NULL; } Py_INCREF(origin); alias->origin = origin; - alias->parameters = parameters; + alias->args = args; + alias->parameters = NULL; _PyObject_GC_TRACK(alias); return (PyObject *)alias; } From 98d479fc198281ab84b79a6fd8ea257e2923fb14 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 14:36:59 -0800 Subject: [PATCH 26/72] Properly implement __parameters__ as the tuple of args that are type vars --- Lib/test/test_genericalias.py | 27 ++++++++++++++++++++ Lib/test/test_typing.py | 12 ++++----- Objects/descrobject.c | 47 +++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 0e7b5a0810a6be..bcb7d37621ed57 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -21,6 +21,7 @@ def test_subscriptable(self): alias = t[int] self.assertIs(alias.__origin__, t) self.assertEqual(alias.__args__, (int,)) + self.assertEqual(alias.__parameters__, ()) def test_unsubscriptable(self): for t in int, str, float: @@ -78,6 +79,7 @@ class MyList(list): t = MyList[int] self.assertIs(t.__origin__, MyList) self.assertEqual(t.__args__, (int,)) + self.assertEqual(t.__parameters__, ()) def test_repr(self): class MyList(list): @@ -94,6 +96,31 @@ def test_exposed_type(self): self.assertEqual(str(a), 'list[int]') self.assertIs(a.__origin__, list) self.assertEqual(a.__args__, (int,)) + self.assertEqual(a.__parameters__, ()) + + def test_parameters(self): + from typing import TypeVar + T = TypeVar('T') + K = TypeVar('K') + V = TypeVar('V') + D0 = dict[str, int] + self.assertEqual(D0.__args__, (str, int)) + self.assertEqual(D0.__parameters__, ()) + D1a = dict[str, V] + self.assertEqual(D1a.__args__, (str, V)) + self.assertEqual(D1a.__parameters__, (V,)) + D1b = dict[K, int] + self.assertEqual(D1b.__args__, (K, int)) + self.assertEqual(D1b.__parameters__, (K,)) + D2 = dict[K, V] + self.assertEqual(D2.__args__, (K, V)) + self.assertEqual(D2.__parameters__, (K, V)) + L0 = list[str] + self.assertEqual(L0.__args__, (str,)) + self.assertEqual(L0.__parameters__, ()) + L1 = list[T] + self.assertEqual(L1.__args__, (T,)) + self.assertEqual(L1.__parameters__, (T,)) if __name__ == "__main__": diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1b7edfb1d1997f..eb703faf570b59 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1653,8 +1653,8 @@ class C(List[T][U][V]): ... class D(C, List[T][U][V]): ... self.assertEqual(C.__parameters__, (V,)) self.assertEqual(D.__parameters__, (V,)) - ## self.assertEqual(C[int].__parameters__, ()) - ## self.assertEqual(D[int].__parameters__, ()) + self.assertEqual(C[int].__parameters__, ()) + self.assertEqual(D[int].__parameters__, ()) self.assertEqual(C[int].__args__, (int,)) self.assertEqual(D[int].__args__, (int,)) self.assertEqual(C.__bases__, (list, Generic)) @@ -1776,10 +1776,10 @@ def __call__(self): self.assertEqual(T1[int, T].__origin__, T1) self.assertEqual(T2.__parameters__, (T,)) - ## with self.assertRaises(TypeError): - ## T1[int] - ## with self.assertRaises(TypeError): - ## T2[int, str] + with self.assertRaises(TypeError): + T1[int] + with self.assertRaises(TypeError): + T2[int, str] self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') self.assertEqual(C2.__parameters__, ()) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 0adb9ebfbe2ea9..fbe4c674446a48 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1958,7 +1958,7 @@ static PyMethodDef ga_methods[] = { static PyMemberDef ga_members[] = { {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, {"__args__", T_OBJECT, offsetof(gaobject, args), READONLY}, - {"__parameters__", T_OBJECT_EX, offsetof(gaobject, parameters), READONLY}, + {"__parameters__", T_OBJECT, offsetof(gaobject, parameters), READONLY}, {0} }; @@ -1997,6 +1997,44 @@ PyTypeObject Py_GenericAliasType = { .tp_free = PyObject_GC_Del, }; +// isinstance(obj, TypeVar) without importing typing.py. If someone +// names some other class TypeVar, it will be mistaken for a TypeVar. +// Maybe that's a feature; or maybe we'll have to see if +// sys.modules['typing'] exists and look for its 'TypeVar' attribute +// (which is roughly what dataclasses.py uses to recognize ClassVar). +static int +is_typevar(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + return strcmp(type->tp_name, "TypeVar") == 0; +} + +// tuple(t for t in args if isinstance(t, TypeVar)) +static PyObject * +make_parameters(PyObject *args) +{ + int len = PyTuple_GET_SIZE(args); + PyObject *parameters = PyTuple_New(len); + if (parameters == NULL) + return NULL; + int iparam = 0; + for (int iarg = 0; iarg < len; iarg++) { + PyObject *t = PyTuple_GET_ITEM(args, iarg); + if (is_typevar(t)) { + Py_INCREF(t); + PyTuple_SET_ITEM(parameters, iparam, t); + iparam++; + } + } + if (iparam < len) { + if (_PyTuple_Resize(¶meters, iparam) < 0) { + Py_XDECREF(parameters); + return NULL; + } + } + return parameters; +} + PyObject * Py_GenericAlias(PyObject *origin, PyObject *args) { @@ -2019,7 +2057,12 @@ Py_GenericAlias(PyObject *origin, PyObject *args) Py_INCREF(origin); alias->origin = origin; alias->args = args; - alias->parameters = NULL; + // TODO: Make __parameters__ a lazy attribute + alias->parameters = make_parameters(args); + if (alias->parameters == NULL) { + Py_DECREF(alias); + return NULL; + } _PyObject_GC_TRACK(alias); return (PyObject *)alias; } From 82bc8060b378c842e9af24a3cbc4bb1d4d39cfb1 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 15:08:51 -0800 Subject: [PATCH 27/72] Set __orig_class__ if possible when instantiating --- Objects/descrobject.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index fbe4c674446a48..889d7dad3b9b0e 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1915,7 +1915,12 @@ static PyObject * ga_call(PyObject *self, PyObject *args, PyObject *kwds) { gaobject *alias = (gaobject *)self; - return PyObject_Call(alias->origin, args, kwds); + PyObject *obj = PyObject_Call(alias->origin, args, kwds); + if (obj != NULL) { + PyObject_SetAttrString(obj, "__orig_class__", self); + PyErr_Clear(); + } + return obj; } static const char* const attr_exceptions[] = { From 2567ce37a5f910ff7be570e69eca90eea03bf7d5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Jan 2020 21:55:20 -0800 Subject: [PATCH 28/72] WIP: Implement ga_getitem so list[T][int] -> list[int] However, dict[T, int][str] -> dict[str], which is wrong! Similar, dict[T, T][str] fails but should return dict[str, str]. Finally, dict[T, T].__parameters__ -> (T, T) but should be (T,). --- Objects/descrobject.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 889d7dad3b9b0e..fbff4e018151bc 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1911,6 +1911,42 @@ ga_repr(PyObject *self) return NULL; } +PyTypeObject Py_GenericAliasType; // Forward + +static PyObject * +ga_getitem(PyObject *self, PyObject *item) +{ + PyObject *parameters = PyObject_GetAttrString(self, "__parameters__"); + if (parameters == NULL) { + PyErr_Clear(); + } + else if (!PyTuple_Check(parameters)) { + Py_DECREF(parameters); + } + else { + int nparams = PyTuple_Size(parameters); + Py_DECREF(parameters); + if (nparams < 0) + return NULL; + int nitems = PyTuple_Check(item) ? PyTuple_Size(item) : 1; + if (nitems < 0) + return NULL; + if (nitems != nparams) { + return PyErr_Format(PyExc_TypeError, "Incorrect parameter count"); + } + } + if (Py_TYPE(self) == &Py_GenericAliasType) { + gaobject *alias = (gaobject *)self; + // TODO: merge alias->args and items + return Py_GenericAlias(alias->origin, item); + } + return PyObject_GetItem((PyObject *)Py_TYPE(self), item); +} + +static PyMappingMethods ga_as_mapping = { + .mp_subscript = ga_getitem, +}; + static PyObject * ga_call(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1985,12 +2021,14 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) // - argument clinic? // - __doc__? // - cache? +// - __eq__ PyTypeObject Py_GenericAliasType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "GenericAlias", .tp_basicsize = sizeof(gaobject), .tp_dealloc = ga_dealloc, .tp_repr = ga_repr, + .tp_as_mapping = &ga_as_mapping, .tp_call = ga_call, .tp_getattro = ga_getattro, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, @@ -2026,6 +2064,7 @@ make_parameters(PyObject *args) for (int iarg = 0; iarg < len; iarg++) { PyObject *t = PyTuple_GET_ITEM(args, iarg); if (is_typevar(t)) { + // TODO: This is wrong! tuple[T, T].__parameters__ should be (T,) Py_INCREF(t); PyTuple_SET_ITEM(parameters, iparam, t); iparam++; From 4f2ced6b4e61d23fa487cfabe0ce3a459e0b3f49 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 11:16:34 -0800 Subject: [PATCH 29/72] Exclude duplicates from __parameters__ Also move declaration of Py_GenericAliasType to descrobject.h. --- Include/descrobject.h | 1 + Lib/test/test_genericalias.py | 9 ++++++--- Objects/descrobject.c | 23 +++++++++++++++++------ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Include/descrobject.h b/Include/descrobject.h index 04c509e4193c07..006361b11940ef 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -104,6 +104,7 @@ PyAPI_DATA(PyTypeObject) PyProperty_Type; // Experimental code to implement PEP 585 (list[int] etc.) PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *); +PyAPI_DATA(PyTypeObject) Py_GenericAliasType; #ifdef __cplusplus } diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index bcb7d37621ed57..3b8d181afd6d94 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -112,9 +112,12 @@ def test_parameters(self): D1b = dict[K, int] self.assertEqual(D1b.__args__, (K, int)) self.assertEqual(D1b.__parameters__, (K,)) - D2 = dict[K, V] - self.assertEqual(D2.__args__, (K, V)) - self.assertEqual(D2.__parameters__, (K, V)) + D2a = dict[K, V] + self.assertEqual(D2a.__args__, (K, V)) + self.assertEqual(D2a.__parameters__, (K, V)) + D2b = dict[T, T] + self.assertEqual(D2b.__args__, (T, T)) + self.assertEqual(D2b.__parameters__, (T,)) L0 = list[str] self.assertEqual(L0.__args__, (str,)) self.assertEqual(L0.__parameters__, ()) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index fbff4e018151bc..3bb560df4629f1 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1911,8 +1911,6 @@ ga_repr(PyObject *self) return NULL; } -PyTypeObject Py_GenericAliasType; // Forward - static PyObject * ga_getitem(PyObject *self, PyObject *item) { @@ -2052,6 +2050,18 @@ is_typevar(PyObject *obj) return strcmp(type->tp_name, "TypeVar") == 0; } +// Index of item in self[:len], or -1 if not found (self is a tuple) +static int +tuple_index(PyObject *self, int len, PyObject *item) +{ + for (int i = 0; i < len; i++) { + if (PyTuple_GET_ITEM(self, i) == item) { + return i; + } + } + return -1; +} + // tuple(t for t in args if isinstance(t, TypeVar)) static PyObject * make_parameters(PyObject *args) @@ -2064,10 +2074,11 @@ make_parameters(PyObject *args) for (int iarg = 0; iarg < len; iarg++) { PyObject *t = PyTuple_GET_ITEM(args, iarg); if (is_typevar(t)) { - // TODO: This is wrong! tuple[T, T].__parameters__ should be (T,) - Py_INCREF(t); - PyTuple_SET_ITEM(parameters, iparam, t); - iparam++; + if (tuple_index(parameters, iparam, t) < 0) { + Py_INCREF(t); + PyTuple_SET_ITEM(parameters, iparam, t); + iparam++; + } } } if (iparam < len) { From 2c25fb0b4cf78836d471094a10142633573411d5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 16:36:55 -0800 Subject: [PATCH 30/72] Implement ga_getitem properly, with tests --- Lib/test/test_genericalias.py | 13 +++++ Objects/descrobject.c | 107 +++++++++++++++++++--------------- 2 files changed, 73 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 3b8d181afd6d94..12822ecb5131ae 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -125,6 +125,19 @@ def test_parameters(self): self.assertEqual(L1.__args__, (T,)) self.assertEqual(L1.__parameters__, (T,)) + def test_parameter_chaining(self): + from typing import TypeVar + T = TypeVar('T') + self.assertEqual(repr(list[T][int]), 'list[int]') + self.assertEqual(repr(dict[str, T][int]), 'dict[str, int]') + self.assertEqual(repr(dict[T, int][str]), 'dict[str, int]') + self.assertEqual(repr(dict[T, T][int]), 'dict[int, int]') + with self.assertRaises(TypeError): + list[int][int] + dict[T, int][str, int] + dict[str, T][str, int] + dict[T, T][str, int] + if __name__ == "__main__": unittest.main() diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 3bb560df4629f1..e606ae89a28813 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1911,34 +1911,71 @@ ga_repr(PyObject *self) return NULL; } +// isinstance(obj, TypeVar) without importing typing.py. If someone +// names some other class TypeVar, it will be mistaken for a TypeVar. +// Maybe that's a feature; or maybe we'll have to see if +// sys.modules['typing'] exists and look for its 'TypeVar' attribute +// (which is roughly what dataclasses.py uses to recognize ClassVar). +static int +is_typevar(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + return strcmp(type->tp_name, "TypeVar") == 0; +} + +// Index of item in self[:len], or -1 if not found (self is a tuple) +static int +tuple_index(PyObject *self, int len, PyObject *item) +{ + for (int i = 0; i < len; i++) { + if (PyTuple_GET_ITEM(self, i) == item) { + return i; + } + } + return -1; +} + static PyObject * ga_getitem(PyObject *self, PyObject *item) { - PyObject *parameters = PyObject_GetAttrString(self, "__parameters__"); - if (parameters == NULL) { - PyErr_Clear(); - } - else if (!PyTuple_Check(parameters)) { - Py_DECREF(parameters); - } - else { - int nparams = PyTuple_Size(parameters); - Py_DECREF(parameters); - if (nparams < 0) - return NULL; - int nitems = PyTuple_Check(item) ? PyTuple_Size(item) : 1; - if (nitems < 0) - return NULL; - if (nitems != nparams) { - return PyErr_Format(PyExc_TypeError, "Incorrect parameter count"); + gaobject *alias = (gaobject *)self; + int nparams = PyTuple_GET_SIZE(alias->parameters); + if (nparams == 0) { + return PyErr_Format(PyExc_TypeError, + "There are no type variables left in %R", + self); + } + int is_tuple = PyTuple_Check(item); + int nitem = is_tuple ? PyTuple_GET_SIZE(item) : 1; + if (nitem != nparams) { + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %R", + nitem > nparams ? "many" : "few", + self); + } + int nargs = PyTuple_GET_SIZE(alias->args); + PyObject *newargs = PyTuple_New(nargs); + if (newargs == NULL) + return NULL; + for (int iarg = 0; iarg < nargs; iarg++) { + PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + if (is_typevar(arg)) { + int iparam = tuple_index(alias->parameters, nparams, arg); + assert(iparam >= 0); + if (is_tuple) { + arg = PyTuple_GET_ITEM(item, iparam); + } + else { + assert(iparam == 0); + arg = item; + } } + Py_INCREF(arg); + PyTuple_SET_ITEM(newargs, iarg, arg); } - if (Py_TYPE(self) == &Py_GenericAliasType) { - gaobject *alias = (gaobject *)self; - // TODO: merge alias->args and items - return Py_GenericAlias(alias->origin, item); - } - return PyObject_GetItem((PyObject *)Py_TYPE(self), item); + PyObject *res = Py_GenericAlias(alias->origin, newargs); + Py_DECREF(newargs); + return res; } static PyMappingMethods ga_as_mapping = { @@ -2038,30 +2075,6 @@ PyTypeObject Py_GenericAliasType = { .tp_free = PyObject_GC_Del, }; -// isinstance(obj, TypeVar) without importing typing.py. If someone -// names some other class TypeVar, it will be mistaken for a TypeVar. -// Maybe that's a feature; or maybe we'll have to see if -// sys.modules['typing'] exists and look for its 'TypeVar' attribute -// (which is roughly what dataclasses.py uses to recognize ClassVar). -static int -is_typevar(PyObject *obj) -{ - PyTypeObject *type = Py_TYPE(obj); - return strcmp(type->tp_name, "TypeVar") == 0; -} - -// Index of item in self[:len], or -1 if not found (self is a tuple) -static int -tuple_index(PyObject *self, int len, PyObject *item) -{ - for (int i = 0; i < len; i++) { - if (PyTuple_GET_ITEM(self, i) == item) { - return i; - } - } - return -1; -} - // tuple(t for t in args if isinstance(t, TypeVar)) static PyObject * make_parameters(PyObject *args) From 9ed17ef14738dd71b65c50394e438350f26f20b0 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 18:15:54 -0800 Subject: [PATCH 31/72] Implement == and != for GA --- Lib/test/test_genericalias.py | 15 +++++++++++---- Lib/test/test_typing.py | 15 ++++++--------- Objects/descrobject.c | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 12822ecb5131ae..36da9f846f7fe7 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -128,16 +128,23 @@ def test_parameters(self): def test_parameter_chaining(self): from typing import TypeVar T = TypeVar('T') - self.assertEqual(repr(list[T][int]), 'list[int]') - self.assertEqual(repr(dict[str, T][int]), 'dict[str, int]') - self.assertEqual(repr(dict[T, int][str]), 'dict[str, int]') - self.assertEqual(repr(dict[T, T][int]), 'dict[int, int]') + self.assertEqual(list[T][int], list[int]) + self.assertEqual(dict[str, T][int], dict[str, int]) + self.assertEqual(dict[T, int][str], dict[str, int]) + self.assertEqual(dict[T, T][int], dict[int, int]) with self.assertRaises(TypeError): list[int][int] dict[T, int][str, int] dict[str, T][str, int] dict[T, T][str, int] + def test_equality(self): + self.assertEqual(list[int], list[int]) + self.assertEqual(dict[str, int], dict[str, int]) + self.assertNotEqual(dict[str, int], dict[str, str]) + self.assertNotEqual(list, list[int]) + self.assertNotEqual(list[int], list) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index eb703faf570b59..a5a2e7ceaa73cb 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1817,21 +1817,18 @@ def test_type_erasure_special(self): # this is the only test that checks type caching self.clear_caches() class MyTup(Tuple[T, T]): ... - tt = MyTup[int] - self.assertIs(tt().__class__, MyTup) - self.assertIs(tt().__orig_class__, tt) + self.assertIs(MyTup[int]().__class__, MyTup) + self.assertEqual(MyTup[int]().__orig_class__, MyTup[int]) class MyCall(Callable[..., T]): def __call__(self): return None self.assertIs(MyCall[T]().__class__, MyCall) self.assertIs(MyCall[T]().__orig_class__, MyCall[T]) class MyDict(typing.Dict[T, T]): ... - dd = MyDict[int] - self.assertIs(dd().__class__, MyDict) - self.assertIs(dd().__orig_class__, dd) + self.assertIs(MyDict[int]().__class__, MyDict) + self.assertEqual(MyDict[int]().__orig_class__, MyDict[int]) class MyDef(typing.DefaultDict[str, T]): ... - df = MyDef[int] - self.assertIs(df().__class__, MyDef) - self.assertIs(df().__orig_class__, df) + self.assertIs(MyDef[int]().__class__, MyDef) + self.assertEqual(MyDef[int]().__orig_class__, MyDef[int]) # ChainMap was added in 3.3 if sys.version_info >= (3, 3): class MyChain(typing.ChainMap[str, T]): ... diff --git a/Objects/descrobject.c b/Objects/descrobject.c index e606ae89a28813..e8d2ccab922c60 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -2019,6 +2019,41 @@ ga_getattro(PyObject *self, PyObject *name) return PyObject_GenericGetAttr(self, name); } +static PyObject * +ga_richcompare(PyObject *a, PyObject *b, int op) +{ + if (Py_TYPE(a) != &Py_GenericAliasType || + Py_TYPE(b) != &Py_GenericAliasType || + (op != Py_EQ && op != Py_NE)) + { + Py_RETURN_NOTIMPLEMENTED; + } + + if (op == Py_NE) { + PyObject *eq = ga_richcompare(a, b, Py_EQ); + if (eq == NULL) + return NULL; + Py_DECREF(eq); + if (eq == Py_True) { + Py_RETURN_FALSE; + } + else { + Py_RETURN_TRUE; + } + } + + gaobject *aa = (gaobject *)a; + gaobject *bb = (gaobject *)b; + PyObject *eq = PyObject_RichCompare(aa->origin, bb->origin, Py_EQ); + if (eq == NULL) + return NULL; + if (eq == Py_False) { + return eq; + } + Py_DECREF(eq); + return PyObject_RichCompare(aa->args, bb->args, Py_EQ); +} + static PyObject * ga_mro_entries(PyObject *self, PyObject *args) { @@ -2068,6 +2103,7 @@ PyTypeObject Py_GenericAliasType = { .tp_getattro = ga_getattro, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_traverse = ga_traverse, + .tp_richcompare = ga_richcompare, .tp_methods = ga_methods, .tp_members = ga_members, .tp_alloc = PyType_GenericAlloc, From bb9a11ffd15b2002f7077ac6dbff7f09b62e63c3 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 30 Jan 2020 18:19:28 -0800 Subject: [PATCH 32/72] Make collections.abc.* generic (#5) --- Lib/_collections_abc.py | 38 +++++++++++++++++++++++++++++++++++ Lib/collections/__init__.py | 6 ------ Lib/test/test_genericalias.py | 22 +++++++++++++++++--- Lib/typing.py | 2 +- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 2b2ddba170e1b1..1f811cc2f6c5ba 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -8,6 +8,7 @@ from abc import ABCMeta, abstractmethod import sys +from types import GenericAlias __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", @@ -110,6 +111,10 @@ def __subclasshook__(cls, C): return _check_methods(C, "__await__") return NotImplemented + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return GenericAlias(cls, item) + class Coroutine(Awaitable): @@ -169,6 +174,10 @@ def __subclasshook__(cls, C): return _check_methods(C, "__aiter__") return NotImplemented + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return GenericAlias(cls, item) + class AsyncIterator(AsyncIterable): @@ -255,6 +264,10 @@ def __subclasshook__(cls, C): return _check_methods(C, "__iter__") return NotImplemented + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return GenericAlias(cls, item) + class Iterator(Iterable): @@ -274,6 +287,7 @@ def __subclasshook__(cls, C): return _check_methods(C, '__iter__', '__next__') return NotImplemented + Iterator.register(bytes_iterator) Iterator.register(bytearray_iterator) #Iterator.register(callable_iterator) @@ -353,6 +367,7 @@ def __subclasshook__(cls, C): 'send', 'throw', 'close') return NotImplemented + Generator.register(generator) @@ -385,6 +400,11 @@ def __subclasshook__(cls, C): return _check_methods(C, "__contains__") return NotImplemented + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return GenericAlias(cls, item) + + class Collection(Sized, Iterable, Container): __slots__ = () @@ -395,6 +415,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__len__", "__iter__", "__contains__") return NotImplemented + class Callable(metaclass=ABCMeta): __slots__ = () @@ -409,6 +430,10 @@ def __subclasshook__(cls, C): return _check_methods(C, "__call__") return NotImplemented + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return GenericAlias(cls, item) + ### SETS ### @@ -550,6 +575,7 @@ def _hash(self): h = 590923713 return h + Set.register(frozenset) @@ -632,6 +658,7 @@ def __isub__(self, it): self.discard(value) return self + MutableSet.register(set) @@ -688,6 +715,7 @@ def __eq__(self, other): __reversed__ = None + Mapping.register(mappingproxy) @@ -704,6 +732,10 @@ def __len__(self): def __repr__(self): return '{0.__class__.__name__}({0._mapping!r})'.format(self) + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return GenericAlias(cls, item) + class KeysView(MappingView, Set): @@ -719,6 +751,7 @@ def __contains__(self, key): def __iter__(self): yield from self._mapping + KeysView.register(dict_keys) @@ -743,6 +776,7 @@ def __iter__(self): for key in self._mapping: yield (key, self._mapping[key]) + ItemsView.register(dict_items) @@ -761,6 +795,7 @@ def __iter__(self): for key in self._mapping: yield self._mapping[key] + ValuesView.register(dict_values) @@ -847,6 +882,7 @@ def setdefault(self, key, default=None): self[key] = default return default + MutableMapping.register(dict) @@ -914,6 +950,7 @@ def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' return sum(1 for v in self if v is value or v == value) + Sequence.register(tuple) Sequence.register(str) Sequence.register(range) @@ -1000,5 +1037,6 @@ def __iadd__(self, values): self.extend(values) return self + MutableSequence.register(list) MutableSequence.register(bytearray) # Multiply inheriting, see ByteString diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 6c6f8b3fe2bd1e..f7f432b145dc4d 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -962,10 +962,6 @@ def __init__(self, dict=None, /, **kwargs): if kwargs: self.update(kwargs) - # It's a generic class, just like dict. - def __class_getitem__(cls, item): - return _GenericAlias(cls, item) - def __len__(self): return len(self.data) def __getitem__(self, key): if key in self.data: @@ -1029,8 +1025,6 @@ def __init__(self, initlist=None): self.data[:] = initlist.data[:] else: self.data = list(initlist) - def __class_getitem__(cls, item): - return _GenericAlias(cls, item) def __repr__(self): return repr(self.data) def __lt__(self, other): return self.data < self.__cast(other) def __le__(self, other): return self.data <= self.__cast(other) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 36da9f846f7fe7..9e54049f07565d 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -1,20 +1,36 @@ """Tests for C-implemented GenericAlias.""" import unittest -from collections import defaultdict, deque +from collections import ( + defaultdict, deque, OrderedDict, Counter, UserDict, UserList +) +from collections.abc import * from contextlib import AbstractContextManager, AbstractAsyncContextManager from io import IOBase from re import Pattern, Match + class BaseTest(unittest.TestCase): """Test basics.""" def test_subscriptable(self): for t in (tuple, list, dict, set, frozenset, - defaultdict, deque, + defaultdict, deque, + OrderedDict, Counter, UserDict, UserList, IOBase, Pattern, Match, AbstractContextManager, AbstractAsyncContextManager, + Awaitable, Coroutine, + AsyncIterable, AsyncIterator, + AsyncGenerator, Generator, + Iterable, Iterator, + Reversible, + Container, Collection, + Callable, + Set, MutableSet, + Mapping, MutableMapping, MappingView, + KeysView, ItemsView, ValuesView, + Sequence, MutableSequence, ): tname = t.__name__ with self.subTest(f"Testing {tname}"): @@ -24,7 +40,7 @@ def test_subscriptable(self): self.assertEqual(alias.__parameters__, ()) def test_unsubscriptable(self): - for t in int, str, float: + for t in int, str, float, Sized, Hashable: tname = t.__name__ with self.subTest(f"Testing {tname}"): with self.assertRaises(TypeError): diff --git a/Lib/typing.py b/Lib/typing.py index 7de3e346eaa79a..954997271c2413 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -946,7 +946,7 @@ class _TypingEllipsis: _SPECIAL_NAMES = ['__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', - '__subclasshook__', '__weakref__'] + '__subclasshook__', '__weakref__', '__class_getitem__'] # These special attributes will be not collected as protocol members. EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS + _SPECIAL_NAMES + ['_MutableMapping__marker'] From 6d3f5357d322ef705fda36b8bbb97af31a627dd9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 30 Jan 2020 18:20:31 -0800 Subject: [PATCH 33/72] Make isinstance and issubclass error for GenericAlias (#6) --- Lib/test/test_genericalias.py | 11 +++++++++++ Objects/descrobject.c | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 9e54049f07565d..92ca83dbc3ad39 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -161,6 +161,17 @@ def test_equality(self): self.assertNotEqual(list, list[int]) self.assertNotEqual(list[int], list) + def test_isinstance(self): + self.assertTrue(isinstance([], list)) + with self.assertRaises(TypeError): + isinstance([], list[str]) + + def test_issubclass(self): + class L(list): ... + self.assertTrue(issubclass(L, list)) + with self.assertRaises(TypeError): + issubclass(L, list[str]) + if __name__ == "__main__": unittest.main() diff --git a/Objects/descrobject.c b/Objects/descrobject.c index e8d2ccab922c60..e778963cb103db 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -2061,8 +2061,26 @@ ga_mro_entries(PyObject *self, PyObject *args) return PyTuple_Pack(1, alias->origin); } +static PyObject * +ga_instancecheck(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyErr_Format(PyExc_TypeError, + "TypeError: Subscripted generics cannot be used with class and instance checks", + self); +} + +static PyObject * +ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyErr_Format(PyExc_TypeError, + "TypeError: Subscripted generics cannot be used with class and instance checks", + self); +} + static PyMethodDef ga_methods[] = { {"__mro_entries__", ga_mro_entries, METH_O}, + {"__instancecheck__", ga_instancecheck, METH_O}, + {"__subclasscheck__", ga_subclasscheck, METH_O}, {0} }; From 4b3ecf287c70017d2001b34677fcf90749db3d2b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 19:27:06 -0800 Subject: [PATCH 34/72] Fix two more AssertIs calls in test_typing.py --- Lib/test/test_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a5a2e7ceaa73cb..e0482114df3ca7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1822,7 +1822,7 @@ class MyTup(Tuple[T, T]): ... class MyCall(Callable[..., T]): def __call__(self): return None self.assertIs(MyCall[T]().__class__, MyCall) - self.assertIs(MyCall[T]().__orig_class__, MyCall[T]) + self.assertEqual(MyCall[T]().__orig_class__, MyCall[T]) class MyDict(typing.Dict[T, T]): ... self.assertIs(MyDict[int]().__class__, MyDict) self.assertEqual(MyDict[int]().__orig_class__, MyDict[int]) @@ -1833,7 +1833,7 @@ class MyDef(typing.DefaultDict[str, T]): ... if sys.version_info >= (3, 3): class MyChain(typing.ChainMap[str, T]): ... self.assertIs(MyChain[int]().__class__, MyChain) - self.assertIs(MyChain[int]().__orig_class__, MyChain[int]) + self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) From 2df602bca7a68fe8bf8c979d71e614e341691b76 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 20:19:22 -0800 Subject: [PATCH 35/72] Admit defeat -- comment out two expected TypeErrors in test_extended_generic_rules_subclassing --- Lib/test/test_typing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index e0482114df3ca7..86048a26a271d6 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1776,10 +1776,11 @@ def __call__(self): self.assertEqual(T1[int, T].__origin__, T1) self.assertEqual(T2.__parameters__, (T,)) - with self.assertRaises(TypeError): - T1[int] - with self.assertRaises(TypeError): - T2[int, str] + # These don't work because of tuple.__class_item__ + ## with self.assertRaises(TypeError): + ## T1[int] + ## with self.assertRaises(TypeError): + ## T2[int, str] self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') self.assertEqual(C2.__parameters__, ()) From eede65a1eeb6a23072ef7c7c214d669a62125cd2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 20:26:50 -0800 Subject: [PATCH 36/72] Fix whitespace to satisfy patchcheck.py --- Lib/test/test_genericalias.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 92ca83dbc3ad39..94c32dc976aa66 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -15,7 +15,7 @@ class BaseTest(unittest.TestCase): def test_subscriptable(self): for t in (tuple, list, dict, set, frozenset, - defaultdict, deque, + defaultdict, deque, OrderedDict, Counter, UserDict, UserList, IOBase, Pattern, Match, From 2b22f6e75618f8327ee1a65681e1b3d2fcbcb945 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 21:06:21 -0800 Subject: [PATCH 37/72] Fix test_site.py --- Lib/test/test_site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 41c4229919507b..231ac86b6023a6 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -537,7 +537,7 @@ def test_startup_imports(self): # http://bugs.python.org/issue19218 collection_mods = {'_collections', 'collections', 'functools', 'heapq', 'itertools', 'keyword', 'operator', - 'reprlib', 'types', 'weakref' + 'reprlib', 'weakref' }.difference(sys.builtin_module_names) self.assertFalse(modules.intersection(collection_mods), stderr) From 065a032653b547a2a98890736b52db7765f7edcd Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 30 Jan 2020 21:15:04 -0800 Subject: [PATCH 38/72] Better way to fix test_site.py: don't import types from collections --- Lib/_collections_abc.py | 3 ++- Lib/collections/__init__.py | 3 ++- Lib/test/test_site.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 1f811cc2f6c5ba..3a218e1d114600 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -8,7 +8,8 @@ from abc import ABCMeta, abstractmethod import sys -from types import GenericAlias + +GenericAlias = type(list[int]) __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index f7f432b145dc4d..4f8fc4911530d3 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -25,7 +25,6 @@ from _weakref import proxy as _proxy from itertools import repeat as _repeat, chain as _chain, starmap as _starmap from reprlib import recursive_repr as _recursive_repr -from types import GenericAlias as _GenericAlias try: from _collections import deque @@ -39,6 +38,8 @@ except ImportError: pass +_GenericAlias = type(list[int]) + ################################################################################ ### OrderedDict diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 231ac86b6023a6..41c4229919507b 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -537,7 +537,7 @@ def test_startup_imports(self): # http://bugs.python.org/issue19218 collection_mods = {'_collections', 'collections', 'functools', 'heapq', 'itertools', 'keyword', 'operator', - 'reprlib', 'weakref' + 'reprlib', 'types', 'weakref' }.difference(sys.builtin_module_names) self.assertFalse(modules.intersection(collection_mods), stderr) From baf9b32218e19b35d63f6225771244df6491f9ff Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 31 Jan 2020 15:40:02 -0800 Subject: [PATCH 39/72] Use Py_ssize_t instead of int for indexing (#8) --- Objects/descrobject.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index e778963cb103db..2e34f9fe0da366 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1924,10 +1924,10 @@ is_typevar(PyObject *obj) } // Index of item in self[:len], or -1 if not found (self is a tuple) -static int -tuple_index(PyObject *self, int len, PyObject *item) +static Py_ssize_t +tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) { - for (int i = 0; i < len; i++) { + for (Py_ssize_t i = 0; i < len; i++) { if (PyTuple_GET_ITEM(self, i) == item) { return i; } @@ -1939,28 +1939,28 @@ static PyObject * ga_getitem(PyObject *self, PyObject *item) { gaobject *alias = (gaobject *)self; - int nparams = PyTuple_GET_SIZE(alias->parameters); + Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters); if (nparams == 0) { return PyErr_Format(PyExc_TypeError, "There are no type variables left in %R", self); } int is_tuple = PyTuple_Check(item); - int nitem = is_tuple ? PyTuple_GET_SIZE(item) : 1; + Py_ssize_t nitem = is_tuple ? PyTuple_GET_SIZE(item) : 1; if (nitem != nparams) { return PyErr_Format(PyExc_TypeError, "Too %s arguments for %R", nitem > nparams ? "many" : "few", self); } - int nargs = PyTuple_GET_SIZE(alias->args); + Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); PyObject *newargs = PyTuple_New(nargs); if (newargs == NULL) return NULL; - for (int iarg = 0; iarg < nargs; iarg++) { + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); if (is_typevar(arg)) { - int iparam = tuple_index(alias->parameters, nparams, arg); + Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg); assert(iparam >= 0); if (is_tuple) { arg = PyTuple_GET_ITEM(item, iparam); @@ -2133,12 +2133,12 @@ PyTypeObject Py_GenericAliasType = { static PyObject * make_parameters(PyObject *args) { - int len = PyTuple_GET_SIZE(args); + Py_ssize_t len = PyTuple_GET_SIZE(args); PyObject *parameters = PyTuple_New(len); if (parameters == NULL) return NULL; - int iparam = 0; - for (int iarg = 0; iarg < len; iarg++) { + Py_ssize_t iparam = 0; + for (Py_ssize_t iarg = 0; iarg < len; iarg++) { PyObject *t = PyTuple_GET_ITEM(args, iarg); if (is_typevar(t)) { if (tuple_index(parameters, iparam, t) < 0) { From 9915ba2bea0c1eb6b2f5838521dbebcaf8c5c19b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 4 Feb 2020 20:36:28 -0800 Subject: [PATCH 40/72] GenericAlias pickle support (#9) --- Lib/test/test_genericalias.py | 12 ++++++ Objects/descrobject.c | 81 ++++++++++++++++++++++------------- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 94c32dc976aa66..0b6413d3010686 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -1,6 +1,7 @@ """Tests for C-implemented GenericAlias.""" import unittest +import pickle from collections import ( defaultdict, deque, OrderedDict, Counter, UserDict, UserList ) @@ -8,7 +9,10 @@ from contextlib import AbstractContextManager, AbstractAsyncContextManager from io import IOBase from re import Pattern, Match +from types import GenericAlias +from typing import TypeVar +T = TypeVar('T') class BaseTest(unittest.TestCase): """Test basics.""" @@ -172,6 +176,14 @@ class L(list): ... with self.assertRaises(TypeError): issubclass(L, list[str]) + def test_pickle(self): + alias = GenericAlias(list, T) + s = pickle.dumps(alias) + loaded = pickle.loads(s) + self.assertEqual(alias.__origin__, loaded.__origin__) + self.assertEqual(alias.__args__, loaded.__args__) + self.assertEqual(alias.__parameters__, loaded.__parameters__) + if __name__ == "__main__": unittest.main() diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 2e34f9fe0da366..747ce75d471c0c 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1935,6 +1935,34 @@ tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) return -1; } +// tuple(t for t in args if isinstance(t, TypeVar)) +static PyObject * +make_parameters(PyObject *args) +{ + Py_ssize_t len = PyTuple_GET_SIZE(args); + PyObject *parameters = PyTuple_New(len); + if (parameters == NULL) + return NULL; + Py_ssize_t iparam = 0; + for (Py_ssize_t iarg = 0; iarg < len; iarg++) { + PyObject *t = PyTuple_GET_ITEM(args, iarg); + if (is_typevar(t)) { + if (tuple_index(parameters, iparam, t) < 0) { + Py_INCREF(t); + PyTuple_SET_ITEM(parameters, iparam, t); + iparam++; + } + } + } + if (iparam < len) { + if (_PyTuple_Resize(¶meters, iparam) < 0) { + Py_XDECREF(parameters); + return NULL; + } + } + return parameters; +} + static PyObject * ga_getitem(PyObject *self, PyObject *item) { @@ -1999,6 +2027,9 @@ static const char* const attr_exceptions[] = { "__args__", "__parameters__", "__mro_entries__", + "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ + "__reduce__", + "__setstate__", NULL, }; @@ -2077,10 +2108,30 @@ ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored)) self); } +static PyObject * +ga_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + gaobject *alias = (gaobject *)self; + return Py_BuildValue("O(OO)", Py_TYPE(alias), + alias->origin, alias->args); +} + +static PyObject * +ga_setstate(PyObject *self, PyObject *state) +{ + gaobject *alias = (gaobject *)self; + PyObject *parameters = make_parameters(alias->args); + Py_INCREF(parameters); + alias->parameters = parameters; + Py_RETURN_NONE; +} + static PyMethodDef ga_methods[] = { {"__mro_entries__", ga_mro_entries, METH_O}, {"__instancecheck__", ga_instancecheck, METH_O}, {"__subclasscheck__", ga_subclasscheck, METH_O}, + {"__reduce__", ga_reduce, METH_NOARGS}, + {"__setstate__", ga_setstate, METH_O}, {0} }; @@ -2112,7 +2163,7 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) // - __eq__ PyTypeObject Py_GenericAliasType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "GenericAlias", + .tp_name = "types.GenericAlias", .tp_basicsize = sizeof(gaobject), .tp_dealloc = ga_dealloc, .tp_repr = ga_repr, @@ -2129,34 +2180,6 @@ PyTypeObject Py_GenericAliasType = { .tp_free = PyObject_GC_Del, }; -// tuple(t for t in args if isinstance(t, TypeVar)) -static PyObject * -make_parameters(PyObject *args) -{ - Py_ssize_t len = PyTuple_GET_SIZE(args); - PyObject *parameters = PyTuple_New(len); - if (parameters == NULL) - return NULL; - Py_ssize_t iparam = 0; - for (Py_ssize_t iarg = 0; iarg < len; iarg++) { - PyObject *t = PyTuple_GET_ITEM(args, iarg); - if (is_typevar(t)) { - if (tuple_index(parameters, iparam, t) < 0) { - Py_INCREF(t); - PyTuple_SET_ITEM(parameters, iparam, t); - iparam++; - } - } - } - if (iparam < len) { - if (_PyTuple_Resize(¶meters, iparam) < 0) { - Py_XDECREF(parameters); - return NULL; - } - } - return parameters; -} - PyObject * Py_GenericAlias(PyObject *origin, PyObject *args) { From 4e1da193e3b36cc62c2113a37ebf329431348acc Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 4 Feb 2020 21:45:14 -0800 Subject: [PATCH 41/72] Make type generic (#7) This change specifically allows type[int], but disallows str[int]. --- Lib/test/test_genericalias.py | 18 ++++++++++++++++-- Objects/abstract.c | 6 ++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 0b6413d3010686..1385d581c8312d 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -18,8 +18,8 @@ class BaseTest(unittest.TestCase): """Test basics.""" def test_subscriptable(self): - for t in (tuple, list, dict, set, frozenset, - defaultdict, deque, + for t in (type, tuple, list, dict, set, frozenset, + defaultdict, deque, OrderedDict, Counter, UserDict, UserList, IOBase, Pattern, Match, @@ -176,6 +176,20 @@ class L(list): ... with self.assertRaises(TypeError): issubclass(L, list[str]) + def test_type_generic(self): + t = type[int] + Test = t('Test', (), {}) + self.assertTrue(isinstance(Test, type)) + test = Test() + self.assertEqual(t(test), Test) + self.assertEqual(t(0), int) + + def test_type_subclass_generic(self): + class MyType(type): + pass + with self.assertRaises(TypeError): + MyType[int] + def test_pickle(self): alias = GenericAlias(list, T) s = pickle.dumps(alias) diff --git a/Objects/abstract.c b/Objects/abstract.c index dc8ba10762de6d..ac6e8fc24ee725 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -175,6 +175,12 @@ PyObject_GetItem(PyObject *o, PyObject *key) if (PyType_Check(o)) { PyObject *meth, *result; _Py_IDENTIFIER(__class_getitem__); + + // Special case type[int], but disallow other types so str[int] fails + if ((PyTypeObject*)o == &PyType_Type) { + return Py_GenericAlias(o, key); + } + if (_PyObject_LookupAttrId(o, &PyId___class_getitem__, &meth) < 0) { return NULL; } From 8926f7aa1131d4805c9a9174c05e38d7cbcff8ce Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 4 Feb 2020 22:00:00 -0800 Subject: [PATCH 42/72] Remove a completed TODO comment --- Objects/descrobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 747ce75d471c0c..fd9811f4469c34 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -2160,7 +2160,6 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) // - argument clinic? // - __doc__? // - cache? -// - __eq__ PyTypeObject Py_GenericAliasType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "types.GenericAlias", From 7b1ad200849e09293b0283633193211731e9396a Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 4 Feb 2020 22:24:46 -0800 Subject: [PATCH 43/72] Remove extraneous whitespace. (#10) --- Lib/test/test_genericalias.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 1385d581c8312d..dfe6fe04c0ec76 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -19,7 +19,7 @@ class BaseTest(unittest.TestCase): def test_subscriptable(self): for t in (type, tuple, list, dict, set, frozenset, - defaultdict, deque, + defaultdict, deque, OrderedDict, Counter, UserDict, UserList, IOBase, Pattern, Match, From a05d9ae1da8e8137e0881fe51226cc08cb195be1 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 7 Feb 2020 10:05:57 -0800 Subject: [PATCH 44/72] Do not make IOBase generic (#11) --- Modules/_io/iobase.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index e72a24f480e75a..d51fc944e1a629 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -824,8 +824,6 @@ static PyMethodDef iobase_methods[] = { _IO__IOBASE_READLINES_METHODDEF _IO__IOBASE_WRITELINES_METHODDEF - {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, {NULL, NULL} }; From d4a372bbc77f9506a6332ae5b4875d5855b271ec Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 7 Feb 2020 15:22:07 -0800 Subject: [PATCH 45/72] Remove __class_getitem__ from _pyio.py too; and don't test for IOBase --- Lib/_pyio.py | 4 ---- Lib/test/test_genericalias.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index a07610a9b4e250..3917f474057c4d 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -341,10 +341,6 @@ def _unsupported(self, name): raise UnsupportedOperation("%s.%s() not supported" % (self.__class__.__name__, name)) - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return types.GenericAlias(cls, item) - ### Positioning ### def seek(self, pos, whence=0): diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index dfe6fe04c0ec76..16cdc764637cc8 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -7,7 +7,6 @@ ) from collections.abc import * from contextlib import AbstractContextManager, AbstractAsyncContextManager -from io import IOBase from re import Pattern, Match from types import GenericAlias @@ -21,7 +20,6 @@ def test_subscriptable(self): for t in (type, tuple, list, dict, set, frozenset, defaultdict, deque, OrderedDict, Counter, UserDict, UserList, - IOBase, Pattern, Match, AbstractContextManager, AbstractAsyncContextManager, Awaitable, Coroutine, From 2f94a36df0fe1ff7d738f5daea438fae0c2cad2d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 24 Feb 2020 19:00:07 -0800 Subject: [PATCH 46/72] Make types.MappingProxyType generic (#13) --- Lib/test/test_genericalias.py | 3 ++- Objects/descrobject.c | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 16cdc764637cc8..d5e919bd2c45d8 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -8,7 +8,7 @@ from collections.abc import * from contextlib import AbstractContextManager, AbstractAsyncContextManager from re import Pattern, Match -from types import GenericAlias +from types import GenericAlias, MappingProxyType from typing import TypeVar T = TypeVar('T') @@ -33,6 +33,7 @@ def test_subscriptable(self): Mapping, MutableMapping, MappingView, KeysView, ItemsView, ValuesView, Sequence, MutableSequence, + MappingProxyType, ): tname = t.__name__ with self.subTest(f"Testing {tname}"): diff --git a/Objects/descrobject.c b/Objects/descrobject.c index af4d1c2f296dd1..c804bf2157c209 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1066,6 +1066,8 @@ static PyMethodDef mappingproxy_methods[] = { PyDoc_STR("D.items() -> list of D's (key, value) pairs, as 2-tuples")}, {"copy", (PyCFunction)mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, {0} }; From 3da279795e5eab8da47eeb2ce34b9925f07bdf5c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 24 Feb 2020 19:09:21 -0800 Subject: [PATCH 47/72] Reword isinstance/subclass checks (#12) --- Objects/descrobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index c804bf2157c209..718ea2f27901f1 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -2098,7 +2098,7 @@ static PyObject * ga_instancecheck(PyObject *self, PyObject *Py_UNUSED(ignored)) { return PyErr_Format(PyExc_TypeError, - "TypeError: Subscripted generics cannot be used with class and instance checks", + "isinstance() argument 2 cannot be a parameterized generic", self); } @@ -2106,7 +2106,7 @@ static PyObject * ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored)) { return PyErr_Format(PyExc_TypeError, - "TypeError: Subscripted generics cannot be used with class and instance checks", + "issubclass() argument 2 cannot be a parameterized generic", self); } From 13d8d6d32e36affb7ae9491a9bea5381b1c6c17e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 24 Feb 2020 19:42:49 -0800 Subject: [PATCH 48/72] Fix test_types (mappingproxy now has a __class_getitem__ method) --- Lib/test/test_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 7b45b7a5895039..d6622dc2bc4c9d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -622,6 +622,7 @@ def test_methods(self): self.assertEqual(attrs, { '__contains__', '__getitem__', + '__class_getitem__', '__iter__', '__len__', 'copy', From 4783c2e8a92fa8e73b8a770f3af8ec43626a23fd Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 24 Feb 2020 21:06:23 -0800 Subject: [PATCH 49/72] Convert some already generic things to GenericAlias to be consistent (#14) --- Lib/os.py | 6 ++++-- Lib/subprocess.py | 28 +++++++--------------------- Lib/tempfile.py | 15 ++++----------- Lib/test/test_os.py | 3 ++- Lib/test/test_subprocess.py | 5 +++-- Lib/test/test_tempfile.py | 5 +++-- 6 files changed, 23 insertions(+), 39 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index ab75b94d4fe45f..2587b6260266e7 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -27,6 +27,7 @@ import stat as st from _collections_abc import _check_methods +from types import GenericAlias _names = sys.builtin_module_names @@ -1060,8 +1061,9 @@ def __subclasshook__(cls, subclass): return _check_methods(subclass, '__fspath__') return NotImplemented - def __class_getitem__(cls, type): - return cls + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return GenericAlias(cls, item) if name == 'nt': diff --git a/Lib/subprocess.py b/Lib/subprocess.py index c8db387091ba39..ce99bc2ce34b34 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -52,6 +52,7 @@ import warnings import contextlib from time import monotonic as _time +import types try: import pwd @@ -446,17 +447,9 @@ def __repr__(self): args.append('stderr={!r}'.format(self.stderr)) return "{}({})".format(type(self).__name__, ', '.join(args)) - def __class_getitem__(cls, type): - """Provide minimal support for using this class as generic - (for example in type annotations). - - See PEP 484 and PEP 560 for more details. For example, - `CompletedProcess[bytes]` is a valid expression at runtime - (type argument `bytes` indicates the type used for stdout). - Note, no type checking happens at runtime, but a static type - checker can be used. - """ - return cls + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return types.GenericAlias(cls, item) def check_returncode(self): @@ -1000,16 +993,9 @@ def __repr__(self): obj_repr = obj_repr[:76] + "...>" return obj_repr - def __class_getitem__(cls, type): - """Provide minimal support for using this class as generic - (for example in type annotations). - - See PEP 484 and PEP 560 for more details. For example, `Popen[bytes]` - is a valid expression at runtime (type argument `bytes` indicates the - type used for stdout). Note, no type checking happens at runtime, but - a static type checker can be used. - """ - return cls + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return types.GenericAlias(cls, item) @property def universal_newlines(self): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 448163f04380d5..86c007b58d426e 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -44,6 +44,7 @@ import errno as _errno from random import Random as _Random import sys as _sys +import types as _types import weakref as _weakref import _thread _allocate_lock = _thread.allocate_lock @@ -643,17 +644,9 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1, 'encoding': encoding, 'newline': newline, 'dir': dir, 'errors': errors} - def __class_getitem__(cls, type): - """Provide minimal support for using this class as generic - (for example in type annotations). - - See PEP 484 and PEP 560 for more details. For example, - `SpooledTemporaryFile[str]` is a valid expression at runtime (type - argument `str` indicates whether the file is open in bytes or text - mode). Note, no type checking happens at runtime, but a static type - checker can be used. - """ - return cls + def __class_getitem__(cls, item): + """Internal: PEP 585.""" + return _types.GenericAlias(cls, item) def _check(self, file): if self._rolled: return diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 9e3a1695dfb34e..1a90901b6d1fe7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -25,6 +25,7 @@ import tempfile import threading import time +import types import unittest import uuid import warnings @@ -4080,7 +4081,7 @@ class A(os.PathLike): self.assertTrue(issubclass(FakePath, os.PathLike)) def test_pathlike_class_getitem(self): - self.assertIs(os.PathLike[bytes], os.PathLike) + self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) class TimesTests(unittest.TestCase): diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 2bbdbaef84e992..eb31b32f664c27 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -11,6 +11,7 @@ import tempfile import time import traceback +import types import selectors import sysconfig import select @@ -1436,8 +1437,8 @@ def test_file_not_found_with_bad_cwd(self): self.assertEqual(c.exception.filename, '/some/nonexistent/directory') def test_class_getitems(self): - self.assertIs(subprocess.Popen[bytes], subprocess.Popen) - self.assertIs(subprocess.CompletedProcess[str], subprocess.CompletedProcess) + self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) + self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 5fe9506b0b7ba1..6126d478b9495e 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -10,6 +10,7 @@ import warnings import contextlib import stat +import types import weakref from unittest import mock @@ -1230,8 +1231,8 @@ def test_truncate_with_size_parameter(self): self.assertEqual(os.fstat(f.fileno()).st_size, 20) def test_class_getitem(self): - self.assertIs(tempfile.SpooledTemporaryFile[bytes], - tempfile.SpooledTemporaryFile) + self.assertIsInstance(tempfile.SpooledTemporaryFile[bytes], + types.GenericAlias) if tempfile.NamedTemporaryFile is not tempfile.TemporaryFile: From e50136dd92c4a8f18932613dcd738a001ac513b9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 24 Feb 2020 21:09:10 -0800 Subject: [PATCH 50/72] Make __parameters__ lazy (#15) Now we can also remove `__setstate__`. --- Objects/descrobject.c | 50 +++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 4a164cbd533143..7172adf0427c91 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1969,6 +1969,15 @@ static PyObject * ga_getitem(PyObject *self, PyObject *item) { gaobject *alias = (gaobject *)self; + // do a lookup for __parameters__ so it gets populated (if not already) + if (alias->parameters == NULL) { + _Py_IDENTIFIER(__parameters__); + PyObject *params = _PyObject_GetAttrId(self, &PyId___parameters__); + if (params == NULL) { + return NULL; + } + Py_DECREF(params); + } Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters); if (nparams == 0) { return PyErr_Format(PyExc_TypeError, @@ -2031,7 +2040,6 @@ static const char* const attr_exceptions[] = { "__mro_entries__", "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ "__reduce__", - "__setstate__", NULL, }; @@ -2118,29 +2126,37 @@ ga_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) alias->origin, alias->args); } -static PyObject * -ga_setstate(PyObject *self, PyObject *state) -{ - gaobject *alias = (gaobject *)self; - PyObject *parameters = make_parameters(alias->args); - Py_INCREF(parameters); - alias->parameters = parameters; - Py_RETURN_NONE; -} - static PyMethodDef ga_methods[] = { {"__mro_entries__", ga_mro_entries, METH_O}, {"__instancecheck__", ga_instancecheck, METH_O}, {"__subclasscheck__", ga_subclasscheck, METH_O}, {"__reduce__", ga_reduce, METH_NOARGS}, - {"__setstate__", ga_setstate, METH_O}, {0} }; static PyMemberDef ga_members[] = { {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, {"__args__", T_OBJECT, offsetof(gaobject, args), READONLY}, - {"__parameters__", T_OBJECT, offsetof(gaobject, parameters), READONLY}, + {0} +}; + +static PyObject * +ga_parameters(PyObject *self, void *unused) +{ + gaobject *alias = (gaobject *)self; + if (alias->parameters == NULL) { + alias->parameters = make_parameters(alias->args); + if (alias->parameters == NULL) { + Py_DECREF(alias->parameters); + return NULL; + } + } + Py_INCREF(alias->parameters); + return alias->parameters; +} + +static PyGetSetDef ga_properties[] = { + {"__parameters__", ga_parameters, (setter)NULL, "Type variables in the GenericAlias.", NULL}, {0} }; @@ -2179,6 +2195,7 @@ PyTypeObject Py_GenericAliasType = { .tp_alloc = PyType_GenericAlloc, .tp_new = ga_new, .tp_free = PyObject_GC_Del, + .tp_getset = ga_properties, }; PyObject * @@ -2203,12 +2220,7 @@ Py_GenericAlias(PyObject *origin, PyObject *args) Py_INCREF(origin); alias->origin = origin; alias->args = args; - // TODO: Make __parameters__ a lazy attribute - alias->parameters = make_parameters(args); - if (alias->parameters == NULL) { - Py_DECREF(alias); - return NULL; - } + alias->parameters = NULL; _PyObject_GC_TRACK(alias); return (PyObject *)alias; } From 01929866fe2278d37c632c0c108f4ef20aed4d79 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 25 Feb 2020 09:19:01 -0800 Subject: [PATCH 51/72] Fix failing test_doctest -- we now find 821 tests --- Lib/test/test_doctest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index aa92777efc3c7a..e9dff2edae92f1 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -661,7 +661,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 800 < len(tests) < 820 # approximate number of objects with docstrings + >>> 810 < len(tests) < 830 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests From b80b0770268424174b2a6605bd7def923694b0b2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 25 Feb 2020 14:57:52 -0800 Subject: [PATCH 52/72] Don't import types in os.py -- it breaks test_site.py --- Lib/os.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index 2587b6260266e7..0bc680d6c25d6d 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -27,7 +27,8 @@ import stat as st from _collections_abc import _check_methods -from types import GenericAlias + +GenericAlias = type(list[int]) _names = sys.builtin_module_names From 3bc4e127cc1807100c4cd68a6815c91c32c2289a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 14 Mar 2020 11:36:56 -0700 Subject: [PATCH 53/72] Move GenericAlias definition to its own file --- Makefile.pre.in | 1 + Objects/descrobject.c | 432 ---------------------------------- Objects/genericaliasobject.c | 433 +++++++++++++++++++++++++++++++++++ 3 files changed, 434 insertions(+), 432 deletions(-) create mode 100644 Objects/genericaliasobject.c diff --git a/Makefile.pre.in b/Makefile.pre.in index 3199a1aa02d65b..b0708c62685950 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -392,6 +392,7 @@ OBJECT_OBJS= \ Objects/descrobject.o \ Objects/enumobject.o \ Objects/exceptions.o \ + Objects/genericaliasobject.o \ Objects/genobject.o \ Objects/fileobject.o \ Objects/floatobject.o \ diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 7172adf0427c91..704e5967968ec3 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1792,435 +1792,3 @@ PyTypeObject PyProperty_Type = { PyType_GenericNew, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; - - -// Experimental code to implement PEP 585 (list[int] etc.) -// TODO: Does this belong in this file? - -typedef struct { - PyObject_HEAD - PyObject *origin; - PyObject *args; - PyObject *parameters; -} gaobject; - -static void -ga_dealloc(PyObject *self) -{ - gaobject *alias = (gaobject *)self; - - _PyObject_GC_UNTRACK(self); - Py_XDECREF(alias->origin); - Py_XDECREF(alias->args); - Py_XDECREF(alias->parameters); - self->ob_type->tp_free(self); -} - -static int -ga_traverse(PyObject *self, visitproc visit, void *arg) -{ - gaobject *alias = (gaobject *)self; - Py_VISIT(alias->origin); - Py_VISIT(alias->args); - Py_VISIT(alias->parameters); - return 0; -} - -static int -ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) -{ - PyObject *qualname = PyObject_GetAttrString(p, "__qualname__"); - PyErr_Clear(); - PyObject *module = PyObject_GetAttrString(p, "__module__"); - PyErr_Clear(); - PyObject *r = NULL; - int err; - if (PyObject_HasAttrString(p, "__origin__") && - PyObject_HasAttrString(p, "__args__")) - { - // It looks like a GenericAlias - r = PyObject_Repr(p); - } - else if (p == Py_Ellipsis) { - // The Ellipsis object - r = PyUnicode_FromString("..."); - } - else if (qualname != NULL && module != NULL) { - // Looks like a class - if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { - // builtins don't need a module name - Py_INCREF(qualname); - r = qualname; - } - else { - r = PyUnicode_FromFormat("%U.%U", module, qualname); - } - } - else { - // fallback - r = PyObject_Repr(p); - } - if (r == NULL) { - // error if any of the above PyObject_Repr/PyUnicode_From* fail - err = -1; - } else { - err = _PyUnicodeWriter_WriteStr(writer, r); - } - Py_XDECREF(qualname); - Py_XDECREF(module); - Py_XDECREF(r); - return err; -} - -static PyObject * -ga_repr(PyObject *self) -{ - gaobject *alias = (gaobject *)self; - Py_ssize_t len = PyTuple_Size(alias->args); - - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); - - if (ga_repr_item(&writer, alias->origin) < 0) { - goto error; - } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) { - goto error; - } - for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { - goto error; - } - } - PyObject *p = PyTuple_GET_ITEM(alias->args, i); - if (ga_repr_item(&writer, p) < 0) { - goto error; - } - } - if (len == 0) { - // for something like tuple[()] we should print a "()" - if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) { - goto error; - } - } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) { - goto error; - } - return _PyUnicodeWriter_Finish(&writer); -error: - _PyUnicodeWriter_Dealloc(&writer); - return NULL; -} - -// isinstance(obj, TypeVar) without importing typing.py. If someone -// names some other class TypeVar, it will be mistaken for a TypeVar. -// Maybe that's a feature; or maybe we'll have to see if -// sys.modules['typing'] exists and look for its 'TypeVar' attribute -// (which is roughly what dataclasses.py uses to recognize ClassVar). -static int -is_typevar(PyObject *obj) -{ - PyTypeObject *type = Py_TYPE(obj); - return strcmp(type->tp_name, "TypeVar") == 0; -} - -// Index of item in self[:len], or -1 if not found (self is a tuple) -static Py_ssize_t -tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) -{ - for (Py_ssize_t i = 0; i < len; i++) { - if (PyTuple_GET_ITEM(self, i) == item) { - return i; - } - } - return -1; -} - -// tuple(t for t in args if isinstance(t, TypeVar)) -static PyObject * -make_parameters(PyObject *args) -{ - Py_ssize_t len = PyTuple_GET_SIZE(args); - PyObject *parameters = PyTuple_New(len); - if (parameters == NULL) - return NULL; - Py_ssize_t iparam = 0; - for (Py_ssize_t iarg = 0; iarg < len; iarg++) { - PyObject *t = PyTuple_GET_ITEM(args, iarg); - if (is_typevar(t)) { - if (tuple_index(parameters, iparam, t) < 0) { - Py_INCREF(t); - PyTuple_SET_ITEM(parameters, iparam, t); - iparam++; - } - } - } - if (iparam < len) { - if (_PyTuple_Resize(¶meters, iparam) < 0) { - Py_XDECREF(parameters); - return NULL; - } - } - return parameters; -} - -static PyObject * -ga_getitem(PyObject *self, PyObject *item) -{ - gaobject *alias = (gaobject *)self; - // do a lookup for __parameters__ so it gets populated (if not already) - if (alias->parameters == NULL) { - _Py_IDENTIFIER(__parameters__); - PyObject *params = _PyObject_GetAttrId(self, &PyId___parameters__); - if (params == NULL) { - return NULL; - } - Py_DECREF(params); - } - Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters); - if (nparams == 0) { - return PyErr_Format(PyExc_TypeError, - "There are no type variables left in %R", - self); - } - int is_tuple = PyTuple_Check(item); - Py_ssize_t nitem = is_tuple ? PyTuple_GET_SIZE(item) : 1; - if (nitem != nparams) { - return PyErr_Format(PyExc_TypeError, - "Too %s arguments for %R", - nitem > nparams ? "many" : "few", - self); - } - Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); - PyObject *newargs = PyTuple_New(nargs); - if (newargs == NULL) - return NULL; - for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { - PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); - if (is_typevar(arg)) { - Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg); - assert(iparam >= 0); - if (is_tuple) { - arg = PyTuple_GET_ITEM(item, iparam); - } - else { - assert(iparam == 0); - arg = item; - } - } - Py_INCREF(arg); - PyTuple_SET_ITEM(newargs, iarg, arg); - } - PyObject *res = Py_GenericAlias(alias->origin, newargs); - Py_DECREF(newargs); - return res; -} - -static PyMappingMethods ga_as_mapping = { - .mp_subscript = ga_getitem, -}; - -static PyObject * -ga_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - gaobject *alias = (gaobject *)self; - PyObject *obj = PyObject_Call(alias->origin, args, kwds); - if (obj != NULL) { - PyObject_SetAttrString(obj, "__orig_class__", self); - PyErr_Clear(); - } - return obj; -} - -static const char* const attr_exceptions[] = { - "__origin__", - "__args__", - "__parameters__", - "__mro_entries__", - "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ - "__reduce__", - NULL, -}; - -static PyObject * -ga_getattro(PyObject *self, PyObject *name) -{ - gaobject *alias = (gaobject *)self; - if (PyUnicode_Check(name)) { - for (const char * const *p = attr_exceptions; ; p++) { - if (*p == NULL) { - return PyObject_GetAttr(alias->origin, name); - } - if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { - break; - } - } - } - return PyObject_GenericGetAttr(self, name); -} - -static PyObject * -ga_richcompare(PyObject *a, PyObject *b, int op) -{ - if (Py_TYPE(a) != &Py_GenericAliasType || - Py_TYPE(b) != &Py_GenericAliasType || - (op != Py_EQ && op != Py_NE)) - { - Py_RETURN_NOTIMPLEMENTED; - } - - if (op == Py_NE) { - PyObject *eq = ga_richcompare(a, b, Py_EQ); - if (eq == NULL) - return NULL; - Py_DECREF(eq); - if (eq == Py_True) { - Py_RETURN_FALSE; - } - else { - Py_RETURN_TRUE; - } - } - - gaobject *aa = (gaobject *)a; - gaobject *bb = (gaobject *)b; - PyObject *eq = PyObject_RichCompare(aa->origin, bb->origin, Py_EQ); - if (eq == NULL) - return NULL; - if (eq == Py_False) { - return eq; - } - Py_DECREF(eq); - return PyObject_RichCompare(aa->args, bb->args, Py_EQ); -} - -static PyObject * -ga_mro_entries(PyObject *self, PyObject *args) -{ - gaobject *alias = (gaobject *)self; - return PyTuple_Pack(1, alias->origin); -} - -static PyObject * -ga_instancecheck(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyErr_Format(PyExc_TypeError, - "isinstance() argument 2 cannot be a parameterized generic", - self); -} - -static PyObject * -ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyErr_Format(PyExc_TypeError, - "issubclass() argument 2 cannot be a parameterized generic", - self); -} - -static PyObject * -ga_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - gaobject *alias = (gaobject *)self; - return Py_BuildValue("O(OO)", Py_TYPE(alias), - alias->origin, alias->args); -} - -static PyMethodDef ga_methods[] = { - {"__mro_entries__", ga_mro_entries, METH_O}, - {"__instancecheck__", ga_instancecheck, METH_O}, - {"__subclasscheck__", ga_subclasscheck, METH_O}, - {"__reduce__", ga_reduce, METH_NOARGS}, - {0} -}; - -static PyMemberDef ga_members[] = { - {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, - {"__args__", T_OBJECT, offsetof(gaobject, args), READONLY}, - {0} -}; - -static PyObject * -ga_parameters(PyObject *self, void *unused) -{ - gaobject *alias = (gaobject *)self; - if (alias->parameters == NULL) { - alias->parameters = make_parameters(alias->args); - if (alias->parameters == NULL) { - Py_DECREF(alias->parameters); - return NULL; - } - } - Py_INCREF(alias->parameters); - return alias->parameters; -} - -static PyGetSetDef ga_properties[] = { - {"__parameters__", ga_parameters, (setter)NULL, "Type variables in the GenericAlias.", NULL}, - {0} -}; - -static PyObject * -ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { - return PyErr_Format(PyExc_TypeError, "GenericAlias does not support keyword arguments"); - } - if (PyTuple_Size(args) != 2) { - return PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); - } - PyObject *origin = PyTuple_GET_ITEM(args, 0); - PyObject *arguments = PyTuple_GET_ITEM(args, 1); - return Py_GenericAlias(origin, arguments); -} - -// TODO: -// - argument clinic? -// - __doc__? -// - cache? -PyTypeObject Py_GenericAliasType = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "types.GenericAlias", - .tp_basicsize = sizeof(gaobject), - .tp_dealloc = ga_dealloc, - .tp_repr = ga_repr, - .tp_as_mapping = &ga_as_mapping, - .tp_call = ga_call, - .tp_getattro = ga_getattro, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = ga_traverse, - .tp_richcompare = ga_richcompare, - .tp_methods = ga_methods, - .tp_members = ga_members, - .tp_alloc = PyType_GenericAlloc, - .tp_new = ga_new, - .tp_free = PyObject_GC_Del, - .tp_getset = ga_properties, -}; - -PyObject * -Py_GenericAlias(PyObject *origin, PyObject *args) -{ - if (!PyTuple_Check(args)) { - args = PyTuple_Pack(1, args); - if (args == NULL) { - return NULL; - } - } - else { - Py_INCREF(args); - } - - gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); - if (alias == NULL) { - Py_DECREF(args); - return NULL; - } - - Py_INCREF(origin); - alias->origin = origin; - alias->args = args; - alias->parameters = NULL; - _PyObject_GC_TRACK(alias); - return (PyObject *)alias; -} diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c new file mode 100644 index 00000000000000..9cd2c81c5117be --- /dev/null +++ b/Objects/genericaliasobject.c @@ -0,0 +1,433 @@ +// types.GenericAlias -- used to represent e.g. list[int]. + +#include "Python.h" +#include "pycore_object.h" +#include "structmember.h" + +typedef struct { + PyObject_HEAD + PyObject *origin; + PyObject *args; + PyObject *parameters; +} gaobject; + +static void +ga_dealloc(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(alias->origin); + Py_XDECREF(alias->args); + Py_XDECREF(alias->parameters); + self->ob_type->tp_free(self); +} + +static int +ga_traverse(PyObject *self, visitproc visit, void *arg) +{ + gaobject *alias = (gaobject *)self; + Py_VISIT(alias->origin); + Py_VISIT(alias->args); + Py_VISIT(alias->parameters); + return 0; +} + +static int +ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) +{ + PyObject *qualname = PyObject_GetAttrString(p, "__qualname__"); + PyErr_Clear(); + PyObject *module = PyObject_GetAttrString(p, "__module__"); + PyErr_Clear(); + PyObject *r = NULL; + int err; + if (PyObject_HasAttrString(p, "__origin__") && + PyObject_HasAttrString(p, "__args__")) + { + // It looks like a GenericAlias + r = PyObject_Repr(p); + } + else if (p == Py_Ellipsis) { + // The Ellipsis object + r = PyUnicode_FromString("..."); + } + else if (qualname != NULL && module != NULL) { + // Looks like a class + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + // builtins don't need a module name + Py_INCREF(qualname); + r = qualname; + } + else { + r = PyUnicode_FromFormat("%U.%U", module, qualname); + } + } + else { + // fallback + r = PyObject_Repr(p); + } + if (r == NULL) { + // error if any of the above PyObject_Repr/PyUnicode_From* fail + err = -1; + } else { + err = _PyUnicodeWriter_WriteStr(writer, r); + } + Py_XDECREF(qualname); + Py_XDECREF(module); + Py_XDECREF(r); + return err; +} + +static PyObject * +ga_repr(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + Py_ssize_t len = PyTuple_Size(alias->args); + + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + + if (ga_repr_item(&writer, alias->origin) < 0) { + goto error; + } + if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) { + goto error; + } + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + goto error; + } + } + PyObject *p = PyTuple_GET_ITEM(alias->args, i); + if (ga_repr_item(&writer, p) < 0) { + goto error; + } + } + if (len == 0) { + // for something like tuple[()] we should print a "()" + if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) { + goto error; + } + } + if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) { + goto error; + } + return _PyUnicodeWriter_Finish(&writer); +error: + _PyUnicodeWriter_Dealloc(&writer); + return NULL; +} + +// isinstance(obj, TypeVar) without importing typing.py. If someone +// names some other class TypeVar, it will be mistaken for a TypeVar. +// Maybe that's a feature; or maybe we'll have to see if +// sys.modules['typing'] exists and look for its 'TypeVar' attribute +// (which is roughly what dataclasses.py uses to recognize ClassVar). +static int +is_typevar(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + return strcmp(type->tp_name, "TypeVar") == 0; +} + +// Index of item in self[:len], or -1 if not found (self is a tuple) +static Py_ssize_t +tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) +{ + for (Py_ssize_t i = 0; i < len; i++) { + if (PyTuple_GET_ITEM(self, i) == item) { + return i; + } + } + return -1; +} + +// tuple(t for t in args if isinstance(t, TypeVar)) +static PyObject * +make_parameters(PyObject *args) +{ + Py_ssize_t len = PyTuple_GET_SIZE(args); + PyObject *parameters = PyTuple_New(len); + if (parameters == NULL) + return NULL; + Py_ssize_t iparam = 0; + for (Py_ssize_t iarg = 0; iarg < len; iarg++) { + PyObject *t = PyTuple_GET_ITEM(args, iarg); + if (is_typevar(t)) { + if (tuple_index(parameters, iparam, t) < 0) { + Py_INCREF(t); + PyTuple_SET_ITEM(parameters, iparam, t); + iparam++; + } + } + } + if (iparam < len) { + if (_PyTuple_Resize(¶meters, iparam) < 0) { + Py_XDECREF(parameters); + return NULL; + } + } + return parameters; +} + +static PyObject * +ga_getitem(PyObject *self, PyObject *item) +{ + gaobject *alias = (gaobject *)self; + // do a lookup for __parameters__ so it gets populated (if not already) + if (alias->parameters == NULL) { + _Py_IDENTIFIER(__parameters__); + PyObject *params = _PyObject_GetAttrId(self, &PyId___parameters__); + if (params == NULL) { + return NULL; + } + Py_DECREF(params); + } + Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters); + if (nparams == 0) { + return PyErr_Format(PyExc_TypeError, + "There are no type variables left in %R", + self); + } + int is_tuple = PyTuple_Check(item); + Py_ssize_t nitem = is_tuple ? PyTuple_GET_SIZE(item) : 1; + if (nitem != nparams) { + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %R", + nitem > nparams ? "many" : "few", + self); + } + Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); + PyObject *newargs = PyTuple_New(nargs); + if (newargs == NULL) + return NULL; + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + if (is_typevar(arg)) { + Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg); + assert(iparam >= 0); + if (is_tuple) { + arg = PyTuple_GET_ITEM(item, iparam); + } + else { + assert(iparam == 0); + arg = item; + } + } + Py_INCREF(arg); + PyTuple_SET_ITEM(newargs, iarg, arg); + } + PyObject *res = Py_GenericAlias(alias->origin, newargs); + Py_DECREF(newargs); + return res; +} + +static PyMappingMethods ga_as_mapping = { + .mp_subscript = ga_getitem, +}; + +static PyObject * +ga_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + gaobject *alias = (gaobject *)self; + PyObject *obj = PyObject_Call(alias->origin, args, kwds); + if (obj != NULL) { + PyObject_SetAttrString(obj, "__orig_class__", self); + PyErr_Clear(); + } + return obj; +} + +static const char* const attr_exceptions[] = { + "__origin__", + "__args__", + "__parameters__", + "__mro_entries__", + "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ + "__reduce__", + NULL, +}; + +static PyObject * +ga_getattro(PyObject *self, PyObject *name) +{ + gaobject *alias = (gaobject *)self; + if (PyUnicode_Check(name)) { + for (const char * const *p = attr_exceptions; ; p++) { + if (*p == NULL) { + return PyObject_GetAttr(alias->origin, name); + } + if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { + break; + } + } + } + return PyObject_GenericGetAttr(self, name); +} + +static PyObject * +ga_richcompare(PyObject *a, PyObject *b, int op) +{ + if (Py_TYPE(a) != &Py_GenericAliasType || + Py_TYPE(b) != &Py_GenericAliasType || + (op != Py_EQ && op != Py_NE)) + { + Py_RETURN_NOTIMPLEMENTED; + } + + if (op == Py_NE) { + PyObject *eq = ga_richcompare(a, b, Py_EQ); + if (eq == NULL) + return NULL; + Py_DECREF(eq); + if (eq == Py_True) { + Py_RETURN_FALSE; + } + else { + Py_RETURN_TRUE; + } + } + + gaobject *aa = (gaobject *)a; + gaobject *bb = (gaobject *)b; + PyObject *eq = PyObject_RichCompare(aa->origin, bb->origin, Py_EQ); + if (eq == NULL) + return NULL; + if (eq == Py_False) { + return eq; + } + Py_DECREF(eq); + return PyObject_RichCompare(aa->args, bb->args, Py_EQ); +} + +static PyObject * +ga_mro_entries(PyObject *self, PyObject *args) +{ + gaobject *alias = (gaobject *)self; + return PyTuple_Pack(1, alias->origin); +} + +static PyObject * +ga_instancecheck(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyErr_Format(PyExc_TypeError, + "isinstance() argument 2 cannot be a parameterized generic", + self); +} + +static PyObject * +ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyErr_Format(PyExc_TypeError, + "issubclass() argument 2 cannot be a parameterized generic", + self); +} + +static PyObject * +ga_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + gaobject *alias = (gaobject *)self; + return Py_BuildValue("O(OO)", Py_TYPE(alias), + alias->origin, alias->args); +} + +static PyMethodDef ga_methods[] = { + {"__mro_entries__", ga_mro_entries, METH_O}, + {"__instancecheck__", ga_instancecheck, METH_O}, + {"__subclasscheck__", ga_subclasscheck, METH_O}, + {"__reduce__", ga_reduce, METH_NOARGS}, + {0} +}; + +static PyMemberDef ga_members[] = { + {"__origin__", T_OBJECT, offsetof(gaobject, origin), READONLY}, + {"__args__", T_OBJECT, offsetof(gaobject, args), READONLY}, + {0} +}; + +static PyObject * +ga_parameters(PyObject *self, void *unused) +{ + gaobject *alias = (gaobject *)self; + if (alias->parameters == NULL) { + alias->parameters = make_parameters(alias->args); + if (alias->parameters == NULL) { + Py_DECREF(alias->parameters); + return NULL; + } + } + Py_INCREF(alias->parameters); + return alias->parameters; +} + +static PyGetSetDef ga_properties[] = { + {"__parameters__", ga_parameters, (setter)NULL, "Type variables in the GenericAlias.", NULL}, + {0} +}; + +static PyObject * +ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { + return PyErr_Format(PyExc_TypeError, "GenericAlias does not support keyword arguments"); + } + if (PyTuple_Size(args) != 2) { + return PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); + } + PyObject *origin = PyTuple_GET_ITEM(args, 0); + PyObject *arguments = PyTuple_GET_ITEM(args, 1); + return Py_GenericAlias(origin, arguments); +} + +// TODO: +// - argument clinic? +// - __doc__? +// - cache? +PyTypeObject Py_GenericAliasType = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "types.GenericAlias", + .tp_basicsize = sizeof(gaobject), + .tp_dealloc = ga_dealloc, + .tp_repr = ga_repr, + .tp_as_mapping = &ga_as_mapping, + .tp_call = ga_call, + .tp_getattro = ga_getattro, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = ga_traverse, + .tp_richcompare = ga_richcompare, + .tp_methods = ga_methods, + .tp_members = ga_members, + .tp_alloc = PyType_GenericAlloc, + .tp_new = ga_new, + .tp_free = PyObject_GC_Del, + .tp_getset = ga_properties, +}; + +PyObject * +Py_GenericAlias(PyObject *origin, PyObject *args) +{ + if (!PyTuple_Check(args)) { + args = PyTuple_Pack(1, args); + if (args == NULL) { + return NULL; + } + } + else { + Py_INCREF(args); + } + + gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); + if (alias == NULL) { + Py_DECREF(args); + return NULL; + } + + Py_INCREF(origin); + alias->origin = origin; + alias->args = args; + alias->parameters = NULL; + _PyObject_GC_TRACK(alias); + return (PyObject *)alias; +} From f658a64f94e16ee543e03fb335d51f0dfaec522b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 14 Mar 2020 14:45:38 -0700 Subject: [PATCH 54/72] Add a hash function to GenericAlias --- Lib/test/test_genericalias.py | 13 +++++++++++++ Objects/genericaliasobject.c | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index d5e919bd2c45d8..197f1dbd21f2cc 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -9,6 +9,7 @@ from contextlib import AbstractContextManager, AbstractAsyncContextManager from re import Pattern, Match from types import GenericAlias, MappingProxyType +import typing from typing import TypeVar T = TypeVar('T') @@ -197,6 +198,18 @@ def test_pickle(self): self.assertEqual(alias.__args__, loaded.__args__) self.assertEqual(alias.__parameters__, loaded.__parameters__) + def test_union(self): + a = typing.Union[list[int], list[str]] + self.assertEqual(a.__args__, (list[int], list[str])) + self.assertEqual(a.__parameters__, ()) + + def test_union_generic(self): + T = typing.TypeVar('T') + a = typing.Union[list[T], tuple[T, ...]] + self.assertEqual(a.__args__, (list[T], tuple[T, ...])) + # TODO: To make this work, would need to update typing.py to recognize list[T]. + # self.assertEqual(a.__parameters__, (T,)) + if __name__ == "__main__": unittest.main() diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 9cd2c81c5117be..d3470a65c468a0 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -228,6 +228,16 @@ static PyMappingMethods ga_as_mapping = { .mp_subscript = ga_getitem, }; +static Py_hash_t +ga_hash(PyObject *self) +{ + gaobject *alias = (gaobject *)self; + // TODO: Hash in the hash for the origin + Py_hash_t h0 = PyObject_Hash(alias->origin); + Py_hash_t h1 = PyObject_Hash(alias->args); + return h0 ^ h1; +} + static PyObject * ga_call(PyObject *self, PyObject *args, PyObject *kwds) { @@ -392,6 +402,7 @@ PyTypeObject Py_GenericAliasType = { .tp_dealloc = ga_dealloc, .tp_repr = ga_repr, .tp_as_mapping = &ga_as_mapping, + .tp_hash = ga_hash, .tp_call = ga_call, .tp_getattro = ga_getattro, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, From bfd6342c0ee4b3c83e6235efad2200b9ff3aa607 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 14 Mar 2020 18:11:00 -0700 Subject: [PATCH 55/72] Make Union[list[T], int] work This involves a rare change in typing.py. --- Lib/test/test_genericalias.py | 3 +-- Lib/typing.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 197f1dbd21f2cc..9d8dbfe04205fd 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -207,8 +207,7 @@ def test_union_generic(self): T = typing.TypeVar('T') a = typing.Union[list[T], tuple[T, ...]] self.assertEqual(a.__args__, (list[T], tuple[T, ...])) - # TODO: To make this work, would need to update typing.py to recognize list[T]. - # self.assertEqual(a.__parameters__, (T,)) + self.assertEqual(a.__parameters__, (T,)) if __name__ == "__main__": diff --git a/Lib/typing.py b/Lib/typing.py index c265e57aafe061..197001928decbf 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -26,7 +26,7 @@ import re as stdlib_re # Avoid confusion with the re we export. import sys import types -from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType +from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias # Please keep __all__ alphabetized within each category. __all__ = [ @@ -180,7 +180,8 @@ def _collect_type_vars(types): for t in types: if isinstance(t, TypeVar) and t not in tvars: tvars.append(t) - if isinstance(t, _GenericAlias) and not t._special: + if ((isinstance(t, _GenericAlias) and not t._special) + or isinstance(t, GenericAlias)): tvars.extend([t for t in t.__parameters__ if t not in tvars]) return tuple(tvars) From 7919fa24526bb014a183563c848865bc869c1ddd Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 15 Mar 2020 19:19:42 -0700 Subject: [PATCH 56/72] Add GenericAlias to Windows build (#16) --- PCbuild/pythoncore.vcxproj | 1 + 1 file changed, 1 insertion(+) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 0acf7f4a8de808..acced0833ce89d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -382,6 +382,7 @@ + From 728084a49bc62b46217d9accbf9a2011bb6ffb88 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 18 Mar 2020 09:58:21 -0700 Subject: [PATCH 57/72] Run CI on PRs for pep585 branch --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6774ae46f7e0f2..da1d4c54671b1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,6 +13,7 @@ on: - '**/*.rst' pull_request: branches: + - pep585 - master - 3.8 - 3.7 From ddbce27bc8716e8fc0acbae0907691b0d41b3cae Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 30 Mar 2020 16:03:00 -0700 Subject: [PATCH 58/72] Add docstring for GenericAlias (#17) Co-authored-by: Guido van Rossum --- Objects/genericaliasobject.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index d3470a65c468a0..863af825dfd3e5 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -398,6 +398,9 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyTypeObject Py_GenericAliasType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "types.GenericAlias", + .tp_doc = "Represent a PEP 585 generic type\n" + "\n" + "E.g. for t = list[int], t.origin is list and t.args is (int,).", .tp_basicsize = sizeof(gaobject), .tp_dealloc = ga_dealloc, .tp_repr = ga_repr, From 4db065d9662b003047450f10e5b8184fd5e4e647 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 30 Mar 2020 21:57:49 -0700 Subject: [PATCH 59/72] Remove unneeded _GenericAlias definition from collections/__init__.py --- Lib/collections/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 55ee8aeee7e500..e19840650dd7eb 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -38,8 +38,6 @@ except ImportError: pass -_GenericAlias = type(list[int]) - def __getattr__(name): # For backwards compatibility, continue to make the collections ABCs From e14625fa30d74a872f2f6af390ce2564c6b3e1b5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 13:52:31 -0700 Subject: [PATCH 60/72] Make is_typevar() harder to fool --- Objects/genericaliasobject.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 863af825dfd3e5..4e903bb2595c9a 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -120,16 +120,24 @@ ga_repr(PyObject *self) return NULL; } -// isinstance(obj, TypeVar) without importing typing.py. If someone -// names some other class TypeVar, it will be mistaken for a TypeVar. -// Maybe that's a feature; or maybe we'll have to see if -// sys.modules['typing'] exists and look for its 'TypeVar' attribute -// (which is roughly what dataclasses.py uses to recognize ClassVar). +// isinstance(obj, TypeVar) without importing typing.py. static int is_typevar(PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - return strcmp(type->tp_name, "TypeVar") == 0; + if (strcmp(type->tp_name, "TypeVar") != 0) { + return 0; + } + PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); + if (module == NULL) { + // Oops. (Bubbling up the error would just complicate the callers.) + PyErr_Clear(); + return 0; + } + int res = PyUnicode_Check(module) + && PyUnicode_CompareWithASCIIString(module, "typing") == 0; + Py_DECREF(module); + return res; } // Index of item in self[:len], or -1 if not found (self is a tuple) From 76c080534dc282ca42c0d7ccbb920b914afe887b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 15:22:48 -0700 Subject: [PATCH 61/72] Rewrite ga_repr_item() to honor exceptions Co-Authored-By: Serhiy Storchaka --- Objects/genericaliasobject.c | 82 +++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 25 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 4e903bb2595c9a..4b375b8411d54b 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -36,46 +36,78 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) static int ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) { - PyObject *qualname = PyObject_GetAttrString(p, "__qualname__"); - PyErr_Clear(); - PyObject *module = PyObject_GetAttrString(p, "__module__"); - PyErr_Clear(); + _Py_IDENTIFIER(__module__); + _Py_IDENTIFIER(__qualname__); + _Py_IDENTIFIER(__origin__); + _Py_IDENTIFIER(__args__); + PyObject *qualname = NULL; + PyObject *module = NULL; PyObject *r = NULL; + PyObject *tmp; int err; - if (PyObject_HasAttrString(p, "__origin__") && - PyObject_HasAttrString(p, "__args__")) - { - // It looks like a GenericAlias - r = PyObject_Repr(p); - } - else if (p == Py_Ellipsis) { + + if (p == Py_Ellipsis) { // The Ellipsis object r = PyUnicode_FromString("..."); + goto done; } - else if (qualname != NULL && module != NULL) { - // Looks like a class - if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { - // builtins don't need a module name - Py_INCREF(qualname); - r = qualname; + + if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) { + goto done; + } + if (tmp != NULL) { + Py_DECREF(tmp); + if (_PyObject_LookupAttrId(p, &PyId___args__, &tmp) < 0) { + goto done; } - else { - r = PyUnicode_FromFormat("%U.%U", module, qualname); + if (tmp != NULL) { + Py_DECREF(tmp); + // It looks like a GenericAlias + goto use_repr; } } + + if (_PyObject_LookupAttrId(p, &PyId___qualname__, &qualname) < 0) { + goto done; + } + if (qualname == NULL) { + goto use_repr; + } + if (_PyObject_LookupAttrId(p, &PyId___module__, &module) < 0) { + goto done; + } + if (module == NULL || module == Py_None) { + goto use_repr; + } + + // Looks like a class + if (PyUnicode_Check(module) && + _PyUnicode_EqualToASCIIString(module, "builtins")) + { + // builtins don't need a module name + Py_INCREF(qualname); + r = qualname; + goto done; + } else { - // fallback - r = PyObject_Repr(p); + r = PyUnicode_FromFormat("%S.%S", module, qualname); + goto done; } + +use_repr: + r = PyObject_Repr(p); + +done: + Py_XDECREF(qualname); + Py_XDECREF(module); if (r == NULL) { // error if any of the above PyObject_Repr/PyUnicode_From* fail err = -1; - } else { + } + else { err = _PyUnicodeWriter_WriteStr(writer, r); + Py_DECREF(r); } - Py_XDECREF(qualname); - Py_XDECREF(module); - Py_XDECREF(r); return err; } From 128aacb965ddd7a130b29053ec8fd75fd95221fc Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 15:19:32 -0700 Subject: [PATCH 62/72] Use __class_getitem__ = classmethod(GenericAlias) --- Lib/_collections_abc.py | 24 ++++++------------------ Lib/_pyio.py | 1 - Lib/contextlib.py | 6 ++---- Lib/os.py | 4 +--- Lib/subprocess.py | 8 ++------ Lib/tempfile.py | 4 +--- 6 files changed, 12 insertions(+), 35 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 3a218e1d114600..36cd9930003475 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -112,9 +112,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__await__") return NotImplemented - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return GenericAlias(cls, item) + __class_getitem__ = classmethod(GenericAlias) class Coroutine(Awaitable): @@ -175,9 +173,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__aiter__") return NotImplemented - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return GenericAlias(cls, item) + __class_getitem__ = classmethod(GenericAlias) class AsyncIterator(AsyncIterable): @@ -265,9 +261,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__iter__") return NotImplemented - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return GenericAlias(cls, item) + __class_getitem__ = classmethod(GenericAlias) class Iterator(Iterable): @@ -401,9 +395,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__contains__") return NotImplemented - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return GenericAlias(cls, item) + __class_getitem__ = classmethod(GenericAlias) class Collection(Sized, Iterable, Container): @@ -431,9 +423,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__call__") return NotImplemented - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return GenericAlias(cls, item) + __class_getitem__ = classmethod(GenericAlias) ### SETS ### @@ -733,9 +723,7 @@ def __len__(self): def __repr__(self): return '{0.__class__.__name__}({0._mapping!r})'.format(self) - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return GenericAlias(cls, item) + __class_getitem__ = classmethod(GenericAlias) class KeysView(MappingView, Set): diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 3800870028c27c..4804ed27cd14d6 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -8,7 +8,6 @@ import errno import stat import sys -import types # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock if sys.platform in {'win32', 'cygwin'}: diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 7601a8f7c20775..ff92d9f913f4c2 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -16,8 +16,7 @@ class AbstractContextManager(abc.ABC): """An abstract base class for context managers.""" - def __class_getitem__(cls, parameters): - return GenericAlias(cls, parameters) + __class_getitem__ = classmethod(GenericAlias) def __enter__(self): """Return `self` upon entering the runtime context.""" @@ -39,8 +38,7 @@ class AbstractAsyncContextManager(abc.ABC): """An abstract base class for asynchronous context managers.""" - def __class_getitem__(cls, parameters): - return GenericAlias(cls, parameters) + __class_getitem__ = classmethod(GenericAlias) async def __aenter__(self): """Return `self` upon entering the runtime context.""" diff --git a/Lib/os.py b/Lib/os.py index b1626b7de6efc2..2c0276483f6ae1 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1080,9 +1080,7 @@ def __subclasshook__(cls, subclass): return _check_methods(subclass, '__fspath__') return NotImplemented - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return GenericAlias(cls, item) + __class_getitem__ = classmethod(GenericAlias) if name == 'nt': diff --git a/Lib/subprocess.py b/Lib/subprocess.py index ce99bc2ce34b34..aa757ecf23da35 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -447,9 +447,7 @@ def __repr__(self): args.append('stderr={!r}'.format(self.stderr)) return "{}({})".format(type(self).__name__, ', '.join(args)) - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return types.GenericAlias(cls, item) + __class_getitem__ = classmethod(types.GenericAlias) def check_returncode(self): @@ -993,9 +991,7 @@ def __repr__(self): obj_repr = obj_repr[:76] + "...>" return obj_repr - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return types.GenericAlias(cls, item) + __class_getitem__ = classmethod(types.GenericAlias) @property def universal_newlines(self): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index ffb1cb2220bcf7..a398345f108d0f 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -644,9 +644,7 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1, 'encoding': encoding, 'newline': newline, 'dir': dir, 'errors': errors} - def __class_getitem__(cls, item): - """Internal: PEP 585.""" - return _types.GenericAlias(cls, item) + __class_getitem__ = classmethod(_types.GenericAlias) def _check(self, file): if self._rolled: return From 94f4095f82f59d705f09db641c265fcc32a590e4 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 15:26:11 -0700 Subject: [PATCH 63/72] Update Objects/genericaliasobject.c Co-Authored-By: Serhiy Storchaka --- Objects/genericaliasobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 4b375b8411d54b..fb6aea98fe8753 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -167,7 +167,7 @@ is_typevar(PyObject *obj) return 0; } int res = PyUnicode_Check(module) - && PyUnicode_CompareWithASCIIString(module, "typing") == 0; + && _PyUnicode_EqualToASCIIString(module, "typing"); Py_DECREF(module); return res; } From 79f41704bb153f7e7b2acedf70885b8b9fed753f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 15:30:08 -0700 Subject: [PATCH 64/72] Apply suggestions from code review Co-Authored-By: Serhiy Storchaka --- Objects/genericaliasobject.c | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index fb6aea98fe8753..9ded4f670189b5 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -218,12 +218,10 @@ ga_getitem(PyObject *self, PyObject *item) gaobject *alias = (gaobject *)self; // do a lookup for __parameters__ so it gets populated (if not already) if (alias->parameters == NULL) { - _Py_IDENTIFIER(__parameters__); - PyObject *params = _PyObject_GetAttrId(self, &PyId___parameters__); - if (params == NULL) { + alias->parameters = make_parameters(alias->args); + if (alias->parameters == NULL) { return NULL; } - Py_DECREF(params); } Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters); if (nparams == 0) { @@ -274,7 +272,13 @@ ga_hash(PyObject *self) gaobject *alias = (gaobject *)self; // TODO: Hash in the hash for the origin Py_hash_t h0 = PyObject_Hash(alias->origin); + if (h0 == -1) { + return -1; + } Py_hash_t h1 = PyObject_Hash(alias->args); + if (h1 == -1) { + return -1; + } return h0 ^ h1; } @@ -284,8 +288,13 @@ ga_call(PyObject *self, PyObject *args, PyObject *kwds) gaobject *alias = (gaobject *)self; PyObject *obj = PyObject_Call(alias->origin, args, kwds); if (obj != NULL) { - PyObject_SetAttrString(obj, "__orig_class__", self); - PyErr_Clear(); + if (PyObject_SetAttrString(obj, "__orig_class__", self) < 0) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(obj); + return NULL; + } + PyErr_Clear(); + } } return obj; } @@ -309,7 +318,7 @@ ga_getattro(PyObject *self, PyObject *name) if (*p == NULL) { return PyObject_GetAttr(alias->origin, name); } - if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { + if (_PyUnicode_EqualToASCIIString(name, *p)) { break; } } @@ -342,13 +351,13 @@ ga_richcompare(PyObject *a, PyObject *b, int op) gaobject *aa = (gaobject *)a; gaobject *bb = (gaobject *)b; - PyObject *eq = PyObject_RichCompare(aa->origin, bb->origin, Py_EQ); - if (eq == NULL) + int eq = PyObject_RichCompareBool(aa->origin, bb->origin, Py_EQ); + if (eq < 0) { return NULL; - if (eq == Py_False) { - return eq; } - Py_DECREF(eq); + if (!eq) { + Py_RETURN_FALSE; + } return PyObject_RichCompare(aa->args, bb->args, Py_EQ); } @@ -421,10 +430,10 @@ static PyObject * ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { - return PyErr_Format(PyExc_TypeError, "GenericAlias does not support keyword arguments"); + return PyErr_SetString(PyExc_TypeError, "GenericAlias does not support keyword arguments"); } - if (PyTuple_Size(args) != 2) { - return PyErr_Format(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); + if (PyTuple_GET_SIZE(args) != 2) { + return PyErr_SetString(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); } PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); From f6f5b810fd1df1c81770080815893f3a62350e6c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 15:24:55 -0700 Subject: [PATCH 65/72] Use PyTuple_GET_SIZE --- Objects/genericaliasobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 9ded4f670189b5..72aec4c1cb7f1b 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -115,7 +115,7 @@ static PyObject * ga_repr(PyObject *self) { gaobject *alias = (gaobject *)self; - Py_ssize_t len = PyTuple_Size(alias->args); + Py_ssize_t len = PyTuple_GET_SIZE(alias->args); _PyUnicodeWriter writer; _PyUnicodeWriter_Init(&writer); From 99cac832a106ac3a4105bb853369278dc3d938d6 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 15:35:40 -0700 Subject: [PATCH 66/72] Use PyErr_SetString correctly --- Objects/genericaliasobject.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 72aec4c1cb7f1b..9dc591cbb2356e 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -371,17 +371,17 @@ ga_mro_entries(PyObject *self, PyObject *args) static PyObject * ga_instancecheck(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return PyErr_Format(PyExc_TypeError, - "isinstance() argument 2 cannot be a parameterized generic", - self); + PyErr_SetString(PyExc_TypeError, + "isinstance() argument 2 cannot be a parameterized generic"); + return NULL; } static PyObject * ga_subclasscheck(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return PyErr_Format(PyExc_TypeError, - "issubclass() argument 2 cannot be a parameterized generic", - self); + PyErr_SetString(PyExc_TypeError, + "issubclass() argument 2 cannot be a parameterized generic"); + return NULL; } static PyObject * @@ -413,7 +413,6 @@ ga_parameters(PyObject *self, void *unused) if (alias->parameters == NULL) { alias->parameters = make_parameters(alias->args); if (alias->parameters == NULL) { - Py_DECREF(alias->parameters); return NULL; } } @@ -430,10 +429,12 @@ static PyObject * ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { - return PyErr_SetString(PyExc_TypeError, "GenericAlias does not support keyword arguments"); + PyErr_SetString(PyExc_TypeError, "GenericAlias does not support keyword arguments"); + return NULL; } if (PyTuple_GET_SIZE(args) != 2) { - return PyErr_SetString(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); + PyErr_SetString(PyExc_TypeError, "GenericAlias expects 2 positional arguments"); + return NULL; } PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); From 604493486829f7ec12c89c24f34f5a8ae97367bb Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 16:34:39 -0700 Subject: [PATCH 67/72] Fix test failure (setattr may raise TypeError) --- Objects/genericaliasobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 9dc591cbb2356e..1aa0d52b35a5d7 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -289,7 +289,9 @@ ga_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject *obj = PyObject_Call(alias->origin, args, kwds); if (obj != NULL) { if (PyObject_SetAttrString(obj, "__orig_class__", self) < 0) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError) && + !PyErr_ExceptionMatches(PyExc_TypeError)) + { Py_DECREF(obj); return NULL; } From 355d2675370649a794f007bedc2ed39e06a005f0 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 Apr 2020 19:44:04 -0700 Subject: [PATCH 68/72] Propagate errors from is_typevar() --- Objects/genericaliasobject.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 1aa0d52b35a5d7..1a4c034c3cdfab 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -153,6 +153,7 @@ ga_repr(PyObject *self) } // isinstance(obj, TypeVar) without importing typing.py. +// Returns -1 for errors. static int is_typevar(PyObject *obj) { @@ -162,9 +163,7 @@ is_typevar(PyObject *obj) } PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); if (module == NULL) { - // Oops. (Bubbling up the error would just complicate the callers.) - PyErr_Clear(); - return 0; + return -1; } int res = PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); @@ -195,7 +194,12 @@ make_parameters(PyObject *args) Py_ssize_t iparam = 0; for (Py_ssize_t iarg = 0; iarg < len; iarg++) { PyObject *t = PyTuple_GET_ITEM(args, iarg); - if (is_typevar(t)) { + int typevar = is_typevar(t); + if (typevar < 0) { + Py_XDECREF(parameters); + return NULL; + } + if (typevar) { if (tuple_index(parameters, iparam, t) < 0) { Py_INCREF(t); PyTuple_SET_ITEM(parameters, iparam, t); @@ -243,7 +247,12 @@ ga_getitem(PyObject *self, PyObject *item) return NULL; for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); - if (is_typevar(arg)) { + int typevar = is_typevar(arg); + if (typevar < 0) { + Py_DECREF(newargs); + return NULL; + } + if (typevar) { Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg); assert(iparam >= 0); if (is_tuple) { From 95b3c2a94873e862b06261fe69a030b27405c26e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 6 Apr 2020 12:01:40 -0700 Subject: [PATCH 69/72] Use PyObject_Str(qualname) Co-Authored-By: Serhiy Storchaka --- Objects/genericaliasobject.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 1a4c034c3cdfab..49f537ef96845a 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -85,8 +85,7 @@ ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) _PyUnicode_EqualToASCIIString(module, "builtins")) { // builtins don't need a module name - Py_INCREF(qualname); - r = qualname; + r = PyObject_Str(qualname); goto done; } else { From d3839eabb23970163ca1234f6c1a3d434c23f9ab Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 6 Apr 2020 13:44:04 -0700 Subject: [PATCH 70/72] Delete mention of pep585 branch in GitHub workflow --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0830b83bcbf485..50d1561518bd82 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,6 @@ on: - '**/*.rst' pull_request: branches: - - pep585 - master - 3.8 - 3.7 From 31fd842170661521d3ccb55fbacaccdc06084e75 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 6 Apr 2020 21:08:18 -0700 Subject: [PATCH 71/72] Move Py_GenericAlias* to genericaliasobject.h --- Include/Python.h | 1 + Include/descrobject.h | 5 ----- Include/genericaliasobject.h | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 Include/genericaliasobject.h diff --git a/Include/Python.h b/Include/Python.h index 969d8e6bea7416..769ec49779c0fc 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -120,6 +120,7 @@ #include "iterobject.h" #include "genobject.h" #include "descrobject.h" +#include "genericaliasobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/descrobject.h b/Include/descrobject.h index 006361b11940ef..ead269d1d2f796 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -101,11 +101,6 @@ PyAPI_FUNC(PyObject *) PyWrapper_New(PyObject *, PyObject *); PyAPI_DATA(PyTypeObject) PyProperty_Type; - -// Experimental code to implement PEP 585 (list[int] etc.) -PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *); -PyAPI_DATA(PyTypeObject) Py_GenericAliasType; - #ifdef __cplusplus } #endif diff --git a/Include/genericaliasobject.h b/Include/genericaliasobject.h new file mode 100644 index 00000000000000..cf002976b27cd7 --- /dev/null +++ b/Include/genericaliasobject.h @@ -0,0 +1,14 @@ +// Implementation of PEP 585: support list[int] etc. +#ifndef Py_GENERICALIASOBJECT_H +#define Py_GENERICALIASOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *); +PyAPI_DATA(PyTypeObject) Py_GenericAliasType; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_GENERICALIASOBJECT_H */ From cdb46c3d80f5e80e681a6fcc06a39637d8c26277 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 7 Apr 2020 08:47:30 -0700 Subject: [PATCH 72/72] Add Py_GenericAlias[Type] to PC/python3.ref --- PC/python3.def | 2 ++ 1 file changed, 2 insertions(+) diff --git a/PC/python3.def b/PC/python3.def index c7aed8789cfaaa..083384e30f648f 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -734,6 +734,8 @@ EXPORTS Py_FileSystemDefaultEncoding=python39.Py_FileSystemDefaultEncoding DATA Py_Finalize=python39.Py_Finalize Py_FinalizeEx=python39.Py_FinalizeEx + Py_GenericAlias=python39.Py_GenericAlias + Py_GenericAliasType=python39.Py_GenericAliasType Py_GetBuildInfo=python39.Py_GetBuildInfo Py_GetCompiler=python39.Py_GetCompiler Py_GetCopyright=python39.Py_GetCopyright