bpo-43901: Lazy-create an empty annotations dict in all unannotated user classes and modules (#25623)

larryhastings · web-flow · commit 2f2b69855d65 · 2021-04-29T20:09:08.000-07:00
Change class and module objects to lazy-create empty annotations dicts on demand.  The annotations dicts are stored in the object's `__dict__` for backwards compatibility.
diff --git a/Lib/test/ann_module4.py b/Lib/test/ann_module4.py
@@ -0,0 +1,5 @@
+# This ann_module isn't for test_typing,
+# it's for test_module
+
+a:int=3
+b:str=4
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
@@ -382,8 +382,7 @@ class CC(metaclass=CMeta):
         self.assertEqual(CC.__annotations__['xx'], 'ANNOT')
 
     def test_var_annot_module_semantics(self):
-        with self.assertRaises(AttributeError):
-            print(test.__annotations__)
+        self.assertEqual(test.__annotations__, {})
         self.assertEqual(ann_module.__annotations__,
                      {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int]})
         self.assertEqual(ann_module.M.__annotations__,
diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py
@@ -286,6 +286,60 @@ class M(ModuleType):
             melon = Descr()
         self.assertRaises(RuntimeError, getattr, M("mymod"), "melon")
 
+    def test_lazy_create_annotations(self):
+        # module objects lazy create their __annotations__ dict on demand.
+        # the annotations dict is stored in module.__dict__.
+        # a freshly created module shouldn't have an annotations dict yet.
+        foo = ModuleType("foo")
+        for i in range(4):
+            self.assertFalse("__annotations__" in foo.__dict__)
+            d = foo.__annotations__
+            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertEqual(foo.__annotations__, d)
+            self.assertEqual(foo.__dict__['__annotations__'], d)
+            if i % 2:
+                del foo.__annotations__
+            else:
+                del foo.__dict__['__annotations__']
+
+    def test_setting_annotations(self):
+        foo = ModuleType("foo")
+        for i in range(4):
+            self.assertFalse("__annotations__" in foo.__dict__)
+            d = {'a': int}
+            foo.__annotations__ = d
+            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertEqual(foo.__annotations__, d)
+            self.assertEqual(foo.__dict__['__annotations__'], d)
+            if i % 2:
+                del foo.__annotations__
+            else:
+                del foo.__dict__['__annotations__']
+
+    def test_annotations_getset_raises(self):
+        # module has no dict, all operations fail
+        foo = ModuleType.__new__(ModuleType)
+        with self.assertRaises(TypeError):
+            print(foo.__annotations__)
+        with self.assertRaises(TypeError):
+            foo.__annotations__ = {}
+        with self.assertRaises(TypeError):
+            del foo.__annotations__
+
+        # double delete
+        foo = ModuleType("foo")
+        foo.__annotations__ = {}
+        del foo.__annotations__
+        with self.assertRaises(AttributeError):
+            del foo.__annotations__
+
+    def test_annotations_are_created_correctly(self):
+        from test import ann_module4
+        self.assertTrue("__annotations__" in ann_module4.__dict__)
+        del ann_module4.__annotations__
+        self.assertFalse("__annotations__" in ann_module4.__dict__)
+
+
     # frozen and namespace module reprs are tested in importlib.
 
 
diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py
@@ -31,10 +31,9 @@ def test_setup_annotations_line(self):
         except OSError:
             pass
 
-    def test_no_annotations_if_not_needed(self):
+    def test_default_annotations_exist(self):
         class C: pass
-        with self.assertRaises(AttributeError):
-            C.__annotations__
+        self.assertEqual(C.__annotations__, {})
 
     def test_use_existing_annotations(self):
         ns = {'__annotations__': {1: 2}}
diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py
@@ -0,0 +1,103 @@
+import unittest
+
+class TypeAnnotationTests(unittest.TestCase):
+
+    def test_lazy_create_annotations(self):
+        # type objects lazy create their __annotations__ dict on demand.
+        # the annotations dict is stored in type.__dict__.
+        # a freshly created type shouldn't have an annotations dict yet.
+        foo = type("Foo", (), {})
+        for i in range(3):
+            self.assertFalse("__annotations__" in foo.__dict__)
+            d = foo.__annotations__
+            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertEqual(foo.__annotations__, d)
+            self.assertEqual(foo.__dict__['__annotations__'], d)
+            del foo.__annotations__
+
+    def test_setting_annotations(self):
+        foo = type("Foo", (), {})
+        for i in range(3):
+            self.assertFalse("__annotations__" in foo.__dict__)
+            d = {'a': int}
+            foo.__annotations__ = d
+            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertEqual(foo.__annotations__, d)
+            self.assertEqual(foo.__dict__['__annotations__'], d)
+            del foo.__annotations__
+
+    def test_annotations_getset_raises(self):
+        # builtin types don't have __annotations__ (yet!)
+        with self.assertRaises(AttributeError):
+            print(float.__annotations__)
+        with self.assertRaises(TypeError):
+            float.__annotations__ = {}
+        with self.assertRaises(TypeError):
+            del float.__annotations__
+
+        # double delete
+        foo = type("Foo", (), {})
+        foo.__annotations__ = {}
+        del foo.__annotations__
+        with self.assertRaises(AttributeError):
+            del foo.__annotations__
+
+    def test_annotations_are_created_correctly(self):
+        class C:
+            a:int=3
+            b:str=4
+        self.assertTrue("__annotations__" in C.__dict__)
+        del C.__annotations__
+        self.assertFalse("__annotations__" in C.__dict__)
+
+    def test_descriptor_still_works(self):
+        class C:
+            def __init__(self, name=None, bases=None, d=None):
+                self.my_annotations = None
+
+            @property
+            def __annotations__(self):
+                if not hasattr(self, 'my_annotations'):
+                    self.my_annotations = {}
+                if not isinstance(self.my_annotations, dict):
+                    self.my_annotations = {}
+                return self.my_annotations
+
+            @__annotations__.setter
+            def __annotations__(self, value):
+                if not isinstance(value, dict):
+                    raise ValueError("can only set __annotations__ to a dict")
+                self.my_annotations = value
+
+            @__annotations__.deleter
+            def __annotations__(self):
+                if hasattr(self, 'my_annotations') and self.my_annotations == None:
+                    raise AttributeError('__annotations__')
+                self.my_annotations = None
+
+        c = C()
+        self.assertEqual(c.__annotations__, {})
+        d = {'a':'int'}
+        c.__annotations__ = d
+        self.assertEqual(c.__annotations__, d)
+        with self.assertRaises(ValueError):
+            c.__annotations__ = 123
+        del c.__annotations__
+        with self.assertRaises(AttributeError):
+            del c.__annotations__
+        self.assertEqual(c.__annotations__, {})
+
+
+        class D(metaclass=C):
+            pass
+
+        self.assertEqual(D.__annotations__, {})
+        d = {'a':'int'}
+        D.__annotations__ = d
+        self.assertEqual(D.__annotations__, d)
+        with self.assertRaises(ValueError):
+            D.__annotations__ = 123
+        del D.__annotations__
+        with self.assertRaises(AttributeError):
+            del D.__annotations__
+        self.assertEqual(D.__annotations__, {})
diff --git a/Lib/typing.py b/Lib/typing.py
@@ -1677,6 +1677,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
             else:
                 base_globals = globalns
             ann = base.__dict__.get('__annotations__', {})
+            if isinstance(ann, types.GetSetDescriptorType):
+                ann = {}
             base_locals = dict(vars(base)) if localns is None else localns
             if localns is None and globalns is None:
                 # This is surprising, but required.  Before Python 3.10,
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-25-22-50-47.bpo-43901.oKjG5E.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-22-50-47.bpo-43901.oKjG5E.rst
@@ -0,0 +1,3 @@
+Change class and module objects to lazy-create empty annotations dicts on
+demand.  The annotations dicts are stored in the object's __dict__ for
+backwards compatibility.
diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c
@@ -12,6 +12,9 @@ static Py_ssize_t max_module_number;
 _Py_IDENTIFIER(__doc__);
 _Py_IDENTIFIER(__name__);
 _Py_IDENTIFIER(__spec__);
+_Py_IDENTIFIER(__dict__);
+_Py_IDENTIFIER(__dir__);
+_Py_IDENTIFIER(__annotations__);
 
 static PyMemberDef module_members[] = {
     {"__dict__", T_OBJECT, offsetof(PyModuleObject, md_dict), READONLY},
@@ -807,8 +810,6 @@ module_clear(PyModuleObject *m)
 static PyObject *
 module_dir(PyObject *self, PyObject *args)
 {
-    _Py_IDENTIFIER(__dict__);
-    _Py_IDENTIFIER(__dir__);
     PyObject *result = NULL;
     PyObject *dict = _PyObject_GetAttrId(self, &PyId___dict__);
 
@@ -841,6 +842,71 @@ static PyMethodDef module_methods[] = {
     {0}
 };
 
+static PyObject *
+module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored))
+{
+    PyObject *dict = _PyObject_GetAttrId((PyObject *)m, &PyId___dict__);
+
+    if ((dict == NULL) || !PyDict_Check(dict)) {
+        PyErr_Format(PyExc_TypeError, "<module>.__dict__ is not a dictionary");
+        return NULL;
+    }
+
+    PyObject *annotations;
+    /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */
+    if (_PyDict_ContainsId(dict, &PyId___annotations__)) {
+        annotations = _PyDict_GetItemIdWithError(dict, &PyId___annotations__);
+        /*
+        ** _PyDict_GetItemIdWithError could still fail,
+        ** for instance with a well-timed Ctrl-C or a MemoryError.
+        ** so let's be totally safe.
+        */
+        if (annotations) {
+            Py_INCREF(annotations);
+        }
+    } else {
+        annotations = PyDict_New();
+        if (annotations) {
+            int result = _PyDict_SetItemId(dict, &PyId___annotations__, annotations);
+            if (result) {
+                Py_CLEAR(annotations);
+            }
+        }
+    }
+    Py_DECREF(dict);
+    return annotations;
+}
+
+static int
+module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored))
+{
+    PyObject *dict = _PyObject_GetAttrId((PyObject *)m, &PyId___dict__);
+
+    if ((dict == NULL) || !PyDict_Check(dict)) {
+        PyErr_Format(PyExc_TypeError, "<module>.__dict__ is not a dictionary");
+        return -1;
+    }
+
+    if (value != NULL) {
+        /* set */
+        return _PyDict_SetItemId(dict, &PyId___annotations__, value);
+    }
+
+    /* delete */
+    if (!_PyDict_ContainsId(dict, &PyId___annotations__)) {
+        PyErr_Format(PyExc_AttributeError, "__annotations__");
+        return -1;
+    }
+
+    return _PyDict_DelItemId(dict, &PyId___annotations__);
+}
+
+
+static PyGetSetDef module_getsets[] = {
+    {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations},
+    {NULL}
+};
+
 PyTypeObject PyModule_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "module",                                   /* tp_name */
@@ -872,7 +938,7 @@ PyTypeObject PyModule_Type = {
     0,                                          /* tp_iternext */
     module_methods,                             /* tp_methods */
     module_members,                             /* tp_members */
-    0,                                          /* tp_getset */
+    module_getsets,                             /* tp_getset */
     0,                                          /* tp_base */
     0,                                          /* tp_dict */
     0,                                          /* tp_descr_get */
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
@@ -52,6 +52,7 @@ typedef struct PySlot_Offset {
 
 /* alphabetical order */
 _Py_IDENTIFIER(__abstractmethods__);
+_Py_IDENTIFIER(__annotations__);
 _Py_IDENTIFIER(__class__);
 _Py_IDENTIFIER(__class_getitem__);
 _Py_IDENTIFIER(__classcell__);
@@ -930,6 +931,73 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context)
     return _PyDict_SetItemId(type->tp_dict, &PyId___doc__, value);
 }
 
+static PyObject *
+type_get_annotations(PyTypeObject *type, void *context)
+{
+    if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+        PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotations__'", type->tp_name);
+        return NULL;
+    }
+
+    PyObject *annotations;
+    /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */
+    if (_PyDict_ContainsId(type->tp_dict, &PyId___annotations__)) {
+        annotations = _PyDict_GetItemIdWithError(type->tp_dict, &PyId___annotations__);
+        /*
+        ** _PyDict_GetItemIdWithError could still fail,
+        ** for instance with a well-timed Ctrl-C or a MemoryError.
+        ** so let's be totally safe.
+        */
+        if (annotations) {
+            if (Py_TYPE(annotations)->tp_descr_get) {
+                annotations = Py_TYPE(annotations)->tp_descr_get(annotations, NULL,
+                                                       (PyObject *)type);
+            } else {
+                Py_INCREF(annotations);
+            }
+        }
+    } else {
+        annotations = PyDict_New();
+        if (annotations) {
+            int result = _PyDict_SetItemId(type->tp_dict, &PyId___annotations__, annotations);
+            if (result) {
+                Py_CLEAR(annotations);
+            } else {
+                PyType_Modified(type);
+            }
+        }
+    }
+    return annotations;
+}
+
+static int
+type_set_annotations(PyTypeObject *type, PyObject *value, void *context)
+{
+    if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+        PyErr_Format(PyExc_TypeError, "can't set attributes of built-in/extension type '%s'", type->tp_name);
+        return -1;
+    }
+
+    int result;
+    if (value != NULL) {
+        /* set */
+        result = _PyDict_SetItemId(type->tp_dict, &PyId___annotations__, value);
+    } else {
+        /* delete */
+        if (!_PyDict_ContainsId(type->tp_dict, &PyId___annotations__)) {
+            PyErr_Format(PyExc_AttributeError, "__annotations__");
+            return -1;
+        }
+        result = _PyDict_DelItemId(type->tp_dict, &PyId___annotations__);
+    }
+
+    if (result == 0) {
+        PyType_Modified(type);
+    }
+    return result;
+}
+
+
 /*[clinic input]
 type.__instancecheck__ -> bool
 
@@ -973,6 +1041,7 @@ static PyGetSetDef type_getsets[] = {
     {"__dict__",  (getter)type_dict,  NULL, NULL},
     {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL},
     {"__text_signature__", (getter)type_get_text_signature, NULL, NULL},
+    {"__annotations__", (getter)type_get_annotations, (setter)type_set_annotations, NULL},
     {0}
 };
 

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +# This ann_module isn't for test_typing,
 +# it's for test_module
++
 +a:int=3
 +b:str=4
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+Change class and module objects to lazy-create empty annotations dicts on`
	`2`	`+demand. The annotations dicts are stored in the object's __dict__ for`
	`3`	`+backwards compatibility.`