diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 32bd19d968b917..bef6a4e2bed9ce 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -226,6 +226,7 @@ extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); extern PyTypeObject _PyBufferWrapper_Type; +extern PyTypeObject _PyAnnotationsDescriptor_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); diff --git a/Lib/doctest.py b/Lib/doctest.py index ea7d275c91db04..3c209c3e6c5895 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1067,6 +1067,7 @@ def _find(self, tests, obj, name, module, source_lines, globs, seen): # Recurse to methods, properties, and nested classes. if ((inspect.isroutine(val) or inspect.isclass(val) or isinstance(val, property)) and + valname != '__annotations__' and self._from_module(module, val)): valname = '%s.%s' % (name, valname) self._find(tests, val, valname, module, source_lines, diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 768c3dcb11ec59..0f5fe090f33331 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -328,8 +328,10 @@ def visiblename(name, all=None, obj=None): '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', '__path__', '__qualname__', '__slots__', '__version__', - '__static_attributes__', '__firstlineno__'}: + '__static_attributes__', '__firstlineno__', '__annotations__'}: return 0 + if name == '__annotate__' and getattr(obj, name, None) is None: + return False # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index b0f86317bfecf6..2dc559a5656ac4 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5123,7 +5123,8 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + self.assertEqual(keys, ['__annotate__', '__annotations__', + '__dict__', '__doc__', '__firstlineno__', '__module__', '__static_attributes__', '__weakref__', 'meth']) @@ -5135,7 +5136,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 7) + self.assertEqual(len(values), 9) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5145,7 +5146,8 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + self.assertEqual(keys, ['__annotate__', '__annotations__', + '__dict__', '__doc__', '__firstlineno__', '__module__', '__static_attributes__', '__weakref__', 'meth']) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index a9be1f5aa84681..04c78231229bc9 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,3 +1,4 @@ +import itertools import textwrap import types import unittest @@ -11,12 +12,8 @@ class TypeAnnotationTests(unittest.TestCase): def test_lazy_create_annotations(self): - # type objects lazy create their __annotations__ dict on demand. - # the annotations dict is stored in type.__dict__. - # a freshly created type shouldn't have an annotations dict yet. foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations__" in foo.__dict__) d = foo.__annotations__ self.assertTrue("__annotations__" in foo.__dict__) self.assertEqual(foo.__annotations__, d) @@ -26,7 +23,6 @@ def test_lazy_create_annotations(self): def test_setting_annotations(self): foo = type("Foo", (), {}) for i in range(3): - self.assertFalse("__annotations__" in foo.__dict__) d = {'a': int} foo.__annotations__ = d self.assertTrue("__annotations__" in foo.__dict__) @@ -270,6 +266,68 @@ def check_annotations(self, f): self.assertIs(f.__annotate__, None) +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(Meta.__annotations__, {"a": int}) + self.assertEqual(Meta.__annotate__(1), {"a": int}) + + self.assertEqual(X.__annotations__, {}) + self.assertIs(X.__annotate__, None) + + self.assertEqual(Y.__annotations__, {"b": float}) + self.assertEqual(Y.__annotate__(1), {"b": float}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + classes[i].__annotations__ + for c in classes: + with self.subTest(c=c): + self.assertEqual(c.__annotations__, c.expected_annotations) + if c.expected_annotations: + self.assertEqual(c.__annotate__(1), c.expected_annotations) + else: + self.assertIs(c.__annotate__, None) + + class DeferredEvaluationTests(unittest.TestCase): def test_function(self): def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-18-19-33-37.gh-issue-119180.JgGJVv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-19-33-37.gh-issue-119180.JgGJVv.rst new file mode 100644 index 00000000000000..e52aeb108dc629 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-18-19-33-37.gh-issue-119180.JgGJVv.rst @@ -0,0 +1,2 @@ +Make lookup of ``__annotate__`` and ``__annotations__`` on classes more +robust in the presence of metaclasses. diff --git a/Objects/object.c b/Objects/object.c index dc6402f0a99cd0..854f9bdfc5e99a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2269,6 +2269,7 @@ static PyTypeObject* static_types[] = { &PyZip_Type, &Py_GenericAliasType, &_PyAnextAwaitable_Type, + &_PyAnnotationsDescriptor_Type, &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1cc6ca79298108..d1233d7ec28baa 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -115,6 +115,9 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound); static int slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value); +static PyObject * +annodescr_new(PyObject *owner); + static inline PyTypeObject * type_from_ref(PyObject *ref) @@ -1888,6 +1891,34 @@ type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) return 0; } +static PyObject * +type_materialize_annotations(PyTypeObject *type) +{ + PyObject *annotate = type_get_annotate(type, NULL); + if (annotate == NULL) { + return NULL; + } + if (PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + PyObject *annotations = _PyObject_CallOneArg(annotate, one); + Py_DECREF(annotate); + if (annotations == NULL) { + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotations); + return NULL; + } + return annotations; + } + else { + Py_DECREF(annotate); + return PyDict_New(); + } +} + static PyObject * type_get_annotations(PyTypeObject *type, void *context) { @@ -1909,40 +1940,17 @@ type_get_annotations(PyTypeObject *type, void *context) } } else { - PyObject *annotate = type_get_annotate(type, NULL); - if (annotate == NULL) { + annotations = type_materialize_annotations(type); + if (annotations == NULL) { Py_DECREF(dict); return NULL; } - if (PyCallable_Check(annotate)) { - PyObject *one = _PyLong_GetOne(); - annotations = _PyObject_CallOneArg(annotate, one); - if (annotations == NULL) { - Py_DECREF(dict); - Py_DECREF(annotate); - return NULL; - } - if (!PyDict_Check(annotations)) { - PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", - Py_TYPE(annotations)->tp_name); - Py_DECREF(annotations); - Py_DECREF(annotate); - Py_DECREF(dict); - return NULL; - } - } - else { - annotations = PyDict_New(); - } - Py_DECREF(annotate); - if (annotations) { - int result = PyDict_SetItem( - dict, &_Py_ID(__annotations__), annotations); - if (result) { - Py_CLEAR(annotations); - } else { - PyType_Modified(type); - } + int result = PyDict_SetItem( + dict, &_Py_ID(__annotations__), annotations); + if (result) { + Py_CLEAR(annotations); + } else { + PyType_Modified(type); } } Py_DECREF(dict); @@ -4198,6 +4206,39 @@ type_new_set_classcell(PyTypeObject *type) return 0; } +static int +type_new_set_annotate(PyTypeObject *type) +{ + PyObject *dict = lookup_tp_dict(type); + // If __annotate__ is not set (i.e., the class has no annotations), + // set it to None + int result = PyDict_Contains(dict, &_Py_ID(__annotate__)); + if (result < 0) { + return -1; + } + else if (result == 0) { + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), Py_None) < 0) { + return -1; + } + } + result = PyDict_Contains(dict, &_Py_ID(__annotations__)); + if (result < 0) { + return -1; + } + else if (result == 0) { + PyObject *descr = annodescr_new((PyObject *)type); + if (descr == NULL) { + return -1; + } + if (PyDict_SetItem(dict, &_Py_ID(__annotations__), descr) < 0) { + Py_DECREF(descr); + return -1; + } + Py_DECREF(descr); + } + return 0; +} + static int type_new_set_classdictcell(PyTypeObject *type) { @@ -4271,6 +4312,9 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) if (type_new_set_classdictcell(type) < 0) { return -1; } + if (type_new_set_annotate(type) < 0) { + return -1; + } return 0; } @@ -11641,3 +11685,255 @@ PyTypeObject PySuper_Type = { PyObject_GC_Del, /* tp_free */ .tp_vectorcall = (vectorcallfunc)super_vectorcall, }; + +/* Annotations descriptor for heap types */ + +typedef struct { + PyObject_HEAD + PyObject *owner; + PyObject *annotations; +} annodescrobject; + +static PyObject * +annodescr_new(PyObject *owner) +{ + annodescrobject *ad = PyObject_GC_New(annodescrobject, &_PyAnnotationsDescriptor_Type); + if (ad == NULL) { + return NULL; + } + ad->owner = Py_NewRef(owner); + ad->annotations = NULL; + _PyObject_GC_TRACK(ad); + return (PyObject *)ad; +} + +static int +annodescr_materialize(annodescrobject *ad) +{ + if (ad->annotations == NULL) { + assert(PyType_Check(ad->owner)); + PyObject *dict = type_materialize_annotations((PyTypeObject *)ad->owner); + if (dict == NULL) { + return -1; + } + ad->annotations = dict; + } + return 0; +} + +static void +annodescr_dealloc(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + PyTypeObject *tp = Py_TYPE(self); + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(ad->owner); + Py_XDECREF(ad->annotations); + tp->tp_free(self); + Py_DECREF(tp); +} + +static int +annodescr_traverse(PyObject *self, visitproc visit, void *arg) +{ + annodescrobject *ad = (annodescrobject *)self; + + Py_VISIT(ad->owner); + Py_VISIT(ad->annotations); + + return 0; +} + +static int +annodescr_clear(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + + Py_CLEAR(ad->annotations); + Py_CLEAR(ad->owner); + return 0; +} + +static PyObject * +annodescr_repr(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + return PyUnicode_FromFormat("", ad->owner); +} + +PyDoc_STRVAR(annodescr_doc, +"Wrapper for a class's annotations dictionary."); + +static PyObject * +annodescr_iter(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyObject_GetIter(ad->annotations); +} + +static Py_ssize_t +annodescr_length(PyObject *self) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return -1; + } + return PyObject_Size(ad->annotations); +} + +static PyObject * +annodescr_getitem(PyObject *self, PyObject *key) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyObject_GetItem(ad->annotations, key); +} + +static int +annodescr_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return -1; + } + return PyObject_SetItem(ad->annotations, key, value); +} + +static PyMappingMethods annodescr_as_mapping = { + .mp_length = annodescr_length, + .mp_subscript = annodescr_getitem, + .mp_ass_subscript = annodescr_setitem, +}; + +static int +annodescr_contains(PyObject *self, PyObject *key) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return -1; + } + return PySequence_Contains(ad->annotations, key); +} + +static PySequenceMethods annodescr_as_sequence = { + .sq_contains = annodescr_contains, +}; + +static PyObject * +annodescr_or(PyObject *self, PyObject *other) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyNumber_Or(ad->annotations, other); +} + +static PyObject * +annodescr_inplace_or(PyObject *self, PyObject *other) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyNumber_InPlaceOr(ad->annotations, other); +} + +static PyNumberMethods annodescr_as_number = { + .nb_or = annodescr_or, + .nb_inplace_or = annodescr_inplace_or, +}; + +static PyObject * +annodescr_richcompare(PyObject *self, PyObject *other, int op) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return PyObject_RichCompare(ad->annotations, other, op); +} + +#define MAKE_DICT_WRAPPER(name) \ + static PyObject *annodescr_wrap_ ## name(PyObject *self, \ + PyObject *args, \ + PyObject *kwargs) \ + { \ + annodescrobject *ad = (annodescrobject *)self; \ + if (annodescr_materialize(ad) < 0) { \ + return NULL; \ + } \ + PyObject *method = PyObject_GetAttrString(ad->annotations, #name); \ + if (method == NULL) { \ + return NULL; \ + } \ + return PyObject_Call(method, args, kwargs); \ + } + +MAKE_DICT_WRAPPER(get) +MAKE_DICT_WRAPPER(setdefault) +MAKE_DICT_WRAPPER(pop) +MAKE_DICT_WRAPPER(popitem) +MAKE_DICT_WRAPPER(keys) +MAKE_DICT_WRAPPER(values) +MAKE_DICT_WRAPPER(items) +MAKE_DICT_WRAPPER(update) +MAKE_DICT_WRAPPER(clear) +MAKE_DICT_WRAPPER(copy) + +#undef MAKE_DICT_WRAPPER + +static PyMethodDef annodescr_methods[] = { + {"get", (PyCFunction)annodescr_wrap_get, METH_VARARGS | METH_KEYWORDS, NULL}, + {"setdefault", (PyCFunction)annodescr_wrap_setdefault, METH_VARARGS | METH_KEYWORDS, NULL}, + {"pop", (PyCFunction)annodescr_wrap_pop, METH_VARARGS | METH_KEYWORDS, NULL}, + {"popitem", (PyCFunction)annodescr_wrap_popitem, METH_VARARGS | METH_KEYWORDS, NULL}, + {"keys", (PyCFunction)annodescr_wrap_keys, METH_VARARGS | METH_KEYWORDS, NULL}, + {"values", (PyCFunction)annodescr_wrap_values, METH_VARARGS | METH_KEYWORDS, NULL}, + {"items", (PyCFunction)annodescr_wrap_items, METH_VARARGS | METH_KEYWORDS, NULL}, + {"update", (PyCFunction)annodescr_wrap_update, METH_VARARGS | METH_KEYWORDS, NULL}, + {"clear", (PyCFunction)annodescr_wrap_clear, METH_VARARGS | METH_KEYWORDS, NULL}, + {"copy", (PyCFunction)annodescr_wrap_copy, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL}, +}; + +static PyObject * +annodescr_descr_get(PyObject *self, PyObject *obj, PyObject *type) +{ + annodescrobject *ad = (annodescrobject *)self; + if (annodescr_materialize(ad) < 0) { + return NULL; + } + return Py_NewRef(ad->annotations); +} + +PyTypeObject _PyAnnotationsDescriptor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "AnnotationsDescriptor", + .tp_basicsize = sizeof(annodescrobject), + .tp_itemsize = 0, + .tp_dealloc = annodescr_dealloc, + .tp_traverse = annodescr_traverse, + .tp_clear = annodescr_clear, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, + .tp_alloc = PyType_GenericAlloc, + .tp_new = PyType_GenericNew, + .tp_free = PyObject_GC_Del, + .tp_doc = annodescr_doc, + .tp_repr = annodescr_repr, + .tp_iter = annodescr_iter, + .tp_as_mapping = &annodescr_as_mapping, + .tp_as_sequence = &annodescr_as_sequence, + .tp_as_number = &annodescr_as_number, + .tp_hash = PyObject_HashNotImplemented, + .tp_getattro = PyObject_GenericGetAttr, + .tp_richcompare = annodescr_richcompare, + .tp_methods = annodescr_methods, + .tp_descr_get = annodescr_descr_get, +};