From 985f8df0801d7cacbe0c7ab274a8e2e4aca6d50d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 20 May 2024 09:59:42 -0400 Subject: [PATCH 1/9] Add __annotate__ descriptors --- Include/cpython/funcobject.h | 1 + .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + .../internal/pycore_runtime_init_generated.h | 2 + .../internal/pycore_unicodeobject_generated.h | 6 ++ Lib/test/test_type_annotations.py | 44 ++++++++ Objects/funcobject.c | 69 +++++++++++- Objects/moduleobject.c | 101 ++++++++++++++++-- Objects/typeobject.c | 93 +++++++++++++++- 9 files changed, 305 insertions(+), 15 deletions(-) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 5433ba48eefc69..598cd330bc9ca9 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -41,6 +41,7 @@ typedef struct { PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ca7355b2b61aa7..158584b504e465 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -590,6 +590,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__all__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__and__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); @@ -989,6 +990,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index fbb25285f0f282..769aa88b54c710 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -79,6 +79,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__all__) STRUCT_FOR_ID(__and__) STRUCT_FOR_ID(__anext__) + STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) STRUCT_FOR_ID(__args__) STRUCT_FOR_ID(__asyncio_running_event_loop__) @@ -478,6 +479,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) + STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 508da40c53422d..065b18efd7922c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -588,6 +588,7 @@ extern "C" { INIT_ID(__all__), \ INIT_ID(__and__), \ INIT_ID(__anext__), \ + INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ INIT_ID(__args__), \ INIT_ID(__asyncio_running_event_loop__), \ @@ -987,6 +988,7 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ + INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index cc2fc15ac5cabf..334dcce03aaf49 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -78,6 +78,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__anext__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__annotate__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__annotations__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1275,6 +1278,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hour); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(id); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ident); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3dbb35afcb620f..55053df5710844 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,4 +1,5 @@ import textwrap +import types import unittest from test.support import run_code @@ -212,3 +213,46 @@ def test_match(self): case 0: x: int = 1 """) + + +class AnnotateTests(unittest.TestCase): + """See PEP 649.""" + def test_manual_annotate_function(self): + def f(): + pass + mod = types.ModuleType("mod") + class X: + pass + + for obj in (f, mod, X): + with self.subTest(obj=obj): + self.check_annotations(obj) + + def check_annotations(self, f): + self.assertEqual(f.__annotations__, {}) + self.assertIs(f.__annotate__, None) + + with self.assertRaises(TypeError): + f.__annotate__ = 42 + f.__annotate__ = lambda: 42 + with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): + print(f.__annotations__) + + f.__annotate__ = lambda x: 42 + with self.assertRaisesRegex(TypeError, r"__annotate__ returned a non-dict"): + print(f.__annotations__) + + f.__annotate__ = lambda x: {"x": x} + self.assertEqual(f.__annotations__, {"x": 1}) + + # Setting annotate to None does not invalidate the cached __annotations__ + f.__annotate__ = None + self.assertEqual(f.__annotations__, {"x": 1}) + + # But setting it to a new callable does + f.__annotate__ = lambda x: {"y": x} + self.assertEqual(f.__annotations__, {"y": 1}) + + # Setting f.__annotations__ also clears __annotate__ + f.__annotations__ = {"z": 43} + self.assertIs(f.__annotate__, None) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 8a30213888ef87..72d74516cc4f53 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -124,6 +125,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = 0; @@ -202,6 +204,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = 0; @@ -512,7 +515,27 @@ static PyObject * func_get_annotation_dict(PyFunctionObject *op) { if (op->func_annotations == NULL) { - return NULL; + if (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate)) { + return NULL; + } + PyObject *one = _PyLong_GetOne(); + PyObject *ann_dict = _PyObject_CallOneArg(op->func_annotate, one); + if (ann_dict == NULL) { + return NULL; + } + if (op->func_annotations != NULL) { + Py_DECREF(ann_dict); + assert(PyDict_Check(op->func_annotations)); + return op->func_annotations; + } + if (!PyDict_Check(ann_dict)) { + PyErr_SetString(PyExc_TypeError, + "__annotate__ returned a non-dict"); + Py_DECREF(ann_dict); + return NULL; + } + Py_XSETREF(op->func_annotations, ann_dict); + return ann_dict; } if (PyTuple_CheckExact(op->func_annotations)) { PyObject *ann_tuple = op->func_annotations; @@ -565,7 +588,9 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); + PyFunctionObject *func = (PyFunctionObject *)op; + Py_XSETREF(func->func_annotations, annotations); + Py_CLEAR(func->func_annotate); return 0; } @@ -763,10 +788,44 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor return 0; } +static PyObject * +func_get_annotate(PyFunctionObject *op, void *Py_UNUSED(ignored)) +{ + if (op->func_annotate == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(op->func_annotate); +} + +static int +func_set_annotate(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "__annotate__ cannot be deleted"); + return -1; + } + if (Py_IsNone(value)) { + Py_XSETREF(op->func_annotate, value); + return 0; + } + else if (PyCallable_Check(value)) { + Py_XSETREF(op->func_annotate, Py_XNewRef(value)); + Py_CLEAR(op->func_annotations); + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "__annotate__ must be set to a callable object or None"); + return -1; + } +} + static PyObject * func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) { - if (op->func_annotations == NULL) { + if (op->func_annotations == NULL && + (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate))) { op->func_annotations = PyDict_New(); if (op->func_annotations == NULL) return NULL; @@ -789,6 +848,7 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno return -1; } Py_XSETREF(op->func_annotations, Py_XNewRef(value)); + Py_CLEAR(op->func_annotate); return 0; } @@ -836,6 +896,7 @@ static PyGetSetDef func_getsetlist[] = { (setter)func_set_kwdefaults}, {"__annotations__", (getter)func_get_annotations, (setter)func_set_annotations}, + {"__annotate__", (getter)func_get_annotate, (setter)func_set_annotate}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, {"__name__", (getter)func_get_name, (setter)func_set_name}, {"__qualname__", (getter)func_get_qualname, (setter)func_set_qualname}, @@ -972,6 +1033,7 @@ func_clear(PyFunctionObject *op) Py_CLEAR(op->func_dict); Py_CLEAR(op->func_closure); Py_CLEAR(op->func_annotations); + Py_CLEAR(op->func_annotate); Py_CLEAR(op->func_typeparams); // Don't Py_CLEAR(op->func_code), since code is always required // to be non-NULL. Similarly, name and qualname shouldn't be NULL. @@ -1028,6 +1090,7 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg) Py_VISIT(f->func_dict); Py_VISIT(f->func_closure); Py_VISIT(f->func_annotations); + Py_VISIT(f->func_annotate); Py_VISIT(f->func_typeparams); Py_VISIT(f->func_qualname); return 0; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 46995b948a28e7..3b378482be72f8 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -5,6 +5,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_interp.h" // PyInterpreterState.importlib +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_AllocNoTrack @@ -1133,7 +1134,7 @@ static PyMethodDef module_methods[] = { }; static PyObject * -module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +module_get_dict(PyModuleObject *m) { PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); if (dict == NULL) { @@ -1144,10 +1145,93 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) Py_DECREF(dict); return NULL; } + return dict; +} + +static PyObject * +module_get_annotate(PyModuleObject *m, void *Py_UNUSED(ignored)) +{ + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } + + PyObject *annotate; + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) == 0) { + annotate = Py_None; + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); + if (result == -1) { + Py_CLEAR(annotate); + } + } + Py_DECREF(dict); + return annotate; +} + +static int +module_set_annotate(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + Py_DECREF(dict); + return -1; + } + + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), value) == -1) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + return -1; + } + } + Py_DECREF(dict); + return 0; +} + +static PyObject * +module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +{ + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } PyObject *annotations; if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) { - annotations = PyDict_New(); + PyObject *annotate; + int annotate_result = PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate); + if (annotate_result < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate_result == 1 && PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ returned a non-dict"); + Py_DECREF(annotations); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -1164,14 +1248,10 @@ static int module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) { int ret = -1; - PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); + PyObject *dict = module_get_dict(m); if (dict == NULL) { return -1; } - if (!PyDict_Check(dict)) { - PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); - goto exit; - } if (value != NULL) { /* set */ @@ -1188,8 +1268,12 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor ret = 0; } } + if (ret == 0) { + if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + ret = -1; + } + } -exit: Py_DECREF(dict); return ret; } @@ -1197,6 +1281,7 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor static PyGetSetDef module_getsets[] = { {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations}, + {"__annotate__", (getter)module_get_annotate, (setter)module_set_annotate}, {NULL} }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b7c3fcf47f23fc..82bd707c8b55cf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7,7 +7,7 @@ #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_lock.h" // _PySeqLock_* -#include "pycore_long.h" // _PyLong_IsNegative() +#include "pycore_long.h" // _PyLong_IsNegative(), _PyLong_GetOne() #include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetDef() @@ -1674,6 +1674,68 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) return PyDict_SetItem(dict, &_Py_ID(__doc__), value); } +static PyObject * +type_get_annotate(PyTypeObject *type, void *Py_UNUSED(ignored)) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotate__'", type->tp_name); + return NULL; + } + + PyObject *annotate; + PyObject *dict = lookup_tp_dict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) { + return NULL; + } + if (annotate) { + descrgetfunc get = Py_TYPE(annotate)->tp_descr_get; + if (get) { + Py_SETREF(annotate, get(annotate, NULL, (PyObject *)type)); + } + } + else { + annotate = Py_None; + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); + if (result == 0) { + PyType_Modified(type); + } + } + return annotate; +} + +static int +type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { + PyErr_Format(PyExc_TypeError, + "cannot set '__annotate__' attribute of immutable type '%s'", + type->tp_name); + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; + } + + PyObject *dict = lookup_tp_dict(type); + assert(PyDict_Check(dict)); + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value); + if (result < 0) { + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + return -1; + } + } + return 0; +} + static PyObject * type_get_annotations(PyTypeObject *type, void *context) { @@ -1694,7 +1756,25 @@ type_get_annotations(PyTypeObject *type, void *context) } } else { - annotations = PyDict_New(); + PyObject *annotate = type_get_annotate(type, NULL); + if (annotate == NULL) { + return NULL; + } + if (PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ returned a non-dict"); + Py_DECREF(annotations); + return NULL; + } + } + else { + annotations = PyDict_New(); + } if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -1731,11 +1811,15 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) return -1; } } + PyType_Modified(type); if (result < 0) { return -1; } - - PyType_Modified(type); + else if (result == 0) { + if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + return -1; + } + } return 0; } @@ -1811,6 +1895,7 @@ static PyGetSetDef type_getsets[] = { {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL}, {"__text_signature__", (getter)type_get_text_signature, NULL, NULL}, {"__annotations__", (getter)type_get_annotations, (setter)type_set_annotations, NULL}, + {"__annotate__", (getter)type_get_annotate, (setter)type_set_annotate, NULL}, {"__type_params__", (getter)type_get_type_params, (setter)type_set_type_params, NULL}, {0} }; From c822ffab59966c02b5acb12c6991b778d03dfade Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 20 May 2024 10:09:30 -0400 Subject: [PATCH 2/9] fix refleaks --- Lib/test/test_type_annotations.py | 4 ++-- Objects/funcobject.c | 2 +- Objects/moduleobject.c | 4 ++++ Objects/typeobject.c | 3 +++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 55053df5710844..ef1569487de18c 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -217,7 +217,7 @@ def test_match(self): class AnnotateTests(unittest.TestCase): """See PEP 649.""" - def test_manual_annotate_function(self): + def test_manual_annotate(self): def f(): pass mod = types.ModuleType("mod") @@ -232,7 +232,7 @@ def check_annotations(self, f): self.assertEqual(f.__annotations__, {}) self.assertIs(f.__annotate__, None) - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "__annotate__ must be callable or None"): f.__annotate__ = 42 f.__annotate__ = lambda: 42 with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 72d74516cc4f53..d957f27a0a6949 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -816,7 +816,7 @@ func_set_annotate(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } else { PyErr_SetString(PyExc_TypeError, - "__annotate__ must be set to a callable object or None"); + "__annotate__ must be callable or None"); return -1; } } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 3b378482be72f8..8ec74a4702b898 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1220,10 +1220,13 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) PyObject *one = _PyLong_GetOne(); annotations = _PyObject_CallOneArg(annotate, one); if (annotations == NULL) { + Py_DECREF(annotate); + Py_DECREF(dict); return NULL; } if (!PyDict_Check(annotations)) { PyErr_SetString(PyExc_TypeError, "__annotate__ returned a non-dict"); + Py_DECREF(annotate); Py_DECREF(annotations); Py_DECREF(dict); return NULL; @@ -1232,6 +1235,7 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) else { annotations = PyDict_New(); } + Py_XDECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 82bd707c8b55cf..14d879236d58d2 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1764,17 +1764,20 @@ type_get_annotations(PyTypeObject *type, void *context) PyObject *one = _PyLong_GetOne(); annotations = _PyObject_CallOneArg(annotate, one); if (annotations == NULL) { + Py_DECREF(annotate); return NULL; } if (!PyDict_Check(annotations)) { PyErr_SetString(PyExc_TypeError, "__annotate__ returned a non-dict"); Py_DECREF(annotations); + Py_DECREF(annotate); return NULL; } } else { annotations = PyDict_New(); } + Py_DECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); From e80095ed46df46cfb37ddb815832513caf6a0014 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 20 May 2024 10:10:57 -0400 Subject: [PATCH 3/9] blurb --- .../2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst new file mode 100644 index 00000000000000..5a88ce097274fb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst @@ -0,0 +1,2 @@ +Add an ``__annotate__`` attribute to functions, classes, and modules as part +of :pep:`649`. Patch by Jelle Zijlstra. From 90ff2c40e562d7e5018189bb011bd42e7919f210 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 20 May 2024 10:25:12 -0400 Subject: [PATCH 4/9] Fix some tests --- Lib/test/test_sys.py | 2 +- Lib/test/test_typing.py | 2 +- Lib/typing.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ee3bd0092f9bf3..8fe1d77756866a 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1564,7 +1564,7 @@ def func(): check(x, size('3Pi2cP7P2ic??2P')) # function def func(): pass - check(func, size('15Pi')) + check(func, size('16Pi')) class c(): @staticmethod def foo(): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 64c4c497eb8934..dac55ceb9e99e0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3723,7 +3723,7 @@ def meth(self): pass acceptable_extra_attrs = { '_is_protocol', '_is_runtime_protocol', '__parameters__', - '__init__', '__annotations__', '__subclasshook__', + '__init__', '__annotations__', '__subclasshook__', '__annotate__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( diff --git a/Lib/typing.py b/Lib/typing.py index 434574559e04fc..be49aa63464f05 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1889,6 +1889,7 @@ class _TypingEllipsis: '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', }) # These special attributes will be not collected as protocol members. From 026c0ff458de7ea60af2569560a9f4dacddc18ae Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 20 May 2024 10:26:17 -0400 Subject: [PATCH 5/9] regen globals --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 3 --- 4 files changed, 6 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 158584b504e465..33133aaaf00893 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -990,7 +990,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 769aa88b54c710..f5ea7b9bd7d433 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -479,7 +479,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) - STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 065b18efd7922c..c73408d6315312 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -988,7 +988,6 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ - INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 334dcce03aaf49..d84c45a6b57887 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1278,9 +1278,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hour); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(id); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ident); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); From 3f5b8c1ad3162f30e97dcae5072680e264a33ae5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 21 May 2024 10:56:41 -0400 Subject: [PATCH 6/9] CR feedback --- Objects/moduleobject.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 8ec74a4702b898..b0d4bc2d5f0592 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1159,8 +1159,7 @@ module_get_annotate(PyModuleObject *m, void *Py_UNUSED(ignored)) PyObject *annotate; if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) == 0) { annotate = Py_None; - int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); - if (result == -1) { + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate) == -1) { Py_CLEAR(annotate); } } @@ -1272,10 +1271,8 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor ret = 0; } } - if (ret == 0) { - if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { - ret = -1; - } + if (ret == 0 && PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + ret = -1; } Py_DECREF(dict); From e2e2bde63299bd1bfe68a7f17165baa8f185db10 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 21 May 2024 10:58:48 -0400 Subject: [PATCH 7/9] better error --- Lib/test/test_type_annotations.py | 2 +- Objects/funcobject.c | 4 ++-- Objects/moduleobject.c | 3 ++- Objects/typeobject.c | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index ef1569487de18c..5e3c3347a41571 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -239,7 +239,7 @@ def check_annotations(self, f): print(f.__annotations__) f.__annotate__ = lambda x: 42 - with self.assertRaisesRegex(TypeError, r"__annotate__ returned a non-dict"): + with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): print(f.__annotations__) f.__annotate__ = lambda x: {"x": x} diff --git a/Objects/funcobject.c b/Objects/funcobject.c index d957f27a0a6949..e78af8d54007e4 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -529,8 +529,8 @@ func_get_annotation_dict(PyFunctionObject *op) return op->func_annotations; } if (!PyDict_Check(ann_dict)) { - PyErr_SetString(PyExc_TypeError, - "__annotate__ returned a non-dict"); + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(ann_dict)->tp_name); Py_DECREF(ann_dict); return NULL; } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b0d4bc2d5f0592..73ad9711b6b0fc 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1224,7 +1224,8 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) return NULL; } if (!PyDict_Check(annotations)) { - PyErr_SetString(PyExc_TypeError, "__annotate__ returned a non-dict"); + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); Py_DECREF(annotate); Py_DECREF(annotations); Py_DECREF(dict); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 14d879236d58d2..6aa00a60323ecb 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1768,7 +1768,8 @@ type_get_annotations(PyTypeObject *type, void *context) return NULL; } if (!PyDict_Check(annotations)) { - PyErr_SetString(PyExc_TypeError, "__annotate__ returned a non-dict"); + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); Py_DECREF(annotations); Py_DECREF(annotate); return NULL; From b0809daba8ccbdaeba4249c2720845da167cd9d0 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 21 May 2024 13:28:09 -0400 Subject: [PATCH 8/9] return None not NULL --- Objects/funcobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index e78af8d54007e4..c85b3470dadadc 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -516,7 +516,7 @@ func_get_annotation_dict(PyFunctionObject *op) { if (op->func_annotations == NULL) { if (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate)) { - return NULL; + Py_RETURN_NONE; } PyObject *one = _PyLong_GetOne(); PyObject *ann_dict = _PyObject_CallOneArg(op->func_annotate, one); From 67cf08f4cc8f53d76d8fdecfdd9ef88d9085c800 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 21 May 2024 14:34:02 -0400 Subject: [PATCH 9/9] Modifications --- Objects/funcobject.c | 5 ----- Objects/typeobject.c | 32 +++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index c85b3470dadadc..4e78252052932c 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -523,11 +523,6 @@ func_get_annotation_dict(PyFunctionObject *op) if (ann_dict == NULL) { return NULL; } - if (op->func_annotations != NULL) { - Py_DECREF(ann_dict); - assert(PyDict_Check(op->func_annotations)); - return op->func_annotations; - } if (!PyDict_Check(ann_dict)) { PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", Py_TYPE(ann_dict)->tp_name); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 6aa00a60323ecb..9f000d8c193bc5 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1683,8 +1683,9 @@ type_get_annotate(PyTypeObject *type, void *Py_UNUSED(ignored)) } PyObject *annotate; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) { + Py_DECREF(dict); return NULL; } if (annotate) { @@ -1696,10 +1697,12 @@ type_get_annotate(PyTypeObject *type, void *Py_UNUSED(ignored)) else { annotate = Py_None; int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); - if (result == 0) { - PyType_Modified(type); + if (result < 0) { + Py_DECREF(dict); + return NULL; } } + Py_DECREF(dict); return annotate; } @@ -1722,17 +1725,22 @@ type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) return -1; } - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); assert(PyDict_Check(dict)); int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value); if (result < 0) { + Py_DECREF(dict); return -1; } if (!Py_IsNone(value)) { if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + PyType_Modified(type); return -1; } } + Py_DECREF(dict); + PyType_Modified(type); return 0; } @@ -1745,8 +1753,9 @@ type_get_annotations(PyTypeObject *type, void *context) } PyObject *annotations; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + Py_DECREF(dict); return NULL; } if (annotations) { @@ -1758,12 +1767,14 @@ type_get_annotations(PyTypeObject *type, void *context) else { PyObject *annotate = type_get_annotate(type, NULL); if (annotate == NULL) { + Py_DECREF(dict); return NULL; } if (PyCallable_Check(annotate)) { PyObject *one = _PyLong_GetOne(); annotations = _PyObject_CallOneArg(annotate, one); if (annotations == NULL) { + Py_DECREF(dict); Py_DECREF(annotate); return NULL; } @@ -1772,6 +1783,7 @@ type_get_annotations(PyTypeObject *type, void *context) Py_TYPE(annotations)->tp_name); Py_DECREF(annotations); Py_DECREF(annotate); + Py_DECREF(dict); return NULL; } } @@ -1789,6 +1801,7 @@ type_get_annotations(PyTypeObject *type, void *context) } } } + Py_DECREF(dict); return annotations; } @@ -1803,7 +1816,7 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) } int result; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (value != NULL) { /* set */ result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); @@ -1812,18 +1825,23 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); if (result == 0) { PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); return -1; } } - PyType_Modified(type); if (result < 0) { + Py_DECREF(dict); return -1; } else if (result == 0) { if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + PyType_Modified(type); + Py_DECREF(dict); return -1; } } + PyType_Modified(type); + Py_DECREF(dict); return 0; }