From 773891f2090b03243c49d4b42f86fb26588aac48 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 08:51:48 +0900 Subject: [PATCH 01/11] introduce Py_id_static_spec type slot --- Include/cpython/object.h | 1 + Include/typeslots.h | 4 ++++ Objects/typeobject.c | 17 +++++++++++++++++ Objects/typeslots.inc | 1 + Objects/typeslots.py | 2 ++ 5 files changed, 25 insertions(+) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 2797051281f3b4..94fd9df54cf7a4 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -271,6 +271,7 @@ typedef struct _heaptypeobject { char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec struct _specialization_cache _spec_cache; // For use by the specializer. /* here are optional user slots, followed by the members. */ + PyType_Spec *ht_static_spec; } PyHeapTypeObject; PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); diff --git a/Include/typeslots.h b/Include/typeslots.h index 506b05580de146..6fdd636f57d1a3 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -86,3 +86,7 @@ /* New in 3.10 */ #define Py_am_send 81 #endif +#if !defined(Py_LIMITED_API) +/* New in 3.13 */ +#define Py_id_static_spec 82 +#endif diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 970c82d2a17ada..2555170897b055 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3587,6 +3587,7 @@ type_new_alloc(type_new_ctx *ctx) et->ht_name = Py_NewRef(ctx->name); et->ht_module = NULL; et->_ht_tpname = NULL; + et->ht_static_spec = NULL; _PyObject_SetDeferredRefcount((PyObject *)et); @@ -4613,11 +4614,23 @@ _PyType_FromMetaclass_impl( } } break; + case Py_id_static_spec: + { + if (slot->pfunc && slot->pfunc != spec) { + PyErr_Format(PyExc_TypeError, + "Py_id_static_spec slot does not match " + "the type spec of '%s'", type->tp_name); + goto finally; + } + res->ht_static_spec = (PyType_Spec *)slot->pfunc; + } + break; default: { /* Copy other slots directly */ PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; short slot_offset = slotoffsets.slot_offset; + assert(slot_offset != Py_id_static_spec); if (slotoffsets.subslot_offset == -1) { /* Set a slot in the main PyTypeObject */ *(void**)((char*)res_start + slot_offset) = slot->pfunc; @@ -4775,6 +4788,9 @@ PyType_GetSlot(PyTypeObject *type, int slot) PyErr_BadInternalCall(); return NULL; } + if (slot == Py_id_static_spec) { + return NULL; + } parent_slot = *(void**)((char*)type + pyslot_offsets[slot].slot_offset); if (parent_slot == NULL) { @@ -5474,6 +5490,7 @@ type_dealloc(PyObject *self) } Py_XDECREF(et->ht_module); PyMem_Free(et->_ht_tpname); + et->ht_static_spec = NULL; Py_TYPE(type)->tp_free((PyObject *)type); } diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index 896daa7d8066b7..89be1a91cb0885 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -80,3 +80,4 @@ {offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, {-1, offsetof(PyTypeObject, tp_finalize)}, {offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, +{-1, Py_id_static_spec}, diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 8ab05f91be12b0..6908bc93894542 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -30,6 +30,8 @@ def generate_typeslots(out=sys.stdout): elif member.startswith("bf_"): member = (f'{{offsetof(PyBufferProcs, {member}),'+ ' offsetof(PyTypeObject, tp_as_buffer)}') + elif member == "id_static_spec": + member = '{-1, Py_id_static_spec}' res[int(m.group(2))] = member M = max(res.keys())+1 From 17b28bb361aeb86bb470cb9c13ac61a36dd32dd0 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 08:52:55 +0900 Subject: [PATCH 02/11] introduce _PyType_GetBaseBySpec() --- Include/internal/pycore_typeobject.h | 1 + Objects/typeobject.c | 41 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 09c4501c38c935..dd2725aad4563a 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -164,6 +164,7 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); +PyAPI_FUNC(PyTypeObject *) _PyType_GetBaseBySpec(PyTypeObject *, PyType_Spec *); // PyType_Ready() must be called if _PyType_IsReady() is false. // See also the Py_TPFLAGS_READY flag. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2555170897b055..55dfe10b9097da 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4883,6 +4883,47 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return res; } +static PyTypeObject * +get_base_by_spec(PyTypeObject *type, PyType_Spec *spec) +{ + PyObject *bases = type->tp_bases; + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (Py_ssize_t i = 0; i < n; i++) { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(bases, i); + if (!_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE)) { + continue; + } + PyType_Spec *base_spec = ((PyHeapTypeObject*)base)->ht_static_spec; + if (base_spec && base_spec == spec) { + return base; + } + base = get_base_by_spec(base, spec); + if (base) { + return base; + } + } + return NULL; +} + +PyTypeObject * +_PyType_GetBaseBySpec(PyTypeObject *type, PyType_Spec *spec) +{ + assert(spec); + assert(PyType_Check(type)); + + BEGIN_TYPE_LOCK() + PyTypeObject *res = get_base_by_spec(type, spec); + END_TYPE_LOCK() + + if (res == NULL) { + PyErr_Format( + PyExc_TypeError, + "PyType_GetBaseBySpec: No superclass of '%s' has the given spec", + type->tp_name); + } + return res; +} + void * PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { From 4d319e45c1a40458fa654a2ffb4710f8008f5f82 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 08:58:28 +0900 Subject: [PATCH 03/11] fix _ctypes mem leak during a finish --- Modules/_ctypes/_ctypes.c | 92 +++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 3cb0b24668eb2a..ee5f755e5c8a2f 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -451,23 +451,44 @@ class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=8389fc5b74a84f2a]*/ + +static PyType_Spec pyctype_type_spec; + + +static inline int +PyStgInfo_FromSpec(PyObject *type, StgInfo **result) +{ + *result = NULL; + PyTypeObject *PyCType_Type = _PyType_GetBaseBySpec(Py_TYPE(type), + &pyctype_type_spec); + if (PyCType_Type == NULL) { + // not a ctypes class. + return -1; + } + StgInfo *info = PyObject_GetTypeData(type, PyCType_Type); + assert(info != NULL); + if (!info->initialized) { + // StgInfo is not initialized. This happens in abstract classes. + return 0; + } + *result = info; + return 1; +} + static int CType_Type_traverse(PyObject *self, visitproc visit, void *arg) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - Py_VISIT(info->proto); - Py_VISIT(info->argtypes); - Py_VISIT(info->converters); - Py_VISIT(info->restype); - Py_VISIT(info->checker); - Py_VISIT(info->module); - } + StgInfo *info; + if (PyStgInfo_FromSpec(self, &info) < 0) { + PyErr_WriteUnraisable(self); + } + if (info) { + Py_VISIT(info->proto); + Py_VISIT(info->argtypes); + Py_VISIT(info->converters); + Py_VISIT(info->restype); + Py_VISIT(info->checker); + Py_VISIT(info->module); } Py_VISIT(Py_TYPE(self)); return PyType_Type.tp_traverse(self, visit, arg); @@ -488,15 +509,12 @@ ctype_clear_stginfo(StgInfo *info) static int CType_Type_clear(PyObject *self) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - ctype_clear_stginfo(info); - } + StgInfo *info; + if (PyStgInfo_FromSpec(self, &info) < 0) { + PyErr_WriteUnraisable(self); + } + if (info) { + ctype_clear_stginfo(info); } return PyType_Type.tp_clear(self); } @@ -504,21 +522,18 @@ CType_Type_clear(PyObject *self) static void CType_Type_dealloc(PyObject *self) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - PyMem_Free(info->ffi_type_pointer.elements); - info->ffi_type_pointer.elements = NULL; - PyMem_Free(info->format); - info->format = NULL; - PyMem_Free(info->shape); - info->shape = NULL; - ctype_clear_stginfo(info); - } + StgInfo *info; + if (PyStgInfo_FromSpec(self, &info) < 0) { + PyErr_WriteUnraisable(self); + } + if (info) { + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + ctype_clear_stginfo(info); } PyTypeObject *tp = Py_TYPE(self); PyType_Type.tp_dealloc(self); @@ -574,6 +589,7 @@ static PyType_Slot ctype_type_slots[] = { {Py_tp_methods, ctype_methods}, // Sequence protocol. {Py_sq_repeat, CType_Type_repeat}, + {Py_id_static_spec, &pyctype_type_spec}, {0, NULL}, }; From de5b42f8e43f75cdf59bffe2818456dba2548086 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 14:27:38 +0900 Subject: [PATCH 04/11] fix redundant base check --- Objects/typeobject.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 55dfe10b9097da..628440b5bf1718 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4893,8 +4893,7 @@ get_base_by_spec(PyTypeObject *type, PyType_Spec *spec) if (!_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE)) { continue; } - PyType_Spec *base_spec = ((PyHeapTypeObject*)base)->ht_static_spec; - if (base_spec && base_spec == spec) { + if (((PyHeapTypeObject*)base)->ht_static_spec == spec) { return base; } base = get_base_by_spec(base, spec); From 3169285c358c9f5259cc3ccf156781745badb935 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 14:51:22 +0900 Subject: [PATCH 05/11] fix a warning --- Objects/typeobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 628440b5bf1718..bdcf22874648a0 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4909,9 +4909,10 @@ _PyType_GetBaseBySpec(PyTypeObject *type, PyType_Spec *spec) { assert(spec); assert(PyType_Check(type)); + PyTypeObject *res; BEGIN_TYPE_LOCK() - PyTypeObject *res = get_base_by_spec(type, spec); + res = get_base_by_spec(type, spec); END_TYPE_LOCK() if (res == NULL) { From 9f7f30964c9927c218c7635f0781750deae046a6 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:28:07 +0900 Subject: [PATCH 06/11] fix failing test --- Include/cpython/object.h | 2 +- Lib/test/test_sys.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 94fd9df54cf7a4..7863b10c0d459f 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -268,10 +268,10 @@ typedef struct _heaptypeobject { PyObject *ht_name, *ht_slots, *ht_qualname; struct _dictkeysobject *ht_cached_keys; PyObject *ht_module; + PyType_Spec *ht_static_spec; char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec struct _specialization_cache _spec_cache; // For use by the specializer. /* here are optional user slots, followed by the members. */ - PyType_Spec *ht_static_spec; } PyHeapTypeObject; PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ab26bf56d9ced9..aaf7c7af95d1ec 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1660,7 +1660,7 @@ def delx(self): del self.__x '3P' # PyMappingMethods '10P' # PySequenceMethods '2P' # PyBufferProcs - '6P' + '7P' '1PIP' # Specializer cache ) class newstyleclass(object): pass From 42a8aa74ef62cb87e839f6ab33d91efeceea59e5 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:17:10 +0900 Subject: [PATCH 07/11] fix typeslots.py --- Objects/typeobject.c | 1 - Objects/typeslots.inc | 2 +- Objects/typeslots.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index bdcf22874648a0..3a455c2b12682e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4630,7 +4630,6 @@ _PyType_FromMetaclass_impl( /* Copy other slots directly */ PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; short slot_offset = slotoffsets.slot_offset; - assert(slot_offset != Py_id_static_spec); if (slotoffsets.subslot_offset == -1) { /* Set a slot in the main PyTypeObject */ *(void**)((char*)res_start + slot_offset) = slot->pfunc; diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index 89be1a91cb0885..303a95d844e34d 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -80,4 +80,4 @@ {offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, {-1, offsetof(PyTypeObject, tp_finalize)}, {offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, -{-1, Py_id_static_spec}, +{-1, offsetof(PyHeapTypeObject, ht_static_spec)}, diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 6908bc93894542..fbad29b4a4707e 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -31,7 +31,7 @@ def generate_typeslots(out=sys.stdout): member = (f'{{offsetof(PyBufferProcs, {member}),'+ ' offsetof(PyTypeObject, tp_as_buffer)}') elif member == "id_static_spec": - member = '{-1, Py_id_static_spec}' + member = '{-1, offsetof(PyHeapTypeObject, ht_static_spec)}' res[int(m.group(2))] = member M = max(res.keys())+1 From 554a8a14b0aaa89e2050bc18c14d9278407b3a7d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 23:18:49 +0900 Subject: [PATCH 08/11] add a comment --- Modules/_ctypes/_ctypes.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ee5f755e5c8a2f..74858aa1f81302 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -589,6 +589,7 @@ static PyType_Slot ctype_type_slots[] = { {Py_tp_methods, ctype_methods}, // Sequence protocol. {Py_sq_repeat, CType_Type_repeat}, + // Memory layout ID. {Py_id_static_spec, &pyctype_type_spec}, {0, NULL}, }; From ea1d78c5e69e0028354e234b71e084982fee8196 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 22 Apr 2024 23:20:00 +0900 Subject: [PATCH 09/11] use MRO if exists --- Objects/typeobject.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 06b4c22e95c84f..1fb530937bc415 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4883,7 +4883,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) } static PyTypeObject * -get_base_by_spec(PyTypeObject *type, PyType_Spec *spec) +get_base_by_spec_recursive(PyTypeObject *type, PyType_Spec *spec) { PyObject *bases = type->tp_bases; Py_ssize_t n = PyTuple_GET_SIZE(bases); @@ -4895,7 +4895,7 @@ get_base_by_spec(PyTypeObject *type, PyType_Spec *spec) if (((PyHeapTypeObject*)base)->ht_static_spec == spec) { return base; } - base = get_base_by_spec(base, spec); + base = get_base_by_spec_recursive(base, spec); if (base) { return base; } @@ -4908,10 +4908,29 @@ _PyType_GetBaseBySpec(PyTypeObject *type, PyType_Spec *spec) { assert(spec); assert(PyType_Check(type)); - PyTypeObject *res; + PyTypeObject *res = NULL; BEGIN_TYPE_LOCK() - res = get_base_by_spec(type, spec); + PyObject *mro = lookup_tp_mro(type); + if (mro == NULL) { + res = get_base_by_spec_recursive(type, spec); + } + else { + // See PyType_GetModuleByDef() implementation + assert(PyTuple_Check(mro)); + assert(PyTuple_GET_SIZE(mro) >= 1); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *super = PyTuple_GET_ITEM(mro, i); + if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + continue; + } + if (((PyHeapTypeObject*)super)->ht_static_spec == spec) { + res = _PyType_CAST(super); + break; + } + } + } END_TYPE_LOCK() if (res == NULL) { From d47b5841ff71ea5c242c46fcde600dd9d22ff84f Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:45:33 +0900 Subject: [PATCH 10/11] walk MRO from top to down --- Objects/typeobject.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1fb530937bc415..d9d8babe17460e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4916,11 +4916,8 @@ _PyType_GetBaseBySpec(PyTypeObject *type, PyType_Spec *spec) res = get_base_by_spec_recursive(type, spec); } else { - // See PyType_GetModuleByDef() implementation assert(PyTuple_Check(mro)); - assert(PyTuple_GET_SIZE(mro) >= 1); - Py_ssize_t n = PyTuple_GET_SIZE(mro); - for (Py_ssize_t i = 0; i < n; i++) { + for (Py_ssize_t i = PyTuple_GET_SIZE(mro) - 1; i >= 0; i--) { PyObject *super = PyTuple_GET_ITEM(mro, i); if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { continue; From 9b6bb44b024e8e26a7983fe58a5954b5df12fd35 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 26 Apr 2024 07:46:52 +0900 Subject: [PATCH 11/11] remove get_module_state_by_def_final() --- Modules/_ctypes/ctypes.h | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 20c68134be2804..feb51b8602d762 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -101,20 +101,6 @@ get_module_state_by_def(PyTypeObject *cls) return get_module_state(mod); } -static inline ctypes_state * -get_module_state_by_def_final(PyTypeObject *cls) -{ - if (cls->tp_mro == NULL) { - return NULL; - } - PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); - if (mod == NULL) { - PyErr_Clear(); - return NULL; - } - return get_module_state(mod); -} - extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec;