From aee85c4449454f4b1f8706452f6900f9452ef229 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Oct 2022 15:17:53 +0200 Subject: [PATCH 01/21] Headers for new API --- Doc/data/stable_abi.dat | 2 ++ Include/cpython/object.h | 1 + Include/descrobject.h | 1 + Include/object.h | 2 ++ Include/typeslots.h | 4 ++++ Lib/test/test_stable_abi_ctypes.py | 2 ++ Misc/stable_abi.toml | 9 +++++++++ PC/python3dll.c | 2 ++ 8 files changed, 23 insertions(+) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 53895bbced8408..3639cc97a119ee 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -516,6 +516,8 @@ function,PyObject_GetAttrString,3.2,, function,PyObject_GetBuffer,3.11,, function,PyObject_GetItem,3.2,, function,PyObject_GetIter,3.2,, +function,PyObject_GetTypeData,3.12,, +function,PyObject_GetTypeDataSize,3.12,, function,PyObject_HasAttr,3.2,, function,PyObject_HasAttrString,3.2,, function,PyObject_Hash,3.2,, diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3abfcb7d44f0fb..4dbad129e027b9 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -506,6 +506,7 @@ Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; Py_TRASHCAN_END; \ } while(0); +PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); diff --git a/Include/descrobject.h b/Include/descrobject.h index 0a420b865dfd1b..fd66d17b497a31 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -83,6 +83,7 @@ struct PyMemberDef { #define Py_READONLY 1 #define Py_AUDIT_READ 2 // Added in 3.10, harmless no-op before that #define _Py_WRITE_RESTRICTED 4 // Deprecated, no-op. Do not reuse the value. +#define Py_RELATIVE_OFFSET 8 PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *); PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *); diff --git a/Include/object.h b/Include/object.h index 75624fe8c77a51..6fced3ee3e4b51 100644 --- a/Include/object.h +++ b/Include/object.h @@ -264,6 +264,8 @@ PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); +PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); +PyAPI_FUNC(Py_ssize_t) PyObject_GetTypeDataSize(PyTypeObject *cls); #endif /* Generic type check */ diff --git a/Include/typeslots.h b/Include/typeslots.h index 506b05580de146..2ea2a61f0164a8 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -86,3 +86,7 @@ /* New in 3.10 */ #define Py_am_send 81 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 +/* New in 3.12 */ +#define Py_tp_inherit_itemsize 82 +#endif diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 67c653428a6dee..0049ab3220c0b1 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -524,6 +524,8 @@ def test_windows_feature_macros(self): "PyObject_GetBuffer", "PyObject_GetItem", "PyObject_GetIter", + "PyObject_GetTypeData", + "PyObject_GetTypeDataSize", "PyObject_HasAttr", "PyObject_HasAttrString", "PyObject_Hash", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index aa12bcc85cebc7..7d5b37609f3b17 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2342,3 +2342,12 @@ added = '3.12' # Before 3.12, available in "structmember.h" w/o Py_ prefix [const.Py_AUDIT_READ] added = '3.12' # Before 3.12, available in "structmember.h" + +[function.PyObject_GetTypeData] + added = '3.12' +[function.PyObject_GetTypeDataSize] + added = '3.12' +[const.Py_tp_inherit_itemsize] + added = '3.12' +[const.Py_RELATIVE_OFFSET] + added = '3.12' diff --git a/PC/python3dll.c b/PC/python3dll.c index 931f316bb99843..b1c628e1f33ab8 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -462,6 +462,8 @@ EXPORT_FUNC(PyObject_GetAttrString) EXPORT_FUNC(PyObject_GetBuffer) EXPORT_FUNC(PyObject_GetItem) EXPORT_FUNC(PyObject_GetIter) +EXPORT_FUNC(PyObject_GetTypeData) +EXPORT_FUNC(PyObject_GetTypeDataSize) EXPORT_FUNC(PyObject_HasAttr) EXPORT_FUNC(PyObject_HasAttrString) EXPORT_FUNC(PyObject_Hash) From f091810ba090cc9a028efe6e1aa7e0c0b191f0cf Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Oct 2022 15:29:19 +0200 Subject: [PATCH 02/21] Implement helper functions --- Objects/typeobject.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ae80f5a8fd88e0..d1e6b4500d1f2d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -17,6 +17,8 @@ #include "structmember.h" // PyMemberDef #include +#include // alignof +#include // ptrdiff_t /*[clinic input] class type "PyTypeObject *" "&PyType_Type" @@ -3522,6 +3524,13 @@ static const PySlot_Offset pyslot_offsets[] = { #include "typeslots.inc" }; +/* Align up to the nearest multiple of alignof(max_align_t) */ +static Py_ssize_t +_align_up(Py_ssize_t size) { + const Py_ssize_t alignment = alignof(max_align_t); + return (size + alignment - 1) & ~(alignment - 1); +} + /* Given a PyType_FromMetaclass `bases` argument (NULL, type, or tuple of * types), return a tuple of types. */ @@ -4089,6 +4098,24 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return NULL; } +void * +PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { + return (char *)obj + _align_up(cls->tp_base->tp_basicsize); +} + +Py_ssize_t +PyObject_GetTypeDataSize(PyTypeObject *cls) { + ptrdiff_t result = cls->tp_basicsize - _align_up(cls->tp_base->tp_basicsize); + if (result < 0) { + return 0; + } + return result; +} + +void * +PyObject_GetItemData(PyObject *obj) { + return (char *)obj + Py_TYPE(obj)->tp_basicsize; +} /* Internal API to look for a name through the MRO, bypassing the method cache. This returns a borrowed reference, and might set an exception. From 0066113978d06ad15c0c4a1cd317e577542e087a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Oct 2022 15:58:11 +0200 Subject: [PATCH 03/21] Implement the relative sizes/offsets --- Objects/typeobject.c | 59 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index d1e6b4500d1f2d..bd395e6efb2b28 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3524,7 +3524,9 @@ static const PySlot_Offset pyslot_offsets[] = { #include "typeslots.inc" }; -/* Align up to the nearest multiple of alignof(max_align_t) */ +/* Align up to the nearest multiple of alignof(max_align_t) + * (like _Py_ALIGN_UP, but for a size rather than pointer) + */ static Py_ssize_t _align_up(Py_ssize_t size) { const Py_ssize_t alignment = alignof(max_align_t); @@ -3634,6 +3636,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, Py_ssize_t nmembers = 0; Py_ssize_t weaklistoffset, dictoffset, vectorcalloffset; char *res_start; + int inherit_itemsize = 0; nmembers = weaklistoffset = dictoffset = vectorcalloffset = 0; for (slot = spec->slots; slot->slot; slot++) { @@ -3670,6 +3673,12 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, assert(memb->flags == READONLY); vectorcalloffset = memb->offset; } + if (memb->flags & Py_RELATIVE_OFFSET) { + assert(spec->basicsize <= 0); + } + else { + assert(spec->basicsize > 0); + } } break; case Py_tp_doc: @@ -3695,6 +3704,21 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, memcpy(tp_doc, slot->pfunc, len); } break; + case Py_tp_inherit_itemsize: + inherit_itemsize = 1; + if (spec->itemsize != 0) { + PyErr_SetString( + PyExc_SystemError, + "With Py_tp_inherit_itemsize, itemsize must be 0."); + goto finally; + } + if (slot->pfunc != NULL) { + PyErr_SetString( + PyExc_SystemError, + "pfunc for Py_tp_inherit_itemsize must be NULL."); + goto finally; + } + break; } } @@ -3799,6 +3823,25 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, // here we just check its work assert(_PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)); + /* Calculate sizes */ + + Py_ssize_t basicsize = spec->basicsize; + Py_ssize_t type_data_offset = spec->basicsize; + if (basicsize == 0) { + /* Inherit */ + basicsize = base->tp_basicsize; + } + else if (basicsize < 0) { + /* Extend */ + type_data_offset = _align_up(base->tp_basicsize); + basicsize = type_data_offset + _align_up(-spec->basicsize); + } + + Py_ssize_t itemsize = spec->itemsize; + if (inherit_itemsize) { + itemsize = spec->itemsize; + } + /* Allocate the new type * * Between here and PyType_Ready, we should limit: @@ -3846,8 +3889,8 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, /* Copy the sizes */ - type->tp_basicsize = spec->basicsize; - type->tp_itemsize = spec->itemsize; + type->tp_basicsize = basicsize; + type->tp_itemsize = itemsize; /* Copy all the ordinary slots */ @@ -3856,6 +3899,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, case Py_tp_base: case Py_tp_bases: case Py_tp_doc: + case Py_tp_inherit_itemsize: /* Processed above */ break; case Py_tp_members: @@ -3864,6 +3908,15 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, size_t len = Py_TYPE(type)->tp_itemsize * nmembers; memcpy(_PyHeapType_GET_MEMBERS(res), slot->pfunc, len); type->tp_members = _PyHeapType_GET_MEMBERS(res); + PyMemberDef *memb; + unsigned i; + for (memb = _PyHeapType_GET_MEMBERS(res), i = nmembers; + i > 0; ++memb, --i) { + if (memb->flags & Py_RELATIVE_OFFSET) { + memb->flags &= ~Py_RELATIVE_OFFSET; + memb->offset += type_data_offset; + } + } } break; default: From 5d7435541dfd22b71c04c3115ce83b9cd7ae29cd Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Oct 2022 16:03:37 +0200 Subject: [PATCH 04/21] Error when Py_RELATIVE_OFFSET is used --- Objects/descrobject.c | 6 ++++++ Python/structmember.c | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index c545b90c6283e1..e2f5c163cd5788 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -978,6 +978,12 @@ PyDescr_NewMember(PyTypeObject *type, PyMemberDef *member) { PyMemberDescrObject *descr; + if (member->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyDescr_NewMember used with Py_RELATIVE_OFFSET"); + return NULL; + } descr = (PyMemberDescrObject *)descr_new(&PyMemberDescr_Type, type, member->name); if (descr != NULL) diff --git a/Python/structmember.c b/Python/structmember.c index 1b8be28dcf2eb2..7f987db4fd2983 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -8,6 +8,12 @@ PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { PyObject *v; + if (l->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyMember_GetOne used with Py_RELATIVE_OFFSET"); + return NULL; + } const char* addr = obj_addr + l->offset; switch (l->type) { @@ -103,6 +109,12 @@ int PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) { PyObject *oldv; + if (l->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyMember_GetOne used with Py_RELATIVE_OFFSET"); + return NULL; + } addr += l->offset; From b8cd8e1d42a37e643848c24e39e8a52e1095aeb7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Oct 2022 17:38:00 +0200 Subject: [PATCH 05/21] Add test infrastructure --- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/heaptype_relative.c | 23 +++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 +++ PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 +++ 6 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 Modules/_testcapi/heaptype_relative.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d64752e8ca9609..2dc81f968e10b3 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/heaptype_relative.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c new file mode 100644 index 00000000000000..b916614545518b --- /dev/null +++ b/Modules/_testcapi/heaptype_relative.c @@ -0,0 +1,23 @@ +#define Py_LIMITED_API 0x030c0000 // 3.12 +#include "parts.h" + +#ifdef LIMITED_API_AVAILABLE + +static PyMethodDef TestMethods[] = { + /* Add module methods here. + * (Empty list left here as template/example, since using + * PyModule_AddFunctions isn't very common.) + */ + {NULL}, +}; + +int +_PyTestCapi_Init_HeaptypeRelative(PyObject *m) { + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + return 0; +} + +#endif // LIMITED_API_AVAILABLE diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 7ba3c4ebff8cde..6fdd47ad8a5b15 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -39,6 +39,7 @@ int _PyTestCapi_Init_Structmember(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); +int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); #endif // LIMITED_API_AVAILABLE #endif diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3617fafe9b4fdd..4303e347d7b0fc 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3975,6 +3975,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_VectorcallLimited(m) < 0) { return NULL; } + if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { + return NULL; + } #endif PyState_AddModule(m, &_testcapimodule); diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 58bf4e1eacbf21..e0bfdd1f8d1789 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -98,6 +98,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 101c5322761634..197d56334c6757 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -24,6 +24,9 @@ Source Files + + Source Files + Source Files From 548c8779bd00ef95e49f1ece0208847d052239a3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 12 Oct 2022 17:28:12 +0200 Subject: [PATCH 06/21] Add tests for non-positive basicsize --- Lib/test/test_capi/test_misc.py | 52 +++++++++++++++++++++ Modules/_testcapi/heaptype_relative.c | 66 +++++++++++++++++++++++++-- 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 1d30adaee9218f..fe5b0f623bcd61 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -17,11 +17,13 @@ import unittest import warnings import weakref +import operator from test import support from test.support import MISSING_C_DOCSTRINGS from test.support import import_helper from test.support import threading_helper from test.support import warnings_helper +from test.support import requires_limited_api from test.support.script_helper import assert_python_failure, assert_python_ok try: import _posixsubprocess @@ -842,6 +844,56 @@ def meth(self): MutableBase.meth = lambda self: 'changed' self.assertEqual(instance.meth(), 'changed') + @requires_limited_api + def test_heaptype_relative_sizes(self): + # Test subclassing using "relative" basicsize, see PEP 697 + def check(extra_base_size, extra_size): + Base, Sub, instance, data_ptr, data_size = ( + _testcapi.make_sized_heaptypes( + extra_base_size, -extra_size)) + + # no alignment shenanigans when inheriting directly + if extra_size == 0: + self.assertEqual(Base.__basicsize__, Sub.__basicsize__) + self.assertEqual(data_size, 0) + + else: + # The following offsets should be in increasing order: + data_offset = data_ptr - id(instance) + offsets = [ + (0, 'start of object'), + (Base.__basicsize__, 'end of base data'), + (data_offset, 'subclass data'), + (data_offset + extra_size, 'end of requested subcls data'), + (data_offset + data_size, 'end of reserved subcls data'), + (Sub.__basicsize__, 'end of object'), + ] + ordered_offsets = sorted(offsets, key=operator.itemgetter(0)) + self.assertEqual( + offsets, ordered_offsets, + msg=f'Offsets not in expected order, got: {ordered_offsets}') + + # end of reserved subcls data == end of object + self.assertEqual(Sub.__basicsize__, data_offset + data_size) + + # we don't reserve (requested + alignment) or more data + self.assertLess(data_size - extra_size, + _testcapi.alignof_max_align_t) + + # Everything should be aligned + self.assertEqual(data_ptr % _testcapi.alignof_max_align_t, 0) + self.assertEqual(data_size % _testcapi.alignof_max_align_t, 0) + + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + args = dict(extra_base_size=extra_base_size, + extra_size=extra_size) + with self.subTest(**args): + check(**args) def test_pynumber_tobase(self): from _testcapi import pynumber_tobase diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index b916614545518b..b4778ebc2f4660 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -1,13 +1,69 @@ #define Py_LIMITED_API 0x030c0000 // 3.12 #include "parts.h" +#include // alignof +#include // max_align_t #ifdef LIMITED_API_AVAILABLE +static PyType_Slot empty_slots[] = { + {0, NULL}, +}; + +static PyObject * +make_sized_heaptypes(PyObject *module, PyObject *args) +{ + PyObject *base = NULL; + PyObject *sub = NULL; + PyObject *instance = NULL; + PyObject *result = NULL; + + int extra_base_size, basicsize; + + int r = PyArg_ParseTuple(args, "ii", &extra_base_size, &basicsize); + if (!r) { + goto finally; + } + + PyType_Spec base_spec = { + .name = "_testcapi.Base", + .basicsize = sizeof(PyObject) + extra_base_size, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = empty_slots, + }; + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = empty_slots, + }; + + base = PyType_FromMetaclass(NULL, module, &base_spec, NULL); + if (!base) { + goto finally; + } + sub = PyType_FromMetaclass(NULL, module, &sub_spec, base); + if (!sub) { + goto finally; + } + instance = PyObject_CallNoArgs(sub); + if (!instance) { + goto finally; + } + void *data_ptr = PyObject_GetTypeData(instance, (PyTypeObject *)sub); + Py_ssize_t data_size = PyObject_GetTypeDataSize((PyTypeObject *)sub); + + result = Py_BuildValue("OOOln", base, sub, instance, (long)data_ptr, + data_size); + finally: + Py_XDECREF(base); + Py_XDECREF(sub); + Py_XDECREF(instance); + return result; +} + + static PyMethodDef TestMethods[] = { - /* Add module methods here. - * (Empty list left here as template/example, since using - * PyModule_AddFunctions isn't very common.) - */ + {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, {NULL}, }; @@ -17,6 +73,8 @@ _PyTestCapi_Init_HeaptypeRelative(PyObject *m) { return -1; } + PyModule_AddIntConstant(m, "alignof_max_align_t", alignof(max_align_t)); + return 0; } From c580c35191e361e77e90960d5ac0ae0facac6265 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 19 Oct 2022 17:59:15 +0200 Subject: [PATCH 07/21] Use Py_slot_inherit_itemsize rather than Py_tp_... --- Include/typeslots.h | 2 +- Misc/stable_abi.toml | 2 +- Objects/typeobject.c | 13 +++++++++---- Objects/typeslots.inc | 1 + Objects/typeslots.py | 2 ++ 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Include/typeslots.h b/Include/typeslots.h index 2ea2a61f0164a8..1b64c3587b815c 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -88,5 +88,5 @@ #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 /* New in 3.12 */ -#define Py_tp_inherit_itemsize 82 +#define Py_slot_inherit_itemsize 82 #endif diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 7d5b37609f3b17..8614290ece0bff 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2347,7 +2347,7 @@ added = '3.12' [function.PyObject_GetTypeDataSize] added = '3.12' -[const.Py_tp_inherit_itemsize] +[const.Py_slot_inherit_itemsize] added = '3.12' [const.Py_RELATIVE_OFFSET] added = '3.12' diff --git a/Objects/typeobject.c b/Objects/typeobject.c index bd395e6efb2b28..17a6fb188e0cd8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3704,18 +3704,18 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, memcpy(tp_doc, slot->pfunc, len); } break; - case Py_tp_inherit_itemsize: + case Py_slot_inherit_itemsize: inherit_itemsize = 1; if (spec->itemsize != 0) { PyErr_SetString( PyExc_SystemError, - "With Py_tp_inherit_itemsize, itemsize must be 0."); + "With Py_slot_inherit_itemsize, itemsize must be 0."); goto finally; } if (slot->pfunc != NULL) { PyErr_SetString( PyExc_SystemError, - "pfunc for Py_tp_inherit_itemsize must be NULL."); + "pfunc for Py_slot_inherit_itemsize must be NULL."); goto finally; } break; @@ -3899,7 +3899,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, case Py_tp_base: case Py_tp_bases: case Py_tp_doc: - case Py_tp_inherit_itemsize: + case Py_slot_inherit_itemsize: /* Processed above */ break; case Py_tp_members: @@ -3925,9 +3925,14 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; short slot_offset = slotoffsets.slot_offset; if (slotoffsets.subslot_offset == -1) { + /* "virtual" slots likePy_slot_inherit_itemsize should + * be processed above */ + assert(slot_offset); + /* Set a slot in the main PyTypeObject */ *(void**)((char*)res_start + slot_offset) = slot->pfunc; } else { + /* Set a slot in one of the tp_as_* tables */ void *procs = *(void**)((char*)res_start + slot_offset); short subslot_offset = slotoffsets.subslot_offset; *(void**)((char*)procs + subslot_offset) = slot->pfunc; diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index 896daa7d8066b7..a10bce100168d9 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -80,3 +80,4 @@ {offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, {-1, offsetof(PyTypeObject, tp_finalize)}, {offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, +{-1, 0} // "virtual", not stored, diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 8ab05f91be12b0..cae9f94b816155 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -30,6 +30,8 @@ def generate_typeslots(out=sys.stdout): elif member.startswith("bf_"): member = (f'{{offsetof(PyBufferProcs, {member}),'+ ' offsetof(PyTypeObject, tp_as_buffer)}') + elif member.startswith("slot_"): + member = '{-1, 0} // "virtual", not stored' res[int(m.group(2))] = member M = max(res.keys())+1 From e8eea516ea725c58c607c59c7c7338659d28a435 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 19 Oct 2022 17:59:44 +0200 Subject: [PATCH 08/21] Test subclassing a variable-size type --- Lib/test/test_capi/test_misc.py | 33 ++++++++ Modules/_testcapi/heaptype.c | 110 ++++++++++++++++++++++++++ Modules/_testcapi/heaptype_relative.c | 81 +++++++++++++++++++ 3 files changed, 224 insertions(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index fe5b0f623bcd61..377e130417d65b 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -895,6 +895,39 @@ def check(extra_base_size, extra_size): with self.subTest(**args): check(**args) + @requires_limited_api + def test_HeapCCollection(self): + """Make sure HeapCCollection works properly by itself""" + collection = _testcapi.HeapCCollection(1, 2, 3) + self.assertEqual(list(collection), [1, 2, 3]) + + @requires_limited_api + def test_heaptype_itemsize_nope(self): + """Test invalid arguments around Py_slot_inherit_itemsize""" + with self.assertRaises(SystemError): + _testcapi.subclass_var_heaptype(_testcapi.HeapCCollection, 0, 0, 1) + with self.assertRaises(SystemError): + _testcapi.subclass_var_heaptype(_testcapi.HeapCCollection, 0, 1, 0) + + @requires_limited_api + def test_heaptype_inherit_itemsize(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_size in sizes: + with self.subTest(extra_size=extra_size): + Sub = _testcapi.subclass_var_heaptype( + _testcapi.HeapCCollection, -extra_size, 0, 0) + collection = Sub(1, 2, 3) + collection.set_data_to_3s() + + self.assertEqual(list(collection), [1, 2, 3]) + mem = collection.get_data() + self.assertGreaterEqual(len(mem), extra_size) + self.assertTrue(set(mem) <= {3}, f'got {mem}') + def test_pynumber_tobase(self): from _testcapi import pynumber_tobase small_number = 123 diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index bf80fd64d80b35..eaa750e92a5fee 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -968,6 +968,109 @@ static PyType_Spec HeapCTypeSetattr_spec = { HeapCTypeSetattr_slots }; +PyDoc_STRVAR(HeapCCollection_doc, +"Tuple-like heap type that uses PyObject_GetItemData for items."); + +static PyObject* +HeapCCollection_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + PyObject *self = NULL; + PyObject *result = NULL; + + Py_ssize_t size = PyTuple_GET_SIZE(args); + self = subtype->tp_alloc(subtype, size); + if (!self) { + goto finally; + } + PyObject **data = PyObject_GetItemData(self); + if (!data) { + goto finally; + } + + for (Py_ssize_t i = 0; i < size; i++) { + data[i] = Py_NewRef(PyTuple_GET_ITEM(args, i)); + } + + result = self; + self = NULL; + finally: + Py_XDECREF(self); + return result; +} + +static Py_ssize_t +HeapCCollection_length(PyVarObject *self) +{ + return Py_SIZE(self); +} + +static PyObject* +HeapCCollection_item(PyObject *self, Py_ssize_t i) +{ + if (i < 0 || i >= Py_SIZE(self)) { + return PyErr_Format(PyExc_IndexError, "index %zd out of range", i); + } + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return NULL; + } + return Py_NewRef(data[i]); +} + +static int +HeapCCollection_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return -1; + } + for (Py_ssize_t i = 0; i < Py_SIZE(self); i++) { + Py_VISIT(data[i]); + } + return 0; +} + +static int +HeapCCollection_clear(PyObject *self) { + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return -1; + } + Py_ssize_t size = Py_SIZE(self); + Py_SET_SIZE(self, 0); + for (Py_ssize_t i = 0; i < size; i++) { + Py_CLEAR(data[i]); + } + return 0; +} + +static void HeapCCollection_dealloc(PyObject *self) { + PyTypeObject *tp = Py_TYPE(self); + HeapCCollection_clear(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot HeapCCollection_slots[] = { + {Py_tp_new, HeapCCollection_new}, + {Py_sq_length, HeapCCollection_length}, + {Py_sq_item, HeapCCollection_item}, + {Py_tp_traverse, HeapCCollection_traverse}, + {Py_tp_clear, HeapCCollection_clear}, + {Py_tp_dealloc, HeapCCollection_dealloc}, + {Py_tp_doc, (char*)HeapCCollection_doc}, + {0, 0}, +}; + +static PyType_Spec HeapCCollection_spec = { + "_testcapi.HeapCCollection", + sizeof(PyVarObject), + sizeof(PyObject*), + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + HeapCCollection_slots +}; + int _PyTestCapi_Init_Heaptype(PyObject *m) { _testcapimodule = PyModule_GetDef(m); @@ -1091,5 +1194,12 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { } PyModule_AddObject(m, "HeapCTypeMetaclassCustomNew", HeapCTypeMetaclassCustomNew); + PyObject *HeapCCollection = PyType_FromMetaclass( + NULL, m, &HeapCCollection_spec, NULL); + if (HeapCCollection == NULL) { + return -1; + } + PyModule_AddObject(m, "HeapCCollection", HeapCCollection); + return 0; } diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index b4778ebc2f4660..ee34764d36ff3b 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -2,6 +2,7 @@ #include "parts.h" #include // alignof #include // max_align_t +#include // memset #ifdef LIMITED_API_AVAILABLE @@ -50,7 +51,13 @@ make_sized_heaptypes(PyObject *module, PyObject *args) goto finally; } void *data_ptr = PyObject_GetTypeData(instance, (PyTypeObject *)sub); + if (!data_ptr) { + goto finally; + } Py_ssize_t data_size = PyObject_GetTypeDataSize((PyTypeObject *)sub); + if (data_size < 0) { + goto finally; + } result = Py_BuildValue("OOOln", base, sub, instance, (long)data_ptr, data_size); @@ -61,9 +68,83 @@ make_sized_heaptypes(PyObject *module, PyObject *args) return result; } +static PyObject * +var_heaptype_set_data_to_3s( + PyObject *self, PyTypeObject *defining_class, + PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + void *data_ptr = PyObject_GetTypeData(self, defining_class); + if (!data_ptr) { + return NULL; + } + Py_ssize_t data_size = PyObject_GetTypeDataSize(defining_class); + if (data_size < 0) { + return NULL; + } + memset(data_ptr, 3, data_size); + Py_RETURN_NONE; +} + +static PyObject * +var_heaptype_get_data(PyObject *self, PyTypeObject *defining_class, + PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + void *data_ptr = PyObject_GetTypeData(self, defining_class); + if (!data_ptr) { + return NULL; + } + Py_ssize_t data_size = PyObject_GetTypeDataSize(defining_class); + if (data_size < 0) { + return NULL; + } + return PyBytes_FromStringAndSize(data_ptr, data_size); +} + +static PyMethodDef var_heaptype_methods[] = { + {"set_data_to_3s", _PyCFunction_CAST(var_heaptype_set_data_to_3s), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS}, + {"get_data", _PyCFunction_CAST(var_heaptype_get_data), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS}, + {NULL}, +}; + +static PyObject * +subclass_var_heaptype(PyObject *module, PyObject *args) +{ + PyObject *result = NULL; + + PyObject *base; // borrowed from args + int basicsize, itemsize; + long pfunc; + + int r = PyArg_ParseTuple(args, "Oiil", &base, &basicsize, &itemsize, &pfunc); + if (!r) { + goto finally; + } + + PyType_Slot slots[] = { + {Py_tp_methods, var_heaptype_methods}, + {Py_slot_inherit_itemsize, (void*)pfunc}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .itemsize = itemsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + return result; +} + static PyMethodDef TestMethods[] = { {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, + {"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS}, {NULL}, }; From f836c7f25d23612829f5ef39a881347c9e74b224 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 31 Oct 2022 16:43:04 +0100 Subject: [PATCH 09/21] Remove check for basicsize > 0 for members without PY_RELATIVE_OFFSET Theoretically it's possible to reuse a member definition from a superclass, or get an absolute offset some other way. --- Objects/typeobject.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 17a6fb188e0cd8..5693e98681829f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3676,9 +3676,6 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, if (memb->flags & Py_RELATIVE_OFFSET) { assert(spec->basicsize <= 0); } - else { - assert(spec->basicsize > 0); - } } break; case Py_tp_doc: From a4b2fb155bfdfcc93017dc1423aef4141db3a2fb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 31 Oct 2022 17:29:49 +0100 Subject: [PATCH 10/21] Avoid BytesWarning in tests --- Lib/test/test_capi/test_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 377e130417d65b..3b809193870dd6 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -926,7 +926,7 @@ def test_heaptype_inherit_itemsize(self): self.assertEqual(list(collection), [1, 2, 3]) mem = collection.get_data() self.assertGreaterEqual(len(mem), extra_size) - self.assertTrue(set(mem) <= {3}, f'got {mem}') + self.assertTrue(set(mem) <= {3}, f'got {mem!r}') def test_pynumber_tobase(self): from _testcapi import pynumber_tobase From 612442891d13041c8c9bd3e8e6a5df21d9b7b823 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 31 Oct 2022 17:44:11 +0100 Subject: [PATCH 11/21] Objects/typeslots.py: Put the comma in individual entries, to allow comments Objects/typeslots.py# modified: Objects/typeobject.c --- Objects/typeslots.inc | 2 +- Objects/typeslots.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index a10bce100168d9..69703f3824e815 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -80,4 +80,4 @@ {offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, {-1, offsetof(PyTypeObject, tp_finalize)}, {offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, -{-1, 0} // "virtual", not stored, +{-1, 0}, // "virtual", not stored diff --git a/Objects/typeslots.py b/Objects/typeslots.py index cae9f94b816155..a139537405ceb1 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -14,30 +14,30 @@ def generate_typeslots(out=sys.stdout): member = m.group(1) if member.startswith("tp_"): - member = f'{{-1, offsetof(PyTypeObject, {member})}}' + member = f'{{-1, offsetof(PyTypeObject, {member})}},' elif member.startswith("am_"): member = (f'{{offsetof(PyAsyncMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_async)}') + ' offsetof(PyTypeObject, tp_as_async)},') elif member.startswith("nb_"): member = (f'{{offsetof(PyNumberMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_number)}') + ' offsetof(PyTypeObject, tp_as_number)},') elif member.startswith("mp_"): member = (f'{{offsetof(PyMappingMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_mapping)}') + ' offsetof(PyTypeObject, tp_as_mapping)},') elif member.startswith("sq_"): member = (f'{{offsetof(PySequenceMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_sequence)}') + ' offsetof(PyTypeObject, tp_as_sequence)},') elif member.startswith("bf_"): member = (f'{{offsetof(PyBufferProcs, {member}),'+ - ' offsetof(PyTypeObject, tp_as_buffer)}') + ' offsetof(PyTypeObject, tp_as_buffer)},') elif member.startswith("slot_"): - member = '{-1, 0} // "virtual", not stored' + member = '{-1, 0}, // "virtual", not stored' res[int(m.group(2))] = member M = max(res.keys())+1 for i in range(1,M): if i in res: - out.write("%s,\n" % res[i]) + out.write("%s\n" % res[i]) else: out.write("{0, 0},\n") From 558121670b5038bde0c4b89c90c4e01e0b07f549 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 1 Nov 2022 16:23:55 +0100 Subject: [PATCH 12/21] Improve tests for PyType_GetSlot --- Include/typeslots.h | 3 +++ Lib/test/test_capi/test_misc.py | 16 ++++++++++++++++ Modules/_testcapimodule.c | 26 +++++++++++++++++++++++++- Objects/typeobject.c | 12 ++++++++++-- Objects/typeslots.py | 15 +++++++++++++-- 5 files changed, 67 insertions(+), 5 deletions(-) diff --git a/Include/typeslots.h b/Include/typeslots.h index 1b64c3587b815c..8e73ee381443ee 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -90,3 +90,6 @@ /* New in 3.12 */ #define Py_slot_inherit_itemsize 82 #endif + +/* Do renumber this one */ +#define _Py_slot_max 82 diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3b809193870dd6..e9be878cee4d8a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -928,6 +928,22 @@ def test_heaptype_inherit_itemsize(self): self.assertGreaterEqual(len(mem), extra_size) self.assertTrue(set(mem) <= {3}, f'got {mem!r}') + def test_pytype_getslot(self): + with self.assertRaisesRegex( + SystemError, + "PyType_GetSlot: slot out of range"): + _testcapi.pytype_getslot(int, 0) + + with self.assertRaisesRegex( + SystemError, + "PyType_GetSlot: slot out of range"): + _testcapi.pytype_getslot(int, -1) + + with self.assertRaisesRegex( + SystemError, + "PyType_GetSlot: incompatible slot"): + _testcapi.pytype_getslot(int, 82) # Py_slot_inherit_itemsize + def test_pynumber_tobase(self): from _testcapi import pynumber_tobase small_number = 123 diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 4303e347d7b0fc..c9d14df08d9628 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -587,11 +587,18 @@ test_get_statictype_slots(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } - void *over_value = PyType_GetSlot(&PyLong_Type, Py_bf_releasebuffer + 1); + void *over_value = PyType_GetSlot(&PyLong_Type, _Py_slot_max + 1); if (over_value != NULL) { PyErr_SetString(PyExc_AssertionError, "mismatch: max+1 of long"); return NULL; } + if (PyErr_ExceptionMatches(PyExc_SystemError)) { + // This is the right exception + PyErr_Clear(); + } + else { + return NULL; + } tp_new = PyType_GetSlot(&PyLong_Type, 0); if (tp_new != NULL) { @@ -609,6 +616,22 @@ test_get_statictype_slots(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } +static PyObject * +pytype_getslot(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + int slot; + int res = PyArg_ParseTuple(args, "O!i:pytype_getslot", + &PyType_Type, &type, &slot); + if (!res) { + return NULL; + } + void *pfunc = PyType_GetSlot(type, slot); + if (PyErr_Occurred()) { + return NULL; + } + return pfunc ? Py_NewRef(Py_True) : Py_NewRef(Py_None); +} static PyType_Slot HeapTypeNameType_slots[] = { {0}, @@ -3174,6 +3197,7 @@ static PyMethodDef TestMethods[] = { {"test_buildvalue_N", test_buildvalue_N, METH_NOARGS}, {"test_buildvalue_issue38913", test_buildvalue_issue38913, METH_NOARGS}, {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, + {"pytype_getslot", pytype_getslot, METH_VARARGS}, {"test_get_type_name", test_get_type_name, METH_NOARGS}, {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5693e98681829f..24c310fdd4c6d1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4065,16 +4065,24 @@ PyType_GetSlot(PyTypeObject *type, int slot) int slots_len = Py_ARRAY_LENGTH(pyslot_offsets); if (slot <= 0 || slot >= slots_len) { - PyErr_BadInternalCall(); + PyErr_SetString(PyExc_SystemError, + "PyType_GetSlot: slot out of range"); return NULL; } - parent_slot = *(void**)((char*)type + pyslot_offsets[slot].slot_offset); + int slot_offset = pyslot_offsets[slot].slot_offset; + + parent_slot = *(void**)((char*)type + slot_offset); if (parent_slot == NULL) { return NULL; } /* Return slot directly if we have no sub slot. */ if (pyslot_offsets[slot].subslot_offset == -1) { + if (slot_offset == 0) { + PyErr_SetString(PyExc_SystemError, + "PyType_GetSlot: incompatible slot"); + return NULL; + } return parent_slot; } return *(void**)((char*)parent_slot + pyslot_offsets[slot].subslot_offset); diff --git a/Objects/typeslots.py b/Objects/typeslots.py index a139537405ceb1..faaa1894a0f93a 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -7,7 +7,14 @@ def generate_typeslots(out=sys.stdout): out.write("/* Generated by typeslots.py */\n") res = {} + last = None for line in sys.stdin: + m = re.match("#define _Py_slot_max ([0-9]+)", line) + if m: + assert last is None + last = int(m[1]) + continue + m = re.match("#define Py_([a-z_]+) ([0-9]+)", line) if not m: continue @@ -34,8 +41,12 @@ def generate_typeslots(out=sys.stdout): member = '{-1, 0}, // "virtual", not stored' res[int(m.group(2))] = member - M = max(res.keys())+1 - for i in range(1,M): + if last is None: + raise ValueError('_Py_slot_max not defined') + if last != max(res.keys()): + raise ValueError("_Py_slot_max doesn't match highest-numbered slot") + + for i in range(1, last+1): if i in res: out.write("%s\n" % res[i]) else: From 6508229bede1b0dc3c5f359450811f8b38455686 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 2 Nov 2022 11:13:50 +0100 Subject: [PATCH 13/21] Add tests for Py_RELATIVE_OFFSET --- Lib/test/test_capi/test_misc.py | 37 +++++++++ Modules/_testcapi/heaptype_relative.c | 109 ++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index e9be878cee4d8a..f433ade2c00fd8 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -928,6 +928,43 @@ def test_heaptype_inherit_itemsize(self): self.assertGreaterEqual(len(mem), extra_size) self.assertTrue(set(mem) <= {3}, f'got {mem!r}') + @requires_limited_api + def test_heaptype_relative_members(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + for offset in sizes: + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): + if offset < extra_size: + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + Base = Sub.mro()[1] + instance = Sub() + self.assertEqual(instance.memb, instance.get_memb()) + instance.set_memb(13) + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 13) + instance.memb = 14 + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 14) + self.assertGreaterEqual(instance.get_memb_offset(), Base.__basicsize__) + self.assertLess(instance.get_memb_offset(), Sub.__basicsize__) + else: + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, extra_size, offset, True) + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, -1, True) + def test_pytype_getslot(self): with self.assertRaisesRegex( SystemError, diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index ee34764d36ff3b..bc4001e14a8d72 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -141,10 +141,119 @@ subclass_var_heaptype(PyObject *module, PyObject *args) return result; } +static PyMemberDef * +heaptype_with_member_extract_and_check_memb(PyObject *self) +{ + PyMemberDef *def = PyType_GetSlot(Py_TYPE(self), Py_tp_members); + if (!def) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, "tp_members is NULL"); + } + return NULL; + } + if (!def[0].name) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] is NULL"); + return NULL; + } + if (def[1].name) { + PyErr_SetString(PyExc_ValueError, "tp_members[1] is not NULL"); + return NULL; + } + if (strcmp(def[0].name, "memb")) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] is not for `memb`"); + return NULL; + } + if (def[0].flags) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] has flags set"); + return NULL; + } + return def; +} + +static PyObject * +heaptype_with_member_get_memb(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + return PyMember_GetOne((const char *)self, def); +} + +static PyObject * +heaptype_with_member_set_memb(PyObject *self, PyObject *value) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + int r = PyMember_SetOne((char *)self, def, value); + if (r < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +get_memb_offset(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + return PyLong_FromSsize_t(def->offset); +} + +static PyMethodDef heaptype_with_member_methods[] = { + {"get_memb", heaptype_with_member_get_memb, METH_NOARGS}, + {"set_memb", heaptype_with_member_set_memb, METH_O}, + {"get_memb_offset", get_memb_offset, METH_NOARGS}, + {NULL}, +}; + +static PyObject * +make_heaptype_with_member(PyObject *module, PyObject *args) +{ + PyObject *base = NULL; + PyObject *result = NULL; + + int extra_base_size, basicsize, offset, add_flag; + + int r = PyArg_ParseTuple(args, "iiip", &extra_base_size, &basicsize, &offset, &add_flag); + if (!r) { + goto finally; + } + + PyType_Spec base_spec = { + .name = "_testcapi.Base", + .basicsize = sizeof(PyObject) + extra_base_size, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = empty_slots, + }; + base = PyType_FromMetaclass(NULL, module, &base_spec, NULL); + if (!base) { + goto finally; + } + + PyMemberDef members[] = { + {"memb", Py_T_BYTE, offset, add_flag ? Py_RELATIVE_OFFSET : 0}, + {0}, + }; + PyType_Slot slots[] = { + {Py_tp_members, members}, + {Py_tp_methods, heaptype_with_member_methods}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + Py_XDECREF(base); + return result; +} + static PyMethodDef TestMethods[] = { {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, {"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS}, + {"make_heaptype_with_member", make_heaptype_with_member, METH_VARARGS}, {NULL}, }; From e81a7bc17d8aca0454debb7a1d06d31a797e85c8 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 9 Nov 2022 18:03:45 +0100 Subject: [PATCH 14/21] Additional checks for Py_RELATIVE_OFFSET --- Objects/typeobject.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 24c310fdd4c6d1..aea3a4b619cdfa 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3674,7 +3674,18 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, vectorcalloffset = memb->offset; } if (memb->flags & Py_RELATIVE_OFFSET) { - assert(spec->basicsize <= 0); + if(spec->basicsize > 0) { + PyErr_SetString( + PyExc_SystemError, + "With Py_RELATIVE_OFFSET, basicsize must be negative."); + goto finally; + } + if(memb->offset < 0 || memb->offset >= -spec->basicsize) { + PyErr_SetString( + PyExc_SystemError, + "Member offset out of range (0..-basicsize)"); + goto finally; + } } } break; From 06f349e6a6e718a42e3bca400f7964f938d1d18b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 10 Nov 2022 17:03:49 +0100 Subject: [PATCH 15/21] Switch _PyHeapType_GET_MEMBERS to PyObject_GetItemData --- Include/internal/pycore_object.h | 5 ----- Objects/typeobject.c | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 33c8c0b75ea742..865a893614ca0f 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -354,11 +354,6 @@ extern int _PyObject_IsInstanceDictEmpty(PyObject *); extern int _PyType_HasSubclasses(PyTypeObject *); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); -// Access macro to the members which are floating "behind" the object -static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { - return (PyMemberDef*)((char*)etype + Py_TYPE(etype)->tp_basicsize); -} - PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); /* C function call trampolines to mitigate bad function pointer casts. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index aea3a4b619cdfa..f3b8bb0d3888da 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1349,6 +1349,12 @@ PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Helpers for subtyping */ +static inline PyMemberDef * +_PyHeapType_GET_MEMBERS(PyHeapTypeObject* type) +{ + return PyObject_GetItemData((PyObject *)type); +} + static int traverse_slots(PyTypeObject *type, PyObject *self, visitproc visit, void *arg) { From 9087846a36df8cf3f64ac291874ecdc04bfa253a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 15 Nov 2022 17:43:12 +0100 Subject: [PATCH 16/21] [TMP] Start on docs --- Doc/c-api/type.rst | 50 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 7b5d1fac40ed87..053a67ef207517 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -327,11 +327,45 @@ The following functions and structs are used to create Name of the type, used to set :c:member:`PyTypeObject.tp_name`. .. c:member:: int PyType_Spec.basicsize + + If positive, specifies the size of the instance in bytes. + It is used to set :c:member:`PyTypeObject.tp_basicsize`. + + If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize` + should be inherited. + + If negative, the absolute value specifies how much space instances of the + class need *in addition* to the superclass. + Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific + memory reserved this way. + + .. versionchanged:: 3.12 + + Previously, this field could not be negative. + .. c:member:: int PyType_Spec.itemsize - Size of the instance in bytes, used to set - :c:member:`PyTypeObject.tp_basicsize` and - :c:member:`PyTypeObject.tp_itemsize`. + Size of one element of a variable-size type, in bytes + Used to set :c:member:`PyTypeObject.tp_itemsize`. + See ``tp_itemsize`` documentation for caveats. + + If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited. + Extending arbitrary variable-sized classes is dangerous, + since some types use a fixed offset for variable-sized memory, + which can then overlap fixed-sized memory used by subclass. + To help prevent mistakes, inheriting ``itemsize`` is only possible + in the following situations: + + - The base :c:member:`~PyTypeObject.tp_itemsize` is zero, i.e. the type + not variable-sized. + - The requested :c:member:`PyType_Spec.basicsize` is positive, + suggesting that the memory layout of the base class is known. + - The :c:member:`PyType_Spec.basicsize` is zero, + suggesting that the subclass does not access the instance's memory + directly. + - The :c:macro:`Py_slot_inherit_itemsize` is set, + asserting that the superclass only uses :c:func:`PyObject_GetItemData` + (or an equivalent) to access variable-sized memory. .. c:member:: int PyType_Spec.flags @@ -387,6 +421,14 @@ The following functions and structs are used to create To avoid issues, use the *bases* argument of :py:func:`PyType_FromSpecWithBases` instead. + An additional slot is available: + + .. c:macro:: Py_slot_inherit_itemsize + + Used to extend a variable-sized class that is known to only use + :c:func:`PyObject_GetItemData` (or an equivalent) to access + variable-sized memory. + .. versionchanged:: 3.9 Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. @@ -396,9 +438,11 @@ The following functions and structs are used to create :c:member:`~PyBufferProcs.bf_releasebuffer` are now available under the limited API. + .. c:member:: void *PyType_Slot.pfunc The desired value of the slot. In most cases, this is a pointer to a function. Slots other than ``Py_tp_doc`` may not be ``NULL``. + From c70b7c1022d3db216382f5b824ee2cad55f8d920 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 16 Nov 2022 13:24:25 +0100 Subject: [PATCH 17/21] Switch to Py_TPFLAGS_ITEMS_AT_END --- Doc/c-api/type.rst | 12 +------ Doc/c-api/typeobj.rst | 20 ++++++++++++ Include/object.h | 3 ++ Include/typeslots.h | 6 +--- Lib/test/test_capi/test_misc.py | 13 -------- Misc/stable_abi.toml | 4 +-- Modules/_testcapi/heaptype_relative.c | 3 +- Objects/typeobject.c | 46 +++++++++++++-------------- 8 files changed, 51 insertions(+), 56 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 053a67ef207517..8ee3f8657c6a75 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -363,9 +363,7 @@ The following functions and structs are used to create - The :c:member:`PyType_Spec.basicsize` is zero, suggesting that the subclass does not access the instance's memory directly. - - The :c:macro:`Py_slot_inherit_itemsize` is set, - asserting that the superclass only uses :c:func:`PyObject_GetItemData` - (or an equivalent) to access variable-sized memory. + - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END`flag. .. c:member:: int PyType_Spec.flags @@ -421,14 +419,6 @@ The following functions and structs are used to create To avoid issues, use the *bases* argument of :py:func:`PyType_FromSpecWithBases` instead. - An additional slot is available: - - .. c:macro:: Py_slot_inherit_itemsize - - Used to extend a variable-sized class that is known to only use - :c:func:`PyObject_GetItemData` (or an equivalent) to access - variable-sized memory. - .. versionchanged:: 3.9 Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 8f8869ec668a8d..c3b19375e95ff1 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1168,6 +1168,26 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. + .. data:: Py_TPFLAGS_ITEMS_AT_END + + Only usable with variable-size types, i.e. ones with non-zero + :c:member:`~PyObject.tp_itemsize``. + + Indicates that the variable-sized portion of an instance of this type is + at the end of the instance's memory area, at an offset of + :c:expr:`Py_TYPE(obj)->tp_basicsize` (which may be different in each + subclass). + + When setting this flag, be sure that all superclasses either + use the memory layout, or are not variable-sized. + Python does not check this. + + .. versionadded:: 3.12 + + **Inheritance:** + + This flag is inherited. + .. XXX Document more flags here? diff --git a/Include/object.h b/Include/object.h index 6fced3ee3e4b51..c5f977734afcbb 100644 --- a/Include/object.h +++ b/Include/object.h @@ -432,6 +432,9 @@ given type object has a specified feature. // subject itself (rather than a mapped attribute on it): #define _Py_TPFLAGS_MATCH_SELF (1UL << 22) +/* Items (ob_size*tp_itemsize) are found at the end of an instance's memory */ +#define Py_TPFLAGS_ITEMS_AT_END (1UL << 23) + /* These flags are used to determine if a type is a subclass. */ #define Py_TPFLAGS_LONG_SUBCLASS (1UL << 24) #define Py_TPFLAGS_LIST_SUBCLASS (1UL << 25) diff --git a/Include/typeslots.h b/Include/typeslots.h index 8e73ee381443ee..01fd407b51bb99 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -86,10 +86,6 @@ /* New in 3.10 */ #define Py_am_send 81 #endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 -/* New in 3.12 */ -#define Py_slot_inherit_itemsize 82 -#endif /* Do renumber this one */ -#define _Py_slot_max 82 +#define _Py_slot_max 81 diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index f433ade2c00fd8..0a204ee603565e 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -901,14 +901,6 @@ def test_HeapCCollection(self): collection = _testcapi.HeapCCollection(1, 2, 3) self.assertEqual(list(collection), [1, 2, 3]) - @requires_limited_api - def test_heaptype_itemsize_nope(self): - """Test invalid arguments around Py_slot_inherit_itemsize""" - with self.assertRaises(SystemError): - _testcapi.subclass_var_heaptype(_testcapi.HeapCCollection, 0, 0, 1) - with self.assertRaises(SystemError): - _testcapi.subclass_var_heaptype(_testcapi.HeapCCollection, 0, 1, 0) - @requires_limited_api def test_heaptype_inherit_itemsize(self): """Test HeapCCollection subclasses work properly""" @@ -976,11 +968,6 @@ def test_pytype_getslot(self): "PyType_GetSlot: slot out of range"): _testcapi.pytype_getslot(int, -1) - with self.assertRaisesRegex( - SystemError, - "PyType_GetSlot: incompatible slot"): - _testcapi.pytype_getslot(int, 82) # Py_slot_inherit_itemsize - def test_pynumber_tobase(self): from _testcapi import pynumber_tobase small_number = 123 diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8614290ece0bff..3d97c72d89c39c 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2347,7 +2347,7 @@ added = '3.12' [function.PyObject_GetTypeDataSize] added = '3.12' -[const.Py_slot_inherit_itemsize] - added = '3.12' [const.Py_RELATIVE_OFFSET] added = '3.12' +[const.Py_TPFLAGS_ITEMS_AT_END] + added = '3.12' diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index bc4001e14a8d72..ba7652b07b6b88 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -124,7 +124,6 @@ subclass_var_heaptype(PyObject *module, PyObject *args) PyType_Slot slots[] = { {Py_tp_methods, var_heaptype_methods}, - {Py_slot_inherit_itemsize, (void*)pfunc}, {0, NULL}, }; @@ -132,7 +131,7 @@ subclass_var_heaptype(PyObject *module, PyObject *args) .name = "_testcapi.Sub", .basicsize = basicsize, .itemsize = itemsize, - .flags = Py_TPFLAGS_DEFAULT, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_ITEMS_AT_END, .slots = slots, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f3b8bb0d3888da..59d83cd758c5c8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3718,21 +3718,6 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, memcpy(tp_doc, slot->pfunc, len); } break; - case Py_slot_inherit_itemsize: - inherit_itemsize = 1; - if (spec->itemsize != 0) { - PyErr_SetString( - PyExc_SystemError, - "With Py_slot_inherit_itemsize, itemsize must be 0."); - goto finally; - } - if (slot->pfunc != NULL) { - PyErr_SetString( - PyExc_SystemError, - "pfunc for Py_slot_inherit_itemsize must be NULL."); - goto finally; - } - break; } } @@ -3849,6 +3834,16 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, /* Extend */ type_data_offset = _align_up(base->tp_basicsize); basicsize = type_data_offset + _align_up(-spec->basicsize); + + /* Inheriting variable-sized types is limited */ + if (base->tp_itemsize + && !((base->tp_flags | spec->flags) & Py_TPFLAGS_ITEMS_AT_END)) + { + PyErr_SetString( + PyExc_SystemError, + "Cannot extend variable-size class without Py_TPFLAGS_ITEMS_AT_END."); + goto finally; + } } Py_ssize_t itemsize = spec->itemsize; @@ -3913,7 +3908,6 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, case Py_tp_base: case Py_tp_bases: case Py_tp_doc: - case Py_slot_inherit_itemsize: /* Processed above */ break; case Py_tp_members: @@ -3939,9 +3933,6 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; short slot_offset = slotoffsets.slot_offset; if (slotoffsets.subslot_offset == -1) { - /* "virtual" slots likePy_slot_inherit_itemsize should - * be processed above */ - assert(slot_offset); /* Set a slot in the main PyTypeObject */ *(void**)((char*)res_start + slot_offset) = slot->pfunc; } @@ -4179,12 +4170,14 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) } void * -PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { +PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) +{ return (char *)obj + _align_up(cls->tp_base->tp_basicsize); } Py_ssize_t -PyObject_GetTypeDataSize(PyTypeObject *cls) { +PyObject_GetTypeDataSize(PyTypeObject *cls) +{ ptrdiff_t result = cls->tp_basicsize - _align_up(cls->tp_base->tp_basicsize); if (result < 0) { return 0; @@ -4193,7 +4186,8 @@ PyObject_GetTypeDataSize(PyTypeObject *cls) { } void * -PyObject_GetItemData(PyObject *obj) { +PyObject_GetItemData(PyObject *obj) +{ return (char *)obj + Py_TYPE(obj)->tp_basicsize; } @@ -6342,6 +6336,11 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) if (PyType_HasFeature(base, _Py_TPFLAGS_MATCH_SELF)) { type->tp_flags |= _Py_TPFLAGS_MATCH_SELF; } + + /* Inherit Py_TPFLAGS_ITEMS_AT_END */ + if (base->tp_flags & Py_TPFLAGS_ITEMS_AT_END) { + type->tp_flags |= Py_TPFLAGS_ITEMS_AT_END; + } } static int @@ -9797,7 +9796,8 @@ PyTypeObject PySuper_Type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ + Py_TPFLAGS_BASETYPE | Py_TPFLAGS_ITEMS_AT_END, + /* tp_flags */ super_doc, /* tp_doc */ super_traverse, /* tp_traverse */ 0, /* tp_clear */ From e2200ced88084914cfc248c0358f1078a41793e9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 29 Nov 2022 14:55:44 +0100 Subject: [PATCH 18/21] Rename PyObject_GetTypeDataSize to PyType_GetTypeDataSize --- Doc/data/stable_abi.dat | 2 +- Include/object.h | 2 +- Lib/test/test_stable_abi_ctypes.py | 2 +- Misc/stable_abi.toml | 2 +- Modules/_testcapi/heaptype_relative.c | 6 +++--- Objects/typeobject.c | 2 +- PC/python3dll.c | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 3639cc97a119ee..f4679fef687a36 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -517,7 +517,6 @@ function,PyObject_GetBuffer,3.11,, function,PyObject_GetItem,3.2,, function,PyObject_GetIter,3.2,, function,PyObject_GetTypeData,3.12,, -function,PyObject_GetTypeDataSize,3.12,, function,PyObject_HasAttr,3.2,, function,PyObject_HasAttrString,3.2,, function,PyObject_Hash,3.2,, @@ -672,6 +671,7 @@ function,PyType_GetModuleState,3.10,, function,PyType_GetName,3.11,, function,PyType_GetQualName,3.11,, function,PyType_GetSlot,3.4,, +function,PyType_GetTypeDataSize,3.12,, function,PyType_IsSubtype,3.2,, function,PyType_Modified,3.2,, function,PyType_Ready,3.2,, diff --git a/Include/object.h b/Include/object.h index c5f977734afcbb..9b9de568d13e03 100644 --- a/Include/object.h +++ b/Include/object.h @@ -265,7 +265,7 @@ PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); -PyAPI_FUNC(Py_ssize_t) PyObject_GetTypeDataSize(PyTypeObject *cls); +PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); #endif /* Generic type check */ diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 0049ab3220c0b1..b8850c632d30fb 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -525,7 +525,6 @@ def test_windows_feature_macros(self): "PyObject_GetItem", "PyObject_GetIter", "PyObject_GetTypeData", - "PyObject_GetTypeDataSize", "PyObject_HasAttr", "PyObject_HasAttrString", "PyObject_Hash", @@ -676,6 +675,7 @@ def test_windows_feature_macros(self): "PyType_GetName", "PyType_GetQualName", "PyType_GetSlot", + "PyType_GetTypeDataSize", "PyType_IsSubtype", "PyType_Modified", "PyType_Ready", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 3d97c72d89c39c..cf28299cc3449d 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2345,7 +2345,7 @@ [function.PyObject_GetTypeData] added = '3.12' -[function.PyObject_GetTypeDataSize] +[function.PyType_GetTypeDataSize] added = '3.12' [const.Py_RELATIVE_OFFSET] added = '3.12' diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index ba7652b07b6b88..4ea41a5405fcd2 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -54,7 +54,7 @@ make_sized_heaptypes(PyObject *module, PyObject *args) if (!data_ptr) { goto finally; } - Py_ssize_t data_size = PyObject_GetTypeDataSize((PyTypeObject *)sub); + Py_ssize_t data_size = PyType_GetTypeDataSize((PyTypeObject *)sub); if (data_size < 0) { goto finally; } @@ -77,7 +77,7 @@ var_heaptype_set_data_to_3s( if (!data_ptr) { return NULL; } - Py_ssize_t data_size = PyObject_GetTypeDataSize(defining_class); + Py_ssize_t data_size = PyType_GetTypeDataSize(defining_class); if (data_size < 0) { return NULL; } @@ -93,7 +93,7 @@ var_heaptype_get_data(PyObject *self, PyTypeObject *defining_class, if (!data_ptr) { return NULL; } - Py_ssize_t data_size = PyObject_GetTypeDataSize(defining_class); + Py_ssize_t data_size = PyType_GetTypeDataSize(defining_class); if (data_size < 0) { return NULL; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 59d83cd758c5c8..ceb182a014a6ad 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4176,7 +4176,7 @@ PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) } Py_ssize_t -PyObject_GetTypeDataSize(PyTypeObject *cls) +PyType_GetTypeDataSize(PyTypeObject *cls) { ptrdiff_t result = cls->tp_basicsize - _align_up(cls->tp_base->tp_basicsize); if (result < 0) { diff --git a/PC/python3dll.c b/PC/python3dll.c index b1c628e1f33ab8..16d0ea2f7f9f8d 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -463,7 +463,6 @@ EXPORT_FUNC(PyObject_GetBuffer) EXPORT_FUNC(PyObject_GetItem) EXPORT_FUNC(PyObject_GetIter) EXPORT_FUNC(PyObject_GetTypeData) -EXPORT_FUNC(PyObject_GetTypeDataSize) EXPORT_FUNC(PyObject_HasAttr) EXPORT_FUNC(PyObject_HasAttrString) EXPORT_FUNC(PyObject_Hash) @@ -615,6 +614,7 @@ EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) EXPORT_FUNC(PyType_GetQualName) EXPORT_FUNC(PyType_GetSlot) +EXPORT_FUNC(PyType_GetTypeDataSize) EXPORT_FUNC(PyType_IsSubtype) EXPORT_FUNC(PyType_Modified) EXPORT_FUNC(PyType_Ready) From 2a7432548eed253ac770864aca514bed3c1bc69c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 29 Nov 2022 14:55:31 +0100 Subject: [PATCH 19/21] Add more docs --- Doc/c-api/object.rst | 34 ++++++++++++++++++++++++++++++++++ Doc/c-api/structures.rst | 15 +++++++++++++++ Doc/c-api/type.rst | 14 +++++++------- Doc/c-api/typeobj.rst | 4 ++-- 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 5a25a2b6c9d3db..ee690d01418c1b 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -386,3 +386,37 @@ Object Protocol returns ``NULL`` if the object cannot be iterated. .. versionadded:: 3.10 + +.. c:function:: PyObject* PyObject_GetTypeData(PyObject *o, PyTypeObject *cls) + + The object *o* must be an instance of *cls*, and *cls* must have been + created using negative :c:member:`PyType_Spec.basicsize`. + + Get a pointer to subclass-specific data reserved for *cls*. + + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.12 + +.. c:function:: PyObject* PyType_GetTypeDataSize(PyTypeObject *cls) + + The type *cls* must have been created using + negative :c:member:`PyType_Spec.basicsize`. + + Return the size of the memory reserved for *cls*, i.e. the size of the + memory :c:func:`PyObject_GetTypeData` returns. + + This may be larger than requested using :c:member:`-PyType_Spec.basicsize `; + it is safe to use this larger size (e.g. with :c:func:`!memset`). + + .. versionadded:: 3.12 + +.. c:function:: PyObject* PyObject_GetItemData(PyObject *o) + + The object *o* must have :c:macro:`Py_TPFLAGS_ITEMS_AT_END` set. + + Get a pointer to the per-item data. + + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.12 diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 827d624fc99edb..f752430eb63b83 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -485,6 +485,21 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: Emit an ``object.__getattr__`` :ref:`audit event ` before reading. +.. c:macro:: Py_RELATIVE_OFFSET + + Can only be used as part of :c:member:`Py_tp_members ` + :c:type:`slot ` when creating a class using negative + :c:member:`~PyTypeDef.basicsize`. + It is mandatory in that case. + + Indicates that the :c:member:`~PyMemberDef.offset` of this ``PyMemberDef`` + entry indicates an offset from the subclass-specific data, rather than + from ``PyObject``. + + This flag is only used in :c:type:`PyTypeSlot`. + During class creation, it is cleared and :c:member:`PyMemberDef.offset` + is set to the offset from the ``PyObject`` struct. + .. index:: single: READ_RESTRICTED single: WRITE_RESTRICTED diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 8ee3f8657c6a75..571a2aa8f92446 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -322,11 +322,11 @@ The following functions and structs are used to create Structure defining a type's behavior. - .. c:member:: const char* PyType_Spec.name + .. c:member:: const char* name Name of the type, used to set :c:member:`PyTypeObject.tp_name`. - .. c:member:: int PyType_Spec.basicsize + .. c:member:: int basicsize If positive, specifies the size of the instance in bytes. It is used to set :c:member:`PyTypeObject.tp_basicsize`. @@ -343,7 +343,7 @@ The following functions and structs are used to create Previously, this field could not be negative. - .. c:member:: int PyType_Spec.itemsize + .. c:member:: int itemsize Size of one element of a variable-size type, in bytes Used to set :c:member:`PyTypeObject.tp_itemsize`. @@ -360,19 +360,19 @@ The following functions and structs are used to create not variable-sized. - The requested :c:member:`PyType_Spec.basicsize` is positive, suggesting that the memory layout of the base class is known. - - The :c:member:`PyType_Spec.basicsize` is zero, + - The requested :c:member:`PyType_Spec.basicsize` is zero, suggesting that the subclass does not access the instance's memory directly. - - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END`flag. + - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. - .. c:member:: int PyType_Spec.flags + .. c:member:: int flags Type flags, used to set :c:member:`PyTypeObject.tp_flags`. If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, :c:func:`PyType_FromSpecWithBases` sets it automatically. - .. c:member:: PyType_Slot *PyType_Spec.slots + .. c:member:: PyType_Slot *slots Array of :c:type:`PyType_Slot` structures. Terminated by the special slot value ``{0, NULL}``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c3b19375e95ff1..6f9033443ecf20 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1168,10 +1168,10 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. - .. data:: Py_TPFLAGS_ITEMS_AT_END + .. c:macro:: Py_TPFLAGS_ITEMS_AT_END Only usable with variable-size types, i.e. ones with non-zero - :c:member:`~PyObject.tp_itemsize``. + :c:member:`~PyObject.tp_itemsize`. Indicates that the variable-sized portion of an instance of this type is at the end of the instance's memory area, at an offset of From 68bd590dd4502bab982331072d235e048c53cdc5 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 29 Nov 2022 15:48:30 +0100 Subject: [PATCH 20/21] Add asserts to the accessors --- Objects/typeobject.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ceb182a014a6ad..15fc0ff71429cf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4172,6 +4172,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) void * PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { + assert(PyObject_TypeCheck(obj, cls)); return (char *)obj + _align_up(cls->tp_base->tp_basicsize); } @@ -4188,6 +4189,7 @@ PyType_GetTypeDataSize(PyTypeObject *cls) void * PyObject_GetItemData(PyObject *obj) { + assert(PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_ITEMS_AT_END)); return (char *)obj + Py_TYPE(obj)->tp_basicsize; } From 53695a2701ef2f54e5549f3835d64f6ba44a5732 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 29 Nov 2022 15:49:00 +0100 Subject: [PATCH 21/21] Add Py_TPFLAGS_ITEMS_AT_END to PyType_Type and a test type --- Modules/_testcapi/heaptype.c | 3 ++- Objects/typeobject.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index eaa750e92a5fee..095c7970b2bb6e 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -1067,7 +1067,8 @@ static PyType_Spec HeapCCollection_spec = { "_testcapi.HeapCCollection", sizeof(PyVarObject), sizeof(PyObject*), - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC + | Py_TPFLAGS_ITEMS_AT_END, HeapCCollection_slots }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 15fc0ff71429cf..080b8c4ab502f3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4923,7 +4923,8 @@ PyTypeObject PyType_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS | - Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */ + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_ITEMS_AT_END, + /* tp_flags */ type_doc, /* tp_doc */ (traverseproc)type_traverse, /* tp_traverse */ (inquiry)type_clear, /* tp_clear */