diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index cf0b148c2b6463..4a7191a562cc10 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { # define _PyGC_BITS_UNREACHABLE (4) # define _PyGC_BITS_FROZEN (8) # define _PyGC_BITS_SHARED (16) +# define _PyGC_BITS_SHARED_INLINE (32) #endif /* True if the object is currently tracked by the GC. */ @@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { #ifdef Py_GIL_DISABLED -/* True if an object is shared between multiple threads and - * needs special purpose when freeing to do the possibility - * of in-flight lock-free reads occurring */ +/* True if memory the object references is shared between + * multiple threads and needs special purpose when freeing + * those references due to the possibility of in-flight + * lock-free reads occurring. The object is responsible + * for calling _PyMem_FreeDelayed on the referenced + * memory. */ static inline int _PyObject_GC_IS_SHARED(PyObject *op) { return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; } @@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { } #define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) +/* True if the memory of the object is shared between multiple + * threads and needs special purpose when freeing due to + * the possibility of in-flight lock-free reads occurring. + * Objects with this bit that are GC objects will automatically + * delay-freed by PyObject_GC_Del. */ +static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) { + return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0; +} +#define _PyObject_GC_IS_SHARED_INLINE(op) \ + _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) { + op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE; +} +#define _PyObject_GC_SET_SHARED_INLINE(op) \ + _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op)) + #endif /* Bit flags for _gc_prev */ diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1aea91abc5d69f..dd6b0762370c92 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. extern void _PyMem_FreeDelayed(void *ptr); +// Enqueue an object to be freed possibly after some delay +extern void _PyObject_FreeDelayed(void *ptr); + // Periodically process delayed free requests. extern void _PyMem_ProcessDelayed(PyThreadState *tstate); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index b2a2286ef22b66..73378816a2aa3b 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -957,7 +957,7 @@ _PyMem_Strdup(const char *str) // A pointer to be freed once the QSBR read sequence reaches qsbr_goal. struct _mem_work_item { - void *ptr; + uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free uint64_t qsbr_goal; }; @@ -971,16 +971,27 @@ struct _mem_work_chunk { struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; }; -void -_PyMem_FreeDelayed(void *ptr) +static void +free_work_item(uintptr_t ptr) +{ + if (ptr & 0x01) { + PyObject_Free((char *)(ptr - 1)); + } + else { + PyMem_Free((void *)ptr); + } +} + +static void +free_delayed(uintptr_t ptr) { #ifndef Py_GIL_DISABLED - PyMem_Free(ptr); + free_work_item(ptr); #else if (_PyRuntime.stoptheworld.world_stopped) { // Free immediately if the world is stopped, including during // interpreter shutdown. - PyMem_Free(ptr); + free_work_item(ptr); return; } @@ -1007,7 +1018,7 @@ _PyMem_FreeDelayed(void *ptr) if (buf == NULL) { // failed to allocate a buffer, free immediately _PyEval_StopTheWorld(tstate->base.interp); - PyMem_Free(ptr); + free_work_item(ptr); _PyEval_StartTheWorld(tstate->base.interp); return; } @@ -1024,6 +1035,20 @@ _PyMem_FreeDelayed(void *ptr) #endif } +void +_PyMem_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed((uintptr_t)ptr); +} + +void +_PyObject_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed(((uintptr_t)ptr)|0x01); +} + static struct _mem_work_chunk * work_queue_first(struct llist_node *head) { @@ -1043,7 +1068,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, return; } - PyMem_Free(item->ptr); + free_work_item(item->ptr); buf->rd_idx++; } @@ -1130,7 +1155,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp) // Free the remaining items immediately. There should be no other // threads accessing the memory at this point during shutdown. struct _mem_work_item *item = &buf->array[buf->rd_idx]; - PyMem_Free(item->ptr); + free_work_item(item->ptr); buf->rd_idx++; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 18893c6c391fff..5dceb5c2c3065e 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1692,6 +1692,7 @@ PyObject_GC_Del(void *op) { size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -1704,8 +1705,13 @@ PyObject_GC_Del(void *op) } record_deallocation(_PyThreadState_GET()); - - PyObject_Free(((char *)op)-presize); + PyObject *self = (PyObject *)op; + if (_PyObject_GC_IS_SHARED_INLINE(self)) { + _PyObject_FreeDelayed(((char *)op)-presize); + } + else { + PyObject_Free(((char *)op)-presize); + } } int