From b982284ef39914632269e69d7016e48132909567 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 28 Oct 2024 13:17:24 +0000 Subject: [PATCH 1/8] gh-126076: Account for relocated objects in tracemalloc --- Include/internal/pycore_object.h | 15 ++++++++++----- ...4-10-28-13-18-16.gh-issue-126076.MebZuS.rst | 3 +++ Objects/bytesobject.c | 1 + Objects/object.c | 18 +++--------------- Objects/tupleobject.c | 1 + Objects/unicodeobject.c | 1 + Python/ceval.c | 6 +----- 7 files changed, 20 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index c7af720b1ce43d..1768c12db1675c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -94,6 +94,15 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( #define _Py_FatalRefcountError(message) \ _Py_FatalRefcountErrorFunc(__func__, (message)) +#define _PyReftracerTrack(obj, operation) \ + do { \ + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \ + if (tracer->tracer_func != NULL) { \ + void* data = tracer->tracer_data; \ + tracer->tracer_func(obj, (operation), data); \ + } \ + } while(0) + #ifdef Py_REF_DEBUG /* The symbol is only exposed in the API for the sake of extensions @@ -208,11 +217,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif - struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; - if (tracer->tracer_func != NULL) { - void* data = tracer->tracer_data; - tracer->tracer_func(op, PyRefTracer_DESTROY, data); - } + _PyReftracerTrack(op, PyRefTracer_DESTROY); destruct(op); } } diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst new file mode 100644 index 00000000000000..b9b9c6d13f4d9b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst @@ -0,0 +1,3 @@ +Relocated objects such as ``tuple``, ``bytes`` and ``unicode`` objects are +properly tracked by :mod:`tracemalloc` and its associated hooks. Patch by +Pablo Galindo diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index dcc1aba76abbed..b007dd8430b861 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3195,6 +3195,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif + _PyReftracerTrack(v, PyRefTracer_DESTROY); *pv = (PyObject *) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { diff --git a/Objects/object.c b/Objects/object.c index 7cc74a8dc0d8eb..0acbdeb298a144 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2457,11 +2457,7 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif - struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; - if (tracer->tracer_func != NULL) { - void* data = tracer->tracer_data; - tracer->tracer_func(op, PyRefTracer_CREATE, data); - } + _PyReftracerTrack(op, PyRefTracer_CREATE); } void @@ -2525,10 +2521,7 @@ _Py_ResurrectReference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif - if (_PyRuntime.ref_tracer.tracer_func != NULL) { - void* data = _PyRuntime.ref_tracer.tracer_data; - _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); - } + _PyReftracerTrack(op, PyRefTracer_CREATE); } @@ -2917,12 +2910,7 @@ _Py_Dealloc(PyObject *op) // Make sure that type->tp_name remains valid Py_INCREF(type); #endif - - struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; - if (tracer->tracer_func != NULL) { - void* data = tracer->tracer_data; - tracer->tracer_func(op, PyRefTracer_DESTROY, data); - } + _PyReftracerTrack(op, PyRefTracer_DESTROY); #ifdef Py_TRACE_REFS _Py_ForgetReference(op); diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index f3132e0933ac30..2f2fb87c472d74 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -961,6 +961,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) #ifdef Py_TRACE_REFS _Py_ForgetReference((PyObject *) v); #endif + _PyReftracerTrack(v, PyRefTracer_DESTROY); /* DECREF items deleted by shrinkage */ for (i = newsize; i < oldsize; i++) { Py_CLEAR(v->ob_item[i]); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 9cd9781e412524..562e3312b63e9a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1129,6 +1129,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length) #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); #endif + _PyReftracerTrack(unicode, PyRefTracer_DESTROY); new_unicode = (PyObject *)PyObject_Realloc(unicode, new_size); if (new_unicode == NULL) { diff --git a/Python/ceval.c b/Python/ceval.c index beee5325cd6259..deb45d50ecb9ec 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -99,11 +99,7 @@ } \ _Py_DECREF_STAT_INC(); \ if (--op->ob_refcnt == 0) { \ - struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \ - if (tracer->tracer_func != NULL) { \ - void* data = tracer->tracer_data; \ - tracer->tracer_func(op, PyRefTracer_DESTROY, data); \ - } \ + _PyReftracerTrack(op, PyRefTracer_DESTROY); \ destructor d = (destructor)(dealloc); \ d(op); \ } \ From ce3cb856a0bafc766aaebc34b6360bf46ba8796b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 28 Oct 2024 17:32:14 +0000 Subject: [PATCH 2/8] Update Objects/tupleobject.c Co-authored-by: Victor Stinner --- Objects/tupleobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 2f2fb87c472d74..d1a07d790d5c10 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -961,7 +961,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) #ifdef Py_TRACE_REFS _Py_ForgetReference((PyObject *) v); #endif - _PyReftracerTrack(v, PyRefTracer_DESTROY); + _PyReftracerTrack((PyObject *)v, PyRefTracer_DESTROY); /* DECREF items deleted by shrinkage */ for (i = newsize; i < oldsize; i++) { Py_CLEAR(v->ob_item[i]); From a78ff5f43b80310521191815a3fdb1689dd04beb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 29 Oct 2024 15:59:04 +0000 Subject: [PATCH 3/8] Update pycore_object.h Co-authored-by: Victor Stinner --- Include/internal/pycore_object.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 1768c12db1675c..f32da77d425830 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -102,7 +102,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( tracer->tracer_func(obj, (operation), data); \ } \ } while(0) - #ifdef Py_REF_DEBUG /* The symbol is only exposed in the API for the sake of extensions From a97000facb83f44bbe2ddcaea88b4cc2c190e3c4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 29 Oct 2024 15:59:16 +0000 Subject: [PATCH 4/8] Update pycore_object.h Co-authored-by: Victor Stinner --- Include/internal/pycore_object.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index f32da77d425830..27c7b8def901db 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -99,7 +99,7 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \ if (tracer->tracer_func != NULL) { \ void* data = tracer->tracer_data; \ - tracer->tracer_func(obj, (operation), data); \ + tracer->tracer_func((obj), (operation), data); \ } \ } while(0) From 77d980b29d5dda8c733467e3d5923250f507d69f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 29 Oct 2024 22:06:49 +0000 Subject: [PATCH 5/8] Address review comments --- Objects/object.c | 3 +-- Objects/tupleobject.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Objects/object.c b/Objects/object.c index 0acbdeb298a144..c68003ce740098 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2521,7 +2521,6 @@ _Py_ResurrectReference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif - _PyReftracerTrack(op, PyRefTracer_CREATE); } @@ -2910,11 +2909,11 @@ _Py_Dealloc(PyObject *op) // Make sure that type->tp_name remains valid Py_INCREF(type); #endif - _PyReftracerTrack(op, PyRefTracer_DESTROY); #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif + _PyReftracerTrack(op, PyRefTracer_DESTROY); (*dealloc)(op); #ifdef Py_DEBUG diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index d1a07d790d5c10..ba0b06124ccba1 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -961,11 +961,11 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) #ifdef Py_TRACE_REFS _Py_ForgetReference((PyObject *) v); #endif - _PyReftracerTrack((PyObject *)v, PyRefTracer_DESTROY); /* DECREF items deleted by shrinkage */ for (i = newsize; i < oldsize; i++) { Py_CLEAR(v->ob_item[i]); } + _PyReftracerTrack((PyObject *)v, PyRefTracer_DESTROY); sv = PyObject_GC_Resize(PyTupleObject, v, newsize); if (sv == NULL) { *pv = NULL; From 9591820ad534b7eb5e38ffad0408a4cb0b977204 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 30 Oct 2024 16:11:01 +0000 Subject: [PATCH 6/8] Update 2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst index b9b9c6d13f4d9b..2e8ee966c5a707 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst @@ -1,3 +1,3 @@ Relocated objects such as ``tuple``, ``bytes`` and ``unicode`` objects are properly tracked by :mod:`tracemalloc` and its associated hooks. Patch by -Pablo Galindo +Pablo Galindo. From a3bd045803469652cf5134a4ff1243a9eccb310f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 19 Nov 2024 10:08:36 +0000 Subject: [PATCH 7/8] Update Include/internal/pycore_object.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Include/internal/pycore_object.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 27c7b8def901db..cafc02f892499c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -98,7 +98,7 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( do { \ struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \ if (tracer->tracer_func != NULL) { \ - void* data = tracer->tracer_data; \ + void *data = tracer->tracer_data; \ tracer->tracer_func((obj), (operation), data); \ } \ } while(0) From 18b01399cff1e99f74700c9c5c051714ae271e24 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 19 Nov 2024 10:09:04 +0000 Subject: [PATCH 8/8] Update Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst --- .../2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst index 2e8ee966c5a707..5108ca52b202c4 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-28-13-18-16.gh-issue-126076.MebZuS.rst @@ -1,3 +1,3 @@ -Relocated objects such as ``tuple``, ``bytes`` and ``unicode`` objects are +Relocated objects such as ``tuple``, ``bytes`` and ``str`` objects are properly tracked by :mod:`tracemalloc` and its associated hooks. Patch by Pablo Galindo.