python · vstinner · Aug 31, 2023 · Aug 29, 2023 · Aug 29, 2023 · Aug 30, 2023
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
@@ -528,28 +528,6 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field.
    This field is inherited by subtypes.
 
 
-.. c:member:: PyObject* PyObject._ob_next
-             PyObject* PyObject._ob_prev
-
-   These fields are only present when the macro ``Py_TRACE_REFS`` is defined
-   (see the :option:`configure --with-trace-refs option <--with-trace-refs>`).
-
-   Their initialization to ``NULL`` is taken care of by the
-   ``PyObject_HEAD_INIT`` macro.  For :ref:`statically allocated objects
-   <static-types>`, these fields always remain ``NULL``.  For :ref:`dynamically
-   allocated objects <heap-types>`, these two fields are used to link the
-   object into a doubly linked list of *all* live objects on the heap.
-
-   This could be used for various debugging purposes; currently the only uses
-   are the :func:`sys.getobjects` function and to print the objects that are
-   still alive at the end of a run when the environment variable
-   :envvar:`PYTHONDUMPREFS` is set.
-
-   **Inheritance:**
-
-   These fields are not inherited by subtypes.
-
-
 PyVarObject Slots
 -----------------
 

@@ -408,8 +408,7 @@ See also the :ref:`Python Development Mode <devmode>` and the
 .. versionchanged:: 3.8
    Release builds and debug builds are now ABI compatible: defining the
    ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the
-   :option:`--with-trace-refs` option), which introduces the only ABI
-   incompatibility.
+   :option:`--with-trace-refs` option).
 
 
 Debug options
@@ -430,8 +429,14 @@ Debug options
    * Add :func:`sys.getobjects` function.
    * Add :envvar:`PYTHONDUMPREFS` environment variable.
 
-   This build is not ABI compatible with release build (default build) or debug
-   build (``Py_DEBUG`` and ``Py_REF_DEBUG`` macros).
+   The :envvar:`PYTHONDUMPREFS` environment variable can be used to dump
+   objects and reference counts still alive at Python exit.
+
+   :ref:`Statically allocated objects <static-types>` are not traced.
+
+   .. versionchanged:: 3.13
+      This build is now ABI compatible with release build and :ref:`debug build
+      <debug-build>`.
 
    .. versionadded:: 3.8
 

diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
@@ -828,6 +828,11 @@ Build Changes
 * SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module.
   (Contributed by Erlend Aasland in :gh:`105875`.)
 
+* Python built with :file:`configure` :option:`--with-trace-refs` (tracing
+  references) is now ABI compatible with Python release build and
+  :ref:`debug build <debug-build>`.
+  (Contributed by Victor Stinner in :gh:`108634`.)
+
 
 C API Changes
 =============
@@ -900,6 +905,10 @@ New Features
   (with an underscore prefix).
   (Contributed by Victor Stinner in :gh:`108014`.)
 
+* Python built with :file:`configure` :option:`--with-trace-refs` (tracing
+  references) now supports the :ref:`Limited API <limited-c-api>`.
+  (Contributed by Victor Stinner in :gh:`108634`.)
+
 Porting to Python 3.13
 ----------------------
 

diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
@@ -276,10 +276,8 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(after_in_child)
         STRUCT_FOR_ID(after_in_parent)
         STRUCT_FOR_ID(aggregate_class)
-        STRUCT_FOR_ID(alias)
         STRUCT_FOR_ID(append)
         STRUCT_FOR_ID(argdefs)
-        STRUCT_FOR_ID(args)
         STRUCT_FOR_ID(arguments)
         STRUCT_FOR_ID(argv)
         STRUCT_FOR_ID(as_integer_ratio)
@@ -403,8 +401,6 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(errors)
         STRUCT_FOR_ID(event)
         STRUCT_FOR_ID(eventmask)
-        STRUCT_FOR_ID(exc_type)
-        STRUCT_FOR_ID(exc_value)
         STRUCT_FOR_ID(excepthook)
         STRUCT_FOR_ID(exception)
         STRUCT_FOR_ID(existing_file_name)
@@ -720,7 +716,6 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(timetuple)
         STRUCT_FOR_ID(top)
         STRUCT_FOR_ID(trace_callback)
-        STRUCT_FOR_ID(traceback)
         STRUCT_FOR_ID(trailers)
         STRUCT_FOR_ID(translate)
         STRUCT_FOR_ID(true)

diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
@@ -49,7 +49,6 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *);
    backwards compatible solution */
 #define _PyObject_HEAD_INIT(type)         \
     {                                     \
-        _PyObject_EXTRA_INIT              \
         .ob_refcnt = _Py_IMMORTAL_REFCNT, \
         .ob_type = (type)                 \
     },
@@ -178,6 +177,8 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
 extern void _PyType_InitCache(PyInterpreterState *interp);
 
 extern void _PyObject_InitState(PyInterpreterState *interp);
+extern void _PyObject_FiniState(PyInterpreterState *interp);
+extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);
 
 /* Inline functions trading binary compatibility for speed:
    _PyObject_Init() is the fast version of PyObject_Init(), and
@@ -296,7 +297,7 @@ extern void _PyDebug_PrintTotalRefs(void);
 #endif
 
 #ifdef Py_TRACE_REFS
-extern void _Py_AddToAllObjects(PyObject *op, int force);
+extern void _Py_AddToAllObjects(PyObject *op);
 extern void _Py_PrintReferences(PyInterpreterState *, FILE *);
 extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *);
 #endif

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_hashtable.h"     // _Py_hashtable_t
+
 struct _py_object_runtime_state {
 #ifdef Py_REF_DEBUG
     Py_ssize_t interpreter_leaks;
@@ -20,11 +22,10 @@ struct _py_object_state {
     Py_ssize_t reftotal;
 #endif
 #ifdef Py_TRACE_REFS
-    /* Head of circular doubly-linked list of all objects.  These are linked
-     * together via the _ob_prev and _ob_next members of a PyObject, which
-     * exist only in a Py_TRACE_REFS build.
-     */
-    PyObject refchain;
+    // Hash table storing all objects. The key is the object pointer
+    // (PyObject*) and the value is always the number 1 (as uintptr_t).
+    // See _PyRefchain_IsTraced() and _PyRefchain_Trace() functions.
+    _Py_hashtable_t *refchain;
 #endif
     int _not_used;
 };

@@ -192,7 +192,7 @@ extern PyTypeObject _PyExc_MemoryError;
 #ifdef Py_TRACE_REFS
 # define _py_object_state_INIT(INTERP) \
     { \
-        .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \
+        .refchain = NULL, \
     }
 #else
 # define _py_object_state_INIT(INTERP) \

diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h
diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h
@@ -111,14 +111,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def);
 #define PYTHON_ABI_VERSION 3
 #define PYTHON_ABI_STRING "3"
 
-#ifdef Py_TRACE_REFS
- /* When we are tracing reference counts, rename module creation functions so
-    modules compiled with incompatible settings will generate a
-    link-time error. */
- #define PyModule_Create2 PyModule_Create2TraceRefs
- #define PyModule_FromDefAndSpec2 PyModule_FromDefAndSpec2TraceRefs
-#endif
-
 PyAPI_FUNC(PyObject *) PyModule_Create2(PyModuleDef*, int apiver);
 
 #ifdef Py_LIMITED_API

diff --git a/Include/object.h b/Include/object.h
@@ -58,23 +58,6 @@ whose size is determined when the object is allocated.
 #  define Py_REF_DEBUG
 #endif
 
-#if defined(Py_LIMITED_API) && defined(Py_TRACE_REFS)
-#  error Py_LIMITED_API is incompatible with Py_TRACE_REFS
-#endif
-
-#ifdef Py_TRACE_REFS
-/* Define pointers to support a doubly-linked list of all live heap objects. */
-#define _PyObject_HEAD_EXTRA            \
-    PyObject *_ob_next;           \
-    PyObject *_ob_prev;
-
-#define _PyObject_EXTRA_INIT _Py_NULL, _Py_NULL,
-
-#else
-#  define _PyObject_HEAD_EXTRA
-#  define _PyObject_EXTRA_INIT
-#endif
-
 /* PyObject_HEAD defines the initial segment of every PyObject. */
 #define PyObject_HEAD                   PyObject ob_base;
 
@@ -130,14 +113,12 @@ check by comparing the reference count field to the immortality reference count.
 #ifdef Py_BUILD_CORE
 #define PyObject_HEAD_INIT(type)    \
     {                               \
-        _PyObject_EXTRA_INIT        \
         { _Py_IMMORTAL_REFCNT },    \
         (type)                      \
     },
 #else
 #define PyObject_HEAD_INIT(type) \
     {                            \
-        _PyObject_EXTRA_INIT     \
         { 1 },                   \
         (type)                   \
     },
@@ -164,8 +145,6 @@ check by comparing the reference count field to the immortality reference count.
  * in addition, be cast to PyVarObject*.
  */
 struct _object {
-    _PyObject_HEAD_EXTRA
-
 #if (defined(__GNUC__) || defined(__clang__)) \
         && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L)
     // On C99 and older, anonymous union is a GCC and clang extension

diff --git a/Include/pyport.h b/Include/pyport.h
@@ -684,12 +684,6 @@ extern char * _getpty(int *, int, mode_t, int);
 #  endif
 #endif
 
-/* Check that ALT_SOABI is consistent with Py_TRACE_REFS:
-   ./configure --with-trace-refs should must be used to define Py_TRACE_REFS */
-#if defined(ALT_SOABI) && defined(Py_TRACE_REFS)
-#  error "Py_TRACE_REFS ABI is not compatible with release and debug ABI"
-#endif
-
 #if defined(__ANDROID__) || defined(__VXWORKS__)
    // Use UTF-8 as the locale encoding, ignore the LC_CTYPE locale.
    // See _Py_GetLocaleEncoding(), PyUnicode_DecodeLocale()

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -779,9 +779,6 @@ def python_is_optimized():
 
 _header = 'nP'
 _align = '0n'
-if hasattr(sys, "getobjects"):
-    _header = '2P' + _header
-    _align = '0P'
 _vheader = _header + 'n'
 
 def calcobjsize(fmt):
@@ -2469,3 +2466,5 @@ def adjust_int_max_str_digits(max_digits):
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
                                 'skipped on s390x')
+
+Py_TRACE_REFS = hasattr(sys, 'getobjects')
diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py
@@ -112,6 +112,9 @@ def test_pyobject_forbidden_bytes_is_freed(self):
     def test_pyobject_freed_is_freed(self):
         self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
 
+    # Python built with Py_TRACE_REFS fail with a fatal error in
+    # _PyRefchain_Trace() on memory allocation error.
+    @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build')
     def test_set_nomemory(self):
         code = """if 1:
             import _testcapi

@@ -1484,6 +1484,9 @@ def recurse_in_body_and_except():
 
 
     @cpython_only
+    # Python built with Py_TRACE_REFS fail with a fatal error in
+    # _PyRefchain_Trace() on memory allocation error.
+    @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build')
     def test_recursion_normalizing_with_no_memory(self):
         # Issue #30697. Test that in the abort that occurs when there is no
         # memory left and the size of the Python frames stack is greater than
@@ -1652,6 +1655,9 @@ def test_unhandled(self):
                 self.assertTrue(report.endswith("\n"))
 
     @cpython_only
+    # Python built with Py_TRACE_REFS fail with a fatal error in
+    # _PyRefchain_Trace() on memory allocation error.
+    @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build')
     def test_memory_error_in_PyErr_PrintEx(self):
         code = """if 1:
             import _testcapi