Skip to content

Commit 626bff8

Browse files
vstinnerdavidmalcolm
vstinner
and
davidmalcolm
authored
bpo-9263: Dump Python object on GC assertion failure (GH-10062)
Changes: * Add _PyObject_AssertFailed() function. * Add _PyObject_ASSERT() and _PyObject_ASSERT_WITH_MSG() macros. * gc_decref(): replace assert() with _PyObject_ASSERT_WITH_MSG() to dump the faulty object if the assertion fails. _PyObject_AssertFailed() calls: * _PyMem_DumpTraceback(): try to log the traceback where the object memory has been allocated if tracemalloc is enabled. * _PyObject_Dump(): log repr(obj). * Py_FatalError(): log the current Python traceback. _PyObject_AssertFailed() uses _PyObject_IsFreed() heuristic to check if the object memory has been freed by a debug hook on Python memory allocators. Initial patch written by David Malcolm. Co-Authored-By: David Malcolm <dmalcolm@redhat.com>
1 parent 18618e6 commit 626bff8

File tree

6 files changed

+179
-13
lines changed

6 files changed

+179
-13
lines changed

‎Include/object.h

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1105,6 +1105,53 @@ PyAPI_FUNC(void)
11051105
_PyObject_DebugTypeStats(FILE *out);
11061106
#endif /* ifndef Py_LIMITED_API */
11071107

1108+
1109+
#ifndef Py_LIMITED_API
1110+
/* Define a pair of assertion macros:
1111+
_PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT().
1112+
1113+
These work like the regular C assert(), in that they will abort the
1114+
process with a message on stderr if the given condition fails to hold,
1115+
but compile away to nothing if NDEBUG is defined.
1116+
1117+
However, before aborting, Python will also try to call _PyObject_Dump() on
1118+
the given object. This may be of use when investigating bugs in which a
1119+
particular object is corrupt (e.g. buggy a tp_visit method in an extension
1120+
module breaking the garbage collector), to help locate the broken objects.
1121+
1122+
The WITH_MSG variant allows you to supply an additional message that Python
1123+
will attempt to print to stderr, after the object dump. */
1124+
#ifdef NDEBUG
1125+
/* No debugging: compile away the assertions: */
1126+
# define _PyObject_ASSERT_WITH_MSG(obj, expr, msg) ((void)0)
1127+
#else
1128+
/* With debugging: generate checks: */
1129+
# define _PyObject_ASSERT_WITH_MSG(obj, expr, msg) \
1130+
((expr) \
1131+
? (void)(0) \
1132+
: _PyObject_AssertFailed((obj), \
1133+
(msg), \
1134+
Py_STRINGIFY(expr), \
1135+
__FILE__, \
1136+
__LINE__, \
1137+
__func__))
1138+
#endif
1139+
1140+
#define _PyObject_ASSERT(obj, expr) _PyObject_ASSERT_WITH_MSG(obj, expr, NULL)
1141+
1142+
/* Declare and define _PyObject_AssertFailed() even when NDEBUG is defined,
1143+
to avoid causing compiler/linker errors when building extensions without
1144+
NDEBUG against a Python built with NDEBUG defined. */
1145+
PyAPI_FUNC(void) _PyObject_AssertFailed(
1146+
PyObject *obj,
1147+
const char *msg,
1148+
const char *expr,
1149+
const char *file,
1150+
int line,
1151+
const char *function);
1152+
#endif /* ifndef Py_LIMITED_API */
1153+
1154+
11081155
#ifdef __cplusplus
11091156
}
11101157
#endif

‎Include/pymem.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
196196
197197
The function does nothing if Python is not compiled is debug mode. */
198198
PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
199-
#endif
199+
#endif /* Py_LIMITED_API */
200200

201201
#ifdef Py_BUILD_CORE
202202
/* Set the memory allocator of the specified domain to the default.

‎Lib/test/test_gc.py

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
import unittest
22
from test.support import (verbose, refcount_test, run_unittest,
33
strip_python_stderr, cpython_only, start_threads,
4-
temp_dir, requires_type_collecting, TESTFN, unlink)
4+
temp_dir, requires_type_collecting, TESTFN, unlink,
5+
import_module)
56
from test.support.script_helper import assert_python_ok, make_script
67

8+
import gc
79
import sys
10+
import sysconfig
11+
import textwrap
12+
import threading
813
import time
9-
import gc
1014
import weakref
11-
import threading
1215

1316
try:
1417
from _testcapi import with_tp_del
@@ -62,6 +65,14 @@ def __init__(self, partner=None):
6265
def __tp_del__(self):
6366
pass
6467

68+
if sysconfig.get_config_vars().get('PY_CFLAGS', ''):
69+
BUILD_WITH_NDEBUG = ('-DNDEBUG' in sysconfig.get_config_vars()['PY_CFLAGS'])
70+
else:
71+
# Usually, sys.gettotalrefcount() is only present if Python has been
72+
# compiled in debug mode. If it's missing, expect that Python has
73+
# been released in release mode: with NDEBUG defined.
74+
BUILD_WITH_NDEBUG = (not hasattr(sys, 'gettotalrefcount'))
75+
6576
### Tests
6677
###############################################################################
6778

@@ -878,6 +889,58 @@ def test_collect_garbage(self):
878889
self.assertEqual(len(gc.garbage), 0)
879890

880891

892+
@unittest.skipIf(BUILD_WITH_NDEBUG,
893+
'built with -NDEBUG')
894+
def test_refcount_errors(self):
895+
self.preclean()
896+
# Verify the "handling" of objects with broken refcounts
897+
898+
# Skip the test if ctypes is not available
899+
import_module("ctypes")
900+
901+
import subprocess
902+
code = textwrap.dedent('''
903+
from test.support import gc_collect, SuppressCrashReport
904+
905+
a = [1, 2, 3]
906+
b = [a]
907+
908+
# Avoid coredump when Py_FatalError() calls abort()
909+
SuppressCrashReport().__enter__()
910+
911+
# Simulate the refcount of "a" being too low (compared to the
912+
# references held on it by live data), but keeping it above zero
913+
# (to avoid deallocating it):
914+
import ctypes
915+
ctypes.pythonapi.Py_DecRef(ctypes.py_object(a))
916+
917+
# The garbage collector should now have a fatal error
918+
# when it reaches the broken object
919+
gc_collect()
920+
''')
921+
p = subprocess.Popen([sys.executable, "-c", code],
922+
stdout=subprocess.PIPE,
923+
stderr=subprocess.PIPE)
924+
stdout, stderr = p.communicate()
925+
p.stdout.close()
926+
p.stderr.close()
927+
# Verify that stderr has a useful error message:
928+
self.assertRegex(stderr,
929+
br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.')
930+
self.assertRegex(stderr,
931+
br'refcount is too small')
932+
self.assertRegex(stderr,
933+
br'object : \[1, 2, 3\]')
934+
self.assertRegex(stderr,
935+
br'type : list')
936+
self.assertRegex(stderr,
937+
br'refcount: 1')
938+
# "address : 0x7fb5062efc18"
939+
# "address : 7FB5062EFC18"
940+
self.assertRegex(stderr,
941+
br'address : [0-9a-fA-Fx]+')
942+
943+
881944
class GCTogglingTests(unittest.TestCase):
882945
def setUp(self):
883946
gc.enable()

‎Modules/_tracemalloc.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,10 +1436,12 @@ _tracemalloc__get_object_traceback(PyObject *module, PyObject *obj)
14361436
traceback_t *traceback;
14371437

14381438
type = Py_TYPE(obj);
1439-
if (PyType_IS_GC(type))
1439+
if (PyType_IS_GC(type)) {
14401440
ptr = (void *)((char *)obj - sizeof(PyGC_Head));
1441-
else
1441+
}
1442+
else {
14421443
ptr = (void *)obj;
1444+
}
14431445

14441446
traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (uintptr_t)ptr);
14451447
if (traceback == NULL)

‎Modules/gcmodule.c

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ module gc
6262
// most gc_list_* functions for it.
6363
#define NEXT_MASK_UNREACHABLE (1)
6464

65+
/* Get an object's GC head */
66+
#define AS_GC(o) ((PyGC_Head *)(o)-1)
67+
68+
/* Get the object given the GC head */
69+
#define FROM_GC(g) ((PyObject *)(((PyGC_Head *)g)+1))
70+
6571
static inline int
6672
gc_is_collecting(PyGC_Head *g)
6773
{
@@ -98,16 +104,12 @@ gc_reset_refs(PyGC_Head *g, Py_ssize_t refs)
98104
static inline void
99105
gc_decref(PyGC_Head *g)
100106
{
101-
assert(gc_get_refs(g) > 0);
107+
_PyObject_ASSERT_WITH_MSG(FROM_GC(g),
108+
gc_get_refs(g) > 0,
109+
"refcount is too small");
102110
g->_gc_prev -= 1 << _PyGC_PREV_SHIFT;
103111
}
104112

105-
/* Get an object's GC head */
106-
#define AS_GC(o) ((PyGC_Head *)(o)-1)
107-
108-
/* Get the object given the GC head */
109-
#define FROM_GC(g) ((PyObject *)(((PyGC_Head *)g)+1))
110-
111113
/* Python string to use if unhandled exception occurs */
112114
static PyObject *gc_str = NULL;
113115

‎Objects/object.c

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
extern "C" {
1111
#endif
1212

13+
/* Defined in tracemalloc.c */
14+
extern void _PyMem_DumpTraceback(int fd, const void *ptr);
15+
1316
_Py_IDENTIFIER(Py_Repr);
1417
_Py_IDENTIFIER(__bytes__);
1518
_Py_IDENTIFIER(__dir__);
@@ -2212,6 +2215,55 @@ _PyTrash_thread_destroy_chain(void)
22122215
--tstate->trash_delete_nesting;
22132216
}
22142217

2218+
2219+
void
2220+
_PyObject_AssertFailed(PyObject *obj, const char *msg, const char *expr,
2221+
const char *file, int line, const char *function)
2222+
{
2223+
fprintf(stderr,
2224+
"%s:%d: %s: Assertion \"%s\" failed",
2225+
file, line, function, expr);
2226+
fflush(stderr);
2227+
2228+
if (msg) {
2229+
fprintf(stderr, "; %s.\n", msg);
2230+
}
2231+
else {
2232+
fprintf(stderr, ".\n");
2233+
}
2234+
fflush(stderr);
2235+
2236+
if (obj == NULL) {
2237+
fprintf(stderr, "<NULL object>\n");
2238+
}
2239+
else if (_PyObject_IsFreed(obj)) {
2240+
/* It seems like the object memory has been freed:
2241+
don't access it to prevent a segmentation fault. */
2242+
fprintf(stderr, "<Freed object>\n");
2243+
}
2244+
else {
2245+
/* Diplay the traceback where the object has been allocated.
2246+
Do it before dumping repr(obj), since repr() is more likely
2247+
to crash than dumping the traceback. */
2248+
void *ptr;
2249+
PyTypeObject *type = Py_TYPE(obj);
2250+
if (PyType_IS_GC(type)) {
2251+
ptr = (void *)((char *)obj - sizeof(PyGC_Head));
2252+
}
2253+
else {
2254+
ptr = (void *)obj;
2255+
}
2256+
_PyMem_DumpTraceback(fileno(stderr), ptr);
2257+
2258+
/* This might succeed or fail, but we're about to abort, so at least
2259+
try to provide any extra info we can: */
2260+
_PyObject_Dump(obj);
2261+
}
2262+
fflush(stderr);
2263+
2264+
Py_FatalError("_PyObject_AssertFailed");
2265+
}
2266+
22152267
#ifndef Py_TRACE_REFS
22162268
/* For Py_LIMITED_API, we need an out-of-line version of _Py_Dealloc.
22172269
Define this here, so we can undefine the macro. */

0 commit comments

Comments
 (0)