From 7ae71e9e38f725348f1ab5c6cbebe021bfedb892 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 5 Jan 2024 21:18:49 +0000 Subject: [PATCH 1/4] gh-113753: Clear finalized bit when allocating PyAsyncGenASend from free-list --- Include/internal/pycore_gc.h | 4 ++++ Objects/genobject.c | 2 ++ 2 files changed, 6 insertions(+) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 2a79c403803ed1..84bbadc390e0e3 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -120,6 +120,10 @@ static inline void _PyGC_SET_FINALIZED(PyObject *op) { PyGC_Head *gc = _Py_AS_GC(op); _PyGCHead_SET_FINALIZED(gc); } +static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { + PyGC_Head *gc = _Py_AS_GC(op); + gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; +} /* GC runtime state */ diff --git a/Objects/genobject.c b/Objects/genobject.c index 9614713883741c..237ad515e91077 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -1913,6 +1914,7 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) if (state->asend_numfree) { state->asend_numfree--; o = state->asend_freelist[state->asend_numfree]; + _PyGC_CLEAR_FINALIZED((PyObject *)o); _Py_NewReference((PyObject *)o); } else From fba7b02a7f1ff70359f19a3be8f3d3696e7cf801 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 5 Jan 2024 21:28:57 +0000 Subject: [PATCH 2/4] Add NEWS --- .../2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst new file mode 100644 index 00000000000000..32cf2cb2a4ae56 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst @@ -0,0 +1,2 @@ +Fix an issue where the finalizer of ``PyAsyncGenASend`` objects might not be +called if they were allocated from a free list. From fe24eecab8d6701fd3b118aa5da786d7ef1b8805 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 10 Jan 2024 15:07:06 +0000 Subject: [PATCH 3/4] Move _PyGC_CLEAR_FINALIZED call --- Objects/genobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 237ad515e91077..c430fbfae94eaa 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1914,7 +1914,6 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) if (state->asend_numfree) { state->asend_numfree--; o = state->asend_freelist[state->asend_numfree]; - _PyGC_CLEAR_FINALIZED((PyObject *)o); _Py_NewReference((PyObject *)o); } else @@ -1953,6 +1952,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) #endif if (state->value_numfree < _PyAsyncGen_MAXFREELIST) { assert(_PyAsyncGenWrappedValue_CheckExact(o)); + _PyGC_CLEAR_FINALIZED((PyObject *)o); state->value_freelist[state->value_numfree++] = o; OBJECT_STAT_INC(to_freelist); } From 614ac7b66b9879d038412a97c7317555c9c74bf7 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 10 Jan 2024 16:02:53 +0000 Subject: [PATCH 4/4] Fix location of _PyGC_CLEAR_FINALIZED call and add test --- Lib/test/test_asyncgen.py | 8 ++++++++ Objects/genobject.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index a49630112af510..7fa0a85100a581 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1701,6 +1701,14 @@ def test_asend(self): async def gen(): yield 1 + # gh-113753: asend objects allocated from a free-list should warn. + # Ensure there is a finalized 'asend' object ready to be reused. + try: + g = gen() + g.asend(None).send(None) + except StopIteration: + pass + msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited" with self.assertWarnsRegex(RuntimeWarning, msg): g = gen() diff --git a/Objects/genobject.c b/Objects/genobject.c index c430fbfae94eaa..f03919c75d70a5 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1740,6 +1740,7 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) #endif if (state->asend_numfree < _PyAsyncGen_MAXFREELIST) { assert(PyAsyncGenASend_CheckExact(o)); + _PyGC_CLEAR_FINALIZED((PyObject *)o); state->asend_freelist[state->asend_numfree++] = o; } else @@ -1952,7 +1953,6 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) #endif if (state->value_numfree < _PyAsyncGen_MAXFREELIST) { assert(_PyAsyncGenWrappedValue_CheckExact(o)); - _PyGC_CLEAR_FINALIZED((PyObject *)o); state->value_freelist[state->value_numfree++] = o; OBJECT_STAT_INC(to_freelist); }