From ede08ee0d4f60ed9992f06526cc6ddb433be670f Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 2 Feb 2024 15:28:55 +0000 Subject: [PATCH 1/6] gh-115103: Implement delayed memory reclamation (QSBR) This adds a safe memory reclamation scheme based on FreeBSD's "GUS" and quiescent state based reclamation (QSBR). The API provides a mechanism for callers to detect when it is safe to free memory that may be concurrently accessed by readers. --- Doc/license.rst | 32 +++ Include/cpython/pyatomic.h | 6 + Include/cpython/pyatomic_gcc.h | 8 + Include/cpython/pyatomic_msc.h | 26 +++ Include/cpython/pyatomic_std.h | 16 ++ Include/internal/pycore_interp.h | 2 + Include/internal/pycore_qsbr.h | 131 +++++++++++ Include/internal/pycore_runtime_init.h | 4 + Include/internal/pycore_tstate.h | 2 + Makefile.pre.in | 2 + Modules/posixmodule.c | 4 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 2 + PCbuild/pythoncore.vcxproj.filters | 6 + Python/ceval_macros.h | 7 + Python/pystate.c | 27 +++ Python/qsbr.c | 290 +++++++++++++++++++++++++ 18 files changed, 569 insertions(+) create mode 100644 Include/internal/pycore_qsbr.h create mode 100644 Python/qsbr.c diff --git a/Doc/license.rst b/Doc/license.rst index 9fc0ff7161a591..cbe918bd1acfe3 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1095,3 +1095,35 @@ which is distributed under the MIT license:: LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Global Unbounded Sequences (GUS) +-------------------------------- + +The file :file:`Python/qsbr.c` is adapted from FreeBSD's "Global Unbounded +Sequences" safe memory reclamation scheme in +`subr_smr.c `_. +The file is distributed under the 2-Clause BSD License:: + + Copyright (c) 2019,2020 Jeffrey Roberson + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice unmodified, this list of conditions, and the following + disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 5314a70436bfc3..98f81564fe36ec 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -469,6 +469,12 @@ _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj); + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 70f2b7e1b5706a..d4c315764f3c39 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -495,6 +495,14 @@ static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 601a0cf65afc1c..766812f26d256d 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -938,6 +938,32 @@ _Py_atomic_load_int_acquire(const int *obj) #endif } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uint64_release" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64_acquire" +#endif +} + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index a05bfaec47e89d..839afbd882800c 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -870,6 +870,22 @@ _Py_atomic_load_int_acquire(const int *obj) memory_order_acquire); } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_release); +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_acquire); +} + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index f7c332ed747cfa..8b72141989c127 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -31,6 +31,7 @@ extern "C" { #include "pycore_mimalloc.h" // struct _mimalloc_interp_state #include "pycore_object_state.h" // struct _py_object_state #include "pycore_obmalloc.h" // struct _obmalloc_state +#include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct types_state @@ -198,6 +199,7 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; struct _stoptheworld_state stoptheworld; + struct _qsbr_shared qsbr; #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h new file mode 100644 index 00000000000000..f42331a973d448 --- /dev/null +++ b/Include/internal/pycore_qsbr.h @@ -0,0 +1,131 @@ +// The QSBR APIs (quiescent state-based reclamation) provide a mechanism for +// the free-threaded build to safely reclaim memory when there may be +// concurrent accesses. +// +// Many operations in the free-threaded build are protected by locks. However, +// in some cases, we want to allow reads to happen concurrently with updates. +// In this case, we need to delay freeing ("reclaiming") any memory that may be +// concurrently accessed by a reader. The QSBR APIs provide a way to do this. +#ifndef Py_INTERNAL_QSBR_H +#define Py_INTERNAL_QSBR_H + +#include +#include +#include "pycore_lock.h" // PyMutex + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct _qsbr_shared; +struct _PyThreadStateImpl; // forward declare to avoid circular dependency + +// Per-thread state +struct _qsbr_thread_state { + // Last observed write sequence (or 0 if detached) + uint64_t seq; + + // Shared (per-interpreter) QSBR state + struct _qsbr_shared *shared; + + // Thread state (or NULL) + PyThreadState *tstate; + + // Used to defer advancing write sequence a fixed number of times + int deferrals; + + // Is this thread state allocated? + bool allocated; + struct _qsbr_thread_state *freelist_next; +}; + +// Padding to avoid false sharing +struct _qsbr_pad { + struct _qsbr_thread_state qsbr; + char __padding[64 - sizeof(struct _qsbr_thread_state)]; +}; + +// Per-interpreter state +struct _qsbr_shared { + // Write sequence: always odd, incremented by two + uint64_t wr_seq; + + // Minimum observed read sequence of all QSBR thread states + uint64_t rd_seq; + + // Array of QSBR thread states. + struct _qsbr_pad *array; + Py_ssize_t size; + + // Freelist of unused _qsbr_thread_states (protected by mutex) + PyMutex mutex; + struct _qsbr_thread_state *freelist; +}; + +static inline uint64_t +_Py_qsbr_shared_current(struct _qsbr_shared *shared) +{ + return _Py_atomic_load_uint64_acquire(&shared->wr_seq); +} + +// Reports a quiescent state: the caller no longer holds any pointer to shared +// data not protected by locks or reference counts. +static inline void +_Py_qsbr_quiescent_state(struct _qsbr_thread_state *qsbr) +{ + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64_release(&qsbr->seq, seq); +} + +// Advance the write sequence and return the new goal. This should be called +// after data is removed. The returned goal is used with `_Py_qsbr_poll()` to +// determine when it is safe to reclaim (free) the memory. +extern uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared); + +// Batches requests to advance the write sequence. This advances the write +// sequence every N calls, which reduces overhead but increases time to +// reclamation. Returns the new goal. +extern uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); + +// Have the read sequences advanced to the given goal? If this returns true, +// it safe to reclaim any memory tagged with the goal (or earlier goal). +extern bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal); + +// Called when thread attaches to interpreter +extern void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr); + +// Called when thread detaches from interpreter +extern void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr); + +// Reserves (allocates) a QSBR state and returns its index. +extern Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp); + +// Associates a PyThreadState with the QSBR state at the given index +extern void +_Py_qsbr_register(struct _PyThreadStateImpl *tstate, + PyInterpreterState *interp, Py_ssize_t index); + +// Disassociates a PyThreadState from the QSBR state and frees the QSBR state. +extern void +_Py_qsbr_unregister(struct _PyThreadStateImpl *tstate); + +extern void +_Py_qsbr_fini(PyInterpreterState *interp); + +extern void +_Py_qsbr_after_fork(struct _PyThreadStateImpl *tstate); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_QSBR_H */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 571a7d612c94e2..3973c7b83700c6 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -169,6 +169,10 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .qsbr = { \ + .wr_seq = 1, \ + .rd_seq = 1, \ + }, \ .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 472fa08154e8f9..e02f83b23ed2d9 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_qsbr.h" // struct qsbr // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -20,6 +21,7 @@ typedef struct _PyThreadStateImpl { PyThreadState base; #ifdef Py_GIL_DISABLED + struct _qsbr_thread_state *qsbr; struct _mimalloc_thread_state mimalloc; struct _Py_freelist_state freelist_state; #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 07b2ec7adde78a..19a09a16aad7cf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -455,6 +455,7 @@ PYTHON_OBJS= \ Python/pystate.o \ Python/pythonrun.o \ Python/pytime.o \ + Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ Python/structmember.o \ @@ -1158,6 +1159,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pystats.h \ $(srcdir)/Include/internal/pycore_pythonrun.h \ $(srcdir)/Include/internal/pycore_pythread.h \ + $(srcdir)/Include/internal/pycore_qsbr.h \ $(srcdir)/Include/internal/pycore_range.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init.h \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 22891135bde0af..9938a7ba21ee24 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -637,6 +637,10 @@ PyOS_AfterFork_Child(void) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif +#ifdef Py_GIL_DISABLED + _Py_qsbr_after_fork((_PyThreadStateImpl *)tstate); +#endif + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 35788ec4503e8f..4869c3a8586bf2 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -252,6 +252,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 7a44179e356105..39975ce38e3b41 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -373,6 +373,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e1ff97659659ee..ec08a4e90c693f 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -274,6 +274,7 @@ + @@ -611,6 +612,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4c55f23006b2f0..d2169fb00507e5 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -747,6 +747,9 @@ Include\internal + + Include\internal + Include\internal @@ -1412,6 +1415,9 @@ Python + + Python + Python diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c2550f53ad6eaa..1043966c9a8277 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -86,6 +86,12 @@ #define PRE_DISPATCH_GOTO() ((void)0) #endif +#ifdef Py_GIL_DISABLED +#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) +#else +#define QSBR_QUIESCENT_STATE(tstate) +#endif + /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ @@ -117,6 +123,7 @@ #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ + QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ if (_Py_HandlePending(tstate) != 0) { \ GOTO_ERROR(error); \ diff --git a/Python/pystate.c b/Python/pystate.c index e77e5bfa7e2df8..22e8690b588d66 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -951,6 +951,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyThread_free_lock(interp->id_mutex); } + _Py_qsbr_fini(interp); + _PyObject_FiniState(interp); free_interpreter(interp); @@ -1372,6 +1374,14 @@ new_threadstate(PyInterpreterState *interp, int whence) if (new_tstate == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + Py_ssize_t qsbr_idx = _Py_qsbr_reserve(interp); + if (qsbr_idx < 0) { + PyMem_RawFree(new_tstate); + return NULL; + } +#endif + /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(runtime); @@ -1398,6 +1408,9 @@ new_threadstate(PyInterpreterState *interp, int whence) sizeof(*tstate)); } +#ifdef Py_GIL_DISABLED + _Py_qsbr_register(tstate, interp, qsbr_idx); +#endif init_threadstate(tstate, interp, id, whence); add_threadstate(interp, (PyThreadState *)tstate, old_head); @@ -1609,6 +1622,10 @@ tstate_delete_common(PyThreadState *tstate) } HEAD_UNLOCK(runtime); +#ifdef Py_GIL_DISABLED + _Py_qsbr_unregister((_PyThreadStateImpl *)tstate); +#endif + // XXX Unbind in PyThreadState_Clear(), or earlier // (and assert not-equal here)? if (tstate->_status.bound_gilstate) { @@ -1650,6 +1667,9 @@ void _PyThreadState_DeleteCurrent(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif tstate_set_detached(tstate); tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); @@ -1871,6 +1891,10 @@ _PyThreadState_Attach(PyThreadState *tstate) tstate_wait_attach(tstate); } +#ifdef Py_GIL_DISABLED + _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif + // Resume previous critical section. This acquires the lock(s) from the // top-most critical section. if (tstate->critical_section != 0) { @@ -1891,6 +1915,9 @@ detach_thread(PyThreadState *tstate, int detached_state) if (tstate->critical_section != 0) { _PyCriticalSection_SuspendAll(tstate); } +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif tstate_deactivate(tstate); tstate_set_detached(tstate); current_fast_clear(&_PyRuntime); diff --git a/Python/qsbr.c b/Python/qsbr.c new file mode 100644 index 00000000000000..17f28b94104128 --- /dev/null +++ b/Python/qsbr.c @@ -0,0 +1,290 @@ +/* + * Implementation of safe memory reclamation scheme using + * quiescent states. + * + * This is dervied from the "GUS" safe memory reclamation technique + * in FreeBSD written by Jeffrey Roberson. It is heavily modified. Any bugs + * in this code are likely due to the modifications. + * + * The original copyright is preserved below. + * + * Copyright (c) 2019,2020 Jeffrey Roberson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "Python.h" +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() +#include "pycore_lock.h" // PyMutex_Lock() +#include "pycore_qsbr.h" +#include "pycore_pystate.h" // _PyThreadState_GET() + + +// Wrap-around safe comparison +#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) +#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) + +// Starting size of the array of qsbr thread states +#define MIN_ARRAY_SIZE 8 + +// The shared write sequence is always odd and incremented by two. Detached +// threads are indicated by a read sequence of zero. +#define QSBR_OFFLINE 0 +#define QSBR_INITIAL 1 +#define QSBR_INCR 2 + +// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing +// the write sequence. +#define QSBR_DEFERRED_LIMIT 10 + +// Allocate a QSBR thread state from the freelist +struct _qsbr_thread_state * +qsbr_allocate(struct _qsbr_shared *shared) +{ + struct _qsbr_thread_state *qsbr = shared->freelist; + if (qsbr == NULL) { + return NULL; + } + shared->freelist = qsbr->freelist_next; + qsbr->freelist_next = NULL; + qsbr->shared = shared; + qsbr->allocated = true; + return qsbr; +} + +// Initialize (or reintialize) the freelist of QSBR thread states +static void +initialize_freelist(struct _qsbr_shared *shared) +{ + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr->tstate != NULL) { + // Update the thread state pointer to its QSBR state + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)qsbr->tstate; + tstate->qsbr = qsbr; + } + if (!qsbr->allocated) { + // Push to freelist + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} + +// Grow the array of QSBR thread states. Returns 0 on success, -1 on failure. +static int +grow_thread_array(struct _qsbr_shared *shared) +{ + Py_ssize_t new_size = shared->size * 2; + if (new_size < MIN_ARRAY_SIZE) { + new_size = MIN_ARRAY_SIZE; + } + + struct _qsbr_pad *array = PyMem_RawCalloc(new_size, sizeof(*array)); + if (array == NULL) { + return -1; + } + + struct _qsbr_pad *old = shared->array; + if (old != NULL) { + memcpy(array, shared->array, shared->size * sizeof(*array)); + } + + shared->array = array; + shared->size = new_size; + shared->freelist = NULL; + initialize_freelist(shared); + + PyMem_RawFree(old); + return 0; +} + +uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared) +{ + return _Py_atomic_add_uint64(&shared->wr_seq, QSBR_INCR) + QSBR_INCR; +} + +uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) +{ + if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { + return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; + } + qsbr->deferrals = 0; + return _Py_qsbr_advance(qsbr->shared); +} + +static uint64_t +qsbr_poll_scan(struct _qsbr_shared *shared) +{ + // Synchronize with store in _Py_qsbr_attach(). We need to ensure that + // the reads from each thread's sequence number are not reordered to see + // earlier "offline" states. + _Py_atomic_fence_seq_cst(); + + // Compute the minimum sequence number of all attached threads + uint64_t min_seq = _Py_atomic_load_uint64(&shared->wr_seq); + struct _qsbr_pad *array = shared->array; + for (Py_ssize_t i = 0, size = shared->size; i != size; i++) { + struct _qsbr_thread_state *qsbr = &array[i].qsbr; + + uint64_t seq = _Py_atomic_load_uint64(&qsbr->seq); + if (seq != QSBR_OFFLINE && QSBR_LT(seq, min_seq)) { + min_seq = seq; + } + } + + // Update the shared read sequence + uint64_t rd_seq = _Py_atomic_load_uint64(&shared->rd_seq); + if (QSBR_LT(rd_seq, min_seq)) { + // It's okay if the compare-exchange failed: another thread updated it + (void)_Py_atomic_compare_exchange_uint64(&shared->rd_seq, &rd_seq, min_seq); + rd_seq = min_seq; + } + + return rd_seq; +} + +bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) +{ + assert(_PyThreadState_GET()->state == _Py_THREAD_ATTACHED); + + uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); + if (QSBR_LEQ(goal, rd_seq)) { + return true; + } + + rd_seq = qsbr_poll_scan(qsbr->shared); + return QSBR_LEQ(goal, rd_seq); +} + +void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq == 0 && "already attached"); + + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64(&qsbr->seq, seq); // needs seq_cst +} + +void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq != 0 && "already detached"); + + _Py_atomic_store_uint64_release(&qsbr->seq, QSBR_OFFLINE); +} + +Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + + PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); + // Try allocating from our internal freelist + struct _qsbr_thread_state *qsbr = qsbr_allocate(shared); + + // If there are no free entries, we pause all threads, grow the array, + // and update the pointers in PyThreadState to entries in the new array. + if (qsbr == NULL) { + _PyEval_StopTheWorld(interp); + if (grow_thread_array(shared) == 0) { + qsbr = qsbr_allocate(shared); + } + _PyEval_StartTheWorld(interp); + } + PyMutex_Unlock(&shared->mutex); + + if (qsbr == NULL) { + return -1; + } + + // Return an index rather than the pointer because the array may be + // resized and the pointer invalidated. + return (struct _qsbr_pad *)qsbr - shared->array; +} + +void +_Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, + Py_ssize_t index) +{ + // Associate the QSBR state with the thread state + struct _qsbr_shared *shared = &interp->qsbr; + + // NOTE: this function is called with runtime locked, so we don't detach + // while waiting for the lock. This prevents a stop-the-world pause + // while the runtime lock is held, which could lead to deadlock. + PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); + struct _qsbr_thread_state *qsbr = &interp->qsbr.array[index].qsbr; + assert(qsbr->allocated && qsbr->tstate == NULL); + qsbr->tstate = (PyThreadState *)tstate; + tstate->qsbr = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_unregister(_PyThreadStateImpl *tstate) +{ + struct _qsbr_thread_state *qsbr = tstate->qsbr; + struct _qsbr_shared *shared = qsbr->shared; + + assert(qsbr->seq == 0 && "thread state must be detached"); + + PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); + assert(qsbr->allocated && qsbr->tstate == (PyThreadState *)tstate); + tstate->qsbr = NULL; + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_fini(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + PyMem_RawFree(shared->array); + shared->array = NULL; + shared->size = 0; + shared->freelist = NULL; +} + +void +_Py_qsbr_after_fork(_PyThreadStateImpl *tstate) +{ + struct _qsbr_thread_state *this_qsbr = tstate->qsbr; + struct _qsbr_shared *shared = this_qsbr->shared; + + _PyMutex_at_fork_reinit(&shared->mutex); + + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr != this_qsbr && qsbr->allocated) { + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} From a33b739edbe6235ef928b1944a8ee6f7436000dc Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 8 Feb 2024 20:43:58 +0000 Subject: [PATCH 2/6] Fix default build --- Include/internal/pycore_tstate.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index e02f83b23ed2d9..950ce540439956 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -20,8 +20,9 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + struct _qsbr_thread_state *qsbr; // only used by free-threaded build + #ifdef Py_GIL_DISABLED - struct _qsbr_thread_state *qsbr; struct _mimalloc_thread_state mimalloc; struct _Py_freelist_state freelist_state; #endif From 39e395416e2397cee04d78c7e4cbf1ddf55a6ac3 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 8 Feb 2024 20:49:18 +0000 Subject: [PATCH 3/6] Mark function as static --- Python/qsbr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/qsbr.c b/Python/qsbr.c index 17f28b94104128..b2d2a94cc84bb3 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -56,7 +56,7 @@ #define QSBR_DEFERRED_LIMIT 10 // Allocate a QSBR thread state from the freelist -struct _qsbr_thread_state * +static struct _qsbr_thread_state * qsbr_allocate(struct _qsbr_shared *shared) { struct _qsbr_thread_state *qsbr = shared->freelist; From f1c374046b0f7144df30b075fc12e66ce836f313 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 8 Feb 2024 21:03:49 +0000 Subject: [PATCH 4/6] Fix indentation in qsbr.c --- Python/qsbr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/qsbr.c b/Python/qsbr.c index b2d2a94cc84bb3..c783e94bef7ab1 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -126,7 +126,7 @@ _Py_qsbr_advance(struct _qsbr_shared *shared) uint64_t _Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) { - if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { + if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; } qsbr->deferrals = 0; From 94a95d5bf7ff81574c2e27a5ddb2461fadd37066 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 15 Feb 2024 14:34:02 +0000 Subject: [PATCH 5/6] Fix potential deadlock and other changes from review --- Include/internal/pycore_qsbr.h | 8 ++++++++ Include/internal/pycore_runtime_init.h | 5 +++-- Python/pystate.c | 9 ++++++--- Python/qsbr.c | 22 ++++++++-------------- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index f42331a973d448..475f00deedc226 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -21,6 +21,14 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +// The shared write sequence is always odd and incremented by two. Detached +// threads are indicated by a read sequence of zero. This avoids collisions +// between the offline state and any valid sequence number even if the +// sequences numbers wrap around. +#define QSBR_OFFLINE 0 +#define QSBR_INITIAL 1 +#define QSBR_INCR 2 + struct _qsbr_shared; struct _PyThreadStateImpl; // forward declare to avoid circular dependency diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 45c8235c2f5e83..be81604d653814 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -17,6 +17,7 @@ extern "C" { #include "pycore_pyhash.h" // pyhash_state_INIT #include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT #include "pycore_pythread.h" // _pythread_RUNTIME_INIT +#include "pycore_qsbr.h" // QSBR_INITIAL #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT @@ -170,8 +171,8 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .qsbr = { \ - .wr_seq = 1, \ - .rd_seq = 1, \ + .wr_seq = QSBR_INITIAL, \ + .rd_seq = QSBR_INITIAL, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ diff --git a/Python/pystate.c b/Python/pystate.c index f384dfaa7103c8..91149e701123c0 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1412,9 +1412,6 @@ new_threadstate(PyInterpreterState *interp, int whence) sizeof(*tstate)); } -#ifdef Py_GIL_DISABLED - _Py_qsbr_register(tstate, interp, qsbr_idx); -#endif init_threadstate(tstate, interp, id, whence); add_threadstate(interp, (PyThreadState *)tstate, old_head); @@ -1423,6 +1420,12 @@ new_threadstate(PyInterpreterState *interp, int whence) // Must be called with lock unlocked to avoid re-entrancy deadlock. PyMem_RawFree(new_tstate); } + +#ifdef Py_GIL_DISABLED + // Must be called with lock unlocked to avoid lock ordering deadlocks. + _Py_qsbr_register(tstate, interp, qsbr_idx); +#endif + return (PyThreadState *)tstate; } diff --git a/Python/qsbr.c b/Python/qsbr.c index c783e94bef7ab1..2fa03c0d6aab6b 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -45,12 +45,6 @@ // Starting size of the array of qsbr thread states #define MIN_ARRAY_SIZE 8 -// The shared write sequence is always odd and incremented by two. Detached -// threads are indicated by a read sequence of zero. -#define QSBR_OFFLINE 0 -#define QSBR_INITIAL 1 -#define QSBR_INCR 2 - // For _Py_qsbr_deferred_advance(): the number of deferrals before advancing // the write sequence. #define QSBR_DEFERRED_LIMIT 10 @@ -72,7 +66,7 @@ qsbr_allocate(struct _qsbr_shared *shared) // Initialize (or reintialize) the freelist of QSBR thread states static void -initialize_freelist(struct _qsbr_shared *shared) +initialize_new_array(struct _qsbr_shared *shared) { for (Py_ssize_t i = 0; i != shared->size; i++) { struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; @@ -111,7 +105,7 @@ grow_thread_array(struct _qsbr_shared *shared) shared->array = array; shared->size = new_size; shared->freelist = NULL; - initialize_freelist(shared); + initialize_new_array(shared); PyMem_RawFree(old); return 0; @@ -120,6 +114,9 @@ grow_thread_array(struct _qsbr_shared *shared) uint64_t _Py_qsbr_advance(struct _qsbr_shared *shared) { + // NOTE: with 64-bit sequence numbers, we don't have to worry too much + // about the wr_seq getting too far ahead of rd_seq, but if we ever use + // 32-bit sequence numbers, we'll need to be more careful. return _Py_atomic_add_uint64(&shared->wr_seq, QSBR_INCR) + QSBR_INCR; } @@ -200,7 +197,7 @@ _Py_qsbr_reserve(PyInterpreterState *interp) { struct _qsbr_shared *shared = &interp->qsbr; - PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); + PyMutex_Lock(&shared->mutex); // Try allocating from our internal freelist struct _qsbr_thread_state *qsbr = qsbr_allocate(shared); @@ -231,10 +228,7 @@ _Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, // Associate the QSBR state with the thread state struct _qsbr_shared *shared = &interp->qsbr; - // NOTE: this function is called with runtime locked, so we don't detach - // while waiting for the lock. This prevents a stop-the-world pause - // while the runtime lock is held, which could lead to deadlock. - PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); + PyMutex_Lock(&shared->mutex); struct _qsbr_thread_state *qsbr = &interp->qsbr.array[index].qsbr; assert(qsbr->allocated && qsbr->tstate == NULL); qsbr->tstate = (PyThreadState *)tstate; @@ -250,7 +244,7 @@ _Py_qsbr_unregister(_PyThreadStateImpl *tstate) assert(qsbr->seq == 0 && "thread state must be detached"); - PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); + PyMutex_Lock(&shared->mutex); assert(qsbr->allocated && qsbr->tstate == (PyThreadState *)tstate); tstate->qsbr = NULL; qsbr->tstate = NULL; From 69d56690165e9992429e5371298aded06ab8423e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 15 Feb 2024 17:58:11 +0000 Subject: [PATCH 6/6] Add comment about QSBR_LT/QSBR_LEQ --- Python/qsbr.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/qsbr.c b/Python/qsbr.c index 2fa03c0d6aab6b..7f7ae03cf60d22 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -38,7 +38,9 @@ #include "pycore_pystate.h" // _PyThreadState_GET() -// Wrap-around safe comparison +// Wrap-around safe comparison. This is a holdover from the FreeBSD +// implementation, which uses 32-bit sequence numbers. We currently use 64-bit +// sequence numbers, so wrap-around is unlikely. #define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) #define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0)