From 0c8dcfc499c9b2f6859719503d0bac5910ca2c22 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 29 Mar 2025 13:04:27 -0400 Subject: [PATCH 01/52] gh-129069: make list ass_slice and memory_repeat safe --- Include/internal/pycore_list.h | 8 ++-- .../internal/pycore_pyatomic_ft_wrappers.h | 45 +++++++++++++++++++ Objects/listobject.c | 26 ++++------- Objects/tupleobject.c | 6 +-- 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 5c66d76a0d6e7f..f67c3bad8da821 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -48,15 +48,17 @@ _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) return _PyList_AppendTakeRefListResize(self, newitem); } -// Repeat the bytes of a buffer in place +// Repeat the bytes of a buffer of pointers in place static inline void -_Py_memory_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) +_Py_memory_ptrs_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) { assert(len_src > 0); + assert(len_src % sizeof(void *) == 0); + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); Py_ssize_t copied = len_src; while (copied < len_dest) { Py_ssize_t bytes_to_copy = Py_MIN(copied, len_dest - copied); - memcpy(dest + copied, dest, (size_t)bytes_to_copy); + FT_ATOMIC_MEMCPY_PTR_RELAXED(dest + copied, dest, (size_t)bytes_to_copy); copied += bytes_to_copy; } } diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index d755d03a5fa190..6111bc8aa6ea33 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -110,6 +110,48 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ _Py_atomic_load_ullong_relaxed(&value) +static inline void * +FT_ATOMIC_MEMCPY_PTR_RELAXED(void *dest, void *src, ssize_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + for (void **d = (void **)dest, **s = (void **)src, **e = d + n / sizeof(void *); d != e; d++, s++) { + void *v = _Py_atomic_load_ptr_relaxed(s); + _Py_atomic_store_ptr_relaxed(d, v); + } + } + + return dest; +} + +static inline void * +FT_ATOMIC_MEMMOVE_PTR_RELAXED(void *dest, void *src, ssize_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { // prefer incrementing copy + for (void **d = (void **)dest, **s = (void **)src, **e = d + n / sizeof(void *); d != e; d++, s++) { + void *v = _Py_atomic_load_ptr_relaxed(s); + _Py_atomic_store_ptr_relaxed(d, v); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + for (void **d = (void **)dest + n, **s = (void **)src + n, **e = (void **)dest - 1; d != e; d--, s--) { + void *v = _Py_atomic_load_ptr_relaxed(s); + _Py_atomic_store_ptr_relaxed(d, v); + } + } + + return dest; +} + + #else #define FT_ATOMIC_LOAD_PTR(value) value #define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value @@ -157,6 +199,9 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_MEMCPY_PTR_RELAXED(dest, src, n) memcpy(dest, src, n) +#define FT_ATOMIC_MEMMOVE_PTR_RELAXED(dest, src, n) memmove(dest, src, n) + #endif #ifdef __cplusplus diff --git a/Objects/listobject.c b/Objects/listobject.c index 917bef1a66ddff..c78b4110bf2834 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -813,10 +813,8 @@ list_repeat_lock_held(PyListObject *a, Py_ssize_t n) _Py_RefcntAdd(*src, n); *dest++ = *src++; } - // TODO: _Py_memory_repeat calls are not safe for shared lists in - // GIL_DISABLED builds. (See issue #129069) - _Py_memory_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, - sizeof(PyObject *)*input_size); + _Py_memory_ptrs_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, + sizeof(PyObject *)*input_size); } Py_SET_SIZE(np, output_size); @@ -949,12 +947,10 @@ list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyO if (d < 0) { /* Delete -d items */ Py_ssize_t tail; tail = (Py_SIZE(a) - ihigh) * sizeof(PyObject *); - // TODO: these memmove/memcpy calls are not safe for shared lists in - // GIL_DISABLED builds. (See issue #129069) - memmove(&item[ihigh+d], &item[ihigh], tail); + FT_ATOMIC_MEMMOVE_PTR_RELAXED(&item[ihigh+d], &item[ihigh], tail); if (list_resize(a, Py_SIZE(a) + d) < 0) { - memmove(&item[ihigh], &item[ihigh+d], tail); - memcpy(&item[ilow], recycle, s); + FT_ATOMIC_MEMMOVE_PTR_RELAXED(&item[ihigh], &item[ihigh+d], tail); + FT_ATOMIC_MEMCPY_PTR_RELAXED(&item[ilow], recycle, s); goto Error; } item = a->ob_item; @@ -964,10 +960,8 @@ list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyO if (list_resize(a, k+d) < 0) goto Error; item = a->ob_item; - // TODO: these memmove/memcpy calls are not safe for shared lists in - // GIL_DISABLED builds. (See issue #129069) - memmove(&item[ihigh+d], &item[ihigh], - (k - ihigh)*sizeof(PyObject *)); + FT_ATOMIC_MEMMOVE_PTR_RELAXED(&item[ihigh+d], &item[ihigh], + (k - ihigh)*sizeof(PyObject *)); } for (k = 0; k < n; k++, ilow++) { PyObject *w = vitem[k]; @@ -1051,10 +1045,8 @@ list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) for (Py_ssize_t j = 0; j < input_size; j++) { _Py_RefcntAdd(items[j], n-1); } - // TODO: _Py_memory_repeat calls are not safe for shared lists in - // GIL_DISABLED builds. (See issue #129069) - _Py_memory_repeat((char *)items, sizeof(PyObject *)*output_size, - sizeof(PyObject *)*input_size); + _Py_memory_ptrs_repeat((char *)items, sizeof(PyObject *)*output_size, + sizeof(PyObject *)*input_size); return 0; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 78ee3fa1fc3aad..4624dee7f32fe3 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -5,7 +5,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_freelist.h" // _Py_FREELIST_PUSH() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_list.h" // _Py_memory_repeat() +#include "pycore_list.h" // _Py_memory_ptrs_repeat() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_stackref.h" // PyStackRef_AsPyObjectSteal() @@ -547,8 +547,8 @@ tuple_repeat(PyObject *self, Py_ssize_t n) *dest++ = *src++; } - _Py_memory_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, - sizeof(PyObject *)*input_size); + _Py_memory_ptrs_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, + sizeof(PyObject *)*input_size); } _PyObject_GC_TRACK(np); return (PyObject *) np; From 67c94593d691d5bf4dad3d8eee6bdd69e36c489e Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 29 Mar 2025 20:12:29 +0000 Subject: [PATCH 02/52] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst new file mode 100644 index 00000000000000..6937f7f6d8fa60 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst @@ -0,0 +1 @@ +Fix data race and make sure no jagged writes in ``list.list_ass_slice``. From e460b5ace0a4927c857f1cec356e21051a71312c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 29 Mar 2025 16:17:10 -0400 Subject: [PATCH 03/52] include atomic wrappers in pycore_list.h --- Include/internal/pycore_list.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index f67c3bad8da821..a4eb1035e4e7dc 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -12,6 +12,8 @@ extern "C" { #include "pycore_stackref.h" #endif +#include "pycore_pyatomic_ft_wrappers.h" + PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); // _PyList_GetItemRef should be used only when the object is known as a list From fcdb63450ba405b861a9bef976209a3d57c4bf18 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 29 Mar 2025 18:34:35 -0400 Subject: [PATCH 04/52] provisional fix to compile on windows --- Include/internal/pycore_pyatomic_ft_wrappers.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 6111bc8aa6ea33..d840635e6af25d 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -110,9 +110,11 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ _Py_atomic_load_ullong_relaxed(&value) +// This should probably be moved into its own file as a real function. static inline void * -FT_ATOMIC_MEMCPY_PTR_RELAXED(void *dest, void *src, ssize_t n) +_Py_atomic_memcpy_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) { + size_t n = (size_t)sn; assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); @@ -127,9 +129,13 @@ FT_ATOMIC_MEMCPY_PTR_RELAXED(void *dest, void *src, ssize_t n) return dest; } +#define FT_ATOMIC_MEMCPY_PTR_RELAXED(dest, src, n) \ + _Py_atomic_memcpy_ptr_relaxed(dest, src, (Py_ssize_t)(n)) + static inline void * -FT_ATOMIC_MEMMOVE_PTR_RELAXED(void *dest, void *src, ssize_t n) +_Py_atomic_memmove_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) { + size_t n = (size_t)sn; assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); @@ -151,6 +157,9 @@ FT_ATOMIC_MEMMOVE_PTR_RELAXED(void *dest, void *src, ssize_t n) return dest; } +#define FT_ATOMIC_MEMMOVE_PTR_RELAXED(dest, src, n) \ + _Py_atomic_memmove_ptr_relaxed(dest, src, (Py_ssize_t)(n)) + #else #define FT_ATOMIC_LOAD_PTR(value) value From b1e92160cbfa9a8cbb76ab006be3e2c51a1c8776 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 1 Apr 2025 06:45:46 -0400 Subject: [PATCH 05/52] requested changes so far --- Include/cpython/pyatomic.h | 58 +++++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 45 -------------- ...-03-29-20-12-28.gh-issue-129069.QwbbWV.rst | 2 +- 3 files changed, 59 insertions(+), 46 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad66..fb15c0c3d0ee44 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,6 +545,64 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) +{ + size_t n = (size_t)sn; + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **d = (void **)dest; + void **s = (void **)src; + void **e = d + n / sizeof(void *); + + for (; d != e; d++, s++) { + void *v = _Py_atomic_load_ptr_relaxed(s); + _Py_atomic_store_ptr_relaxed(d, v); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) +{ + size_t n = (size_t)sn; + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **d = (void **)dest; + void **s = (void **)src; + void **e = d + n / sizeof(void *); + + for (; d != e; d++, s++) { + void *v = _Py_atomic_load_ptr_relaxed(s); + _Py_atomic_store_ptr_relaxed(d, v); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **d = (void **)dest + n; + void **s = (void **)src + n; + void **e = (void **)dest - 1; + + for (; d != e; d--, s--) { + void *v = _Py_atomic_load_ptr_relaxed(s); + _Py_atomic_store_ptr_relaxed(d, v); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index d840635e6af25d..4db094582da0a0 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -110,53 +110,8 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ _Py_atomic_load_ullong_relaxed(&value) -// This should probably be moved into its own file as a real function. -static inline void * -_Py_atomic_memcpy_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) -{ - size_t n = (size_t)sn; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - for (void **d = (void **)dest, **s = (void **)src, **e = d + n / sizeof(void *); d != e; d++, s++) { - void *v = _Py_atomic_load_ptr_relaxed(s); - _Py_atomic_store_ptr_relaxed(d, v); - } - } - - return dest; -} - #define FT_ATOMIC_MEMCPY_PTR_RELAXED(dest, src, n) \ _Py_atomic_memcpy_ptr_relaxed(dest, src, (Py_ssize_t)(n)) - -static inline void * -_Py_atomic_memmove_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) -{ - size_t n = (size_t)sn; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { // prefer incrementing copy - for (void **d = (void **)dest, **s = (void **)src, **e = d + n / sizeof(void *); d != e; d++, s++) { - void *v = _Py_atomic_load_ptr_relaxed(s); - _Py_atomic_store_ptr_relaxed(d, v); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - for (void **d = (void **)dest + n, **s = (void **)src + n, **e = (void **)dest - 1; d != e; d--, s--) { - void *v = _Py_atomic_load_ptr_relaxed(s); - _Py_atomic_store_ptr_relaxed(d, v); - } - } - - return dest; -} - #define FT_ATOMIC_MEMMOVE_PTR_RELAXED(dest, src, n) \ _Py_atomic_memmove_ptr_relaxed(dest, src, (Py_ssize_t)(n)) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst index 6937f7f6d8fa60..1cb09541dfa36b 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst @@ -1 +1 @@ -Fix data race and make sure no jagged writes in ``list.list_ass_slice``. +Fix data race and avoid jagged writes in ``list.list_ass_slice``. From 0f96d5838c889808a9a0072013a4567120b33d91 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 8 Apr 2025 07:49:00 -0400 Subject: [PATCH 06/52] remove atomic load and rename --- Include/cpython/pyatomic.h | 13 +++++-------- Include/internal/pycore_list.h | 2 +- Include/internal/pycore_pyatomic_ft_wrappers.h | 12 ++++++------ Objects/listobject.c | 10 +++++----- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index fb15c0c3d0ee44..d9b39525f93202 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -548,7 +548,7 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ static inline void * -_Py_atomic_memcpy_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t sn) { size_t n = (size_t)sn; assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); @@ -561,8 +561,7 @@ _Py_atomic_memcpy_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) void **e = d + n / sizeof(void *); for (; d != e; d++, s++) { - void *v = _Py_atomic_load_ptr_relaxed(s); - _Py_atomic_store_ptr_relaxed(d, v); + _Py_atomic_store_ptr_relaxed(d, *s); } } @@ -570,7 +569,7 @@ _Py_atomic_memcpy_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) } static inline void * -_Py_atomic_memmove_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, Py_ssize_t sn) { size_t n = (size_t)sn; assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); @@ -583,8 +582,7 @@ _Py_atomic_memmove_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) void **e = d + n / sizeof(void *); for (; d != e; d++, s++) { - void *v = _Py_atomic_load_ptr_relaxed(s); - _Py_atomic_store_ptr_relaxed(d, v); + _Py_atomic_store_ptr_relaxed(d, *s); } } else if (dest > src) { @@ -594,8 +592,7 @@ _Py_atomic_memmove_ptr_relaxed(void *dest, void *src, Py_ssize_t sn) void **e = (void **)dest - 1; for (; d != e; d--, s--) { - void *v = _Py_atomic_load_ptr_relaxed(s); - _Py_atomic_store_ptr_relaxed(d, v); + _Py_atomic_store_ptr_relaxed(d, *s); } } diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index a4eb1035e4e7dc..b9bd0c4754ce7a 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -60,7 +60,7 @@ _Py_memory_ptrs_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) Py_ssize_t copied = len_src; while (copied < len_dest) { Py_ssize_t bytes_to_copy = Py_MIN(copied, len_dest - copied); - FT_ATOMIC_MEMCPY_PTR_RELAXED(dest + copied, dest, (size_t)bytes_to_copy); + FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest + copied, dest, (size_t)bytes_to_copy); copied += bytes_to_copy; } } diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 4db094582da0a0..392e7eeebe7522 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -110,10 +110,10 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ _Py_atomic_load_ullong_relaxed(&value) -#define FT_ATOMIC_MEMCPY_PTR_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_relaxed(dest, src, (Py_ssize_t)(n)) -#define FT_ATOMIC_MEMMOVE_PTR_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_relaxed(dest, src, (Py_ssize_t)(n)) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) #else @@ -163,8 +163,8 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value -#define FT_ATOMIC_MEMCPY_PTR_RELAXED(dest, src, n) memcpy(dest, src, n) -#define FT_ATOMIC_MEMMOVE_PTR_RELAXED(dest, src, n) memmove(dest, src, n) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) #endif diff --git a/Objects/listobject.c b/Objects/listobject.c index 3798437619c39e..f4aee7b8f15fec 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -947,10 +947,10 @@ list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyO if (d < 0) { /* Delete -d items */ Py_ssize_t tail; tail = (Py_SIZE(a) - ihigh) * sizeof(PyObject *); - FT_ATOMIC_MEMMOVE_PTR_RELAXED(&item[ihigh+d], &item[ihigh], tail); + FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(&item[ihigh+d], &item[ihigh], tail); if (list_resize(a, Py_SIZE(a) + d) < 0) { - FT_ATOMIC_MEMMOVE_PTR_RELAXED(&item[ihigh], &item[ihigh+d], tail); - FT_ATOMIC_MEMCPY_PTR_RELAXED(&item[ilow], recycle, s); + FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(&item[ihigh], &item[ihigh+d], tail); + FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(&item[ilow], recycle, s); goto Error; } item = a->ob_item; @@ -960,8 +960,8 @@ list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyO if (list_resize(a, k+d) < 0) goto Error; item = a->ob_item; - FT_ATOMIC_MEMMOVE_PTR_RELAXED(&item[ihigh+d], &item[ihigh], - (k - ihigh)*sizeof(PyObject *)); + FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(&item[ihigh+d], &item[ihigh], + (k - ihigh)*sizeof(PyObject *)); } for (k = 0; k < n; k++, ilow++) { PyObject *w = vitem[k]; From 26ee081202b49196c681ea997d783fc79463c436 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 18 May 2025 07:48:16 -0400 Subject: [PATCH 07/52] requested variable name changes --- Include/cpython/pyatomic.h | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index d9b39525f93202..43797a146e2055 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -548,20 +548,20 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t sn) +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) { - size_t n = (size_t)sn; + size_t size = (size_t)n; assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(size % sizeof(void *) == 0); if (dest != src) { - void **d = (void **)dest; - void **s = (void **)src; - void **e = d + n / sizeof(void *); + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + size / sizeof(void *); - for (; d != e; d++, s++) { - _Py_atomic_store_ptr_relaxed(d, *s); + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); } } @@ -569,30 +569,30 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t sn) } static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, Py_ssize_t sn) +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) { - size_t n = (size_t)sn; + size_t size = (size_t)n; assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(size % sizeof(void *) == 0); - if (dest < src || dest >= (void *)((char *)src + n)) { - void **d = (void **)dest; - void **s = (void **)src; - void **e = d + n / sizeof(void *); + if (dest < src || dest >= (void *)((char *)src + size)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + size / sizeof(void *); - for (; d != e; d++, s++) { - _Py_atomic_store_ptr_relaxed(d, *s); + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); } } else if (dest > src) { - n = n / sizeof(void *) - 1; - void **d = (void **)dest + n; - void **s = (void **)src + n; - void **e = (void **)dest - 1; + size = size / sizeof(void *) - 1; + void **dest_ = (void **)dest + size; + void **src_ = (void **)src + size; + void **end = (void **)dest - 1; - for (; d != e; d--, s--) { - _Py_atomic_store_ptr_relaxed(d, *s); + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); } } From ac4e968575a108b54290c7d2354dcc0ea2259c5c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 1 Aug 2025 09:00:52 -0400 Subject: [PATCH 08/52] use _Py_IS_ALIGNED() --- Include/cpython/pyatomic.h | 8 ++++---- Include/internal/pycore_list.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 43797a146e2055..99a1d7b9058a6b 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -551,8 +551,8 @@ static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) { size_t size = (size_t)n; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); assert(size % sizeof(void *) == 0); if (dest != src) { @@ -572,8 +572,8 @@ static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) { size_t size = (size_t)n; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); assert(size % sizeof(void *) == 0); if (dest < src || dest >= (void *)((char *)src + size)) { diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 0604d2ff9ba09c..3bda1b6beb3f08 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -59,7 +59,7 @@ _Py_memory_ptrs_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) { assert(len_src > 0); assert(len_src % sizeof(void *) == 0); - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); Py_ssize_t copied = len_src; while (copied < len_dest) { Py_ssize_t bytes_to_copy = Py_MIN(copied, len_dest - copied); From 7c250fa97d739cdeb628f3cfe2677cec02e090d2 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 2 Aug 2025 08:46:21 -0400 Subject: [PATCH 09/52] Py_ssize_t arg -> size_t --- Include/cpython/pyatomic.h | 22 +++++++++---------- .../internal/pycore_pyatomic_ft_wrappers.h | 4 ++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 99a1d7b9058a6b..b4d62fb0ae60cb 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -548,17 +548,16 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - size_t size = (size_t)n; assert(_Py_IS_ALIGNED(dest, sizeof(void *))); assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(size % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); if (dest != src) { void **dest_ = (void **)dest; void **src_ = (void **)src; - void **end = dest_ + size / sizeof(void *); + void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { _Py_atomic_store_ptr_relaxed(dest_, *src_); @@ -569,26 +568,25 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) } static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - size_t size = (size_t)n; assert(_Py_IS_ALIGNED(dest, sizeof(void *))); assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(size % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); - if (dest < src || dest >= (void *)((char *)src + size)) { + if (dest < src || dest >= (void *)((char *)src + n)) { void **dest_ = (void **)dest; void **src_ = (void **)src; - void **end = dest_ + size / sizeof(void *); + void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { _Py_atomic_store_ptr_relaxed(dest_, *src_); } } else if (dest > src) { - size = size / sizeof(void *) - 1; - void **dest_ = (void **)dest + size; - void **src_ = (void **)src + size; + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; void **end = (void **)dest - 1; for (; dest_ != end; dest_--, src_--) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 2d1d4b44e98213..aca0778a23ad23 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -113,9 +113,9 @@ extern "C" { (void)_Py_atomic_add_ssize(&value, new_value) #define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) #define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) #else From 80c12d2e985a3ff0927916404d2e131ad99f0357 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 2 Aug 2025 16:25:54 -0400 Subject: [PATCH 10/52] add tests and remove suppressions --- Modules/_testcapi/pyatomic.c | 75 ++++++++++++++++++++++ Tools/tsan/suppressions_free_threading.txt | 6 -- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 850de6f9c3366b..ff835adc49ad28 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -5,6 +5,7 @@ */ #include "parts.h" +#include "pyconfig.h" // SIZEOF_VOID_P // We define atomic bitwise operations on these types #define FOR_BITWISE_TYPES(V) \ @@ -156,6 +157,78 @@ test_atomic_load_store_int_release_acquire(PyObject *self, PyObject *obj) { \ Py_RETURN_NONE; } +static PyObject * +test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { +#if SIZEOF_VOID_P == 8 +#define p0 (void *)0x5555555555555555 +#define p1 (void *)0xaaaaaaaaaaaaaaaa +#define p2 (void *)0xfedcba9876543210 +#define p3 (void *)0x0123456789abcdef +#else +#if SIZEOF_VOID_P == 4 +#define p0 (void *)0x55555555 +#define p1 (void *)0xaaaaaaaa +#define p2 (void *)0x76543210 +#define p3 (void *)0x01234567 +#else +#error "unexpected sizeof(void *), expecting 8 or 4" +#endif +#endif + void *src[4] = { (void *)0, p2, p3, (void *)0 }; + void *dst[4] = { p0, (void *)0, (void *)0, p1 }; + assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); + assert(dst[0] == p0); + assert(dst[1] == p2); + assert(dst[2] == p3); + assert(dst[3] == p1); + Py_RETURN_NONE; +#undef p3 +#undef p2 +#undef p1 +#undef p0 +} + +static PyObject * +test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { +#if SIZEOF_VOID_P == 8 +#define p0 (void *)0x5555555555555555 +#define p1 (void *)0xaaaaaaaaaaaaaaaa +#define p2 (void *)0xfedcba9876543210 +#define p3 (void *)0x0123456789abcdef +#define p4 (void *)0x0f2d4b6987a5c3e1 +#else +#if SIZEOF_VOID_P == 4 +#define p0 (void *)0x55555555 +#define p1 (void *)0xaaaaaaaa +#define p2 (void *)0x76543210 +#define p3 (void *)0x01234567 +#define p4 (void *)0x07254361 +#else +#error "unexpected sizeof(void *), expecting 8 or 4" +#endif +#endif + void *back[5] = { p0, p2, p3, p4, p1 }; + assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); + assert(back[0] == p0); + assert(back[1] == p3); + assert(back[2] == p4); + assert(back[3] == p4); + assert(back[4] == p1); + void *fwd[5] = { p0, p2, p3, p4, p1 }; + assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); + assert(fwd[0] == p0); + assert(fwd[1] == p2); + assert(fwd[2] == p2); + assert(fwd[3] == p3); + assert(fwd[4] == p1); + Py_RETURN_NONE; +#undef p4 +#undef p3 +#undef p2 +#undef p1 +#undef p0 +} + // NOTE: all tests should start with "test_atomic_" to be included // in test_pyatomic.py @@ -179,6 +252,8 @@ static PyMethodDef test_methods[] = { {"test_atomic_fences", test_atomic_fences, METH_NOARGS}, {"test_atomic_release_acquire", test_atomic_release_acquire, METH_NOARGS}, {"test_atomic_load_store_int_release_acquire", test_atomic_load_store_int_release_acquire, METH_NOARGS}, + {"test_atomic_memcpy_ptr_store_relaxed", test_atomic_memcpy_ptr_store_relaxed, METH_NOARGS}, + {"test_atomic_memmove_ptr_store_relaxed", test_atomic_memmove_ptr_store_relaxed, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 52d7c25a5bb37a..e69106a4d4635e 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -32,12 +32,6 @@ thread:pthread_create # Range iteration is not thread-safe yet (issue #129068) race_top:rangeiter_next -# List resizing happens through different paths ending in memcpy or memmove -# (for efficiency), which will probably need to rewritten as explicit loops -# of ptr-sized copies to be thread-safe. (Issue #129069) -race:list_ass_slice_lock_held -race:list_inplace_repeat_lock_held - # PyObject_Realloc internally does memcpy which isn't atomic so can race # with non-locking reads. See #132070 race:PyObject_Realloc From f220d8393d132f5caf9cdabafd5e5751cd835c93 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 15 Aug 2025 12:40:44 -0400 Subject: [PATCH 11/52] add list free-threading tsan test to test_list.py --- Lib/test/test_list.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 223f34fb696a6e..9ec5f7afb14a6e 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -1,8 +1,10 @@ import signal import sys import textwrap +import threading from test import list_tests, support from test.support import cpython_only +from test.support import threading_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_failure, assert_python_ok import pickle @@ -379,6 +381,33 @@ def foo(x): self.assertEqual(foo(list(range(10))), 45) + @unittest.skipUnless(support.Py_GIL_DISABLED and + support.check_sanitizer(thread=True), + 'this test can only possibly fail with GIL disabled') + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_free_threading(self): + def mutate(b, l): + d = [None] * 100 + b.wait() + + for _ in range(100): + l.extend(d) + del l[:] + + NUM_THREADS = 10 + barrier = threading.Barrier(len(NUM_THREADS)) + threads = [] + l = [] + + for _ in range(NUM_THREADS): + thread = threading.Thread(target=mutate, args=(barrier, l)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + if __name__ == "__main__": unittest.main() From bed61d9f16315d13a787c07d120c9e4d3245b1eb Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 15 Aug 2025 13:50:33 -0400 Subject: [PATCH 12/52] misc fix test_list.test_free_threading, no tsan req --- Lib/test/test_list.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 9ec5f7afb14a6e..9b03abb7408470 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -381,22 +381,22 @@ def foo(x): self.assertEqual(foo(list(range(10))), 45) - @unittest.skipUnless(support.Py_GIL_DISABLED and - support.check_sanitizer(thread=True), + @unittest.skipUnless(support.Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_free_threading(self): def mutate(b, l): - d = [None] * 100 + d = [None] * 500 b.wait() + l.extend(d) - for _ in range(100): - l.extend(d) - del l[:] + for _ in range(1000): + del l[:360] + l[1:-1] = d - NUM_THREADS = 10 - barrier = threading.Barrier(len(NUM_THREADS)) + NUM_THREADS = 20 + barrier = threading.Barrier(NUM_THREADS) threads = [] l = [] From 656a5a055a78e9b587795ab01dbbebf177f3da7d Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 27 Aug 2025 09:30:28 -0400 Subject: [PATCH 13/52] fix list tsan test --- Lib/test/test_list.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 9b03abb7408470..8d48e7b80046ec 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -386,27 +386,28 @@ def foo(x): @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_free_threading(self): - def mutate(b, l): - d = [None] * 500 - b.wait() - l.extend(d) + def read(done, l): + while not done.is_set(): + for _ in range(100): + _ = l[1] - for _ in range(1000): - del l[:360] - l[1:-1] = d - - NUM_THREADS = 20 - barrier = threading.Barrier(NUM_THREADS) + NUM_THREADS = 10 + done = threading.Event() threads = [] - l = [] + l = [None] * 500 + d = [None] * 498 for _ in range(NUM_THREADS): - thread = threading.Thread(target=mutate, args=(barrier, l)) + thread = threading.Thread(target=read, args=(done, l)) threads.append(thread) with threading_helper.start_threads(threads): - pass + for _ in range(100): + del l[:498] + l[1:-1] = d + + done.set() if __name__ == "__main__": From d34d11a607e13c070b8fcc75a11180e368676fa5 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 27 Aug 2025 14:16:06 -0400 Subject: [PATCH 14/52] change news --- .../2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst index 1cb09541dfa36b..63d7d60c73b278 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-29-20-12-28.gh-issue-129069.QwbbWV.rst @@ -1 +1 @@ -Fix data race and avoid jagged writes in ``list.list_ass_slice``. +Fix data race when assigning list slices concurrently. From ce439d7d964df7e0b41f5418238eb2edc040e6cc Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 13:34:47 -0400 Subject: [PATCH 15/52] test build fix --- Include/cpython/pyatomic.h | 50 ++----------------------------- Include/cpython/pyatomic_gcc.h | 54 ++++++++++++++++++++++++++++++++++ Include/cpython/pyatomic_msc.h | 54 ++++++++++++++++++++++++++++++++++ Include/cpython/pyatomic_std.h | 53 +++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 48 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b4d62fb0ae60cb..e0e66227735109 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -548,56 +548,10 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - - +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b..358b0f633c0ed2 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,6 +600,60 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf..67c7c12219b843 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,6 +1154,60 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5f..e066ea73deed45 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,6 +1088,59 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void From a16cf5bfe8120fc79d8b90d3dec76d9d8ebdee44 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 13:38:22 -0400 Subject: [PATCH 16/52] control test with original main pyatomic.h --- Include/cpython/pyatomic.h | 7 --- Include/cpython/pyatomic_gcc.h | 56 +------------------ Include/cpython/pyatomic_msc.h | 54 ------------------ Include/cpython/pyatomic_std.h | 53 ------------------ .../internal/pycore_pyatomic_ft_wrappers.h | 6 +- 5 files changed, 3 insertions(+), 173 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index e0e66227735109..2a0c11e7b3ad66 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,13 +545,6 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 358b0f633c0ed2..292f1a3a9eac5f 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,61 +600,7 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - - -// --- _Py_atomic_fence ------------------------------------------------------ +z// --- _Py_atomic_fence ------------------------------------------------------ static inline void _Py_atomic_fence_seq_cst(void) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 67c7c12219b843..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,60 +1154,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index e066ea73deed45..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,59 +1088,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 53683e1691dd65..927a3834f633e0 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,10 +114,8 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) #else From 8ab9d16e515c194624918401ed63bb638db57efd Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 13:40:23 -0400 Subject: [PATCH 17/52] fix typo --- Include/cpython/pyatomic_gcc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 292f1a3a9eac5f..1566b83b9f6a1b 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,7 +600,7 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } -z// --- _Py_atomic_fence ------------------------------------------------------ +// --- _Py_atomic_fence ------------------------------------------------------ static inline void _Py_atomic_fence_seq_cst(void) From 8e2d83ec299860f09332a43bb57cb30c26f8d99d Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 13:45:59 -0400 Subject: [PATCH 18/52] remove capi tests --- Modules/_testcapi/pyatomic.c | 146 +++++++++++++++++------------------ 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index ff835adc49ad28..f911b2689c2245 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -5,7 +5,7 @@ */ #include "parts.h" -#include "pyconfig.h" // SIZEOF_VOID_P +// #include "pyconfig.h" // SIZEOF_VOID_P // We define atomic bitwise operations on these types #define FOR_BITWISE_TYPES(V) \ @@ -157,77 +157,77 @@ test_atomic_load_store_int_release_acquire(PyObject *self, PyObject *obj) { \ Py_RETURN_NONE; } -static PyObject * -test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { -#if SIZEOF_VOID_P == 8 -#define p0 (void *)0x5555555555555555 -#define p1 (void *)0xaaaaaaaaaaaaaaaa -#define p2 (void *)0xfedcba9876543210 -#define p3 (void *)0x0123456789abcdef -#else -#if SIZEOF_VOID_P == 4 -#define p0 (void *)0x55555555 -#define p1 (void *)0xaaaaaaaa -#define p2 (void *)0x76543210 -#define p3 (void *)0x01234567 -#else -#error "unexpected sizeof(void *), expecting 8 or 4" -#endif -#endif - void *src[4] = { (void *)0, p2, p3, (void *)0 }; - void *dst[4] = { p0, (void *)0, (void *)0, p1 }; - assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); - assert(dst[0] == p0); - assert(dst[1] == p2); - assert(dst[2] == p3); - assert(dst[3] == p1); - Py_RETURN_NONE; -#undef p3 -#undef p2 -#undef p1 -#undef p0 -} +// static PyObject * +// test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { +// #if SIZEOF_VOID_P == 8 +// #define p0 (void *)0x5555555555555555 +// #define p1 (void *)0xaaaaaaaaaaaaaaaa +// #define p2 (void *)0xfedcba9876543210 +// #define p3 (void *)0x0123456789abcdef +// #else +// #if SIZEOF_VOID_P == 4 +// #define p0 (void *)0x55555555 +// #define p1 (void *)0xaaaaaaaa +// #define p2 (void *)0x76543210 +// #define p3 (void *)0x01234567 +// #else +// #error "unexpected sizeof(void *), expecting 8 or 4" +// #endif +// #endif +// void *src[4] = { (void *)0, p2, p3, (void *)0 }; +// void *dst[4] = { p0, (void *)0, (void *)0, p1 }; +// assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); +// assert(dst[0] == p0); +// assert(dst[1] == p2); +// assert(dst[2] == p3); +// assert(dst[3] == p1); +// Py_RETURN_NONE; +// #undef p3 +// #undef p2 +// #undef p1 +// #undef p0 +// } -static PyObject * -test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { -#if SIZEOF_VOID_P == 8 -#define p0 (void *)0x5555555555555555 -#define p1 (void *)0xaaaaaaaaaaaaaaaa -#define p2 (void *)0xfedcba9876543210 -#define p3 (void *)0x0123456789abcdef -#define p4 (void *)0x0f2d4b6987a5c3e1 -#else -#if SIZEOF_VOID_P == 4 -#define p0 (void *)0x55555555 -#define p1 (void *)0xaaaaaaaa -#define p2 (void *)0x76543210 -#define p3 (void *)0x01234567 -#define p4 (void *)0x07254361 -#else -#error "unexpected sizeof(void *), expecting 8 or 4" -#endif -#endif - void *back[5] = { p0, p2, p3, p4, p1 }; - assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); - assert(back[0] == p0); - assert(back[1] == p3); - assert(back[2] == p4); - assert(back[3] == p4); - assert(back[4] == p1); - void *fwd[5] = { p0, p2, p3, p4, p1 }; - assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); - assert(fwd[0] == p0); - assert(fwd[1] == p2); - assert(fwd[2] == p2); - assert(fwd[3] == p3); - assert(fwd[4] == p1); - Py_RETURN_NONE; -#undef p4 -#undef p3 -#undef p2 -#undef p1 -#undef p0 -} +// static PyObject * +// test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { +// #if SIZEOF_VOID_P == 8 +// #define p0 (void *)0x5555555555555555 +// #define p1 (void *)0xaaaaaaaaaaaaaaaa +// #define p2 (void *)0xfedcba9876543210 +// #define p3 (void *)0x0123456789abcdef +// #define p4 (void *)0x0f2d4b6987a5c3e1 +// #else +// #if SIZEOF_VOID_P == 4 +// #define p0 (void *)0x55555555 +// #define p1 (void *)0xaaaaaaaa +// #define p2 (void *)0x76543210 +// #define p3 (void *)0x01234567 +// #define p4 (void *)0x07254361 +// #else +// #error "unexpected sizeof(void *), expecting 8 or 4" +// #endif +// #endif +// void *back[5] = { p0, p2, p3, p4, p1 }; +// assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); +// assert(back[0] == p0); +// assert(back[1] == p3); +// assert(back[2] == p4); +// assert(back[3] == p4); +// assert(back[4] == p1); +// void *fwd[5] = { p0, p2, p3, p4, p1 }; +// assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); +// assert(fwd[0] == p0); +// assert(fwd[1] == p2); +// assert(fwd[2] == p2); +// assert(fwd[3] == p3); +// assert(fwd[4] == p1); +// Py_RETURN_NONE; +// #undef p4 +// #undef p3 +// #undef p2 +// #undef p1 +// #undef p0 +// } // NOTE: all tests should start with "test_atomic_" to be included // in test_pyatomic.py @@ -252,8 +252,8 @@ static PyMethodDef test_methods[] = { {"test_atomic_fences", test_atomic_fences, METH_NOARGS}, {"test_atomic_release_acquire", test_atomic_release_acquire, METH_NOARGS}, {"test_atomic_load_store_int_release_acquire", test_atomic_load_store_int_release_acquire, METH_NOARGS}, - {"test_atomic_memcpy_ptr_store_relaxed", test_atomic_memcpy_ptr_store_relaxed, METH_NOARGS}, - {"test_atomic_memmove_ptr_store_relaxed", test_atomic_memmove_ptr_store_relaxed, METH_NOARGS}, + // {"test_atomic_memcpy_ptr_store_relaxed", test_atomic_memcpy_ptr_store_relaxed, METH_NOARGS}, + // {"test_atomic_memmove_ptr_store_relaxed", test_atomic_memmove_ptr_store_relaxed, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; From 1c14be7410763b443ac0e39af8ef0d5677a023fb Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 13:49:02 -0400 Subject: [PATCH 19/52] restore atomic mem funcs --- Include/cpython/pyatomic.h | 53 +++++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad66..b4d62fb0ae60cb 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,6 +545,59 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 927a3834f633e0..53683e1691dd65 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,8 +114,10 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) #else From 9bf93f2ebbd9e9e1aec3d3f00e34951b980a5acb Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 13:54:15 -0400 Subject: [PATCH 20/52] another test --- Include/cpython/pyatomic.h | 102 +++++++++--------- .../internal/pycore_pyatomic_ft_wrappers.h | 6 +- Modules/_testcapi/pyatomic.c | 2 +- 3 files changed, 54 insertions(+), 56 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b4d62fb0ae60cb..276b52bd941e54 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,57 +545,57 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} +// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +// static inline void * +// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); +// assert(_Py_IS_ALIGNED(src, sizeof(void *))); +// assert(n % sizeof(void *) == 0); + +// if (dest != src) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// _Py_atomic_store_ptr_relaxed(dest_, *src_); +// } +// } + +// return dest; +// } + +// static inline void * +// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); +// assert(_Py_IS_ALIGNED(src, sizeof(void *))); +// assert(n % sizeof(void *) == 0); + +// if (dest < src || dest >= (void *)((char *)src + n)) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// _Py_atomic_store_ptr_relaxed(dest_, *src_); +// } +// } +// else if (dest > src) { +// n = n / sizeof(void *) - 1; +// void **dest_ = (void **)dest + n; +// void **src_ = (void **)src + n; +// void **end = (void **)dest - 1; + +// for (; dest_ != end; dest_--, src_--) { +// _Py_atomic_store_ptr_relaxed(dest_, *src_); +// } +// } + +// return dest; +// } diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 53683e1691dd65..927a3834f633e0 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,10 +114,8 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) #else diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index f911b2689c2245..c3089e2eb0bef0 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -5,7 +5,7 @@ */ #include "parts.h" -// #include "pyconfig.h" // SIZEOF_VOID_P +#include "pyconfig.h" // SIZEOF_VOID_P // We define atomic bitwise operations on these types #define FOR_BITWISE_TYPES(V) \ From 272fec6d8d2d9c23aeab14fa21d775ee2c9ede58 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 13:59:16 -0400 Subject: [PATCH 21/52] another test --- Include/cpython/pyatomic.h | 8 +++++++- Include/cpython/pyatomic_gcc.h | 16 ++++++++++++++++ Include/cpython/pyatomic_msc.h | 16 ++++++++++++++++ Include/cpython/pyatomic_std.h | 15 +++++++++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++-- 5 files changed, 58 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 276b52bd941e54..b46a7eabc508d9 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,7 +545,13 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); -// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); // static inline void * // _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b..68398f635895f8 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,6 +600,22 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memcpy(dest, src, n); +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memmove(dest, src, n); +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf..6968d24ee0801c 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,6 +1154,22 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memcpy(dest, src, n); +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memmove(dest, src, n); +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5f..71547a5059eddc 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,6 +1088,21 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memcpy(dest, src, n); +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memmove(dest, src, n); +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 927a3834f633e0..53683e1691dd65 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,8 +114,10 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) #else From 07213219254a4d7d242776873ba5917de782c5b2 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 14:03:46 -0400 Subject: [PATCH 22/52] final test hopefully to diagnose --- Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 53683e1691dd65..927a3834f633e0 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,10 +114,8 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) #else From e34fa034099e10c59f80dd9381f2cff59e22d0cc Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 14:08:03 -0400 Subject: [PATCH 23/52] hunch --- Include/cpython/pyatomic.h | 50 ---------------------------------- Include/cpython/pyatomic_gcc.h | 16 ----------- Include/cpython/pyatomic_msc.h | 16 ----------- Include/cpython/pyatomic_std.h | 15 ---------- 4 files changed, 97 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b46a7eabc508d9..fe500399e7e536 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -553,56 +553,6 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); -// static inline void * -// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); -// assert(_Py_IS_ALIGNED(src, sizeof(void *))); -// assert(n % sizeof(void *) == 0); - -// if (dest != src) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// _Py_atomic_store_ptr_relaxed(dest_, *src_); -// } -// } - -// return dest; -// } - -// static inline void * -// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); -// assert(_Py_IS_ALIGNED(src, sizeof(void *))); -// assert(n % sizeof(void *) == 0); - -// if (dest < src || dest >= (void *)((char *)src + n)) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// _Py_atomic_store_ptr_relaxed(dest_, *src_); -// } -// } -// else if (dest > src) { -// n = n / sizeof(void *) - 1; -// void **dest_ = (void **)dest + n; -// void **src_ = (void **)src + n; -// void **end = (void **)dest - 1; - -// for (; dest_ != end; dest_--, src_--) { -// _Py_atomic_store_ptr_relaxed(dest_, *src_); -// } -// } - -// return dest; -// } - diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 68398f635895f8..1566b83b9f6a1b 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,22 +600,6 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - return memcpy(dest, src, n); -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - return memmove(dest, src, n); -} - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 6968d24ee0801c..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,22 +1154,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - return memcpy(dest, src, n); -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - return memmove(dest, src, n); -} - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 71547a5059eddc..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,21 +1088,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - return memcpy(dest, src, n); -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - return memmove(dest, src, n); -} - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void From a7850be43cd05ffb86d5a1169bfeab240a12c342 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 14:22:17 -0400 Subject: [PATCH 24/52] fingers crossed --- Include/cpython/pyatomic.h | 50 ++++++++++++++++ Include/cpython/pyatomic_gcc.h | 54 ++++++++++++++++++ Include/cpython/pyatomic_msc.h | 53 +++++++++++++++++ Include/cpython/pyatomic_std.h | 57 +++++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 +- 5 files changed, 218 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index fe500399e7e536..b46a7eabc508d9 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -553,6 +553,56 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); +// static inline void * +// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); +// assert(_Py_IS_ALIGNED(src, sizeof(void *))); +// assert(n % sizeof(void *) == 0); + +// if (dest != src) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// _Py_atomic_store_ptr_relaxed(dest_, *src_); +// } +// } + +// return dest; +// } + +// static inline void * +// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); +// assert(_Py_IS_ALIGNED(src, sizeof(void *))); +// assert(n % sizeof(void *) == 0); + +// if (dest < src || dest >= (void *)((char *)src + n)) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// _Py_atomic_store_ptr_relaxed(dest_, *src_); +// } +// } +// else if (dest > src) { +// n = n / sizeof(void *) - 1; +// void **dest_ = (void **)dest + n; +// void **src_ = (void **)src + n; +// void **end = (void **)dest - 1; + +// for (; dest_ != end; dest_--, src_--) { +// _Py_atomic_store_ptr_relaxed(dest_, *src_); +// } +// } + +// return dest; +// } + diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b..df6bc6ea90cb07 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,6 +600,60 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf..4101c7abf703e1 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,6 +1154,59 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *(void * volatile *)dest_ = *src_; + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *(void * volatile *)dest_ = *src_; + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + *(void * volatile *)dest_ = *src_; + } + } + + return dest; +} + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5f..4796062ca2c6a2 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,6 +1088,63 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + _Py_USING_STD; + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 927a3834f633e0..53683e1691dd65 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,8 +114,10 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) #else From be4f118db8a47ae9c4be71a965d797bce11b8afb Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 14:27:14 -0400 Subject: [PATCH 25/52] misc --- Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 53683e1691dd65..927a3834f633e0 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,10 +114,8 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) #else From 23c1bcec3433819e2840da559c87b61e11803fd5 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 14:49:19 -0400 Subject: [PATCH 26/52] test as #define --- Include/cpython/pyatomic.h | 59 ------------------- Include/cpython/pyatomic_gcc.h | 54 ----------------- Include/cpython/pyatomic_msc.h | 53 ----------------- Include/cpython/pyatomic_std.h | 57 ------------------ .../internal/pycore_pyatomic_ft_wrappers.h | 20 ++++++- 5 files changed, 18 insertions(+), 225 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b46a7eabc508d9..2a0c11e7b3ad66 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,65 +545,6 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); - -// static inline void * -// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); -// assert(_Py_IS_ALIGNED(src, sizeof(void *))); -// assert(n % sizeof(void *) == 0); - -// if (dest != src) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// _Py_atomic_store_ptr_relaxed(dest_, *src_); -// } -// } - -// return dest; -// } - -// static inline void * -// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// assert(_Py_IS_ALIGNED(dest, sizeof(void *))); -// assert(_Py_IS_ALIGNED(src, sizeof(void *))); -// assert(n % sizeof(void *) == 0); - -// if (dest < src || dest >= (void *)((char *)src + n)) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// _Py_atomic_store_ptr_relaxed(dest_, *src_); -// } -// } -// else if (dest > src) { -// n = n / sizeof(void *) - 1; -// void **dest_ = (void **)dest + n; -// void **src_ = (void **)src + n; -// void **end = (void **)dest - 1; - -// for (; dest_ != end; dest_--, src_--) { -// _Py_atomic_store_ptr_relaxed(dest_, *src_); -// } -// } - -// return dest; -// } - - // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index df6bc6ea90cb07..1566b83b9f6a1b 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,60 +600,6 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } - - return dest; -} - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 4101c7abf703e1..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,59 +1154,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - *(void * volatile *)dest_ = *src_; - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - *(void * volatile *)dest_ = *src_; - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - *(void * volatile *)dest_ = *src_; - } - } - - return dest; -} - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 4796062ca2c6a2..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,63 +1088,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - _Py_USING_STD; - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } - - return dest; -} - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 927a3834f633e0..27b0204706397e 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -114,8 +114,24 @@ extern "C" { #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) -#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) memcpy(dest, src, n) -#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) memmove(dest, src, n) +#define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ +{ \ + void *_dest = (dest); \ + void *_src = (src); \ + \ + if (dest != src) { \ + void **dest_ = (void **)_dest; \ + void **src_ = (void **)_src; \ + void **end = dest_ + (Py_ssize_t)(n) / sizeof(void *); \ + \ + for (; dest_ != end; dest_++, src_++) { \ + _Py_atomic_store_ptr_relaxed(dest_, *src_); \ + } \ + } \ +} + +#define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ + memmove(dest, src, (size_t)(n)) #else From aef4baa9d662b7204ec2730d41a0676ac56150a7 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 15:06:24 -0400 Subject: [PATCH 27/52] reorder to end of pyatomic.h --- Include/cpython/pyatomic.h | 53 +++++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 18 +------ Modules/_testcapi/pyatomic.c | 2 +- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad66..b3d76c3858a430 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -612,3 +612,56 @@ static inline void _Py_atomic_fence_release(void); #else # error "long must be 4 or 8 bytes in size" #endif // SIZEOF_LONG + + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 27b0204706397e..53683e1691dd65 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -115,23 +115,9 @@ extern "C" { #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ -{ \ - void *_dest = (dest); \ - void *_src = (src); \ - \ - if (dest != src) { \ - void **dest_ = (void **)_dest; \ - void **src_ = (void **)_src; \ - void **end = dest_ + (Py_ssize_t)(n) / sizeof(void *); \ - \ - for (; dest_ != end; dest_++, src_++) { \ - _Py_atomic_store_ptr_relaxed(dest_, *src_); \ - } \ - } \ -} - + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) #define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - memmove(dest, src, (size_t)(n)) + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) #else diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index c3089e2eb0bef0..f911b2689c2245 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -5,7 +5,7 @@ */ #include "parts.h" -#include "pyconfig.h" // SIZEOF_VOID_P +// #include "pyconfig.h" // SIZEOF_VOID_P // We define atomic bitwise operations on these types #define FOR_BITWISE_TYPES(V) \ From b26b262848e8c7519566844715ada4c0f23cb367 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 15:28:28 -0400 Subject: [PATCH 28/52] revert _Py_IS_ALIGNED() to raw checks --- Include/cpython/pyatomic.h | 106 ++++++++++++------------ Include/internal/pycore_list.h | 2 +- Modules/_testcapi/pyatomic.c | 142 ++++++++++++++++----------------- 3 files changed, 125 insertions(+), 125 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b3d76c3858a430..32f1c2c7b72476 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,6 +545,59 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ @@ -612,56 +665,3 @@ static inline void _Py_atomic_fence_release(void); #else # error "long must be 4 or 8 bytes in size" #endif // SIZEOF_LONG - - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 3bda1b6beb3f08..0604d2ff9ba09c 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -59,7 +59,7 @@ _Py_memory_ptrs_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) { assert(len_src > 0); assert(len_src % sizeof(void *) == 0); - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); Py_ssize_t copied = len_src; while (copied < len_dest) { Py_ssize_t bytes_to_copy = Py_MIN(copied, len_dest - copied); diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index f911b2689c2245..e6052fe711023f 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -5,7 +5,7 @@ */ #include "parts.h" -// #include "pyconfig.h" // SIZEOF_VOID_P +#include "pyconfig.h" // SIZEOF_VOID_P // We define atomic bitwise operations on these types #define FOR_BITWISE_TYPES(V) \ @@ -157,77 +157,77 @@ test_atomic_load_store_int_release_acquire(PyObject *self, PyObject *obj) { \ Py_RETURN_NONE; } -// static PyObject * -// test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { -// #if SIZEOF_VOID_P == 8 -// #define p0 (void *)0x5555555555555555 -// #define p1 (void *)0xaaaaaaaaaaaaaaaa -// #define p2 (void *)0xfedcba9876543210 -// #define p3 (void *)0x0123456789abcdef -// #else -// #if SIZEOF_VOID_P == 4 -// #define p0 (void *)0x55555555 -// #define p1 (void *)0xaaaaaaaa -// #define p2 (void *)0x76543210 -// #define p3 (void *)0x01234567 -// #else -// #error "unexpected sizeof(void *), expecting 8 or 4" -// #endif -// #endif -// void *src[4] = { (void *)0, p2, p3, (void *)0 }; -// void *dst[4] = { p0, (void *)0, (void *)0, p1 }; -// assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); -// assert(dst[0] == p0); -// assert(dst[1] == p2); -// assert(dst[2] == p3); -// assert(dst[3] == p1); -// Py_RETURN_NONE; -// #undef p3 -// #undef p2 -// #undef p1 -// #undef p0 -// } +static PyObject * +test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { +#if SIZEOF_VOID_P == 8 +#define p0 (void *)0x5555555555555555 +#define p1 (void *)0xaaaaaaaaaaaaaaaa +#define p2 (void *)0xfedcba9876543210 +#define p3 (void *)0x0123456789abcdef +#else +#if SIZEOF_VOID_P == 4 +#define p0 (void *)0x55555555 +#define p1 (void *)0xaaaaaaaa +#define p2 (void *)0x76543210 +#define p3 (void *)0x01234567 +#else +#error "unexpected sizeof(void *), expecting 8 or 4" +#endif +#endif + void *src[4] = { (void *)0, p2, p3, (void *)0 }; + void *dst[4] = { p0, (void *)0, (void *)0, p1 }; + assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); + assert(dst[0] == p0); + assert(dst[1] == p2); + assert(dst[2] == p3); + assert(dst[3] == p1); + Py_RETURN_NONE; +#undef p3 +#undef p2 +#undef p1 +#undef p0 +} -// static PyObject * -// test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { -// #if SIZEOF_VOID_P == 8 -// #define p0 (void *)0x5555555555555555 -// #define p1 (void *)0xaaaaaaaaaaaaaaaa -// #define p2 (void *)0xfedcba9876543210 -// #define p3 (void *)0x0123456789abcdef -// #define p4 (void *)0x0f2d4b6987a5c3e1 -// #else -// #if SIZEOF_VOID_P == 4 -// #define p0 (void *)0x55555555 -// #define p1 (void *)0xaaaaaaaa -// #define p2 (void *)0x76543210 -// #define p3 (void *)0x01234567 -// #define p4 (void *)0x07254361 -// #else -// #error "unexpected sizeof(void *), expecting 8 or 4" -// #endif -// #endif -// void *back[5] = { p0, p2, p3, p4, p1 }; -// assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); -// assert(back[0] == p0); -// assert(back[1] == p3); -// assert(back[2] == p4); -// assert(back[3] == p4); -// assert(back[4] == p1); -// void *fwd[5] = { p0, p2, p3, p4, p1 }; -// assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); -// assert(fwd[0] == p0); -// assert(fwd[1] == p2); -// assert(fwd[2] == p2); -// assert(fwd[3] == p3); -// assert(fwd[4] == p1); -// Py_RETURN_NONE; -// #undef p4 -// #undef p3 -// #undef p2 -// #undef p1 -// #undef p0 -// } +static PyObject * +test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { +#if SIZEOF_VOID_P == 8 +#define p0 (void *)0x5555555555555555 +#define p1 (void *)0xaaaaaaaaaaaaaaaa +#define p2 (void *)0xfedcba9876543210 +#define p3 (void *)0x0123456789abcdef +#define p4 (void *)0x0f2d4b6987a5c3e1 +#else +#if SIZEOF_VOID_P == 4 +#define p0 (void *)0x55555555 +#define p1 (void *)0xaaaaaaaa +#define p2 (void *)0x76543210 +#define p3 (void *)0x01234567 +#define p4 (void *)0x07254361 +#else +#error "unexpected sizeof(void *), expecting 8 or 4" +#endif +#endif + void *back[5] = { p0, p2, p3, p4, p1 }; + assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); + assert(back[0] == p0); + assert(back[1] == p3); + assert(back[2] == p4); + assert(back[3] == p4); + assert(back[4] == p1); + void *fwd[5] = { p0, p2, p3, p4, p1 }; + assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); + assert(fwd[0] == p0); + assert(fwd[1] == p2); + assert(fwd[2] == p2); + assert(fwd[3] == p3); + assert(fwd[4] == p1); + Py_RETURN_NONE; +#undef p4 +#undef p3 +#undef p2 +#undef p1 +#undef p0 +} // NOTE: all tests should start with "test_atomic_" to be included // in test_pyatomic.py From 839f7729aa570f55b8079db2baf171abb47fd7fb Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 15:49:45 -0400 Subject: [PATCH 29/52] revert size_t to Py_ssize_t --- Include/cpython/pyatomic.h | 106 ++++++------- .../internal/pycore_pyatomic_ft_wrappers.h | 4 +- Modules/_testcapi/pyatomic.c | 142 +++++++++--------- 3 files changed, 126 insertions(+), 126 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 32f1c2c7b72476..5b84ab01e9be20 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,59 +545,6 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - - // --- _Py_atomic_fence ------------------------------------------------------ @@ -665,3 +612,56 @@ static inline void _Py_atomic_fence_release(void); #else # error "long must be 4 or 8 bytes in size" #endif // SIZEOF_LONG + + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 53683e1691dd65..1f22513142af01 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -115,9 +115,9 @@ extern "C" { #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) #define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) #else diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index e6052fe711023f..f911b2689c2245 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -5,7 +5,7 @@ */ #include "parts.h" -#include "pyconfig.h" // SIZEOF_VOID_P +// #include "pyconfig.h" // SIZEOF_VOID_P // We define atomic bitwise operations on these types #define FOR_BITWISE_TYPES(V) \ @@ -157,77 +157,77 @@ test_atomic_load_store_int_release_acquire(PyObject *self, PyObject *obj) { \ Py_RETURN_NONE; } -static PyObject * -test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { -#if SIZEOF_VOID_P == 8 -#define p0 (void *)0x5555555555555555 -#define p1 (void *)0xaaaaaaaaaaaaaaaa -#define p2 (void *)0xfedcba9876543210 -#define p3 (void *)0x0123456789abcdef -#else -#if SIZEOF_VOID_P == 4 -#define p0 (void *)0x55555555 -#define p1 (void *)0xaaaaaaaa -#define p2 (void *)0x76543210 -#define p3 (void *)0x01234567 -#else -#error "unexpected sizeof(void *), expecting 8 or 4" -#endif -#endif - void *src[4] = { (void *)0, p2, p3, (void *)0 }; - void *dst[4] = { p0, (void *)0, (void *)0, p1 }; - assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); - assert(dst[0] == p0); - assert(dst[1] == p2); - assert(dst[2] == p3); - assert(dst[3] == p1); - Py_RETURN_NONE; -#undef p3 -#undef p2 -#undef p1 -#undef p0 -} +// static PyObject * +// test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { +// #if SIZEOF_VOID_P == 8 +// #define p0 (void *)0x5555555555555555 +// #define p1 (void *)0xaaaaaaaaaaaaaaaa +// #define p2 (void *)0xfedcba9876543210 +// #define p3 (void *)0x0123456789abcdef +// #else +// #if SIZEOF_VOID_P == 4 +// #define p0 (void *)0x55555555 +// #define p1 (void *)0xaaaaaaaa +// #define p2 (void *)0x76543210 +// #define p3 (void *)0x01234567 +// #else +// #error "unexpected sizeof(void *), expecting 8 or 4" +// #endif +// #endif +// void *src[4] = { (void *)0, p2, p3, (void *)0 }; +// void *dst[4] = { p0, (void *)0, (void *)0, p1 }; +// assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); +// assert(dst[0] == p0); +// assert(dst[1] == p2); +// assert(dst[2] == p3); +// assert(dst[3] == p1); +// Py_RETURN_NONE; +// #undef p3 +// #undef p2 +// #undef p1 +// #undef p0 +// } -static PyObject * -test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { -#if SIZEOF_VOID_P == 8 -#define p0 (void *)0x5555555555555555 -#define p1 (void *)0xaaaaaaaaaaaaaaaa -#define p2 (void *)0xfedcba9876543210 -#define p3 (void *)0x0123456789abcdef -#define p4 (void *)0x0f2d4b6987a5c3e1 -#else -#if SIZEOF_VOID_P == 4 -#define p0 (void *)0x55555555 -#define p1 (void *)0xaaaaaaaa -#define p2 (void *)0x76543210 -#define p3 (void *)0x01234567 -#define p4 (void *)0x07254361 -#else -#error "unexpected sizeof(void *), expecting 8 or 4" -#endif -#endif - void *back[5] = { p0, p2, p3, p4, p1 }; - assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); - assert(back[0] == p0); - assert(back[1] == p3); - assert(back[2] == p4); - assert(back[3] == p4); - assert(back[4] == p1); - void *fwd[5] = { p0, p2, p3, p4, p1 }; - assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); - assert(fwd[0] == p0); - assert(fwd[1] == p2); - assert(fwd[2] == p2); - assert(fwd[3] == p3); - assert(fwd[4] == p1); - Py_RETURN_NONE; -#undef p4 -#undef p3 -#undef p2 -#undef p1 -#undef p0 -} +// static PyObject * +// test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { +// #if SIZEOF_VOID_P == 8 +// #define p0 (void *)0x5555555555555555 +// #define p1 (void *)0xaaaaaaaaaaaaaaaa +// #define p2 (void *)0xfedcba9876543210 +// #define p3 (void *)0x0123456789abcdef +// #define p4 (void *)0x0f2d4b6987a5c3e1 +// #else +// #if SIZEOF_VOID_P == 4 +// #define p0 (void *)0x55555555 +// #define p1 (void *)0xaaaaaaaa +// #define p2 (void *)0x76543210 +// #define p3 (void *)0x01234567 +// #define p4 (void *)0x07254361 +// #else +// #error "unexpected sizeof(void *), expecting 8 or 4" +// #endif +// #endif +// void *back[5] = { p0, p2, p3, p4, p1 }; +// assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); +// assert(back[0] == p0); +// assert(back[1] == p3); +// assert(back[2] == p4); +// assert(back[3] == p4); +// assert(back[4] == p1); +// void *fwd[5] = { p0, p2, p3, p4, p1 }; +// assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); +// assert(fwd[0] == p0); +// assert(fwd[1] == p2); +// assert(fwd[2] == p2); +// assert(fwd[3] == p3); +// assert(fwd[4] == p1); +// Py_RETURN_NONE; +// #undef p4 +// #undef p3 +// #undef p2 +// #undef p1 +// #undef p0 +// } // NOTE: all tests should start with "test_atomic_" to be included // in test_pyatomic.py From c8fb961986dade54482d38970db5ed0c1ffaab86 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 15:59:19 -0400 Subject: [PATCH 30/52] explicitly include string.h for test --- Include/cpython/pyatomic.h | 110 +++++++++--------- .../internal/pycore_pyatomic_ft_wrappers.h | 4 +- 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 5b84ab01e9be20..b685ccf305f6a6 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -87,6 +87,8 @@ # error "this header file must not be included directly" #endif +#include + // --- _Py_atomic_add -------------------------------------------------------- // Atomically adds `value` to `obj` and returns the previous value @@ -545,6 +547,61 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memcpy(dest, src, n); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); + + // if (dest != src) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); + + // for (; dest_ != end; dest_++, src_++) { + // _Py_atomic_store_ptr_relaxed(dest_, *src_); + // } + // } + + // return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + return memmove(dest, src, n); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); + + // if (dest < src || dest >= (void *)((char *)src + n)) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); + + // for (; dest_ != end; dest_++, src_++) { + // _Py_atomic_store_ptr_relaxed(dest_, *src_); + // } + // } + // else if (dest > src) { + // n = n / sizeof(void *) - 1; + // void **dest_ = (void **)dest + n; + // void **src_ = (void **)src + n; + // void **end = (void **)dest - 1; + + // for (; dest_ != end; dest_--, src_--) { + // _Py_atomic_store_ptr_relaxed(dest_, *src_); + // } + // } + + // return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ @@ -612,56 +669,3 @@ static inline void _Py_atomic_fence_release(void); #else # error "long must be 4 or 8 bytes in size" #endif // SIZEOF_LONG - - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) -{ - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, Py_ssize_t n) -{ - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 1f22513142af01..53683e1691dd65 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -115,9 +115,9 @@ extern "C" { #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #define FT_ATOMIC_MEMCPY_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) + _Py_atomic_memcpy_ptr_store_relaxed(dest, src, (size_t)(n)) #define FT_ATOMIC_MEMMOVE_PTR_STORE_RELAXED(dest, src, n) \ - _Py_atomic_memmove_ptr_store_relaxed(dest, src, (Py_ssize_t)(n)) + _Py_atomic_memmove_ptr_store_relaxed(dest, src, (size_t)(n)) #else From 32238cb3a87794f16527b47b3be5af4fae8b4b18 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:05:20 -0400 Subject: [PATCH 31/52] double check _Py_IS_ALIGNED() --- Include/cpython/pyatomic.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b685ccf305f6a6..f02111d4ef835d 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -552,6 +552,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); return memcpy(dest, src, n); // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); @@ -573,6 +576,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { + assert(_Py_IS_ALIGNED(dest, sizeof(void *))); + assert(_Py_IS_ALIGNED(src, sizeof(void *))); + assert(n % sizeof(void *) == 0); return memmove(dest, src, n); // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); From 08298eeccd89e2a24bcbe5c9ffcf1f36a33988a7 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:12:41 -0400 Subject: [PATCH 32/52] again --- Include/cpython/pyatomic.h | 56 ++------------------------------ Include/cpython/pyatomic_gcc.h | 54 +++++++++++++++++++++++++++++++ Include/cpython/pyatomic_msc.h | 54 +++++++++++++++++++++++++++++++ Include/cpython/pyatomic_std.h | 58 ++++++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 54 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index f02111d4ef835d..06ef5ca6481588 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -550,62 +550,10 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - return memcpy(dest, src, n); - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // if (dest != src) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); - - // for (; dest_ != end; dest_++, src_++) { - // _Py_atomic_store_ptr_relaxed(dest_, *src_); - // } - // } - - // return dest; -} +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(_Py_IS_ALIGNED(dest, sizeof(void *))); - assert(_Py_IS_ALIGNED(src, sizeof(void *))); - assert(n % sizeof(void *) == 0); - return memmove(dest, src, n); - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // if (dest < src || dest >= (void *)((char *)src + n)) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); - - // for (; dest_ != end; dest_++, src_++) { - // _Py_atomic_store_ptr_relaxed(dest_, *src_); - // } - // } - // else if (dest > src) { - // n = n / sizeof(void *) - 1; - // void **dest_ = (void **)dest + n; - // void **src_ = (void **)src + n; - // void **end = (void **)dest - 1; - - // for (; dest_ != end; dest_--, src_--) { - // _Py_atomic_store_ptr_relaxed(dest_, *src_); - // } - // } - - // return dest; -} +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b..db08f43594d99e 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,6 +600,60 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf..6b18d6376fa71f 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,6 +1154,60 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *(void * volatile *)dest_ = *src_; + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *(void * volatile *)dest_ = *src_; + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + *(void * volatile *)dest_ = *src_; + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5f..1f48a0b1c32df6 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,6 +1088,64 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + _Py_USING_STD; + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + _Py_USING_STD; + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ static inline void From 5d372b7a65d5353410ab7455e1625d00c60d9a71 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:19:09 -0400 Subject: [PATCH 33/52] and again --- Include/cpython/pyatomic.h | 2 -- Include/cpython/pyatomic_gcc.h | 60 +++++++++++++++++-------------- Include/cpython/pyatomic_msc.h | 60 +++++++++++++++++-------------- Include/cpython/pyatomic_std.h | 66 ++++++++++++++++++---------------- 4 files changed, 102 insertions(+), 86 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 06ef5ca6481588..fe500399e7e536 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -87,8 +87,6 @@ # error "this header file must not be included directly" #endif -#include - // --- _Py_atomic_add -------------------------------------------------------- // Atomically adds `value` to `obj` and returns the previous value diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index db08f43594d99e..188cac1596eaa8 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -603,6 +603,8 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ +#include + static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { @@ -610,15 +612,17 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); + memcpy(dest, src, n); + + // if (dest != src) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); - for (; dest_ != end; dest_++, src_++) { - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } + // for (; dest_ != end; dest_++, src_++) { + // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + // } + // } return dest; } @@ -630,25 +634,27 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } + memmove(dest, src, n); + + // if (dest < src || dest >= (void *)((char *)src + n)) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); + + // for (; dest_ != end; dest_++, src_++) { + // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + // } + // } + // else if (dest > src) { + // n = n / sizeof(void *) - 1; + // void **dest_ = (void **)dest + n; + // void **src_ = (void **)src + n; + // void **end = (void **)dest - 1; + + // for (; dest_ != end; dest_--, src_--) { + // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + // } + // } return dest; } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 6b18d6376fa71f..9f9359fc3b129f 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1157,6 +1157,8 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ +#include + static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { @@ -1164,15 +1166,17 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); + memcpy(dest, src, n); - for (; dest_ != end; dest_++, src_++) { - *(void * volatile *)dest_ = *src_; - } - } + // if (dest != src) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); + + // for (; dest_ != end; dest_++, src_++) { + // *(void * volatile *)dest_ = *src_; + // } + // } return dest; } @@ -1184,25 +1188,27 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - *(void * volatile *)dest_ = *src_; - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - *(void * volatile *)dest_ = *src_; - } - } + memmove(dest, src, n); + + // if (dest < src || dest >= (void *)((char *)src + n)) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); + + // for (; dest_ != end; dest_++, src_++) { + // *(void * volatile *)dest_ = *src_; + // } + // } + // else if (dest > src) { + // n = n / sizeof(void *) - 1; + // void **dest_ = (void **)dest + n; + // void **src_ = (void **)src + n; + // void **end = (void **)dest - 1; + + // for (; dest_ != end; dest_--, src_--) { + // *(void * volatile *)dest_ = *src_; + // } + // } return dest; } diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 1f48a0b1c32df6..ee482f78fc97c1 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1090,6 +1090,8 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ +#include + static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { @@ -1098,16 +1100,18 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); + memcpy(dest, src, n); + + // if (dest != src) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); - for (; dest_ != end; dest_++, src_++) { - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } + // for (; dest_ != end; dest_++, src_++) { + // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + // memory_order_relaxed); + // } + // } return dest; } @@ -1120,27 +1124,29 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } + memmove(dest, src, n); + + // if (dest < src || dest >= (void *)((char *)src + n)) { + // void **dest_ = (void **)dest; + // void **src_ = (void **)src; + // void **end = dest_ + n / sizeof(void *); + + // for (; dest_ != end; dest_++, src_++) { + // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + // memory_order_relaxed); + // } + // } + // else if (dest > src) { + // n = n / sizeof(void *) - 1; + // void **dest_ = (void **)dest + n; + // void **src_ = (void **)src + n; + // void **end = (void **)dest - 1; + + // for (; dest_ != end; dest_--, src_--) { + // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + // memory_order_relaxed); + // } + // } return dest; } From a6eb55002bbb08f28de6625d10819745b136cf0f Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:23:46 -0400 Subject: [PATCH 34/52] uintptr_t? --- Include/cpython/pyatomic_gcc.h | 8 ++++---- Include/cpython/pyatomic_msc.h | 8 ++++---- Include/cpython/pyatomic_std.h | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 188cac1596eaa8..1f69b97b2f2178 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -608,8 +608,8 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); memcpy(dest, src, n); @@ -630,8 +630,8 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 9f9359fc3b129f..e27cdb2580eb1d 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1162,8 +1162,8 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); memcpy(dest, src, n); @@ -1184,8 +1184,8 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index ee482f78fc97c1..ebea8bedde4e34 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1096,8 +1096,8 @@ static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { _Py_USING_STD; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); memcpy(dest, src, n); @@ -1120,8 +1120,8 @@ static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { _Py_USING_STD; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); assert(n % sizeof(void *) == 0); memmove(dest, src, n); From e2c5f2ecbbb99a3f6eb7609f6c9f511f05429528 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:27:02 -0400 Subject: [PATCH 35/52] completely empty functions --- Include/cpython/pyatomic_gcc.h | 10 +++++----- Include/cpython/pyatomic_msc.h | 10 +++++----- Include/cpython/pyatomic_std.h | 14 +++++++------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1f69b97b2f2178..48aef95e0cf4cd 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -603,16 +603,16 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ -#include +// #include static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(n % sizeof(void *) == 0); - memcpy(dest, src, n); + // memcpy(dest, src, n); // if (dest != src) { // void **dest_ = (void **)dest; @@ -632,9 +632,9 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(n % sizeof(void *) == 0); - memmove(dest, src, n); + // memmove(dest, src, n); // if (dest < src || dest >= (void *)((char *)src + n)) { // void **dest_ = (void **)dest; diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index e27cdb2580eb1d..621ed1418f4cc2 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1157,16 +1157,16 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ -#include +// #include static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(n % sizeof(void *) == 0); - memcpy(dest, src, n); + // memcpy(dest, src, n); // if (dest != src) { // void **dest_ = (void **)dest; @@ -1186,9 +1186,9 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(n % sizeof(void *) == 0); - memmove(dest, src, n); + // memmove(dest, src, n); // if (dest < src || dest >= (void *)((char *)src + n)) { // void **dest_ = (void **)dest; diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index ebea8bedde4e34..31529c661e67c5 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1090,17 +1090,17 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ -#include +// #include static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - _Py_USING_STD; + // _Py_USING_STD; // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(n % sizeof(void *) == 0); - memcpy(dest, src, n); + // memcpy(dest, src, n); // if (dest != src) { // void **dest_ = (void **)dest; @@ -1119,12 +1119,12 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - _Py_USING_STD; + // _Py_USING_STD; // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(n % sizeof(void *) == 0); - memmove(dest, src, n); + // memmove(dest, src, n); // if (dest < src || dest >= (void *)((char *)src + n)) { // void **dest_ = (void **)dest; From c3a23fbcdc2c9f901321c80d145b30821e42d9f4 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:33:25 -0400 Subject: [PATCH 36/52] loop and normal store --- Include/cpython/pyatomic_gcc.h | 57 +++++++++++++++-------------- Include/cpython/pyatomic_msc.h | 59 +++++++++++++++--------------- Include/cpython/pyatomic_std.h | 65 ++++++++++++++++++---------------- 3 files changed, 95 insertions(+), 86 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 48aef95e0cf4cd..240aff2cf765cc 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -614,15 +614,16 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) // memcpy(dest, src, n); - // if (dest != src) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); - // for (; dest_ != end; dest_++, src_++) { - // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - // } - // } + for (; dest_ != end; dest_++, src_++) { + *dest_ = *src_; + // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } return dest; } @@ -636,25 +637,27 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) // memmove(dest, src, n); - // if (dest < src || dest >= (void *)((char *)src + n)) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); - - // for (; dest_ != end; dest_++, src_++) { - // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - // } - // } - // else if (dest > src) { - // n = n / sizeof(void *) - 1; - // void **dest_ = (void **)dest + n; - // void **src_ = (void **)src + n; - // void **end = (void **)dest - 1; - - // for (; dest_ != end; dest_--, src_--) { - // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - // } - // } + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *dest_ = *src_; + // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + *dest_ = *src_; + // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + } + } return dest; } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 621ed1418f4cc2..759a80fc6630b3 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1168,15 +1168,16 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) // memcpy(dest, src, n); - // if (dest != src) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); - - // for (; dest_ != end; dest_++, src_++) { - // *(void * volatile *)dest_ = *src_; - // } - // } + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *dest_ = *src_; + // *(void * volatile *)dest_ = *src_; + } + } return dest; } @@ -1190,25 +1191,27 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) // memmove(dest, src, n); - // if (dest < src || dest >= (void *)((char *)src + n)) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); - - // for (; dest_ != end; dest_++, src_++) { - // *(void * volatile *)dest_ = *src_; - // } - // } - // else if (dest > src) { - // n = n / sizeof(void *) - 1; - // void **dest_ = (void **)dest + n; - // void **src_ = (void **)src + n; - // void **end = (void **)dest - 1; - - // for (; dest_ != end; dest_--, src_--) { - // *(void * volatile *)dest_ = *src_; - // } - // } + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *dest_ = *src_; + // *(void * volatile *)dest_ = *src_; + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + *dest_ = *src_; + // *(void * volatile *)dest_ = *src_; + } + } return dest; } diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 31529c661e67c5..275e9eb65f65fe 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1102,16 +1102,17 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) // memcpy(dest, src, n); - // if (dest != src) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); - - // for (; dest_ != end; dest_++, src_++) { - // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - // memory_order_relaxed); - // } - // } + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *dest_ = *src_; + // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + // memory_order_relaxed); + } + } return dest; } @@ -1126,27 +1127,29 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) // memmove(dest, src, n); - // if (dest < src || dest >= (void *)((char *)src + n)) { - // void **dest_ = (void **)dest; - // void **src_ = (void **)src; - // void **end = dest_ + n / sizeof(void *); - - // for (; dest_ != end; dest_++, src_++) { - // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - // memory_order_relaxed); - // } - // } - // else if (dest > src) { - // n = n / sizeof(void *) - 1; - // void **dest_ = (void **)dest + n; - // void **src_ = (void **)src + n; - // void **end = (void **)dest - 1; - - // for (; dest_ != end; dest_--, src_--) { - // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - // memory_order_relaxed); - // } - // } + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + *dest_ = *src_; + // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + // memory_order_relaxed); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + *dest_ = *src_; + // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + // memory_order_relaxed); + } + } return dest; } From 58136554e649bbdf6770b39bd4f775d3c7b37e73 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:37:27 -0400 Subject: [PATCH 37/52] put back assert --- Include/cpython/pyatomic_gcc.h | 12 ++++++------ Include/cpython/pyatomic_msc.h | 12 ++++++------ Include/cpython/pyatomic_std.h | 14 +++++++------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 240aff2cf765cc..3c51c2e8664fef 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -608,9 +608,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -631,9 +631,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 759a80fc6630b3..76e6c79521e37c 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1162,9 +1162,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1185,9 +1185,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 275e9eb65f65fe..fcbe58720eab76 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1096,9 +1096,9 @@ static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { // _Py_USING_STD; - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1121,10 +1121,10 @@ static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { // _Py_USING_STD; - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - + assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + assert(n % sizeof(void *) == 0); +` // memmove(dest, src, n); if (dest < src || dest >= (void *)((char *)src + n)) { From 635d2170b3c40dd7fb390dfccd0130a216cbd6c0 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:41:30 -0400 Subject: [PATCH 38/52] _Py_USING_STD and normal store --- Include/cpython/pyatomic_gcc.h | 12 ++++++------ Include/cpython/pyatomic_msc.h | 12 ++++++------ Include/cpython/pyatomic_std.h | 18 +++++++++--------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 3c51c2e8664fef..240aff2cf765cc 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -608,9 +608,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -631,9 +631,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 76e6c79521e37c..759a80fc6630b3 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1162,9 +1162,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1185,9 +1185,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index fcbe58720eab76..357e106d86142f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1095,10 +1095,10 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - // _Py_USING_STD; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + _Py_USING_STD; + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1120,11 +1120,11 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - // _Py_USING_STD; - assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); -` + _Py_USING_STD; + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); + // memmove(dest, src, n); if (dest < src || dest >= (void *)((char *)src + n)) { From 088c40ed1a9d3752e585d961d09c799054328a2e Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:44:35 -0400 Subject: [PATCH 39/52] pyatomic_gcc store atomic --- Include/cpython/pyatomic_gcc.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 240aff2cf765cc..0e78a457b5fff6 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -620,8 +620,8 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { - *dest_ = *src_; - // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + // *dest_ = *src_; + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); } } @@ -643,8 +643,8 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { - *dest_ = *src_; - // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + // *dest_ = *src_; + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); } } else if (dest > src) { @@ -654,8 +654,8 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = (void **)dest - 1; for (; dest_ != end; dest_--, src_--) { - *dest_ = *src_; - // __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); + // *dest_ = *src_; + __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); } } From 2063f73a48b1a9a6f1766685ce39a497be5868c3 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:48:18 -0400 Subject: [PATCH 40/52] pyatomic_std store atomic --- Include/cpython/pyatomic_std.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 357e106d86142f..9199ae699ffb42 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1108,9 +1108,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { - *dest_ = *src_; - // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - // memory_order_relaxed); + // *dest_ = *src_; + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); } } @@ -1133,9 +1133,9 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { - *dest_ = *src_; - // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - // memory_order_relaxed); + // *dest_ = *src_; + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); } } else if (dest > src) { @@ -1145,9 +1145,9 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = (void **)dest - 1; for (; dest_ != end; dest_--, src_--) { - *dest_ = *src_; - // atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - // memory_order_relaxed); + // *dest_ = *src_; + atomic_store_explicit((_Atomic(void*)*)dest_, *src_, + memory_order_relaxed); } } From 11b616c1fb497f6e0e5760ebfe8ea62d5e291e85 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:52:20 -0400 Subject: [PATCH 41/52] pyatomic_msc store atomic --- Include/cpython/pyatomic_msc.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 759a80fc6630b3..1b0915c3ff3013 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1174,8 +1174,8 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { - *dest_ = *src_; - // *(void * volatile *)dest_ = *src_; + // *dest_ = *src_; + *(void * volatile *)dest_ = *src_; } } @@ -1197,8 +1197,8 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = dest_ + n / sizeof(void *); for (; dest_ != end; dest_++, src_++) { - *dest_ = *src_; - // *(void * volatile *)dest_ = *src_; + // *dest_ = *src_; + *(void * volatile *)dest_ = *src_; } } else if (dest > src) { @@ -1208,8 +1208,8 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) void **end = (void **)dest - 1; for (; dest_ != end; dest_--, src_--) { - *dest_ = *src_; - // *(void * volatile *)dest_ = *src_; + // *dest_ = *src_; + *(void * volatile *)dest_ = *src_; } } From 191c2619bf64afb3ff88a07a304b709c7cdf68ec Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 16:56:54 -0400 Subject: [PATCH 42/52] put back assert n --- Include/cpython/pyatomic_gcc.h | 4 ++-- Include/cpython/pyatomic_msc.h | 4 ++-- Include/cpython/pyatomic_std.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 0e78a457b5fff6..b98da0b2bbdfb1 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -610,7 +610,7 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -633,7 +633,7 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 1b0915c3ff3013..aa9592f64ef3be 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1164,7 +1164,7 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1187,7 +1187,7 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 9199ae699ffb42..139cc0571da80d 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1098,7 +1098,7 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) _Py_USING_STD; // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1123,7 +1123,7 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) _Py_USING_STD; // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + assert(n % sizeof(void *) == 0); // memmove(dest, src, n); From 0784d1912e0257278a6ecbf75fe5fed06024501e Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:02:25 -0400 Subject: [PATCH 43/52] typecase to size_t in assert --- Include/cpython/pyatomic_gcc.h | 4 ++-- Include/cpython/pyatomic_msc.h | 4 ++-- Include/cpython/pyatomic_std.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index b98da0b2bbdfb1..1ec4a14a7d484a 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -610,7 +610,7 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(n % (size_t)sizeof(void *) == 0); // memcpy(dest, src, n); @@ -633,7 +633,7 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(n % (size_t)sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index aa9592f64ef3be..960eaafff92812 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1164,7 +1164,7 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(n % (size_t)sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1187,7 +1187,7 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(n % (size_t)sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 139cc0571da80d..15ff1c7c431bff 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1098,7 +1098,7 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) _Py_USING_STD; // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(n % (size_t)sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1123,7 +1123,7 @@ _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) _Py_USING_STD; // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % sizeof(void *) == 0); + assert(n % (size_t)sizeof(void *) == 0); // memmove(dest, src, n); From 154a132ca9306991333c30dc72ffb1f58bfa5bf1 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:06:53 -0400 Subject: [PATCH 44/52] size assert no work, try other with cast --- Include/cpython/pyatomic_gcc.h | 12 ++++++------ Include/cpython/pyatomic_msc.h | 12 ++++++------ Include/cpython/pyatomic_std.h | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1ec4a14a7d484a..391dc7bfbbc7a0 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -608,9 +608,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -631,9 +631,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 960eaafff92812..ebeea706812ac4 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1162,9 +1162,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1185,9 +1185,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memmove(dest, src, n); diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 15ff1c7c431bff..cb4f07dd21316b 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1096,9 +1096,9 @@ static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { _Py_USING_STD; - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memcpy(dest, src, n); @@ -1121,9 +1121,9 @@ static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { _Py_USING_STD; - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); // memmove(dest, src, n); From a5f4d9e0edcce2aee92eaa25f629c42206c7a014 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:10:16 -0400 Subject: [PATCH 45/52] last test of copy at pyatomic.h level --- Include/cpython/pyatomic.h | 62 +++++++++++++--- Include/cpython/pyatomic_gcc.h | 120 +++++++++++++++--------------- Include/cpython/pyatomic_msc.h | 120 +++++++++++++++--------------- Include/cpython/pyatomic_std.h | 130 ++++++++++++++++----------------- 4 files changed, 238 insertions(+), 194 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index fe500399e7e536..26e09e184be8fb 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,15 +545,6 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n); - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n); - - // --- _Py_atomic_fence ------------------------------------------------------ @@ -621,3 +612,56 @@ static inline void _Py_atomic_fence_release(void); #else # error "long must be 4 or 8 bytes in size" #endif // SIZEOF_LONG + + +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 391dc7bfbbc7a0..8ff0e0f158d883 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -601,66 +601,66 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -// #include - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // memcpy(dest, src, n); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - // *dest_ = *src_; - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // memmove(dest, src, n); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - // *dest_ = *src_; - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - // *dest_ = *src_; - __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); - } - } - - return dest; -} +// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +// // #include + +// static inline void * +// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); +// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); +// // assert(n % sizeof(void *) == 0); + +// // memcpy(dest, src, n); + +// if (dest != src) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// // *dest_ = *src_; +// __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); +// } +// } + +// return dest; +// } + +// static inline void * +// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); +// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); +// // assert(n % sizeof(void *) == 0); + +// // memmove(dest, src, n); + +// if (dest < src || dest >= (void *)((char *)src + n)) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// // *dest_ = *src_; +// __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); +// } +// } +// else if (dest > src) { +// n = n / sizeof(void *) - 1; +// void **dest_ = (void **)dest + n; +// void **src_ = (void **)src + n; +// void **end = (void **)dest - 1; + +// for (; dest_ != end; dest_--, src_--) { +// // *dest_ = *src_; +// __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); +// } +// } + +// return dest; +// } // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index ebeea706812ac4..3062d6e3700a93 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1155,66 +1155,66 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -// #include - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // memcpy(dest, src, n); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - // *dest_ = *src_; - *(void * volatile *)dest_ = *src_; - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // memmove(dest, src, n); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - // *dest_ = *src_; - *(void * volatile *)dest_ = *src_; - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - // *dest_ = *src_; - *(void * volatile *)dest_ = *src_; - } - } - - return dest; -} +// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +// // #include + +// static inline void * +// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); +// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); +// // assert(n % sizeof(void *) == 0); + +// // memcpy(dest, src, n); + +// if (dest != src) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// // *dest_ = *src_; +// *(void * volatile *)dest_ = *src_; +// } +// } + +// return dest; +// } + +// static inline void * +// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); +// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); +// // assert(n % sizeof(void *) == 0); + +// // memmove(dest, src, n); + +// if (dest < src || dest >= (void *)((char *)src + n)) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// // *dest_ = *src_; +// *(void * volatile *)dest_ = *src_; +// } +// } +// else if (dest > src) { +// n = n / sizeof(void *) - 1; +// void **dest_ = (void **)dest + n; +// void **src_ = (void **)src + n; +// void **end = (void **)dest - 1; + +// for (; dest_ != end; dest_--, src_--) { +// // *dest_ = *src_; +// *(void * volatile *)dest_ = *src_; +// } +// } + +// return dest; +// } // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index cb4f07dd21316b..232e135ddb8364 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,71 +1088,71 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -// #include - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - _Py_USING_STD; - assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // memcpy(dest, src, n); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - // *dest_ = *src_; - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - _Py_USING_STD; - assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - // memmove(dest, src, n); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - // *dest_ = *src_; - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - // *dest_ = *src_; - atomic_store_explicit((_Atomic(void*)*)dest_, *src_, - memory_order_relaxed); - } - } - - return dest; -} +// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +// // #include + +// static inline void * +// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// _Py_USING_STD; +// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); +// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); +// // assert(n % sizeof(void *) == 0); + +// // memcpy(dest, src, n); + +// if (dest != src) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// // *dest_ = *src_; +// atomic_store_explicit((_Atomic(void*)*)dest_, *src_, +// memory_order_relaxed); +// } +// } + +// return dest; +// } + +// static inline void * +// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +// { +// _Py_USING_STD; +// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); +// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); +// // assert(n % sizeof(void *) == 0); + +// // memmove(dest, src, n); + +// if (dest < src || dest >= (void *)((char *)src + n)) { +// void **dest_ = (void **)dest; +// void **src_ = (void **)src; +// void **end = dest_ + n / sizeof(void *); + +// for (; dest_ != end; dest_++, src_++) { +// // *dest_ = *src_; +// atomic_store_explicit((_Atomic(void*)*)dest_, *src_, +// memory_order_relaxed); +// } +// } +// else if (dest > src) { +// n = n / sizeof(void *) - 1; +// void **dest_ = (void **)dest + n; +// void **src_ = (void **)src + n; +// void **end = (void **)dest - 1; + +// for (; dest_ != end; dest_--, src_--) { +// // *dest_ = *src_; +// atomic_store_explicit((_Atomic(void*)*)dest_, *src_, +// memory_order_relaxed); +// } +// } + +// return dest; +// } // --- _Py_atomic_fence ------------------------------------------------------ From 73c7cc1f2fe67ab3042d7682d7d805ae7ab5e62a Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:14:26 -0400 Subject: [PATCH 46/52] test location dependence --- Include/cpython/pyatomic.h | 106 ++++++++++++++++----------------- Include/cpython/pyatomic_gcc.h | 63 -------------------- Include/cpython/pyatomic_msc.h | 63 -------------------- Include/cpython/pyatomic_std.h | 67 --------------------- 4 files changed, 53 insertions(+), 246 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 26e09e184be8fb..6cdff0b19f35f2 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -545,6 +545,59 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); +// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ + +static inline void * +_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); + + if (dest != src) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + +static inline void * +_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) +{ + // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); + // assert(n % sizeof(void *) == 0); + + if (dest < src || dest >= (void *)((char *)src + n)) { + void **dest_ = (void **)dest; + void **src_ = (void **)src; + void **end = dest_ + n / sizeof(void *); + + for (; dest_ != end; dest_++, src_++) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + else if (dest > src) { + n = n / sizeof(void *) - 1; + void **dest_ = (void **)dest + n; + void **src_ = (void **)src + n; + void **end = (void **)dest - 1; + + for (; dest_ != end; dest_--, src_--) { + _Py_atomic_store_ptr_relaxed(dest_, *src_); + } + } + + return dest; +} + + // --- _Py_atomic_fence ------------------------------------------------------ @@ -612,56 +665,3 @@ static inline void _Py_atomic_fence_release(void); #else # error "long must be 4 or 8 bytes in size" #endif // SIZEOF_LONG - - -// --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -static inline void * -_Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - if (dest != src) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} - -static inline void * -_Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -{ - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); - - if (dest < src || dest >= (void *)((char *)src + n)) { - void **dest_ = (void **)dest; - void **src_ = (void **)src; - void **end = dest_ + n / sizeof(void *); - - for (; dest_ != end; dest_++, src_++) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - else if (dest > src) { - n = n / sizeof(void *) - 1; - void **dest_ = (void **)dest + n; - void **src_ = (void **)src + n; - void **end = (void **)dest - 1; - - for (; dest_ != end; dest_--, src_--) { - _Py_atomic_store_ptr_relaxed(dest_, *src_); - } - } - - return dest; -} diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 8ff0e0f158d883..1566b83b9f6a1b 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -600,69 +600,6 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } - -// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -// // #include - -// static inline void * -// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); -// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); -// // assert(n % sizeof(void *) == 0); - -// // memcpy(dest, src, n); - -// if (dest != src) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// // *dest_ = *src_; -// __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); -// } -// } - -// return dest; -// } - -// static inline void * -// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); -// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); -// // assert(n % sizeof(void *) == 0); - -// // memmove(dest, src, n); - -// if (dest < src || dest >= (void *)((char *)src + n)) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// // *dest_ = *src_; -// __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); -// } -// } -// else if (dest > src) { -// n = n / sizeof(void *) - 1; -// void **dest_ = (void **)dest + n; -// void **src_ = (void **)src + n; -// void **end = (void **)dest - 1; - -// for (; dest_ != end; dest_--, src_--) { -// // *dest_ = *src_; -// __atomic_store_n((void **)dest_, *src_, __ATOMIC_RELAXED); -// } -// } - -// return dest; -// } - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 3062d6e3700a93..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1154,69 +1154,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) #endif } - -// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -// // #include - -// static inline void * -// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); -// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); -// // assert(n % sizeof(void *) == 0); - -// // memcpy(dest, src, n); - -// if (dest != src) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// // *dest_ = *src_; -// *(void * volatile *)dest_ = *src_; -// } -// } - -// return dest; -// } - -// static inline void * -// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); -// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); -// // assert(n % sizeof(void *) == 0); - -// // memmove(dest, src, n); - -// if (dest < src || dest >= (void *)((char *)src + n)) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// // *dest_ = *src_; -// *(void * volatile *)dest_ = *src_; -// } -// } -// else if (dest > src) { -// n = n / sizeof(void *) - 1; -// void **dest_ = (void **)dest + n; -// void **src_ = (void **)src + n; -// void **end = (void **)dest - 1; - -// for (; dest_ != end; dest_--, src_--) { -// // *dest_ = *src_; -// *(void * volatile *)dest_ = *src_; -// } -// } - -// return dest; -// } - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 232e135ddb8364..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1088,73 +1088,6 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) } -// // --- _Py_atomic_memcpy / _Py_atomic_memmove ------------ - -// // #include - -// static inline void * -// _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// _Py_USING_STD; -// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); -// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); -// // assert(n % sizeof(void *) == 0); - -// // memcpy(dest, src, n); - -// if (dest != src) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// // *dest_ = *src_; -// atomic_store_explicit((_Atomic(void*)*)dest_, *src_, -// memory_order_relaxed); -// } -// } - -// return dest; -// } - -// static inline void * -// _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) -// { -// _Py_USING_STD; -// // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); -// // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); -// // assert(n % sizeof(void *) == 0); - -// // memmove(dest, src, n); - -// if (dest < src || dest >= (void *)((char *)src + n)) { -// void **dest_ = (void **)dest; -// void **src_ = (void **)src; -// void **end = dest_ + n / sizeof(void *); - -// for (; dest_ != end; dest_++, src_++) { -// // *dest_ = *src_; -// atomic_store_explicit((_Atomic(void*)*)dest_, *src_, -// memory_order_relaxed); -// } -// } -// else if (dest > src) { -// n = n / sizeof(void *) - 1; -// void **dest_ = (void **)dest + n; -// void **src_ = (void **)src + n; -// void **end = (void **)dest - 1; - -// for (; dest_ != end; dest_--, src_--) { -// // *dest_ = *src_; -// atomic_store_explicit((_Atomic(void*)*)dest_, *src_, -// memory_order_relaxed); -// } -// } - -// return dest; -// } - - // --- _Py_atomic_fence ------------------------------------------------------ static inline void From 93b90c73d4109e74d38687301dcb6a111d40ae9a Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:18:21 -0400 Subject: [PATCH 47/52] put back test --- Modules/_testcapi/pyatomic.c | 146 +++++++++++++++++------------------ 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index f911b2689c2245..ff835adc49ad28 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -5,7 +5,7 @@ */ #include "parts.h" -// #include "pyconfig.h" // SIZEOF_VOID_P +#include "pyconfig.h" // SIZEOF_VOID_P // We define atomic bitwise operations on these types #define FOR_BITWISE_TYPES(V) \ @@ -157,77 +157,77 @@ test_atomic_load_store_int_release_acquire(PyObject *self, PyObject *obj) { \ Py_RETURN_NONE; } -// static PyObject * -// test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { -// #if SIZEOF_VOID_P == 8 -// #define p0 (void *)0x5555555555555555 -// #define p1 (void *)0xaaaaaaaaaaaaaaaa -// #define p2 (void *)0xfedcba9876543210 -// #define p3 (void *)0x0123456789abcdef -// #else -// #if SIZEOF_VOID_P == 4 -// #define p0 (void *)0x55555555 -// #define p1 (void *)0xaaaaaaaa -// #define p2 (void *)0x76543210 -// #define p3 (void *)0x01234567 -// #else -// #error "unexpected sizeof(void *), expecting 8 or 4" -// #endif -// #endif -// void *src[4] = { (void *)0, p2, p3, (void *)0 }; -// void *dst[4] = { p0, (void *)0, (void *)0, p1 }; -// assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); -// assert(dst[0] == p0); -// assert(dst[1] == p2); -// assert(dst[2] == p3); -// assert(dst[3] == p1); -// Py_RETURN_NONE; -// #undef p3 -// #undef p2 -// #undef p1 -// #undef p0 -// } +static PyObject * +test_atomic_memcpy_ptr_store_relaxed(PyObject *self, PyObject *obj) { +#if SIZEOF_VOID_P == 8 +#define p0 (void *)0x5555555555555555 +#define p1 (void *)0xaaaaaaaaaaaaaaaa +#define p2 (void *)0xfedcba9876543210 +#define p3 (void *)0x0123456789abcdef +#else +#if SIZEOF_VOID_P == 4 +#define p0 (void *)0x55555555 +#define p1 (void *)0xaaaaaaaa +#define p2 (void *)0x76543210 +#define p3 (void *)0x01234567 +#else +#error "unexpected sizeof(void *), expecting 8 or 4" +#endif +#endif + void *src[4] = { (void *)0, p2, p3, (void *)0 }; + void *dst[4] = { p0, (void *)0, (void *)0, p1 }; + assert(_Py_atomic_memcpy_ptr_store_relaxed(&dst[1], &src[1], SIZEOF_VOID_P * 2) == &dst[1]); + assert(dst[0] == p0); + assert(dst[1] == p2); + assert(dst[2] == p3); + assert(dst[3] == p1); + Py_RETURN_NONE; +#undef p3 +#undef p2 +#undef p1 +#undef p0 +} -// static PyObject * -// test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { -// #if SIZEOF_VOID_P == 8 -// #define p0 (void *)0x5555555555555555 -// #define p1 (void *)0xaaaaaaaaaaaaaaaa -// #define p2 (void *)0xfedcba9876543210 -// #define p3 (void *)0x0123456789abcdef -// #define p4 (void *)0x0f2d4b6987a5c3e1 -// #else -// #if SIZEOF_VOID_P == 4 -// #define p0 (void *)0x55555555 -// #define p1 (void *)0xaaaaaaaa -// #define p2 (void *)0x76543210 -// #define p3 (void *)0x01234567 -// #define p4 (void *)0x07254361 -// #else -// #error "unexpected sizeof(void *), expecting 8 or 4" -// #endif -// #endif -// void *back[5] = { p0, p2, p3, p4, p1 }; -// assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); -// assert(back[0] == p0); -// assert(back[1] == p3); -// assert(back[2] == p4); -// assert(back[3] == p4); -// assert(back[4] == p1); -// void *fwd[5] = { p0, p2, p3, p4, p1 }; -// assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); -// assert(fwd[0] == p0); -// assert(fwd[1] == p2); -// assert(fwd[2] == p2); -// assert(fwd[3] == p3); -// assert(fwd[4] == p1); -// Py_RETURN_NONE; -// #undef p4 -// #undef p3 -// #undef p2 -// #undef p1 -// #undef p0 -// } +static PyObject * +test_atomic_memmove_ptr_store_relaxed(PyObject *self, PyObject *obj) { +#if SIZEOF_VOID_P == 8 +#define p0 (void *)0x5555555555555555 +#define p1 (void *)0xaaaaaaaaaaaaaaaa +#define p2 (void *)0xfedcba9876543210 +#define p3 (void *)0x0123456789abcdef +#define p4 (void *)0x0f2d4b6987a5c3e1 +#else +#if SIZEOF_VOID_P == 4 +#define p0 (void *)0x55555555 +#define p1 (void *)0xaaaaaaaa +#define p2 (void *)0x76543210 +#define p3 (void *)0x01234567 +#define p4 (void *)0x07254361 +#else +#error "unexpected sizeof(void *), expecting 8 or 4" +#endif +#endif + void *back[5] = { p0, p2, p3, p4, p1 }; + assert(_Py_atomic_memmove_ptr_store_relaxed(&back[1], &back[2], SIZEOF_VOID_P * 2) == &back[1]); + assert(back[0] == p0); + assert(back[1] == p3); + assert(back[2] == p4); + assert(back[3] == p4); + assert(back[4] == p1); + void *fwd[5] = { p0, p2, p3, p4, p1 }; + assert(_Py_atomic_memmove_ptr_store_relaxed(&fwd[2], &fwd[1], SIZEOF_VOID_P * 2) == &fwd[2]); + assert(fwd[0] == p0); + assert(fwd[1] == p2); + assert(fwd[2] == p2); + assert(fwd[3] == p3); + assert(fwd[4] == p1); + Py_RETURN_NONE; +#undef p4 +#undef p3 +#undef p2 +#undef p1 +#undef p0 +} // NOTE: all tests should start with "test_atomic_" to be included // in test_pyatomic.py @@ -252,8 +252,8 @@ static PyMethodDef test_methods[] = { {"test_atomic_fences", test_atomic_fences, METH_NOARGS}, {"test_atomic_release_acquire", test_atomic_release_acquire, METH_NOARGS}, {"test_atomic_load_store_int_release_acquire", test_atomic_load_store_int_release_acquire, METH_NOARGS}, - // {"test_atomic_memcpy_ptr_store_relaxed", test_atomic_memcpy_ptr_store_relaxed, METH_NOARGS}, - // {"test_atomic_memmove_ptr_store_relaxed", test_atomic_memmove_ptr_store_relaxed, METH_NOARGS}, + {"test_atomic_memcpy_ptr_store_relaxed", test_atomic_memcpy_ptr_store_relaxed, METH_NOARGS}, + {"test_atomic_memmove_ptr_store_relaxed", test_atomic_memmove_ptr_store_relaxed, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; From bf15e937e1d753daf14c026dc35279c14b9a75d7 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:31:12 -0400 Subject: [PATCH 48/52] test basic assert --- Include/cpython/pyatomic.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 6cdff0b19f35f2..71dec11d4fe0e0 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -550,6 +550,7 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { + assert(1 == 1); // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); // assert(n % sizeof(void *) == 0); From 39bee16b73762e011c178803f487b2754aa9660a Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:36:13 -0400 Subject: [PATCH 49/52] leave assertions commented out --- Include/cpython/pyatomic.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 71dec11d4fe0e0..68b2a7f1653b96 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -550,10 +550,12 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - assert(1 == 1); - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + // XXX: The assertions below currently fail on Android ARM build. + // In fact, assert(1 == 1) fails on that build. + + // assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % (size_t)sizeof(void *) == 0); if (dest != src) { void **dest_ = (void **)dest; @@ -571,9 +573,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (sizeof (void *) - 1)) == 0); - // assert(n % sizeof(void *) == 0); + // assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + // assert(n % (size_t)sizeof(void *) == 0); if (dest < src || dest >= (void *)((char *)src + n)) { void **dest_ = (void **)dest; From 6f4ef28009d48906f44187cf82d8e2635226bd4b Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:39:08 -0400 Subject: [PATCH 50/52] test assert(1 == 1) with latest main --- Include/cpython/pyatomic.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 68b2a7f1653b96..f8062f969ef4d9 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -553,6 +553,8 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) // XXX: The assertions below currently fail on Android ARM build. // In fact, assert(1 == 1) fails on that build. + assert(1 == 1); + // assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); // assert(n % (size_t)sizeof(void *) == 0); From c2de54273bff1d61d41b8eda9488facc71a3da97 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:42:00 -0400 Subject: [PATCH 51/52] remove assert(1 == 1) test, now we know --- Include/cpython/pyatomic.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index f8062f969ef4d9..68b2a7f1653b96 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -553,8 +553,6 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) // XXX: The assertions below currently fail on Android ARM build. // In fact, assert(1 == 1) fails on that build. - assert(1 == 1); - // assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); // assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); // assert(n % (size_t)sizeof(void *) == 0); From e6d3e5cb8d5d39cd3f79c38c6072d5ec713cd141 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 31 Aug 2025 17:47:12 -0400 Subject: [PATCH 52/52] #include --- Include/cpython/pyatomic.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 68b2a7f1653b96..75d1e896d5fdf7 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -87,6 +87,8 @@ # error "this header file must not be included directly" #endif +#include + // --- _Py_atomic_add -------------------------------------------------------- // Atomically adds `value` to `obj` and returns the previous value @@ -550,12 +552,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); static inline void * _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) { - // XXX: The assertions below currently fail on Android ARM build. - // In fact, assert(1 == 1) fails on that build. - - // assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(n % (size_t)sizeof(void *) == 0); if (dest != src) { void **dest_ = (void **)dest; @@ -573,9 +572,9 @@ _Py_atomic_memcpy_ptr_store_relaxed(void *dest, void *src, size_t n) static inline void * _Py_atomic_memmove_ptr_store_relaxed(void *dest, void *src, size_t n) { - // assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); - // assert(n % (size_t)sizeof(void *) == 0); + assert(((uintptr_t)dest & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(((uintptr_t)src & (uintptr_t)(sizeof (void *) - 1)) == 0); + assert(n % (size_t)sizeof(void *) == 0); if (dest < src || dest >= (void *)((char *)src + n)) { void **dest_ = (void **)dest;