From a3913872693959d2929f6d32c6080821a6c8679f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 17 Mar 2025 16:09:57 -0500 Subject: [PATCH 01/16] Enable CPython free-threaded wheel builds --- .github/workflows/build_dist.yml | 11 ++++++++- lz4/_version.c | 4 +++ lz4/block/_block.c | 4 +++ lz4/frame/_frame.c | 4 +++ lz4/stream/_stream.c | 4 +++ tests/block/conftest.py | 18 ++++++++++++++ tests/block/test_block_0.py | 14 ++++++++++- tests/block/test_block_3.py | 1 + tests/frame/test_frame_2.py | 7 ++++++ tests/frame/test_frame_5.py | 8 +++--- tests/frame/test_frame_6.py | 42 +++++++++++++++++++------------- tests/frame/test_frame_8.py | 8 +++--- tests/frame/test_frame_9.py | 25 +++++++++++-------- tests/stream/test_stream_0.py | 1 + tests/stream/test_stream_3.py | 1 + 15 files changed, 117 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index ac70871a..b1e325c3 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: - macos-13 # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] steps: - name: Check out repository uses: actions/checkout@v4 @@ -63,6 +63,14 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: all + + - name: Setup free-threading variables + if: ${{ endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" + echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" + echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.21 env: @@ -110,6 +118,7 @@ jobs: CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" + CIBW_ENABLE: cpython-freethreading CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels diff --git a/lz4/_version.c b/lz4/_version.c index c611f0b3..77952415 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -113,5 +113,9 @@ PyInit__version(void) if (module == NULL) return NULL; + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 3e904a03..daa9fd5b 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -518,5 +518,9 @@ PyInit__block(void) Py_INCREF(LZ4BlockError); PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 34606653..d081cce9 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1677,5 +1677,9 @@ PyInit__frame(void) PyModule_AddIntConstant (module, "BLOCKSIZE_MAX1MB", LZ4F_max1MB); PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 522fdeda..6351d969 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -1649,5 +1649,9 @@ PyInit__stream(void) Py_INCREF (LZ4StreamError); PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/tests/block/conftest.py b/tests/block/conftest.py index 0b3578ea..b25c239e 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -3,6 +3,24 @@ import sys +class EmptyMemoryView(): + def __init__(self): + self.data = b'' + self.view = None + + def __buffer__(self, flags: int, /) -> memoryview: + if self.view is None: + self.view = memoryview(self.data) + return self.view + + def __release_buffer__(self, buffer: memoryview, /): + breakpoint() + buffer.release() + + def __len__(self): + return 0 + + test_data = [ (b''), (os.urandom(8 * 1024)), diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 8fc0f488..31513b47 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -1,6 +1,9 @@ import lz4.block from multiprocessing.pool import ThreadPool import sys +import copy +import inspect +import pytest from functools import partial if sys.version_info <= (3, 2): import struct @@ -79,10 +82,19 @@ def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dicti # Test multi threaded usage with all valid variations of input +@pytest.mark.thread_unsafe def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) - data_in = [data for i in range(32)] + def copy_buf(data): + data_x = data + if isinstance(data, memoryview): + data_x = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data_x = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + return data_x + + data_in = [copy_buf(data) for i in range(32)] pool = ThreadPool(2) rt = partial(roundtrip, c_kwargs=c_kwargs, diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 3fcb175b..88461b7a 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -18,6 +18,7 @@ def data(request): return request.param +@pytest.mark.thread_unsafe def test_block_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index 80b44b87..14a3f6b0 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -1,6 +1,8 @@ import lz4.frame as lz4frame import pytest import os +import copy +import inspect import sys from . helpers import ( get_chunked, @@ -41,6 +43,11 @@ def test_roundtrip_chunked(data, block_size, block_linked, data, c_chunks, d_chunks = data + if isinstance(data, memoryview): + data = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + c_context = lz4frame.create_compression_context() kwargs = {} diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 05daf283..40836462 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -8,6 +8,8 @@ (b'a' * 1024 * 1024), ] +pytestmark = pytest.mark.thread_unsafe + @pytest.fixture( params=test_data, @@ -66,17 +68,17 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot -def test_frame_open_decompress_mem_usage(data): +def test_frame_open_decompress_mem_usage(tmp_path, data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() - with lz4.frame.open('test.lz4', 'w') as f: + with lz4.frame.open(tmp_path / 'test.lz4', 'w') as f: f.write(data) prev_snapshot = None for i in range(1000): - with lz4.frame.open('test.lz4', 'r') as f: + with lz4.frame.open(tmp_path / 'test.lz4', 'r') as f: decompressed = f.read() # noqa: F841 if i % 100 == 0: diff --git a/tests/frame/test_frame_6.py b/tests/frame/test_frame_6.py index c20a4f31..4f4185ee 100644 --- a/tests/frame/test_frame_6.py +++ b/tests/frame/test_frame_6.py @@ -1,5 +1,6 @@ import os import pytest +import threading import lz4.frame as lz4frame test_data = [ @@ -33,40 +34,45 @@ def compression_level(request): return request.param -def test_lz4frame_open_write(data): - with lz4frame.open('testfile', mode='wb') as fp: +def test_lz4frame_open_write(tmp_path, data): + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp: fp.write(data) -def test_lz4frame_open_write_read_defaults(data): - with lz4frame.open('testfile', mode='wb') as fp: +def test_lz4frame_open_write_read_defaults(tmp_path, data): + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp: fp.write(data) - with lz4frame.open('testfile', mode='r') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_open_write_read_text(): +def test_lz4frame_open_write_read_text(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_open_write_read_text_iter(): +def test_lz4frame_open_write_read_text_iter(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) data_out = '' - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: for line in fp: data_out += line assert data_out == data def test_lz4frame_open_write_read( + tmp_path, data, compression_level, block_linked, @@ -91,29 +97,31 @@ def test_lz4frame_open_write_read( kwargs['return_bytearray'] = return_bytearray kwargs['mode'] = 'wb' - with lz4frame.open('testfile', **kwargs) as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', **kwargs) as fp: fp.write(data) - with lz4frame.open('testfile', mode='r') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_flush(): +def test_lz4frame_flush(tmp_path): data_1 = b"This is a..." data_2 = b" test string!" + thread_id = threading.get_native_id() - with lz4frame.open("testfile", mode="w") as fp_write: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="w") as fp_write: fp_write.write(data_1) fp_write.flush() fp_write.write(data_2) - with lz4frame.open("testfile", mode="r") as fp_read: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read: assert fp_read.read() == data_1 fp_write.flush() - with lz4frame.open("testfile", mode="r") as fp_read: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read: assert fp_read.read() == data_1 + data_2 diff --git a/tests/frame/test_frame_8.py b/tests/frame/test_frame_8.py index 159534ae..cfaeaace 100644 --- a/tests/frame/test_frame_8.py +++ b/tests/frame/test_frame_8.py @@ -1,12 +1,14 @@ +import threading import lz4.frame as lz4frame -def test_lz4frame_open_write_read_text_iter(): +def test_lz4frame_open_write_read_text_iter(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) data_out = '' - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: for line in fp: data_out += line assert data_out == data diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py index 51433934..c5335aed 100644 --- a/tests/frame/test_frame_9.py +++ b/tests/frame/test_frame_9.py @@ -3,11 +3,12 @@ import io import pickle import sys +import threading import lz4.frame import pytest -def test_issue_172_1(): +def test_issue_172_1(tmp_path): """Test reproducer for issue 172 Issue 172 is a reported failure occurring on Windows 10 only. This bug was @@ -16,34 +17,38 @@ def test_issue_172_1(): """ input_data = 8 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'wb') as fp: + thread_id = threading.get_native_id() + + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(10) assert len(data) == 10 -def test_issue_172_2(): +def test_issue_172_2(tmp_path): input_data = 9 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'w') as fp: + thread_id = threading.get_native_id() + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'w') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'r') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'r') as fp: data = fp.read(10) assert len(data) == 10 -def test_issue_172_3(): +def test_issue_172_3(tmp_path): input_data = 9 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'wb') as fp: + thread_id = threading.get_native_id() + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(10) assert len(data) == 10 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(16 * 1024 - 1) assert len(data) == 9 * 1024 assert data == input_data diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index 03b19f3f..cac07bdd 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -96,6 +96,7 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size, # Test single threaded usage with all valid variations of input +@pytest.mark.thread_unsafe def test_1(data, strategy, mode, buffer_size, store_comp_size, c_return_bytearray, d_return_bytearray, dictionary): if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])): diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 2b52d6b5..fed93d2c 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -71,6 +71,7 @@ def data(request): return request.param +@pytest.mark.thread_unsafe def test_block_decompress_mem_usage(data, buffer_size): kwargs = { 'strategy': "double_buffer", From 0474866fc2172da12c9601abb9e83eef0758001c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 11:42:55 -0500 Subject: [PATCH 02/16] Update cibuildwheel version --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index b1e325c3..38f40dda 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -72,7 +72,7 @@ jobs: echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.21 + uses: pypa/cibuildwheel@v2.23.2 env: CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" From 9990420ab9d57cf7e576447652d441772564f0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 12:32:44 -0500 Subject: [PATCH 03/16] Variable misspell fix --- lz4/_version.c | 2 +- lz4/block/_block.c | 2 +- lz4/frame/_frame.c | 2 +- lz4/stream/_stream.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lz4/_version.c b/lz4/_version.c index 77952415..af606abe 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -114,7 +114,7 @@ PyInit__version(void) return NULL; #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/block/_block.c b/lz4/block/_block.c index daa9fd5b..993cc44c 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -519,7 +519,7 @@ PyInit__block(void) PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index d081cce9..e62c72c6 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1678,7 +1678,7 @@ PyInit__frame(void) PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 6351d969..f0dfad57 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -1650,7 +1650,7 @@ PyInit__stream(void) PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; From 073bb4f24681bfb5972c75dfb734c479dad1f4d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 14:38:33 -0500 Subject: [PATCH 04/16] Update environment variables --- .github/workflows/build_dist.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 38f40dda..77862f1a 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,15 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" - echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" + - name: Setup environment + if: ${{ !endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4="False" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: - CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches From 42ab28c69df1f88ca9a5301dddd7a24a504a47fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 14:44:07 -0500 Subject: [PATCH 05/16] Ensure CIBW_ENVIRONMENT is set --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 77862f1a..0c659e3f 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,12 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4="False" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: From 4f4b633d8eab5830f71118265bf853318f9ad251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:28:12 -0500 Subject: [PATCH 06/16] Pass tox overrides through CLI --- .github/workflows/build_dist.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 0c659e3f..62d84da4 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,14 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: From 5b1c27132a7712868e1d0fc4b8fe73c32c5f95c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:39:14 -0500 Subject: [PATCH 07/16] Ensure test command is applied --- .github/workflows/build_dist.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 62d84da4..e6541dda 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -84,9 +84,9 @@ jobs: CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64" + CIBW_ENABLE: cpython-freethreading CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" - CIBW_TEST_COMMAND: "tox -c {project}" CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels @@ -106,7 +106,7 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] steps: - name: Check out repository uses: actions/checkout@v4 @@ -116,15 +116,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.x + - name: Setup free-threading variables + if: ${{ endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" + - name: Setup environment + if: ${{ !endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.21 env: - CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_ENABLE: cpython-freethreading - CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v4 From 0cf7fb0caae42550ba547b90b038644a0058c245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:57:10 -0500 Subject: [PATCH 08/16] Check why using temp paths increase memory usage --- tests/frame/test_frame_5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 40836462..dcbe4aea 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -68,17 +68,17 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot -def test_frame_open_decompress_mem_usage(tmp_path, data): +def test_frame_open_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() - with lz4.frame.open(tmp_path / 'test.lz4', 'w') as f: + with lz4.frame.open('test.lz4', 'w') as f: f.write(data) prev_snapshot = None for i in range(1000): - with lz4.frame.open(tmp_path / 'test.lz4', 'r') as f: + with lz4.frame.open('test.lz4', 'r') as f: decompressed = f.read() # noqa: F841 if i % 100 == 0: From 78231aa9d829a6a13491f2009939537e9328d24a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:13:03 -0500 Subject: [PATCH 09/16] Copy bytearrays directly --- tests/block/conftest.py | 18 ------------------ tests/block/test_block_0.py | 3 ++- tests/frame/test_frame_2.py | 4 +++- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/block/conftest.py b/tests/block/conftest.py index b25c239e..0b3578ea 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -3,24 +3,6 @@ import sys -class EmptyMemoryView(): - def __init__(self): - self.data = b'' - self.view = None - - def __buffer__(self, flags: int, /) -> memoryview: - if self.view is None: - self.view = memoryview(self.data) - return self.view - - def __release_buffer__(self, buffer: memoryview, /): - breakpoint() - buffer.release() - - def __len__(self): - return 0 - - test_data = [ (b''), (os.urandom(8 * 1024)), diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 31513b47..8a22e57f 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -91,7 +91,8 @@ def copy_buf(data): if isinstance(data, memoryview): data_x = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): - data_x = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + data_x = bytearray() + data_x[:] = data return data_x data_in = [copy_buf(data) for i in range(32)] diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index 14a3f6b0..f7e6fa3f 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -46,7 +46,9 @@ def test_roundtrip_chunked(data, block_size, block_linked, if isinstance(data, memoryview): data = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): - data = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + data_2 = bytearray() + data_2[:] = data + data = data_2 c_context = lz4frame.create_compression_context() From b4c9e832accede222a38873162a5fedb3f49a6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:22:17 -0500 Subject: [PATCH 10/16] Remove unused import --- tests/block/test_block_0.py | 1 - tests/frame/test_frame_2.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 8a22e57f..f2c3cd18 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -2,7 +2,6 @@ from multiprocessing.pool import ThreadPool import sys import copy -import inspect import pytest from functools import partial if sys.version_info <= (3, 2): diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index f7e6fa3f..230867e6 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -2,7 +2,6 @@ import pytest import os import copy -import inspect import sys from . helpers import ( get_chunked, From 739aaecbc5e76bf2c4bb45a2c866b7520152822f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:41:35 -0500 Subject: [PATCH 11/16] Copy memoryview on test_1 --- tests/block/test_block_0.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index f2c3cd18..561b1858 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -70,6 +70,13 @@ def setup_kwargs(mode, store_size, c_return_bytearray=None, d_return_bytearray=N # Test single threaded usage with all valid variations of input def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dictionary): + if isinstance(data, memoryview): + data = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data_x = bytearray() + data_x[:] = data + data = data_x + (c_kwargs, d_kwargs) = setup_kwargs( mode, store_size, c_return_bytearray, d_return_bytearray) From 461a4c447a61473bed9c840835b1daed65af569b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 12:26:53 -0500 Subject: [PATCH 12/16] Upgrade pypa/cibuildwheel action in arm64 job --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e6541dda..4af62b60 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -130,7 +130,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.21 + uses: pypa/cibuildwheel@v2.23.2 env: CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} From 5164e75820d8e681fbe78913407ba31a3887f83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 14:45:57 -0500 Subject: [PATCH 13/16] CI: set parallelism to 2 in aarch64 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 4af62b60..e3276807 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -121,7 +121,7 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=2" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} From 6f81f992a96931ff3d70cad44cf3a3daa20414e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 15:11:26 -0500 Subject: [PATCH 14/16] CI: disable pytest-run-parallel altogeogether in aarch64 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e3276807..6765e991 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -121,7 +121,7 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=2" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} From 50393d216ce858cd0c902f79781393793c6f7b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 15:24:42 -0500 Subject: [PATCH 15/16] CI: disable armv7l --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 6765e991..9b9964b9 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -132,7 +132,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: - CIBW_ARCHS_LINUX: "aarch64 armv7l" + CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_ENABLE: cpython-freethreading From 69a66d01a20ebb2688d491bd637cd46fe0c54783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 9 Apr 2025 16:43:32 -0500 Subject: [PATCH 16/16] Address review comments --- .github/workflows/build_dist.yml | 2 ++ tests/block/test_block_0.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 9b9964b9..b907acc0 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -120,6 +120,8 @@ jobs: if: ${{ endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | + # Variables are set in order to be passed down to both cibuildwheel and the + # Docker image spawned by that action echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 561b1858..a7731c3a 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -93,12 +93,13 @@ def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) def copy_buf(data): - data_x = data if isinstance(data, memoryview): data_x = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): data_x = bytearray() data_x[:] = data + else: + data_x = data return data_x data_in = [copy_buf(data) for i in range(32)]