python-lz4 · andfoy · Mar 17, 2025 · Apr 1, 2025 · Apr 1, 2025 · Apr 1, 2025
diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml
@@ -48,7 +48,7 @@ jobs:
           - macos-13 # x86
           - macos-latest # arm
           - windows-latest
-        cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*]
+        cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*]
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -63,17 +63,30 @@ jobs:
         uses: docker/setup-qemu-action@v3
         with:
           platforms: all
+
+      - name: Setup free-threading variables
+        if: ${{ endsWith(matrix.cibw_build, 't-*') }}
+        shell: bash -l {0}
+        run: |
+          echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV"
+          echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV"
+          echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV"
+      - name: Setup environment
+        if: ${{ !endsWith(matrix.cibw_build, 't-*') }}
+        shell: bash -l {0}
+        run: |
+          echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV"
+          echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV"
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21
+        uses: pypa/cibuildwheel@v2.23.2
         env:
-          CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False"
           # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64"
           CIBW_ARCHS_LINUX: "x86_64 i686"
           CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches
           CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64"
+          CIBW_ENABLE: cpython-freethreading
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_SKIP: "cp*-musllinux*"
-          CIBW_TEST_COMMAND: "tox -c {project}"
           CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64"
           CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox"
       - name: Save wheels
@@ -93,7 +106,7 @@ jobs:
       matrix:
         os:
           - ubuntu-24.04-arm
-        cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*]
+        cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*]
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -103,14 +116,28 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.x
+      - name: Setup free-threading variables
+        if: ${{ endsWith(matrix.cibw_build, 't-*') }}
+        shell: bash -l {0}
+        run: |
+          # Variables are set in order to be passed down to both cibuildwheel and the
+          # Docker image spawned by that action
+          echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV"
+          echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV"
+          echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV"
+      - name: Setup environment
+        if: ${{ !endsWith(matrix.cibw_build, 't-*') }}
+        shell: bash -l {0}
+        run: |
+          echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV"
+          echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV"
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21
+        uses: pypa/cibuildwheel@v2.23.2
         env:
-          CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False"
-          CIBW_ARCHS_LINUX: "aarch64 armv7l"
+          CIBW_ARCHS_LINUX: "aarch64"
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_SKIP: "cp*-musllinux*"
-          CIBW_TEST_COMMAND: "tox -c {project}"
+          CIBW_ENABLE: cpython-freethreading
           CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox"
       - name: Save wheels
         uses: actions/upload-artifact@v4

diff --git a/lz4/_version.c b/lz4/_version.c
@@ -113,5 +113,9 @@ PyInit__version(void)
   if (module == NULL)
     return NULL;
 
+  #ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+  #endif
+
   return module;
 }
diff --git a/lz4/block/_block.c b/lz4/block/_block.c
@@ -518,5 +518,9 @@ PyInit__block(void)
   Py_INCREF(LZ4BlockError);
   PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError);
 
+  #ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+  #endif
+
   return module;
 }
diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c
@@ -1677,5 +1677,9 @@ PyInit__frame(void)
   PyModule_AddIntConstant (module, "BLOCKSIZE_MAX1MB", LZ4F_max1MB);
   PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB);
 
+  #ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+  #endif
+
   return module;
 }
diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c
@@ -1649,5 +1649,9 @@ PyInit__stream(void)
   Py_INCREF (LZ4StreamError);
   PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError);
 
+  #ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+  #endif
+
   return module;
 }
diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py
@@ -1,6 +1,8 @@
 import lz4.block
 from multiprocessing.pool import ThreadPool
 import sys
+import copy
+import pytest
 from functools import partial
 if sys.version_info <= (3, 2):
     import struct
@@ -68,6 +70,13 @@ def setup_kwargs(mode, store_size, c_return_bytearray=None, d_return_bytearray=N
 
 # Test single threaded usage with all valid variations of input
 def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dictionary):
+    if isinstance(data, memoryview):
+        data = memoryview(copy.deepcopy(data.obj))
+    elif isinstance(data, bytearray):
+        data_x = bytearray()
+        data_x[:] = data
+        data = data_x
+
     (c_kwargs, d_kwargs) = setup_kwargs(
         mode, store_size, c_return_bytearray, d_return_bytearray)
 
@@ -79,10 +88,21 @@ def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dicti
 
 
 # Test multi threaded usage with all valid variations of input
+@pytest.mark.thread_unsafe
 def test_2(data, mode, store_size, dictionary):
     (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size)
 
-    data_in = [data for i in range(32)]
+    def copy_buf(data):
+        if isinstance(data, memoryview):
+            data_x = memoryview(copy.deepcopy(data.obj))
+        elif isinstance(data, bytearray):
+            data_x = bytearray()
+            data_x[:] = data
+        else:
+            data_x = data
+        return data_x
+
+    data_in = [copy_buf(data) for i in range(32)]
 
     pool = ThreadPool(2)
     rt = partial(roundtrip, c_kwargs=c_kwargs,

diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py
@@ -18,6 +18,7 @@ def data(request):
     return request.param
 
 
+@pytest.mark.thread_unsafe
 def test_block_decompress_mem_usage(data):
     tracemalloc = pytest.importorskip('tracemalloc')
 

diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py
@@ -1,6 +1,7 @@
 import lz4.frame as lz4frame
 import pytest
 import os
+import copy
 import sys
 from . helpers import (
     get_chunked,
@@ -41,6 +42,13 @@ def test_roundtrip_chunked(data, block_size, block_linked,
 
     data, c_chunks, d_chunks = data
 
+    if isinstance(data, memoryview):
+        data = memoryview(copy.deepcopy(data.obj))
+    elif isinstance(data, bytearray):
+        data_2 = bytearray()
+        data_2[:] = data
+        data = data_2
+
     c_context = lz4frame.create_compression_context()
 
     kwargs = {}

diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py
@@ -8,6 +8,8 @@
     (b'a' * 1024 * 1024),
 ]
 
+pytestmark = pytest.mark.thread_unsafe
+
 
 @pytest.fixture(
     params=test_data,

diff --git a/tests/frame/test_frame_6.py b/tests/frame/test_frame_6.py
@@ -1,5 +1,6 @@
 import os
 import pytest
+import threading
 import lz4.frame as lz4frame
 
 test_data = [
@@ -33,40 +34,45 @@ def compression_level(request):
     return request.param
 
 
-def test_lz4frame_open_write(data):
-    with lz4frame.open('testfile', mode='wb') as fp:
+def test_lz4frame_open_write(tmp_path, data):
+    thread_id = threading.get_native_id()
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp:
         fp.write(data)
 
 
-def test_lz4frame_open_write_read_defaults(data):
-    with lz4frame.open('testfile', mode='wb') as fp:
+def test_lz4frame_open_write_read_defaults(tmp_path, data):
+    thread_id = threading.get_native_id()
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp:
         fp.write(data)
-    with lz4frame.open('testfile', mode='r') as fp:
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp:
         data_out = fp.read()
     assert data_out == data
 
 
-def test_lz4frame_open_write_read_text():
+def test_lz4frame_open_write_read_text(tmp_path):
     data = u'This is a test string'
-    with lz4frame.open('testfile', mode='wt') as fp:
+    thread_id = threading.get_native_id()
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp:
         fp.write(data)
-    with lz4frame.open('testfile', mode='rt') as fp:
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp:
         data_out = fp.read()
     assert data_out == data
 
 
-def test_lz4frame_open_write_read_text_iter():
+def test_lz4frame_open_write_read_text_iter(tmp_path):
     data = u'This is a test string'
-    with lz4frame.open('testfile', mode='wt') as fp:
+    thread_id = threading.get_native_id()
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp:
         fp.write(data)
     data_out = ''
-    with lz4frame.open('testfile', mode='rt') as fp:
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp:
         for line in fp:
             data_out += line
     assert data_out == data
 
 
 def test_lz4frame_open_write_read(
+        tmp_path,
         data,
         compression_level,
         block_linked,
@@ -91,29 +97,31 @@ def test_lz4frame_open_write_read(
     kwargs['return_bytearray'] = return_bytearray
     kwargs['mode'] = 'wb'
 
-    with lz4frame.open('testfile', **kwargs) as fp:
+    thread_id = threading.get_native_id()
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', **kwargs) as fp:
         fp.write(data)
 
-    with lz4frame.open('testfile', mode='r') as fp:
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp:
         data_out = fp.read()
 
     assert data_out == data
 
 
-def test_lz4frame_flush():
+def test_lz4frame_flush(tmp_path):
     data_1 = b"This is a..."
     data_2 = b" test string!"
+    thread_id = threading.get_native_id()
 
-    with lz4frame.open("testfile", mode="w") as fp_write:
+    with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="w") as fp_write:
         fp_write.write(data_1)
         fp_write.flush()
 
         fp_write.write(data_2)
 
-        with lz4frame.open("testfile", mode="r") as fp_read:
+        with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read:
             assert fp_read.read() == data_1
 
         fp_write.flush()
 
-        with lz4frame.open("testfile", mode="r") as fp_read:
+        with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read:
             assert fp_read.read() == data_1 + data_2
diff --git a/tests/frame/test_frame_8.py b/tests/frame/test_frame_8.py
@@ -1,12 +1,14 @@
+import threading
 import lz4.frame as lz4frame
 
 
-def test_lz4frame_open_write_read_text_iter():
+def test_lz4frame_open_write_read_text_iter(tmp_path):
     data = u'This is a test string'
-    with lz4frame.open('testfile', mode='wt') as fp:
+    thread_id = threading.get_native_id()
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp:
         fp.write(data)
     data_out = ''
-    with lz4frame.open('testfile', mode='rt') as fp:
+    with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp:
         for line in fp:
             data_out += line
     assert data_out == data
diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py
@@ -3,11 +3,12 @@
 import io
 import pickle
 import sys
+import threading
 import lz4.frame
 import pytest
 
 
-def test_issue_172_1():
+def test_issue_172_1(tmp_path):
     """Test reproducer for issue 172
 
     Issue 172 is a reported failure occurring on Windows 10 only. This bug was
@@ -16,34 +17,38 @@ def test_issue_172_1():
 
     """
     input_data = 8 * os.urandom(1024)
-    with lz4.frame.open('testfile_small', 'wb') as fp:
+    thread_id = threading.get_native_id()
+
+    with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp:
         bytes_written = fp.write(input_data)  # noqa: F841
 
-    with lz4.frame.open('testfile_small', 'rb') as fp:
+    with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp:
         data = fp.read(10)
         assert len(data) == 10
 
 
-def test_issue_172_2():
+def test_issue_172_2(tmp_path):
     input_data = 9 * os.urandom(1024)
-    with lz4.frame.open('testfile_small', 'w') as fp:
+    thread_id = threading.get_native_id()
+    with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'w') as fp:
         bytes_written = fp.write(input_data)  # noqa: F841
 
-    with lz4.frame.open('testfile_small', 'r') as fp:
+    with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'r') as fp:
         data = fp.read(10)
         assert len(data) == 10
 
 
-def test_issue_172_3():
+def test_issue_172_3(tmp_path):
     input_data = 9 * os.urandom(1024)
-    with lz4.frame.open('testfile_small', 'wb') as fp:
+    thread_id = threading.get_native_id()
+    with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp:
         bytes_written = fp.write(input_data)  # noqa: F841
 
-    with lz4.frame.open('testfile_small', 'rb') as fp:
+    with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp:
         data = fp.read(10)
         assert len(data) == 10
 
-    with lz4.frame.open('testfile_small', 'rb') as fp:
+    with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp:
         data = fp.read(16 * 1024 - 1)
         assert len(data) == 9 * 1024
         assert data == input_data

diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py
@@ -96,6 +96,7 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size,
 
 
 # Test single threaded usage with all valid variations of input
+@pytest.mark.thread_unsafe
 def test_1(data, strategy, mode, buffer_size, store_comp_size,
            c_return_bytearray, d_return_bytearray, dictionary):
     if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])):