Skip to content

Commit 7633d8d

Browse files
andfoyjonathanunderwood
andfoy
authored and
jonathanunderwood
committed
Enable CPython free-threaded wheel builds
1 parent b20fdf3 commit 7633d8d

15 files changed

+117
-35
lines changed

‎.github/workflows/build_dist.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
- macos-13 # x86
4949
- macos-latest # arm
5050
- windows-latest
51-
cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*]
51+
cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*]
5252
steps:
5353
- name: Check out repository
5454
uses: actions/checkout@v4
@@ -63,6 +63,14 @@ jobs:
6363
uses: docker/setup-qemu-action@v3
6464
with:
6565
platforms: all
66+
67+
- name: Setup free-threading variables
68+
if: ${{ endsWith(matrix.cibw_build, 't-*') }}
69+
shell: bash -l {0}
70+
run: |
71+
echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV"
72+
echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV"
73+
echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV"
6674
- name: Build wheels
6775
uses: pypa/cibuildwheel@v2.21
6876
env:
@@ -110,6 +118,7 @@ jobs:
110118
CIBW_ARCHS_LINUX: "aarch64 armv7l"
111119
CIBW_BUILD: ${{ matrix.cibw_build }}
112120
CIBW_SKIP: "cp*-musllinux*"
121+
CIBW_ENABLE: cpython-freethreading
113122
CIBW_TEST_COMMAND: "tox -c {project}"
114123
CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox"
115124
- name: Save wheels

‎lz4/_version.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,5 +113,9 @@ PyInit__version(void)
113113
if (module == NULL)
114114
return NULL;
115115

116+
#ifdef Py_GIL_DISABLED
117+
PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED);
118+
#endif
119+
116120
return module;
117121
}

‎lz4/block/_block.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,5 +518,9 @@ PyInit__block(void)
518518
Py_INCREF(LZ4BlockError);
519519
PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError);
520520

521+
#ifdef Py_GIL_DISABLED
522+
PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED);
523+
#endif
524+
521525
return module;
522526
}

‎lz4/frame/_frame.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1677,5 +1677,9 @@ PyInit__frame(void)
16771677
PyModule_AddIntConstant (module, "BLOCKSIZE_MAX1MB", LZ4F_max1MB);
16781678
PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB);
16791679

1680+
#ifdef Py_GIL_DISABLED
1681+
PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED);
1682+
#endif
1683+
16801684
return module;
16811685
}

‎lz4/stream/_stream.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1649,5 +1649,9 @@ PyInit__stream(void)
16491649
Py_INCREF (LZ4StreamError);
16501650
PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError);
16511651

1652+
#ifdef Py_GIL_DISABLED
1653+
PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED);
1654+
#endif
1655+
16521656
return module;
16531657
}

‎tests/block/conftest.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,24 @@
33
import sys
44

55

6+
class EmptyMemoryView():
7+
def __init__(self):
8+
self.data = b''
9+
self.view = None
10+
11+
def __buffer__(self, flags: int, /) -> memoryview:
12+
if self.view is None:
13+
self.view = memoryview(self.data)
14+
return self.view
15+
16+
def __release_buffer__(self, buffer: memoryview, /):
17+
breakpoint()
18+
buffer.release()
19+
20+
def __len__(self):
21+
return 0
22+
23+
624
test_data = [
725
(b''),
826
(os.urandom(8 * 1024)),

‎tests/block/test_block_0.py

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import lz4.block
22
from multiprocessing.pool import ThreadPool
33
import sys
4+
import copy
5+
import inspect
6+
import pytest
47
from functools import partial
58
if sys.version_info <= (3, 2):
69
import struct
@@ -79,10 +82,19 @@ def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dicti
7982

8083

8184
# Test multi threaded usage with all valid variations of input
85+
@pytest.mark.thread_unsafe
8286
def test_2(data, mode, store_size, dictionary):
8387
(c_kwargs, d_kwargs) = setup_kwargs(mode, store_size)
8488

85-
data_in = [data for i in range(32)]
89+
def copy_buf(data):
90+
data_x = data
91+
if isinstance(data, memoryview):
92+
data_x = memoryview(copy.deepcopy(data.obj))
93+
elif isinstance(data, bytearray):
94+
data_x = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj))
95+
return data_x
96+
97+
data_in = [copy_buf(data) for i in range(32)]
8698

8799
pool = ThreadPool(2)
88100
rt = partial(roundtrip, c_kwargs=c_kwargs,

‎tests/block/test_block_3.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ def data(request):
1818
return request.param
1919

2020

21+
@pytest.mark.thread_unsafe
2122
def test_block_decompress_mem_usage(data):
2223
tracemalloc = pytest.importorskip('tracemalloc')
2324

‎tests/frame/test_frame_2.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import lz4.frame as lz4frame
22
import pytest
33
import os
4+
import copy
5+
import inspect
46
import sys
57
from . helpers import (
68
get_chunked,
@@ -41,6 +43,11 @@ def test_roundtrip_chunked(data, block_size, block_linked,
4143

4244
data, c_chunks, d_chunks = data
4345

46+
if isinstance(data, memoryview):
47+
data = memoryview(copy.deepcopy(data.obj))
48+
elif isinstance(data, bytearray):
49+
data = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj))
50+
4451
c_context = lz4frame.create_compression_context()
4552

4653
kwargs = {}

‎tests/frame/test_frame_5.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
(b'a' * 1024 * 1024),
99
]
1010

11+
pytestmark = pytest.mark.thread_unsafe
12+
1113

1214
@pytest.fixture(
1315
params=test_data,
@@ -66,17 +68,17 @@ def test_frame_decompress_chunk_mem_usage(data):
6668
prev_snapshot = snapshot
6769

6870

69-
def test_frame_open_decompress_mem_usage(data):
71+
def test_frame_open_decompress_mem_usage(tmp_path, data):
7072
tracemalloc = pytest.importorskip('tracemalloc')
7173
tracemalloc.start()
7274

73-
with lz4.frame.open('test.lz4', 'w') as f:
75+
with lz4.frame.open(tmp_path / 'test.lz4', 'w') as f:
7476
f.write(data)
7577

7678
prev_snapshot = None
7779

7880
for i in range(1000):
79-
with lz4.frame.open('test.lz4', 'r') as f:
81+
with lz4.frame.open(tmp_path / 'test.lz4', 'r') as f:
8082
decompressed = f.read() # noqa: F841
8183

8284
if i % 100 == 0:

‎tests/frame/test_frame_6.py

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import os
22
import pytest
3+
import threading
34
import lz4.frame as lz4frame
45

56
test_data = [
@@ -33,40 +34,45 @@ def compression_level(request):
3334
return request.param
3435

3536

36-
def test_lz4frame_open_write(data):
37-
with lz4frame.open('testfile', mode='wb') as fp:
37+
def test_lz4frame_open_write(tmp_path, data):
38+
thread_id = threading.get_native_id()
39+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp:
3840
fp.write(data)
3941

4042

41-
def test_lz4frame_open_write_read_defaults(data):
42-
with lz4frame.open('testfile', mode='wb') as fp:
43+
def test_lz4frame_open_write_read_defaults(tmp_path, data):
44+
thread_id = threading.get_native_id()
45+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp:
4346
fp.write(data)
44-
with lz4frame.open('testfile', mode='r') as fp:
47+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp:
4548
data_out = fp.read()
4649
assert data_out == data
4750

4851

49-
def test_lz4frame_open_write_read_text():
52+
def test_lz4frame_open_write_read_text(tmp_path):
5053
data = u'This is a test string'
51-
with lz4frame.open('testfile', mode='wt') as fp:
54+
thread_id = threading.get_native_id()
55+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp:
5256
fp.write(data)
53-
with lz4frame.open('testfile', mode='rt') as fp:
57+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp:
5458
data_out = fp.read()
5559
assert data_out == data
5660

5761

58-
def test_lz4frame_open_write_read_text_iter():
62+
def test_lz4frame_open_write_read_text_iter(tmp_path):
5963
data = u'This is a test string'
60-
with lz4frame.open('testfile', mode='wt') as fp:
64+
thread_id = threading.get_native_id()
65+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp:
6166
fp.write(data)
6267
data_out = ''
63-
with lz4frame.open('testfile', mode='rt') as fp:
68+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp:
6469
for line in fp:
6570
data_out += line
6671
assert data_out == data
6772

6873

6974
def test_lz4frame_open_write_read(
75+
tmp_path,
7076
data,
7177
compression_level,
7278
block_linked,
@@ -91,29 +97,31 @@ def test_lz4frame_open_write_read(
9197
kwargs['return_bytearray'] = return_bytearray
9298
kwargs['mode'] = 'wb'
9399

94-
with lz4frame.open('testfile', **kwargs) as fp:
100+
thread_id = threading.get_native_id()
101+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', **kwargs) as fp:
95102
fp.write(data)
96103

97-
with lz4frame.open('testfile', mode='r') as fp:
104+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp:
98105
data_out = fp.read()
99106

100107
assert data_out == data
101108

102109

103-
def test_lz4frame_flush():
110+
def test_lz4frame_flush(tmp_path):
104111
data_1 = b"This is a..."
105112
data_2 = b" test string!"
113+
thread_id = threading.get_native_id()
106114

107-
with lz4frame.open("testfile", mode="w") as fp_write:
115+
with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="w") as fp_write:
108116
fp_write.write(data_1)
109117
fp_write.flush()
110118

111119
fp_write.write(data_2)
112120

113-
with lz4frame.open("testfile", mode="r") as fp_read:
121+
with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read:
114122
assert fp_read.read() == data_1
115123

116124
fp_write.flush()
117125

118-
with lz4frame.open("testfile", mode="r") as fp_read:
126+
with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read:
119127
assert fp_read.read() == data_1 + data_2

‎tests/frame/test_frame_8.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
1+
import threading
12
import lz4.frame as lz4frame
23

34

4-
def test_lz4frame_open_write_read_text_iter():
5+
def test_lz4frame_open_write_read_text_iter(tmp_path):
56
data = u'This is a test string'
6-
with lz4frame.open('testfile', mode='wt') as fp:
7+
thread_id = threading.get_native_id()
8+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp:
79
fp.write(data)
810
data_out = ''
9-
with lz4frame.open('testfile', mode='rt') as fp:
11+
with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp:
1012
for line in fp:
1113
data_out += line
1214
assert data_out == data

‎tests/frame/test_frame_9.py

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@
33
import io
44
import pickle
55
import sys
6+
import threading
67
import lz4.frame
78
import pytest
89

910

10-
def test_issue_172_1():
11+
def test_issue_172_1(tmp_path):
1112
"""Test reproducer for issue 172
1213
1314
Issue 172 is a reported failure occurring on Windows 10 only. This bug was
@@ -16,34 +17,38 @@ def test_issue_172_1():
1617
1718
"""
1819
input_data = 8 * os.urandom(1024)
19-
with lz4.frame.open('testfile_small', 'wb') as fp:
20+
thread_id = threading.get_native_id()
21+
22+
with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp:
2023
bytes_written = fp.write(input_data) # noqa: F841
2124

22-
with lz4.frame.open('testfile_small', 'rb') as fp:
25+
with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp:
2326
data = fp.read(10)
2427
assert len(data) == 10
2528

2629

27-
def test_issue_172_2():
30+
def test_issue_172_2(tmp_path):
2831
input_data = 9 * os.urandom(1024)
29-
with lz4.frame.open('testfile_small', 'w') as fp:
32+
thread_id = threading.get_native_id()
33+
with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'w') as fp:
3034
bytes_written = fp.write(input_data) # noqa: F841
3135

32-
with lz4.frame.open('testfile_small', 'r') as fp:
36+
with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'r') as fp:
3337
data = fp.read(10)
3438
assert len(data) == 10
3539

3640

37-
def test_issue_172_3():
41+
def test_issue_172_3(tmp_path):
3842
input_data = 9 * os.urandom(1024)
39-
with lz4.frame.open('testfile_small', 'wb') as fp:
43+
thread_id = threading.get_native_id()
44+
with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp:
4045
bytes_written = fp.write(input_data) # noqa: F841
4146

42-
with lz4.frame.open('testfile_small', 'rb') as fp:
47+
with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp:
4348
data = fp.read(10)
4449
assert len(data) == 10
4550

46-
with lz4.frame.open('testfile_small', 'rb') as fp:
51+
with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp:
4752
data = fp.read(16 * 1024 - 1)
4853
assert len(data) == 9 * 1024
4954
assert data == input_data

‎tests/stream/test_stream_0.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size,
9696

9797

9898
# Test single threaded usage with all valid variations of input
99+
@pytest.mark.thread_unsafe
99100
def test_1(data, strategy, mode, buffer_size, store_comp_size,
100101
c_return_bytearray, d_return_bytearray, dictionary):
101102
if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])):

‎tests/stream/test_stream_3.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ def data(request):
7171
return request.param
7272

7373

74+
@pytest.mark.thread_unsafe
7475
def test_block_decompress_mem_usage(data, buffer_size):
7576
kwargs = {
7677
'strategy': "double_buffer",

0 commit comments

Comments
 (0)