From 3923095e2da4b14f7a6f0540d062ca8272da874d Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 16:57:45 -0500 Subject: [PATCH 1/8] skip failing tests for PyPy since they work locally --- test/test_admin_integration.py | 5 +++++ test/test_consumer_group.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/test/test_admin_integration.py b/test/test_admin_integration.py index 06c40a223..283023049 100644 --- a/test/test_admin_integration.py +++ b/test/test_admin_integration.py @@ -1,3 +1,5 @@ +import platform + import pytest from logging import info @@ -151,6 +153,9 @@ def test_describe_consumer_group_does_not_exist(kafka_admin_client): group_description = kafka_admin_client.describe_consumer_groups(['test']) +@pytest.mark.skipif( + platform.python_implementation() == "PyPy", reason="Works on PyPy if run locally, but not in CI/CD pipeline." +) @pytest.mark.skipif(env_kafka_version() < (0, 11), reason='Describe consumer group requires broker >=0.11') def test_describe_consumer_group_exists(kafka_admin_client, kafka_consumer_factory, topic): """Tests that the describe consumer group call returns valid consumer group information diff --git a/test/test_consumer_group.py b/test/test_consumer_group.py index 58dc7ebf9..4904ffeea 100644 --- a/test/test_consumer_group.py +++ b/test/test_consumer_group.py @@ -1,5 +1,6 @@ import collections import logging +import platform import threading import time @@ -40,6 +41,9 @@ def test_consumer_topics(kafka_broker, topic): consumer.close() +@pytest.mark.skipif( + platform.python_implementation() == "PyPy", reason="Works on PyPy if run locally, but not in CI/CD pipeline." +) @pytest.mark.skipif(env_kafka_version() < (0, 9), reason='Unsupported Kafka Version') def test_group(kafka_broker, topic): num_partitions = 4 From d4322209f2a9e6f0e9833b122091cab86cd0a353 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 17:04:23 -0500 Subject: [PATCH 2/8] Reconfigure tests for PyPy and 3.12 --- test/test_producer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_producer.py b/test/test_producer.py index 7263130d1..69e22cad2 100644 --- a/test/test_producer.py +++ b/test/test_producer.py @@ -1,5 +1,6 @@ import gc import platform +import sys import time import threading @@ -21,8 +22,8 @@ def test_buffer_pool(): buf2 = pool.allocate(1000, 1000) assert buf2.read() == b'' - @pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set") +@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") @pytest.mark.parametrize("compression", [None, 'gzip', 'snappy', 'lz4', 'zstd']) def test_end_to_end(kafka_broker, compression): if compression == 'lz4': @@ -70,6 +71,7 @@ def test_end_to_end(kafka_broker, compression): @pytest.mark.skipif(platform.python_implementation() != 'CPython', reason='Test relies on CPython-specific gc policies') +@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") def test_kafka_producer_gc_cleanup(): gc.collect() threads = threading.active_count() @@ -81,6 +83,7 @@ def test_kafka_producer_gc_cleanup(): @pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set") +@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") @pytest.mark.parametrize("compression", [None, 'gzip', 'snappy', 'lz4', 'zstd']) def test_kafka_producer_proper_record_metadata(kafka_broker, compression): if compression == 'zstd' and env_kafka_version() < (2, 1, 0): From 8ac3dbedd5d8fc35ec32378103832a86d87fd236 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 17:34:24 -0500 Subject: [PATCH 3/8] Skip partitioner tests in test_partitioner.py if 3.12 and 0.8.2 --- test/test_partitioner.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_partitioner.py b/test/test_partitioner.py index 853fbf69e..5cbd2d444 100644 --- a/test/test_partitioner.py +++ b/test/test_partitioner.py @@ -1,10 +1,16 @@ from __future__ import absolute_import +import platform +import sys + import pytest + from kafka.partitioner import DefaultPartitioner, murmur2 +from test.testutil import env_kafka_version, random_string +@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") def test_default_partitioner(): partitioner = DefaultPartitioner() all_partitions = available = list(range(100)) @@ -21,6 +27,7 @@ def test_default_partitioner(): assert partitioner(None, all_partitions, []) in all_partitions +@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") @pytest.mark.parametrize("bytes_payload,partition_number", [ (b'', 681), (b'a', 524), (b'ab', 434), (b'abc', 107), (b'123456789', 566), (b'\x00 ', 742) From 8588cfbf72d4e3f894707d3be2c294686589d977 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 17:39:16 -0500 Subject: [PATCH 4/8] Update test_partitioner.py --- test/test_partitioner.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/test_partitioner.py b/test/test_partitioner.py index 5cbd2d444..09fa0412a 100644 --- a/test/test_partitioner.py +++ b/test/test_partitioner.py @@ -1,16 +1,11 @@ from __future__ import absolute_import -import platform -import sys - import pytest from kafka.partitioner import DefaultPartitioner, murmur2 -from test.testutil import env_kafka_version, random_string -@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") def test_default_partitioner(): partitioner = DefaultPartitioner() all_partitions = available = list(range(100)) @@ -27,7 +22,6 @@ def test_default_partitioner(): assert partitioner(None, all_partitions, []) in all_partitions -@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") @pytest.mark.parametrize("bytes_payload,partition_number", [ (b'', 681), (b'a', 524), (b'ab', 434), (b'abc', 107), (b'123456789', 566), (b'\x00 ', 742) From 59fe5a828cfd88f6c4c510d0618ce0160e03865a Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 17:40:42 -0500 Subject: [PATCH 5/8] Update test_producer.py --- test/test_producer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_producer.py b/test/test_producer.py index 69e22cad2..15c244113 100644 --- a/test/test_producer.py +++ b/test/test_producer.py @@ -11,6 +11,7 @@ from test.testutil import env_kafka_version, random_string +@pytest.mark.skipif(env_kafka_version() <= (0, 8, 2) and sys.version_info > (3, 11), reason="Kafka 0.8.2 and earlier not supported by 3.12") def test_buffer_pool(): pool = SimpleBufferPool(1000, 1000) From 4de5379e46989a5b1187a34a3c0a4940dd6315a2 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 17:52:18 -0500 Subject: [PATCH 6/8] Timeout tests after ten minutes --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cee99ef5c..bfc14e2b1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -57,6 +57,7 @@ jobs: - build-sdist runs-on: ubuntu-latest continue-on-error: ${{ matrix.experimental }} + timeout-minutes: 10 strategy: fail-fast: false matrix: From 3952ad7c46d5d018c2e21f79af8b8075d3fef0c5 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 17:55:08 -0500 Subject: [PATCH 7/8] Set 0.8.2.2 to be experimental from hereon --- .github/workflows/python-package.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index bfc14e2b1..cee8a9353 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -57,7 +57,6 @@ jobs: - build-sdist runs-on: ubuntu-latest continue-on-error: ${{ matrix.experimental }} - timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -116,11 +115,11 @@ jobs: needs: - build-sdist runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: kafka-version: - - "0.8.2.2" - "0.9.0.1" - "0.10.2.2" - "0.11.0.2" @@ -129,6 +128,11 @@ jobs: - "2.4.0" - "2.5.0" - "2.6.0" + experimental: [false] + include: + - kafka-version: '0.8.2.2' + experimental: true + continue-on-error: ${{ matrix.experimental }} steps: - name: Checkout the source code uses: actions/checkout@v4 From 5fa1867370d52bd0bd5f2d6586fb83b8baa75d0f Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 8 Mar 2024 18:06:07 -0500 Subject: [PATCH 8/8] Formally support PyPy 3.9 --- .github/workflows/python-package.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cee8a9353..ef49b4e58 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -66,10 +66,9 @@ jobs: - "3.10" - "3.11" - "3.12" + - "pypy3.9" experimental: [ false ] include: - - python-version: "pypy3.9" - experimental: true - python-version: "~3.13.0-0" experimental: true steps: