From 272fa2ba541384226dbc6909132cc9b34608c2c6 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Mon, 3 Feb 2025 21:08:05 -0800 Subject: [PATCH 01/12] Refactor Makefile: handle artifact downloads; export env vars --- Makefile | 76 ++++++++++++++++++++++++++++++++++++++++++-------------- tox.ini | 3 +-- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index fc8fa5b21..8178b131b 100644 --- a/Makefile +++ b/Makefile @@ -1,35 +1,32 @@ # Some simple testing tasks (sorry, UNIX only). -FLAGS= -KAFKA_VERSION=0.11.0.2 -SCALA_VERSION=2.12 +TEST_FLAGS ?= +export KAFKA_VERSION ?= 0.11.0.2 +KAFKA_ARTIFACT ?= kafka_$(SCALA_VERSION)-$(KAFKA_VERSION).tgz +DIST_BASE_URL ?= https://archive.apache.org/dist/kafka/ +TOX_ENV ?= 312 + +# Required to support testing old kafka versions on newer java releases +# The performance opts defaults are set in each kafka brokers bin/kafka_run_class.sh file +# The values here are taken from the 2.4.0 release. +export KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true setup: pip install -r requirements-dev.txt pip install -Ue . -servers/$(KAFKA_VERSION)/kafka-bin: - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) ./build_integration.sh - -build-integration: servers/$(KAFKA_VERSION)/kafka-bin - # Test and produce coverage using tox. This is the same as is run on Travis -test37: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py37 -- $(FLAGS) - -test27: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py27 -- $(FLAGS) +test: build-integration + tox -e $(TOX_ENV) -- $(TEST_FLAGS) # Test using pytest directly if you want to use local python. Useful for other # platforms that require manual installation for C libraries, ie. Windows. test-local: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \ - --pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF $(FLAGS) kafka test + pytest --pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF $(TEST_FLAGS) kafka test cov-local: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \ - --pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka \ - --cov-config=.covrc --cov-report html $(FLAGS) kafka test + pytest --pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka \ + --cov-config=.covrc --cov-report html $(TEST_FLAGS) kafka test @echo "open file://`pwd`/htmlcov/index.html" # Check the readme for syntax errors, which can lead to invalid formatting on @@ -56,4 +53,45 @@ doc: make -C docs html @echo "open file://`pwd`/docs/_build/html/index.html" -.PHONY: all test37 test27 test-local cov-local clean doc +dist: + python setup.py bdist_wheel + python setup.py sdist + +publish: + twine upload dist/kafka-python-${KAFKA_PYTHON_VERSION}.tar.gz dist/kafka_python-${KAFKA_PYTHON_VERSION}-py2.py3-none-any.whl + +.PHONY: all test test-local cov-local clean doc dist publish + +kafka_artifact_version=$(lastword $(subst -, ,$(1))) +kafka_scala_0_8_0=2.8.0 +scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(kafka_scala_$(subst .,_,$(1))),$(kafka_scala_$(subst .,_,$(1))),"2.12")) +kafka_artifact_name=kafka_$(call scala_version,$(1))-$(1).$(if $(filter 0.8.0,$(1)),tar.gz,tgz) + +build-integration: servers/$(KAFKA_VERSION)/kafka-bin + +servers/dist: + mkdir -p servers/dist + +servers/dist/kafka_%.tgz servers/dist/kafka_%.tar.gz: + @echo "Downloading $(@F)" + wget -P servers/dist/ -N $(DIST_BASE_URL)$(call kafka_artifact_version,$*)/$(@F) + +servers/dist/jakarta.xml.bind-api-2.3.3.jar: + wget -P servers/dist/ -N https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar + +# to allow us to derive the prerequisite artifact name from the target name +.SECONDEXPANSION: + +servers/%/kafka-bin: servers/dist/$$(call kafka_artifact_name,$$*) | servers/dist + @echo "Extracting kafka $* binaries from $<" + if [ -d "$@" ]; then rm -rf $@.bak; mv $@ $@.bak; fi + mkdir $@ + tar xzvf $< -C $@ --strip-components 1 + +servers/patch-libs/%: servers/dist/jakarta.xml.bind-api-2.3.3.jar | servers/$$*/kafka-bin + cp $< servers/$*/kafka-bin/libs/ + +servers/download/%: servers/dist/$$(call kafka_artifact_name,$$*) | servers/dist ; + +# Avoid removing any pattern match targets as intermediates (without this, .tgz artifacts are removed by make after extraction) +.SECONDARY: diff --git a/tox.ini b/tox.ini index 7417387ed..71e443dec 100644 --- a/tox.ini +++ b/tox.ini @@ -33,8 +33,7 @@ commands = setenv = CRC32C_SW_MODE = auto PROJECT_ROOT = {toxinidir} -passenv = KAFKA_VERSION - +passenv = KAFKA_* [testenv:pypy] # pylint is super slow on pypy... From 9d0346e0a8c0d10e2beccffd5948d907cddf76bd Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 10:18:39 -0800 Subject: [PATCH 02/12] no progressbar for wget --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8178b131b..c61feb132 100644 --- a/Makefile +++ b/Makefile @@ -74,10 +74,10 @@ servers/dist: servers/dist/kafka_%.tgz servers/dist/kafka_%.tar.gz: @echo "Downloading $(@F)" - wget -P servers/dist/ -N $(DIST_BASE_URL)$(call kafka_artifact_version,$*)/$(@F) + wget -nv -P servers/dist/ -N $(DIST_BASE_URL)$(call kafka_artifact_version,$*)/$(@F) servers/dist/jakarta.xml.bind-api-2.3.3.jar: - wget -P servers/dist/ -N https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar + wget -nv -P servers/dist/ -N https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar # to allow us to derive the prerequisite artifact name from the target name .SECONDEXPANSION: From 9b1e8c24d0c96f5524aa6babc89ef5834b4a84af Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 22:28:35 -0800 Subject: [PATCH 03/12] Add lint: target; call pytest directly in test: recipe --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c61feb132..c4811ef92 100644 --- a/Makefile +++ b/Makefile @@ -15,9 +15,11 @@ setup: pip install -r requirements-dev.txt pip install -Ue . -# Test and produce coverage using tox. This is the same as is run on Travis +lint: + pylint --recursive=y --errors-only kafka test + test: build-integration - tox -e $(TOX_ENV) -- $(TEST_FLAGS) + pytest --durations=10 kafka test # Test using pytest directly if you want to use local python. Useful for other # platforms that require manual installation for C libraries, ie. Windows. From 6295ef397ebdae54c027821af2386c0d3b8116e4 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 22:29:30 -0800 Subject: [PATCH 04/12] Use make targets in gh workflow; use java 21; drop java helper script --- .github/workflows/python-package.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0b4a8e6c4..cb9f5849c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -70,16 +70,10 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 11 - - name: Check Java installation - run: source travis_java_install.sh - - name: Pull Kafka releases - run: ./build_integration.sh - env: - PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ matrix.kafka }} + java-version: 21 + - name: Pull Kafka release + run: make servers/${{ matrix.kafka }}/kafka-bin - name: Test with tox - run: tox + run: make test env: - PLATFORM: ${{ matrix.platform }} KAFKA_VERSION: ${{ matrix.kafka }} From d7e1f9a4745016174ce1932d090e9ac3461fc1e4 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 22:34:02 -0800 Subject: [PATCH 05/12] Drop unused vars + reorder --- Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c4811ef92..d5de50bd3 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,8 @@ -# Some simple testing tasks (sorry, UNIX only). +# Some simple testing tasks -TEST_FLAGS ?= -export KAFKA_VERSION ?= 0.11.0.2 -KAFKA_ARTIFACT ?= kafka_$(SCALA_VERSION)-$(KAFKA_VERSION).tgz +export KAFKA_VERSION ?= 2.4.0 DIST_BASE_URL ?= https://archive.apache.org/dist/kafka/ -TOX_ENV ?= 312 +TEST_FLAGS ?= # Required to support testing old kafka versions on newer java releases # The performance opts defaults are set in each kafka brokers bin/kafka_run_class.sh file From 233e7bf388c7bf1d9db5bb6245e84cbd07f9231d Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 22:46:41 -0800 Subject: [PATCH 06/12] Fill out remaining scala version mappings --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d5de50bd3..35fde0f57 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,18 @@ publish: .PHONY: all test test-local cov-local clean doc dist publish kafka_artifact_version=$(lastword $(subst -, ,$(1))) + +# Mappings for artifacts -> scala version; any unlisted will use default 2.12 kafka_scala_0_8_0=2.8.0 -scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(kafka_scala_$(subst .,_,$(1))),$(kafka_scala_$(subst .,_,$(1))),"2.12")) +kafka_scala_0_8_1=2.10 +kafka_scala_0_8_1_1=2.10 +kafka_scala_0_9_0_0=2.11 +kafka_scala_0_9_0_1=2.11 +kafka_scala_0_10_0_0=2.11 +kafka_scala_0_10_0_1=2.11 +kafka_scala_0_10_1_0=2.11 +scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(kafka_scala_$(subst .,_,$(1))),$(kafka_scala_$(subst .,_,$(1))),2.12)) + kafka_artifact_name=kafka_$(call scala_version,$(1))-$(1).$(if $(filter 0.8.0,$(1)),tar.gz,tgz) build-integration: servers/$(KAFKA_VERSION)/kafka-bin From f37600154da159da56295365090e5ef0c685f61e Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 22:46:46 -0800 Subject: [PATCH 07/12] Pytest name --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cb9f5849c..5b2505682 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -73,7 +73,7 @@ jobs: java-version: 21 - name: Pull Kafka release run: make servers/${{ matrix.kafka }}/kafka-bin - - name: Test with tox + - name: Pytest run: make test env: KAFKA_VERSION: ${{ matrix.kafka }} From ecd1e84de42a6c166878be4b6c20818aed33d33c Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 22:49:15 -0800 Subject: [PATCH 08/12] add zstandard to requirements-dev --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1fa933da2..e272d1ff7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ Sphinx sphinx-rtd-theme tox xxhash +zstandard From 1af3c9c4f839d82c3a149fc360eeaa5b9b70f3d4 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 23:17:54 -0800 Subject: [PATCH 09/12] More kafka_scala mappings; patch libs on install for kafka < 1 --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 35fde0f57..3a228d216 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,9 @@ kafka_artifact_version=$(lastword $(subst -, ,$(1))) kafka_scala_0_8_0=2.8.0 kafka_scala_0_8_1=2.10 kafka_scala_0_8_1_1=2.10 +kafka_scala_0_8_2_0=2.11 +kafka_scala_0_8_2_1=2.11 +kafka_scala_0_8_2_2=2.11 kafka_scala_0_9_0_0=2.11 kafka_scala_0_9_0_1=2.11 kafka_scala_0_10_0_0=2.11 @@ -97,6 +100,7 @@ servers/%/kafka-bin: servers/dist/$$(call kafka_artifact_name,$$*) | servers/dis if [ -d "$@" ]; then rm -rf $@.bak; mv $@ $@.bak; fi mkdir $@ tar xzvf $< -C $@ --strip-components 1 + if [[ "$*" < "1" ]]; then make servers/patch-libs/$*; fi servers/patch-libs/%: servers/dist/jakarta.xml.bind-api-2.3.3.jar | servers/$$*/kafka-bin cp $< servers/$*/kafka-bin/libs/ From 069c137947a7d7618d09c7d203dba3e393a98f63 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 4 Feb 2025 23:27:33 -0800 Subject: [PATCH 10/12] use bash --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 3a228d216..edc4ea792 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ # Some simple testing tasks +SHELL = bash + export KAFKA_VERSION ?= 2.4.0 DIST_BASE_URL ?= https://archive.apache.org/dist/kafka/ TEST_FLAGS ?= From 6d845b215b61ab1b73a5cf11441eaab476f3f88a Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 5 Feb 2025 22:04:46 -0800 Subject: [PATCH 11/12] drop dist/publish targets --- Makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Makefile b/Makefile index edc4ea792..da692c02c 100644 --- a/Makefile +++ b/Makefile @@ -55,13 +55,6 @@ doc: make -C docs html @echo "open file://`pwd`/docs/_build/html/index.html" -dist: - python setup.py bdist_wheel - python setup.py sdist - -publish: - twine upload dist/kafka-python-${KAFKA_PYTHON_VERSION}.tar.gz dist/kafka_python-${KAFKA_PYTHON_VERSION}-py2.py3-none-any.whl - .PHONY: all test test-local cov-local clean doc dist publish kafka_artifact_version=$(lastword $(subst -, ,$(1))) From d623d2b82bdfaf1d76889188090c9abf60d9f3cc Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 5 Feb 2025 22:09:21 -0800 Subject: [PATCH 12/12] drop empty TEST_FLAGS default --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index da692c02c..d384043a7 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,6 @@ SHELL = bash export KAFKA_VERSION ?= 2.4.0 DIST_BASE_URL ?= https://archive.apache.org/dist/kafka/ -TEST_FLAGS ?= # Required to support testing old kafka versions on newer java releases # The performance opts defaults are set in each kafka brokers bin/kafka_run_class.sh file