From b942019a94497bcc03701141dffb69d70adf5cc3 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Mon, 9 Dec 2024 15:10:27 +0100 Subject: [PATCH 1/5] remove limit on cryptography, update all lock files --- pyproject.toml | 6 ++---- requirements-dev.txt | 4 ++-- requirements-runtime.txt | 4 ++-- requirements-test.txt | 4 ++-- requirements-typehint.txt | 4 ++-- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a4e09c93b1953..de0b35c43fcc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,15 +80,13 @@ runtime = [ # pinned / updated by ASF update action "awscli>=1.32.117", "airspeed-ext>=0.6.3", - # TODO upgrade to kclpy 3+ - "amazon_kclpy>=2.0.6,!=2.1.0,!=2.1.4,<3.0.0", + "amazon_kclpy>=2.0.6,!=2.1.0,!=2.1.4", # antlr4-python3-runtime: exact pin because antlr4 runtime is tightly coupled to the generated parser code "antlr4-python3-runtime==4.13.2", "apispec>=5.1.1", "aws-sam-translator>=1.15.1", "crontab>=0.22.6", - # TODO remove upper limit once https://github.com/getmoto/moto/pull/7876 is in our moto-ext version - "cryptography>=41.0.5,<43.0.0", + "cryptography>=41.0.5", # allow Python programs full access to Java class libraries. Used for opt-in event ruler. "jpype1-ext>=0.0.1", "json5>=0.9.11", diff --git a/requirements-dev.txt b/requirements-dev.txt index 2ab3a769cd965..991b9eff33a7f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # airspeed-ext==0.6.7 # via localstack-core -amazon-kclpy==2.1.5 +amazon-kclpy==3.0.1 # via localstack-core annotated-types==0.7.0 # via pydantic @@ -107,7 +107,7 @@ coveralls==4.0.1 # via localstack-core (pyproject.toml) crontab==1.0.1 # via localstack-core -cryptography==42.0.8 +cryptography==44.0.0 # via # joserfc # localstack-core diff --git a/requirements-runtime.txt b/requirements-runtime.txt index fb8f1c8b8e3eb..e06eef3e58097 100644 --- a/requirements-runtime.txt +++ b/requirements-runtime.txt @@ -6,7 +6,7 @@ # airspeed-ext==0.6.7 # via localstack-core (pyproject.toml) -amazon-kclpy==2.1.5 +amazon-kclpy==3.0.1 # via localstack-core (pyproject.toml) annotated-types==0.7.0 # via pydantic @@ -78,7 +78,7 @@ constantly==23.10.4 # via localstack-twisted crontab==1.0.1 # via localstack-core (pyproject.toml) -cryptography==42.0.8 +cryptography==44.0.0 # via # joserfc # localstack-core diff --git a/requirements-test.txt b/requirements-test.txt index 9136facb709dd..edc8fc7459b68 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -6,7 +6,7 @@ # airspeed-ext==0.6.7 # via localstack-core -amazon-kclpy==2.1.5 +amazon-kclpy==3.0.1 # via localstack-core annotated-types==0.7.0 # via pydantic @@ -101,7 +101,7 @@ coverage==7.6.9 # via localstack-core (pyproject.toml) crontab==1.0.1 # via localstack-core -cryptography==42.0.8 +cryptography==44.0.0 # via # joserfc # localstack-core diff --git a/requirements-typehint.txt b/requirements-typehint.txt index 9f38be20f5c59..cd039f0eccf22 100644 --- a/requirements-typehint.txt +++ b/requirements-typehint.txt @@ -6,7 +6,7 @@ # airspeed-ext==0.6.7 # via localstack-core -amazon-kclpy==2.1.5 +amazon-kclpy==3.0.1 # via localstack-core annotated-types==0.7.0 # via pydantic @@ -111,7 +111,7 @@ coveralls==4.0.1 # via localstack-core crontab==1.0.1 # via localstack-core -cryptography==42.0.8 +cryptography==44.0.0 # via # joserfc # localstack-core From 3ed707e9c8fb9cfceea27c2b4ca346e114deb84d Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Tue, 10 Dec 2024 10:42:21 +0100 Subject: [PATCH 2/5] upgrade kclipy helper for v3 in compat mode --- localstack-core/localstack/utils/kinesis/kclipy_helper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/localstack-core/localstack/utils/kinesis/kclipy_helper.py b/localstack-core/localstack/utils/kinesis/kclipy_helper.py index ea03f997afc40..1c6f74f5cba61 100644 --- a/localstack-core/localstack/utils/kinesis/kclipy_helper.py +++ b/localstack-core/localstack/utils/kinesis/kclipy_helper.py @@ -93,16 +93,17 @@ def create_config_file( **kwargs, ): if not credentialsProvider: - credentialsProvider = "DefaultAWSCredentialsProviderChain" + credentialsProvider = "DefaultCredentialsProvider" content = f""" executableName = {executableName} streamName = {streamName} applicationName = {applicationName} AWSCredentialsProvider = {credentialsProvider} + clientVersionConfig = CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2x kinesisCredentialsProvider = {credentialsProvider} dynamoDBCredentialsProvider = {credentialsProvider} cloudWatchCredentialsProvider = {credentialsProvider} - processingLanguage = python/3.10 + processingLanguage = python/3.11 shardSyncIntervalMillis = 2000 parentShardPollIntervalMillis = 2000 idleTimeBetweenReadsInMillis = 1000 From e0aa05e2853bec98a03f7227fd0939708bf99435 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Tue, 10 Dec 2024 12:08:32 +0100 Subject: [PATCH 3/5] increase timeout on flaky test_firehose_kinesis_to_s3 --- tests/aws/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aws/test_integration.py b/tests/aws/test_integration.py index 6da17331eff45..ef239d0252195 100644 --- a/tests/aws/test_integration.py +++ b/tests/aws/test_integration.py @@ -188,7 +188,7 @@ def _assert_active(): ) assert stream_info["DeliveryStreamDescription"]["DeliveryStreamStatus"] == "ACTIVE" - retry(_assert_active, sleep=1, retries=30) + retry(_assert_active, sleep=1, retries=60) # create target S3 bucket s3_create_bucket(Bucket=TEST_BUCKET_NAME) From e775b464b716a47b7841ff0575d26f7b3b7e9d0c Mon Sep 17 00:00:00 2001 From: Alexander Rashed <2796604+alexrashed@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:28:55 +0100 Subject: [PATCH 4/5] upgrade kclpy min version, optimize config Co-authored-by: Daniel Fangl --- localstack-core/localstack/utils/kinesis/kclipy_helper.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/localstack-core/localstack/utils/kinesis/kclipy_helper.py b/localstack-core/localstack/utils/kinesis/kclipy_helper.py index 1c6f74f5cba61..347210fa8e1e0 100644 --- a/localstack-core/localstack/utils/kinesis/kclipy_helper.py +++ b/localstack-core/localstack/utils/kinesis/kclipy_helper.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import sys from glob import glob from amazon_kclpy import kcl @@ -99,11 +100,10 @@ def create_config_file( streamName = {streamName} applicationName = {applicationName} AWSCredentialsProvider = {credentialsProvider} - clientVersionConfig = CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2x kinesisCredentialsProvider = {credentialsProvider} dynamoDBCredentialsProvider = {credentialsProvider} cloudWatchCredentialsProvider = {credentialsProvider} - processingLanguage = python/3.11 + processingLanguage = python/{sys.version_info.major}.{sys.version_info.minor} shardSyncIntervalMillis = 2000 parentShardPollIntervalMillis = 2000 idleTimeBetweenReadsInMillis = 1000 diff --git a/pyproject.toml b/pyproject.toml index de0b35c43fcc5..be492b6561838 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ runtime = [ # pinned / updated by ASF update action "awscli>=1.32.117", "airspeed-ext>=0.6.3", - "amazon_kclpy>=2.0.6,!=2.1.0,!=2.1.4", + "amazon_kclpy>=3.0.0", # antlr4-python3-runtime: exact pin because antlr4 runtime is tightly coupled to the generated parser code "antlr4-python3-runtime==4.13.2", "apispec>=5.1.1", From a3c8398e5d555de632b32923ea6c41a4afda2728 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Wed, 11 Dec 2024 09:43:05 +0100 Subject: [PATCH 5/5] re-add v2 compatibility mode for KCL --- localstack-core/localstack/utils/kinesis/kclipy_helper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/localstack-core/localstack/utils/kinesis/kclipy_helper.py b/localstack-core/localstack/utils/kinesis/kclipy_helper.py index 347210fa8e1e0..0f06fe5000aac 100644 --- a/localstack-core/localstack/utils/kinesis/kclipy_helper.py +++ b/localstack-core/localstack/utils/kinesis/kclipy_helper.py @@ -95,11 +95,13 @@ def create_config_file( ): if not credentialsProvider: credentialsProvider = "DefaultCredentialsProvider" + # TODO properly migrate to v3 of KCL and remove the clientVersionConfig content = f""" executableName = {executableName} streamName = {streamName} applicationName = {applicationName} AWSCredentialsProvider = {credentialsProvider} + clientVersionConfig = CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2x kinesisCredentialsProvider = {credentialsProvider} dynamoDBCredentialsProvider = {credentialsProvider} cloudWatchCredentialsProvider = {credentialsProvider}