From fe233dc7ac8be16e29eebf1376ac56ca46731494 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Tue, 7 Jan 2025 17:18:56 +0100 Subject: [PATCH 1/8] Make JPype JSONata package installable --- .../services/stepfunctions/asl/jsonata/jsonata.py | 10 ++++++---- .../stepfunctions/{asl/jsonata => }/packages.py | 12 +----------- .../localstack/services/stepfunctions/plugins.py | 9 +++++++++ 3 files changed, 16 insertions(+), 15 deletions(-) rename localstack-core/localstack/services/stepfunctions/{asl/jsonata => }/packages.py (52%) create mode 100644 localstack-core/localstack/services/stepfunctions/plugins.py diff --git a/localstack-core/localstack/services/stepfunctions/asl/jsonata/jsonata.py b/localstack-core/localstack/services/stepfunctions/asl/jsonata/jsonata.py index bf1ad25843035..459dba2ed80b2 100644 --- a/localstack-core/localstack/services/stepfunctions/asl/jsonata/jsonata.py +++ b/localstack-core/localstack/services/stepfunctions/asl/jsonata/jsonata.py @@ -8,8 +8,8 @@ import jpype import jpype.imports -from localstack.services.stepfunctions.asl.jsonata.packages import jsonata_package from localstack.services.stepfunctions.asl.utils.encoding import to_json_str +from localstack.services.stepfunctions.packages import jpype_jsonata_package from localstack.utils.objects import singleton_factory JSONataExpression = str @@ -40,17 +40,19 @@ class _JSONataJVMBridge: _java_JSONATA: "com.dashjoin.jsonata.Jsonata.jsonata" # noqa def __init__(self): - installer = jsonata_package.get_installer() + installer = jpype_jsonata_package.get_installer() installer.install() from jpype import config as jpype_config jpype_config.destroy_jvm = False + # Limitation: We can only start one JVM instance within LocalStack and using JPype for another purpose + # (e.g., event-ruler) fails unless we change the way we load/reload the classpath. jvm_path = installer.get_java_lib_path() jsonata_libs_path = Path(installer.get_installed_dir()) - event_ruler_libs_pattern = jsonata_libs_path.joinpath("*") - jpype.startJVM(jvm_path, classpath=[event_ruler_libs_pattern], interrupt=False) + jsonata_libs_pattern = jsonata_libs_path.joinpath("*") + jpype.startJVM(jvm_path, classpath=[jsonata_libs_pattern], interrupt=False) from com.fasterxml.jackson.databind import ObjectMapper # noqa from com.dashjoin.jsonata.Jsonata import jsonata # noqa diff --git a/localstack-core/localstack/services/stepfunctions/asl/jsonata/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py similarity index 52% rename from localstack-core/localstack/services/stepfunctions/asl/jsonata/packages.py rename to localstack-core/localstack/services/stepfunctions/packages.py index 277959f9143fa..9899260bec1fd 100644 --- a/localstack-core/localstack/services/stepfunctions/asl/jsonata/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -3,31 +3,21 @@ from localstack.packages.java import JavaInstallerMixin JSONATA_DEFAULT_VERSION = "0.9.7" -JACKSON_DEFAULT_VERSION = "2.16.2" - -JSONATA_JACKSON_VERSION_STORE = {JSONATA_DEFAULT_VERSION: JACKSON_DEFAULT_VERSION} class JSONataPackage(Package): def __init__(self): super().__init__("JSONataLibs", JSONATA_DEFAULT_VERSION) - def get_versions(self) -> list[str]: - return list(JSONATA_JACKSON_VERSION_STORE.keys()) - def _get_installer(self, version: str) -> PackageInstaller: return JSONataPackageInstaller(version) class JSONataPackageInstaller(JavaInstallerMixin, MavenPackageInstaller): def __init__(self, version: str): - jackson_version = JSONATA_JACKSON_VERSION_STORE[version] super().__init__( f"pkg:maven/com.dashjoin/jsonata@{version}", - f"pkg:maven/com.fasterxml.jackson.core/jackson-core@{jackson_version}", - f"pkg:maven/com.fasterxml.jackson.core/jackson-annotations@{jackson_version}", - f"pkg:maven/com.fasterxml.jackson.core/jackson-databind@{jackson_version}", ) -jsonata_package = JSONataPackage() +jpype_jsonata_package = JSONataPackage() diff --git a/localstack-core/localstack/services/stepfunctions/plugins.py b/localstack-core/localstack/services/stepfunctions/plugins.py new file mode 100644 index 0000000000000..b407ee2875396 --- /dev/null +++ b/localstack-core/localstack/services/stepfunctions/plugins.py @@ -0,0 +1,9 @@ +from localstack.packages import Package, package + + +@package(name="jpype-jsonata") +def jpype_jsonata_package() -> Package: + """The Java-based jsonata library uses JPype and depends on a JVM installation.""" + from localstack.services.stepfunctions.packages import jpype_jsonata_package + + return jpype_jsonata_package From f4170f6aa2589b52d343e1f80c4dee6970300e06 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Tue, 7 Jan 2025 17:19:41 +0100 Subject: [PATCH 2/8] Install jpype-jsonata lpm package by default --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ae5edace42519..98a5dfe13c9ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,6 +153,7 @@ RUN --mount=type=cache,target=/root/.cache \ source .venv/bin/activate && \ python -m localstack.cli.lpm install \ lambda-runtime \ + jpype-jsonata \ dynamodb-local && \ chown -R localstack:localstack /usr/lib/localstack && \ chmod -R 777 /usr/lib/localstack From b90570e6753541de5f56973c94ca9c88d5913fc2 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Tue, 7 Jan 2025 17:20:12 +0100 Subject: [PATCH 3/8] Clarify help for mount-source dev run --- localstack-core/localstack/dev/run/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localstack-core/localstack/dev/run/__main__.py b/localstack-core/localstack/dev/run/__main__.py index 610c77e4fac09..39ab236c9e3c2 100644 --- a/localstack-core/localstack/dev/run/__main__.py +++ b/localstack-core/localstack/dev/run/__main__.py @@ -36,7 +36,7 @@ type=str, required=False, help="Overwrite the container image to be used (defaults to localstack/localstack or " - "localstack/localstack-pro.", + "localstack/localstack-pro).", ) @click.option( "--volume-dir", @@ -66,7 +66,7 @@ "--mount-source/--no-mount-source", is_flag=True, default=True, - help="Mount source files from localstack, localstack-ext, and moto into the container.", + help="Mount source files from localstack and localstack-ext. Use --local-packages for optional dependencies such as moto.", ) @click.option( "--mount-dependencies/--no-mount-dependencies", From 3093c5792a598ccce35fb0220ae2b210f198f81e Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Tue, 7 Jan 2025 17:49:51 +0100 Subject: [PATCH 4/8] Unify JRE version with dynamodb-local --- localstack-core/localstack/services/stepfunctions/packages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/localstack-core/localstack/services/stepfunctions/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py index 9899260bec1fd..9d01eb9560d29 100644 --- a/localstack-core/localstack/services/stepfunctions/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -9,6 +9,9 @@ class JSONataPackage(Package): def __init__(self): super().__init__("JSONataLibs", JSONATA_DEFAULT_VERSION) + # Match the dynamodb-local JRE version to reduce the LocalStack image size by sharing the same JRE version + self.java_version = "21" + def _get_installer(self, version: str) -> PackageInstaller: return JSONataPackageInstaller(version) From 81565cc5ea37f7c6c081debe8436fa97f775025d Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Tue, 7 Jan 2025 18:05:16 +0100 Subject: [PATCH 5/8] Fix missing installer versions method --- localstack-core/localstack/services/stepfunctions/packages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/localstack-core/localstack/services/stepfunctions/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py index 9d01eb9560d29..4f0e27d7d7d38 100644 --- a/localstack-core/localstack/services/stepfunctions/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -15,6 +15,9 @@ def __init__(self): def _get_installer(self, version: str) -> PackageInstaller: return JSONataPackageInstaller(version) + def get_versions(self) -> list[str]: + return [JSONATA_DEFAULT_VERSION] + class JSONataPackageInstaller(JavaInstallerMixin, MavenPackageInstaller): def __init__(self, version: str): From bef9653506cf1fefd99df3c925a077a4a7750b44 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Wed, 8 Jan 2025 11:34:04 +0100 Subject: [PATCH 6/8] Revert back to default JRE 11 to mitigate conflict --- localstack-core/localstack/services/stepfunctions/packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localstack-core/localstack/services/stepfunctions/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py index 4f0e27d7d7d38..68cdfe60aadb1 100644 --- a/localstack-core/localstack/services/stepfunctions/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -9,8 +9,8 @@ class JSONataPackage(Package): def __init__(self): super().__init__("JSONataLibs", JSONATA_DEFAULT_VERSION) - # Match the dynamodb-local JRE version to reduce the LocalStack image size by sharing the same JRE version - self.java_version = "21" + # Warning: The `java_version` should be unique in LocalStack because JPype can only start a single JVM instance! + # Hence, we should avoid any conflicts with other JVM usages. def _get_installer(self, version: str) -> PackageInstaller: return JSONataPackageInstaller(version) From a748b1a8f2836ac6de46b66f73c65d57bab20c56 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Wed, 8 Jan 2025 12:40:32 +0100 Subject: [PATCH 7/8] Re-introduce and clarify Jackson dependency --- .../services/stepfunctions/packages.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/localstack-core/localstack/services/stepfunctions/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py index 68cdfe60aadb1..50f0a5af15880 100644 --- a/localstack-core/localstack/services/stepfunctions/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -3,26 +3,33 @@ from localstack.packages.java import JavaInstallerMixin JSONATA_DEFAULT_VERSION = "0.9.7" +JACKSON_DEFAULT_VERSION = "2.16.2" + +JSONATA_JACKSON_VERSION_STORE = {JSONATA_DEFAULT_VERSION: JACKSON_DEFAULT_VERSION} class JSONataPackage(Package): def __init__(self): super().__init__("JSONataLibs", JSONATA_DEFAULT_VERSION) - # Warning: The `java_version` should be unique in LocalStack because JPype can only start a single JVM instance! - # Hence, we should avoid any conflicts with other JVM usages. + def get_versions(self) -> list[str]: + return list(JSONATA_JACKSON_VERSION_STORE.keys()) def _get_installer(self, version: str) -> PackageInstaller: return JSONataPackageInstaller(version) - def get_versions(self) -> list[str]: - return [JSONATA_DEFAULT_VERSION] - class JSONataPackageInstaller(JavaInstallerMixin, MavenPackageInstaller): def __init__(self, version: str): + jackson_version = JSONATA_JACKSON_VERSION_STORE[version] super().__init__( f"pkg:maven/com.dashjoin/jsonata@{version}", + # jackson-databind is imported in jsonata.py as "from com.fasterxml.jackson.databind import ObjectMapper" + # jackson-annotations and jackson-core are dependencies of jackson-databind: + # https://central.sonatype.com/artifact/com.fasterxml.jackson.core/jackson-databind/dependencies + f"pkg:maven/com.fasterxml.jackson.core/jackson-core@{jackson_version}", + f"pkg:maven/com.fasterxml.jackson.core/jackson-annotations@{jackson_version}", + f"pkg:maven/com.fasterxml.jackson.core/jackson-databind@{jackson_version}", ) From c3b502c9bb85f5471eea965e8c8188bcbcdc682e Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Wed, 8 Jan 2025 14:00:34 +0100 Subject: [PATCH 8/8] Retry sharing JRE after restoring the required dependency --- localstack-core/localstack/services/stepfunctions/packages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/localstack-core/localstack/services/stepfunctions/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py index 50f0a5af15880..b96f7a8d775f0 100644 --- a/localstack-core/localstack/services/stepfunctions/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -12,6 +12,9 @@ class JSONataPackage(Package): def __init__(self): super().__init__("JSONataLibs", JSONATA_DEFAULT_VERSION) + # Match the dynamodb-local JRE version to reduce the LocalStack image size by sharing the same JRE version + self.java_version = "21" + def get_versions(self) -> list[str]: return list(JSONATA_JACKSON_VERSION_STORE.keys())