localstack · alexrashed · Sep 26, 2023 · Apr 6, 2023 · Apr 6, 2023 · Apr 6, 2023
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,10 +3,10 @@ version: 2.1
 parameters:
   ubuntu-amd64-machine-image:
     type: string
-    default: "ubuntu-2004:2022.04.1"
+    default: "ubuntu-2204:2023.02.1"
   ubuntu-arm64-machine-image:
     type: string
-    default: "ubuntu-2004:2022.04.1"
+    default: "ubuntu-2204:2023.02.1"
 
 executors:
   ubuntu-machine-amd64:
@@ -39,13 +39,6 @@ jobs:
       - checkout
       - restore_cache:
           key: python-requirements-{{ checksum "setup.cfg" }}
-      - run:
-          name: Install prerequisites
-          command: |
-            # fix for: https://discuss.circleci.com/t/heroku-gpg-issues-in-ubuntu-images/43834/3
-            sudo rm -rf /etc/apt/sources.list.d/heroku.list
-            sudo apt-get update
-            sudo apt-get install -y libsasl2-dev
       - run:
           name: Setup environment
           command: |
@@ -106,6 +99,14 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - prepare-pytest-tinybird
+      - run:
+          # legacy tests are executed locally, new runners ship with Java 17, downgrade to Java 11 for stepfunctions
+          name: Install OpenJDK 11
+          command: |
+            sudo apt-get update && sudo apt-get install openjdk-11-jdk
+            sudo update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
+            sudo update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
+            java -version
       - run:
           name: Test 'local' Lambda executor
           environment:

diff --git a/.github/workflows/asf-updates.yml b/.github/workflows/asf-updates.yml
@@ -17,13 +17,13 @@ jobs:
       - name: Set up system wide dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install libsasl2-dev jq
+          sudo apt-get install jq
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.11
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Cache LocalStack community dependencies (venv)
         uses: actions/cache@v3

diff --git a/.github/workflows/marker-report-issue.yml b/.github/workflows/marker-report-issue.yml
@@ -33,7 +33,7 @@ jobs:
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install dependencies
         run: make install-dev

diff --git a/.github/workflows/marker-report.yml b/.github/workflows/marker-report.yml
@@ -23,7 +23,7 @@ jobs:
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Cache LocalStack community dependencies (venv)
         uses: actions/cache@v3

diff --git a/.github/workflows/tests-podman.yml b/.github/workflows/tests-podman.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install podman and test dependencies
         run: |

diff --git a/.github/workflows/tests-pro-integration.yml b/.github/workflows/tests-pro-integration.yml
@@ -150,11 +150,11 @@ jobs:
           token: ${{ secrets.PRO_ACCESS_TOKEN }}
           path: localstack-ext
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Set up Node 18.x
         uses: actions/setup-node@v3
@@ -175,7 +175,7 @@ jobs:
       - name: Install OS packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y --allow-downgrades libsasl2-dev jq postgresql-14=14.9-0ubuntu0* postgresql-client postgresql-plpython3
+          sudo apt-get install -y --allow-downgrades libsnappy-dev jq postgresql-14=14.9-0ubuntu0* postgresql-client postgresql-plpython3
 
       - name: Cache Ext Dependencies (venv)
         if: inputs.disableCaching != true

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # builder: Stage to build a custom JRE (with jlink)
-FROM python:3.10.13-slim-bullseye@sha256:5277d9d2e03ced08b332eb2a8fb86bb0800b45ac9bf16c235b9e63d532b9ca38 as java-builder
+FROM python:3.11.5-slim-buster@sha256:9f35f3a6420693c209c11bba63dcf103d88e47ebe0b205336b5168c122967edf as java-builder
 ARG TARGETARCH
 
 # install OpenJDK 11
@@ -37,7 +37,7 @@ jdk.localedata --include-locales en,th \
 
 
 # base: Stage which installs necessary runtime dependencies (OS packages, java,...)
-FROM python:3.10.13-slim-bullseye@sha256:5277d9d2e03ced08b332eb2a8fb86bb0800b45ac9bf16c235b9e63d532b9ca38 as base
+FROM python:3.11.5-slim-buster@sha256:9f35f3a6420693c209c11bba63dcf103d88e47ebe0b205336b5168c122967edf as base
 ARG TARGETARCH
 
 # Install runtime OS package dependencies
@@ -167,9 +167,9 @@ RUN --mount=type=cache,target=/root/.cache \
     chmod -R 777 /usr/lib/localstack
 
 # link the python package installer virtual environments into the localstack venv
-RUN echo /var/lib/localstack/lib/python-packages/lib/python3.10/site-packages > localstack-var-python-packages-venv.pth && \
+RUN echo /var/lib/localstack/lib/python-packages/lib/python3.11/site-packages > localstack-var-python-packages-venv.pth && \
     mv localstack-var-python-packages-venv.pth .venv/lib/python*/site-packages/
-RUN echo /usr/lib/localstack/python-packages/lib/python3.10/site-packages > localstack-static-python-packages-venv.pth && \
+RUN echo /usr/lib/localstack/python-packages/lib/python3.11/site-packages > localstack-static-python-packages-venv.pth && \
     mv localstack-static-python-packages-venv.pth .venv/lib/python*/site-packages/
 
 # Install the latest version of the LocalStack Persistence Plugin

diff --git a/Makefile b/Makefile
@@ -168,7 +168,7 @@ docker-run:        		  ## Run Docker image locally
 
 docker-mount-run:
 	MOTO_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages/moto | awk '{print $$NF}'); echo MOTO_DIR $$MOTO_DIR; \
-		DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/" make docker-run
+		DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.11/site-packages/moto/" make docker-run
 
 docker-cp-coverage:
 	@echo 'Extracting .coverage file from Docker image'; \
@@ -194,13 +194,13 @@ test-docker-mount:		  ## Run automated tests in Docker (mounting local code)
 	# TODO: find a cleaner way to mount/copy the dependencies into the container...
 	VENV_DIR=$$(pwd)/.venv/; \
 		PKG_DIR=$$(echo $$VENV_DIR/lib/python*/site-packages | awk '{print $$NF}'); \
-		PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.10/site-packages; \
+		PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.11/site-packages; \
 		echo "#!/usr/bin/env python" > /tmp/pytest.ls.bin; cat $$VENV_DIR/bin/pytest >> /tmp/pytest.ls.bin; chmod +x /tmp/pytest.ls.bin; \
 		DOCKER_FLAGS="-v `pwd`/tests:/opt/code/localstack/tests -v /tmp/pytest.ls.bin:/opt/code/localstack/.venv/bin/pytest -v $$PKG_DIR/deepdiff:$$PKG_DIR_CON/deepdiff -v $$PKG_DIR/ordered_set:$$PKG_DIR_CON/ordered_set -v $$PKG_DIR/py:$$PKG_DIR_CON/py -v $$PKG_DIR/pluggy:$$PKG_DIR_CON/pluggy -v $$PKG_DIR/iniconfig:$$PKG_DIR_CON/iniconfig -v $$PKG_DIR/jsonpath_ng:$$PKG_DIR_CON/jsonpath_ng -v $$PKG_DIR/packaging:$$PKG_DIR_CON/packaging -v $$PKG_DIR/pytest:$$PKG_DIR_CON/pytest -v $$PKG_DIR/pytest_httpserver:$$PKG_DIR_CON/pytest_httpserver -v $$PKG_DIR/_pytest:$$PKG_DIR_CON/_pytest -v $$PKG_DIR/_pytest:$$PKG_DIR_CON/orjson" make test-docker-mount-code
 
 test-docker-mount-code:
 	PACKAGES_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages | awk '{print $$NF}'); \
-		DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run
+		DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.11/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run
 
 lint:              		  ## Run code linter to check code style
 	($(VENV_RUN); python -m pflake8 --show-source)

diff --git a/localstack/dev/run/paths.py b/localstack/dev/run/paths.py
@@ -38,7 +38,7 @@ class ContainerPaths:
     """Important paths in the container"""
 
     project_dir: str = "/opt/code/localstack"
-    site_packages_target_dir: str = "/opt/code/localstack/.venv/lib/python3.10/site-packages"
+    site_packages_target_dir: str = "/opt/code/localstack/.venv/lib/python3.11/site-packages"
     docker_entrypoint: str = "/usr/local/bin/docker-entrypoint.sh"
     localstack_supervisor: str = "/usr/local/bin/localstack-supervisor"
     localstack_source_dir: str

diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py
@@ -1,16 +1,12 @@
 import logging
-import os
 import select
 import socket
-from typing import Tuple, Union
+from typing import Union
 
 from localstack.constants import BIND_HOST, LOCALHOST_IP
-from localstack.utils.asyncio import ensure_event_loop
-from localstack.utils.files import new_tmp_file, save_file
 from localstack.utils.functions import run_safe
 from localstack.utils.numbers import is_number
-from localstack.utils.ssl import create_ssl_cert
-from localstack.utils.threads import TMP_THREADS, FuncThread, start_worker_thread
+from localstack.utils.threads import start_worker_thread
 
 LOG = logging.getLogger(__name__)
 
@@ -77,94 +73,3 @@ def handle_request(s_src, thread):
             start_worker_thread(lambda *args, _thread: handle_request(src_socket, _thread))
         except socket.timeout:
             pass
-
-
-def start_ssl_proxy(
-    port: int,
-    target: PortOrUrl,
-    target_ssl=False,
-    client_cert_key: Tuple[str, str] = None,
-    asynchronous: bool = False,
-):
-    """Start a proxy server that accepts SSL requests and forwards requests to a backend (either SSL or non-SSL)"""
-
-    def _run(*args):
-        return _do_start_ssl_proxy(
-            port, target, target_ssl=target_ssl, client_cert_key=client_cert_key
-        )
-
-    if not asynchronous:
-        return _run()
-    proxy = FuncThread(_run, name="ssl-proxy")
-    TMP_THREADS.append(proxy)
-    proxy.start()
-    return proxy
-
-
-def _save_cert_keys(client_cert_key: Tuple[str, str]) -> Tuple[str, str]:
-    """
-    Save the given cert / key into files and returns their filename
-
-    :param client_cert_key: tuple with (client_cert, client_key)
-    :return: tuple of paths to files containing (client_cert, client_key)
-    """
-    cert_file = client_cert_key[0]
-    if not os.path.exists(cert_file):
-        cert_file = new_tmp_file()
-        save_file(cert_file, client_cert_key[0])
-    key_file = client_cert_key[1]
-    if not os.path.exists(key_file):
-        key_file = new_tmp_file()
-        save_file(key_file, client_cert_key[1])
-    return cert_file, key_file
-
-
-def _do_start_ssl_proxy(
-    port: int,
-    target: PortOrUrl,
-    target_ssl=False,
-    client_cert_key: Tuple[str, str] = None,
-    bind_address: str = "0.0.0.0",
-):
-    """
-    Starts a tcp proxy (with tls) on the specified port
-
-    :param port: Port the proxy should bind to
-    :param target: Target of the proxy. If a port, it will connect to localhost:
-    :param target_ssl: Specify if the proxy should connect to the target using SSL/TLS
-    :param client_cert_key: Client certificate for the target connection. Only set if target_ssl=True
-    :param bind_address: Bind address of the proxy server
-    """
-    import pproxy
-
-    if ":" not in str(target):
-        target = f"127.0.0.1:{target}"
-    LOG.debug("Starting SSL proxy server %s -> %s", port, target)
-
-    # create server and remote connection
-    server = pproxy.Server(f"secure+tunnel://{bind_address}:{port}")
-    target_proto = "ssl+tunnel" if target_ssl else "tunnel"
-    remote = pproxy.Connection(f"{target_proto}://{target}")
-    if client_cert_key:
-        # TODO verify client certs server side?
-        LOG.debug("Configuring ssl proxy to use client certs")
-        cert_file, key_file = _save_cert_keys(client_cert_key=client_cert_key)
-        remote.sslclient.load_cert_chain(certfile=cert_file, keyfile=key_file)
-    args = dict(rserver=[remote])
-
-    # set SSL contexts
-    _, cert_file_name, key_file_name = create_ssl_cert()
-    for context in pproxy.server.sslcontexts:
-        context.load_cert_chain(cert_file_name, key_file_name)
-
-    loop = ensure_event_loop()
-    handler = loop.run_until_complete(server.start_server(args))
-    try:
-        loop.run_forever()
-    except KeyboardInterrupt:
-        print("exit!")
-
-    handler.close()
-    loop.run_until_complete(handler.wait_closed())
-    loop.run_until_complete(loop.shutdown_asyncgens())
-    loop.close()
diff --git a/setup.cfg b/setup.cfg
@@ -7,7 +7,7 @@ author = LocalStack Contributors
 author_email = info@localstack.cloud
 license = Apache License 2.0
 classifiers =
-    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
     License :: OSI Approved :: Apache Software License
     Topic :: Internet
     Topic :: Software Development :: Testing
@@ -28,7 +28,7 @@ install_requires =
     click>=7.0
     cachetools~=5.0.0
     cryptography
-    dill==0.3.2
+    dill==0.3.6
     dnslib>=0.9.10
     dnspython>=1.16.0
     plux>=1.3.1
@@ -86,7 +86,6 @@ runtime =
     localstack-client>=2.0
     moto-ext[all]==4.2.4.post1
     opensearch-py==2.1.1
-    pproxy>=2.7.0
     pymongo>=4.2.0
     pyopenssl>=23.0.0
     Quart>=0.18

diff --git a/tests/aws/services/lambda_/test_lambda.py b/tests/aws/services/lambda_/test_lambda.py
@@ -108,7 +108,7 @@
 PYTHON_TEST_RUNTIMES = (
     RUNTIMES_AGGREGATED["python"]
     if (not is_old_provider() or use_docker()) and get_arch() != Arch.arm64
-    else [Runtime.python3_10]
+    else [Runtime.python3_11]
 )
 NODE_TEST_RUNTIMES = (
     RUNTIMES_AGGREGATED["nodejs"]

diff --git a/tests/unit/test_proxy.py b/tests/unit/test_proxy.py
@@ -14,7 +14,6 @@
     to_str,
     wait_for_port_open,
 )
-from localstack.utils.server.proxy_server import start_ssl_proxy
 
 LOG = logging.getLogger(__name__)
 
@@ -43,39 +42,6 @@ def test_start_and_stop(self, monkeypatch):
 
         assert not is_port_open(proxy_port)
 
-    def test_ssl_proxy_server(self):
-        class MyListener(ProxyListener):
-            def forward_request(self, *args, **kwargs):
-                invocations.append((args, kwargs))
-                return {"foo": "bar"}
-
-        invocations = []
-
-        # start SSL proxy
-        listener = MyListener()
-        port = get_free_tcp_port()
-        server = start_proxy_server(port, update_listener=listener, use_ssl=True)
-        wait_for_port_open(port)
-
-        # start SSL proxy
-        proxy_port = get_free_tcp_port()
-        proxy = start_ssl_proxy(proxy_port, port, asynchronous=True)
-        wait_for_port_open(proxy_port)
-
-        # invoke SSL proxy server
-        url = f"https://{LOCALHOST_HOSTNAME}:{proxy_port}"
-        num_requests = 3
-        for i in range(num_requests):
-            response = requests.get(url, verify=False)
-            assert response.status_code == 200
-
-        # assert backend server has been invoked
-        assert len(invocations) == num_requests
-
-        # clean up
-        proxy.stop()
-        server.stop()
-
     def test_static_route(self):
         class MyListener(ProxyListener):
             def forward_request(self, method, path, *args, **kwargs):