From 1a834339b37ac471cedaafb704bcd961b1043040 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Thu, 6 Apr 2023 11:56:41 +0200 Subject: [PATCH 01/15] first stab at python3.11 --- .circleci/config.yml | 4 ++-- .github/workflows/tests-pro-integration.yml | 2 +- Dockerfile | 8 ++++---- setup.cfg | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9383fb520f6f..84426c0bd522e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,10 +3,10 @@ version: 2.1 parameters: ubuntu-amd64-machine-image: type: string - default: "ubuntu-2004:2022.04.1" + default: "ubuntu-2004:2023.02.1" ubuntu-arm64-machine-image: type: string - default: "ubuntu-2004:2022.04.1" + default: "ubuntu-2004:2023.02.1" executors: ubuntu-machine-amd64: diff --git a/.github/workflows/tests-pro-integration.yml b/.github/workflows/tests-pro-integration.yml index 73bad0e59f128..744a2ec8b4fe4 100644 --- a/.github/workflows/tests-pro-integration.yml +++ b/.github/workflows/tests-pro-integration.yml @@ -154,7 +154,7 @@ jobs: id: setup-python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up Node 18.x uses: actions/setup-node@v3 diff --git a/Dockerfile b/Dockerfile index 9c0edeb9d532f..4d84f78f51f0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # builder: Stage to build a custom JRE (with jlink) -FROM python:3.10.13-slim-bullseye@sha256:5277d9d2e03ced08b332eb2a8fb86bb0800b45ac9bf16c235b9e63d532b9ca38 as java-builder +FROM python:3.11.5-slim-buster@sha256:9f35f3a6420693c209c11bba63dcf103d88e47ebe0b205336b5168c122967edf as java-builder ARG TARGETARCH # install OpenJDK 11 @@ -37,7 +37,7 @@ jdk.localedata --include-locales en,th \ # base: Stage which installs necessary runtime dependencies (OS packages, java,...) -FROM python:3.10.13-slim-bullseye@sha256:5277d9d2e03ced08b332eb2a8fb86bb0800b45ac9bf16c235b9e63d532b9ca38 as base +FROM python:3.11.5-slim-buster@sha256:9f35f3a6420693c209c11bba63dcf103d88e47ebe0b205336b5168c122967edf as base ARG TARGETARCH # Install runtime OS package dependencies @@ -167,9 +167,9 @@ RUN --mount=type=cache,target=/root/.cache \ chmod -R 777 /usr/lib/localstack # link the python package installer virtual environments into the localstack venv -RUN echo /var/lib/localstack/lib/python-packages/lib/python3.10/site-packages > localstack-var-python-packages-venv.pth && \ +RUN echo /var/lib/localstack/lib/python-packages/lib/python3.11/site-packages > localstack-var-python-packages-venv.pth && \ mv localstack-var-python-packages-venv.pth .venv/lib/python*/site-packages/ -RUN echo /usr/lib/localstack/python-packages/lib/python3.10/site-packages > localstack-static-python-packages-venv.pth && \ +RUN echo /usr/lib/localstack/python-packages/lib/python3.11/site-packages > localstack-static-python-packages-venv.pth && \ mv localstack-static-python-packages-venv.pth .venv/lib/python*/site-packages/ # Install the latest version of the LocalStack Persistence Plugin diff --git a/setup.cfg b/setup.cfg index e4372bff66fba..01f818375f102 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ author = LocalStack Contributors author_email = info@localstack.cloud license = Apache License 2.0 classifiers = - Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 License :: OSI Approved :: Apache Software License Topic :: Internet Topic :: Software Development :: Testing From 5f0447f66a82e2b0655f7775882d7f0662ca0c79 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Thu, 6 Apr 2023 13:27:56 +0200 Subject: [PATCH 02/15] fix sns_api snapshot transform regex --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84426c0bd522e..c4ad776ab39a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,10 +3,10 @@ version: 2.1 parameters: ubuntu-amd64-machine-image: type: string - default: "ubuntu-2004:2023.02.1" + default: "ubuntu-2204:2023.02.1" ubuntu-arm64-machine-image: type: string - default: "ubuntu-2004:2023.02.1" + default: "ubuntu-2204:2023.02.1" executors: ubuntu-machine-amd64: From 0d3e3f6c32c42c1ad7d5ccc51f483c991d35406c Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Thu, 6 Apr 2023 13:41:39 +0200 Subject: [PATCH 03/15] update dill --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 01f818375f102..4131e7ee48f58 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ install_requires = click>=7.0 cachetools~=5.0.0 cryptography - dill==0.3.2 + dill==0.3.6 dnslib>=0.9.10 dnspython>=1.16.0 plux>=1.3.1 From dcec85274bae0765aa350ec6a51fcf1bdbc55c52 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Thu, 13 Jul 2023 09:44:04 +0200 Subject: [PATCH 04/15] update python in pipelines --- .github/workflows/asf-updates.yml | 4 ++-- .github/workflows/tests-podman.yml | 2 +- .github/workflows/tests-pro-integration.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/asf-updates.yml b/.github/workflows/asf-updates.yml index d479f0094156e..e38bec452180f 100644 --- a/.github/workflows/asf-updates.yml +++ b/.github/workflows/asf-updates.yml @@ -19,11 +19,11 @@ jobs: sudo apt-get update sudo apt-get install libsasl2-dev jq - - name: Set up Python 3.8 + - name: Set up Python 3.11 id: setup-python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Cache LocalStack community dependencies (venv) uses: actions/cache@v3 diff --git a/.github/workflows/tests-podman.yml b/.github/workflows/tests-podman.yml index da12cf9933928..c3bd21f831b49 100644 --- a/.github/workflows/tests-podman.yml +++ b/.github/workflows/tests-podman.yml @@ -29,7 +29,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install podman and test dependencies run: | diff --git a/.github/workflows/tests-pro-integration.yml b/.github/workflows/tests-pro-integration.yml index 744a2ec8b4fe4..65f5a58b94c56 100644 --- a/.github/workflows/tests-pro-integration.yml +++ b/.github/workflows/tests-pro-integration.yml @@ -150,7 +150,7 @@ jobs: token: ${{ secrets.PRO_ACCESS_TOKEN }} path: localstack-ext - - name: Set up Python 3.10 + - name: Set up Python 3.11 id: setup-python uses: actions/setup-python@v4 with: From 90d47a215df75bca8c7c0f108b5c9e5660f924c8 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Thu, 13 Jul 2023 10:10:54 +0200 Subject: [PATCH 05/15] remove sasl OS package installations, add libsnappy-dev --- .circleci/config.yml | 7 ------- .github/workflows/asf-updates.yml | 2 +- .github/workflows/tests-pro-integration.yml | 2 +- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4ad776ab39a8..4d70de300f871 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,13 +39,6 @@ jobs: - checkout - restore_cache: key: python-requirements-{{ checksum "setup.cfg" }} - - run: - name: Install prerequisites - command: | - # fix for: https://discuss.circleci.com/t/heroku-gpg-issues-in-ubuntu-images/43834/3 - sudo rm -rf /etc/apt/sources.list.d/heroku.list - sudo apt-get update - sudo apt-get install -y libsasl2-dev - run: name: Setup environment command: | diff --git a/.github/workflows/asf-updates.yml b/.github/workflows/asf-updates.yml index e38bec452180f..cdc8b25bd44fd 100644 --- a/.github/workflows/asf-updates.yml +++ b/.github/workflows/asf-updates.yml @@ -17,7 +17,7 @@ jobs: - name: Set up system wide dependencies run: | sudo apt-get update - sudo apt-get install libsasl2-dev jq + sudo apt-get install jq - name: Set up Python 3.11 id: setup-python diff --git a/.github/workflows/tests-pro-integration.yml b/.github/workflows/tests-pro-integration.yml index 65f5a58b94c56..c9d1a8f85ae37 100644 --- a/.github/workflows/tests-pro-integration.yml +++ b/.github/workflows/tests-pro-integration.yml @@ -175,7 +175,7 @@ jobs: - name: Install OS packages run: | sudo apt-get update - sudo apt-get install -y --allow-downgrades libsasl2-dev jq postgresql-14=14.9-0ubuntu0* postgresql-client postgresql-plpython3 + sudo apt-get install -y --allow-downgrades libsnappy-dev jq postgresql-14=14.9-0ubuntu0* postgresql-client postgresql-plpython3 - name: Cache Ext Dependencies (venv) if: inputs.disableCaching != true From 6f345f1a09dbe5d16c541c4d920c9432d33e8af2 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Mon, 26 Jun 2023 15:56:12 +0200 Subject: [PATCH 06/15] remove pproxy --- localstack/utils/server/proxy_server.py | 99 +------------------------ setup.cfg | 1 - tests/unit/test_proxy.py | 34 --------- 3 files changed, 2 insertions(+), 132 deletions(-) diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py index c5af8f8716380..fad25e321acdb 100644 --- a/localstack/utils/server/proxy_server.py +++ b/localstack/utils/server/proxy_server.py @@ -1,16 +1,12 @@ import logging -import os import select import socket -from typing import Tuple, Union +from typing import Union from localstack.constants import BIND_HOST, LOCALHOST_IP -from localstack.utils.asyncio import ensure_event_loop -from localstack.utils.files import new_tmp_file, save_file from localstack.utils.functions import run_safe from localstack.utils.numbers import is_number -from localstack.utils.ssl import create_ssl_cert -from localstack.utils.threads import TMP_THREADS, FuncThread, start_worker_thread +from localstack.utils.threads import start_worker_thread LOG = logging.getLogger(__name__) @@ -77,94 +73,3 @@ def handle_request(s_src, thread): start_worker_thread(lambda *args, _thread: handle_request(src_socket, _thread)) except socket.timeout: pass - - -def start_ssl_proxy( - port: int, - target: PortOrUrl, - target_ssl=False, - client_cert_key: Tuple[str, str] = None, - asynchronous: bool = False, -): - """Start a proxy server that accepts SSL requests and forwards requests to a backend (either SSL or non-SSL)""" - - def _run(*args): - return _do_start_ssl_proxy( - port, target, target_ssl=target_ssl, client_cert_key=client_cert_key - ) - - if not asynchronous: - return _run() - proxy = FuncThread(_run, name="ssl-proxy") - TMP_THREADS.append(proxy) - proxy.start() - return proxy - - -def _save_cert_keys(client_cert_key: Tuple[str, str]) -> Tuple[str, str]: - """ - Save the given cert / key into files and returns their filename - - :param client_cert_key: tuple with (client_cert, client_key) - :return: tuple of paths to files containing (client_cert, client_key) - """ - cert_file = client_cert_key[0] - if not os.path.exists(cert_file): - cert_file = new_tmp_file() - save_file(cert_file, client_cert_key[0]) - key_file = client_cert_key[1] - if not os.path.exists(key_file): - key_file = new_tmp_file() - save_file(key_file, client_cert_key[1]) - return cert_file, key_file - - -def _do_start_ssl_proxy( - port: int, - target: PortOrUrl, - target_ssl=False, - client_cert_key: Tuple[str, str] = None, - bind_address: str = "0.0.0.0", -): - """ - Starts a tcp proxy (with tls) on the specified port - - :param port: Port the proxy should bind to - :param target: Target of the proxy. If a port, it will connect to localhost: - :param target_ssl: Specify if the proxy should connect to the target using SSL/TLS - :param client_cert_key: Client certificate for the target connection. Only set if target_ssl=True - :param bind_address: Bind address of the proxy server - """ - import pproxy - - if ":" not in str(target): - target = f"127.0.0.1:{target}" - LOG.debug("Starting SSL proxy server %s -> %s", port, target) - - # create server and remote connection - server = pproxy.Server(f"secure+tunnel://{bind_address}:{port}") - target_proto = "ssl+tunnel" if target_ssl else "tunnel" - remote = pproxy.Connection(f"{target_proto}://{target}") - if client_cert_key: - # TODO verify client certs server side? - LOG.debug("Configuring ssl proxy to use client certs") - cert_file, key_file = _save_cert_keys(client_cert_key=client_cert_key) - remote.sslclient.load_cert_chain(certfile=cert_file, keyfile=key_file) - args = dict(rserver=[remote]) - - # set SSL contexts - _, cert_file_name, key_file_name = create_ssl_cert() - for context in pproxy.server.sslcontexts: - context.load_cert_chain(cert_file_name, key_file_name) - - loop = ensure_event_loop() - handler = loop.run_until_complete(server.start_server(args)) - try: - loop.run_forever() - except KeyboardInterrupt: - print("exit!") - - handler.close() - loop.run_until_complete(handler.wait_closed()) - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() diff --git a/setup.cfg b/setup.cfg index 4131e7ee48f58..5675e2b48a26b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -86,7 +86,6 @@ runtime = localstack-client>=2.0 moto-ext[all]==4.2.4.post1 opensearch-py==2.1.1 - pproxy>=2.7.0 pymongo>=4.2.0 pyopenssl>=23.0.0 Quart>=0.18 diff --git a/tests/unit/test_proxy.py b/tests/unit/test_proxy.py index 92d5aa0bcb278..2f1031d7cf7cc 100644 --- a/tests/unit/test_proxy.py +++ b/tests/unit/test_proxy.py @@ -14,7 +14,6 @@ to_str, wait_for_port_open, ) -from localstack.utils.server.proxy_server import start_ssl_proxy LOG = logging.getLogger(__name__) @@ -43,39 +42,6 @@ def test_start_and_stop(self, monkeypatch): assert not is_port_open(proxy_port) - def test_ssl_proxy_server(self): - class MyListener(ProxyListener): - def forward_request(self, *args, **kwargs): - invocations.append((args, kwargs)) - return {"foo": "bar"} - - invocations = [] - - # start SSL proxy - listener = MyListener() - port = get_free_tcp_port() - server = start_proxy_server(port, update_listener=listener, use_ssl=True) - wait_for_port_open(port) - - # start SSL proxy - proxy_port = get_free_tcp_port() - proxy = start_ssl_proxy(proxy_port, port, asynchronous=True) - wait_for_port_open(proxy_port) - - # invoke SSL proxy server - url = f"https://{LOCALHOST_HOSTNAME}:{proxy_port}" - num_requests = 3 - for i in range(num_requests): - response = requests.get(url, verify=False) - assert response.status_code == 200 - - # assert backend server has been invoked - assert len(invocations) == num_requests - - # clean up - proxy.stop() - server.stop() - def test_static_route(self): class MyListener(ProxyListener): def forward_request(self, method, path, *args, **kwargs): From 549e8aab853190ba79ab70e34856236e3368a3ef Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Mon, 24 Jul 2023 16:24:57 +0200 Subject: [PATCH 07/15] first version of a different approach (tls proxy) --- localstack/utils/server/proxy_server.py | 101 ++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py index fad25e321acdb..a71ba91d8edcc 100644 --- a/localstack/utils/server/proxy_server.py +++ b/localstack/utils/server/proxy_server.py @@ -1,11 +1,17 @@ import logging +import os import select import socket +import ssl +from concurrent.futures import ThreadPoolExecutor from typing import Union from localstack.constants import BIND_HOST, LOCALHOST_IP +from localstack.utils.files import new_tmp_file, save_file from localstack.utils.functions import run_safe from localstack.utils.numbers import is_number +from localstack.utils.serving import Server +from localstack.utils.ssl import create_ssl_cert from localstack.utils.threads import start_worker_thread LOG = logging.getLogger(__name__) @@ -73,3 +79,98 @@ def handle_request(s_src, thread): start_worker_thread(lambda *args, _thread: handle_request(src_socket, _thread)) except socket.timeout: pass + + +def _save_cert_keys(client_cert_key: tuple[str, str]) -> tuple[str, str]: + """ + Save the given cert / key into files and returns their filename + :param client_cert_key: tuple with (client_cert, client_key) + :return: tuple of paths to files containing (client_cert, client_key) + """ + cert_file = client_cert_key[0] + if not os.path.exists(cert_file): + cert_file = new_tmp_file() + save_file(cert_file, client_cert_key[0]) + key_file = client_cert_key[1] + if not os.path.exists(key_file): + key_file = new_tmp_file() + save_file(key_file, client_cert_key[1]) + return cert_file, key_file + + +class TLSProxyServer(Server): + thread_pool: ThreadPoolExecutor + client_certs: tuple[str, str] + socket: socket.socket | None + target_host: str + target_port: str + + def __init__( + self, + port: int, + target: str, + host: str = "localhost", + client_certs: tuple[str, str] | None = None, + ): + super().__init__(port, host) + self.target_host, _, self.target_port = target.partition(":") + self.thread_pool = ThreadPoolExecutor() + self.client_certs = client_certs + self.socket = None + + def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> None: + LOG.debug( + "Handling connection from %s to %s:%s", + client_address, + self.target_host, + self.target_port, + ) + try: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + if self.client_certs: + LOG.debug("Configuring ssl proxy to use client certs") + cert_file, key_file = _save_cert_keys(client_cert_key=self.client_certs) + context.load_cert_chain(certfile=cert_file, keyfile=key_file) + with socket.create_connection((self.target_host, int(self.target_port))) as sock: + with context.wrap_socket(sock, server_hostname=self.target_host) as target_socket: + sockets = [source_socket, target_socket] + while not self._stopped.is_set(): + s_read, _, _ = select.select(sockets, [], []) + + for s in s_read: + data = s.recv(BUFFER_SIZE) + if not data: + return + + if s == source_socket: + target_socket.sendall(data) + elif s == target_socket: + source_socket.sendall(data) + except Exception as e: + LOG.warning( + "Error while proxying SSL request: %s", e, exc_info=LOG.isEnabledFor(logging.DEBUG) + ) + + def do_run(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + + _, cert_file_name, key_file_name = create_ssl_cert() + context.load_cert_chain(cert_file_name, key_file_name) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: + self.socket = sock + sock.bind((self.host, self.port)) + sock.listen() + with context.wrap_socket(sock, server_side=True) as ssock: + while not self._stopped.is_set(): + try: + conn, addr = ssock.accept() + self.thread_pool.submit(self._handle_socket, conn, addr) + except Exception as e: + LOG.exception("Error accepting socket: %s", e) + + def do_shutdown(self): + if self.socket: + self.socket.close() From dd9da53e964dc133542588af5bd925dbc75c0d16 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Tue, 25 Jul 2023 17:20:19 +0200 Subject: [PATCH 08/15] fix tls proxy due to select only affecting underlying socket --- localstack/utils/server/proxy_server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py index a71ba91d8edcc..ce01269b9087e 100644 --- a/localstack/utils/server/proxy_server.py +++ b/localstack/utils/server/proxy_server.py @@ -17,6 +17,7 @@ LOG = logging.getLogger(__name__) BUFFER_SIZE = 2**10 # 1024 +TLS_BUFFER_SIZE = 16384 # 16 KB, max TLS record size PortOrUrl = Union[str, int] @@ -140,7 +141,7 @@ def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> N s_read, _, _ = select.select(sockets, [], []) for s in s_read: - data = s.recv(BUFFER_SIZE) + data = s.recv(TLS_BUFFER_SIZE) if not data: return @@ -152,6 +153,9 @@ def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> N LOG.warning( "Error while proxying SSL request: %s", e, exc_info=LOG.isEnabledFor(logging.DEBUG) ) + finally: + source_socket.close() + LOG.debug("Connection finished!") def do_run(self): context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) @@ -168,6 +172,8 @@ def do_run(self): try: conn, addr = ssock.accept() self.thread_pool.submit(self._handle_socket, conn, addr) + except ssl.SSLZeroReturnError: + pass except Exception as e: LOG.exception("Error accepting socket: %s", e) From 0af9ee4d1e975c2a597503f594101bc966a186f4 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Thu, 14 Sep 2023 12:32:45 +0200 Subject: [PATCH 09/15] update newly created references to Python 3.10 to Python 3.11 --- .github/workflows/marker-report-issue.yml | 2 +- .github/workflows/marker-report.yml | 2 +- Makefile | 6 +++--- localstack/dev/run/paths.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/marker-report-issue.yml b/.github/workflows/marker-report-issue.yml index 18dc7717bd798..60dd7d38c760a 100644 --- a/.github/workflows/marker-report-issue.yml +++ b/.github/workflows/marker-report-issue.yml @@ -33,7 +33,7 @@ jobs: id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: make install-dev diff --git a/.github/workflows/marker-report.yml b/.github/workflows/marker-report.yml index cc8e777e923ea..95205b99de43e 100644 --- a/.github/workflows/marker-report.yml +++ b/.github/workflows/marker-report.yml @@ -23,7 +23,7 @@ jobs: id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Cache LocalStack community dependencies (venv) uses: actions/cache@v3 diff --git a/Makefile b/Makefile index a842accd06f25..cf7bc522cc0b7 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ docker-run: ## Run Docker image locally docker-mount-run: MOTO_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages/moto | awk '{print $$NF}'); echo MOTO_DIR $$MOTO_DIR; \ - DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/" make docker-run + DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.11/site-packages/moto/" make docker-run docker-cp-coverage: @echo 'Extracting .coverage file from Docker image'; \ @@ -194,13 +194,13 @@ test-docker-mount: ## Run automated tests in Docker (mounting local code) # TODO: find a cleaner way to mount/copy the dependencies into the container... VENV_DIR=$$(pwd)/.venv/; \ PKG_DIR=$$(echo $$VENV_DIR/lib/python*/site-packages | awk '{print $$NF}'); \ - PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.10/site-packages; \ + PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.11/site-packages; \ echo "#!/usr/bin/env python" > /tmp/pytest.ls.bin; cat $$VENV_DIR/bin/pytest >> /tmp/pytest.ls.bin; chmod +x /tmp/pytest.ls.bin; \ DOCKER_FLAGS="-v `pwd`/tests:/opt/code/localstack/tests -v /tmp/pytest.ls.bin:/opt/code/localstack/.venv/bin/pytest -v $$PKG_DIR/deepdiff:$$PKG_DIR_CON/deepdiff -v $$PKG_DIR/ordered_set:$$PKG_DIR_CON/ordered_set -v $$PKG_DIR/py:$$PKG_DIR_CON/py -v $$PKG_DIR/pluggy:$$PKG_DIR_CON/pluggy -v $$PKG_DIR/iniconfig:$$PKG_DIR_CON/iniconfig -v $$PKG_DIR/jsonpath_ng:$$PKG_DIR_CON/jsonpath_ng -v $$PKG_DIR/packaging:$$PKG_DIR_CON/packaging -v $$PKG_DIR/pytest:$$PKG_DIR_CON/pytest -v $$PKG_DIR/pytest_httpserver:$$PKG_DIR_CON/pytest_httpserver -v $$PKG_DIR/_pytest:$$PKG_DIR_CON/_pytest -v $$PKG_DIR/_pytest:$$PKG_DIR_CON/orjson" make test-docker-mount-code test-docker-mount-code: PACKAGES_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages | awk '{print $$NF}'); \ - DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run + DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.11/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run lint: ## Run code linter to check code style ($(VENV_RUN); python -m pflake8 --show-source) diff --git a/localstack/dev/run/paths.py b/localstack/dev/run/paths.py index c17c9b5ba97e0..7578076be905e 100644 --- a/localstack/dev/run/paths.py +++ b/localstack/dev/run/paths.py @@ -38,7 +38,7 @@ class ContainerPaths: """Important paths in the container""" project_dir: str = "/opt/code/localstack" - site_packages_target_dir: str = "/opt/code/localstack/.venv/lib/python3.10/site-packages" + site_packages_target_dir: str = "/opt/code/localstack/.venv/lib/python3.11/site-packages" docker_entrypoint: str = "/usr/local/bin/docker-entrypoint.sh" localstack_supervisor: str = "/usr/local/bin/localstack-supervisor" localstack_source_dir: str From 69b8a421178911cc659e5c555c250b47fcb82ca8 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Thu, 14 Sep 2023 13:07:39 +0200 Subject: [PATCH 10/15] upgrade default lambda python runtime for host mode tests --- tests/aws/services/lambda_/test_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aws/services/lambda_/test_lambda.py b/tests/aws/services/lambda_/test_lambda.py index dd83707c8aad4..25b7ce8772a67 100644 --- a/tests/aws/services/lambda_/test_lambda.py +++ b/tests/aws/services/lambda_/test_lambda.py @@ -108,7 +108,7 @@ PYTHON_TEST_RUNTIMES = ( RUNTIMES_AGGREGATED["python"] if (not is_old_provider() or use_docker()) and get_arch() != Arch.arm64 - else [Runtime.python3_10] + else [Runtime.python3_11] ) NODE_TEST_RUNTIMES = ( RUNTIMES_AGGREGATED["nodejs"] From 2713fa04e3349279aa99d4b8ab8189f162581abf Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Thu, 14 Sep 2023 15:01:45 +0200 Subject: [PATCH 11/15] downgrade Java from 17 to 11 for locally executed tests --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d70de300f871..f95a261eb8b25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,6 +99,14 @@ jobs: - attach_workspace: at: /tmp/workspace - prepare-pytest-tinybird + - run: + # legacy tests are executed locally, new runners ship with Java 17, downgrade to Java 11 for stepfunctions + name: Install OpenJDK 11 + command: | + sudo apt-get update && sudo apt-get install openjdk-11-jdk + sudo update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + sudo update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + java -version - run: name: Test 'local' Lambda executor environment: From 78f6e3a9fc46622d74ce123cb4ebab9b320982b4 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Wed, 20 Sep 2023 15:59:08 +0200 Subject: [PATCH 12/15] slightly refactor tls proxy --- localstack/utils/server/proxy_server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py index ce01269b9087e..645b639f5dfa0 100644 --- a/localstack/utils/server/proxy_server.py +++ b/localstack/utils/server/proxy_server.py @@ -115,7 +115,7 @@ def __init__( ): super().__init__(port, host) self.target_host, _, self.target_port = target.partition(":") - self.thread_pool = ThreadPoolExecutor() + self.thread_pool = ThreadPoolExecutor(32) self.client_certs = client_certs self.socket = None @@ -141,9 +141,12 @@ def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> N s_read, _, _ = select.select(sockets, [], []) for s in s_read: - data = s.recv(TLS_BUFFER_SIZE) + data = s.recv(1024) if not data: return + pending_data = s.pending() + if pending_data: + data += s.recv(pending_data) if s == source_socket: target_socket.sendall(data) From d6fd7cca581b04f814d42a9ed6d7d0994a538f42 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Mon, 25 Sep 2023 14:39:19 +0200 Subject: [PATCH 13/15] Revert "slightly refactor tls proxy" This reverts commit 3d4296114f6bb2c090bfcc212c24ac713dc09dba. --- localstack/utils/server/proxy_server.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py index 645b639f5dfa0..ce01269b9087e 100644 --- a/localstack/utils/server/proxy_server.py +++ b/localstack/utils/server/proxy_server.py @@ -115,7 +115,7 @@ def __init__( ): super().__init__(port, host) self.target_host, _, self.target_port = target.partition(":") - self.thread_pool = ThreadPoolExecutor(32) + self.thread_pool = ThreadPoolExecutor() self.client_certs = client_certs self.socket = None @@ -141,12 +141,9 @@ def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> N s_read, _, _ = select.select(sockets, [], []) for s in s_read: - data = s.recv(1024) + data = s.recv(TLS_BUFFER_SIZE) if not data: return - pending_data = s.pending() - if pending_data: - data += s.recv(pending_data) if s == source_socket: target_socket.sendall(data) From b9a7824dd67f507b2a5b205c569a1fdc9a191f74 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Mon, 25 Sep 2023 14:39:37 +0200 Subject: [PATCH 14/15] Revert "fix tls proxy due to select only affecting underlying socket" This reverts commit 4fa343645679540b1155da5b929cdded013a9e3d. --- localstack/utils/server/proxy_server.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py index ce01269b9087e..a71ba91d8edcc 100644 --- a/localstack/utils/server/proxy_server.py +++ b/localstack/utils/server/proxy_server.py @@ -17,7 +17,6 @@ LOG = logging.getLogger(__name__) BUFFER_SIZE = 2**10 # 1024 -TLS_BUFFER_SIZE = 16384 # 16 KB, max TLS record size PortOrUrl = Union[str, int] @@ -141,7 +140,7 @@ def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> N s_read, _, _ = select.select(sockets, [], []) for s in s_read: - data = s.recv(TLS_BUFFER_SIZE) + data = s.recv(BUFFER_SIZE) if not data: return @@ -153,9 +152,6 @@ def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> N LOG.warning( "Error while proxying SSL request: %s", e, exc_info=LOG.isEnabledFor(logging.DEBUG) ) - finally: - source_socket.close() - LOG.debug("Connection finished!") def do_run(self): context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) @@ -172,8 +168,6 @@ def do_run(self): try: conn, addr = ssock.accept() self.thread_pool.submit(self._handle_socket, conn, addr) - except ssl.SSLZeroReturnError: - pass except Exception as e: LOG.exception("Error accepting socket: %s", e) From 99c768683108ed930509b18ed700cd42f2e7925c Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Mon, 25 Sep 2023 14:39:53 +0200 Subject: [PATCH 15/15] Revert "first version of a different approach (tls proxy)" This reverts commit f32537586f2084fd664341a06a54b120e8b98904. --- localstack/utils/server/proxy_server.py | 101 ------------------------ 1 file changed, 101 deletions(-) diff --git a/localstack/utils/server/proxy_server.py b/localstack/utils/server/proxy_server.py index a71ba91d8edcc..fad25e321acdb 100644 --- a/localstack/utils/server/proxy_server.py +++ b/localstack/utils/server/proxy_server.py @@ -1,17 +1,11 @@ import logging -import os import select import socket -import ssl -from concurrent.futures import ThreadPoolExecutor from typing import Union from localstack.constants import BIND_HOST, LOCALHOST_IP -from localstack.utils.files import new_tmp_file, save_file from localstack.utils.functions import run_safe from localstack.utils.numbers import is_number -from localstack.utils.serving import Server -from localstack.utils.ssl import create_ssl_cert from localstack.utils.threads import start_worker_thread LOG = logging.getLogger(__name__) @@ -79,98 +73,3 @@ def handle_request(s_src, thread): start_worker_thread(lambda *args, _thread: handle_request(src_socket, _thread)) except socket.timeout: pass - - -def _save_cert_keys(client_cert_key: tuple[str, str]) -> tuple[str, str]: - """ - Save the given cert / key into files and returns their filename - :param client_cert_key: tuple with (client_cert, client_key) - :return: tuple of paths to files containing (client_cert, client_key) - """ - cert_file = client_cert_key[0] - if not os.path.exists(cert_file): - cert_file = new_tmp_file() - save_file(cert_file, client_cert_key[0]) - key_file = client_cert_key[1] - if not os.path.exists(key_file): - key_file = new_tmp_file() - save_file(key_file, client_cert_key[1]) - return cert_file, key_file - - -class TLSProxyServer(Server): - thread_pool: ThreadPoolExecutor - client_certs: tuple[str, str] - socket: socket.socket | None - target_host: str - target_port: str - - def __init__( - self, - port: int, - target: str, - host: str = "localhost", - client_certs: tuple[str, str] | None = None, - ): - super().__init__(port, host) - self.target_host, _, self.target_port = target.partition(":") - self.thread_pool = ThreadPoolExecutor() - self.client_certs = client_certs - self.socket = None - - def _handle_socket(self, source_socket: ssl.SSLSocket, client_address: str) -> None: - LOG.debug( - "Handling connection from %s to %s:%s", - client_address, - self.target_host, - self.target_port, - ) - try: - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - if self.client_certs: - LOG.debug("Configuring ssl proxy to use client certs") - cert_file, key_file = _save_cert_keys(client_cert_key=self.client_certs) - context.load_cert_chain(certfile=cert_file, keyfile=key_file) - with socket.create_connection((self.target_host, int(self.target_port))) as sock: - with context.wrap_socket(sock, server_hostname=self.target_host) as target_socket: - sockets = [source_socket, target_socket] - while not self._stopped.is_set(): - s_read, _, _ = select.select(sockets, [], []) - - for s in s_read: - data = s.recv(BUFFER_SIZE) - if not data: - return - - if s == source_socket: - target_socket.sendall(data) - elif s == target_socket: - source_socket.sendall(data) - except Exception as e: - LOG.warning( - "Error while proxying SSL request: %s", e, exc_info=LOG.isEnabledFor(logging.DEBUG) - ) - - def do_run(self): - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - - _, cert_file_name, key_file_name = create_ssl_cert() - context.load_cert_chain(cert_file_name, key_file_name) - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: - self.socket = sock - sock.bind((self.host, self.port)) - sock.listen() - with context.wrap_socket(sock, server_side=True) as ssock: - while not self._stopped.is_set(): - try: - conn, addr = ssock.accept() - self.thread_pool.submit(self._handle_socket, conn, addr) - except Exception as e: - LOG.exception("Error accepting socket: %s", e) - - def do_shutdown(self): - if self.socket: - self.socket.close()