localstack · MEPalma · Nov 18, 2024 · Sep 7, 2024 · Sep 7, 2024 · Sep 12, 2024
@@ -15,7 +15,10 @@
     InitializationType,
     OtherServiceEndpoint,
 )
-from localstack.utils.lambda_debug_mode.lambda_debug_mode import is_lambda_debug_enabled_for
+from localstack.utils.lambda_debug_mode.lambda_debug_mode import (
+    is_lambda_debug_enabled_for,
+    is_lambda_debug_timeout_enabled_for,
+)
 
 LOG = logging.getLogger(__name__)
 
@@ -76,7 +79,10 @@ def get_environment(
 
         try:
             yield execution_environment
-            execution_environment.release()
+            if is_lambda_debug_timeout_enabled_for(lambda_arn=function_version.qualified_arn):
+                self.stop_environment(execution_environment)
+            else:
+                execution_environment.release()
         except InvalidStatusException as invalid_e:
             LOG.error("InvalidStatusException: %s", invalid_e)
         except Exception as e:

@@ -225,6 +225,7 @@
 )
 from localstack.utils.bootstrap import is_api_enabled
 from localstack.utils.collections import PaginatedList
+from localstack.utils.lambda_debug_mode.lambda_debug_mode_session import LambdaDebugModeSession
 from localstack.utils.strings import get_random_hex, short_uid, to_bytes, to_str
 from localstack.utils.sync import poll_condition
 from localstack.utils.urls import localstack_host
@@ -263,6 +264,17 @@ def __init__(self) -> None:
     def accept_state_visitor(self, visitor: StateVisitor):
         visitor.visit(lambda_stores)
 
+    def on_before_start(self):
+        # Attempt to start the Lambda Debug Mode session object.
+        try:
+            lambda_debug_mode_session = LambdaDebugModeSession.get()
+            lambda_debug_mode_session.ensure_running()
+        except Exception as ex:
+            LOG.error(
+                "Unexpected error encountered when attempting to initialise Lambda Debug Mode '%s'.",
+                ex,
+            )
+
     def on_before_state_reset(self):
         self.lambda_service.stop()
 
@@ -371,6 +383,15 @@ def on_after_init(self):
     def on_before_stop(self) -> None:
         # TODO: should probably unregister routes?
         self.lambda_service.stop()
+        # Attempt to signal to the Lambda Debug Mode session object to stop.
+        try:
+            lambda_debug_mode_session = LambdaDebugModeSession.get()
+            lambda_debug_mode_session.signal_stop()
+        except Exception as ex:
+            LOG.error(
+                "Unexpected error encountered when attempting to signal Lambda Debug Mode to stop '%s'.",
+                ex,
+            )
 
     @staticmethod
     def _get_function(function_name: str, account_id: str, region: str) -> Function:

diff --git a/localstack-core/localstack/utils/lambda_debug_mode/lambda_debug_mode_session.py b/localstack-core/localstack/utils/lambda_debug_mode/lambda_debug_mode_session.py
@@ -1,6 +1,9 @@
 from __future__ import annotations
 
 import logging
+import os
+import time
+from threading import Event, Thread
 from typing import Optional
 
 from localstack.aws.api.lambda_ import Arn
@@ -17,55 +20,158 @@
 
 class LambdaDebugModeSession:
     _is_lambda_debug_mode: bool
+
+    _configuration_file_path: Optional[str]
+    _watch_thread: Optional[Thread]
+    _initialised_event: Optional[Event]
+    _stop_event: Optional[Event]
     _config: Optional[LambdaDebugModeConfig]
 
     def __init__(self):
         self._is_lambda_debug_mode = bool(LAMBDA_DEBUG_MODE)
-        self._configuration = self._load_lambda_debug_mode_config()
+
+        # Disabled Lambda Debug Mode state initialisation.
+        self._configuration_file_path = None
+        self._watch_thread = None
+        self._initialised_event = None
+        self._stop_event = None
+        self._config = None
+
+        # Lambda Debug Mode is not enabled: leave as disabled state and return.
+        if not self._is_lambda_debug_mode:
+            return
+
+        # Lambda Debug Mode is enabled.
+        # Instantiate the configuration requirements if a configuration file is given.
+        self._configuration_file_path = LAMBDA_DEBUG_MODE_CONFIG_PATH
+        if not self._configuration_file_path:
+            return
+
+        # A configuration file path is given: initialised the resources to load and watch the file.
+
+        # Signal and block on first loading to ensure this is enforced from the very first
+        # invocation, as this module is not loaded at startup. The LambdaDebugModeConfigWatch
+        # thread will then take care of updating the configuration periodically and asynchronously.
+        # This may somewhat slow down the first upstream thread loading this module, but not
+        # future calls. On the other hand, avoiding this mechanism means that first Lambda calls
+        # occur with no Debug configuration.
+        self._initialised_event = Event()
+
+        # Signals when a shutdown signal from the application is registered.
+        self._stop_event = Event()
+
+        self._watch_thread = Thread(
+            target=self._watch_logic, args=(), daemon=True, name="LambdaDebugModeConfigWatch"
+        )
+        self._watch_thread.start()
 
     @staticmethod
     @singleton_factory
     def get() -> LambdaDebugModeSession:
         """Returns a singleton instance of the Lambda Debug Mode session."""
         return LambdaDebugModeSession()
 
-    def _load_lambda_debug_mode_config(self) -> Optional[LambdaDebugModeConfig]:
-        file_path = LAMBDA_DEBUG_MODE_CONFIG_PATH
-        if not self._is_lambda_debug_mode or file_path is None:
-            return None
+    def ensure_running(self) -> None:
+        # Nothing to start.
+        if self._watch_thread is None or self._watch_thread.is_alive():
+            return
+        try:
+            self._watch_thread.start()
+        except Exception as exception:
+            exception_str = str(exception)
+            # The thread was already restarted by another process.
+            if (
+                isinstance(exception, RuntimeError)
+                and exception_str
+                and "threads can only be started once" in exception_str
+            ):
+                return
+            LOG.error(
+                "Lambda Debug Mode could not restart the "
+                "hot reloading of the configuration file, '%s'",
+                exception_str,
+            )
+
+    def signal_stop(self) -> None:
+        stop_event = self._stop_event
+        if stop_event is not None:
+            stop_event.set()
 
+    def _load_lambda_debug_mode_config(self):
         yaml_configuration_string = None
         try:
-            with open(file_path, "r") as df:
+            with open(self._configuration_file_path, "r") as df:
                 yaml_configuration_string = df.read()
         except FileNotFoundError:
-            LOG.error("Error: The file lambda debug config " "file '%s' was not found.", file_path)
+            LOG.error(
+                "Error: The file lambda debug config " "file '%s' was not found.",
+                self._configuration_file_path,
+            )
         except IsADirectoryError:
             LOG.error(
                 "Error: Expected a lambda debug config file " "but found a directory at '%s'.",
-                file_path,
+                self._configuration_file_path,
             )
         except PermissionError:
             LOG.error(
                 "Error: Permission denied while trying to read "
                 "the lambda debug config file '%s'.",
-                file_path,
+                self._configuration_file_path,
             )
         except Exception as ex:
             LOG.error(
                 "Error: An unexpected error occurred while reading "
                 "lambda debug config '%s': '%s'",
-                file_path,
+                self._configuration_file_path,
                 ex,
             )
         if not yaml_configuration_string:
             return None
 
-        config = load_lambda_debug_mode_config(yaml_configuration_string)
-        return config
+        self._config = load_lambda_debug_mode_config(yaml_configuration_string)
+        if self._config is not None:
+            LOG.info("Lambda Debug Mode is now enforcing the latest configuration.")
+        else:
+            LOG.warning(
+                "Lambda Debug Mode could not load the latest configuration due to an error, "
+                "check logs for more details."
+            )
+
+    def _config_file_epoch_last_modified_or_now(self) -> int:
+        try:
+            modified_time = os.path.getmtime(self._configuration_file_path)
+            return int(modified_time)
+        except Exception as e:
+            LOG.warning("Lambda Debug Mode could not access the configuration file: %s", e)
+            epoch_now = int(time.time())
+            return epoch_now
+
+    def _watch_logic(self) -> None:
+        # TODO: consider relying on system calls (watchdog lib for cross-platform support)
+        #  instead of monitoring last modified dates.
+        # Run the first load and signal as initialised.
+        epoch_last_loaded: int = self._config_file_epoch_last_modified_or_now()
+        self._load_lambda_debug_mode_config()
+        self._initialised_event.set()
+
+        # Monitor for file changes whilst the application is running.
+        while not self._stop_event.is_set():
+            time.sleep(1)
+            epoch_last_modified = self._config_file_epoch_last_modified_or_now()
+            if epoch_last_modified > epoch_last_loaded:
+                epoch_last_loaded = epoch_last_modified
+                self._load_lambda_debug_mode_config()
+
+    def _get_initialised_config(self) -> Optional[LambdaDebugModeConfig]:
+        # Check the session is not initialising, and if so then wait for initialisation to finish.
+        # Note: the initialisation event is otherwise left set since after first initialisation has terminated.
+        if self._initialised_event is not None:
+            self._initialised_event.wait()
+        return self._config
 
     def is_lambda_debug_mode(self) -> bool:
         return self._is_lambda_debug_mode
 
     def debug_config_for(self, lambda_arn: Arn) -> Optional[LambdaDebugConfig]:
-        return self._configuration.functions.get(lambda_arn) if self._configuration else None
+        config = self._get_initialised_config()
+        return config.functions.get(lambda_arn) if config else None