diff --git a/.github/workflows/execute_tests.yml b/.github/workflows/execute_tests.yml new file mode 100644 index 0000000..57a6fdf --- /dev/null +++ b/.github/workflows/execute_tests.yml @@ -0,0 +1,32 @@ +name: Execute tests + +on: push + +jobs: + execute_tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9'] + name: Python ${{ matrix.python-version }} tests + steps: + - uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Print information + run: | + echo "python version $(python --version) running" + echo "pip version $(pip --version) running" + + - name: Build + run: | + pip install wheel + python setup.py sdist bdist_wheel + + - name: Test + run: | + python setup.py test diff --git a/.github/workflows/push_to_pypi.yml b/.github/workflows/push_to_pypi.yml new file mode 100644 index 0000000..a5b1afd --- /dev/null +++ b/.github/workflows/push_to_pypi.yml @@ -0,0 +1,40 @@ +name: Push package to pypi + +on: + pull_request: + types: + - closed + branches: + - release + +env: + PYTHON_VERSION: "3.9" + +jobs: + push_to_pypi: + if: github.event.pull_request.merged + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Print information + run: | + echo "python version $(python --version) running" + echo "pip version $(pip --version) running" + + - name: Build + run: | + pip install wheel + python setup.py sdist bdist_wheel + + - name: Deploy + env: + PYPI_USERNAME: "${{ secrets.PYPI_USERNAME }}" + PYPI_PASSWORD: "${{ secrets.PYPI_PASSWORD }}" + run: | + sh deploy.sh diff --git a/.gitignore b/.gitignore index 4b36cff..6a5671f 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,10 @@ target/ # Python environment .python-version + +# Vitual Environments +venv/ +.env/ + +# direnv +.envrc diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 5776446..d732b63 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -22,3 +22,51 @@ Version 0.1.4 ------------- - Support Python 3.6. + +Version 0.1.5 +------------- + + - Support reading environment variables from file. + +Version 0.1.6 +------------- + + - Add timeout feature for Windows. + +Version 0.1.7 +------------- + + - Add tests in CI. + +Version 0.1.8 +------------- + + - Support running as a library. + +Version 0.1.9 +------------- + + - Support latest boto3. + +Version 0.1.10 +-------------- + + - Fix traceback output when exception happens. + +Version 0.1.11 +-------------- + + - Test on python 3.8. + +Version 0.1.12 +-------------- + + - Fix error when running on Windows with Python 3. + +Version 0.1.13 +-------------- + + - Drop support of Python 2.7. + - Various update and cleanup, add Python 3.9 support. + - Add __main__ file for using the package as a module. + - Implement tests via Github Actions. diff --git a/LICENSE b/LICENSE index a3a3ed5..9416dcb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2017 HDE, Inc. +Copyright (c) 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index c314974..6cd0b85 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # python-lambda-local [![Join the chat at https://gitter.im/HDE/python-lambda-local](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/HDE/python-lambda-local?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![wercker status](https://app.wercker.com/status/04f5bc5b7de3d5c6f13eb5b871035226/s "wercker status")](https://app.wercker.com/project/bykey/04f5bc5b7de3d5c6f13eb5b871035226) +[![Github Actions status](https://github.com/HDE/python-lambda-local/actions/workflows/execute_tests.yml/badge.svg)](https://github.com/HDE/python-lambda-local/actions/) [![PyPI version](https://badge.fury.io/py/python-lambda-local.svg)](https://badge.fury.io/py/python-lambda-local) Run lambda function on local machine ## Prepare development environment -Please use a newly created virtualenv of Python 2.7 or Python 3.6 . +Please use a newly created virtualenv of Python 3.7+. ## Installation @@ -21,14 +21,14 @@ $ pip install python-lambda-local This will install the package with name `python-lambda-local` in the virtualenv. Now you can use the command `python-lambda-local` to run your AWS Lambda function written in Python on your own machine. -## Usage +## Usage as a shell command Run `python-lambda-local -h` to see the help. ``` usage: python-lambda-local [-h] [-l LIBRARY_PATH] [-f HANDLER_FUNCTION] [-t TIMEOUT] [-a ARN_STRING] [-v VERSION_NAME] - [--version] + [-e ENVIRONMENT_VARIABLES] [--version] FILE EVENT Run AWS Lambda function written in Python on local machine. @@ -68,7 +68,7 @@ Suppose your project directory is like this: │   │   ├── ... (package content of rx) ... │   │   └── testscheduler.py -│   └── Rx-1.2.3.dist-info +│   └── Rx-1.6.1.dist-info │   ├── DESCRIPTION.rst │   ├── METADATA │   ├── metadata.json @@ -118,16 +118,50 @@ python-lambda-local -l lib/ -f handler -t 5 test.py event.json The output will be like: ``` -[root - INFO - 2017-04-19 12:39:05,512] Event: {u'answer': 42} -[root - INFO - 2017-04-19 12:39:05,512] START RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 +[root - INFO - 2018-11-20 17:10:53,352] Event: {'answer': 42} +[root - INFO - 2018-11-20 17:10:53,352] START RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Version: 0 49 196 441 784 1225 -[root - INFO - 2017-04-19 12:39:05,515] END RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 -[root - INFO - 2017-04-19 12:39:05,515] RESULT: +[root - INFO - 2018-11-20 17:10:53,359] END RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 +[root - INFO - 2018-11-20 17:10:53,360] REPORT RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Duration: 2.17 ms +[root - INFO - 2018-11-20 17:10:53,360] RESULT: None -[root - INFO - 2017-04-19 12:39:05,515] REPORT RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 Duration: 2.27 ms +``` + +## Usage as a library + +### API signature + +``` python +call(func, event, context, environment_variables={}) +``` + +Call a handler function `func` with given `event`, `context` and custom `environment_variables`. + +### Sample + +1. Make sure the 3rd party libraries used in the AWS Lambda function can be imported. + +``` bash +pip install rx==1.6.1 +``` + +2. To call the lambda function above with your python code: + +``` python +from lambda_local.main import call +from lambda_local.context import Context + +import test + +event = { + "answer": 42 +} +context = Context(5) + +call(test.handler, event, context) ``` diff --git a/README.rst b/README.rst index deb21a0..84e9653 100644 --- a/README.rst +++ b/README.rst @@ -1,15 +1,14 @@ python-lambda-local =================== -|Join the chat at https://gitter.im/HDE/python-lambda-local| |wercker -status| |PyPI version| +|Join the chat at https://gitter.im/HDE/python-lambda-local| |Github Actions status| |PyPI version| Run lambda function on local machine Prepare development environment ------------------------------- -Please use a newly created virtualenv of Python 2.7 or Python 3.6 . +Please use a newly created virtualenv of Python 3.7+. Installation ------------ @@ -18,43 +17,45 @@ Within virtualenv, run the following command. .. code:: bash - $ pip install python-lambda-local + $ pip install python-lambda-local This will install the package with name ``python-lambda-local`` in the virtualenv. Now you can use the command ``python-lambda-local`` to run your AWS Lambda function written in Python on your own machine. -Usage ------ +Usage as a shell command +------------------------ Run ``python-lambda-local -h`` to see the help. :: - usage: python-lambda-local [-h] [-l LIBRARY_PATH] [-f HANDLER_FUNCTION] - [-t TIMEOUT] [-a ARN_STRING] [-v VERSION_NAME] - [--version] - FILE EVENT - - Run AWS Lambda function written in Python on local machine. - - positional arguments: - FILE lambda function file name - EVENT event data file name - - optional arguments: - -h, --help show this help message and exit - -l LIBRARY_PATH, --library LIBRARY_PATH - path of 3rd party libraries - -f HANDLER_FUNCTION, --function HANDLER_FUNCTION - lambda function handler name, default: "handler" - -t TIMEOUT, --timeout TIMEOUT - seconds until lambda function timeout, default: 3 - -a ARN_STRING, --arn-string ARN_STRING - ARN string for lambda function - -v VERSION_NAME, --version-name VERSION_NAME - lambda function version name - --version print the version of python-lambda-local and exit + usage: python-lambda-local [-h] [-l LIBRARY_PATH] [-f HANDLER_FUNCTION] + [-t TIMEOUT] [-a ARN_STRING] [-v VERSION_NAME] + [-e ENVIRONMENT_VARIABLES] [--version] + FILE EVENT + + Run AWS Lambda function written in Python on local machine. + + positional arguments: + FILE lambda function file name + EVENT event data file name + + optional arguments: + -h, --help show this help message and exit + -l LIBRARY_PATH, --library LIBRARY_PATH + path of 3rd party libraries + -f HANDLER_FUNCTION, --function HANDLER_FUNCTION + lambda function handler name, default: "handler" + -t TIMEOUT, --timeout TIMEOUT + seconds until lambda function timeout, default: 3 + -a ARN_STRING, --arn-string ARN_STRING + ARN string for lambda function + -v VERSION_NAME, --version-name VERSION_NAME + lambda function version name + -e ENVIRONMENT_VARIABLES, --environment-variables ENVIRONMENT_VARIABLES + path to flat json file with environment variables + --version print the version of python-lambda-local and exit Prepare development directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -66,25 +67,25 @@ Suppose your project directory is like this: :: - ├── event.json - ├── lib - │   ├── rx - │   │   ├── abstractobserver.py - │   │   ├── ... (package content of rx) - ... - │   │   └── testscheduler.py - │   └── Rx-1.2.3.dist-info - │   ├── DESCRIPTION.rst - │   ├── METADATA - │   ├── metadata.json - │   ├── pbr.json - │   ├── RECORD - │   ├── top_level.txt - │   ├── WHEEL - │   └── zip-safe - └── test.py - -The handler's code is in ``test.py`` and the function name of the + ├── event.json + ├── lib + │   ├── rx + │   │   ├── abstractobserver.py + │   │   ├── ... (package content of rx) + ... + │   │   └── testscheduler.py + │   └── Rx-1.6.1.dist-info + │   ├── DESCRIPTION.rst + │   ├── METADATA + │   ├── metadata.json + │   ├── pbr.json + │   ├── RECORD + │   ├── top_level.txt + │   ├── WHEEL + │   └── zip-safe + └── test.py + +The handler’s code is in ``test.py`` and the function name of the handler is ``handler``. The source depends on 3rd party library ``rx`` and it is installed in the directory ``lib``. The test event in json format is in ``event.json`` file. @@ -94,25 +95,25 @@ Content of ``test.py``: .. code:: python - from __future__ import print_function - from rx import Observable + from __future__ import print_function + from rx import Observable - def handler(event, context): - xs = Observable.from_(range(event['answer'])) - ys = xs.to_blocking() - zs = (x*x for x in ys if x % 7 == 0) - for x in zs: - print(x) + def handler(event, context): + xs = Observable.from_(range(event['answer'])) + ys = xs.to_blocking() + zs = (x*x for x in ys if x % 7 == 0) + for x in zs: + print(x) Content of ``event.json``: ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code:: json - { - "answer": 42 - } + { + "answer": 42 + } Run the lambda function ^^^^^^^^^^^^^^^^^^^^^^^ @@ -122,28 +123,67 @@ the following command :: - python-lambda-local -l lib/ -f handler -t 5 test.py event.json + python-lambda-local -l lib/ -f handler -t 5 test.py event.json The output will be like: :: - [root - INFO - 2017-04-19 12:39:05,512] Event: {u'answer': 42} - [root - INFO - 2017-04-19 12:39:05,512] START RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 - 0 - 49 - 196 - 441 - 784 - 1225 - [root - INFO - 2017-04-19 12:39:05,515] END RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 - [root - INFO - 2017-04-19 12:39:05,515] RESULT: - None - [root - INFO - 2017-04-19 12:39:05,515] REPORT RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 Duration: 2.27 ms + [root - INFO - 2018-11-20 17:10:53,352] Event: {'answer': 42} + [root - INFO - 2018-11-20 17:10:53,352] START RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Version: + 0 + 49 + 196 + 441 + 784 + 1225 + [root - INFO - 2018-11-20 17:10:53,359] END RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 + [root - INFO - 2018-11-20 17:10:53,360] REPORT RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Duration: 2.17 ms + [root - INFO - 2018-11-20 17:10:53,360] RESULT: + None + +Usage as a library +------------------ + +API signature +~~~~~~~~~~~~~ + +.. code:: python + + call(func, event, context, environment_variables={}) + +Call a handler function ``func`` with given ``event``, ``context`` and +custom ``environment_variables``. + +Sample +~~~~~~ + +1. Make sure the 3rd party libraries used in the AWS Lambda function can + be imported. + +.. code:: bash + + pip install rx==1.6.1 + +2. To call the lambda function above with your python code: + +.. code:: python + + from lambda_local.main import call + from lambda_local.context import Context + + import test + + event = { + "answer": 42 + } + context = Context(5) + + call(test.handler, event, context) .. |Join the chat at https://gitter.im/HDE/python-lambda-local| image:: https://badges.gitter.im/Join%20Chat.svg :target: https://gitter.im/HDE/python-lambda-local?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -.. |wercker status| image:: https://app.wercker.com/status/04f5bc5b7de3d5c6f13eb5b871035226/s - :target: https://app.wercker.com/project/bykey/04f5bc5b7de3d5c6f13eb5b871035226 +.. |Github Actions status| image:: https://github.com/HDE/python-lambda-local/actions/workflows/execute_tests.yml/badge.svg + :target: https://github.com/HDE/python-lambda-local/actions/ .. |PyPI version| image:: https://badge.fury.io/py/python-lambda-local.svg :target: https://badge.fury.io/py/python-lambda-local diff --git a/deploy.sh b/deploy.sh index 109fc59..2661370 100644 --- a/deploy.sh +++ b/deploy.sh @@ -11,4 +11,5 @@ username: ${PYPI_USERNAME} password: ${PYPI_PASSWORD} EOF -python setup.py sdist upload +pip install twine +twine upload -r pypi dist/* diff --git a/lambda_local/__init__.py b/lambda_local/__init__.py index f56a061..e5899ab 100644 --- a/lambda_local/__init__.py +++ b/lambda_local/__init__.py @@ -1,30 +1,22 @@ ''' python-lambda-local: Main module -Copyright 2015-2017 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' from __future__ import print_function import argparse -import sys -from multiprocessing import Process import pkg_resources from .main import run - __version__ = pkg_resources.require("python-lambda-local")[0].version def main(): args = parse_args() - - p = Process(target=run, args=(args,)) - p.start() - p.join() - - sys.exit(p.exitcode) + run(args) def parse_args(): diff --git a/lambda_local/__main__.py b/lambda_local/__main__.py new file mode 100644 index 0000000..868d99e --- /dev/null +++ b/lambda_local/__main__.py @@ -0,0 +1,4 @@ +from . import main + +if __name__ == "__main__": + main() diff --git a/lambda_local/context.py b/lambda_local/context.py index 4be412e..1c84b72 100644 --- a/lambda_local/context.py +++ b/lambda_local/context.py @@ -1,33 +1,48 @@ ''' -Copyright 2015-2017 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' +from __future__ import print_function from datetime import datetime from datetime import timedelta +import uuid class Context(object): - def __init__(self, timeout, arn_string, version_name): - self.function_name = "undefined" - self.function_version = version_name - self.invoked_function_arn = arn_string - self.memory_limit_in_mb = 0 - self.aws_request_id = "undefined" - self.log_group_name = "undefined" - self.log_stream_name = "undefined" - self.identity = None - self.client_context = None - self.timeout = timeout - self.duration = timedelta(seconds=timeout) + def __init__(self, timeout_in_seconds, + aws_request_id=uuid.uuid4(), + function_name="undefined", + function_version="$LATEST", + log_group_name="undefined", + log_stream_name="undefined", + invoked_function_arn="undefined", + memory_limit_in_mb='0', + client_context=None, + identity=None): + self.function_name = function_name + self.function_version = function_version + self.invoked_function_arn = invoked_function_arn + self.memory_limit_in_mb = memory_limit_in_mb + self.aws_request_id = aws_request_id + self.log_group_name = log_group_name + self.log_stream_name = log_stream_name + self.identity = identity + self.client_context = client_context + + self._timeout_in_seconds = timeout_in_seconds + self._duration = timedelta(seconds=timeout_in_seconds) def get_remaining_time_in_millis(self): - if self.timelimit is None: + if self._timelimit is None: raise Exception("Context not activated.") - return millis_interval(datetime.now(), self.timelimit) + return millis_interval(datetime.now(), self._timelimit) + + def log(self, msg): + print(msg) - def activate(self): - self.timelimit = datetime.now() + self.duration + def _activate(self): + self._timelimit = datetime.now() + self._duration return self diff --git a/lambda_local/environment_variables.py b/lambda_local/environment_variables.py index b518309..484dde7 100644 --- a/lambda_local/environment_variables.py +++ b/lambda_local/environment_variables.py @@ -1,7 +1,16 @@ +''' +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) +Licensed under MIT. +''' import json import os +def export_variables(environment_variables): + for env_name, env_value in environment_variables.items(): + os.environ[str(env_name)] = str(env_value) + + def set_environment_variables(json_file_path): """ Read and set environment variables from a flat json file. @@ -25,5 +34,4 @@ def set_environment_variables(json_file_path): with open(json_file_path) as json_file: env_vars = json.loads(json_file.read()) - for env_name, env_value in env_vars.items(): - os.environ[str(env_name)] = str(env_value) + export_variables(env_vars) diff --git a/lambda_local/event.py b/lambda_local/event.py index 3030ff1..a831caf 100644 --- a/lambda_local/event.py +++ b/lambda_local/event.py @@ -1,5 +1,5 @@ ''' -Copyright 2015-2017 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' diff --git a/lambda_local/main.py b/lambda_local/main.py index 95d4358..55b22c9 100644 --- a/lambda_local/main.py +++ b/lambda_local/main.py @@ -1,28 +1,25 @@ ''' -Copyright 2015-2017 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' -import imp import sys import traceback import json import logging -import uuid import os import timeit -from botocore.vendored.requests.packages import urllib3 +import multiprocessing from . import event from . import context -from .environment_variables import set_environment_variables +from .environment_variables import set_environment_variables, export_variables from .timeout import time_limit from .timeout import TimeoutException logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='[%(name)s - %(levelname)s - %(asctime)s] %(message)s') -urllib3.disable_warnings() ERR_TYPE_EXCEPTION = 0 @@ -31,55 +28,115 @@ EXITCODE_ERR = 1 +class ContextFilter(logging.Filter): + def __init__(self, context): + super(ContextFilter, self).__init__() + self.context = context + + def filter(self, record): + record.aws_request_id = self.context.aws_request_id + return True + + +class FunctionLoader(): + def __init__(self, + request_id=None, + source=None, + function_name=None, + library_path=None, + func=None): + self.request_id = request_id + self.source = source + self.function_name = function_name + self.library_path = library_path + + self.func = func + + def load(self): + if self.library_path is not None: + load_lib(self.library_path) + + self.func = load_source( + self.request_id, self.source, self.function_name) + + +def call(func, event, context, environment_variables={}): + export_variables(environment_variables) + loader = FunctionLoader(func=func) + return _runner(loader, event, context) + + def run(args): # set env vars if path to json file was given set_environment_variables(args.environment_variables) e = event.read_event(args.event) - c = context.Context(args.timeout, args.arn_string, args.version_name) - if args.library is not None: - load_lib(args.library) - request_id = uuid.uuid4() - func = load(request_id, args.file, args.function) + c = context.Context( + args.timeout, + invoked_function_arn=args.arn_string, + function_version=args.version_name) + loader = FunctionLoader( + request_id=c.aws_request_id, + source=args.file, + function_name=args.function, + library_path=args.library) + + (result, err_type) = _runner(loader, e, c) - logger = logging.getLogger() - result = None + if err_type is not None: + sys.exit(EXITCODE_ERR) - logger.info("Event: {}".format(e)) - logger.info("START RequestId: {}".format(request_id)) +def _runner(loader, event, context): + logger = logging.getLogger() - start_time = timeit.default_timer() - result, err_type = execute(func, e, c) - end_time = timeit.default_timer() + logger.info("Event: {}".format(event)) + logger.info("START RequestId: {} Version: {}".format( + context.aws_request_id, context.function_version)) - logger.info("END RequestId: {}".format(request_id)) + queue = multiprocessing.Queue() + p = multiprocessing.Process( + target=execute_in_process, + args=(queue, loader, event, context,)) + p.start() + (result, err_type, duration) = queue.get() + p.join() + logger.info("END RequestId: {}".format(context.aws_request_id)) + duration = "{0:.2f} ms".format(duration) + logger.info("REPORT RequestId: {}\tDuration: {}".format( + context.aws_request_id, duration)) if type(result) is TimeoutException: logger.error("RESULT:\n{}".format(result)) else: logger.info("RESULT:\n{}".format(result)) - duration = "{0:.2f} ms".format((end_time - start_time) * 1000) - logger.info("REPORT RequestId: {}\tDuration: {}".format( - request_id, duration)) - - if err_type is not None: - sys.exit(EXITCODE_ERR) + return (result, err_type) def load_lib(path): sys.path.append(os.path.abspath(path)) -def load(request_id, path, function_name): +def load_source(request_id, path, function_name): mod_name = 'request-' + str(request_id) file_path = os.path.abspath(path) file_directory = os.path.dirname(file_path) sys.path.append(file_directory) - mod = imp.load_source(mod_name, path) + if sys.version_info.major == 2: + import imp + mod = imp.load_source(mod_name, path) + elif sys.version_info.major == 3 and sys.version_info.minor >= 5: + import importlib + spec = importlib.util.spec_from_file_location(mod_name, path) + mod = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = mod + spec.loader.exec_module(mod) + else: + raise Exception("unsupported python version") + func = getattr(mod, function_name) return func @@ -87,9 +144,13 @@ def load(request_id, path, function_name): def execute(func, event, context): err_type = None + logger = logging.getLogger() + log_filter = ContextFilter(context) + logger.addFilter(log_filter) + try: - with time_limit(context.timeout): - result = func(event, context.activate()) + with time_limit(context._timeout_in_seconds): + result = func(event, context._activate()) except TimeoutException as err: result = err err_type = ERR_TYPE_TIMEOUT @@ -97,9 +158,20 @@ def execute(func, event, context): err = sys.exc_info() result = json.dumps({ "errorMessage": str(err[1]), - "stackTrace": traceback.extract_tb(err[2]), + "stackTrace": traceback.format_tb(err[2]), "errorType": err[0].__name__ }, indent=4, separators=(',', ': ')) err_type = ERR_TYPE_EXCEPTION return result, err_type + + +def execute_in_process(queue, loader, event, context): + if loader.func is None: + loader.load() + start_time = timeit.default_timer() + result, err_type = execute(loader.func, event, context) + end_time = timeit.default_timer() + duration = (end_time - start_time) * 1000 + + queue.put((result, err_type, duration)) diff --git a/lambda_local/timeout.py b/lambda_local/timeout.py index 17646c3..9492844 100644 --- a/lambda_local/timeout.py +++ b/lambda_local/timeout.py @@ -1,12 +1,12 @@ ''' -Copyright 2015-2017 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' import signal import threading from contextlib import contextmanager -from six.moves import _thread +import _thread class TimeoutException(Exception): diff --git a/setup.cfg b/setup.cfg index bbfda55..444a061 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,6 @@ tag_build = tag_svn_revision = false -[pytest] +[tool:pytest] addopts = --ignore=setup.py --ignore=build --ignore=dist --doctest-modules norecursedirs=*.egg diff --git a/setup.py b/setup.py index 595797c..7bcbf9c 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ ''' python-lambda-local: Run lambda function in python on local machine. -Copyright 2015-2017 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' +import io import sys from setuptools import setup, find_packages from setuptools.command.test import test as TestCommand @@ -22,29 +23,32 @@ def run_tests(self): sys.exit(pytest.main(self.test_args)) -version = "0.1.6" +version = "0.1.13" + +TEST_REQUIRE = ['pytest'] setup(name="python-lambda-local", version=version, description="Run lambda function in python on local machine.", - long_description=open("README.rst").read(), + long_description=io.open("README.rst", encoding="utf-8").read(), classifiers=[ 'Development Status :: 3 - Alpha', 'Operating System :: POSIX', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: MIT License' ], keywords="AWS Lambda", - author="YANG Xudong", - author_email="xudong.yang@hde.co.jp", + author="YANG Xudong, Iskandar Setiadi", + author_email="iskandar.setiadi@hennge.com", url="https://github.com/HDE/python-lambda-local", license="MIT", packages=find_packages(exclude=['examples', 'tests']), include_package_data=True, zip_safe=False, - tests_require=['pytest'], + tests_require=TEST_REQUIRE, cmdclass={'test': PyTest}, install_requires=['boto3'], entry_points={ diff --git a/tests/__init__.py b/tests/__init__.py index 98c7f83..9052941 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,6 @@ Organize tests into files, each named xxx_test.py Read more here: http://pytest.org/ -Copyright 2015 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT ''' diff --git a/tests/basic_test.py b/tests/basic_test.py index 427173d..0dddec8 100644 --- a/tests/basic_test.py +++ b/tests/basic_test.py @@ -5,7 +5,7 @@ Write each test as a function named test_. Read more here: http://pytest.org/ -Copyright 2015 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT ''' diff --git a/tests/test_direct_invocations.py b/tests/test_direct_invocations.py new file mode 100644 index 0000000..6b4857f --- /dev/null +++ b/tests/test_direct_invocations.py @@ -0,0 +1,88 @@ +''' +python-lambda-local: Test Direct Invocations +(command-line and direct). + +Meant for use with py.test. + +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) +Licensed under MIT +''' +import json +import argparse +from multiprocessing import Process +import os +from lambda_local.main import run as lambda_run +from lambda_local.main import call as lambda_call +from lambda_local.main import ERR_TYPE_EXCEPTION +from lambda_local.context import Context + + +def my_lambda_function(event, context): + print("Hello World from My Lambda Function!") + return 42 + + +def my_failing_lambda_function(event, context): + raise Exception('Oh no') + + +def test_function_call_for_pytest(): + (result, error_type) = lambda_call( + my_lambda_function, {}, Context(1)) + + assert error_type is None + + assert result == 42 + + +def test_handle_exceptions_gracefully(): + (result, error_type) = lambda_call( + my_failing_lambda_function, {}, Context(1)) + + assert error_type is ERR_TYPE_EXCEPTION + + +def test_check_command_line(): + request = json.dumps({}) + request_file = 'check_command_line_event.json' + with open(request_file, "w") as f: + f.write(request) + + args = argparse.Namespace(event=request_file, + file='tests/test_direct_invocations.py', + function='my_lambda_function', + timeout=1, + environment_variables='', + library=None, + version_name='', + arn_string='' + ) + p = Process(target=lambda_run, args=(args,)) + p.start() + p.join() + + os.remove(request_file) + assert p.exitcode == 0 + + +def test_check_command_line_error(): + request = json.dumps({}) + request_file = 'check_command_line_event.json' + with open(request_file, "w") as f: + f.write(request) + + args = argparse.Namespace(event=request_file, + file='tests/test_direct_invocations.py', + function='my_failing_lambda_function', + timeout=1, + environment_variables='', + library=None, + version_name='', + arn_string='' + ) + p = Process(target=lambda_run, args=(args,)) + p.start() + p.join() + + os.remove(request_file) + assert p.exitcode == 1 diff --git a/tests/test_environment_variables.py b/tests/test_environment_variables.py index 6582779..7221f89 100644 --- a/tests/test_environment_variables.py +++ b/tests/test_environment_variables.py @@ -1,3 +1,7 @@ +''' +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) +Licensed under MIT. +''' import os from lambda_local.environment_variables import set_environment_variables diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index 25613d5..0000000 --- a/wercker.yml +++ /dev/null @@ -1,57 +0,0 @@ -box: python:2.7-slim - -build: - steps: - -build-py2: - box: python:2.7-slim - steps: - - script: - name: virtualenv install - code: | - pip install virtualenv - - - virtualenv: - name: setup virtual environment - install_wheel: true # Enable wheel to speed up builds (experimental) - - - script: - name: echo python information - code: | - echo "python version $(python --version) running" - echo "pip version $(pip --version) running" - - - script: - name: build - code: | - python setup.py sdist bdist_wheel - -build-py3: - box: python:3.6-slim - steps: - - script: - name: virtualenv install - code: | - pip install virtualenv - - - virtualenv: - name: setup virtual environment - install_wheel: true # Enable wheel to speed up builds (experimental) - - - script: - name: echo python information - code: | - echo "python version $(python --version) running" - echo "pip version $(pip --version) running" - - - script: - name: build - code: | - python setup.py sdist bdist_wheel - -deploy: - pypi: - - script: - name: deploy to pypi - code: | - sh deploy.sh