diff --git a/.github/workflows/execute_tests.yml b/.github/workflows/execute_tests.yml new file mode 100644 index 0000000..57a6fdf --- /dev/null +++ b/.github/workflows/execute_tests.yml @@ -0,0 +1,32 @@ +name: Execute tests + +on: push + +jobs: + execute_tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9'] + name: Python ${{ matrix.python-version }} tests + steps: + - uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Print information + run: | + echo "python version $(python --version) running" + echo "pip version $(pip --version) running" + + - name: Build + run: | + pip install wheel + python setup.py sdist bdist_wheel + + - name: Test + run: | + python setup.py test diff --git a/.github/workflows/push_to_pypi.yml b/.github/workflows/push_to_pypi.yml new file mode 100644 index 0000000..a5b1afd --- /dev/null +++ b/.github/workflows/push_to_pypi.yml @@ -0,0 +1,40 @@ +name: Push package to pypi + +on: + pull_request: + types: + - closed + branches: + - release + +env: + PYTHON_VERSION: "3.9" + +jobs: + push_to_pypi: + if: github.event.pull_request.merged + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Print information + run: | + echo "python version $(python --version) running" + echo "pip version $(pip --version) running" + + - name: Build + run: | + pip install wheel + python setup.py sdist bdist_wheel + + - name: Deploy + env: + PYPI_USERNAME: "${{ secrets.PYPI_USERNAME }}" + PYPI_PASSWORD: "${{ secrets.PYPI_PASSWORD }}" + run: | + sh deploy.sh diff --git a/.gitignore b/.gitignore index 4f5a7f6..6a5671f 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,7 @@ target/ # Vitual Environments venv/ +.env/ + +# direnv +.envrc diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 5776446..d732b63 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -22,3 +22,51 @@ Version 0.1.4 ------------- - Support Python 3.6. + +Version 0.1.5 +------------- + + - Support reading environment variables from file. + +Version 0.1.6 +------------- + + - Add timeout feature for Windows. + +Version 0.1.7 +------------- + + - Add tests in CI. + +Version 0.1.8 +------------- + + - Support running as a library. + +Version 0.1.9 +------------- + + - Support latest boto3. + +Version 0.1.10 +-------------- + + - Fix traceback output when exception happens. + +Version 0.1.11 +-------------- + + - Test on python 3.8. + +Version 0.1.12 +-------------- + + - Fix error when running on Windows with Python 3. + +Version 0.1.13 +-------------- + + - Drop support of Python 2.7. + - Various update and cleanup, add Python 3.9 support. + - Add __main__ file for using the package as a module. + - Implement tests via Github Actions. diff --git a/LICENSE b/LICENSE index c4d8dfd..9416dcb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2018 HDE, Inc. +Copyright (c) 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 26c2b2c..b88a7a9 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # python-lambda-local [![Join the chat at https://gitter.im/HDE/python-lambda-local](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/HDE/python-lambda-local?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![wercker status](https://app.wercker.com/status/04f5bc5b7de3d5c6f13eb5b871035226/s "wercker status")](https://app.wercker.com/project/bykey/04f5bc5b7de3d5c6f13eb5b871035226) +[![Github Actions status](https://github.com/HDE/python-lambda-local/actions/workflows/execute_tests.yml/badge.svg)](https://github.com/HDE/python-lambda-local/actions/) [![PyPI version](https://badge.fury.io/py/python-lambda-local.svg)](https://badge.fury.io/py/python-lambda-local) Run lambda function on local machine ## Prepare development environment -Please use a newly created virtualenv of Python 2.7 or Python 3.6. +Please use a newly created virtualenv of Python 3.7+. ## Installation @@ -68,7 +68,7 @@ Suppose your project directory is like this: │   │   ├── ... (package content of rx) ... │   │   └── testscheduler.py -│   └── Rx-1.2.3.dist-info +│   └── Rx-1.6.1.dist-info │   ├── DESCRIPTION.rst │   ├── METADATA │   ├── metadata.json @@ -84,6 +84,12 @@ The handler's code is in `test.py` and the function name of the handler is `hand The source depends on 3rd party library `rx` and it is installed in the directory `lib`. The test event in json format is in `event.json` file. +#### Installing `rx` library in `lib/` + +``` bash +pip install --target lib rx==1.6.1 +``` + #### Content of `test.py`: ``` python @@ -147,7 +153,7 @@ Call a handler function `func` with given `event`, `context` and custom `environ 1. Make sure the 3rd party libraries used in the AWS Lambda function can be imported. ``` bash -pip install rx +pip install rx==1.6.1 ``` 2. To call the lambda function above with your python code: @@ -163,5 +169,6 @@ event = { } context = Context(5) -call(test.handler, event, context) +if __name__ == '__main__': + call(test.handler, event, context) ``` diff --git a/README.rst b/README.rst index b77f67e..ca11041 100644 --- a/README.rst +++ b/README.rst @@ -1,15 +1,14 @@ python-lambda-local =================== -|Join the chat at https://gitter.im/HDE/python-lambda-local| |wercker -status| |PyPI version| +|Join the chat at https://gitter.im/HDE/python-lambda-local| |Github Actions status| |PyPI version| Run lambda function on local machine Prepare development environment ------------------------------- -Please use a newly created virtualenv of Python 2.7 or Python 3.6. +Please use a newly created virtualenv of Python 3.7+. Installation ------------ @@ -75,7 +74,7 @@ Suppose your project directory is like this: │   │   ├── ... (package content of rx) ... │   │   └── testscheduler.py - │   └── Rx-1.2.3.dist-info + │   └── Rx-1.6.1.dist-info │   ├── DESCRIPTION.rst │   ├── METADATA │   ├── metadata.json @@ -91,6 +90,13 @@ handler is ``handler``. The source depends on 3rd party library ``rx`` and it is installed in the directory ``lib``. The test event in json format is in ``event.json`` file. +Installing ``rx`` library in ``lib/``: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: bash + + pip install --target lib rx==1.6.1 + Content of ``test.py``: ^^^^^^^^^^^^^^^^^^^^^^^ @@ -164,7 +170,7 @@ Sample .. code:: bash - pip install rx + pip install rx==1.6.1 2. To call the lambda function above with your python code: @@ -180,11 +186,12 @@ Sample } context = Context(5) - call(test.handler, event, context) + if __name__ == '__main__': + call(test.handler, event, context) .. |Join the chat at https://gitter.im/HDE/python-lambda-local| image:: https://badges.gitter.im/Join%20Chat.svg :target: https://gitter.im/HDE/python-lambda-local?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -.. |wercker status| image:: https://app.wercker.com/status/04f5bc5b7de3d5c6f13eb5b871035226/s - :target: https://app.wercker.com/project/bykey/04f5bc5b7de3d5c6f13eb5b871035226 +.. |Github Actions status| image:: https://github.com/HDE/python-lambda-local/actions/workflows/execute_tests.yml/badge.svg + :target: https://github.com/HDE/python-lambda-local/actions/ .. |PyPI version| image:: https://badge.fury.io/py/python-lambda-local.svg :target: https://badge.fury.io/py/python-lambda-local diff --git a/deploy.sh b/deploy.sh index 4fc9305..2661370 100644 --- a/deploy.sh +++ b/deploy.sh @@ -13,4 +13,3 @@ EOF pip install twine twine upload -r pypi dist/* - diff --git a/lambda_local/__init__.py b/lambda_local/__init__.py index 45b457b..d024373 100644 --- a/lambda_local/__init__.py +++ b/lambda_local/__init__.py @@ -1,17 +1,24 @@ ''' python-lambda-local: Main module -Copyright 2015-2018 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' from __future__ import print_function import argparse -import pkg_resources -from .main import run +# Get the version of python-lambda-local +try: + from importlib.metadata import version as get_version + __version__ = get_version("python-lambda-local") + +# If importlib.metadata is not available, use pkg_resources (older versions of Python) +except ImportError: + from pkg_resources import require + __version__ = require("python-lambda-local")[0].version -__version__ = pkg_resources.require("python-lambda-local")[0].version +from .main import run def main(): diff --git a/lambda_local/__main__.py b/lambda_local/__main__.py new file mode 100644 index 0000000..868d99e --- /dev/null +++ b/lambda_local/__main__.py @@ -0,0 +1,4 @@ +from . import main + +if __name__ == "__main__": + main() diff --git a/lambda_local/context.py b/lambda_local/context.py index 1f21657..1c84b72 100644 --- a/lambda_local/context.py +++ b/lambda_local/context.py @@ -1,5 +1,5 @@ ''' -Copyright 2015-2018 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' from __future__ import print_function diff --git a/lambda_local/environment_variables.py b/lambda_local/environment_variables.py index ea102bd..484dde7 100644 --- a/lambda_local/environment_variables.py +++ b/lambda_local/environment_variables.py @@ -1,3 +1,7 @@ +''' +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) +Licensed under MIT. +''' import json import os diff --git a/lambda_local/event.py b/lambda_local/event.py index 3d0f23f..a831caf 100644 --- a/lambda_local/event.py +++ b/lambda_local/event.py @@ -1,5 +1,5 @@ ''' -Copyright 2015-2018 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' diff --git a/lambda_local/main.py b/lambda_local/main.py index 315962e..55b22c9 100644 --- a/lambda_local/main.py +++ b/lambda_local/main.py @@ -1,16 +1,14 @@ ''' -Copyright 2015-2018 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' -import imp import sys import traceback import json import logging import os import timeit -from botocore.vendored.requests.packages import urllib3 import multiprocessing from . import event @@ -22,7 +20,6 @@ logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='[%(name)s - %(levelname)s - %(asctime)s] %(message)s') -urllib3.disable_warnings() ERR_TYPE_EXCEPTION = 0 @@ -41,10 +38,32 @@ def filter(self, record): return True +class FunctionLoader(): + def __init__(self, + request_id=None, + source=None, + function_name=None, + library_path=None, + func=None): + self.request_id = request_id + self.source = source + self.function_name = function_name + self.library_path = library_path + + self.func = func + + def load(self): + if self.library_path is not None: + load_lib(self.library_path) + + self.func = load_source( + self.request_id, self.source, self.function_name) + + def call(func, event, context, environment_variables={}): export_variables(environment_variables) - - return _runner(func, event, context) + loader = FunctionLoader(func=func) + return _runner(loader, event, context) def run(args): @@ -56,17 +75,19 @@ def run(args): args.timeout, invoked_function_arn=args.arn_string, function_version=args.version_name) - if args.library is not None: - load_lib(args.library) - func = load(c.aws_request_id, args.file, args.function) + loader = FunctionLoader( + request_id=c.aws_request_id, + source=args.file, + function_name=args.function, + library_path=args.library) - (result, err_type) = _runner(func, e, c) + (result, err_type) = _runner(loader, e, c) if err_type is not None: sys.exit(EXITCODE_ERR) -def _runner(func, event, context): +def _runner(loader, event, context): logger = logging.getLogger() logger.info("Event: {}".format(event)) @@ -76,7 +97,7 @@ def _runner(func, event, context): queue = multiprocessing.Queue() p = multiprocessing.Process( target=execute_in_process, - args=(queue, func, event, context,)) + args=(queue, loader, event, context,)) p.start() (result, err_type, duration) = queue.get() p.join() @@ -97,14 +118,25 @@ def load_lib(path): sys.path.append(os.path.abspath(path)) -def load(request_id, path, function_name): +def load_source(request_id, path, function_name): mod_name = 'request-' + str(request_id) file_path = os.path.abspath(path) file_directory = os.path.dirname(file_path) sys.path.append(file_directory) - mod = imp.load_source(mod_name, path) + if sys.version_info.major == 2: + import imp + mod = imp.load_source(mod_name, path) + elif sys.version_info.major == 3 and sys.version_info.minor >= 5: + import importlib + spec = importlib.util.spec_from_file_location(mod_name, path) + mod = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = mod + spec.loader.exec_module(mod) + else: + raise Exception("unsupported python version") + func = getattr(mod, function_name) return func @@ -126,7 +158,7 @@ def execute(func, event, context): err = sys.exc_info() result = json.dumps({ "errorMessage": str(err[1]), - "stackTrace": traceback.extract_tb(err[2]), + "stackTrace": traceback.format_tb(err[2]), "errorType": err[0].__name__ }, indent=4, separators=(',', ': ')) err_type = ERR_TYPE_EXCEPTION @@ -134,9 +166,11 @@ def execute(func, event, context): return result, err_type -def execute_in_process(queue, func, event, context): +def execute_in_process(queue, loader, event, context): + if loader.func is None: + loader.load() start_time = timeit.default_timer() - result, err_type = execute(func, event, context) + result, err_type = execute(loader.func, event, context) end_time = timeit.default_timer() duration = (end_time - start_time) * 1000 diff --git a/lambda_local/timeout.py b/lambda_local/timeout.py index 9d0354a..9492844 100644 --- a/lambda_local/timeout.py +++ b/lambda_local/timeout.py @@ -1,12 +1,12 @@ ''' -Copyright 2015-2018 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' import signal import threading from contextlib import contextmanager -from six.moves import _thread +import _thread class TimeoutException(Exception): diff --git a/setup.py b/setup.py index b569510..7bcbf9c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ ''' python-lambda-local: Run lambda function in python on local machine. -Copyright 2015-2018 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT. ''' import io @@ -23,7 +23,9 @@ def run_tests(self): sys.exit(pytest.main(self.test_args)) -version = "0.1.8" +version = "0.1.13" + +TEST_REQUIRE = ['pytest'] setup(name="python-lambda-local", version=version, @@ -33,19 +35,20 @@ def run_tests(self): 'Development Status :: 3 - Alpha', 'Operating System :: POSIX', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: MIT License' ], keywords="AWS Lambda", - author="YANG Xudong", - author_email="xudong.yang@hde.co.jp", + author="YANG Xudong, Iskandar Setiadi", + author_email="iskandar.setiadi@hennge.com", url="https://github.com/HDE/python-lambda-local", license="MIT", packages=find_packages(exclude=['examples', 'tests']), include_package_data=True, zip_safe=False, - tests_require=['pytest'], + tests_require=TEST_REQUIRE, cmdclass={'test': PyTest}, install_requires=['boto3'], entry_points={ diff --git a/tests/__init__.py b/tests/__init__.py index 98c7f83..9052941 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,6 @@ Organize tests into files, each named xxx_test.py Read more here: http://pytest.org/ -Copyright 2015 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT ''' diff --git a/tests/basic_test.py b/tests/basic_test.py index 427173d..0dddec8 100644 --- a/tests/basic_test.py +++ b/tests/basic_test.py @@ -5,7 +5,7 @@ Write each test as a function named test_. Read more here: http://pytest.org/ -Copyright 2015 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT ''' diff --git a/tests/test_direct_invocations.py b/tests/test_direct_invocations.py index 129666c..6b4857f 100644 --- a/tests/test_direct_invocations.py +++ b/tests/test_direct_invocations.py @@ -1,10 +1,10 @@ ''' -python-lambda-local: Test Direct Inovactions +python-lambda-local: Test Direct Invocations (command-line and direct). Meant for use with py.test. -Copyright 2015 HDE, Inc. +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) Licensed under MIT ''' import json @@ -13,6 +13,7 @@ import os from lambda_local.main import run as lambda_run from lambda_local.main import call as lambda_call +from lambda_local.main import ERR_TYPE_EXCEPTION from lambda_local.context import Context @@ -21,6 +22,10 @@ def my_lambda_function(event, context): return 42 +def my_failing_lambda_function(event, context): + raise Exception('Oh no') + + def test_function_call_for_pytest(): (result, error_type) = lambda_call( my_lambda_function, {}, Context(1)) @@ -30,6 +35,13 @@ def test_function_call_for_pytest(): assert result == 42 +def test_handle_exceptions_gracefully(): + (result, error_type) = lambda_call( + my_failing_lambda_function, {}, Context(1)) + + assert error_type is ERR_TYPE_EXCEPTION + + def test_check_command_line(): request = json.dumps({}) request_file = 'check_command_line_event.json' @@ -51,3 +63,26 @@ def test_check_command_line(): os.remove(request_file) assert p.exitcode == 0 + + +def test_check_command_line_error(): + request = json.dumps({}) + request_file = 'check_command_line_event.json' + with open(request_file, "w") as f: + f.write(request) + + args = argparse.Namespace(event=request_file, + file='tests/test_direct_invocations.py', + function='my_failing_lambda_function', + timeout=1, + environment_variables='', + library=None, + version_name='', + arn_string='' + ) + p = Process(target=lambda_run, args=(args,)) + p.start() + p.join() + + os.remove(request_file) + assert p.exitcode == 1 diff --git a/tests/test_environment_variables.py b/tests/test_environment_variables.py index 6582779..7221f89 100644 --- a/tests/test_environment_variables.py +++ b/tests/test_environment_variables.py @@ -1,3 +1,7 @@ +''' +Copyright 2015-2022 HENNGE K.K. (formerly known as HDE, Inc.) +Licensed under MIT. +''' import os from lambda_local.environment_variables import set_environment_variables diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index 7d3c6bf..0000000 --- a/wercker.yml +++ /dev/null @@ -1,67 +0,0 @@ -box: python:2.7-slim - -build: - steps: - -build-py2: - box: python:2.7-slim - steps: - - script: - name: virtualenv install - code: | - pip install virtualenv - - - virtualenv: - name: setup virtual environment - install_wheel: true # Enable wheel to speed up builds (experimental) - - - script: - name: echo python information - code: | - echo "python version $(python --version) running" - echo "pip version $(pip --version) running" - - - script: - name: build - code: | - python setup.py sdist bdist_wheel - - - script: - name: test - code: | - python setup.py test - -build-py3: - box: python:3.6-slim - steps: - - script: - name: virtualenv install - code: | - pip install virtualenv - - - virtualenv: - name: setup virtual environment - install_wheel: true # Enable wheel to speed up builds (experimental) - - - script: - name: echo python information - code: | - echo "python version $(python --version) running" - echo "pip version $(pip --version) running" - - - script: - name: build - code: | - python setup.py sdist bdist_wheel - - - script: - name: test - code: | - python setup.py test - -deploy: - pypi: - - script: - name: deploy to pypi - code: | - sh deploy.sh