From ea2604ed29483ad04dbcf3a04d58961e88c6b3fb Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Fri, 17 Nov 2023 12:24:22 -0500 Subject: [PATCH 01/12] Update the description of myself (#802) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6032725a..37a9d1df 100644 --- a/README.md +++ b/README.md @@ -567,7 +567,7 @@ package: ## Contributors -- [@dschep](https://github.com/dschep) - Lead developer & original maintainer +- [@dschep](https://github.com/dschep) - Original developer - [@azurelogic](https://github.com/azurelogic) - logging & documentation fixes - [@abetomo](https://github.com/abetomo) - style & linting - [@angstwad](https://github.com/angstwad) - `deploy --function` support From 421e9a6e9a168b741dbd0ce9b6c1d39f7d8f55b8 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 24 Nov 2023 23:48:09 +0100 Subject: [PATCH 02/12] ci: Pin versions to speed up poetry deps resolution (#806) --- tests/poetry/pyproject.toml | 8 ++++---- tests/poetry_individually/module1/pyproject.toml | 8 ++++---- tests/poetry_packages/pyproject.toml | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/poetry/pyproject.toml b/tests/poetry/pyproject.toml index b813968a..896b48e7 100644 --- a/tests/poetry/pyproject.toml +++ b/tests/poetry/pyproject.toml @@ -5,13 +5,13 @@ description = "" authors = ["Your Name "] [tool.poetry.dependencies] -python = "^3.6" -Flask = "^1.0" +python = "^3.7" +Flask = "2.0" bottle = {git = "https://git@github.com/bottlepy/bottle.git", tag = "0.12.16"} -boto3 = "^1.9" +boto3 = "1.29.6" [tool.poetry.dev-dependencies] [build-system] -requires = ["poetry>=0.12"] +requires = ["poetry"] build-backend = "poetry.masonry.api" diff --git a/tests/poetry_individually/module1/pyproject.toml b/tests/poetry_individually/module1/pyproject.toml index b813968a..896b48e7 100644 --- a/tests/poetry_individually/module1/pyproject.toml +++ b/tests/poetry_individually/module1/pyproject.toml @@ -5,13 +5,13 @@ description = "" authors = ["Your Name "] [tool.poetry.dependencies] -python = "^3.6" -Flask = "^1.0" +python = "^3.7" +Flask = "2.0" bottle = {git = "https://git@github.com/bottlepy/bottle.git", tag = "0.12.16"} -boto3 = "^1.9" +boto3 = "1.29.6" [tool.poetry.dev-dependencies] [build-system] -requires = ["poetry>=0.12"] +requires = ["poetry"] build-backend = "poetry.masonry.api" diff --git a/tests/poetry_packages/pyproject.toml b/tests/poetry_packages/pyproject.toml index 7bbe30bf..0f9fc705 100644 --- a/tests/poetry_packages/pyproject.toml +++ b/tests/poetry_packages/pyproject.toml @@ -5,14 +5,14 @@ description = "" authors = ["Your Name "] [tool.poetry.dependencies] -python = "^3.6" -Flask = "^1.0" +python = "^3.7" +Flask = "2.0" [tool.poetry.group.custom1.dependencies] bottle = {git = "https://git@github.com/bottlepy/bottle.git", tag = "0.12.16"} [tool.poetry.group.custom2.dependencies] -boto3 = "^1.9" +boto3 = "1.29.6" [build-system] requires = ["poetry-core"] From e33b02da750acfc40b3d341c35edc71a7bea08aa Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Sat, 25 Nov 2023 00:11:54 +0100 Subject: [PATCH 03/12] [ci] Update to Node18 (#803) --- .github/workflows/publish.yml | 4 ++-- .github/workflows/validate.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6eee5b45..21d7cb71 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,12 +26,12 @@ jobs: path: | ~/.npm node_modules - key: npm-v14-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} + key: npm-v18-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} - name: Install Node.js and npm uses: actions/setup-node@v1 with: - node-version: 14.x + node-version: 18.x registry-url: https://registry.npmjs.org - name: Publish new version diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8957e7f7..03eea961 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -10,8 +10,8 @@ env: FORCE_COLOR: 1 jobs: - linuxNode14: - name: '[Linux] Node.js v14: Lint, Eventual Commitlint, Eventual Changelog, Formatting & Unit tests' + linuxNode18: + name: '[Linux] Node.js v18: Lint, Eventual Commitlint, Eventual Changelog, Formatting & Unit tests' runs-on: ubuntu-latest strategy: matrix: @@ -38,10 +38,10 @@ jobs: path: | ~/.npm node_modules - key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + key: npm-v18-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} restore-keys: | - npm-v14-${{ runner.os }}-${{ github.ref }}- - npm-v14-${{ runner.os }}-refs/heads/master- + npm-v18-${{ runner.os }}-${{ github.ref }}- + npm-v18-${{ runner.os }}-refs/heads/master- - name: Set up Python 3.7 uses: actions/setup-python@v2 @@ -51,7 +51,7 @@ jobs: - name: Install Node.js and npm uses: actions/setup-node@v1 with: - node-version: 14.x + node-version: 18.x - name: Check python version run: | From 4bbb80ed0c0150e04696513f37537eb3ab0002a4 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Sat, 25 Nov 2023 21:51:04 +0100 Subject: [PATCH 04/12] test: Update to py3.9 (#808) --- .github/workflows/validate.yml | 4 +- example/serverless.yml | 2 +- example_native_deps/serverless.yml | 2 +- test.js | 117 +++++++++++----------- tests/base/serverless.yml | 2 +- tests/individually/serverless.yml | 2 +- tests/non_build_pyproject/serverless.yml | 2 +- tests/non_poetry_pyproject/serverless.yml | 2 +- tests/pipenv/serverless.yml | 2 +- tests/poetry/serverless.yml | 2 +- tests/poetry_individually/serverless.yml | 2 +- tests/poetry_packages/serverless.yml | 2 +- 12 files changed, 72 insertions(+), 69 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 03eea961..61935c3e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -43,10 +43,10 @@ jobs: npm-v18-${{ runner.os }}-${{ github.ref }}- npm-v18-${{ runner.os }}-refs/heads/master- - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.9 - name: Install Node.js and npm uses: actions/setup-node@v1 diff --git a/example/serverless.yml b/example/serverless.yml index 349cdcb8..e5c4c924 100644 --- a/example/serverless.yml +++ b/example/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements diff --git a/example_native_deps/serverless.yml b/example_native_deps/serverless.yml index 4deed44a..cfbd4913 100644 --- a/example_native_deps/serverless.yml +++ b/example_native_deps/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements diff --git a/test.js b/test.js index a4bb992b..f4afca45 100644 --- a/test.js +++ b/test.js @@ -234,7 +234,7 @@ test('default pythonBin can package flask with default options', async (t) => { t.end(); }); -test('py3.7 packages have the same hash', async (t) => { +test('py3.9 packages have the same hash', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -249,7 +249,7 @@ test('py3.7 packages have the same hash', async (t) => { t.end(); }); -test('py3.7 can package flask with default options', async (t) => { +test('py3.9 can package flask with default options', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -261,7 +261,7 @@ test('py3.7 can package flask with default options', async (t) => { }); test( - 'py3.7 can package flask with hashes', + 'py3.9 can package flask with hashes', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -279,7 +279,7 @@ test( { skip: brokenOn('win32') } ); -test('py3.7 can package flask with nested', async (t) => { +test('py3.9 can package flask with nested', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -295,7 +295,7 @@ test('py3.7 can package flask with nested', async (t) => { t.end(); }); -test('py3.7 can package flask with zip option', async (t) => { +test('py3.9 can package flask with zip option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -313,7 +313,7 @@ test('py3.7 can package flask with zip option', async (t) => { t.end(); }); -test('py3.7 can package flask with slim option', async (t) => { +test('py3.9 can package flask with slim option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -332,7 +332,7 @@ test('py3.7 can package flask with slim option', async (t) => { t.end(); }); -test('py3.7 can package flask with slim & slimPatterns options', async (t) => { +test('py3.9 can package flask with slim & slimPatterns options', async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -353,7 +353,7 @@ test('py3.7 can package flask with slim & slimPatterns options', async (t) => { t.end(); }); -test("py3.7 doesn't package bottle with noDeploy option", async (t) => { +test("py3.9 doesn't package bottle with noDeploy option", async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -371,7 +371,7 @@ test("py3.7 doesn't package bottle with noDeploy option", async (t) => { t.end(); }); -test('py3.7 can package boto3 with editable', async (t) => { +test('py3.9 can package boto3 with editable', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -391,7 +391,7 @@ test('py3.7 can package boto3 with editable', async (t) => { }); test( - 'py3.7 can package flask with dockerizePip option', + 'py3.9 can package flask with dockerizePip option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -406,7 +406,7 @@ test( ); test( - 'py3.7 can package flask with slim & dockerizePip option', + 'py3.9 can package flask with slim & dockerizePip option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -430,7 +430,7 @@ test( ); test( - 'py3.7 can package flask with slim & dockerizePip & slimPatterns options', + 'py3.9 can package flask with slim & dockerizePip & slimPatterns options', async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -455,7 +455,7 @@ test( ); test( - 'py3.7 can package flask with zip & dockerizePip option', + 'py3.9 can package flask with zip & dockerizePip option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -487,7 +487,7 @@ test( ); test( - 'py3.7 can package flask with zip & slim & dockerizePip option', + 'py3.9 can package flask with zip & slim & dockerizePip option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -520,7 +520,7 @@ test( { skip: !canUseDocker() || brokenOn('win32') } ); -test('pipenv py3.7 can package flask with default options', async (t) => { +test('pipenv py3.9 can package flask with default options', async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -535,7 +535,7 @@ test('pipenv py3.7 can package flask with default options', async (t) => { t.end(); }); -test('pipenv py3.7 can package flask with slim option', async (t) => { +test('pipenv py3.9 can package flask with slim option', async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -554,7 +554,7 @@ test('pipenv py3.7 can package flask with slim option', async (t) => { t.end(); }); -test('pipenv py3.7 can package flask with slim & slimPatterns options', async (t) => { +test('pipenv py3.9 can package flask with slim & slimPatterns options', async (t) => { process.chdir('tests/pipenv'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -576,7 +576,7 @@ test('pipenv py3.7 can package flask with slim & slimPatterns options', async (t t.end(); }); -test('pipenv py3.7 can package flask with zip option', async (t) => { +test('pipenv py3.9 can package flask with zip option', async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -594,7 +594,7 @@ test('pipenv py3.7 can package flask with zip option', async (t) => { t.end(); }); -test("pipenv py3.7 doesn't package bottle with noDeploy option", async (t) => { +test("pipenv py3.9 doesn't package bottle with noDeploy option", async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -633,7 +633,7 @@ test('non poetry pyproject.toml without requirements.txt packages handler only', t.end(); }); -test('poetry py3.7 can package flask with default options', async (t) => { +test('poetry py3.9 can package flask with default options', async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -645,7 +645,7 @@ test('poetry py3.7 can package flask with default options', async (t) => { t.end(); }); -test('poetry py3.7 can package flask with slim option', async (t) => { +test('poetry py3.9 can package flask with slim option', async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -664,7 +664,7 @@ test('poetry py3.7 can package flask with slim option', async (t) => { t.end(); }); -test('poetry py3.7 can package flask with slim & slimPatterns options', async (t) => { +test('poetry py3.9 can package flask with slim & slimPatterns options', async (t) => { process.chdir('tests/poetry'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -686,7 +686,7 @@ test('poetry py3.7 can package flask with slim & slimPatterns options', async (t t.end(); }); -test('poetry py3.7 can package flask with zip option', async (t) => { +test('poetry py3.9 can package flask with zip option', async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -704,7 +704,7 @@ test('poetry py3.7 can package flask with zip option', async (t) => { t.end(); }); -test("poetry py3.7 doesn't package bottle with noDeploy option", async (t) => { +test("poetry py3.9 doesn't package bottle with noDeploy option", async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -722,7 +722,7 @@ test("poetry py3.7 doesn't package bottle with noDeploy option", async (t) => { t.end(); }); -test('py3.7 can package flask with zip option and no explicit include', async (t) => { +test('py3.9 can package flask with zip option and no explicit include', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -742,7 +742,7 @@ test('py3.7 can package flask with zip option and no explicit include', async (t t.end(); }); -test('py3.7 can package lambda-decorators using vendor option', async (t) => { +test('py3.9 can package lambda-decorators using vendor option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -805,7 +805,7 @@ test( { skip: process.platform === 'win32' } ); -test('py3.7 can package flask in a project with a space in it', async (t) => { +test('py3.9 can package flask in a project with a space in it', async (t) => { copySync('tests/base', 'tests/base with a space'); process.chdir('tests/base with a space'); const path = npm(['pack', '../..']); @@ -818,7 +818,7 @@ test('py3.7 can package flask in a project with a space in it', async (t) => { }); test( - 'py3.7 can package flask in a project with a space in it with docker', + 'py3.9 can package flask in a project with a space in it with docker', async (t) => { copySync('tests/base', 'tests/base with a space'); process.chdir('tests/base with a space'); @@ -833,7 +833,7 @@ test( { skip: !canUseDocker() || brokenOn('win32') } ); -test('py3.7 supports custom file name with fileName option', async (t) => { +test('py3.9 supports custom file name with fileName option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); writeFileSync('puck', 'requests'); @@ -849,7 +849,7 @@ test('py3.7 supports custom file name with fileName option', async (t) => { t.end(); }); -test("py3.7 doesn't package bottle with zip option", async (t) => { +test("py3.9 doesn't package bottle with zip option", async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -885,7 +885,7 @@ test("py3.7 doesn't package bottle with zip option", async (t) => { t.end(); }); -test('py3.7 can package flask with slim, slimPatterns & slimPatternsAppendDefaults=false options', async (t) => { +test('py3.9 can package flask with slim, slimPatterns & slimPatternsAppendDefaults=false options', async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -908,7 +908,7 @@ test('py3.7 can package flask with slim, slimPatterns & slimPatternsAppendDefaul }); test( - 'py3.7 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', + 'py3.9 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -937,7 +937,7 @@ test( { skip: !canUseDocker() || brokenOn('win32') } ); -test('pipenv py3.7 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', async (t) => { +test('pipenv py3.9 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', async (t) => { process.chdir('tests/pipenv'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -960,7 +960,7 @@ test('pipenv py3.7 can package flask with slim & slimPatterns & slimPatternsAppe t.end(); }); -test('poetry py3.7 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', async (t) => { +test('poetry py3.9 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', async (t) => { process.chdir('tests/poetry'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -983,7 +983,7 @@ test('poetry py3.7 can package flask with slim & slimPatterns & slimPatternsAppe t.end(); }); -test('poetry py3.7 can package flask with package individually option', async (t) => { +test('poetry py3.9 can package flask with package individually option', async (t) => { process.chdir('tests/poetry_individually'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -998,7 +998,7 @@ test('poetry py3.7 can package flask with package individually option', async (t t.end(); }); -test('py3.7 can package flask with package individually option', async (t) => { +test('py3.9 can package flask with package individually option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1080,7 +1080,7 @@ test('py3.7 can package flask with package individually option', async (t) => { t.end(); }); -test('py3.7 can package flask with package individually & slim option', async (t) => { +test('py3.9 can package flask with package individually & slim option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1162,7 +1162,7 @@ test('py3.7 can package flask with package individually & slim option', async (t t.end(); }); -test('py3.7 can package only requirements of module', async (t) => { +test('py3.9 can package only requirements of module', async (t) => { process.chdir('tests/individually'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1218,7 +1218,7 @@ test('py3.7 can package only requirements of module', async (t) => { t.end(); }); -test('py3.7 can package lambda-decorators using vendor and invidiually option', async (t) => { +test('py3.9 can package lambda-decorators using vendor and invidiually option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1373,7 +1373,8 @@ test( { skip: !canUseDocker() || process.platform === 'win32' } ); -test('py3.7 uses download cache by default option', +test( + 'py3.9 uses download cache by default option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1381,7 +1382,7 @@ test('py3.7 uses download cache by default option', sls(['package'], { env: {} }); const cachepath = getUserCachePath(); t.true( - pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), + pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), 'cache directory exists' ); t.end(); @@ -1389,7 +1390,8 @@ test('py3.7 uses download cache by default option', { skip: true } ); -test('py3.7 uses download cache by default', +test( + 'py3.9 uses download cache by default', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1405,7 +1407,7 @@ test('py3.7 uses download cache by default', ); test( - 'py3.7 uses download cache with dockerizePip option', + 'py3.9 uses download cache with dockerizePip option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1423,7 +1425,7 @@ test( ); test( - 'py3.7 uses download cache with dockerizePip by default option', + 'py3.9 uses download cache with dockerizePip by default option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1441,7 +1443,8 @@ test( { skip: true } ); -test('py3.7 uses static and download cache', +test( + 'py3.9 uses static and download cache', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1466,7 +1469,7 @@ test('py3.7 uses static and download cache', ); test( - 'py3.7 uses static and download cache with dockerizePip option', + 'py3.9 uses static and download cache with dockerizePip option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1490,7 +1493,7 @@ test( { skip: !canUseDocker() || brokenOn('win32') } ); -test('py3.7 uses static cache', async (t) => { +test('py3.9 uses static cache', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1511,7 +1514,7 @@ test('py3.7 uses static cache', async (t) => { '.completed_requirements exists in static-cache' ); - // py3.7 checking that static cache actually pulls from cache (by poisoning it) + // py3.9 checking that static cache actually pulls from cache (by poisoning it) writeFileSync( `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' @@ -1526,7 +1529,7 @@ test('py3.7 uses static cache', async (t) => { t.end(); }); -test('py3.7 uses static cache with cacheLocation option', async (t) => { +test('py3.9 uses static cache with cacheLocation option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1550,7 +1553,7 @@ test('py3.7 uses static cache with cacheLocation option', async (t) => { }); test( - 'py3.7 uses static cache with dockerizePip & slim option', + 'py3.9 uses static cache with dockerizePip & slim option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1572,7 +1575,7 @@ test( '.completed_requirements exists in static-cache' ); - // py3.7 checking that static cache actually pulls from cache (by poisoning it) + // py3.9 checking that static cache actually pulls from cache (by poisoning it) writeFileSync( `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' @@ -1595,7 +1598,7 @@ test( ); test( - 'py3.7 uses download cache with dockerizePip & slim option', + 'py3.9 uses download cache with dockerizePip & slim option', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); @@ -1620,7 +1623,7 @@ test( { skip: !canUseDocker() || brokenOn('win32') } ); -test('py3.7 can ignore functions defined with `image`', async (t) => { +test('py3.9 can ignore functions defined with `image`', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1646,7 +1649,7 @@ test('py3.7 can ignore functions defined with `image`', async (t) => { t.end(); }); -test('poetry py3.7 fails packaging if poetry.lock is missing and flag requirePoetryLockFile is set to true', async (t) => { +test('poetry py3.9 fails packaging if poetry.lock is missing and flag requirePoetryLockFile is set to true', async (t) => { copySync('tests/poetry', 'tests/base with a space'); process.chdir('tests/base with a space'); removeSync('poetry.lock'); @@ -1678,7 +1681,7 @@ test('works with provider.runtime not being python', async (t) => { t.end(); }); -test('poetry py3.7 packages additional optional packages', async (t) => { +test('poetry py3.9 packages additional optional packages', async (t) => { process.chdir('tests/poetry_packages'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1694,7 +1697,7 @@ test('poetry py3.7 packages additional optional packages', async (t) => { t.end(); }); -test('poetry py3.7 skips additional optional packages specified in withoutGroups', async (t) => { +test('poetry py3.9 skips additional optional packages specified in withoutGroups', async (t) => { process.chdir('tests/poetry_packages'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1711,7 +1714,7 @@ test('poetry py3.7 skips additional optional packages specified in withoutGroups t.end(); }); -test('poetry py3.7 only installs optional packages specified in onlyGroups', async (t) => { +test('poetry py3.9 only installs optional packages specified in onlyGroups', async (t) => { process.chdir('tests/poetry_packages'); const path = npm(['pack', '../..']); npm(['i', path]); diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index ef48e901..a82187ff 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: ${env:runtime, 'python3.7'} + runtime: ${env:runtime, 'python3.9'} plugins: - serverless-python-requirements diff --git a/tests/individually/serverless.yml b/tests/individually/serverless.yml index d73d613a..6409532b 100644 --- a/tests/individually/serverless.yml +++ b/tests/individually/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test-indiv provider: name: aws - runtime: python3.7 + runtime: python3.9 package: individually: true diff --git a/tests/non_build_pyproject/serverless.yml b/tests/non_build_pyproject/serverless.yml index b0436e61..d1bbaee6 100644 --- a/tests/non_build_pyproject/serverless.yml +++ b/tests/non_build_pyproject/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements diff --git a/tests/non_poetry_pyproject/serverless.yml b/tests/non_poetry_pyproject/serverless.yml index 2b16790c..7338b10b 100644 --- a/tests/non_poetry_pyproject/serverless.yml +++ b/tests/non_poetry_pyproject/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements diff --git a/tests/pipenv/serverless.yml b/tests/pipenv/serverless.yml index 315f6741..2b471526 100644 --- a/tests/pipenv/serverless.yml +++ b/tests/pipenv/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements diff --git a/tests/poetry/serverless.yml b/tests/poetry/serverless.yml index 2d032acd..d10c4997 100644 --- a/tests/poetry/serverless.yml +++ b/tests/poetry/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements diff --git a/tests/poetry_individually/serverless.yml b/tests/poetry_individually/serverless.yml index 527a2846..86dbb547 100644 --- a/tests/poetry_individually/serverless.yml +++ b/tests/poetry_individually/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements diff --git a/tests/poetry_packages/serverless.yml b/tests/poetry_packages/serverless.yml index 03652968..c6972ede 100644 --- a/tests/poetry_packages/serverless.yml +++ b/tests/poetry_packages/serverless.yml @@ -2,7 +2,7 @@ service: sls-py-req-test provider: name: aws - runtime: python3.7 + runtime: python3.9 plugins: - serverless-python-requirements From 16c0e68b850d62eb1ce127b9c3886857ca955574 Mon Sep 17 00:00:00 2001 From: Carl Walsh Date: Mon, 1 Jan 2024 10:59:03 -0800 Subject: [PATCH 05/12] docs: Describe `cleanCache` in README (#794) --- README.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 37a9d1df..91172bf9 100644 --- a/README.md +++ b/README.md @@ -450,12 +450,27 @@ functions: vendor: ./hello-vendor # The option is also available at the function level ``` -## Manual invocations +## Manual invocation -The `.requirements` and `requirements.zip`(if using zip support) files are left -behind to speed things up on subsequent deploys. To clean them up, run -`sls requirements clean`. You can also create them (and `unzip_requirements` if -using zip support) manually with `sls requirements install`. +The `.requirements` and `requirements.zip` (if using zip support) files are left +behind to speed things up on subsequent deploys. To clean them up, run: + +```plaintext +sls requirements clean +``` + +You can also create them (and `unzip_requirements` if +using zip support) manually with: + +```plaintext +sls requirements install +``` + +The pip download/static cache is outside the serverless folder, and should be manually cleaned when i.e. changing python versions: + +```plaintext +sls requirements cleanCache +``` ## Invalidate requirements caches on package From 1b0faaeb6aadd2bc4b1b53526e35298a98d00aca Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Sun, 14 Jan 2024 22:46:13 +0100 Subject: [PATCH 06/12] feat: Support Scaleway provider (#812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andy Méry --- README.md | 25 +++++++++++++++++ index.js | 5 +++- lib/inject.js | 23 +++++++++++++-- test.js | 17 ++++++++++++ tests/scaleway_provider/_slimPatterns.yml | 2 ++ tests/scaleway_provider/handler.py | 5 ++++ tests/scaleway_provider/package.json | 15 ++++++++++ tests/scaleway_provider/requirements.txt | 3 ++ tests/scaleway_provider/serverless.yml | 34 +++++++++++++++++++++++ 9 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 tests/scaleway_provider/_slimPatterns.yml create mode 100644 tests/scaleway_provider/handler.py create mode 100644 tests/scaleway_provider/package.json create mode 100644 tests/scaleway_provider/requirements.txt create mode 100644 tests/scaleway_provider/serverless.yml diff --git a/README.md b/README.md index 91172bf9..d9127adb 100644 --- a/README.md +++ b/README.md @@ -580,6 +580,31 @@ package: - '**' ``` +## Custom Provider Support + +### Scaleway + +This plugin is compatible with the [Scaleway Serverless Framework Plugin](https://github.com/scaleway/serverless-scaleway-functions) to package dependencies for Python functions deployed on [Scaleway](https://www.scaleway.com/en/serverless-functions/). To use it, add the following to your `serverless.yml`: + +```yaml +provider: + name: scaleway + runtime: python311 + +plugins: + - serverless-python-requirements + - serverless-scaleway-functions +``` + +To handle native dependencies, it's recommended to use the Docker builder with the image provided by Scaleway: + +```yaml +custom: + pythonRequirements: + # Can use any Python version supported by Scaleway + dockerImage: rg.fr-par.scw.cloud/scwfunctionsruntimes-public/python-dep:3.11 +``` + ## Contributors - [@dschep](https://github.com/dschep) - Original developer diff --git a/index.js b/index.js index 246b121e..25cc34cd 100644 --- a/index.js +++ b/index.js @@ -72,7 +72,10 @@ class ServerlessPythonRequirements { ) { options.pythonBin = 'python'; } - + if (/python3[0-9]+/.test(options.pythonBin)) { + // "google" and "scaleway" providers' runtimes uses python3XX + options.pythonBin = options.pythonBin.replace(/3([0-9]+)/, '3.$1'); + } if (options.dockerizePip === 'non-linux') { options.dockerizePip = process.platform !== 'linux'; } diff --git a/lib/inject.js b/lib/inject.js index ea20e58d..12267376 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -13,10 +13,16 @@ BbPromise.promisifyAll(fse); * Inject requirements into packaged application. * @param {string} requirementsPath requirements folder path * @param {string} packagePath target package path + * @param {string} injectionRelativePath installation directory in target package * @param {Object} options our options object * @return {Promise} the JSZip object constructed. */ -function injectRequirements(requirementsPath, packagePath, options) { +function injectRequirements( + requirementsPath, + packagePath, + injectionRelativePath, + options +) { const noDeploy = new Set(options.noDeploy || []); return fse @@ -29,7 +35,13 @@ function injectRequirements(requirementsPath, packagePath, options) { dot: true, }) ) - .map((file) => [file, path.relative(requirementsPath, file)]) + .map((file) => [ + file, + path.join( + injectionRelativePath, + path.relative(requirementsPath, file) + ), + ]) .filter( ([file, relativeFile]) => !file.endsWith('/') && @@ -101,6 +113,11 @@ async function injectAllRequirements(funcArtifact) { this.serverless.cli.log('Injecting required Python packages to package...'); } + let injectionRelativePath = '.'; + if (this.serverless.service.provider.name == 'scaleway') { + injectionRelativePath = 'package'; + } + try { if (this.serverless.service.package.individually) { await BbPromise.resolve(this.targetFuncs) @@ -138,6 +155,7 @@ async function injectAllRequirements(funcArtifact) { : injectRequirements( path.join('.serverless', func.module, 'requirements'), func.package.artifact, + injectionRelativePath, this.options ); }); @@ -145,6 +163,7 @@ async function injectAllRequirements(funcArtifact) { await injectRequirements( path.join('.serverless', 'requirements'), this.serverless.service.package.artifact || funcArtifact, + injectionRelativePath, this.options ); } diff --git a/test.js b/test.js index f4afca45..fad21273 100644 --- a/test.js +++ b/test.js @@ -1729,3 +1729,20 @@ test('poetry py3.9 only installs optional packages specified in onlyGroups', asy t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); t.end(); }); + +test('py3.7 injects dependencies into `package` folder when using scaleway provider', async (t) => { + process.chdir('tests/scaleway_provider'); + const path = npm(['pack', '../..']); + npm(['i', path]); + sls(['package'], { env: {} }); + const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); + t.true( + zipfiles.includes(`package${sep}flask${sep}__init__.py`), + 'flask is packaged' + ); + t.true( + zipfiles.includes(`package${sep}boto3${sep}__init__.py`), + 'boto3 is packaged' + ); + t.end(); +}); diff --git a/tests/scaleway_provider/_slimPatterns.yml b/tests/scaleway_provider/_slimPatterns.yml new file mode 100644 index 00000000..443af9a0 --- /dev/null +++ b/tests/scaleway_provider/_slimPatterns.yml @@ -0,0 +1,2 @@ +slimPatterns: + - '**/__main__.py' diff --git a/tests/scaleway_provider/handler.py b/tests/scaleway_provider/handler.py new file mode 100644 index 00000000..5e2e67ff --- /dev/null +++ b/tests/scaleway_provider/handler.py @@ -0,0 +1,5 @@ +import requests + + +def hello(event, context): + return requests.get('https://httpbin.org/get').json() diff --git a/tests/scaleway_provider/package.json b/tests/scaleway_provider/package.json new file mode 100644 index 00000000..d54b88e0 --- /dev/null +++ b/tests/scaleway_provider/package.json @@ -0,0 +1,15 @@ +{ + "name": "example", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz", + "serverless-scaleway-functions": "^0.4.8" + } +} diff --git a/tests/scaleway_provider/requirements.txt b/tests/scaleway_provider/requirements.txt new file mode 100644 index 00000000..23bfb7a6 --- /dev/null +++ b/tests/scaleway_provider/requirements.txt @@ -0,0 +1,3 @@ +flask==0.12.5 +bottle +boto3 diff --git a/tests/scaleway_provider/serverless.yml b/tests/scaleway_provider/serverless.yml new file mode 100644 index 00000000..5d827bdf --- /dev/null +++ b/tests/scaleway_provider/serverless.yml @@ -0,0 +1,34 @@ +service: sls-py-req-test + +configValidationMode: off + +provider: + name: scaleway + runtime: python39 + +plugins: + - serverless-python-requirements + - serverless-scaleway-functions + +custom: + pythonRequirements: + zip: ${env:zip, self:custom.defaults.zip} + slim: ${env:slim, self:custom.defaults.slim} + slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} + slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} + dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} + defaults: + zip: false + slimPatterns: false + slimPatternsAppendDefaults: true + slim: false + dockerizePip: false + +package: + patterns: + - '!**/*' + - 'handler.py' + +functions: + hello: + handler: handler.hello From 787b4791306e9a3ded5f0177c304cfbce081c119 Mon Sep 17 00:00:00 2001 From: Justin Lyons Date: Sat, 10 Feb 2024 04:26:35 -0500 Subject: [PATCH 07/12] feat: Improved pip failure logging (#813) Co-authored-by: Justin Lyons --- lib/pip.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pip.js b/lib/pip.js index 149c0285..060ce829 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -423,8 +423,8 @@ async function installRequirements(targetFolder, pluginInstance) { } if (log) { - log.info(`Stdout: ${e.stdoutBuffer}`); - log.info(`Stderr: ${e.stderrBuffer}`); + log.error(`Stdout: ${e.stdoutBuffer}`); + log.error(`Stderr: ${e.stderrBuffer}`); } else { serverless.cli.log(`Stdout: ${e.stdoutBuffer}`); serverless.cli.log(`Stderr: ${e.stderrBuffer}`); From 27b70f4d6a7e43fd0e9711bbb56752fee2762901 Mon Sep 17 00:00:00 2001 From: Stijn IJzermans Date: Sat, 10 Feb 2024 10:27:37 +0100 Subject: [PATCH 08/12] fix: Ensure proper support for mixed runtimes and architectures (#815) * feat: Use function runtime & arch for docker * docs: Update readme for python3.9 * feat: Do not zip req for non-py functions * ci: Bump internal package version / python version * fix: Rename mixed test name to be more descriptive --------- Co-authored-by: Stijn IJzermans --- .python-version | 2 +- CONTRIBUTING.md | 10 ++- index.js | 19 +++-- lib/pip.js | 13 +-- lib/zip.js | 5 ++ test.js | 83 +++++++++++++++++++ tests/base/package.json | 2 +- tests/individually/package.json | 2 +- .../module1/handler1.ts | 3 + .../module2/handler2.py | 6 ++ .../module2/requirements.txt | 1 + tests/individually_mixed_runtime/package.json | 14 ++++ .../requirements-common.txt | 1 + .../individually_mixed_runtime/serverless.yml | 39 +++++++++ tests/non_build_pyproject/package.json | 2 +- tests/non_poetry_pyproject/package.json | 2 +- tests/pipenv/package.json | 2 +- tests/poetry/package.json | 2 +- tests/poetry_individually/package.json | 2 +- tests/poetry_packages/package.json | 2 +- 20 files changed, 187 insertions(+), 25 deletions(-) create mode 100644 tests/individually_mixed_runtime/module1/handler1.ts create mode 100644 tests/individually_mixed_runtime/module2/handler2.py create mode 100644 tests/individually_mixed_runtime/module2/requirements.txt create mode 100644 tests/individually_mixed_runtime/package.json create mode 100644 tests/individually_mixed_runtime/requirements-common.txt create mode 100644 tests/individually_mixed_runtime/serverless.yml diff --git a/.python-version b/.python-version index 475ba515..bd28b9c5 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.7 +3.9 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4616858b..900a425b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,12 +13,14 @@ Welcome, and thanks in advance for your help! ## Setup Pre-Reqs: -* Python 3.7 -* [poetry](https://python-poetry.org/docs/) (if you use multiple versions of Python be sure to install it with python 3.7) -* Perl (used in the tests) -* Node v14 or v16 + +- Python 3.9 +- [poetry](https://python-poetry.org/docs/) (if you use multiple versions of Python be sure to install it with python 3.9) +- Perl (used in the tests) +- Node v14 or v16 Then, to begin development: + 1. fork the repository 2. `npm install -g serverless@` (check the peer dependencies in the root `package.json` file for the version) 3. run `npm install` in its root folder diff --git a/index.js b/index.js index 25cc34cd..81e50981 100644 --- a/index.js +++ b/index.js @@ -106,13 +106,8 @@ class ServerlessPythonRequirements { throw new Error( 'Python Requirements: you can provide a dockerImage or a dockerFile option, not both.' ); - } else if (!options.dockerFile) { - // If no dockerFile is provided, use default image - const architecture = - this.serverless.service.provider.architecture || 'x86_64'; - const defaultImage = `public.ecr.aws/sam/build-${this.serverless.service.provider.runtime}:latest-${architecture}`; - options.dockerImage = options.dockerImage || defaultImage; } + if (options.layer) { // If layer was set as a boolean, set it to an empty object to use the layer defaults. if (options.layer === true) { @@ -188,6 +183,18 @@ class ServerlessPythonRequirements { this.commands.requirements.type = 'container'; } + this.dockerImageForFunction = (funcOptions) => { + const runtime = + funcOptions.runtime || this.serverless.service.provider.runtime; + + const architecture = + funcOptions.architecture || + this.serverless.service.provider.architecture || + 'x86_64'; + const defaultImage = `public.ecr.aws/sam/build-${runtime}:latest-${architecture}`; + return this.options.dockerImage || defaultImage; + }; + const isFunctionRuntimePython = (args) => { // If functionObj.runtime is undefined, python. if (!args[1].functionObj || !args[1].functionObj.runtime) { diff --git a/lib/pip.js b/lib/pip.js index 060ce829..16a802b0 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -125,12 +125,13 @@ async function pipAcceptsSystem(pythonBin, pluginInstance) { /** * Install requirements described from requirements in the targetFolder into that same targetFolder * @param {string} targetFolder - * @param {Object} serverless - * @param {Object} options + * @param {Object} pluginInstance + * @param {Object} funcOptions * @return {undefined} */ -async function installRequirements(targetFolder, pluginInstance) { - const { options, serverless, log, progress } = pluginInstance; +async function installRequirements(targetFolder, pluginInstance, funcOptions) { + const { options, serverless, log, progress, dockerImageForFunction } = + pluginInstance; const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt'); let installProgress; @@ -253,7 +254,7 @@ async function installRequirements(targetFolder, pluginInstance) { buildDockerImageProgress && buildDockerImageProgress.remove(); } } else { - dockerImage = options.dockerImage; + dockerImage = dockerImageForFunction(funcOptions); } if (log) { log.info(`Docker Image: ${dockerImage}`); @@ -691,7 +692,7 @@ async function installRequirementsIfNeeded( fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt')); // Then install our requirements from this folder - await installRequirements(workingReqsFolder, pluginInstance); + await installRequirements(workingReqsFolder, pluginInstance, funcOptions); // Copy vendor libraries to requirements folder if (options.vendor) { diff --git a/lib/zip.js b/lib/zip.js index 4b652f98..3c21bbbf 100644 --- a/lib/zip.js +++ b/lib/zip.js @@ -114,6 +114,11 @@ function packRequirements() { if (this.options.zip) { if (this.serverless.service.package.individually) { return BbPromise.resolve(this.targetFuncs) + .filter((func) => { + return ( + func.runtime || this.serverless.service.provider.runtime + ).match(/^python.*/); + }) .map((f) => { if (!get(f, 'module')) { set(f, ['module'], '.'); diff --git a/test.js b/test.js index fad21273..b97f3fdc 100644 --- a/test.js +++ b/test.js @@ -1373,6 +1373,89 @@ test( { skip: !canUseDocker() || process.platform === 'win32' } ); +test( + 'py3.9 can package flask running in docker with module runtime & architecture of function', + async (t) => { + process.chdir('tests/individually_mixed_runtime'); + const path = npm(['pack', '../..']); + npm(['i', path]); + + sls(['package'], { + env: { dockerizePip: 'true' }, + }); + + const zipfiles_hello2 = await listZipFiles( + '.serverless/module2-sls-py-req-test-indiv-mixed-runtime-dev-hello2.zip' + ); + t.true( + zipfiles_hello2.includes('handler2.py'), + 'handler2.py is packaged at root level in function hello2' + ); + t.true( + zipfiles_hello2.includes(`flask${sep}__init__.py`), + 'flask is packaged in function hello2' + ); + }, + { + skip: !canUseDocker() || process.platform === 'win32', + } +); + +test( + 'py3.9 can package flask succesfully when using mixed architecture, docker and zipping', + async (t) => { + process.chdir('tests/individually_mixed_runtime'); + const path = npm(['pack', '../..']); + + npm(['i', path]); + sls(['package'], { env: { dockerizePip: 'true', zip: 'true' } }); + + const zipfiles_hello = await listZipFiles('.serverless/hello1.zip'); + t.true( + zipfiles_hello.includes(`module1${sep}handler1.ts`), + 'handler1.ts is packaged in module dir for hello1' + ); + t.false( + zipfiles_hello.includes('handler2.py'), + 'handler2.py is NOT packaged at root level in function hello1' + ); + t.false( + zipfiles_hello.includes(`flask${sep}__init__.py`), + 'flask is NOT packaged in function hello1' + ); + + const zipfiles_hello2 = await listZipFiles( + '.serverless/module2-sls-py-req-test-indiv-mixed-runtime-dev-hello2.zip' + ); + const zippedReqs = await listRequirementsZipFiles( + '.serverless/module2-sls-py-req-test-indiv-mixed-runtime-dev-hello2.zip' + ); + t.true( + zipfiles_hello2.includes('handler2.py'), + 'handler2.py is packaged at root level in function hello2' + ); + t.false( + zipfiles_hello2.includes(`module1${sep}handler1.ts`), + 'handler1.ts is NOT included at module1 level in hello2' + ); + t.false( + zipfiles_hello2.includes(`pyaml${sep}__init__.py`), + 'pyaml is NOT packaged in function hello2' + ); + t.false( + zipfiles_hello2.includes(`boto3${sep}__init__.py`), + 'boto3 is NOT included in zipfile' + ); + t.true( + zippedReqs.includes(`flask${sep}__init__.py`), + 'flask is packaged in function hello2 in requirements.zip' + ); + + t.end(); + }, + { skip: !canUseDocker() || process.platform === 'win32' } +); + test( 'py3.9 uses download cache by default option', async (t) => { diff --git a/tests/base/package.json b/tests/base/package.json index 781a4259..b07744c9 100644 --- a/tests/base/package.json +++ b/tests/base/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } diff --git a/tests/individually/package.json b/tests/individually/package.json index 781a4259..b07744c9 100644 --- a/tests/individually/package.json +++ b/tests/individually/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } diff --git a/tests/individually_mixed_runtime/module1/handler1.ts b/tests/individually_mixed_runtime/module1/handler1.ts new file mode 100644 index 00000000..b8062f8b --- /dev/null +++ b/tests/individually_mixed_runtime/module1/handler1.ts @@ -0,0 +1,3 @@ +function hello() { + return "hello" +} diff --git a/tests/individually_mixed_runtime/module2/handler2.py b/tests/individually_mixed_runtime/module2/handler2.py new file mode 100644 index 00000000..d9f5c465 --- /dev/null +++ b/tests/individually_mixed_runtime/module2/handler2.py @@ -0,0 +1,6 @@ +import flask + +def hello(event, context): + return { + 'status': 200, + } diff --git a/tests/individually_mixed_runtime/module2/requirements.txt b/tests/individually_mixed_runtime/module2/requirements.txt new file mode 100644 index 00000000..c09d0264 --- /dev/null +++ b/tests/individually_mixed_runtime/module2/requirements.txt @@ -0,0 +1 @@ +flask==2.0.3 diff --git a/tests/individually_mixed_runtime/package.json b/tests/individually_mixed_runtime/package.json new file mode 100644 index 00000000..b07744c9 --- /dev/null +++ b/tests/individually_mixed_runtime/package.json @@ -0,0 +1,14 @@ +{ + "name": "example", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" + } +} diff --git a/tests/individually_mixed_runtime/requirements-common.txt b/tests/individually_mixed_runtime/requirements-common.txt new file mode 100644 index 00000000..30ddf823 --- /dev/null +++ b/tests/individually_mixed_runtime/requirements-common.txt @@ -0,0 +1 @@ +boto3 diff --git a/tests/individually_mixed_runtime/serverless.yml b/tests/individually_mixed_runtime/serverless.yml new file mode 100644 index 00000000..7c602239 --- /dev/null +++ b/tests/individually_mixed_runtime/serverless.yml @@ -0,0 +1,39 @@ +service: sls-py-req-test-indiv-mixed-runtime + +provider: + name: aws + runtime: nodejs18.x + architecture: arm64 + +package: + individually: true + +custom: + pythonRequirements: + dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} + zip: ${env:zip, self:custom.defaults.zip} + defaults: + dockerizePip: false + zip: false + +functions: + hello1: + handler: handler1.hello + architecture: x86_64 + package: + patterns: + - '!**' + - 'module1/**' + + hello2: + handler: handler2.hello + module: module2 + runtime: python3.9 + architecture: x86_64 + package: + patterns: + - '!**' + - 'module2/**' + +plugins: + - serverless-python-requirements diff --git a/tests/non_build_pyproject/package.json b/tests/non_build_pyproject/package.json index 781a4259..b07744c9 100644 --- a/tests/non_build_pyproject/package.json +++ b/tests/non_build_pyproject/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } diff --git a/tests/non_poetry_pyproject/package.json b/tests/non_poetry_pyproject/package.json index 781a4259..b07744c9 100644 --- a/tests/non_poetry_pyproject/package.json +++ b/tests/non_poetry_pyproject/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } diff --git a/tests/pipenv/package.json b/tests/pipenv/package.json index 781a4259..b07744c9 100644 --- a/tests/pipenv/package.json +++ b/tests/pipenv/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } diff --git a/tests/poetry/package.json b/tests/poetry/package.json index 781a4259..b07744c9 100644 --- a/tests/poetry/package.json +++ b/tests/poetry/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } diff --git a/tests/poetry_individually/package.json b/tests/poetry_individually/package.json index 781a4259..b07744c9 100644 --- a/tests/poetry_individually/package.json +++ b/tests/poetry_individually/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } diff --git a/tests/poetry_packages/package.json b/tests/poetry_packages/package.json index 781a4259..b07744c9 100644 --- a/tests/poetry_packages/package.json +++ b/tests/poetry_packages/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz" } } From 549aba00ff320e6ddabb6f0be44999e7922fc616 Mon Sep 17 00:00:00 2001 From: Jackson Borneman <11304426+jax-b@users.noreply.github.com> Date: Sat, 24 Feb 2024 18:41:31 -0500 Subject: [PATCH 09/12] test: Bump node version in test configs --- tests/base/serverless.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index a82187ff..87423210 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -47,7 +47,7 @@ functions: handler: handler.hello hello3: handler: handler.hello - runtime: nodejs8.10 + runtime: nodejs14.x hello4: handler: fn2_handler.hello module: fn2 From fa9ac03ea7ffa3b583aaf69bd8e615ec112cabcc Mon Sep 17 00:00:00 2001 From: hayden Date: Mon, 26 Feb 2024 00:42:20 +0900 Subject: [PATCH 10/12] Add docker rootless feature flag and its implementation for supporting docke rootless environment (#818) --- index.js | 1 + lib/pip.js | 33 +++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/index.js b/index.js index 81e50981..ca8b191f 100644 --- a/index.js +++ b/index.js @@ -50,6 +50,7 @@ class ServerlessPythonRequirements { dockerBuildCmdExtraArgs: [], dockerRunCmdExtraArgs: [], dockerExtraFiles: [], + dockerRootless: false, useStaticCache: true, useDownloadCache: true, cacheLocation: false, diff --git a/lib/pip.js b/lib/pip.js index 16a802b0..40140d36 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -328,12 +328,17 @@ async function installRequirements(targetFolder, pluginInstance, funcOptions) { } // Install requirements with pip // Set the ownership of the current folder to user - pipCmds.push([ - 'chown', - '-R', - `${process.getuid()}:${process.getgid()}`, - '/var/task', - ]); + // If you use docker-rootless, you don't need to set the ownership + if (options.dockerRootless !== true) { + pipCmds.push([ + 'chown', + '-R', + `${process.getuid()}:${process.getgid()}`, + '/var/task', + ]); + } else { + pipCmds.push(['chown', '-R', '0:0', '/var/task']); + } } else { // Use same user so --cache-dir works dockerCmd.push('-u', await getDockerUid(bindPath, pluginInstance)); @@ -346,12 +351,16 @@ async function installRequirements(targetFolder, pluginInstance, funcOptions) { if (process.platform === 'linux') { if (options.useDownloadCache) { // Set the ownership of the download cache dir back to user - pipCmds.push([ - 'chown', - '-R', - `${process.getuid()}:${process.getgid()}`, - dockerDownloadCacheDir, - ]); + if (options.dockerRootless !== true) { + pipCmds.push([ + 'chown', + '-R', + `${process.getuid()}:${process.getgid()}`, + dockerDownloadCacheDir, + ]); + } else { + pipCmds.push(['chown', '-R', '0:0', dockerDownloadCacheDir]); + } } } From ca617bb3bc503db14f68d4e41cf532f798b40704 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Thu, 28 Mar 2024 00:05:29 +0100 Subject: [PATCH 11/12] chore: Bump dependencies --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index c420e4d6..6ec63fa3 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ }, "devDependencies": { "cross-spawn": "*", - "eslint": "^8.52.0", + "eslint": "^8.57.0", "git-list-updated": "^1.2.1", "github-release-from-cc-changelog": "^2.3.0", "lodash": "^4.17.21", @@ -72,7 +72,7 @@ "lodash.uniqby": "^4.7.0", "lodash.values": "^4.3.0", "rimraf": "^3.0.2", - "semver": "^7.5.4", + "semver": "^7.6.0", "set-value": "^4.1.0", "sha256-file": "1.0.0", "shell-quote": "^1.8.1" From 840d28dd319340f61aa9627cb4ca68af643780eb Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Thu, 28 Mar 2024 00:09:48 +0100 Subject: [PATCH 12/12] chore: Release v6.1.0 --- CHANGELOG.md | 11 +++++++++++ package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8667134..c63f9d06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [6.1.0](https://github.com/UnitedIncome/serverless-python-requirements/compare/v6.0.1...v6.1.0) (2024-03-27) + +### Features + +- Support Scaleway provider ([#812](https://github.com/UnitedIncome/serverless-python-requirements/issues/812)) ([1b0faae](https://github.com/UnitedIncome/serverless-python-requirements/commit/1b0faaeb6aadd2bc4b1b53526e35298a98d00aca)) ([Andy Méry](https://github.com/cyclimse)) +- Improved pip failure logging ([#813](https://github.com/UnitedIncome/serverless-python-requirements/issues/813)) ([787b479](https://github.com/UnitedIncome/serverless-python-requirements/commit/787b4791306e9a3ded5f0177c304cfbce081c119)) ([Justin Lyons](https://github.com/babyhuey)) + +### Bug Fixes + +- Ensure proper support for mixed runtimes and architectures ([#815](https://github.com/UnitedIncome/serverless-python-requirements/issues/815)) ([27b70f4](https://github.com/UnitedIncome/serverless-python-requirements/commit/27b70f4d6a7e43fd0e9711bbb56752fee2762901)) ([Stijn IJzermans](https://github.com/stijzermans)) + ### [6.0.1](https://github.com/UnitedIncome/serverless-python-requirements/compare/v6.0.0...v6.0.1) (2023-10-22) ### Bug Fixes diff --git a/package.json b/package.json index 6ec63fa3..3612c2cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "serverless-python-requirements", - "version": "6.0.1", + "version": "6.1.0", "engines": { "node": ">=12.0" },