From ec84948747a2fe578f7f112b443a3710e6202b42 Mon Sep 17 00:00:00 2001 From: Marcin Szleszynski <64603095+martinezpl@users.noreply.github.com> Date: Sun, 13 Feb 2022 15:30:22 +0100 Subject: [PATCH 1/8] test: Refactor tests to use `env` instead of `opt` vars (#672) --- test.js | 360 +++++++++++++++--------------- tests/base/serverless.yml | 22 +- tests/individually/serverless.yml | 2 +- tests/pipenv/serverless.yml | 8 +- tests/poetry/serverless.yml | 8 +- 5 files changed, 196 insertions(+), 204 deletions(-) diff --git a/test.js b/test.js index ccd1920c..11a7cce5 100644 --- a/test.js +++ b/test.js @@ -22,15 +22,15 @@ const initialWorkingDir = process.cwd(); const mkCommand = (cmd) => (args, options = {}) => { + options['env'] = Object.assign( + { SLS_DEBUG: 't' }, + process.env, + options['env'] + ); const { error, stdout, stderr, status } = crossSpawn.sync( cmd, args, - Object.assign( - { - env: Object.assign({}, process.env, { SLS_DEBUG: 't' }), - }, - options - ) + options ); if (error) { console.error(`Error running: ${quote([cmd, ...args])}`); // eslint-disable-line no-console @@ -45,6 +45,7 @@ const mkCommand = } return stdout && stdout.toString().trim(); }; + const sls = mkCommand('sls'); const git = mkCommand('git'); const npm = mkCommand('npm'); @@ -205,7 +206,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -220,9 +221,9 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const fileHash = sha256File('.serverless/sls-py-req-test.zip'); - sls(['package']); + sls(['package'], { env: {} }); t.equal( sha256File('.serverless/sls-py-req-test.zip'), fileHash, @@ -239,7 +240,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([`--pythonBin=${getPythonBin(3)}`, 'package']); + sls(['package'], { env: { pythonBin: getPythonBin(3) } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -254,11 +255,12 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(3)}`, - '--fileName=requirements-w-hashes.txt', - 'package', - ]); + sls(['package'], { + env: { + fileName: 'requirements-w-hashes.txt', + pythonBin: getPythonBin(3), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.end(); @@ -272,11 +274,12 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(3)}`, - '--fileName=requirements-w-nested.txt', - 'package', - ]); + sls(['package'], { + env: { + fileName: 'requirements-w-nested.txt', + pythonBin: getPythonBin(3), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -291,7 +294,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']); + sls(['package'], { env: { zip: 'true', pythonBin: getPythonBin(3) } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes('.requirements.zip'), @@ -316,7 +319,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([`--pythonBin=${getPythonBin(3)}`, '--slim=true', 'package']); + sls(['package'], { env: { slim: 'true', pythonBin: getPythonBin(3) } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -341,7 +344,7 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', 'package']); + sls(['package'], { env: { slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -372,7 +375,7 @@ test( 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', 'serverless.yml', ]); - sls([`--pythonBin=${getPythonBin(3)}`, 'package']); + sls(['package'], { env: { pythonBin: getPythonBin(3) } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.false(zipfiles.includes(`bottle.py`), 'bottle is NOT packaged'); @@ -387,11 +390,12 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(3)}`, - '--fileName=requirements-w-editable.txt', - 'package', - ]); + sls(['package'], { + env: { + fileName: 'requirements-w-editable.txt', + pythonBin: getPythonBin(3), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); t.true( @@ -409,8 +413,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', 'package']); - + sls(['package'], { env: { dockerizePip: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -425,7 +428,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', '--slim=true', 'package']); + sls(['package'], { env: { dockerizePip: 'true', slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -450,7 +453,7 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', '--slim=true', 'package']); + sls(['package'], { env: { dockerizePip: 'true', slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -474,8 +477,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', '--zip=true', 'package']); - + sls(['package'], { env: { dockerizePip: 'true', zip: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); const zippedReqs = await listRequirementsZipFiles( '.serverless/sls-py-req-test.zip' @@ -507,8 +509,9 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', '--zip=true', '--slim=true', 'package']); - + sls(['package'], { + env: { dockerizePip: 'true', zip: 'true', slim: 'true' }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); const zippedReqs = await listRequirementsZipFiles( '.serverless/sls-py-req-test.zip' @@ -540,7 +543,9 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([`--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', 'package']); + sls(['package'], { + env: { runtime: 'python2.7', pythonBin: getPythonBin(2) }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -555,12 +560,9 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--slim=true', - 'package', - ]); + sls(['package'], { + env: { runtime: 'python2.7', slim: 'true', pythonBin: getPythonBin(2) }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -584,12 +586,9 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--zip=true', - 'package', - ]); + sls(['package'], { + env: { runtime: 'python2.7', zip: 'true', pythonBin: getPythonBin(2) }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes('.requirements.zip'), @@ -616,13 +615,14 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--dockerizePip=true', - '--slim=true', - 'package', - ]); + sls(['package'], { + env: { + runtime: 'python2.7', + dockerizePip: 'true', + slim: 'true', + pythonBin: getPythonBin(2), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -653,7 +653,9 @@ test( 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', 'serverless.yml', ]); - sls([`--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', 'package']); + sls(['package'], { + env: { runtime: 'python2.7', pythonBin: getPythonBin(2) }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.false(zipfiles.includes(`bottle.py`), 'bottle is NOT packaged'); @@ -668,14 +670,14 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--dockerizePip=true', - '--zip=true', - 'package', - ]); - + sls(['package'], { + env: { + runtime: 'python2.7', + dockerizePip: 'true', + zip: 'true', + pythonBin: getPythonBin(2), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); const zippedReqs = await listRequirementsZipFiles( '.serverless/sls-py-req-test.zip' @@ -707,15 +709,15 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--dockerizePip=true', - '--zip=true', - '--slim=true', - 'package', - ]); - + sls(['package'], { + env: { + runtime: 'python2.7', + dockerizePip: 'true', + zip: 'true', + slim: 'true', + pythonBin: getPythonBin(2), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); const zippedReqs = await listRequirementsZipFiles( '.serverless/sls-py-req-test.zip' @@ -747,13 +749,13 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--dockerizePip=true', - 'package', - ]); - + sls(['package'], { + env: { + runtime: 'python2.7', + dockerizePip: 'true', + pythonBin: getPythonBin(2), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -768,13 +770,14 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--dockerizePip=true', - '--slim=true', - 'package', - ]); + sls(['package'], { + env: { + runtime: 'python2.7', + dockerizePip: 'true', + slim: 'true', + pythonBin: getPythonBin(2), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -800,13 +803,14 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - `--pythonBin=${getPythonBin(2)}`, - '--runtime=python2.7', - '--dockerizePip=true', - '--slim=true', - 'package', - ]); + sls(['package'], { + env: { + runtime: 'python2.7', + dockerizePip: 'true', + slim: 'true', + pythonBin: getPythonBin(2), + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -830,7 +834,7 @@ test( process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -849,7 +853,7 @@ test( process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', 'package']); + sls(['package'], { env: { slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -875,7 +879,7 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', 'package']); + sls(['package'], { env: { slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -899,7 +903,7 @@ test( process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']); + sls(['package'], { env: { zip: 'true', pythonBin: getPythonBin(3) } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes('.requirements.zip'), @@ -931,7 +935,7 @@ test( 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', 'serverless.yml', ]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.false(zipfiles.includes(`bottle.py`), 'bottle is NOT packaged'); @@ -946,7 +950,7 @@ test( process.chdir('tests/non_build_pyproject'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -961,7 +965,7 @@ test( process.chdir('tests/non_poetry_pyproject'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`handler.py`), 'handler is packaged'); t.end(); @@ -975,7 +979,7 @@ test( process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`bottle.py`), 'bottle is packaged'); @@ -991,7 +995,7 @@ test( process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', 'package']); + sls(['package'], { env: { slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -1017,7 +1021,7 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', 'package']); + sls(['package'], { env: { slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( @@ -1041,7 +1045,7 @@ test( process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']); + sls(['package'], { env: { zip: 'true', pythonBin: getPythonBin(3) } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes('.requirements.zip'), @@ -1073,7 +1077,7 @@ test( 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', 'serverless.yml', ]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.false(zipfiles.includes(`bottle.py`), 'bottle is NOT packaged'); @@ -1090,7 +1094,7 @@ test( npm(['i', path]); perl(['-p', '-i.bak', '-e', 's/include://', 'serverless.yml']); perl(['-p', '-i.bak', '-e', 's/^.*handler.py.*$//', 'serverless.yml']); - sls(['--zip=true', 'package']); + sls(['package'], { env: { zip: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes('.requirements.zip'), @@ -1115,7 +1119,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([`--vendor=./vendor`, 'package']); + sls(['package'], { env: { vendor: './vendor' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -1145,8 +1149,7 @@ test( ]); writeFileSync(`foobar`, ''); chmodSync(`foobar`, perm); - sls(['--vendor=./vendor', 'package']); - + sls(['package'], { env: { vendor: './vendor' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -1184,7 +1187,7 @@ test( process.chdir('tests/base with a space'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -1200,7 +1203,7 @@ test( process.chdir('tests/base with a space'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', 'package']); + sls(['package'], { env: { dockerizePip: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -1216,7 +1219,7 @@ test( const path = npm(['pack', '../..']); writeFileSync('puck', 'requests'); npm(['i', path]); - sls(['--fileName=puck', 'package']); + sls(['package'], { env: { fileName: 'puck' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes(`requests${sep}__init__.py`), @@ -1248,7 +1251,7 @@ test( 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', 'serverless.yml', ]); - sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']); + sls(['package'], { env: { zip: 'true', pythonBin: getPythonBin(3) } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); const zippedReqs = await listRequirementsZipFiles( '.serverless/sls-py-req-test.zip' @@ -1285,8 +1288,9 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']); - + sls(['package'], { + env: { slim: 'true', slimPatternsAppendDefaults: 'false' }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( @@ -1310,13 +1314,13 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - '--dockerizePip=true', - '--slim=true', - '--slimPatternsAppendDefaults=false', - 'package', - ]); - + sls(['package'], { + env: { + dockerizePip: 'true', + slim: 'true', + slimPatternsAppendDefaults: 'false', + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( @@ -1340,13 +1344,13 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - '--runtime=python2.7', - '--slim=true', - '--slimPatternsAppendDefaults=false', - 'package', - ]); - + sls(['package'], { + env: { + runtime: 'python2.7', + slim: 'true', + slimPatternsAppendDefaults: 'false', + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( @@ -1370,13 +1374,14 @@ test( copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - '--dockerizePip=true', - '--runtime=python2.7', - '--slim=true', - '--slimPatternsAppendDefaults=false', - 'package', - ]); + sls(['package'], { + env: { + dockerizePip: 'true', + runtime: 'python2.7', + slim: 'true', + slimPatternsAppendDefaults: 'false', + }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( @@ -1401,7 +1406,9 @@ test( const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']); + sls(['package'], { + env: { slim: 'true', slimPatternsAppendDefaults: 'false' }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( @@ -1426,7 +1433,9 @@ test( const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']); + sls(['package'], { + env: { slim: 'true', slimPatternsAppendDefaults: 'false' }, + }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( @@ -1449,8 +1458,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--individually=true', 'package']); - + sls(['package'], { env: { individually: 'true' } }); const zipfiles_hello = await listZipFiles('.serverless/hello.zip'); t.false( zipfiles_hello.includes(`fn2${sep}__init__.py`), @@ -1536,8 +1544,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--individually=true', '--slim=true', 'package']); - + sls(['package'], { env: { individually: 'true', slim: 'true' } }); const zipfiles_hello = await listZipFiles('.serverless/hello.zip'); t.true( zipfiles_hello.includes('handler.py'), @@ -1623,8 +1630,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--individually=true', '--runtime=python2.7', 'package']); - + sls(['package'], { env: { individually: 'true', runtime: 'python2.7' } }); const zipfiles_hello = await listZipFiles('.serverless/hello.zip'); t.true( zipfiles_hello.includes('handler.py'), @@ -1694,13 +1700,9 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - '--individually=true', - '--runtime=python2.7', - '--slim=true', - 'package', - ]); - + sls(['package'], { + env: { individually: 'true', runtime: 'python2.7', slim: 'true' }, + }); const zipfiles_hello = await listZipFiles('.serverless/hello.zip'); t.true( zipfiles_hello.includes('handler.py'), @@ -1785,8 +1787,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--individually=true', '--runtime=python2.7', 'package']); - + sls(['package'], { env: { individually: 'true', runtime: 'python2.7' } }); t.true( pathExistsSync('.serverless/hello.zip'), 'function hello is packaged' @@ -1819,8 +1820,7 @@ test( process.chdir('tests/individually'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); - + sls(['package'], { env: {} }); const zipfiles_hello = await listZipFiles( '.serverless/module1-sls-py-req-test-indiv-dev-hello1.zip' ); @@ -1880,8 +1880,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--individually=true', '--vendor=./vendor', 'package']); - + sls(['package'], { env: { individually: 'true', vendor: './vendor' } }); const zipfiles_hello = await listZipFiles('.serverless/hello.zip'); t.true( zipfiles_hello.includes('handler.py'), @@ -1966,8 +1965,7 @@ test( chmodSync(`module1${sep}foobar`, perm); npm(['i', path]); - sls(['package']); - + sls(['package'], { env: {} }); const zipfiles_hello1 = await listZipFilesWithMetaData( '.serverless/hello1.zip' ); @@ -2006,8 +2004,7 @@ test( chmodSync(`module1${sep}foobar`, perm); npm(['i', path]); - sls(['--dockerizePip=true', 'package']); - + sls(['package'], { env: { dockerizePip: 'true' } }); const zipfiles_hello = await listZipFilesWithMetaData( '.serverless/hello1.zip' ); @@ -2042,7 +2039,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const cachepath = getUserCachePath(); t.true( pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), @@ -2059,7 +2056,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--cacheLocation=.requirements-cache', 'package']); + sls(['package'], { env: { cacheLocation: '.requirements-cache' } }); t.true( pathExistsSync(`.requirements-cache${sep}downloadCacheslspyc${sep}http`), 'cache directory exists' @@ -2075,7 +2072,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', 'package']); + sls(['package'], { env: { dockerizePip: 'true' } }); const cachepath = getUserCachePath(); t.true( pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), @@ -2092,11 +2089,9 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls([ - '--dockerizePip=true', - '--cacheLocation=.requirements-cache', - 'package', - ]); + sls(['package'], { + env: { dockerizePip: 'true', cacheLocation: '.requirements-cache' }, + }); t.true( pathExistsSync(`.requirements-cache${sep}downloadCacheslspyc${sep}http`), 'cache directory exists' @@ -2112,7 +2107,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); const arch = 'x86_64'; @@ -2137,7 +2132,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', 'package']); + sls(['package'], { env: { dockerizePip: 'true' } }); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); const arch = 'x86_64'; @@ -2162,7 +2157,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package']); + sls(['package'], { env: {} }); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); const arch = 'x86_64'; @@ -2184,8 +2179,7 @@ test( `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' ); - sls(['package']); - + sls(['package'], { env: {} }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes('injected_file_is_bad_form'), @@ -2204,7 +2198,7 @@ test( const path = npm(['pack', '../..']); npm(['i', path]); const cachepath = '.requirements-cache'; - sls([`--cacheLocation=${cachepath}`, 'package']); + sls(['package'], { env: { cacheLocation: cachepath } }); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); const arch = 'x86_64'; t.true( @@ -2230,7 +2224,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', '--slim=true', 'package']); + sls(['package'], { env: { dockerizePip: 'true', slim: 'true' } }); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); const arch = 'x86_64'; @@ -2252,8 +2246,7 @@ test( `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' ); - sls(['--dockerizePip=true', '--slim=true', 'package']); - + sls(['package'], { env: { dockerizePip: 'true', slim: 'true' } }); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( zipfiles.includes('injected_file_is_bad_form'), @@ -2276,7 +2269,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--dockerizePip=true', '--slim=true', 'package']); + sls(['package'], { env: { dockerizePip: 'true', slim: 'true' } }); const cachepath = getUserCachePath(); t.true( pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), @@ -2302,8 +2295,7 @@ test( process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['--individually=true', 'package']); - + sls(['package'], { env: { individually: 'true' } }); t.true( pathExistsSync('.serverless/hello.zip'), 'function hello is packaged' diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index 0b360e9b..37238158 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -2,22 +2,22 @@ service: sls-py-req-test provider: name: aws - runtime: ${opt:runtime, 'python3.6'} + runtime: ${env:runtime, 'python3.6'} plugins: - serverless-python-requirements custom: pythonRequirements: - zip: ${opt:zip, self:custom.defaults.zip} - dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip} - slim: ${opt:slim, self:custom.defaults.slim} + zip: ${env:zip, self:custom.defaults.zip} + dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} + slim: ${env:slim, self:custom.defaults.slim} slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} - slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} - vendor: ${opt:vendor, ''} - fileName: ${opt:fileName, 'requirements.txt'} - useStaticCache: ${opt:useStaticCache, self:custom.defaults.useStaticCache} - useDownloadCache: ${opt:useDownloadCache, self:custom.defaults.useDownloadCache} - cacheLocation: ${opt:cacheLocation, ''} + slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} + vendor: ${env:vendor, ''} + fileName: ${env:fileName, 'requirements.txt'} + useStaticCache: ${env:useStaticCache, self:custom.defaults.useStaticCache} + useDownloadCache: ${env:useDownloadCache, self:custom.defaults.useDownloadCache} + cacheLocation: ${env:cacheLocation, ''} defaults: slim: false slimPatterns: false @@ -29,7 +29,7 @@ custom: useDownloadCache: true package: - individually: ${opt:individually, self:custom.defaults.individually} + individually: ${env:individually, self:custom.defaults.individually} patterns: - '!**/*' - 'handler.py' diff --git a/tests/individually/serverless.yml b/tests/individually/serverless.yml index 121bd89d..a83ac7e0 100644 --- a/tests/individually/serverless.yml +++ b/tests/individually/serverless.yml @@ -10,7 +10,7 @@ package: - '!node_modules/**' custom: pythonRequirements: - dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip} + dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} defaults: dockerizePip: false diff --git a/tests/pipenv/serverless.yml b/tests/pipenv/serverless.yml index dd93e290..4b343bfc 100644 --- a/tests/pipenv/serverless.yml +++ b/tests/pipenv/serverless.yml @@ -8,11 +8,11 @@ plugins: - serverless-python-requirements custom: pythonRequirements: - zip: ${opt:zip, self:custom.defaults.zip} - slim: ${opt:slim, self:custom.defaults.slim} + zip: ${env:zip, self:custom.defaults.zip} + slim: ${env:slim, self:custom.defaults.slim} slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} - slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} - dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip} + slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} + dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} defaults: zip: false slimPatterns: false diff --git a/tests/poetry/serverless.yml b/tests/poetry/serverless.yml index dd93e290..4b343bfc 100644 --- a/tests/poetry/serverless.yml +++ b/tests/poetry/serverless.yml @@ -8,11 +8,11 @@ plugins: - serverless-python-requirements custom: pythonRequirements: - zip: ${opt:zip, self:custom.defaults.zip} - slim: ${opt:slim, self:custom.defaults.slim} + zip: ${env:zip, self:custom.defaults.zip} + slim: ${env:slim, self:custom.defaults.slim} slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} - slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} - dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip} + slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} + dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} defaults: zip: false slimPatterns: false From 769bc820eed1f65c8ae41ccfa74749c650560e6a Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Sun, 27 Feb 2022 22:26:21 +0100 Subject: [PATCH 2/8] ci: Upgrade `setup-python` github action --- .github/workflows/integrate.yml | 6 +++--- .github/workflows/validate.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integrate.yml b/.github/workflows/integrate.yml index 953951df..0d77acfc 100644 --- a/.github/workflows/integrate.yml +++ b/.github/workflows/integrate.yml @@ -31,7 +31,7 @@ jobs: restore-keys: npm-v14-${{ runner.os }}-${{ github.ref }}- - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -82,7 +82,7 @@ jobs: restore-keys: npm-v14-${{ runner.os }}-${{ github.ref }}- - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -133,7 +133,7 @@ jobs: restore-keys: npm-v12-${{ runner.os }}-${{ github.ref }}- - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9215eee1..b4c245f5 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -44,7 +44,7 @@ jobs: npm-v14-${{ runner.os }}-refs/heads/master- - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -111,7 +111,7 @@ jobs: npm-v14-${{ runner.os }}-refs/heads/master- - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -164,7 +164,7 @@ jobs: npm-v12-${{ runner.os }}-refs/heads/master- - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From ff11497cbcf42fe7f7d73fb2e8e2642c542dd8d7 Mon Sep 17 00:00:00 2001 From: Andrei Zhemaituk Date: Mon, 28 Feb 2022 07:10:15 -0500 Subject: [PATCH 3/8] refactor: Log child process command output on error (#679) --- lib/pip.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pip.js b/lib/pip.js index 7a0a0ceb..79dec42a 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -415,6 +415,13 @@ async function installRequirements(targetFolder, pluginInstance) { 'PYTHON_REQUIREMENTS_COMMAND_NOT_FOUND' ); } + if (log) { + log.info(`Stdout: ${e.stdoutBuffer}`); + log.info(`Stderr: ${e.stderrBuffer}`); + } else { + serverless.cli.log(`Stdout: ${e.stdoutBuffer}`); + serverless.cli.log(`Stderr: ${e.stderrBuffer}`); + } throw e; } } From 3edf0e0cabeeb11ffadd9dcac6f198f22aee4a16 Mon Sep 17 00:00:00 2001 From: Marc Hassan Date: Wed, 2 Mar 2022 07:11:36 -0500 Subject: [PATCH 4/8] refactor: Replace `lodash.set` with `set-value` (#676) --- lib/inject.js | 2 +- lib/pip.js | 2 +- lib/zip.js | 2 +- package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/inject.js b/lib/inject.js index f32c9d46..ea20e58d 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -2,7 +2,7 @@ const BbPromise = require('bluebird'); const fse = require('fs-extra'); const glob = require('glob-all'); const get = require('lodash.get'); -const set = require('lodash.set'); +const set = require('set-value'); const path = require('path'); const JSZip = require('jszip'); const { writeZip, zipFile } = require('./zipTree'); diff --git a/lib/pip.js b/lib/pip.js index 79dec42a..9e7c592e 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -2,7 +2,7 @@ const fse = require('fs-extra'); const rimraf = require('rimraf'); const path = require('path'); const get = require('lodash.get'); -const set = require('lodash.set'); +const set = require('set-value'); const spawn = require('child-process-ext/spawn'); const { quote } = require('shell-quote'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); diff --git a/lib/zip.js b/lib/zip.js index cba29450..4b652f98 100644 --- a/lib/zip.js +++ b/lib/zip.js @@ -1,7 +1,7 @@ const fse = require('fs-extra'); const path = require('path'); const get = require('lodash.get'); -const set = require('lodash.set'); +const set = require('set-value'); const uniqBy = require('lodash.uniqby'); const BbPromise = require('bluebird'); const JSZip = require('jszip'); diff --git a/package.json b/package.json index 50ef2246..d9422de5 100644 --- a/package.json +++ b/package.json @@ -69,10 +69,10 @@ "is-wsl": "^2.2.0", "jszip": "^3.7.1", "lodash.get": "^4.4.2", - "lodash.set": "^4.3.2", "lodash.uniqby": "^4.7.0", "lodash.values": "^4.3.0", "rimraf": "^3.0.2", + "set-value": "^4.1.0", "sha256-file": "1.0.0", "shell-quote": "^1.7.3" }, From 915bcadad2f8a3be5434d6e42771bc835271baf8 Mon Sep 17 00:00:00 2001 From: Marcin Szleszynski <64603095+martinezpl@users.noreply.github.com> Date: Tue, 8 Mar 2022 13:00:56 +0100 Subject: [PATCH 5/8] feat: Support `dockerPrivateKey` to specify path to SSH key (#674) --- README.md | 19 ++++++++++++++++--- index.js | 7 +++++-- lib/pip.js | 8 ++++++-- test.js | 33 ++++++++++++++++++++++++++++++--- tests/base/custom_ssh | 1 + tests/base/serverless.yml | 6 ++++++ 6 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 tests/base/custom_ssh diff --git a/README.md b/README.md index abe6a175..63b1a32a 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,20 @@ custom: ``` The `dockerSsh` option will mount your `$HOME/.ssh/id_rsa` and `$HOME/.ssh/known_hosts` as a -volume in the docker container. If your SSH key is password protected, you can use `ssh-agent` -because `$SSH_AUTH_SOCK` is also mounted & the env var set. +volume in the docker container. + +In case you want to use a different key, you can specify the path (absolute) to it through `dockerPrivateKey` option: + +```yaml +custom: + pythonRequirements: + dockerizePip: true + dockerSsh: true + dockerPrivateKey: /home/.ssh/id_ed25519 +``` + +If your SSH key is password protected, you can use `ssh-agent` +because `$SSH_AUTH_SOCK` is also mounted & the env var is set. It is important that the host of your private repositories has already been added in your `$HOME/.ssh/known_hosts` file, as the install process will fail otherwise due to host authenticity failure. @@ -213,7 +225,7 @@ the names in `slimPatterns` #### Option not to strip binaries -In some cases, stripping binaries leads to problems like "ELF load command address/offset not properly aligned", even when done in the Docker environment. You can still slim down the package without `*.so` files with +In some cases, stripping binaries leads to problems like "ELF load command address/offset not properly aligned", even when done in the Docker environment. You can still slim down the package without `*.so` files with: ```yaml custom: @@ -566,3 +578,4 @@ package: - [@jacksgt](https://github.com/jacksgt) - Fixing pip issues - [@lephuongbg](https://github.com/lephuongbg) - Fixing single function deployment - [@rileypriddle](https://github.com/rileypriddle) - Introducing schema validation for `module` property +- [@martinezpl](https://github.com/martinezpl) - Fixing test issues, adding `dockerPrivateKey` option diff --git a/index.js b/index.js index 7741a7f8..c6577fe0 100644 --- a/index.js +++ b/index.js @@ -15,7 +15,6 @@ const { installAllRequirements } = require('./lib/pip'); const { pipfileToRequirements } = require('./lib/pipenv'); const { pyprojectTomlToRequirements } = require('./lib/poetry'); const { cleanup, cleanupCache } = require('./lib/clean'); - BbPromise.promisifyAll(fse); /** @@ -45,6 +44,7 @@ class ServerlessPythonRequirements { : this.serverless.service.provider.runtime || 'python', dockerizePip: false, dockerSsh: false, + dockerPrivateKey: null, dockerImage: null, dockerFile: null, dockerEnv: false, @@ -71,7 +71,10 @@ class ServerlessPythonRequirements { } if ( !options.dockerizePip && - (options.dockerSsh || options.dockerImage || options.dockerFile) + (options.dockerSsh || + options.dockerImage || + options.dockerFile || + options.dockerPrivateKey) ) { if (!this.warningLogged) { if (this.log) { diff --git a/lib/pip.js b/lib/pip.js index 9e7c592e..9f950664 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -275,12 +275,16 @@ async function installRequirements(targetFolder, pluginInstance) { dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { + const homePath = require('os').homedir(); + const sshKeyPath = + options.dockerPrivateKey || `${homePath}/.ssh/id_rsa`; + // Mount necessary ssh files to work with private repos dockerCmd.push( '-v', - `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`, + `${sshKeyPath}:/root/.ssh/${sshKeyPath.split('/').splice(-1)[0]}:z`, '-v', - `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, + `${homePath}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, '-v', `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`, '-e', diff --git a/test.js b/test.js index 11a7cce5..b228805e 100644 --- a/test.js +++ b/test.js @@ -3,6 +3,7 @@ const glob = require('glob-all'); const JSZip = require('jszip'); const sha256File = require('sha256-file'); const tape = require('tape-promise/tape'); + const { chmodSync, removeSync, @@ -23,7 +24,7 @@ const mkCommand = (cmd) => (args, options = {}) => { options['env'] = Object.assign( - { SLS_DEBUG: 't' }, + { SLS_DEBUG: 'true' }, process.env, options['env'] ); @@ -32,11 +33,11 @@ const mkCommand = args, options ); - if (error) { + if (error && !options['noThrow']) { console.error(`Error running: ${quote([cmd, ...args])}`); // eslint-disable-line no-console throw error; } - if (status) { + if (status && !options['noThrow']) { console.error('STDOUT: ', stdout.toString()); // eslint-disable-line no-console console.error('STDERR: ', stderr.toString()); // eslint-disable-line no-console throw new Error( @@ -200,6 +201,32 @@ const canUseDocker = () => { // Skip if running on these platforms. const brokenOn = (...platforms) => platforms.indexOf(process.platform) != -1; +test( + 'dockerPrivateKey option correctly resolves docker command', + async (t) => { + process.chdir('tests/base'); + const path = npm(['pack', '../..']); + npm(['i', path]); + const stdout = sls(['package'], { + noThrow: true, + env: { + dockerizePip: true, + dockerSsh: true, + dockerPrivateKey: `${__dirname}${sep}tests${sep}base${sep}custom_ssh`, + dockerImage: 'break the build to log the command', + }, + }); + t.true( + stdout.includes( + `-v ${__dirname}${sep}tests${sep}base${sep}custom_ssh:/root/.ssh/custom_ssh:z` + ), + 'docker command properly resolved' + ); + t.end(); + }, + { skip: !canUseDocker() || brokenOn('win32') } +); + test( 'default pythonBin can package flask with default options', async (t) => { diff --git a/tests/base/custom_ssh b/tests/base/custom_ssh new file mode 100644 index 00000000..8a7c4203 --- /dev/null +++ b/tests/base/custom_ssh @@ -0,0 +1 @@ +SOME KEY diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index 37238158..6526246c 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -10,6 +10,9 @@ custom: pythonRequirements: zip: ${env:zip, self:custom.defaults.zip} dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} + dockerSsh: ${env:dockerSsh, self:custom.defaults.dockerSsh} + dockerPrivateKey: ${env:dockerPrivateKey, self:custom.defaults.dockerPrivateKey} + dockerImage: ${env:dockerImage, self:custom.defaults.dockerImage} slim: ${env:slim, self:custom.defaults.slim} slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} @@ -24,6 +27,9 @@ custom: slimPatternsAppendDefaults: true zip: false dockerizePip: false + dockerSsh: false + dockerPrivateKey: '' + dockerImage: '' individually: false useStaticCache: true useDownloadCache: true From ebd12cb14ea352fb08c0957f213bda7dcce800df Mon Sep 17 00:00:00 2001 From: Brandon White Date: Mon, 14 Mar 2022 11:50:12 -0500 Subject: [PATCH 6/8] feat: Support individual packaging with `poetry` (#682) --- .gitignore | 1 - index.js | 2 -- lib/pip.js | 23 ++++--------- lib/poetry.js | 33 ++++++++++--------- test.js | 19 +++++++++++ tests/base/package.json | 2 +- tests/non_build_pyproject/package.json | 2 +- tests/non_poetry_pyproject/package.json | 2 +- tests/pipenv/package.json | 2 +- tests/poetry/package.json | 2 +- tests/poetry_individually/module1/handler.py | 5 +++ .../module1/pyproject.toml | 17 ++++++++++ tests/poetry_individually/package.json | 14 ++++++++ tests/poetry_individually/serverless.yml | 32 ++++++++++++++++++ 14 files changed, 116 insertions(+), 40 deletions(-) create mode 100644 tests/poetry_individually/module1/handler.py create mode 100644 tests/poetry_individually/module1/pyproject.toml create mode 100644 tests/poetry_individually/package.json create mode 100644 tests/poetry_individually/serverless.yml diff --git a/.gitignore b/.gitignore index ab0317f3..3707ff1e 100644 --- a/.gitignore +++ b/.gitignore @@ -59,7 +59,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ diff --git a/index.js b/index.js index c6577fe0..ebfc4017 100644 --- a/index.js +++ b/index.js @@ -13,7 +13,6 @@ const { injectAllRequirements } = require('./lib/inject'); const { layerRequirements } = require('./lib/layer'); const { installAllRequirements } = require('./lib/pip'); const { pipfileToRequirements } = require('./lib/pipenv'); -const { pyprojectTomlToRequirements } = require('./lib/poetry'); const { cleanup, cleanupCache } = require('./lib/clean'); BbPromise.promisifyAll(fse); @@ -203,7 +202,6 @@ class ServerlessPythonRequirements { } return BbPromise.bind(this) .then(pipfileToRequirements) - .then(pyprojectTomlToRequirements) .then(addVendorHelper) .then(installAllRequirements) .then(packRequirements) diff --git a/lib/pip.js b/lib/pip.js index 9f950664..ccb809c3 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -7,7 +7,7 @@ const spawn = require('child-process-ext/spawn'); const { quote } = require('shell-quote'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); const { getStripCommand, getStripMode, deleteFiles } = require('./slim'); -const { isPoetryProject } = require('./poetry'); +const { isPoetryProject, pyprojectTomlToRequirements } = require('./poetry'); const { checkForAndDeleteMaxCacheVersions, sha256Path, @@ -60,16 +60,9 @@ function generateRequirementsFile( pluginInstance ) { const { serverless, servicePath, options, log } = pluginInstance; - if ( - options.usePoetry && - fse.existsSync(path.join(servicePath, 'pyproject.toml')) && - isPoetryProject(servicePath) - ) { - filterRequirementsFile( - path.join(servicePath, '.serverless/requirements.txt'), - targetFile, - pluginInstance - ); + const modulePath = path.dirname(requirementsPath); + if (options.usePoetry && isPoetryProject(modulePath)) { + filterRequirementsFile(targetFile, targetFile, pluginInstance); if (log) { log.info(`Parsed requirements.txt from pyproject.toml in ${targetFile}`); } else { @@ -570,11 +563,7 @@ function copyVendors(vendorFolder, targetFolder, { serverless, log }) { * @param {string} fileName */ function requirementsFileExists(servicePath, options, fileName) { - if ( - options.usePoetry && - fse.existsSync(path.join(servicePath, 'pyproject.toml')) && - isPoetryProject(servicePath) - ) { + if (options.usePoetry && isPoetryProject(path.dirname(fileName))) { return true; } @@ -609,6 +598,8 @@ async function installRequirementsIfNeeded( // Our source requirements, under our service path, and our module path (if specified) const fileName = path.join(servicePath, modulePath, options.fileName); + await pyprojectTomlToRequirements(modulePath, pluginInstance); + // Skip requirements generation, if requirements file doesn't exist if (!requirementsFileExists(servicePath, options, fileName)) { return false; diff --git a/lib/poetry.js b/lib/poetry.js index 23f43dc0..4003c1df 100644 --- a/lib/poetry.js +++ b/lib/poetry.js @@ -8,24 +8,25 @@ const tomlParse = require('@iarna/toml/parse-string'); /** * poetry install */ -async function pyprojectTomlToRequirements() { - if (!this.options.usePoetry || !isPoetryProject(this.servicePath)) { +async function pyprojectTomlToRequirements(modulePath, pluginInstance) { + const { serverless, servicePath, options, log, progress } = pluginInstance; + + const moduleProjectPath = path.join(servicePath, modulePath); + if (!options.usePoetry || !isPoetryProject(moduleProjectPath)) { return; } let generateRequirementsProgress; - if (this.progress && this.log) { - generateRequirementsProgress = this.progress.get( + if (progress && log) { + generateRequirementsProgress = progress.get( 'python-generate-requirements-toml' ); generateRequirementsProgress.update( 'Generating requirements.txt from "pyproject.toml"' ); - this.log.info('Generating requirements.txt from "pyproject.toml"'); + log.info('Generating requirements.txt from "pyproject.toml"'); } else { - this.serverless.cli.log( - 'Generating requirements.txt from pyproject.toml...' - ); + serverless.cli.log('Generating requirements.txt from pyproject.toml...'); } try { @@ -42,7 +43,7 @@ async function pyprojectTomlToRequirements() { '--with-credentials', ], { - cwd: this.servicePath, + cwd: moduleProjectPath, } ); } catch (e) { @@ -50,7 +51,7 @@ async function pyprojectTomlToRequirements() { e.stderrBuffer && e.stderrBuffer.toString().includes('command not found') ) { - throw new this.serverless.classes.Error( + throw new serverless.classes.Error( `poetry not found! Install it according to the poetry docs.`, 'PYTHON_REQUIREMENTS_POETRY_NOT_FOUND' ); @@ -59,16 +60,16 @@ async function pyprojectTomlToRequirements() { } const editableFlag = new RegExp(/^-e /gm); - const sourceRequirements = path.join(this.servicePath, 'requirements.txt'); + const sourceRequirements = path.join(moduleProjectPath, 'requirements.txt'); const requirementsContents = fse.readFileSync(sourceRequirements, { encoding: 'utf-8', }); if (requirementsContents.match(editableFlag)) { - if (this.log) { - this.log.info('The generated file contains -e flags, removing them'); + if (log) { + log.info('The generated file contains -e flags, removing them'); } else { - this.serverless.cli.log( + serverless.cli.log( 'The generated file contains -e flags, removing them...' ); } @@ -78,10 +79,10 @@ async function pyprojectTomlToRequirements() { ); } - fse.ensureDirSync(path.join(this.servicePath, '.serverless')); + fse.ensureDirSync(path.join(servicePath, '.serverless')); fse.moveSync( sourceRequirements, - path.join(this.servicePath, '.serverless', 'requirements.txt'), + path.join(servicePath, '.serverless', modulePath, 'requirements.txt'), { overwrite: true } ); } finally { diff --git a/test.js b/test.js index b228805e..e2bbdc2c 100644 --- a/test.js +++ b/test.js @@ -1479,6 +1479,25 @@ test( { skip: !hasPython(3.6) } ); +test( + 'poetry py3.6 can package flask with package individually option', + async (t) => { + process.chdir('tests/poetry_individually'); + const path = npm(['pack', '../..']); + npm(['i', path]); + + sls(['package'], { env: {} }); + const zipfiles = await listZipFiles( + '.serverless/module1-sls-py-req-test-dev-hello.zip' + ); + t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); + t.true(zipfiles.includes(`bottle.py`), 'bottle is packaged'); + t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); + t.end(); + }, + { skip: !hasPython(3.6) } +); + test( 'py3.6 can package flask with package individually option', async (t) => { diff --git a/tests/base/package.json b/tests/base/package.json index 43ce4eee..38630491 100644 --- a/tests/base/package.json +++ b/tests/base/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-5.1.1.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-5.3.1.tgz" } } diff --git a/tests/non_build_pyproject/package.json b/tests/non_build_pyproject/package.json index 43ce4eee..38630491 100644 --- a/tests/non_build_pyproject/package.json +++ b/tests/non_build_pyproject/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-5.1.1.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-5.3.1.tgz" } } diff --git a/tests/non_poetry_pyproject/package.json b/tests/non_poetry_pyproject/package.json index 43ce4eee..38630491 100644 --- a/tests/non_poetry_pyproject/package.json +++ b/tests/non_poetry_pyproject/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-5.1.1.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-5.3.1.tgz" } } diff --git a/tests/pipenv/package.json b/tests/pipenv/package.json index 43ce4eee..38630491 100644 --- a/tests/pipenv/package.json +++ b/tests/pipenv/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-5.1.1.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-5.3.1.tgz" } } diff --git a/tests/poetry/package.json b/tests/poetry/package.json index 43ce4eee..38630491 100644 --- a/tests/poetry/package.json +++ b/tests/poetry/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "serverless-python-requirements": "file:serverless-python-requirements-5.1.1.tgz" + "serverless-python-requirements": "file:serverless-python-requirements-5.3.1.tgz" } } diff --git a/tests/poetry_individually/module1/handler.py b/tests/poetry_individually/module1/handler.py new file mode 100644 index 00000000..5e2e67ff --- /dev/null +++ b/tests/poetry_individually/module1/handler.py @@ -0,0 +1,5 @@ +import requests + + +def hello(event, context): + return requests.get('https://httpbin.org/get').json() diff --git a/tests/poetry_individually/module1/pyproject.toml b/tests/poetry_individually/module1/pyproject.toml new file mode 100644 index 00000000..b813968a --- /dev/null +++ b/tests/poetry_individually/module1/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "poetry" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.6" +Flask = "^1.0" +bottle = {git = "https://git@github.com/bottlepy/bottle.git", tag = "0.12.16"} +boto3 = "^1.9" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/tests/poetry_individually/package.json b/tests/poetry_individually/package.json new file mode 100644 index 00000000..38630491 --- /dev/null +++ b/tests/poetry_individually/package.json @@ -0,0 +1,14 @@ +{ + "name": "example", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "serverless-python-requirements": "file:serverless-python-requirements-5.3.1.tgz" + } +} diff --git a/tests/poetry_individually/serverless.yml b/tests/poetry_individually/serverless.yml new file mode 100644 index 00000000..2cb2d160 --- /dev/null +++ b/tests/poetry_individually/serverless.yml @@ -0,0 +1,32 @@ +service: sls-py-req-test + +provider: + name: aws + runtime: python3.6 + +plugins: + - serverless-python-requirements +custom: + pythonRequirements: + zip: ${env:zip, self:custom.defaults.zip} + slim: ${env:slim, self:custom.defaults.slim} + slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} + slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} + dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} + defaults: + zip: false + slimPatterns: false + slimPatternsAppendDefaults: true + slim: false + dockerizePip: false + +package: + individually: true + +functions: + hello: + handler: handler.hello + module: module1 + package: + patterns: + - 'module1/**' From 33f5d5a0dc5fd166086b9d548615e1dfdb0cbd12 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Mon, 14 Mar 2022 18:01:56 +0100 Subject: [PATCH 7/8] chore: Bump dependencies --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index d9422de5..c40b9cbf 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "cross-spawn": "*", "eslint": "^7.32.0", "git-list-updated": "^1.2.1", - "github-release-from-cc-changelog": "^2.2.0", + "github-release-from-cc-changelog": "^2.2.1", "lodash": "^4.17.21", "prettier": "^2", "standard-version": "^9.3.2", @@ -65,7 +65,7 @@ "bluebird": "^3.7.2", "child-process-ext": "^2.1.1", "fs-extra": "^9.1.0", - "glob-all": "^3.2.1", + "glob-all": "^3.3.0", "is-wsl": "^2.2.0", "jszip": "^3.7.1", "lodash.get": "^4.4.2", From 3a898e5e707658c76f6063f44938366935b41812 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Mon, 14 Mar 2022 18:04:22 +0100 Subject: [PATCH 8/8] chore: Release v5.4.0 --- CHANGELOG.md | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbe979bf..41041fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [5.4.0](https://github.com/UnitedIncome/serverless-python-requirements/compare/v5.3.1...v5.4.0) (2022-03-14) + +### Features + +- Support `dockerPrivateKey` to specify path to SSH key ([#674](https://github.com/UnitedIncome/serverless-python-requirements/issues/674)) ([915bcad](https://github.com/UnitedIncome/serverless-python-requirements/commit/915bcadad2f8a3be5434d6e42771bc835271baf8)) ([Marcin Szleszynski](https://github.com/martinezpl)) +- Support individual packaging with `poetry` ([#682](https://github.com/UnitedIncome/serverless-python-requirements/issues/682)) ([ebd12cb](https://github.com/UnitedIncome/serverless-python-requirements/commit/ebd12cb14ea352fb08c0957f213bda7dcce800df)) ([Brandon White](https://github.com/BrandonLWhite)) + +### Maintenance Improvements + +- Log child process command output on error ([#679](https://github.com/UnitedIncome/serverless-python-requirements/issues/679)) ([ff11497](https://github.com/UnitedIncome/serverless-python-requirements/commit/ff11497cbcf42fe7f7d73fb2e8e2642c542dd8d7)) ([Andrei Zhemaituk](https://github.com/zhemaituk)) +- Replace `lodash.set` with `set-value` ([#676](https://github.com/UnitedIncome/serverless-python-requirements/issues/676)) ([3edf0e0](https://github.com/UnitedIncome/serverless-python-requirements/commit/3edf0e0cabeeb11ffadd9dcac6f198f22aee4a16)) ([Marc Hassan](https://github.com/mhassan1)) + ### [5.3.1](https://github.com/UnitedIncome/serverless-python-requirements/compare/v5.3.0...v5.3.1) (2022-01-28) ### Bug Fixes diff --git a/package.json b/package.json index c40b9cbf..7985cb60 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "serverless-python-requirements", - "version": "5.3.1", + "version": "5.4.0", "engines": { "node": ">=12.0" },