From 50c2850874ded795fd50ae377f1db817a0212e7d Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Wed, 24 Nov 2021 11:42:35 +0100 Subject: [PATCH 01/27] refactor: Adapt to `async` version of `spawn` --- lib/docker.js | 37 ++++++------- lib/pip.js | 147 +++++++++++++++++++++++++++----------------------- lib/pipenv.js | 28 +++++----- lib/poetry.js | 47 ++++++++-------- package.json | 1 + 5 files changed, 136 insertions(+), 124 deletions(-) diff --git a/lib/docker.js b/lib/docker.js index 328e3088..94229b21 100644 --- a/lib/docker.js +++ b/lib/docker.js @@ -1,4 +1,4 @@ -const { spawnSync } = require('child_process'); +const spawn = require('child-process-ext/spawn'); const isWsl = require('is-wsl'); const fse = require('fs-extra'); const path = require('path'); @@ -8,18 +8,19 @@ const path = require('path'); * @param {string[]} options * @return {Object} */ -function dockerCommand(options) { +async function dockerCommand(options) { const cmd = 'docker'; - const ps = spawnSync(cmd, options, { encoding: 'utf-8' }); - if (ps.error) { - if (ps.error.code === 'ENOENT') { + try { + return await spawn(cmd, options, { encoding: 'utf-8' }); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { throw new Error('docker not found! Please install it.'); } - throw new Error(ps.error); - } else if (ps.status !== 0) { - throw new Error(ps.stderr); + throw e; } - return ps; } /** @@ -28,7 +29,7 @@ function dockerCommand(options) { * @param {string[]} extraArgs * @return {string} The name of the built docker image. */ -function buildImage(dockerFile, extraArgs) { +async function buildImage(dockerFile, extraArgs) { const imageName = 'sls-py-reqs-custom'; const options = ['build', '-f', dockerFile, '-t', imageName]; @@ -40,7 +41,7 @@ function buildImage(dockerFile, extraArgs) { options.push('.'); - dockerCommand(options); + await dockerCommand(options); return imageName; } @@ -72,7 +73,7 @@ function findTestFile(servicePath) { * @param {string} bindPath * @return {boolean} */ -function tryBindPath(serverless, bindPath, testFile) { +async function tryBindPath(serverless, bindPath, testFile) { const debug = process.env.SLS_DEBUG; const options = [ 'run', @@ -85,7 +86,7 @@ function tryBindPath(serverless, bindPath, testFile) { ]; try { if (debug) serverless.cli.log(`Trying bindPath ${bindPath} (${options})`); - const ps = dockerCommand(options); + const ps = await dockerCommand(options); if (debug) serverless.cli.log(ps.stdout.trim()); return ps.stdout.trim() === `/test/${testFile}`; } catch (err) { @@ -100,14 +101,14 @@ function tryBindPath(serverless, bindPath, testFile) { * @param {string} servicePath * @return {string} The bind path. */ -function getBindPath(serverless, servicePath) { +async function getBindPath(serverless, servicePath) { // Determine bind path if (process.platform !== 'win32' && !isWsl) { return servicePath; } // test docker is available - dockerCommand(['version']); + await dockerCommand(['version']); // find good bind path for Windows let bindPaths = []; @@ -144,7 +145,7 @@ function getBindPath(serverless, servicePath) { for (let i = 0; i < bindPaths.length; i++) { const bindPath = bindPaths[i]; - if (tryBindPath(serverless, bindPath, testFile)) { + if (await tryBindPath(serverless, bindPath, testFile)) { return bindPath; } } @@ -157,7 +158,7 @@ function getBindPath(serverless, servicePath) { * @param {string} bindPath * @return {boolean} */ -function getDockerUid(bindPath) { +async function getDockerUid(bindPath) { const options = [ 'run', '--rm', @@ -169,7 +170,7 @@ function getDockerUid(bindPath) { '%u', '/bin/sh', ]; - const ps = dockerCommand(options); + const ps = await dockerCommand(options); return ps.stdout.trim(); } diff --git a/lib/pip.js b/lib/pip.js index 244010c8..78af2e20 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -3,7 +3,7 @@ const rimraf = require('rimraf'); const path = require('path'); const get = require('lodash.get'); const set = require('lodash.set'); -const { spawnSync } = require('child_process'); +const spawn = require('child-process-ext/spawn'); const { quote } = require('shell-quote'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); const { getStripCommand, getStripMode, deleteFiles } = require('./slim'); @@ -96,16 +96,23 @@ function generateRequirementsFile( } } -function pipAcceptsSystem(pythonBin) { +async function pipAcceptsSystem(pythonBin) { // Check if pip has Debian's --system option and set it if so - const pipTestRes = spawnSync(pythonBin, ['-m', 'pip', 'help', 'install']); - if (pipTestRes.error) { - if (pipTestRes.error.code === 'ENOENT') { + try { + const pipTestRes = await spawn(pythonBin, ['-m', 'pip', 'help', 'install']); + return ( + pipTestRes.stdoutBuffer && + pipTestRes.stdoutBuffer.toString().indexOf('--system') >= 0 + ); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { throw new Error(`${pythonBin} not found! Try the pythonBin option.`); } - throw pipTestRes.error; + throw e; } - return pipTestRes.stdout.toString().indexOf('--system') >= 0; } /** @@ -115,7 +122,7 @@ function pipAcceptsSystem(pythonBin) { * @param {Object} options * @return {undefined} */ -function installRequirements(targetFolder, serverless, options) { +async function installRequirements(targetFolder, serverless, options) { const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt'); serverless.cli.log( @@ -176,7 +183,7 @@ function installRequirements(targetFolder, serverless, options) { pipCmd.push('--cache-dir', downloadCacheDir); } - if (pipAcceptsSystem(options.pythonBin)) { + if (await pipAcceptsSystem(options.pythonBin)) { pipCmd.push('--system'); } } @@ -191,7 +198,7 @@ function installRequirements(targetFolder, serverless, options) { serverless.cli.log( `Building custom docker image from ${options.dockerFile}...` ); - dockerImage = buildImage( + dockerImage = await buildImage( options.dockerFile, options.dockerBuildCmdExtraArgs ); @@ -201,7 +208,9 @@ function installRequirements(targetFolder, serverless, options) { serverless.cli.log(`Docker Image: ${dockerImage}`); // Prepare bind path depending on os platform - const bindPath = dockerPathForWin(getBindPath(serverless, targetFolder)); + const bindPath = dockerPathForWin( + await getBindPath(serverless, targetFolder) + ); dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { @@ -233,7 +242,7 @@ function installRequirements(targetFolder, serverless, options) { fse.closeSync( fse.openSync(path.join(downloadCacheDir, 'requirements.txt'), 'w') ); - const windowsized = getBindPath(serverless, downloadCacheDir); + const windowsized = await getBindPath(serverless, downloadCacheDir); // And now push it to a volume mount and to pip... dockerCmd.push('-v', `${windowsized}:${dockerDownloadCacheDir}:z`); pipCmd.push('--cache-dir', dockerDownloadCacheDir); @@ -262,7 +271,7 @@ function installRequirements(targetFolder, serverless, options) { ]); } else { // Use same user so --cache-dir works - dockerCmd.push('-u', getDockerUid(bindPath)); + dockerCmd.push('-u', await getDockerUid(bindPath)); } for (let path of options.dockerExtraFiles) { @@ -315,22 +324,23 @@ function installRequirements(targetFolder, serverless, options) { serverless.cli.log(`Running ${quote(dockerCmd)}...`); - filterCommands(mainCmds).forEach(([cmd, ...args]) => { - const res = spawnSync(cmd, args); - if (res.error) { - if (res.error.code === 'ENOENT') { + for (const [cmd, ...args] of mainCmds) { + try { + await spawn(cmd, args); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { const advice = cmd.indexOf('python') > -1 ? 'Try the pythonBin option' : 'Please install it'; throw new Error(`${cmd} not found! ${advice}`); } - throw res.error; - } - if (res.status !== 0) { - throw new Error(`STDOUT: ${res.stdout}\n\nSTDERR: ${res.stderr}`); + throw e; } - }); + } // If enabled slimming, delete files in slimPatterns if (options.slim === true || options.slim === 'true') { deleteFiles(options, targetFolder); @@ -489,7 +499,7 @@ function requirementsFileExists(servicePath, options, fileName) { * @param {Object} serverless * @return {string} */ -function installRequirementsIfNeeded( +async function installRequirementsIfNeeded( servicePath, modulePath, options, @@ -573,7 +583,7 @@ function installRequirementsIfNeeded( fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt')); // Then install our requirements from this folder - installRequirements(workingReqsFolder, serverless, options); + await installRequirements(workingReqsFolder, serverless, options); // Copy vendor libraries to requirements folder if (options.vendor) { @@ -596,7 +606,7 @@ function installRequirementsIfNeeded( * pip install the requirements to the requirements directory * @return {undefined} */ -function installAllRequirements() { +async function installAllRequirements() { // fse.ensureDirSync(path.join(this.servicePath, '.serverless')); // First, check and delete cache versions, if enabled checkForAndDeleteMaxCacheVersions(this.options, this.serverless); @@ -604,55 +614,56 @@ function installAllRequirements() { // Then if we're going to package functions individually... if (this.serverless.service.package.individually) { let doneModules = []; - this.targetFuncs - .filter((func) => - (func.runtime || this.serverless.service.provider.runtime).match( - /^python.*/ - ) + const filteredFuncs = this.targetFuncs.filter((func) => + (func.runtime || this.serverless.service.provider.runtime).match( + /^python.*/ ) - .map((f) => { - if (!get(f, 'module')) { - set(f, ['module'], '.'); - } - // If we didn't already process a module (functions can re-use modules) - if (!doneModules.includes(f.module)) { - const reqsInstalledAt = installRequirementsIfNeeded( - this.servicePath, - f.module, - this.options, - f, - this.serverless - ); - // Add modulePath into .serverless for each module so it's easier for injecting and for users to see where reqs are - let modulePath = path.join( - this.servicePath, - '.serverless', - `${f.module}`, - 'requirements' - ); - // Only do if we didn't already do it - if ( - reqsInstalledAt && - !fse.existsSync(modulePath) && - reqsInstalledAt != modulePath - ) { - if (this.options.useStaticCache) { - // Windows can't symlink so we have to copy on Windows, - // it's not as fast, but at least it works - if (process.platform == 'win32') { - fse.copySync(reqsInstalledAt, modulePath); - } else { - fse.symlink(reqsInstalledAt, modulePath); - } + ); + + for (const f of filteredFuncs) { + if (!get(f, 'module')) { + set(f, ['module'], '.'); + } + + // If we didn't already process a module (functions can re-use modules) + if (!doneModules.includes(f.module)) { + const reqsInstalledAt = await installRequirementsIfNeeded( + this.servicePath, + f.module, + this.options, + f, + this.serverless + ); + // Add modulePath into .serverless for each module so it's easier for injecting and for users to see where reqs are + let modulePath = path.join( + this.servicePath, + '.serverless', + `${f.module}`, + 'requirements' + ); + // Only do if we didn't already do it + if ( + reqsInstalledAt && + !fse.existsSync(modulePath) && + reqsInstalledAt != modulePath + ) { + if (this.options.useStaticCache) { + // Windows can't symlink so we have to copy on Windows, + // it's not as fast, but at least it works + if (process.platform == 'win32') { + fse.copySync(reqsInstalledAt, modulePath); } else { - fse.rename(reqsInstalledAt, modulePath); + fse.symlink(reqsInstalledAt, modulePath); } + } else { + fse.rename(reqsInstalledAt, modulePath); } - doneModules.push(f.module); } - }); + doneModules.push(f.module); + } + } } else { - const reqsInstalledAt = installRequirementsIfNeeded( + const reqsInstalledAt = await installRequirementsIfNeeded( this.servicePath, '', this.options, diff --git a/lib/pipenv.js b/lib/pipenv.js index 063fb5d8..e5731aaf 100644 --- a/lib/pipenv.js +++ b/lib/pipenv.js @@ -1,12 +1,12 @@ const fse = require('fs-extra'); const path = require('path'); -const { spawnSync } = require('child_process'); +const spawn = require('child-process-ext/spawn'); const { EOL } = require('os'); /** * pipenv install */ -function pipfileToRequirements() { +async function pipfileToRequirements() { if ( !this.options.usePipenv || !fse.existsSync(path.join(this.servicePath, 'Pipfile')) @@ -16,28 +16,26 @@ function pipfileToRequirements() { this.serverless.cli.log('Generating requirements.txt from Pipfile...'); - const res = spawnSync( - 'pipenv', - ['lock', '--requirements', '--keep-outdated'], - { + let res; + try { + res = await spawn('pipenv', ['lock', '--requirements', '--keep-outdated'], { cwd: this.servicePath, - } - ); - if (res.error) { - if (res.error.code === 'ENOENT') { + }); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { throw new Error( `pipenv not found! Install it with 'pip install pipenv'.` ); } - throw new Error(res.error); - } - if (res.status !== 0) { - throw new Error(res.stderr); + throw e; } fse.ensureDirSync(path.join(this.servicePath, '.serverless')); fse.writeFileSync( path.join(this.servicePath, '.serverless/requirements.txt'), - removeEditableFlagFromRequirementsString(res.stdout) + removeEditableFlagFromRequirementsString(res.stdoutBuffer) ); } diff --git a/lib/poetry.js b/lib/poetry.js index 553a1392..55f83289 100644 --- a/lib/poetry.js +++ b/lib/poetry.js @@ -1,44 +1,45 @@ const fs = require('fs'); const fse = require('fs-extra'); const path = require('path'); -const { spawnSync } = require('child_process'); +const spawn = require('child-process-ext/spawn'); const tomlParse = require('@iarna/toml/parse-string'); /** * poetry install */ -function pyprojectTomlToRequirements() { +async function pyprojectTomlToRequirements() { if (!this.options.usePoetry || !isPoetryProject(this.servicePath)) { return; } this.serverless.cli.log('Generating requirements.txt from pyproject.toml...'); - const res = spawnSync( - 'poetry', - [ - 'export', - '--without-hashes', - '-f', - 'requirements.txt', - '-o', - 'requirements.txt', - '--with-credentials', - ], - { - cwd: this.servicePath, - } - ); - if (res.error) { - if (res.error.code === 'ENOENT') { + try { + await spawn( + 'poetry', + [ + 'export', + '--without-hashes', + '-f', + 'requirements.txt', + '-o', + 'requirements.txt', + '--with-credentials', + ], + { + cwd: this.servicePath, + } + ); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { throw new Error( `poetry not found! Install it according to the poetry docs.` ); } - throw new Error(res.error); - } - if (res.status !== 0) { - throw new Error(res.stderr); + throw e; } const editableFlag = new RegExp(/^-e /gm); diff --git a/package.json b/package.json index 1fed4c39..c9d247a3 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@iarna/toml": "^2.2.5", "appdirectory": "^0.1.0", "bluebird": "^3.7.2", + "child-process-ext": "^2.1.1", "fs-extra": "^9.1.0", "glob-all": "^3.2.1", "is-wsl": "^2.2.0", From a79899ae5f6f66aa0c65e7fda8e0186d38ff446e Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 10:57:15 +0100 Subject: [PATCH 02/27] refactor: Adapt v3 log writing interfaces --- index.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 2072bbc1..975ff750 100644 --- a/index.js +++ b/index.js @@ -109,9 +109,10 @@ class ServerlessPythonRequirements { * The plugin constructor * @param {Object} serverless * @param {Object} options + * @param {Object} v3Utils * @return {undefined} */ - constructor(serverless) { + constructor(serverless, cliOptions, v3Utils) { this.serverless = serverless; this.servicePath = this.serverless.config.servicePath; this.warningLogged = false; @@ -127,6 +128,13 @@ class ServerlessPythonRequirements { }, }); } + + if (v3Utils) { + this.log = v3Utils.log; + this.progress = v3Utils.progress; + this.writeText = v3Utils.writeText; + }; + this.commands = { requirements: { commands: { From 9e952df5e91abb98679ce9ea700a0c5409198205 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 12:09:29 +0100 Subject: [PATCH 03/27] refactor: Adapt `poetry` for modern logs --- lib/poetry.js | 104 +++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/lib/poetry.js b/lib/poetry.js index 55f83289..65970cc4 100644 --- a/lib/poetry.js +++ b/lib/poetry.js @@ -1,6 +1,7 @@ const fs = require('fs'); const fse = require('fs-extra'); const path = require('path'); + const spawn = require('child-process-ext/spawn'); const tomlParse = require('@iarna/toml/parse-string'); @@ -12,58 +13,75 @@ async function pyprojectTomlToRequirements() { return; } - this.serverless.cli.log('Generating requirements.txt from pyproject.toml...'); + let generateRequirementsProgress; + if (this.progress) { + generateRequirementsProgress = this.progress.get( + 'python-generate-requirements-toml' + ); + } else { + this.serverless.cli.log( + 'Generating requirements.txt from pyproject.toml...' + ); + } try { - await spawn( - 'poetry', - [ - 'export', - '--without-hashes', - '-f', - 'requirements.txt', - '-o', - 'requirements.txt', - '--with-credentials', - ], - { - cwd: this.servicePath, - } - ); - } catch (e) { - if ( - e.stderrBuffer && - e.stderrBuffer.toString().includes('command not found') - ) { - throw new Error( - `poetry not found! Install it according to the poetry docs.` + try { + await spawn( + 'poetry', + [ + 'export', + '--without-hashes', + '-f', + 'requirements.txt', + '-o', + 'requirements.txt', + '--with-credentials', + ], + { + cwd: this.servicePath, + } ); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + throw new Error( + `poetry not found! Install it according to the poetry docs.` + ); + } + throw e; } - throw e; - } - const editableFlag = new RegExp(/^-e /gm); - const sourceRequirements = path.join(this.servicePath, 'requirements.txt'); - const requirementsContents = fse.readFileSync(sourceRequirements, { - encoding: 'utf-8', - }); + const editableFlag = new RegExp(/^-e /gm); + const sourceRequirements = path.join(this.servicePath, 'requirements.txt'); + const requirementsContents = fse.readFileSync(sourceRequirements, { + encoding: 'utf-8', + }); - if (requirementsContents.match(editableFlag)) { - this.serverless.cli.log( - 'The generated file contains -e flags, removing them...' - ); - fse.writeFileSync( + if (requirementsContents.match(editableFlag)) { + if (this.log) { + this.log.info('The generated file contains -e flags, removing them'); + } else { + this.serverless.cli.log( + 'The generated file contains -e flags, removing them...' + ); + } + fse.writeFileSync( + sourceRequirements, + requirementsContents.replace(editableFlag, '') + ); + } + + fse.ensureDirSync(path.join(this.servicePath, '.serverless')); + fse.moveSync( sourceRequirements, - requirementsContents.replace(editableFlag, '') + path.join(this.servicePath, '.serverless', 'requirements.txt'), + { overwrite: true } ); + } finally { + generateRequirementsProgress && generateRequirementsProgress.remove(); } - - fse.ensureDirSync(path.join(this.servicePath, '.serverless')); - fse.moveSync( - sourceRequirements, - path.join(this.servicePath, '.serverless', 'requirements.txt'), - { overwrite: true } - ); } /** From e3afe7bf4162b0be77764874641a5e024be5a91a Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 12:09:46 +0100 Subject: [PATCH 04/27] refactor: Adapt `pipenv` to modern logs --- lib/pipenv.js | 55 ++++++++++++++++++++++++++++++++++----------------- lib/poetry.js | 4 ++++ 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/lib/pipenv.js b/lib/pipenv.js index e5731aaf..5100a810 100644 --- a/lib/pipenv.js +++ b/lib/pipenv.js @@ -14,29 +14,48 @@ async function pipfileToRequirements() { return; } - this.serverless.cli.log('Generating requirements.txt from Pipfile...'); + let generateRequirementsProgress; + if (this.progress) { + generateRequirementsProgress = this.progress.get( + 'python-generate-requirements-pipfile' + ); + generateRequirementsProgress.update( + 'Generating requirements.txt from Pipfile', + { isMainEvent: true } + ); + } else { + this.serverless.cli.log('Generating requirements.txt from Pipfile...'); + } - let res; try { - res = await spawn('pipenv', ['lock', '--requirements', '--keep-outdated'], { - cwd: this.servicePath, - }); - } catch (e) { - if ( - e.stderrBuffer && - e.stderrBuffer.toString().includes('command not found') - ) { - throw new Error( - `pipenv not found! Install it with 'pip install pipenv'.` + let res; + try { + res = await spawn( + 'pipenv', + ['lock', '--requirements', '--keep-outdated'], + { + cwd: this.servicePath, + } ); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + throw new Error( + `pipenv not found! Install it with 'pip install pipenv'.` + ); + } + throw e; } - throw e; + fse.ensureDirSync(path.join(this.servicePath, '.serverless')); + fse.writeFileSync( + path.join(this.servicePath, '.serverless/requirements.txt'), + removeEditableFlagFromRequirementsString(res.stdoutBuffer) + ); + } finally { + generateRequirementsProgress && generateRequirementsProgress.remove(); } - fse.ensureDirSync(path.join(this.servicePath, '.serverless')); - fse.writeFileSync( - path.join(this.servicePath, '.serverless/requirements.txt'), - removeEditableFlagFromRequirementsString(res.stdoutBuffer) - ); } /** diff --git a/lib/poetry.js b/lib/poetry.js index 65970cc4..81988742 100644 --- a/lib/poetry.js +++ b/lib/poetry.js @@ -18,6 +18,10 @@ async function pyprojectTomlToRequirements() { generateRequirementsProgress = this.progress.get( 'python-generate-requirements-toml' ); + generateRequirementsProgress.update( + 'Generating requirements.txt from "pyproject.toml"', + { isMainEvent: true } + ); } else { this.serverless.cli.log( 'Generating requirements.txt from pyproject.toml...' From 1c8f911ef57c91b3efd5fd3c030df9548aa6ceae Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 12:12:29 +0100 Subject: [PATCH 05/27] refactor: Adapt `clean` to modern logs --- lib/clean.js | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/clean.js b/lib/clean.js index e0bff238..88d7d03e 100644 --- a/lib/clean.js +++ b/lib/clean.js @@ -38,8 +38,16 @@ function cleanup() { function cleanupCache() { const cacheLocation = getUserCachePath(this.options); if (fse.existsSync(cacheLocation)) { + let cleanupProgress; if (this.serverless) { - this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`); + if (this.progress) { + cleanupProgress = this.progress.get('python-cleanup-cache'); + cleanupProgress.notice(`Removing static caches at: ${cacheLocation}`, { + isMainEvent: true, + }); + } else { + this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`); + } } // Only remove cache folders that we added, just incase someone accidentally puts a weird @@ -50,10 +58,19 @@ function cleanupCache() { .forEach((file) => { promises.push(fse.removeAsync(file)); }); - return BbPromise.all(promises); + return BbPromise.all(promises) + .then(() => cleanupProgress && cleanupProgress.remove()) + .catch((e) => { + cleanupProgress && cleanupProgress.remove(); + throw e; + }); } else { if (this.serverless) { - this.serverless.cli.log(`No static cache found`); + if (this.log) { + this.log.info(`No static cache found`); + } else { + this.serverless.cli.log(`No static cache found`); + } } return BbPromise.resolve(); } From 8ff97e6b7c279334e417dbdb65e64d0de2656986 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 12:12:44 +0100 Subject: [PATCH 06/27] refactor: Adapt `shared` to modern logs --- lib/shared.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/shared.js b/lib/shared.js index 79b60cef..7baee58b 100644 --- a/lib/shared.js +++ b/lib/shared.js @@ -12,7 +12,7 @@ const sha256File = require('sha256-file'); * @param {Object} serverless * @return {undefined} */ -function checkForAndDeleteMaxCacheVersions(options, serverless) { +function checkForAndDeleteMaxCacheVersions({ serverless, options, log }) { // If we're using the static cache, and we have static cache max versions enabled if ( options.useStaticCache && @@ -42,10 +42,17 @@ function checkForAndDeleteMaxCacheVersions(options, serverless) { rimraf.sync(files[i]); items++; } + // Log the number of cache files flushed - serverless.cli.log( - `Removed ${items} items from cache because of staticCacheMaxVersions` - ); + if (log) { + log.info( + `Removed ${items} items from cache because of staticCacheMaxVersions` + ); + } else { + serverless.cli.log( + `Removed ${items} items from cache because of staticCacheMaxVersions` + ); + } } } } From 1162275d6eb95a756d174f87b40b9cfecd892bc7 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 12:13:23 +0100 Subject: [PATCH 07/27] refactor: Adapt `zip` to modern logs --- lib/zip.js | 81 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 19 deletions(-) diff --git a/lib/zip.js b/lib/zip.js index 2e872aa9..de61ce0f 100644 --- a/lib/zip.js +++ b/lib/zip.js @@ -30,9 +30,13 @@ function addVendorHelper() { }) .then((functions) => uniqBy(functions, (func) => func.module)) .map((f) => { - this.serverless.cli.log( - `Adding Python requirements helper to ${f.module}...` - ); + if (this.log) { + this.log.info(`Adding Python requirements helper to ${f.module}`); + } else { + this.serverless.cli.log( + `Adding Python requirements helper to ${f.module}...` + ); + } return fse.copyAsync( path.resolve(__dirname, '../unzip_requirements.py'), @@ -40,7 +44,11 @@ function addVendorHelper() { ); }); } else { - this.serverless.cli.log('Adding Python requirements helper...'); + if (this.log) { + this.log.info('Adding Python requirements helper'); + } else { + this.serverless.cli.log('Adding Python requirements helper...'); + } if (!get(this.serverless.service, 'package.patterns')) { set(this.serverless.service, ['package', 'patterns'], []); @@ -72,15 +80,25 @@ function removeVendorHelper() { }) .then((funcs) => uniqBy(funcs, (f) => f.module)) .map((f) => { - this.serverless.cli.log( - `Removing Python requirements helper from ${f.module}...` - ); + if (this.log) { + this.log.info( + `Removing Python requirements helper from ${f.module}` + ); + } else { + this.serverless.cli.log( + `Removing Python requirements helper from ${f.module}...` + ); + } return fse.removeAsync( path.join(this.servicePath, f.module, 'unzip_requirements.py') ); }); } else { - this.serverless.cli.log('Removing Python requirements helper...'); + if (this.log) { + this.log.info('Removing Python requirements helper'); + } else { + this.serverless.cli.log('Removing Python requirements helper...'); + } return fse.removeAsync( path.join(this.servicePath, 'unzip_requirements.py') ); @@ -104,21 +122,46 @@ function packRequirements() { }) .then((funcs) => uniqBy(funcs, (f) => f.module)) .map((f) => { - this.serverless.cli.log( - `Zipping required Python packages for ${f.module}...` - ); + let packProgress; + if (this.progress) { + packProgress = this.progress.get( + `python-pack-requirements-${f.module}` + ); + packProgress.update( + `Zipping required Python packages for ${f.module}`, + { isMainEvent: true } + ); + } else { + this.serverless.cli.log( + `Zipping required Python packages for ${f.module}...` + ); + } f.package.patterns.push(`${f.module}/.requirements.zip`); - return addTree( - new JSZip(), - `.serverless/${f.module}/requirements` - ).then((zip) => writeZip(zip, `${f.module}/.requirements.zip`)); + return addTree(new JSZip(), `.serverless/${f.module}/requirements`) + .then((zip) => writeZip(zip, `${f.module}/.requirements.zip`)) + .then(() => packProgress && packProgress.remove()) + .catch((e) => { + packProgress && packProgress.remove(); + throw e; + }); }); } else { - this.serverless.cli.log('Zipping required Python packages...'); + let packProgress; + if (this.progress) { + packProgress = this.progress.get(`python-pack-requirements`); + } else { + this.serverless.cli.log('Zipping required Python packages...'); + } this.serverless.service.package.patterns.push('.requirements.zip'); - return addTree(new JSZip(), '.serverless/requirements').then((zip) => - writeZip(zip, path.join(this.servicePath, '.requirements.zip')) - ); + return addTree(new JSZip(), '.serverless/requirements') + .then((zip) => + writeZip(zip, path.join(this.servicePath, '.requirements.zip')) + ) + .then(() => packProgress && packProgress.remove()) + .catch((e) => { + packProgress && packProgress.remove(); + throw e; + }); } } } From b7902aa047bd91359e41d65b42efca324ec15997 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 13:14:35 +0100 Subject: [PATCH 08/27] refactor: Adapt `pip` to modern logs --- index.js | 10 +- lib/pip.js | 566 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 331 insertions(+), 245 deletions(-) diff --git a/index.js b/index.js index 975ff750..26616295 100644 --- a/index.js +++ b/index.js @@ -74,9 +74,13 @@ class ServerlessPythonRequirements { (options.dockerSsh || options.dockerImage || options.dockerFile) ) { if (!this.warningLogged) { - this.serverless.cli.log( - 'WARNING: You provided a docker related option but dockerizePip is set to false.' - ); + if (this.log) { + this.log.warning('You provided a docker related option but dockerizePip is set to false.'); + } else { + this.serverless.cli.log( + 'WARNING: You provided a docker related option but dockerizePip is set to false.' + ); + } this.warningLogged = true; } } diff --git a/lib/pip.js b/lib/pip.js index 78af2e20..89e8e74a 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -57,10 +57,9 @@ function mergeCommands(commands) { function generateRequirementsFile( requirementsPath, targetFile, - serverless, - servicePath, - options + pluginInstance ) { + const { serverless, servicePath, options, log } = pluginInstance; if ( options.usePoetry && fse.existsSync(path.join(servicePath, 'pyproject.toml')) && @@ -69,12 +68,15 @@ function generateRequirementsFile( filterRequirementsFile( path.join(servicePath, '.serverless/requirements.txt'), targetFile, - options, - serverless - ); - serverless.cli.log( - `Parsed requirements.txt from pyproject.toml in ${targetFile}...` + pluginInstance ); + if (log) { + log.info(`Parsed requirements.txt from pyproject.toml in ${targetFile}`); + } else { + serverless.cli.log( + `Parsed requirements.txt from pyproject.toml in ${targetFile}...` + ); + } } else if ( options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile')) @@ -82,17 +84,26 @@ function generateRequirementsFile( filterRequirementsFile( path.join(servicePath, '.serverless/requirements.txt'), targetFile, - options, - serverless - ); - serverless.cli.log( - `Parsed requirements.txt from Pipfile in ${targetFile}...` + pluginInstance ); + if (log) { + log.info(`Parsed requirements.txt from Pipfile in ${targetFile}`); + } else { + serverless.cli.log( + `Parsed requirements.txt from Pipfile in ${targetFile}...` + ); + } } else { - filterRequirementsFile(requirementsPath, targetFile, options, serverless); - serverless.cli.log( - `Generated requirements from ${requirementsPath} in ${targetFile}...` - ); + filterRequirementsFile(requirementsPath, targetFile, pluginInstance); + if (log) { + log.info( + `Generated requirements from ${requirementsPath} in ${targetFile}` + ); + } else { + serverless.cli.log( + `Generated requirements from ${requirementsPath} in ${targetFile}...` + ); + } } } @@ -122,228 +133,290 @@ async function pipAcceptsSystem(pythonBin) { * @param {Object} options * @return {undefined} */ -async function installRequirements(targetFolder, serverless, options) { +async function installRequirements( + targetFolder, + { options, serverless, log, progress } +) { const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt'); - serverless.cli.log( - `Installing requirements from ${targetRequirementsTxt} ...` - ); - - const dockerCmd = []; - const pipCmd = [options.pythonBin, '-m', 'pip', 'install']; - - if ( - Array.isArray(options.pipCmdExtraArgs) && - options.pipCmdExtraArgs.length > 0 - ) { - options.pipCmdExtraArgs.forEach((cmd) => { - const parts = cmd.split(/\s+/, 2); - pipCmd.push(...parts); - }); + let installProgress; + if (progress) { + installProgress = progress.get('python-install'); + installProgress.update( + `Installing requirements from "${targetRequirementsTxt}"`, + { isMainEvent: true } + ); + } else { + serverless.cli.log( + `Installing requirements from ${targetRequirementsTxt} ...` + ); } - const pipCmds = [pipCmd]; - const postCmds = []; - // Check if we're using the legacy --cache-dir command... - if (options.pipCmdExtraArgs.indexOf('--cache-dir') > -1) { - if (options.dockerizePip) { - throw ( - 'Error: You can not use --cache-dir with Docker any more, please\n' + - ' use the new option useDownloadCache instead. Please see:\n' + - ' https://github.com/UnitedIncome/serverless-python-requirements#caching' - ); - } else { - serverless.cli.log('=================================================='); - serverless.cli.log( - 'Warning: You are using a deprecated --cache-dir inside\n' + - ' your pipCmdExtraArgs which may not work properly, please use the\n' + - ' useDownloadCache option instead. Please see: \n' + - ' https://github.com/UnitedIncome/serverless-python-requirements#caching' - ); - serverless.cli.log('=================================================='); - } - } + try { + const dockerCmd = []; + const pipCmd = [options.pythonBin, '-m', 'pip', 'install']; - if (!options.dockerizePip) { - // Push our local OS-specific paths for requirements and target directory - pipCmd.push( - '-t', - dockerPathForWin(targetFolder), - '-r', - dockerPathForWin(targetRequirementsTxt) - ); - // If we want a download cache... - if (options.useDownloadCache) { - const downloadCacheDir = path.join( - getUserCachePath(options), - 'downloadCacheslspyc' - ); - serverless.cli.log(`Using download cache directory ${downloadCacheDir}`); - fse.ensureDirSync(downloadCacheDir); - pipCmd.push('--cache-dir', downloadCacheDir); + if ( + Array.isArray(options.pipCmdExtraArgs) && + options.pipCmdExtraArgs.length > 0 + ) { + options.pipCmdExtraArgs.forEach((cmd) => { + const parts = cmd.split(/\s+/, 2); + pipCmd.push(...parts); + }); } - if (await pipAcceptsSystem(options.pythonBin)) { - pipCmd.push('--system'); - } - } - // If we are dockerizing pip - if (options.dockerizePip) { - // Push docker-specific paths for requirements and target directory - pipCmd.push('-t', '/var/task/', '-r', '/var/task/requirements.txt'); - - // Build docker image if required - let dockerImage; - if (options.dockerFile) { - serverless.cli.log( - `Building custom docker image from ${options.dockerFile}...` - ); - dockerImage = await buildImage( - options.dockerFile, - options.dockerBuildCmdExtraArgs - ); - } else { - dockerImage = options.dockerImage; + const pipCmds = [pipCmd]; + const postCmds = []; + // Check if we're using the legacy --cache-dir command... + if (options.pipCmdExtraArgs.indexOf('--cache-dir') > -1) { + if (options.dockerizePip) { + throw ( + 'Error: You can not use --cache-dir with Docker any more, please\n' + + ' use the new option useDownloadCache instead. Please see:\n' + + ' https://github.com/UnitedIncome/serverless-python-requirements#caching' + ); + } else { + if (log) { + log.warning( + 'You are using a deprecated --cache-dir inside\n' + + ' your pipCmdExtraArgs which may not work properly, please use the\n' + + ' useDownloadCache option instead. Please see: \n' + + ' https://github.com/UnitedIncome/serverless-python-requirements#caching' + ); + } else { + serverless.cli.log( + '==================================================' + ); + serverless.cli.log( + 'Warning: You are using a deprecated --cache-dir inside\n' + + ' your pipCmdExtraArgs which may not work properly, please use the\n' + + ' useDownloadCache option instead. Please see: \n' + + ' https://github.com/UnitedIncome/serverless-python-requirements#caching' + ); + serverless.cli.log( + '==================================================' + ); + } + } } - serverless.cli.log(`Docker Image: ${dockerImage}`); - - // Prepare bind path depending on os platform - const bindPath = dockerPathForWin( - await getBindPath(serverless, targetFolder) - ); - dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); - if (options.dockerSsh) { - // Mount necessary ssh files to work with private repos - dockerCmd.push( - '-v', - `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`, - '-v', - `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, - '-v', - `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`, - '-e', - 'SSH_AUTH_SOCK=/tmp/ssh_sock' + if (!options.dockerizePip) { + // Push our local OS-specific paths for requirements and target directory + pipCmd.push( + '-t', + dockerPathForWin(targetFolder), + '-r', + dockerPathForWin(targetRequirementsTxt) ); - } + // If we want a download cache... + if (options.useDownloadCache) { + const downloadCacheDir = path.join( + getUserCachePath(options), + 'downloadCacheslspyc' + ); + if (log) { + log.info(`Using download cache directory ${downloadCacheDir}`); + } else { + serverless.cli.log( + `Using download cache directory ${downloadCacheDir}` + ); + } + fse.ensureDirSync(downloadCacheDir); + pipCmd.push('--cache-dir', downloadCacheDir); + } - // If we want a download cache... - const dockerDownloadCacheDir = '/var/useDownloadCache'; - if (options.useDownloadCache) { - const downloadCacheDir = path.join( - getUserCachePath(options), - 'downloadCacheslspyc' - ); - serverless.cli.log(`Using download cache directory ${downloadCacheDir}`); - fse.ensureDirSync(downloadCacheDir); - // This little hack is necessary because getBindPath requires something inside of it to test... - // Ugh, this is so ugly, but someone has to fix getBindPath in some other way (eg: make it use - // its own temp file) - fse.closeSync( - fse.openSync(path.join(downloadCacheDir, 'requirements.txt'), 'w') - ); - const windowsized = await getBindPath(serverless, downloadCacheDir); - // And now push it to a volume mount and to pip... - dockerCmd.push('-v', `${windowsized}:${dockerDownloadCacheDir}:z`); - pipCmd.push('--cache-dir', dockerDownloadCacheDir); + if (pipAcceptsSystem(options.pythonBin)) { + pipCmd.push('--system'); + } } - if (options.dockerEnv) { - // Add environment variables to docker run cmd - options.dockerEnv.forEach(function (item) { - dockerCmd.push('-e', item); - }); - } + // If we are dockerizing pip + if (options.dockerizePip) { + // Push docker-specific paths for requirements and target directory + pipCmd.push('-t', '/var/task/', '-r', '/var/task/requirements.txt'); + + // Build docker image if required + let dockerImage; + if (options.dockerFile) { + let buildDockerImageProgress; + if (progress) { + buildDockerImageProgress = progress.get( + 'python-install-build-docker' + ); + buildDockerImageProgress.update( + `Building custom docker image from ${options.dockerFile}` + ); + } else { + serverless.cli.log( + `Building custom docker image from ${options.dockerFile}...` + ); + } + try { + dockerImage = buildImage( + options.dockerFile, + options.dockerBuildCmdExtraArgs + ); + } finally { + buildDockerImageProgress && buildDockerImageProgress.remove(); + } + } else { + dockerImage = options.dockerImage; + } + if (log) { + log.info(`Docker Image: ${dockerImage}`); + } else { + serverless.cli.log(`Docker Image: ${dockerImage}`); + } - if (process.platform === 'linux') { - // Use same user so requirements folder is not root and so --cache-dir works + // Prepare bind path depending on os platform + const bindPath = dockerPathForWin(getBindPath(serverless, targetFolder)); + + dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); + if (options.dockerSsh) { + // Mount necessary ssh files to work with private repos + dockerCmd.push( + '-v', + `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`, + '-v', + `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, + '-v', + `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`, + '-e', + 'SSH_AUTH_SOCK=/tmp/ssh_sock' + ); + } + + // If we want a download cache... + const dockerDownloadCacheDir = '/var/useDownloadCache'; if (options.useDownloadCache) { - // Set the ownership of the download cache dir to root - pipCmds.unshift(['chown', '-R', '0:0', dockerDownloadCacheDir]); + const downloadCacheDir = path.join( + getUserCachePath(options), + 'downloadCacheslspyc' + ); + if (log) { + log.info(`Using download cache directory ${downloadCacheDir}`); + } else { + serverless.cli.log( + `Using download cache directory ${downloadCacheDir}` + ); + } + fse.ensureDirSync(downloadCacheDir); + // This little hack is necessary because getBindPath requires something inside of it to test... + // Ugh, this is so ugly, but someone has to fix getBindPath in some other way (eg: make it use + // its own temp file) + fse.closeSync( + fse.openSync(path.join(downloadCacheDir, 'requirements.txt'), 'w') + ); + const windowsized = getBindPath(serverless, downloadCacheDir); + // And now push it to a volume mount and to pip... + dockerCmd.push('-v', `${windowsized}:${dockerDownloadCacheDir}:z`); + pipCmd.push('--cache-dir', dockerDownloadCacheDir); } - // Install requirements with pip - // Set the ownership of the current folder to user - pipCmds.push([ - 'chown', - '-R', - `${process.getuid()}:${process.getgid()}`, - '/var/task', - ]); - } else { - // Use same user so --cache-dir works - dockerCmd.push('-u', await getDockerUid(bindPath)); - } - for (let path of options.dockerExtraFiles) { - pipCmds.push(['cp', path, '/var/task/']); - } + if (options.dockerEnv) { + // Add environment variables to docker run cmd + options.dockerEnv.forEach(function (item) { + dockerCmd.push('-e', item); + }); + } - if (process.platform === 'linux') { - if (options.useDownloadCache) { - // Set the ownership of the download cache dir back to user + if (process.platform === 'linux') { + // Use same user so requirements folder is not root and so --cache-dir works + if (options.useDownloadCache) { + // Set the ownership of the download cache dir to root + pipCmds.unshift(['chown', '-R', '0:0', dockerDownloadCacheDir]); + } + // Install requirements with pip + // Set the ownership of the current folder to user pipCmds.push([ 'chown', '-R', `${process.getuid()}:${process.getgid()}`, - dockerDownloadCacheDir, + '/var/task', ]); + } else { + // Use same user so --cache-dir works + dockerCmd.push('-u', getDockerUid(bindPath)); + } + + for (let path of options.dockerExtraFiles) { + pipCmds.push(['cp', path, '/var/task/']); } + + if (process.platform === 'linux') { + if (options.useDownloadCache) { + // Set the ownership of the download cache dir back to user + pipCmds.push([ + 'chown', + '-R', + `${process.getuid()}:${process.getgid()}`, + dockerDownloadCacheDir, + ]); + } + } + + if (Array.isArray(options.dockerRunCmdExtraArgs)) { + dockerCmd.push(...options.dockerRunCmdExtraArgs); + } else { + throw new Error('dockerRunCmdExtraArgs option must be an array'); + } + + dockerCmd.push(dockerImage); } - if (Array.isArray(options.dockerRunCmdExtraArgs)) { - dockerCmd.push(...options.dockerRunCmdExtraArgs); - } else { - throw new Error('dockerRunCmdExtraArgs option must be an array'); + // If enabled slimming, strip so files + switch (getStripMode(options)) { + case 'docker': + pipCmds.push(getStripCommand(options, '/var/task')); + break; + case 'direct': + postCmds.push(getStripCommand(options, dockerPathForWin(targetFolder))); + break; } - dockerCmd.push(dockerImage); - } + let spawnArgs = { shell: true }; + if (process.env.SLS_DEBUG) { + spawnArgs.stdio = 'inherit'; + } + let mainCmds = []; + if (dockerCmd.length) { + dockerCmd.push(...mergeCommands(pipCmds)); + mainCmds = [dockerCmd]; + } else { + mainCmds = pipCmds; + } + mainCmds.push(...postCmds); - // If enabled slimming, strip so files - switch (getStripMode(options)) { - case 'docker': - pipCmds.push(getStripCommand(options, '/var/task')); - break; - case 'direct': - postCmds.push(getStripCommand(options, dockerPathForWin(targetFolder))); - break; - } + if (log) { + log.info(`Running ${quote(dockerCmd)}...`); + } else { + serverless.cli.log(`Running ${quote(dockerCmd)}...`); + } - let spawnArgs = { shell: true }; - if (process.env.SLS_DEBUG) { - spawnArgs.stdio = 'inherit'; - } - let mainCmds = []; - if (dockerCmd.length) { - dockerCmd.push(...mergeCommands(pipCmds)); - mainCmds = [dockerCmd]; - } else { - mainCmds = pipCmds; - } - mainCmds.push(...postCmds); - - serverless.cli.log(`Running ${quote(dockerCmd)}...`); - - for (const [cmd, ...args] of mainCmds) { - try { - await spawn(cmd, args); - } catch (e) { - if ( - e.stderrBuffer && - e.stderrBuffer.toString().includes('command not found') - ) { - const advice = - cmd.indexOf('python') > -1 - ? 'Try the pythonBin option' - : 'Please install it'; - throw new Error(`${cmd} not found! ${advice}`); + for (const [cmd, ...args] of mainCmds) { + try { + await spawn(cmd, args); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + const advice = + cmd.indexOf('python') > -1 + ? 'Try the pythonBin option' + : 'Please install it'; + throw new Error(`${cmd} not found! ${advice}`); + } + throw e; } - throw e; } - } - // If enabled slimming, delete files in slimPatterns - if (options.slim === true || options.slim === 'true') { - deleteFiles(options, targetFolder); + // If enabled slimming, delete files in slimPatterns + if (options.slim === true || options.slim === 'true') { + deleteFiles(options, targetFolder); + } + } finally { + installProgress && installProgress.remove(); } } @@ -392,7 +465,7 @@ function getRequirements(source) { * @param {string} target requirements where results are written * @param {Object} options */ -function filterRequirementsFile(source, target, options, serverless) { +function filterRequirementsFile(source, target, { options, serverless, log }) { const noDeploy = new Set(options.noDeploy || []); const requirements = getRequirements(source); var prepend = []; @@ -414,9 +487,13 @@ function filterRequirementsFile(source, target, options, serverless) { // not required inside final archive and avoids pip bugs // see https://github.com/UnitedIncome/serverless-python-requirements/issues/240 req = req.split('-e')[1].trim(); - serverless.cli.log( - `Warning: Stripping -e flag from requirement ${req}` - ); + if (log) { + log.warning(`Stripping -e flag from requirement ${req}`); + } else { + serverless.cli.log( + `Warning: Stripping -e flag from requirement ${req}` + ); + } } // Keep options for later @@ -444,13 +521,19 @@ function filterRequirementsFile(source, target, options, serverless) { * @param {Object} serverless * @return {undefined} */ -function copyVendors(vendorFolder, targetFolder, serverless) { +function copyVendors(vendorFolder, targetFolder, { serverless, log }) { // Create target folder if it does not exist fse.ensureDirSync(targetFolder); - serverless.cli.log( - `Copying vendor libraries from ${vendorFolder} to ${targetFolder}...` - ); + if (log) { + log.info( + `Copying vendor libraries from ${vendorFolder} to ${targetFolder}` + ); + } else { + serverless.cli.log( + `Copying vendor libraries from ${vendorFolder} to ${targetFolder}...` + ); + } fse.readdirSync(vendorFolder).map((file) => { let source = path.join(vendorFolder, file); @@ -500,12 +583,11 @@ function requirementsFileExists(servicePath, options, fileName) { * @return {string} */ async function installRequirementsIfNeeded( - servicePath, modulePath, - options, funcOptions, - serverless + pluginInstance ) { + const { servicePath, options, serverless } = pluginInstance; // Our source requirements, under our service path, and our module path (if specified) const fileName = path.join(servicePath, modulePath, options.fileName); @@ -528,19 +610,19 @@ async function installRequirementsIfNeeded( fse.ensureDirSync(requirementsTxtDirectory); const slsReqsTxt = path.join(requirementsTxtDirectory, 'requirements.txt'); - generateRequirementsFile( - fileName, - slsReqsTxt, - serverless, - servicePath, - options - ); + generateRequirementsFile(fileName, slsReqsTxt, pluginInstance); // If no requirements file or an empty requirements file, then do nothing if (!fse.existsSync(slsReqsTxt) || fse.statSync(slsReqsTxt).size == 0) { - serverless.cli.log( - `Skipping empty output requirements.txt file from ${slsReqsTxt}` - ); + if (pluginInstance.log) { + pluginInstance.log.info( + `Skipping empty output requirements.txt file from ${slsReqsTxt}` + ); + } else { + serverless.cli.log( + `Skipping empty output requirements.txt file from ${slsReqsTxt}` + ); + } return false; } @@ -560,9 +642,15 @@ async function installRequirementsIfNeeded( fse.existsSync(path.join(workingReqsFolder, '.completed_requirements')) && workingReqsFolder.endsWith('_slspyc') ) { - serverless.cli.log( - `Using static cache of requirements found at ${workingReqsFolder} ...` - ); + if (pluginInstance.log) { + pluginInstance.log.info( + `Using static cache of requirements found at ${workingReqsFolder}` + ); + } else { + serverless.cli.log( + `Using static cache of requirements found at ${workingReqsFolder} ...` + ); + } // We'll "touch" the folder, as to bring it to the start of the FIFO cache fse.utimesSync(workingReqsFolder, new Date(), new Date()); return workingReqsFolder; @@ -583,14 +671,14 @@ async function installRequirementsIfNeeded( fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt')); // Then install our requirements from this folder - await installRequirements(workingReqsFolder, serverless, options); + await installRequirements(workingReqsFolder, pluginInstance); // Copy vendor libraries to requirements folder if (options.vendor) { - copyVendors(options.vendor, workingReqsFolder, serverless); + copyVendors(options.vendor, workingReqsFolder, pluginInstance); } if (funcOptions.vendor) { - copyVendors(funcOptions.vendor, workingReqsFolder, serverless); + copyVendors(funcOptions.vendor, workingReqsFolder, pluginInstance); } // Then touch our ".completed_requirements" file so we know we can use this for static cache @@ -609,7 +697,7 @@ async function installRequirementsIfNeeded( async function installAllRequirements() { // fse.ensureDirSync(path.join(this.servicePath, '.serverless')); // First, check and delete cache versions, if enabled - checkForAndDeleteMaxCacheVersions(this.options, this.serverless); + checkForAndDeleteMaxCacheVersions(this); // Then if we're going to package functions individually... if (this.serverless.service.package.individually) { @@ -663,13 +751,7 @@ async function installAllRequirements() { } } } else { - const reqsInstalledAt = await installRequirementsIfNeeded( - this.servicePath, - '', - this.options, - {}, - this.serverless - ); + const reqsInstalledAt = await installRequirementsIfNeeded('', {}, this); // Add symlinks into .serverless for so it's easier for injecting and for users to see where reqs are let symlinkPath = path.join( this.servicePath, From f43acea2a5ee207874f4b5120a95a6b0164bd405 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 13:23:49 +0100 Subject: [PATCH 09/27] refactor: Adapt `layer` to modern logs --- lib/layer.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 12d338ec..ddc90f6a 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -53,9 +53,24 @@ function layerRequirements() { return BbPromise.resolve(); } - this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); + let layerProgress; + if (this.progress) { + layerProgress = this.progress.get('python-layer-requirements'); + layerProgress.update('Packaging Python Requirements Lambda Layer', { + isMainEvent: true, + }); + } else { + this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); + } - return BbPromise.bind(this).then(zipRequirements).then(createLayers); + return BbPromise.bind(this) + .then(zipRequirements) + .then(createLayers) + .then(() => layerProgress && layerProgress.remove()) + .catch((e) => { + layerProgress && layerProgress.remove(); + throw e; + }); } module.exports = { From cbd7e9c4ecb335457ed59b6d4942636b0639a53a Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 13:34:52 +0100 Subject: [PATCH 10/27] refactor: Adapt `inject` to modern logs --- lib/inject.js | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/inject.js b/lib/inject.js index 3cad758d..2d1bdc2b 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -92,10 +92,19 @@ function injectAllRequirements(funcArtifact) { return BbPromise.resolve(); } - this.serverless.cli.log('Injecting required Python packages to package...'); + let injectProgress; + if (this.progress) { + injectProgress = this.progress.get('python-inject-requirements'); + injectProgress.update('Injecting required Python packages to package', { + isMainEvent: true, + }); + } else { + this.serverless.cli.log('Injecting required Python packages to package...'); + } + let returnPromise; if (this.serverless.service.package.individually) { - return BbPromise.resolve(this.targetFuncs) + returnPromise = BbPromise.resolve(this.targetFuncs) .filter((func) => (func.runtime || this.serverless.service.provider.runtime).match( /^python.*/ @@ -132,12 +141,19 @@ function injectAllRequirements(funcArtifact) { ); }); } else if (!this.options.zip) { - return injectRequirements( + returnPromise = injectRequirements( path.join('.serverless', 'requirements'), this.serverless.service.package.artifact || funcArtifact, this.options ); } + + return returnPromise + .then(() => injectProgress && injectProgress.remove()) + .catch((e) => { + injectProgress && injectProgress.remove(); + throw e; + }); } module.exports = { injectAllRequirements }; From d70ca215eb8d0644697aed3d9515755c89c701e1 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 23 Nov 2021 13:43:51 +0100 Subject: [PATCH 11/27] refactor: Adapt `docker` for modern logs --- index.js | 2 +- lib/docker.js | 34 ++++++++++++++++++++++++++-------- lib/inject.js | 15 +++++++++------ lib/pip.js | 26 +++++++++++--------------- 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/index.js b/index.js index 26616295..cf2af38e 100644 --- a/index.js +++ b/index.js @@ -137,7 +137,7 @@ class ServerlessPythonRequirements { this.log = v3Utils.log; this.progress = v3Utils.progress; this.writeText = v3Utils.writeText; - }; + } this.commands = { requirements: { diff --git a/lib/docker.js b/lib/docker.js index 94229b21..9da6da6a 100644 --- a/lib/docker.js +++ b/lib/docker.js @@ -73,7 +73,7 @@ function findTestFile(servicePath) { * @param {string} bindPath * @return {boolean} */ -async function tryBindPath(serverless, bindPath, testFile) { +async function tryBindPath(bindPath, testFile, { serverless, log }) { const debug = process.env.SLS_DEBUG; const options = [ 'run', @@ -85,12 +85,30 @@ async function tryBindPath(serverless, bindPath, testFile) { `/test/${testFile}`, ]; try { - if (debug) serverless.cli.log(`Trying bindPath ${bindPath} (${options})`); + if (debug) { + if (log) { + log.debug(`Trying bindPath ${bindPath} (${options})`); + } else { + serverless.cli.log(`Trying bindPath ${bindPath} (${options})`); + } + } const ps = await dockerCommand(options); - if (debug) serverless.cli.log(ps.stdout.trim()); - return ps.stdout.trim() === `/test/${testFile}`; + if (debug) { + if (log) { + log.debug(ps.stdoutBuffer.trim()); + } else { + serverless.cli.log(ps.stdoutBuffer.trim()); + } + } + return ps.stdoutBuffer.trim() === `/test/${testFile}`; } catch (err) { - if (debug) serverless.cli.log(`Finding bindPath failed with ${err}`); + if (debug) { + if (log) { + log.debug(`Finding bindPath failed with ${err}`); + } else { + serverless.cli.log(`Finding bindPath failed with ${err}`); + } + } return false; } } @@ -101,7 +119,7 @@ async function tryBindPath(serverless, bindPath, testFile) { * @param {string} servicePath * @return {string} The bind path. */ -async function getBindPath(serverless, servicePath) { +async function getBindPath(servicePath, pluginInstance) { // Determine bind path if (process.platform !== 'win32' && !isWsl) { return servicePath; @@ -145,7 +163,7 @@ async function getBindPath(serverless, servicePath) { for (let i = 0; i < bindPaths.length; i++) { const bindPath = bindPaths[i]; - if (await tryBindPath(serverless, bindPath, testFile)) { + if (await tryBindPath(bindPath, testFile, pluginInstance)) { return bindPath; } } @@ -171,7 +189,7 @@ async function getDockerUid(bindPath) { '/bin/sh', ]; const ps = await dockerCommand(options); - return ps.stdout.trim(); + return ps.stdoutBuffer.trim(); } module.exports = { buildImage, getBindPath, getDockerUid }; diff --git a/lib/inject.js b/lib/inject.js index 2d1bdc2b..85bdf597 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -148,12 +148,15 @@ function injectAllRequirements(funcArtifact) { ); } - return returnPromise - .then(() => injectProgress && injectProgress.remove()) - .catch((e) => { - injectProgress && injectProgress.remove(); - throw e; - }); + return ( + returnPromise && + returnPromise + .then(() => injectProgress && injectProgress.remove()) + .catch((e) => { + injectProgress && injectProgress.remove(); + throw e; + }) + ); } module.exports = { injectAllRequirements }; diff --git a/lib/pip.js b/lib/pip.js index 89e8e74a..d88f5c7d 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -133,19 +133,15 @@ async function pipAcceptsSystem(pythonBin) { * @param {Object} options * @return {undefined} */ -async function installRequirements( - targetFolder, - { options, serverless, log, progress } -) { +async function installRequirements(targetFolder, pluginInstance) { + const { options, serverless, log, progress } = pluginInstance; const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt'); let installProgress; if (progress) { + log.info(`Installing requirements from "${targetRequirementsTxt}"`); installProgress = progress.get('python-install'); - installProgress.update( - `Installing requirements from "${targetRequirementsTxt}"`, - { isMainEvent: true } - ); + installProgress.update('Installing requirements'); } else { serverless.cli.log( `Installing requirements from ${targetRequirementsTxt} ...` @@ -226,7 +222,7 @@ async function installRequirements( pipCmd.push('--cache-dir', downloadCacheDir); } - if (pipAcceptsSystem(options.pythonBin)) { + if (await pipAcceptsSystem(options.pythonBin)) { pipCmd.push('--system'); } } @@ -253,7 +249,7 @@ async function installRequirements( ); } try { - dockerImage = buildImage( + dockerImage = await buildImage( options.dockerFile, options.dockerBuildCmdExtraArgs ); @@ -270,7 +266,9 @@ async function installRequirements( } // Prepare bind path depending on os platform - const bindPath = dockerPathForWin(getBindPath(serverless, targetFolder)); + const bindPath = dockerPathForWin( + await getBindPath(targetFolder, pluginInstance) + ); dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { @@ -308,7 +306,7 @@ async function installRequirements( fse.closeSync( fse.openSync(path.join(downloadCacheDir, 'requirements.txt'), 'w') ); - const windowsized = getBindPath(serverless, downloadCacheDir); + const windowsized = await getBindPath(downloadCacheDir, pluginInstance); // And now push it to a volume mount and to pip... dockerCmd.push('-v', `${windowsized}:${dockerDownloadCacheDir}:z`); pipCmd.push('--cache-dir', dockerDownloadCacheDir); @@ -337,7 +335,7 @@ async function installRequirements( ]); } else { // Use same user so --cache-dir works - dockerCmd.push('-u', getDockerUid(bindPath)); + dockerCmd.push('-u', await getDockerUid(bindPath)); } for (let path of options.dockerExtraFiles) { @@ -716,9 +714,7 @@ async function installAllRequirements() { // If we didn't already process a module (functions can re-use modules) if (!doneModules.includes(f.module)) { const reqsInstalledAt = await installRequirementsIfNeeded( - this.servicePath, f.module, - this.options, f, this.serverless ); From 44b9591f01157a1811e3ca8b43e21265a155a976 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Wed, 24 Nov 2021 21:44:40 +0100 Subject: [PATCH 12/27] refactor: Ensure proper verbose progress logs --- lib/clean.js | 7 +++---- lib/inject.js | 7 +++---- lib/layer.js | 7 +++---- lib/pip.js | 2 +- lib/pipenv.js | 6 +++--- lib/poetry.js | 6 +++--- lib/zip.js | 6 +++--- 7 files changed, 19 insertions(+), 22 deletions(-) diff --git a/lib/clean.js b/lib/clean.js index 88d7d03e..e972f567 100644 --- a/lib/clean.js +++ b/lib/clean.js @@ -40,11 +40,10 @@ function cleanupCache() { if (fse.existsSync(cacheLocation)) { let cleanupProgress; if (this.serverless) { - if (this.progress) { + if (this.progress && this.log) { cleanupProgress = this.progress.get('python-cleanup-cache'); - cleanupProgress.notice(`Removing static caches at: ${cacheLocation}`, { - isMainEvent: true, - }); + cleanupProgress.notice('Removing static caches'); + this.log.info(`Removing static caches at: ${cacheLocation}`); } else { this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`); } diff --git a/lib/inject.js b/lib/inject.js index 85bdf597..9f3ad77a 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -93,11 +93,10 @@ function injectAllRequirements(funcArtifact) { } let injectProgress; - if (this.progress) { + if (this.progress && this.log) { injectProgress = this.progress.get('python-inject-requirements'); - injectProgress.update('Injecting required Python packages to package', { - isMainEvent: true, - }); + injectProgress.update('Injecting required Python packages to package'); + this.log.info('Injecting required Python packages to package'); } else { this.serverless.cli.log('Injecting required Python packages to package...'); } diff --git a/lib/layer.js b/lib/layer.js index ddc90f6a..141d1fd7 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -54,11 +54,10 @@ function layerRequirements() { } let layerProgress; - if (this.progress) { + if (this.progress && this.log) { layerProgress = this.progress.get('python-layer-requirements'); - layerProgress.update('Packaging Python Requirements Lambda Layer', { - isMainEvent: true, - }); + layerProgress.update('Packaging Python Requirements Lambda Layer'); + this.log.info('Packaging Python Requirements Lambda Layer'); } else { this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); } diff --git a/lib/pip.js b/lib/pip.js index d88f5c7d..2f6d4571 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -716,7 +716,7 @@ async function installAllRequirements() { const reqsInstalledAt = await installRequirementsIfNeeded( f.module, f, - this.serverless + this ); // Add modulePath into .serverless for each module so it's easier for injecting and for users to see where reqs are let modulePath = path.join( diff --git a/lib/pipenv.js b/lib/pipenv.js index 5100a810..4949e924 100644 --- a/lib/pipenv.js +++ b/lib/pipenv.js @@ -15,14 +15,14 @@ async function pipfileToRequirements() { } let generateRequirementsProgress; - if (this.progress) { + if (this.progress && this.log) { generateRequirementsProgress = this.progress.get( 'python-generate-requirements-pipfile' ); generateRequirementsProgress.update( - 'Generating requirements.txt from Pipfile', - { isMainEvent: true } + 'Generating requirements.txt from Pipfile' ); + this.log.info('Generating requirements.txt from Pipfile'); } else { this.serverless.cli.log('Generating requirements.txt from Pipfile...'); } diff --git a/lib/poetry.js b/lib/poetry.js index 81988742..12904fd9 100644 --- a/lib/poetry.js +++ b/lib/poetry.js @@ -14,14 +14,14 @@ async function pyprojectTomlToRequirements() { } let generateRequirementsProgress; - if (this.progress) { + if (this.progress && this.log) { generateRequirementsProgress = this.progress.get( 'python-generate-requirements-toml' ); generateRequirementsProgress.update( - 'Generating requirements.txt from "pyproject.toml"', - { isMainEvent: true } + 'Generating requirements.txt from "pyproject.toml"' ); + this.log.info('Generating requirements.txt from "pyproject.toml"'); } else { this.serverless.cli.log( 'Generating requirements.txt from pyproject.toml...' diff --git a/lib/zip.js b/lib/zip.js index de61ce0f..aabb2333 100644 --- a/lib/zip.js +++ b/lib/zip.js @@ -123,14 +123,14 @@ function packRequirements() { .then((funcs) => uniqBy(funcs, (f) => f.module)) .map((f) => { let packProgress; - if (this.progress) { + if (this.progress && this.log) { packProgress = this.progress.get( `python-pack-requirements-${f.module}` ); packProgress.update( - `Zipping required Python packages for ${f.module}`, - { isMainEvent: true } + `Zipping required Python packages for ${f.module}` ); + this.log.info(`Zipping required Python packages for ${f.module}`); } else { this.serverless.cli.log( `Zipping required Python packages for ${f.module}...` From 9479a90b1d262f55a6808a9d12c478f220258da9 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Thu, 25 Nov 2021 11:49:11 +0100 Subject: [PATCH 13/27] refactor: Cleanup and use `finally` for code simplification --- lib/clean.js | 11 +++--- lib/inject.js | 99 ++++++++++++++++++++++++--------------------------- lib/layer.js | 6 +--- lib/zip.js | 12 ++----- 4 files changed, 54 insertions(+), 74 deletions(-) diff --git a/lib/clean.js b/lib/clean.js index e972f567..8aaf331e 100644 --- a/lib/clean.js +++ b/lib/clean.js @@ -40,7 +40,7 @@ function cleanupCache() { if (fse.existsSync(cacheLocation)) { let cleanupProgress; if (this.serverless) { - if (this.progress && this.log) { + if (this.log) { cleanupProgress = this.progress.get('python-cleanup-cache'); cleanupProgress.notice('Removing static caches'); this.log.info(`Removing static caches at: ${cacheLocation}`); @@ -57,12 +57,9 @@ function cleanupCache() { .forEach((file) => { promises.push(fse.removeAsync(file)); }); - return BbPromise.all(promises) - .then(() => cleanupProgress && cleanupProgress.remove()) - .catch((e) => { - cleanupProgress && cleanupProgress.remove(); - throw e; - }); + return BbPromise.all(promises).finally( + () => cleanupProgress && cleanupProgress.remove() + ); } else { if (this.serverless) { if (this.log) { diff --git a/lib/inject.js b/lib/inject.js index 9f3ad77a..f32c9d46 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -86,7 +86,7 @@ function moveModuleUp(source, target, module) { * Inject requirements into packaged application. * @return {Promise} the combined promise for requirements injection. */ -function injectAllRequirements(funcArtifact) { +async function injectAllRequirements(funcArtifact) { if (this.options.layer) { // The requirements will be placed in a Layer, so just resolve return BbPromise.resolve(); @@ -101,61 +101,56 @@ function injectAllRequirements(funcArtifact) { this.serverless.cli.log('Injecting required Python packages to package...'); } - let returnPromise; - if (this.serverless.service.package.individually) { - returnPromise = BbPromise.resolve(this.targetFuncs) - .filter((func) => - (func.runtime || this.serverless.service.provider.runtime).match( - /^python.*/ + try { + if (this.serverless.service.package.individually) { + await BbPromise.resolve(this.targetFuncs) + .filter((func) => + (func.runtime || this.serverless.service.provider.runtime).match( + /^python.*/ + ) ) - ) - .map((func) => { - if (!get(func, 'module')) { - set(func, ['module'], '.'); - } - return func; - }) - .map((func) => { - if (func.module !== '.') { - const artifact = func.package ? func.package.artifact : funcArtifact; - const newArtifact = path.join( - '.serverless', - `${func.module}-${func.name}.zip` - ); - func.package.artifact = newArtifact; - return moveModuleUp(artifact, newArtifact, func.module).then( - () => func - ); - } else { + .map((func) => { + if (!get(func, 'module')) { + set(func, ['module'], '.'); + } return func; - } - }) - .map((func) => { - return this.options.zip - ? func - : injectRequirements( - path.join('.serverless', func.module, 'requirements'), - func.package.artifact, - this.options + }) + .map((func) => { + if (func.module !== '.') { + const artifact = func.package + ? func.package.artifact + : funcArtifact; + const newArtifact = path.join( + '.serverless', + `${func.module}-${func.name}.zip` ); - }); - } else if (!this.options.zip) { - returnPromise = injectRequirements( - path.join('.serverless', 'requirements'), - this.serverless.service.package.artifact || funcArtifact, - this.options - ); + func.package.artifact = newArtifact; + return moveModuleUp(artifact, newArtifact, func.module).then( + () => func + ); + } else { + return func; + } + }) + .map((func) => { + return this.options.zip + ? func + : injectRequirements( + path.join('.serverless', func.module, 'requirements'), + func.package.artifact, + this.options + ); + }); + } else if (!this.options.zip) { + await injectRequirements( + path.join('.serverless', 'requirements'), + this.serverless.service.package.artifact || funcArtifact, + this.options + ); + } + } finally { + injectProgress && injectProgress.remove(); } - - return ( - returnPromise && - returnPromise - .then(() => injectProgress && injectProgress.remove()) - .catch((e) => { - injectProgress && injectProgress.remove(); - throw e; - }) - ); } module.exports = { injectAllRequirements }; diff --git a/lib/layer.js b/lib/layer.js index 141d1fd7..fe2a4a00 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -65,11 +65,7 @@ function layerRequirements() { return BbPromise.bind(this) .then(zipRequirements) .then(createLayers) - .then(() => layerProgress && layerProgress.remove()) - .catch((e) => { - layerProgress && layerProgress.remove(); - throw e; - }); + .finally(() => layerProgress && layerProgress.remove()); } module.exports = { diff --git a/lib/zip.js b/lib/zip.js index aabb2333..cba29450 100644 --- a/lib/zip.js +++ b/lib/zip.js @@ -139,11 +139,7 @@ function packRequirements() { f.package.patterns.push(`${f.module}/.requirements.zip`); return addTree(new JSZip(), `.serverless/${f.module}/requirements`) .then((zip) => writeZip(zip, `${f.module}/.requirements.zip`)) - .then(() => packProgress && packProgress.remove()) - .catch((e) => { - packProgress && packProgress.remove(); - throw e; - }); + .finally(() => packProgress && packProgress.remove()); }); } else { let packProgress; @@ -157,11 +153,7 @@ function packRequirements() { .then((zip) => writeZip(zip, path.join(this.servicePath, '.requirements.zip')) ) - .then(() => packProgress && packProgress.remove()) - .catch((e) => { - packProgress && packProgress.remove(); - throw e; - }); + .finally(() => packProgress && packProgress.remove()); } } } From cdb71110bc9c69b5087b6e18fb353d65962afe4a Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 12:16:27 +0100 Subject: [PATCH 14/27] refactor: Use `ServerlessError` in `docker` --- lib/docker.js | 36 ++++++++++++++++++++++-------------- lib/pip.js | 5 +++-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/docker.js b/lib/docker.js index 9da6da6a..5157803f 100644 --- a/lib/docker.js +++ b/lib/docker.js @@ -8,7 +8,7 @@ const path = require('path'); * @param {string[]} options * @return {Object} */ -async function dockerCommand(options) { +async function dockerCommand(options, pluginInstance) { const cmd = 'docker'; try { return await spawn(cmd, options, { encoding: 'utf-8' }); @@ -17,7 +17,10 @@ async function dockerCommand(options) { e.stderrBuffer && e.stderrBuffer.toString().includes('command not found') ) { - throw new Error('docker not found! Please install it.'); + throw new pluginInstance.serverless.classes.Error( + 'docker not found! Please install it.', + 'PYTHON_REQUIREMENTS_DOCKER_NOT_FOUND' + ); } throw e; } @@ -29,19 +32,22 @@ async function dockerCommand(options) { * @param {string[]} extraArgs * @return {string} The name of the built docker image. */ -async function buildImage(dockerFile, extraArgs) { +async function buildImage(dockerFile, extraArgs, pluginInstance) { const imageName = 'sls-py-reqs-custom'; const options = ['build', '-f', dockerFile, '-t', imageName]; if (Array.isArray(extraArgs)) { options.push(...extraArgs); } else { - throw new Error('dockerRunCmdExtraArgs option must be an array'); + throw new pluginInstance.serverless.classes.Error( + 'dockerRunCmdExtraArgs option must be an array', + 'PYTHON_REQUIREMENTS_INVALID_DOCKER_EXTRA_ARGS' + ); } options.push('.'); - await dockerCommand(options); + await dockerCommand(options, pluginInstance); return imageName; } @@ -50,7 +56,7 @@ async function buildImage(dockerFile, extraArgs) { * @param {string} servicePath * @return {string} file name */ -function findTestFile(servicePath) { +function findTestFile(servicePath, pluginInstance) { if (fse.pathExistsSync(path.join(servicePath, 'serverless.yml'))) { return 'serverless.yml'; } @@ -63,8 +69,9 @@ function findTestFile(servicePath) { if (fse.pathExistsSync(path.join(servicePath, 'requirements.txt'))) { return 'requirements.txt'; } - throw new Error( - 'Unable to find serverless.{yml|yaml|json} or requirements.txt for getBindPath()' + throw new pluginInstance.serverless.classes.Error( + 'Unable to find serverless.{yml|yaml|json} or requirements.txt for getBindPath()', + 'PYTHON_REQUIREMENTS_MISSING_GET_BIND_PATH_FILE' ); } @@ -73,7 +80,8 @@ function findTestFile(servicePath) { * @param {string} bindPath * @return {boolean} */ -async function tryBindPath(bindPath, testFile, { serverless, log }) { +async function tryBindPath(bindPath, testFile, pluginInstance) { + const { serverless, log } = pluginInstance; const debug = process.env.SLS_DEBUG; const options = [ 'run', @@ -92,7 +100,7 @@ async function tryBindPath(bindPath, testFile, { serverless, log }) { serverless.cli.log(`Trying bindPath ${bindPath} (${options})`); } } - const ps = await dockerCommand(options); + const ps = await dockerCommand(options, pluginInstance); if (debug) { if (log) { log.debug(ps.stdoutBuffer.trim()); @@ -126,7 +134,7 @@ async function getBindPath(servicePath, pluginInstance) { } // test docker is available - await dockerCommand(['version']); + await dockerCommand(['version'], pluginInstance); // find good bind path for Windows let bindPaths = []; @@ -159,7 +167,7 @@ async function getBindPath(servicePath, pluginInstance) { bindPaths.push(`/mnt/${drive.toUpperCase()}/${path}`); bindPaths.push(`${drive.toUpperCase()}:/${path}`); - const testFile = findTestFile(servicePath); + const testFile = findTestFile(servicePath, pluginInstance); for (let i = 0; i < bindPaths.length; i++) { const bindPath = bindPaths[i]; @@ -176,7 +184,7 @@ async function getBindPath(servicePath, pluginInstance) { * @param {string} bindPath * @return {boolean} */ -async function getDockerUid(bindPath) { +async function getDockerUid(bindPath, pluginInstance) { const options = [ 'run', '--rm', @@ -188,7 +196,7 @@ async function getDockerUid(bindPath) { '%u', '/bin/sh', ]; - const ps = await dockerCommand(options); + const ps = await dockerCommand(options, pluginInstance); return ps.stdoutBuffer.trim(); } diff --git a/lib/pip.js b/lib/pip.js index 2f6d4571..24d4c55f 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -251,7 +251,8 @@ async function installRequirements(targetFolder, pluginInstance) { try { dockerImage = await buildImage( options.dockerFile, - options.dockerBuildCmdExtraArgs + options.dockerBuildCmdExtraArgs, + pluginInstance ); } finally { buildDockerImageProgress && buildDockerImageProgress.remove(); @@ -335,7 +336,7 @@ async function installRequirements(targetFolder, pluginInstance) { ]); } else { // Use same user so --cache-dir works - dockerCmd.push('-u', await getDockerUid(bindPath)); + dockerCmd.push('-u', await getDockerUid(bindPath, pluginInstance)); } for (let path of options.dockerExtraFiles) { From 395082761ae574c2664f1c272ea4970cfa3fd1f7 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 12:22:22 +0100 Subject: [PATCH 15/27] refactor: Use `ServerlessError` in `poetry` --- lib/poetry.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/poetry.js b/lib/poetry.js index 12904fd9..23f43dc0 100644 --- a/lib/poetry.js +++ b/lib/poetry.js @@ -50,8 +50,9 @@ async function pyprojectTomlToRequirements() { e.stderrBuffer && e.stderrBuffer.toString().includes('command not found') ) { - throw new Error( - `poetry not found! Install it according to the poetry docs.` + throw new this.serverless.classes.Error( + `poetry not found! Install it according to the poetry docs.`, + 'PYTHON_REQUIREMENTS_POETRY_NOT_FOUND' ); } throw e; From 618ef76c830349f34f5f8414dc6e381b167cfa43 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 12:23:04 +0100 Subject: [PATCH 16/27] refactor: Use `ServerlessError` in `pipenv` --- lib/pipenv.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pipenv.js b/lib/pipenv.js index 4949e924..5856d47b 100644 --- a/lib/pipenv.js +++ b/lib/pipenv.js @@ -42,8 +42,9 @@ async function pipfileToRequirements() { e.stderrBuffer && e.stderrBuffer.toString().includes('command not found') ) { - throw new Error( - `pipenv not found! Install it with 'pip install pipenv'.` + throw new this.serverless.classes.Error( + `pipenv not found! Install it according to the poetry docs.`, + 'PYTHON_REQUIREMENTS_PIPENV_NOT_FOUND' ); } throw e; From 8a4bc83025b41acd7f6e88982cd72af24f8d9967 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 12:28:02 +0100 Subject: [PATCH 17/27] refactor: Use `ServerlessError` in `pip` --- lib/pip.js | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/lib/pip.js b/lib/pip.js index 24d4c55f..ce348532 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -107,7 +107,7 @@ function generateRequirementsFile( } } -async function pipAcceptsSystem(pythonBin) { +async function pipAcceptsSystem(pythonBin, pluginInstance) { // Check if pip has Debian's --system option and set it if so try { const pipTestRes = await spawn(pythonBin, ['-m', 'pip', 'help', 'install']); @@ -120,7 +120,10 @@ async function pipAcceptsSystem(pythonBin) { e.stderrBuffer && e.stderrBuffer.toString().includes('command not found') ) { - throw new Error(`${pythonBin} not found! Try the pythonBin option.`); + throw new pluginInstance.serverless.classes.Error( + `${pythonBin} not found! Install it according to the poetry docs.`, + 'PYTHON_REQUIREMENTS_PYTHON_NOT_FOUND' + ); } throw e; } @@ -167,10 +170,9 @@ async function installRequirements(targetFolder, pluginInstance) { // Check if we're using the legacy --cache-dir command... if (options.pipCmdExtraArgs.indexOf('--cache-dir') > -1) { if (options.dockerizePip) { - throw ( - 'Error: You can not use --cache-dir with Docker any more, please\n' + - ' use the new option useDownloadCache instead. Please see:\n' + - ' https://github.com/UnitedIncome/serverless-python-requirements#caching' + throw new pluginInstance.serverless.classes.Error( + 'You cannot use --cache-dir with Docker any more, please use the new option useDownloadCache instead. Please see: https://github.com/UnitedIncome/serverless-python-requirements#caching for more details.', + 'PYTHON_REQUIREMENTS_CACHE_DIR_DOCKER_INVALID' ); } else { if (log) { @@ -222,7 +224,7 @@ async function installRequirements(targetFolder, pluginInstance) { pipCmd.push('--cache-dir', downloadCacheDir); } - if (await pipAcceptsSystem(options.pythonBin)) { + if (await pipAcceptsSystem(options.pythonBin, pluginInstance)) { pipCmd.push('--system'); } } @@ -358,7 +360,10 @@ async function installRequirements(targetFolder, pluginInstance) { if (Array.isArray(options.dockerRunCmdExtraArgs)) { dockerCmd.push(...options.dockerRunCmdExtraArgs); } else { - throw new Error('dockerRunCmdExtraArgs option must be an array'); + throw new pluginInstance.serverless.classes.Error( + 'dockerRunCmdExtraArgs option must be an array', + 'PYTHON_REQUIREMENTS_INVALID_DOCKER_EXTRA_ARGS' + ); } dockerCmd.push(dockerImage); @@ -405,7 +410,10 @@ async function installRequirements(targetFolder, pluginInstance) { cmd.indexOf('python') > -1 ? 'Try the pythonBin option' : 'Please install it'; - throw new Error(`${cmd} not found! ${advice}`); + throw new pluginInstance.serverless.classes.Error( + `${cmd} not found! ${advice}`, + 'PYTHON_REQUIREMENTS_COMMAND_NOT_FOUND' + ); } throw e; } From 1f0804cfa95e85375b5a075188b1fee1fcdcb42e Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 13:05:20 +0100 Subject: [PATCH 18/27] ci: Update validate CI workflow --- .github/workflows/lint.yml | 20 --- .github/workflows/test.yml | 54 -------- .github/workflows/validate.yml | 240 +++++++++++++++++++++++++++++++++ package.json | 11 +- 4 files changed, 248 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/validate.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 1e6b9ee8..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Lint - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Node ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: 14 - - - name: Install deps - run: npm install - - - name: Lint - run: npm run ci:lint diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index f89486bf..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Test - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - python-version: [2.7, 3.6] - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up Node - uses: actions/setup-node@v1 - with: - node-version: 14 - - - name: Check python version - run: | - python --version - - - name: Install setuptools - run: python -m pip install --force setuptools wheel - - - name: Install pipenv / poetry - run: python -m pip install pipenv poetry - - - name: Install serverless - run: npm install -g serverless@2 - - - name: Install deps - run: npm install - - - name: Test - run: npm run test - env: - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - if: matrix.os != 'macOS-latest' - - - name: Test (Mac) - run: npm run test - env: - LC_ALL: en_US.UTF-8 - LANG: en_US.UTF-8 - if: matrix.os == 'macOS-latest' diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..3a9af5c4 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,240 @@ +# PR's only + +name: Validate + +on: + pull_request: + branches: [master] + +env: + FORCE_COLOR: 1 + +jobs: + linuxNode16: + name: '[Linux] Node.js v16: Lint, Formatting & Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # For commitlint purpose ensure to have complete list of PR commits + # It's loose and imperfect assumption that PR has no more than 30 commits + fetch-depth: 30 + + - name: Retrieve last master commit (for `git diff` purposes) + run: | + git checkout -b pr + git fetch --prune --depth=30 origin +refs/heads/master:refs/remotes/origin/master + git checkout master + git checkout pr + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: | + npm-v16-${{ runner.os }}-${{ github.ref }}- + npm-v16-${{ runner.os }}-refs/heads/master- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 16.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Validate Prettier formatting + run: npm run prettier-check:updated + - name: Validate ESLint rules + run: npm run lint:updated + - name: Unit tests + run: script -e -c "npm test" + + windowsNode16: + name: '[Windows] Node.js v16: Unit tests' + runs-on: windows-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: | + npm-v16-${{ runner.os }}-${{ github.ref }}- + npm-v16-${{ runner.os }}-refs/heads/master- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 16.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: npm test + + linuxNode14: + name: '[Linux] Node.js 14: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: | + npm-v14-${{ runner.os }}-${{ github.ref }}- + npm-v14-${{ runner.os }}-refs/heads/master- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + # Some tests depend on TTY support, which is missing in GA runner + # Workaround taken from https://github.com/actions/runner/issues/241#issuecomment-577360161 + run: script -e -c "npm test" + + linuxNode12: + name: '[Linux] Node.js v12: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v12-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: | + npm-v12-${{ runner.os }}-${{ github.ref }}- + npm-v12-${{ runner.os }}-refs/heads/master- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: script -e -c "npm test" diff --git a/package.json b/package.json index c9d247a3..9d1f5852 100644 --- a/package.json +++ b/package.json @@ -38,14 +38,19 @@ "main": "index.js", "bin": {}, "scripts": { - "ci:lint": "eslint *.js lib/*.js --format junit --output-file ~/reports/eslint.xml && prettier -c '{.,lib}/*.{js,md}'", - "test": "node test.js", + "format": "prettier --write '{.,lib}/*.{js,md}'", "lint": "eslint *.js lib/*.js && prettier -c '{.,lib}/*.{js,md}'", - "format": "prettier --write '{.,lib}/*.{js,md}'" + "lint:updated": "pipe-git-updated --ext=js -- eslint", + "prettier-check": "prettier -c --ignore-path .gitignore \"**/*.{css,html,js,json,md,yaml,yml}\"", + "prettier-check:updated": "pipe-git-updated --ext=css --ext=html --ext=js --ext=json --ext=md --ext=yaml --ext=yml -- prettier -c", + "prettify": "prettier --write --ignore-path .gitignore \"**/*.{css,html,js,json,md,yaml,yml}\"", + "prettify:updated": "pipe-git-updated --ext=css --ext=html --ext=js --ext=json --ext=md --ext=yaml --ext=yml -- prettier --write", + "test": "node test.js" }, "devDependencies": { "cross-spawn": "*", "eslint": "^7.32.0", + "git-list-updated": "^1.2.1", "lodash": "^4.17.21", "prettier": "^2", "tape": "*", From 080b0ba4e834e4ba1d60db243d5dc9962fa9c9f0 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 13:57:45 +0100 Subject: [PATCH 19/27] ci: Introduce integrate CI workflow --- .github/workflows/integrate.yml | 239 ++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 .github/workflows/integrate.yml diff --git a/.github/workflows/integrate.yml b/.github/workflows/integrate.yml new file mode 100644 index 00000000..d241971c --- /dev/null +++ b/.github/workflows/integrate.yml @@ -0,0 +1,239 @@ +# master only + +name: Integrate + +on: + push: + branches: [master] + +env: + FORCE_COLOR: 1 + +jobs: + linuxNode16: + name: '[Linux] Node.js v16: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v16-${{ runner.os }}-${{ github.ref }}- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 16.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: script -e -c "npm test" + + windowsNode16: + name: '[Windows] Node.js v16: Unit tests' + runs-on: windows-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v16-${{ runner.os }}-${{ github.ref }}- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 16.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: npm test + + linuxNode14: + name: '[Linux] Node.js 14: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v14-${{ runner.os }}-${{ github.ref }}- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: script -e -c "npm test" + + linuxNode12: + name: '[Linux] Node.js v12: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v12-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v12-${{ runner.os }}-${{ github.ref }}- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: script -e -c "npm test" + + tagIfNewVersion: + name: Tag if new version + runs-on: ubuntu-latest + needs: [linuxNode14, windowsNode14, linuxNode16, linuxNode12] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # Ensure to have complete history of commits pushed with given push operation + # It's loose and imperfect assumption that no more than 30 commits will be pushed at once + fetch-depth: 30 + # Tag needs to be pushed with real user token, otherwise pushed tag won't trigger the actions workflow + # Hence we're passing 'serverless-ci' user authentication token + token: ${{ secrets.USER_GITHUB_TOKEN }} + + - name: Tag if new version + run: | + NEW_VERSION=`git diff -U0 ${{ github.event.before }} package.json | grep '"version": "' | tail -n 1 | grep -oE "[0-9]+\.[0-9]+\.[0-9]+"` || : + if [ -n "$NEW_VERSION" ]; + then + git tag v$NEW_VERSION + git push --tags + fi From f4d87b459a82422347758f647774c77542525774 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 14:05:10 +0100 Subject: [PATCH 20/27] ci: Introduce new CI publish workflow --- .github/workflows/publish.yml | 50 +++++++++++++++++++++++++++++------ CHANGELOG.md | 4 +++ package.json | 2 ++ 3 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6a1e7d26..6eee5b45 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,18 +1,52 @@ +# Version tags only + name: Publish -on: [release] +on: + push: + tags: + - v[0-9]+.[0-9]+.[0-9]+ jobs: - publish-npm: + publish: + name: Publish runs-on: ubuntu-latest + env: + # It'll work with secrets.GITHUB_TOKEN (which is provided by GitHub unconditionally) + # Still then release author would be "github-actions". It's better if it's dedicated repo bot + GITHUB_TOKEN: ${{ secrets.USER_GITHUB_TOKEN }} steps: - - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve node_modules from cache + id: cacheNodeModules + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} - - uses: actions/setup-node@v2 + - name: Install Node.js and npm + uses: actions/setup-node@v1 with: - version: 14 - registry-url: https://registry.npmjs.org/ + node-version: 14.x + registry-url: https://registry.npmjs.org - - run: npm publish + - name: Publish new version env: - NODE_AUTH_TOKEN: ${{secrets.npm_token}} + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish + + # Note: No need to install dependencies as: + # 1. We have retrieved cached `node_modules` for very same `package.json` + # as stored with recent `master `build + # 2. If for some reason cache retrieval fails `npx` will download and install + # `github-release-from-cc-changelog` + + - name: Publish release notes + run: | + TEMP_ARRAY=($(echo $GITHUB_REF | tr "/" "\n")) + TAG=${TEMP_ARRAY[@]: -1} + npx github-release-from-cc-changelog $TAG diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..cda73dee --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. + diff --git a/package.json b/package.json index 9d1f5852..ec73734b 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "format": "prettier --write '{.,lib}/*.{js,md}'", "lint": "eslint *.js lib/*.js && prettier -c '{.,lib}/*.{js,md}'", "lint:updated": "pipe-git-updated --ext=js -- eslint", + "prepare-release": "standard-version && prettier --write CHANGELOG.md", "prettier-check": "prettier -c --ignore-path .gitignore \"**/*.{css,html,js,json,md,yaml,yml}\"", "prettier-check:updated": "pipe-git-updated --ext=css --ext=html --ext=js --ext=json --ext=md --ext=yaml --ext=yml -- prettier -c", "prettify": "prettier --write --ignore-path .gitignore \"**/*.{css,html,js,json,md,yaml,yml}\"", @@ -51,6 +52,7 @@ "cross-spawn": "*", "eslint": "^7.32.0", "git-list-updated": "^1.2.1", + "github-release-from-cc-changelog": "^2.2.0", "lodash": "^4.17.21", "prettier": "^2", "tape": "*", From 274b8e52d9499612afbf096c023d73e75e0dd71a Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 14:07:57 +0100 Subject: [PATCH 21/27] ci: Add commitlint job to CI --- .github/workflows/validate.yml | 12 +++++++++++- CHANGELOG.md | 1 - commitlint.config.js | 31 +++++++++++++++++++++++++++++++ package.json | 1 + 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 commitlint.config.js diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 3a9af5c4..9f546619 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -11,7 +11,7 @@ env: jobs: linuxNode16: - name: '[Linux] Node.js v16: Lint, Formatting & Unit tests' + name: '[Linux] Node.js v16: Lint, Eventual Commitlint, Eventual Changelog, Formatting & Unit tests' runs-on: ubuntu-latest strategy: matrix: @@ -75,6 +75,16 @@ jobs: run: npm run prettier-check:updated - name: Validate ESLint rules run: npm run lint:updated + - name: Validate commit messages + if: github.event.pull_request.base.repo.id == github.event.pull_request.head.repo.id + run: npx commitlint -f master + - name: Validate changelog (if new version) + run: | + NEW_VERSION=`git diff -U0 master package.json | grep '"version": "' | tail -n 1 | grep -oE "[0-9]+\.[0-9]+\.[0-9]+"` || : + if [ -n "$NEW_VERSION" ]; + then + npx dump-release-notes-from-cc-changelog $NEW_VERSION + fi - name: Unit tests run: script -e -c "npm test" diff --git a/CHANGELOG.md b/CHANGELOG.md index cda73dee..5c79a3c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,3 @@ # Changelog All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. - diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 00000000..d23a0d6b --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,31 @@ +'use strict'; + +module.exports = { + rules: { + 'body-leading-blank': [2, 'always'], + 'footer-leading-blank': [2, 'always'], + 'header-max-length': [2, 'always', 72], + 'scope-enum': [2, 'always', ['', 'Config', 'Log']], + 'subject-case': [2, 'always', 'sentence-case'], + 'subject-empty': [2, 'never'], + 'subject-full-stop': [2, 'never', '.'], + 'type-case': [2, 'always', 'lower-case'], + 'type-empty': [2, 'never'], + 'type-enum': [ + 2, + 'always', + [ + 'build', + 'chore', + 'ci', + 'docs', + 'feat', + 'fix', + 'perf', + 'refactor', + 'style', + 'test', + ], + ], + }, +}; diff --git a/package.json b/package.json index ec73734b..be7bc48d 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "main": "index.js", "bin": {}, "scripts": { + "commitlint": "commitlint -f HEAD@{15}", "format": "prettier --write '{.,lib}/*.{js,md}'", "lint": "eslint *.js lib/*.js && prettier -c '{.,lib}/*.{js,md}'", "lint:updated": "pipe-git-updated --ext=js -- eslint", From 29f957dd04505174e7f23f20945a2fb8307a7942 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 14:09:09 +0100 Subject: [PATCH 22/27] chore: Reformat with eslint & prettier --- .github/dependabot.yml | 20 +- .github/workflows/integrate.yml | 8 +- .github/workflows/publish.yml | 4 +- .github/workflows/validate.yml | 8 +- README.md | 12 +- index.js | 44 ++-- package.json | 11 +- test.js | 371 ++++++++++++++++---------------- tests/base/_slimPatterns.yml | 2 +- tests/base/serverless.yml | 3 - tests/pipenv/_slimPatterns.yml | 2 +- tests/poetry/_slimPatterns.yml | 2 +- 12 files changed, 249 insertions(+), 238 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ac29398e..ab487438 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,12 +1,12 @@ version: 2 updates: -- package-ecosystem: npm - directory: "/" - schedule: - interval: daily - time: "10:00" - open-pull-requests-limit: 10 - ignore: - - dependency-name: eslint - versions: - - "> 7.22.0" + - package-ecosystem: npm + directory: '/' + schedule: + interval: daily + time: '10:00' + open-pull-requests-limit: 10 + ignore: + - dependency-name: eslint + versions: + - '> 7.22.0' diff --git a/.github/workflows/integrate.yml b/.github/workflows/integrate.yml index d241971c..64396542 100644 --- a/.github/workflows/integrate.yml +++ b/.github/workflows/integrate.yml @@ -59,7 +59,7 @@ jobs: npm update --no-save npm update --save-dev --no-save - name: Unit tests - run: script -e -c "npm test" + run: npm test windowsNode16: name: '[Windows] Node.js v16: Unit tests' @@ -161,7 +161,7 @@ jobs: npm update --no-save npm update --save-dev --no-save - name: Unit tests - run: script -e -c "npm test" + run: npm test linuxNode12: name: '[Linux] Node.js v12: Unit tests' @@ -212,12 +212,12 @@ jobs: npm update --no-save npm update --save-dev --no-save - name: Unit tests - run: script -e -c "npm test" + run: npm test tagIfNewVersion: name: Tag if new version runs-on: ubuntu-latest - needs: [linuxNode14, windowsNode14, linuxNode16, linuxNode12] + needs: [linuxNode16, windowsNode16, linuxNode14, linuxNode12] steps: - name: Checkout repository uses: actions/checkout@v2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6eee5b45..b44da770 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,12 +26,12 @@ jobs: path: | ~/.npm node_modules - key: npm-v14-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} + key: npm-v16-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} - name: Install Node.js and npm uses: actions/setup-node@v1 with: - node-version: 14.x + node-version: 16.x registry-url: https://registry.npmjs.org - name: Publish new version diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9f546619..d541cec7 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -86,7 +86,7 @@ jobs: npx dump-release-notes-from-cc-changelog $NEW_VERSION fi - name: Unit tests - run: script -e -c "npm test" + run: npm test windowsNode16: name: '[Windows] Node.js v16: Unit tests' @@ -192,9 +192,7 @@ jobs: npm update --no-save npm update --save-dev --no-save - name: Unit tests - # Some tests depend on TTY support, which is missing in GA runner - # Workaround taken from https://github.com/actions/runner/issues/241#issuecomment-577360161 - run: script -e -c "npm test" + run: npm test linuxNode12: name: '[Linux] Node.js v12: Unit tests' @@ -247,4 +245,4 @@ jobs: npm update --no-save npm update --save-dev --no-save - name: Unit tests - run: script -e -c "npm test" + run: npm test diff --git a/README.md b/README.md index 518d5ce1..abe6a175 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,13 @@ A Serverless v1.x plugin to automatically bundle dependencies from `requirements.txt` and make them available in your `PYTHONPATH`. ---- +--- _Originally developed by [**Capital One**](https://www.capitalone.com/tech/open-source/), now maintained in scope of Serverless, Inc_ _Capital One considers itself the bank a technology company would build. It's delivering best-in-class innovation so that its millions of customers can manage their finances with ease. Capital One is all-in on the cloud and is a leader in the adoption of open source, RESTful APIs, microservices and containers. We build our own products and release them with a speed and agility that allows us to get new customer experiences to market quickly. Our engineers use artificial intelligence and machine learning to transform real-time data, software and algorithms into the future of finance, reimagined._ ---- +--- ## Install @@ -557,10 +557,10 @@ package: - [@andrewfarley](https://github.com/andrewfarley) - Implemented download caching and static caching - [@bweigel](https://github.com/bweigel) - adding the `slimPatternsAppendDefaults` option & fixing per-function packaging when some functions don't have requirements & Porting tests from bats to js! - Poetry support - - [@squaresurf](https://github.com/squaresurf) - - [@drice](https://github.com/drice) - - [@ofercaspi](https://github.com/ofercaspi) - - [@tpansino](https://github.com/tpansino) + - [@squaresurf](https://github.com/squaresurf) + - [@drice](https://github.com/drice) + - [@ofercaspi](https://github.com/ofercaspi) + - [@tpansino](https://github.com/tpansino) - [@david-mk-lawrence](https://github.com/david-mk-lawrence) - added Lambda Layer support - [@bryantbriggs](https://github.com/bryantbiggs) - Fixing CI/CD - [@jacksgt](https://github.com/jacksgt) - Fixing pip issues diff --git a/index.js b/index.js index cf2af38e..7741a7f8 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ const values = require('lodash.values'); const { addVendorHelper, removeVendorHelper, - packRequirements + packRequirements, } = require('./lib/zip'); const { injectAllRequirements } = require('./lib/inject'); const { layerRequirements } = require('./lib/layer'); @@ -57,7 +57,7 @@ class ServerlessPythonRequirements { staticCacheMaxVersions: 0, pipCmdExtraArgs: [], noDeploy: [], - vendor: '' + vendor: '', }, (this.serverless.service.custom && this.serverless.service.custom.pythonRequirements) || @@ -75,7 +75,9 @@ class ServerlessPythonRequirements { ) { if (!this.warningLogged) { if (this.log) { - this.log.warning('You provided a docker related option but dockerizePip is set to false.'); + this.log.warning( + 'You provided a docker related option but dockerizePip is set to false.' + ); } else { this.serverless.cli.log( 'WARNING: You provided a docker related option but dockerizePip is set to false.' @@ -144,31 +146,31 @@ class ServerlessPythonRequirements { commands: { clean: { usage: 'Remove .requirements and requirements.zip', - lifecycleEvents: ['clean'] + lifecycleEvents: ['clean'], }, install: { usage: 'install requirements manually', - lifecycleEvents: ['install'] + lifecycleEvents: ['install'], }, cleanCache: { usage: 'Removes all items in the pip download/static cache (if present)', - lifecycleEvents: ['cleanCache'] - } - } - } + lifecycleEvents: ['cleanCache'], + }, + }, + }, }; - if (this.serverless.cli.generateCommandsHelp) { - Object.assign(this.commands.requirements, { - usage: 'Serverless plugin to bundle Python packages', - lifecycleEvents: ['requirements'] - }); - } else { - this.commands.requirements.type = 'container'; - } + if (this.serverless.cli.generateCommandsHelp) { + Object.assign(this.commands.requirements, { + usage: 'Serverless plugin to bundle Python packages', + lifecycleEvents: ['requirements'], + }); + } else { + this.commands.requirements.type = 'container'; + } - const isFunctionRuntimePython = args => { + const isFunctionRuntimePython = (args) => { // If functionObj.runtime is undefined, python. if (!args[1].functionObj || !args[1].functionObj.runtime) { return true; @@ -177,9 +179,7 @@ class ServerlessPythonRequirements { }; const clean = () => - BbPromise.bind(this) - .then(cleanup) - .then(removeVendorHelper); + BbPromise.bind(this).then(cleanup).then(removeVendorHelper); const setupArtifactPathCapturing = () => { // Reference: @@ -243,7 +243,7 @@ class ServerlessPythonRequirements { }, 'requirements:install:install': before, 'requirements:clean:clean': clean, - 'requirements:cleanCache:cleanCache': cleanCache + 'requirements:cleanCache:cleanCache': cleanCache, }; } } diff --git a/package.json b/package.json index be7bc48d..4db9497a 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,7 @@ "bin": {}, "scripts": { "commitlint": "commitlint -f HEAD@{15}", - "format": "prettier --write '{.,lib}/*.{js,md}'", - "lint": "eslint *.js lib/*.js && prettier -c '{.,lib}/*.{js,md}'", + "lint": "eslint .", "lint:updated": "pipe-git-updated --ext=js -- eslint", "prepare-release": "standard-version && prettier --write CHANGELOG.md", "prettier-check": "prettier -c --ignore-path .gitignore \"**/*.{css,html,js,json,md,yaml,yml}\"", @@ -79,6 +78,14 @@ "peerDependencies": { "serverless": "^2.32" }, + "lint-staged": { + "*.js": [ + "eslint" + ], + "*.{css,html,js,json,md,yaml,yml}": [ + "prettier -c" + ] + }, "eslintConfig": { "extends": "eslint:recommended", "env": { diff --git a/test.js b/test.js index 0322ab91..f234229a 100644 --- a/test.js +++ b/test.js @@ -10,7 +10,7 @@ const { copySync, writeFileSync, statSync, - pathExistsSync + pathExistsSync, } = require('fs-extra'); const { quote } = require('shell-quote'); const { sep } = require('path'); @@ -19,30 +19,32 @@ const { getUserCachePath, sha256Path } = require('./lib/shared'); const initialWorkingDir = process.cwd(); -const mkCommand = cmd => (args, options = {}) => { - const { error, stdout, stderr, status } = crossSpawn.sync( - cmd, - args, - Object.assign( - { - env: Object.assign({}, process.env, { SLS_DEBUG: 't' }) - }, - options - ) - ); - if (error) { - console.error(`Error running: ${quote([cmd, ...args])}`); // eslint-disable-line no-console - throw error; - } - if (status) { - console.error('STDOUT: ', stdout.toString()); // eslint-disable-line no-console - console.error('STDERR: ', stderr.toString()); // eslint-disable-line no-console - throw new Error( - `${quote([cmd, ...args])} failed with status code ${status}` - ); - } - return stdout && stdout.toString().trim(); -}; +const mkCommand = + (cmd) => + (args, options = {}) => { + const { error, stdout, stderr, status } = crossSpawn.sync( + cmd, + args, + Object.assign( + { + env: Object.assign({}, process.env, { SLS_DEBUG: 't' }), + }, + options + ) + ); + if (error) { + console.error(`Error running: ${quote([cmd, ...args])}`); // eslint-disable-line no-console + throw error; + } + if (status) { + console.error('STDOUT: ', stdout.toString()); // eslint-disable-line no-console + console.error('STDERR: ', stderr.toString()); // eslint-disable-line no-console + throw new Error( + `${quote([cmd, ...args])} failed with status code ${status}` + ); + } + return stdout && stdout.toString().trim(); + }; const sls = mkCommand('sls'); const git = mkCommand('git'); const npm = mkCommand('npm'); @@ -73,8 +75,8 @@ const teardown = () => { 'serverless.yml.bak', 'module1/foobar', getUserCachePath(), - ...glob.sync('serverless-python-requirements-*.tgz') - ].map(path => removeSync(path)); + ...glob.sync('serverless-python-requirements-*.tgz'), + ].map((path) => removeSync(path)); if (!cwd.endsWith('base with a space')) { try { git(['checkout', 'serverless.yml']); @@ -93,15 +95,17 @@ const teardown = () => { const testFilter = (() => { const elems = process.argv.slice(2); // skip ['node', 'test.js'] if (elems.length) { - return desc => - elems.some(text => desc.search(text) != -1) ? tape.test : tape.test.skip; + return (desc) => + elems.some((text) => desc.search(text) != -1) + ? tape.test + : tape.test.skip; } else { return () => tape.test; } })(); const test = (desc, func, opts = {}) => - testFilter(desc)(desc, opts, async t => { + testFilter(desc)(desc, opts, async (t) => { setup(); let ended = false; try { @@ -124,7 +128,7 @@ const availablePythons = (() => { const mapping = {}; if (process.env.USE_PYTHON) { binaries.push( - ...process.env.USE_PYTHON.split(',').map(v => v.toString().trim()) + ...process.env.USE_PYTHON.split(',').map((v) => v.toString().trim()) ); } else { // For running outside of CI @@ -135,7 +139,7 @@ const availablePythons = (() => { const python = `${bin}${exe}`; const { stdout, status } = crossSpawn.sync(python, [ '-c', - 'import sys; sys.stdout.write(".".join(map(str, sys.version_info[:2])))' + 'import sys; sys.stdout.write(".".join(map(str, sys.version_info[:2])))', ]); const ver = stdout && stdout.toString().trim(); if (!status && ver) { @@ -152,29 +156,29 @@ const availablePythons = (() => { return mapping; })(); -const getPythonBin = version => { +const getPythonBin = (version) => { const bin = availablePythons[String(version)]; if (!bin) throw new Error(`No python version ${version} available`); return bin; }; -const hasPython = version => { +const hasPython = (version) => { return Boolean(availablePythons[String(version)]); }; -const listZipFiles = async function(filename) { +const listZipFiles = async function (filename) { const file = await readFile(filename); const zip = await new JSZip().loadAsync(file); return Object.keys(zip.files); }; -const listZipFilesWithMetaData = async function(filename) { +const listZipFilesWithMetaData = async function (filename) { const file = await readFile(filename); const zip = await new JSZip().loadAsync(file); return Object(zip.files); }; -const listRequirementsZipFiles = async function(filename) { +const listRequirementsZipFiles = async function (filename) { const file = await readFile(filename); const zip = await new JSZip().loadAsync(file); const reqsBuffer = await zip.file('.requirements.zip').async('nodebuffer'); @@ -197,7 +201,7 @@ const brokenOn = (...platforms) => platforms.indexOf(process.platform) != -1; test( 'default pythonBin can package flask with default options', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -212,7 +216,7 @@ test( test( 'py3.6 packages have the same hash', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -231,7 +235,7 @@ test( test( 'py3.6 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -246,14 +250,14 @@ test( test( 'py3.6 can package flask with hashes', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ `--pythonBin=${getPythonBin(3)}`, '--fileName=requirements-w-hashes.txt', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); @@ -264,14 +268,14 @@ test( test( 'py3.6 can package flask with nested', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ `--pythonBin=${getPythonBin(3)}`, '--fileName=requirements-w-nested.txt', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); @@ -283,7 +287,7 @@ test( test( 'py3.6 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -308,7 +312,7 @@ test( test( 'py3.6 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -316,12 +320,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -331,7 +336,7 @@ test( test( 'py3.6 can package flask with slim & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -340,12 +345,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -356,7 +361,7 @@ test( test( "py3.6 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -365,7 +370,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls([`--pythonBin=${getPythonBin(3)}`, 'package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -378,14 +383,14 @@ test( test( 'py3.6 can package boto3 with editable', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ `--pythonBin=${getPythonBin(3)}`, '--fileName=requirements-w-editable.txt', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -400,7 +405,7 @@ test( test( 'py3.6 can package flask with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -416,7 +421,7 @@ test( test( 'py3.6 can package flask with slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -424,12 +429,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are NOT packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -439,7 +445,7 @@ test( test( 'py3.6 can package flask with slim & dockerizePip & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -448,12 +454,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -464,7 +470,7 @@ test( test( 'py3.6 can package flask with zip & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -497,7 +503,7 @@ test( test( 'py3.6 can package flask with zip & slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -530,7 +536,7 @@ test( test( 'py2.7 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -545,7 +551,7 @@ test( test( 'py2.7 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -553,17 +559,18 @@ test( `--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -573,7 +580,7 @@ test( test( 'py2.7 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -581,7 +588,7 @@ test( `--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', '--zip=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( @@ -603,7 +610,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -614,17 +621,17 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -635,7 +642,7 @@ test( test( "py2.7 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -644,7 +651,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls([`--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', 'package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -657,7 +664,7 @@ test( test( 'py2.7 can package flask with zip & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -666,7 +673,7 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--zip=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -696,7 +703,7 @@ test( test( 'py2.7 can package flask with zip & slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -706,7 +713,7 @@ test( '--dockerizePip=true', '--zip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -736,7 +743,7 @@ test( test( 'py2.7 can package flask with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -744,7 +751,7 @@ test( `--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', '--dockerizePip=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -757,7 +764,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -766,17 +773,18 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are NOT packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -786,7 +794,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -797,17 +805,17 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -818,7 +826,7 @@ test( test( 'pipenv py3.6 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -837,7 +845,7 @@ test( test( 'pipenv py3.6 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -845,12 +853,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -860,7 +869,7 @@ test( test( 'pipenv py3.6 can package flask with slim & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/pipenv'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -870,12 +879,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -886,7 +895,7 @@ test( test( 'pipenv py3.6 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -911,7 +920,7 @@ test( test( "pipenv py3.6 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -920,7 +929,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls(['package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -933,7 +942,7 @@ test( test( 'non build pyproject.toml uses requirements.txt', - async t => { + async (t) => { process.chdir('tests/non_build_pyproject'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -948,7 +957,7 @@ test( test( 'non poetry pyproject.toml without requirements.txt packages handler only', - async t => { + async (t) => { process.chdir('tests/non_poetry_pyproject'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -962,7 +971,7 @@ test( test( 'poetry py3.6 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -978,7 +987,7 @@ test( test( 'poetry py3.6 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -986,12 +995,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -1001,7 +1011,7 @@ test( test( 'poetry py3.6 can package flask with slim & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/poetry'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -1011,12 +1021,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1027,7 +1037,7 @@ test( test( 'poetry py3.6 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1052,7 +1062,7 @@ test( test( "poetry py3.6 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1061,7 +1071,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls(['package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -1074,7 +1084,7 @@ test( test( 'py3.6 can package flask with zip option and no explicit include', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1101,7 +1111,7 @@ test( test( 'py3.6 can package lambda-decorators using vendor option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1120,7 +1130,7 @@ test( test( "Don't nuke execute perms", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); const perm = '755'; @@ -1131,7 +1141,7 @@ test( '-i.bak', '-e', 's/(handler.py.*$)/$1\n - foobar/', - 'serverless.yml' + 'serverless.yml', ]); writeFileSync(`foobar`, ''); chmodSync(`foobar`, perm); @@ -1169,7 +1179,7 @@ test( test( 'py3.6 can package flask in a project with a space in it', - async t => { + async (t) => { copySync('tests/base', 'tests/base with a space'); process.chdir('tests/base with a space'); const path = npm(['pack', '../..']); @@ -1185,7 +1195,7 @@ test( test( 'py3.6 can package flask in a project with a space in it with docker', - async t => { + async (t) => { copySync('tests/base', 'tests/base with a space'); process.chdir('tests/base with a space'); const path = npm(['pack', '../..']); @@ -1201,7 +1211,7 @@ test( test( 'py3.6 supports custom file name with fileName option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); writeFileSync('puck', 'requests'); @@ -1227,7 +1237,7 @@ test( test( "py3.6 doesn't package bottle with zip option", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1236,7 +1246,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -1270,7 +1280,7 @@ test( test( 'py3.6 can package flask with slim, slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1280,11 +1290,11 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1295,7 +1305,7 @@ test( test( 'py3.6 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1304,17 +1314,17 @@ test( '--dockerizePip=true', '--slim=true', '--slimPatternsAppendDefaults=false', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1325,7 +1335,7 @@ test( test( 'py2.7 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1334,17 +1344,17 @@ test( '--runtime=python2.7', '--slim=true', '--slimPatternsAppendDefaults=false', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1355,7 +1365,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1365,16 +1375,16 @@ test( '--runtime=python2.7', '--slim=true', '--slimPatternsAppendDefaults=false', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1385,7 +1395,7 @@ test( test( 'pipenv py3.6 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', - async t => { + async (t) => { process.chdir('tests/pipenv'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1395,11 +1405,11 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1410,7 +1420,7 @@ test( test( 'poetry py3.6 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', - async t => { + async (t) => { process.chdir('tests/poetry'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1420,11 +1430,11 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1435,7 +1445,7 @@ test( test( 'py3.6 can package flask with package individually option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1520,11 +1530,9 @@ test( { skip: !hasPython(3.6) } ); - - test( 'py3.6 can package flask with package individually & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1536,7 +1544,7 @@ test( 'handler.py is packaged in function hello' ); t.deepEqual( - zipfiles_hello.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello' ); @@ -1555,7 +1563,7 @@ test( 'handler.py is packaged in function hello2' ); t.deepEqual( - zipfiles_hello2.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello2.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello2' ); @@ -1574,7 +1582,7 @@ test( 'handler.py is packaged in function hello3' ); t.deepEqual( - zipfiles_hello3.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello3.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello3' ); @@ -1599,7 +1607,7 @@ test( 'flask is NOT packaged in function hello4' ); t.deepEqual( - zipfiles_hello4.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello4.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello4' ); @@ -1611,7 +1619,7 @@ test( test( 'py2.7 can package flask with package individually option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1682,7 +1690,7 @@ test( test( 'py2.7 can package flask with package individually & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1690,7 +1698,7 @@ test( '--individually=true', '--runtime=python2.7', '--slim=true', - 'package' + 'package', ]); const zipfiles_hello = await listZipFiles('.serverless/hello.zip'); @@ -1699,7 +1707,7 @@ test( 'handler.py is packaged in function hello' ); t.deepEqual( - zipfiles_hello.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello' ); @@ -1718,7 +1726,7 @@ test( 'handler.py is packaged in function hello2' ); t.deepEqual( - zipfiles_hello2.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello2.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello2' ); @@ -1737,7 +1745,7 @@ test( 'handler.py is packaged in function hello3' ); t.deepEqual( - zipfiles_hello3.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello3.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello3' ); @@ -1773,7 +1781,7 @@ test( test( 'py2.7 can ignore functions defined with `image`', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1807,7 +1815,7 @@ test( test( 'py3.6 can package only requirements of module', - async t => { + async (t) => { process.chdir('tests/individually'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1868,7 +1876,7 @@ test( test( 'py3.6 can package lambda-decorators using vendor and invidiually option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1950,7 +1958,7 @@ test( test( "Don't nuke execute perms when using individually", - async t => { + async (t) => { process.chdir('tests/individually'); const path = npm(['pack', '../..']); const perm = '755'; @@ -1974,8 +1982,9 @@ test( const zipfiles_hello2 = await listZipFilesWithMetaData( '.serverless/module2-sls-py-req-test-indiv-dev-hello2.zip' ); - const flaskPerm = statSync('.serverless/module2/requirements/bin/flask') - .mode; + const flaskPerm = statSync( + '.serverless/module2/requirements/bin/flask' + ).mode; t.true( zipfiles_hello2['bin/flask'].unixPermissions === flaskPerm, @@ -1989,7 +1998,7 @@ test( test( "Don't nuke execute perms when using individually w/docker", - async t => { + async (t) => { process.chdir('tests/individually'); const path = npm(['pack', '../..']); const perm = '755'; @@ -2013,8 +2022,9 @@ test( const zipfiles_hello2 = await listZipFilesWithMetaData( '.serverless/module2-sls-py-req-test-indiv-dev-hello2.zip' ); - const flaskPerm = statSync('.serverless/module2/requirements/bin/flask') - .mode; + const flaskPerm = statSync( + '.serverless/module2/requirements/bin/flask' + ).mode; t.true( zipfiles_hello2['bin/flask'].unixPermissions === flaskPerm, @@ -2028,7 +2038,7 @@ test( test( 'py3.6 uses download cache by default option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2045,7 +2055,7 @@ test( test( 'py3.6 uses download cache by default', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2061,7 +2071,7 @@ test( test( 'py3.6 uses download cache with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2078,14 +2088,14 @@ test( test( 'py3.6 uses download cache with dockerizePip by default option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ '--dockerizePip=true', '--cacheLocation=.requirements-cache', - 'package' + 'package', ]); t.true( pathExistsSync(`.requirements-cache${sep}downloadCacheslspyc${sep}http`), @@ -2098,7 +2108,7 @@ test( test( 'py3.6 uses static and download cache', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2120,7 +2130,7 @@ test( test( 'py3.6 uses static and download cache with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2142,7 +2152,7 @@ test( test( 'py3.6 uses static cache', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2180,7 +2190,7 @@ test( test( 'py3.6 uses static cache with cacheLocation option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2204,7 +2214,7 @@ test( test( 'py3.6 uses static cache with dockerizePip & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2235,7 +2245,7 @@ test( "static cache is really used when running 'sls package' again" ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files are packaged' ); @@ -2247,7 +2257,7 @@ test( test( 'py3.6 uses download cache with dockerizePip & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2261,7 +2271,7 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files are packaged' ); @@ -2273,13 +2283,12 @@ test( test( 'py3.6 can ignore functions defined with `image`', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls(['--individually=true', 'package']); - t.true( pathExistsSync('.serverless/hello.zip'), 'function hello is packaged' diff --git a/tests/base/_slimPatterns.yml b/tests/base/_slimPatterns.yml index 02c631b4..443af9a0 100644 --- a/tests/base/_slimPatterns.yml +++ b/tests/base/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "**/__main__.py" + - '**/__main__.py' diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index 6bb1f322..0b360e9b 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -50,6 +50,3 @@ functions: - 'fn2/**' hello5: image: 000000000000.dkr.ecr.sa-east-1.amazonaws.com/test-lambda-docker@sha256:6bb600b4d6e1d7cf521097177dd0c4e9ea373edb91984a505333be8ac9455d38 - - - diff --git a/tests/pipenv/_slimPatterns.yml b/tests/pipenv/_slimPatterns.yml index 02c631b4..443af9a0 100644 --- a/tests/pipenv/_slimPatterns.yml +++ b/tests/pipenv/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "**/__main__.py" + - '**/__main__.py' diff --git a/tests/poetry/_slimPatterns.yml b/tests/poetry/_slimPatterns.yml index 02c631b4..443af9a0 100644 --- a/tests/poetry/_slimPatterns.yml +++ b/tests/poetry/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "**/__main__.py" + - '**/__main__.py' From 9b84abf826d95a5ad152d6a70de7c722770b467b Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Fri, 26 Nov 2021 17:04:17 +0100 Subject: [PATCH 23/27] chore: Remove dependabot --- .github/dependabot.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index ab487438..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 -updates: - - package-ecosystem: npm - directory: '/' - schedule: - interval: daily - time: '10:00' - open-pull-requests-limit: 10 - ignore: - - dependency-name: eslint - versions: - - '> 7.22.0' From 89b3bababd48d496e159c9694546b8ab18b2955b Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Mon, 29 Nov 2021 15:03:46 +0100 Subject: [PATCH 24/27] chore: Remove Node16 tests --- .github/workflows/integrate.yml | 63 +++--------------------------- .github/workflows/publish.yml | 4 +- .github/workflows/validate.yml | 69 ++++----------------------------- 3 files changed, 16 insertions(+), 120 deletions(-) diff --git a/.github/workflows/integrate.yml b/.github/workflows/integrate.yml index 64396542..953951df 100644 --- a/.github/workflows/integrate.yml +++ b/.github/workflows/integrate.yml @@ -10,59 +10,8 @@ env: FORCE_COLOR: 1 jobs: - linuxNode16: - name: '[Linux] Node.js v16: Unit tests' - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [2.7, 3.6] - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Retrieve dependencies from cache - id: cacheNpm - uses: actions/cache@v2 - with: - path: | - ~/.npm - node_modules - key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} - restore-keys: npm-v16-${{ runner.os }}-${{ github.ref }}- - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Node.js and npm - uses: actions/setup-node@v1 - with: - node-version: 16.x - - - name: Check python version - run: | - python --version - - - name: Install setuptools - run: python -m pip install --force setuptools wheel - - - name: Install pipenv / poetry - run: python -m pip install pipenv poetry - - - name: Install serverless - run: npm install -g serverless@2 - - - name: Install dependencies - if: steps.cacheNpm.outputs.cache-hit != 'true' - run: | - npm update --no-save - npm update --save-dev --no-save - - name: Unit tests - run: npm test - - windowsNode16: - name: '[Windows] Node.js v16: Unit tests' + windowsNode14: + name: '[Windows] Node.js v14: Unit tests' runs-on: windows-latest strategy: matrix: @@ -78,8 +27,8 @@ jobs: path: | ~/.npm node_modules - key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} - restore-keys: npm-v16-${{ runner.os }}-${{ github.ref }}- + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v14-${{ runner.os }}-${{ github.ref }}- - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 @@ -89,7 +38,7 @@ jobs: - name: Install Node.js and npm uses: actions/setup-node@v1 with: - node-version: 16.x + node-version: 14.x - name: Check python version run: | @@ -217,7 +166,7 @@ jobs: tagIfNewVersion: name: Tag if new version runs-on: ubuntu-latest - needs: [linuxNode16, windowsNode16, linuxNode14, linuxNode12] + needs: [windowsNode14, linuxNode14, linuxNode12] steps: - name: Checkout repository uses: actions/checkout@v2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b44da770..6eee5b45 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,12 +26,12 @@ jobs: path: | ~/.npm node_modules - key: npm-v16-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} + key: npm-v14-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} - name: Install Node.js and npm uses: actions/setup-node@v1 with: - node-version: 16.x + node-version: 14.x registry-url: https://registry.npmjs.org - name: Publish new version diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d541cec7..9215eee1 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -10,8 +10,8 @@ env: FORCE_COLOR: 1 jobs: - linuxNode16: - name: '[Linux] Node.js v16: Lint, Eventual Commitlint, Eventual Changelog, Formatting & Unit tests' + linuxNode14: + name: '[Linux] Node.js v14: Lint, Eventual Commitlint, Eventual Changelog, Formatting & Unit tests' runs-on: ubuntu-latest strategy: matrix: @@ -38,10 +38,10 @@ jobs: path: | ~/.npm node_modules - key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} restore-keys: | - npm-v16-${{ runner.os }}-${{ github.ref }}- - npm-v16-${{ runner.os }}-refs/heads/master- + npm-v14-${{ runner.os }}-${{ github.ref }}- + npm-v14-${{ runner.os }}-refs/heads/master- - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 @@ -51,7 +51,7 @@ jobs: - name: Install Node.js and npm uses: actions/setup-node@v1 with: - node-version: 16.x + node-version: 14.x - name: Check python version run: | @@ -88,62 +88,9 @@ jobs: - name: Unit tests run: npm test - windowsNode16: - name: '[Windows] Node.js v16: Unit tests' + windowsNode14: + name: '[Windows] Node.js v14: Unit tests' runs-on: windows-latest - strategy: - matrix: - python-version: [2.7, 3.6] - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Retrieve dependencies from cache - id: cacheNpm - uses: actions/cache@v2 - with: - path: | - ~/.npm - node_modules - key: npm-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} - restore-keys: | - npm-v16-${{ runner.os }}-${{ github.ref }}- - npm-v16-${{ runner.os }}-refs/heads/master- - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Node.js and npm - uses: actions/setup-node@v1 - with: - node-version: 16.x - - - name: Check python version - run: | - python --version - - - name: Install setuptools - run: python -m pip install --force setuptools wheel - - - name: Install pipenv / poetry - run: python -m pip install pipenv poetry - - - name: Install serverless - run: npm install -g serverless@2 - - - name: Install dependencies - if: steps.cacheNpm.outputs.cache-hit != 'true' - run: | - npm update --no-save - npm update --save-dev --no-save - - name: Unit tests - run: npm test - - linuxNode14: - name: '[Linux] Node.js 14: Unit tests' - runs-on: ubuntu-latest strategy: matrix: python-version: [2.7, 3.6] From 328cb016e58231d3c72399918553c4b10d4aa6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Wilczy=C5=84ski?= Date: Wed, 1 Dec 2021 20:13:59 +0100 Subject: [PATCH 25/27] feat: Add architecture to requirements cache directory name (#645) --- lib/pip.js | 3 ++- lib/shared.js | 7 +++++-- test.js | 35 +++++++++++++++++++++++++---------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/lib/pip.js b/lib/pip.js index ce348532..7a0a0ceb 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -640,7 +640,8 @@ async function installRequirementsIfNeeded( const workingReqsFolder = getRequirementsWorkingPath( reqChecksum, requirementsTxtDirectory, - options + options, + serverless ); // Check if our static cache is present and is valid diff --git a/lib/shared.js b/lib/shared.js index 7baee58b..426d6c50 100644 --- a/lib/shared.js +++ b/lib/shared.js @@ -62,17 +62,20 @@ function checkForAndDeleteMaxCacheVersions({ serverless, options, log }) { * @param {string} subfolder * @param {string} servicePath * @param {Object} options + * @param {Object} serverless * @return {string} */ function getRequirementsWorkingPath( subfolder, requirementsTxtDirectory, - options + options, + serverless ) { // If we want to use the static cache if (options && options.useStaticCache) { if (subfolder) { - subfolder = subfolder + '_slspyc'; + const architecture = serverless.service.provider.architecture || 'x86_64'; + subfolder = `${subfolder}_${architecture}_slspyc`; } // If we have max number of cache items... diff --git a/test.js b/test.js index f234229a..ccd1920c 100644 --- a/test.js +++ b/test.js @@ -2115,12 +2115,15 @@ test( sls(['package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), 'http exists in download-cache' ); t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.end(); @@ -2137,12 +2140,15 @@ test( sls(['--dockerizePip=true', 'package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), 'http exists in download-cache' ); t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.end(); @@ -2159,20 +2165,23 @@ test( sls(['package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.true( pathExistsSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}.completed_requirements` + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}.completed_requirements` ), '.completed_requirements exists in static-cache' ); // py3.6 checking that static cache actually pulls from cache (by poisoning it) writeFileSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}injected_file_is_bad_form`, + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' ); sls(['package']); @@ -2197,13 +2206,16 @@ test( const cachepath = '.requirements-cache'; sls([`--cacheLocation=${cachepath}`, 'package']); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.true( pathExistsSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}.completed_requirements` + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}.completed_requirements` ), '.completed_requirements exists in static-cache' ); @@ -2221,20 +2233,23 @@ test( sls(['--dockerizePip=true', '--slim=true', 'package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.true( pathExistsSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}.completed_requirements` + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}.completed_requirements` ), '.completed_requirements exists in static-cache' ); // py3.6 checking that static cache actually pulls from cache (by poisoning it) writeFileSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}injected_file_is_bad_form`, + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' ); sls(['--dockerizePip=true', '--slim=true', 'package']); From 347245cc5a7d97c56dc32f0da4285e0be59cf535 Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 30 Nov 2021 16:57:55 +0100 Subject: [PATCH 26/27] chore: Add `standard-version` config --- package.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/package.json b/package.json index 4db9497a..6a9923c4 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "github-release-from-cc-changelog": "^2.2.0", "lodash": "^4.17.21", "prettier": "^2", + "standard-version": "^9.3.2", "tape": "*", "tape-promise": "*" }, @@ -100,6 +101,30 @@ "no-console": "off" } }, + "standard-version": { + "skip": { + "commit": true, + "tag": true + }, + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "perf", + "section": "Performance Improvements" + }, + { + "type": "refactor", + "section": "Maintenance Improvements" + } + ] + }, "prettier": { "semi": true, "singleQuote": true From e3d9ebcdd3ec72cc2e3301d33e10dd10ef6a594d Mon Sep 17 00:00:00 2001 From: Piotr Grzesik Date: Tue, 30 Nov 2021 17:01:53 +0100 Subject: [PATCH 27/27] chore: Release v5.2.1 --- CHANGELOG.md | 14 ++++++++++++++ package.json | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c79a3c3..bd87ce72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ # Changelog All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. + +### [5.2.1](https://github.com/UnitedIncome/serverless-python-requirements/compare/v5.2.0...v5.2.1) (2021-11-30) + +### Maintenance Improvements + +- Adapt plugin to modern logs ([#646](https://github.com/serverless/serverless-python-requirements/pull/646)) ([8ff97e6](https://github.com/UnitedIncome/serverless-python-requirements/commit/8ff97e6b7c279334e417dbdb65e64d0de2656986)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Adapt to `async` version of `spawn` ([#648](https://github.com/serverless/serverless-python-requirements/pull/648)) ([50c2850](https://github.com/UnitedIncome/serverless-python-requirements/commit/50c2850874ded795fd50ae377f1db817a0212e7d)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Adapt v3 log writing interfaces ([#646](https://github.com/serverless/serverless-python-requirements/pull/646)) ([a79899a](https://github.com/UnitedIncome/serverless-python-requirements/commit/a79899ae5f6f66aa0c65e7fda8e0186d38ff446e)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Ensure proper verbose progress logs ([#646](https://github.com/serverless/serverless-python-requirements/pull/646)) ([44b9591](https://github.com/UnitedIncome/serverless-python-requirements/commit/44b9591f01157a1811e3ca8b43e21265a155a976)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Use `ServerlessError` ([#649](https://github.com/serverless/serverless-python-requirements/pull/649)) ([cdb7111](https://github.com/UnitedIncome/serverless-python-requirements/commit/cdb71110bc9c69b5087b6e18fb353d65962afe4a)) ([Piotr Grzesik](https://github.com/pgrzesik)) + +# Changelog + +All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. diff --git a/package.json b/package.json index 6a9923c4..c6a1f5e6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "serverless-python-requirements", - "version": "5.2.0", + "version": "5.2.1", "engines": { "node": ">=12.0" },