diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index ac29398e..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 -updates: -- package-ecosystem: npm - directory: "/" - schedule: - interval: daily - time: "10:00" - open-pull-requests-limit: 10 - ignore: - - dependency-name: eslint - versions: - - "> 7.22.0" diff --git a/.github/workflows/integrate.yml b/.github/workflows/integrate.yml new file mode 100644 index 00000000..953951df --- /dev/null +++ b/.github/workflows/integrate.yml @@ -0,0 +1,188 @@ +# master only + +name: Integrate + +on: + push: + branches: [master] + +env: + FORCE_COLOR: 1 + +jobs: + windowsNode14: + name: '[Windows] Node.js v14: Unit tests' + runs-on: windows-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v14-${{ runner.os }}-${{ github.ref }}- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: npm test + + linuxNode14: + name: '[Linux] Node.js 14: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v14-${{ runner.os }}-${{ github.ref }}- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: npm test + + linuxNode12: + name: '[Linux] Node.js v12: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v12-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: npm-v12-${{ runner.os }}-${{ github.ref }}- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: npm test + + tagIfNewVersion: + name: Tag if new version + runs-on: ubuntu-latest + needs: [windowsNode14, linuxNode14, linuxNode12] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # Ensure to have complete history of commits pushed with given push operation + # It's loose and imperfect assumption that no more than 30 commits will be pushed at once + fetch-depth: 30 + # Tag needs to be pushed with real user token, otherwise pushed tag won't trigger the actions workflow + # Hence we're passing 'serverless-ci' user authentication token + token: ${{ secrets.USER_GITHUB_TOKEN }} + + - name: Tag if new version + run: | + NEW_VERSION=`git diff -U0 ${{ github.event.before }} package.json | grep '"version": "' | tail -n 1 | grep -oE "[0-9]+\.[0-9]+\.[0-9]+"` || : + if [ -n "$NEW_VERSION" ]; + then + git tag v$NEW_VERSION + git push --tags + fi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 1e6b9ee8..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Lint - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Node ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: 14 - - - name: Install deps - run: npm install - - - name: Lint - run: npm run ci:lint diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6a1e7d26..6eee5b45 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,18 +1,52 @@ +# Version tags only + name: Publish -on: [release] +on: + push: + tags: + - v[0-9]+.[0-9]+.[0-9]+ jobs: - publish-npm: + publish: + name: Publish runs-on: ubuntu-latest + env: + # It'll work with secrets.GITHUB_TOKEN (which is provided by GitHub unconditionally) + # Still then release author would be "github-actions". It's better if it's dedicated repo bot + GITHUB_TOKEN: ${{ secrets.USER_GITHUB_TOKEN }} steps: - - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve node_modules from cache + id: cacheNodeModules + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-refs/heads/master-${{ hashFiles('package.json') }} - - uses: actions/setup-node@v2 + - name: Install Node.js and npm + uses: actions/setup-node@v1 with: - version: 14 - registry-url: https://registry.npmjs.org/ + node-version: 14.x + registry-url: https://registry.npmjs.org - - run: npm publish + - name: Publish new version env: - NODE_AUTH_TOKEN: ${{secrets.npm_token}} + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish + + # Note: No need to install dependencies as: + # 1. We have retrieved cached `node_modules` for very same `package.json` + # as stored with recent `master `build + # 2. If for some reason cache retrieval fails `npx` will download and install + # `github-release-from-cc-changelog` + + - name: Publish release notes + run: | + TEMP_ARRAY=($(echo $GITHUB_REF | tr "/" "\n")) + TAG=${TEMP_ARRAY[@]: -1} + npx github-release-from-cc-changelog $TAG diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index f89486bf..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Test - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - python-version: [2.7, 3.6] - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up Node - uses: actions/setup-node@v1 - with: - node-version: 14 - - - name: Check python version - run: | - python --version - - - name: Install setuptools - run: python -m pip install --force setuptools wheel - - - name: Install pipenv / poetry - run: python -m pip install pipenv poetry - - - name: Install serverless - run: npm install -g serverless@2 - - - name: Install deps - run: npm install - - - name: Test - run: npm run test - env: - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - if: matrix.os != 'macOS-latest' - - - name: Test (Mac) - run: npm run test - env: - LC_ALL: en_US.UTF-8 - LANG: en_US.UTF-8 - if: matrix.os == 'macOS-latest' diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..9215eee1 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,195 @@ +# PR's only + +name: Validate + +on: + pull_request: + branches: [master] + +env: + FORCE_COLOR: 1 + +jobs: + linuxNode14: + name: '[Linux] Node.js v14: Lint, Eventual Commitlint, Eventual Changelog, Formatting & Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # For commitlint purpose ensure to have complete list of PR commits + # It's loose and imperfect assumption that PR has no more than 30 commits + fetch-depth: 30 + + - name: Retrieve last master commit (for `git diff` purposes) + run: | + git checkout -b pr + git fetch --prune --depth=30 origin +refs/heads/master:refs/remotes/origin/master + git checkout master + git checkout pr + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: | + npm-v14-${{ runner.os }}-${{ github.ref }}- + npm-v14-${{ runner.os }}-refs/heads/master- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Validate Prettier formatting + run: npm run prettier-check:updated + - name: Validate ESLint rules + run: npm run lint:updated + - name: Validate commit messages + if: github.event.pull_request.base.repo.id == github.event.pull_request.head.repo.id + run: npx commitlint -f master + - name: Validate changelog (if new version) + run: | + NEW_VERSION=`git diff -U0 master package.json | grep '"version": "' | tail -n 1 | grep -oE "[0-9]+\.[0-9]+\.[0-9]+"` || : + if [ -n "$NEW_VERSION" ]; + then + npx dump-release-notes-from-cc-changelog $NEW_VERSION + fi + - name: Unit tests + run: npm test + + windowsNode14: + name: '[Windows] Node.js v14: Unit tests' + runs-on: windows-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: | + npm-v14-${{ runner.os }}-${{ github.ref }}- + npm-v14-${{ runner.os }}-refs/heads/master- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: npm test + + linuxNode12: + name: '[Linux] Node.js v12: Unit tests' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.6] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Retrieve dependencies from cache + id: cacheNpm + uses: actions/cache@v2 + with: + path: | + ~/.npm + node_modules + key: npm-v12-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('package.json') }} + restore-keys: | + npm-v12-${{ runner.os }}-${{ github.ref }}- + npm-v12-${{ runner.os }}-refs/heads/master- + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Node.js and npm + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - name: Check python version + run: | + python --version + + - name: Install setuptools + run: python -m pip install --force setuptools wheel + + - name: Install pipenv / poetry + run: python -m pip install pipenv poetry + + - name: Install serverless + run: npm install -g serverless@2 + + - name: Install dependencies + if: steps.cacheNpm.outputs.cache-hit != 'true' + run: | + npm update --no-save + npm update --save-dev --no-save + - name: Unit tests + run: npm test diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..bd87ce72 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. + +### [5.2.1](https://github.com/UnitedIncome/serverless-python-requirements/compare/v5.2.0...v5.2.1) (2021-11-30) + +### Maintenance Improvements + +- Adapt plugin to modern logs ([#646](https://github.com/serverless/serverless-python-requirements/pull/646)) ([8ff97e6](https://github.com/UnitedIncome/serverless-python-requirements/commit/8ff97e6b7c279334e417dbdb65e64d0de2656986)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Adapt to `async` version of `spawn` ([#648](https://github.com/serverless/serverless-python-requirements/pull/648)) ([50c2850](https://github.com/UnitedIncome/serverless-python-requirements/commit/50c2850874ded795fd50ae377f1db817a0212e7d)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Adapt v3 log writing interfaces ([#646](https://github.com/serverless/serverless-python-requirements/pull/646)) ([a79899a](https://github.com/UnitedIncome/serverless-python-requirements/commit/a79899ae5f6f66aa0c65e7fda8e0186d38ff446e)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Ensure proper verbose progress logs ([#646](https://github.com/serverless/serverless-python-requirements/pull/646)) ([44b9591](https://github.com/UnitedIncome/serverless-python-requirements/commit/44b9591f01157a1811e3ca8b43e21265a155a976)) ([Piotr Grzesik](https://github.com/pgrzesik)) +- Use `ServerlessError` ([#649](https://github.com/serverless/serverless-python-requirements/pull/649)) ([cdb7111](https://github.com/UnitedIncome/serverless-python-requirements/commit/cdb71110bc9c69b5087b6e18fb353d65962afe4a)) ([Piotr Grzesik](https://github.com/pgrzesik)) + +# Changelog + +All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. diff --git a/README.md b/README.md index 518d5ce1..abe6a175 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,13 @@ A Serverless v1.x plugin to automatically bundle dependencies from `requirements.txt` and make them available in your `PYTHONPATH`. ---- +--- _Originally developed by [**Capital One**](https://www.capitalone.com/tech/open-source/), now maintained in scope of Serverless, Inc_ _Capital One considers itself the bank a technology company would build. It's delivering best-in-class innovation so that its millions of customers can manage their finances with ease. Capital One is all-in on the cloud and is a leader in the adoption of open source, RESTful APIs, microservices and containers. We build our own products and release them with a speed and agility that allows us to get new customer experiences to market quickly. Our engineers use artificial intelligence and machine learning to transform real-time data, software and algorithms into the future of finance, reimagined._ ---- +--- ## Install @@ -557,10 +557,10 @@ package: - [@andrewfarley](https://github.com/andrewfarley) - Implemented download caching and static caching - [@bweigel](https://github.com/bweigel) - adding the `slimPatternsAppendDefaults` option & fixing per-function packaging when some functions don't have requirements & Porting tests from bats to js! - Poetry support - - [@squaresurf](https://github.com/squaresurf) - - [@drice](https://github.com/drice) - - [@ofercaspi](https://github.com/ofercaspi) - - [@tpansino](https://github.com/tpansino) + - [@squaresurf](https://github.com/squaresurf) + - [@drice](https://github.com/drice) + - [@ofercaspi](https://github.com/ofercaspi) + - [@tpansino](https://github.com/tpansino) - [@david-mk-lawrence](https://github.com/david-mk-lawrence) - added Lambda Layer support - [@bryantbriggs](https://github.com/bryantbiggs) - Fixing CI/CD - [@jacksgt](https://github.com/jacksgt) - Fixing pip issues diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 00000000..d23a0d6b --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,31 @@ +'use strict'; + +module.exports = { + rules: { + 'body-leading-blank': [2, 'always'], + 'footer-leading-blank': [2, 'always'], + 'header-max-length': [2, 'always', 72], + 'scope-enum': [2, 'always', ['', 'Config', 'Log']], + 'subject-case': [2, 'always', 'sentence-case'], + 'subject-empty': [2, 'never'], + 'subject-full-stop': [2, 'never', '.'], + 'type-case': [2, 'always', 'lower-case'], + 'type-empty': [2, 'never'], + 'type-enum': [ + 2, + 'always', + [ + 'build', + 'chore', + 'ci', + 'docs', + 'feat', + 'fix', + 'perf', + 'refactor', + 'style', + 'test', + ], + ], + }, +}; diff --git a/index.js b/index.js index 2072bbc1..7741a7f8 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ const values = require('lodash.values'); const { addVendorHelper, removeVendorHelper, - packRequirements + packRequirements, } = require('./lib/zip'); const { injectAllRequirements } = require('./lib/inject'); const { layerRequirements } = require('./lib/layer'); @@ -57,7 +57,7 @@ class ServerlessPythonRequirements { staticCacheMaxVersions: 0, pipCmdExtraArgs: [], noDeploy: [], - vendor: '' + vendor: '', }, (this.serverless.service.custom && this.serverless.service.custom.pythonRequirements) || @@ -74,9 +74,15 @@ class ServerlessPythonRequirements { (options.dockerSsh || options.dockerImage || options.dockerFile) ) { if (!this.warningLogged) { - this.serverless.cli.log( - 'WARNING: You provided a docker related option but dockerizePip is set to false.' - ); + if (this.log) { + this.log.warning( + 'You provided a docker related option but dockerizePip is set to false.' + ); + } else { + this.serverless.cli.log( + 'WARNING: You provided a docker related option but dockerizePip is set to false.' + ); + } this.warningLogged = true; } } @@ -109,9 +115,10 @@ class ServerlessPythonRequirements { * The plugin constructor * @param {Object} serverless * @param {Object} options + * @param {Object} v3Utils * @return {undefined} */ - constructor(serverless) { + constructor(serverless, cliOptions, v3Utils) { this.serverless = serverless; this.servicePath = this.serverless.config.servicePath; this.warningLogged = false; @@ -127,36 +134,43 @@ class ServerlessPythonRequirements { }, }); } + + if (v3Utils) { + this.log = v3Utils.log; + this.progress = v3Utils.progress; + this.writeText = v3Utils.writeText; + } + this.commands = { requirements: { commands: { clean: { usage: 'Remove .requirements and requirements.zip', - lifecycleEvents: ['clean'] + lifecycleEvents: ['clean'], }, install: { usage: 'install requirements manually', - lifecycleEvents: ['install'] + lifecycleEvents: ['install'], }, cleanCache: { usage: 'Removes all items in the pip download/static cache (if present)', - lifecycleEvents: ['cleanCache'] - } - } - } + lifecycleEvents: ['cleanCache'], + }, + }, + }, }; - if (this.serverless.cli.generateCommandsHelp) { - Object.assign(this.commands.requirements, { - usage: 'Serverless plugin to bundle Python packages', - lifecycleEvents: ['requirements'] - }); - } else { - this.commands.requirements.type = 'container'; - } + if (this.serverless.cli.generateCommandsHelp) { + Object.assign(this.commands.requirements, { + usage: 'Serverless plugin to bundle Python packages', + lifecycleEvents: ['requirements'], + }); + } else { + this.commands.requirements.type = 'container'; + } - const isFunctionRuntimePython = args => { + const isFunctionRuntimePython = (args) => { // If functionObj.runtime is undefined, python. if (!args[1].functionObj || !args[1].functionObj.runtime) { return true; @@ -165,9 +179,7 @@ class ServerlessPythonRequirements { }; const clean = () => - BbPromise.bind(this) - .then(cleanup) - .then(removeVendorHelper); + BbPromise.bind(this).then(cleanup).then(removeVendorHelper); const setupArtifactPathCapturing = () => { // Reference: @@ -231,7 +243,7 @@ class ServerlessPythonRequirements { }, 'requirements:install:install': before, 'requirements:clean:clean': clean, - 'requirements:cleanCache:cleanCache': cleanCache + 'requirements:cleanCache:cleanCache': cleanCache, }; } } diff --git a/lib/clean.js b/lib/clean.js index e0bff238..8aaf331e 100644 --- a/lib/clean.js +++ b/lib/clean.js @@ -38,8 +38,15 @@ function cleanup() { function cleanupCache() { const cacheLocation = getUserCachePath(this.options); if (fse.existsSync(cacheLocation)) { + let cleanupProgress; if (this.serverless) { - this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`); + if (this.log) { + cleanupProgress = this.progress.get('python-cleanup-cache'); + cleanupProgress.notice('Removing static caches'); + this.log.info(`Removing static caches at: ${cacheLocation}`); + } else { + this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`); + } } // Only remove cache folders that we added, just incase someone accidentally puts a weird @@ -50,10 +57,16 @@ function cleanupCache() { .forEach((file) => { promises.push(fse.removeAsync(file)); }); - return BbPromise.all(promises); + return BbPromise.all(promises).finally( + () => cleanupProgress && cleanupProgress.remove() + ); } else { if (this.serverless) { - this.serverless.cli.log(`No static cache found`); + if (this.log) { + this.log.info(`No static cache found`); + } else { + this.serverless.cli.log(`No static cache found`); + } } return BbPromise.resolve(); } diff --git a/lib/docker.js b/lib/docker.js index 328e3088..5157803f 100644 --- a/lib/docker.js +++ b/lib/docker.js @@ -1,4 +1,4 @@ -const { spawnSync } = require('child_process'); +const spawn = require('child-process-ext/spawn'); const isWsl = require('is-wsl'); const fse = require('fs-extra'); const path = require('path'); @@ -8,18 +8,22 @@ const path = require('path'); * @param {string[]} options * @return {Object} */ -function dockerCommand(options) { +async function dockerCommand(options, pluginInstance) { const cmd = 'docker'; - const ps = spawnSync(cmd, options, { encoding: 'utf-8' }); - if (ps.error) { - if (ps.error.code === 'ENOENT') { - throw new Error('docker not found! Please install it.'); + try { + return await spawn(cmd, options, { encoding: 'utf-8' }); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + throw new pluginInstance.serverless.classes.Error( + 'docker not found! Please install it.', + 'PYTHON_REQUIREMENTS_DOCKER_NOT_FOUND' + ); } - throw new Error(ps.error); - } else if (ps.status !== 0) { - throw new Error(ps.stderr); + throw e; } - return ps; } /** @@ -28,19 +32,22 @@ function dockerCommand(options) { * @param {string[]} extraArgs * @return {string} The name of the built docker image. */ -function buildImage(dockerFile, extraArgs) { +async function buildImage(dockerFile, extraArgs, pluginInstance) { const imageName = 'sls-py-reqs-custom'; const options = ['build', '-f', dockerFile, '-t', imageName]; if (Array.isArray(extraArgs)) { options.push(...extraArgs); } else { - throw new Error('dockerRunCmdExtraArgs option must be an array'); + throw new pluginInstance.serverless.classes.Error( + 'dockerRunCmdExtraArgs option must be an array', + 'PYTHON_REQUIREMENTS_INVALID_DOCKER_EXTRA_ARGS' + ); } options.push('.'); - dockerCommand(options); + await dockerCommand(options, pluginInstance); return imageName; } @@ -49,7 +56,7 @@ function buildImage(dockerFile, extraArgs) { * @param {string} servicePath * @return {string} file name */ -function findTestFile(servicePath) { +function findTestFile(servicePath, pluginInstance) { if (fse.pathExistsSync(path.join(servicePath, 'serverless.yml'))) { return 'serverless.yml'; } @@ -62,8 +69,9 @@ function findTestFile(servicePath) { if (fse.pathExistsSync(path.join(servicePath, 'requirements.txt'))) { return 'requirements.txt'; } - throw new Error( - 'Unable to find serverless.{yml|yaml|json} or requirements.txt for getBindPath()' + throw new pluginInstance.serverless.classes.Error( + 'Unable to find serverless.{yml|yaml|json} or requirements.txt for getBindPath()', + 'PYTHON_REQUIREMENTS_MISSING_GET_BIND_PATH_FILE' ); } @@ -72,7 +80,8 @@ function findTestFile(servicePath) { * @param {string} bindPath * @return {boolean} */ -function tryBindPath(serverless, bindPath, testFile) { +async function tryBindPath(bindPath, testFile, pluginInstance) { + const { serverless, log } = pluginInstance; const debug = process.env.SLS_DEBUG; const options = [ 'run', @@ -84,12 +93,30 @@ function tryBindPath(serverless, bindPath, testFile) { `/test/${testFile}`, ]; try { - if (debug) serverless.cli.log(`Trying bindPath ${bindPath} (${options})`); - const ps = dockerCommand(options); - if (debug) serverless.cli.log(ps.stdout.trim()); - return ps.stdout.trim() === `/test/${testFile}`; + if (debug) { + if (log) { + log.debug(`Trying bindPath ${bindPath} (${options})`); + } else { + serverless.cli.log(`Trying bindPath ${bindPath} (${options})`); + } + } + const ps = await dockerCommand(options, pluginInstance); + if (debug) { + if (log) { + log.debug(ps.stdoutBuffer.trim()); + } else { + serverless.cli.log(ps.stdoutBuffer.trim()); + } + } + return ps.stdoutBuffer.trim() === `/test/${testFile}`; } catch (err) { - if (debug) serverless.cli.log(`Finding bindPath failed with ${err}`); + if (debug) { + if (log) { + log.debug(`Finding bindPath failed with ${err}`); + } else { + serverless.cli.log(`Finding bindPath failed with ${err}`); + } + } return false; } } @@ -100,14 +127,14 @@ function tryBindPath(serverless, bindPath, testFile) { * @param {string} servicePath * @return {string} The bind path. */ -function getBindPath(serverless, servicePath) { +async function getBindPath(servicePath, pluginInstance) { // Determine bind path if (process.platform !== 'win32' && !isWsl) { return servicePath; } // test docker is available - dockerCommand(['version']); + await dockerCommand(['version'], pluginInstance); // find good bind path for Windows let bindPaths = []; @@ -140,11 +167,11 @@ function getBindPath(serverless, servicePath) { bindPaths.push(`/mnt/${drive.toUpperCase()}/${path}`); bindPaths.push(`${drive.toUpperCase()}:/${path}`); - const testFile = findTestFile(servicePath); + const testFile = findTestFile(servicePath, pluginInstance); for (let i = 0; i < bindPaths.length; i++) { const bindPath = bindPaths[i]; - if (tryBindPath(serverless, bindPath, testFile)) { + if (await tryBindPath(bindPath, testFile, pluginInstance)) { return bindPath; } } @@ -157,7 +184,7 @@ function getBindPath(serverless, servicePath) { * @param {string} bindPath * @return {boolean} */ -function getDockerUid(bindPath) { +async function getDockerUid(bindPath, pluginInstance) { const options = [ 'run', '--rm', @@ -169,8 +196,8 @@ function getDockerUid(bindPath) { '%u', '/bin/sh', ]; - const ps = dockerCommand(options); - return ps.stdout.trim(); + const ps = await dockerCommand(options, pluginInstance); + return ps.stdoutBuffer.trim(); } module.exports = { buildImage, getBindPath, getDockerUid }; diff --git a/lib/inject.js b/lib/inject.js index 3cad758d..f32c9d46 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -86,57 +86,70 @@ function moveModuleUp(source, target, module) { * Inject requirements into packaged application. * @return {Promise} the combined promise for requirements injection. */ -function injectAllRequirements(funcArtifact) { +async function injectAllRequirements(funcArtifact) { if (this.options.layer) { // The requirements will be placed in a Layer, so just resolve return BbPromise.resolve(); } - this.serverless.cli.log('Injecting required Python packages to package...'); + let injectProgress; + if (this.progress && this.log) { + injectProgress = this.progress.get('python-inject-requirements'); + injectProgress.update('Injecting required Python packages to package'); + this.log.info('Injecting required Python packages to package'); + } else { + this.serverless.cli.log('Injecting required Python packages to package...'); + } - if (this.serverless.service.package.individually) { - return BbPromise.resolve(this.targetFuncs) - .filter((func) => - (func.runtime || this.serverless.service.provider.runtime).match( - /^python.*/ + try { + if (this.serverless.service.package.individually) { + await BbPromise.resolve(this.targetFuncs) + .filter((func) => + (func.runtime || this.serverless.service.provider.runtime).match( + /^python.*/ + ) ) - ) - .map((func) => { - if (!get(func, 'module')) { - set(func, ['module'], '.'); - } - return func; - }) - .map((func) => { - if (func.module !== '.') { - const artifact = func.package ? func.package.artifact : funcArtifact; - const newArtifact = path.join( - '.serverless', - `${func.module}-${func.name}.zip` - ); - func.package.artifact = newArtifact; - return moveModuleUp(artifact, newArtifact, func.module).then( - () => func - ); - } else { + .map((func) => { + if (!get(func, 'module')) { + set(func, ['module'], '.'); + } return func; - } - }) - .map((func) => { - return this.options.zip - ? func - : injectRequirements( - path.join('.serverless', func.module, 'requirements'), - func.package.artifact, - this.options + }) + .map((func) => { + if (func.module !== '.') { + const artifact = func.package + ? func.package.artifact + : funcArtifact; + const newArtifact = path.join( + '.serverless', + `${func.module}-${func.name}.zip` ); - }); - } else if (!this.options.zip) { - return injectRequirements( - path.join('.serverless', 'requirements'), - this.serverless.service.package.artifact || funcArtifact, - this.options - ); + func.package.artifact = newArtifact; + return moveModuleUp(artifact, newArtifact, func.module).then( + () => func + ); + } else { + return func; + } + }) + .map((func) => { + return this.options.zip + ? func + : injectRequirements( + path.join('.serverless', func.module, 'requirements'), + func.package.artifact, + this.options + ); + }); + } else if (!this.options.zip) { + await injectRequirements( + path.join('.serverless', 'requirements'), + this.serverless.service.package.artifact || funcArtifact, + this.options + ); + } + } finally { + injectProgress && injectProgress.remove(); } } diff --git a/lib/layer.js b/lib/layer.js index 12d338ec..fe2a4a00 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -53,9 +53,19 @@ function layerRequirements() { return BbPromise.resolve(); } - this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); + let layerProgress; + if (this.progress && this.log) { + layerProgress = this.progress.get('python-layer-requirements'); + layerProgress.update('Packaging Python Requirements Lambda Layer'); + this.log.info('Packaging Python Requirements Lambda Layer'); + } else { + this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); + } - return BbPromise.bind(this).then(zipRequirements).then(createLayers); + return BbPromise.bind(this) + .then(zipRequirements) + .then(createLayers) + .finally(() => layerProgress && layerProgress.remove()); } module.exports = { diff --git a/lib/pip.js b/lib/pip.js index 244010c8..7a0a0ceb 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -3,7 +3,7 @@ const rimraf = require('rimraf'); const path = require('path'); const get = require('lodash.get'); const set = require('lodash.set'); -const { spawnSync } = require('child_process'); +const spawn = require('child-process-ext/spawn'); const { quote } = require('shell-quote'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); const { getStripCommand, getStripMode, deleteFiles } = require('./slim'); @@ -57,10 +57,9 @@ function mergeCommands(commands) { function generateRequirementsFile( requirementsPath, targetFile, - serverless, - servicePath, - options + pluginInstance ) { + const { serverless, servicePath, options, log } = pluginInstance; if ( options.usePoetry && fse.existsSync(path.join(servicePath, 'pyproject.toml')) && @@ -69,12 +68,15 @@ function generateRequirementsFile( filterRequirementsFile( path.join(servicePath, '.serverless/requirements.txt'), targetFile, - options, - serverless - ); - serverless.cli.log( - `Parsed requirements.txt from pyproject.toml in ${targetFile}...` + pluginInstance ); + if (log) { + log.info(`Parsed requirements.txt from pyproject.toml in ${targetFile}`); + } else { + serverless.cli.log( + `Parsed requirements.txt from pyproject.toml in ${targetFile}...` + ); + } } else if ( options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile')) @@ -82,30 +84,49 @@ function generateRequirementsFile( filterRequirementsFile( path.join(servicePath, '.serverless/requirements.txt'), targetFile, - options, - serverless - ); - serverless.cli.log( - `Parsed requirements.txt from Pipfile in ${targetFile}...` + pluginInstance ); + if (log) { + log.info(`Parsed requirements.txt from Pipfile in ${targetFile}`); + } else { + serverless.cli.log( + `Parsed requirements.txt from Pipfile in ${targetFile}...` + ); + } } else { - filterRequirementsFile(requirementsPath, targetFile, options, serverless); - serverless.cli.log( - `Generated requirements from ${requirementsPath} in ${targetFile}...` - ); + filterRequirementsFile(requirementsPath, targetFile, pluginInstance); + if (log) { + log.info( + `Generated requirements from ${requirementsPath} in ${targetFile}` + ); + } else { + serverless.cli.log( + `Generated requirements from ${requirementsPath} in ${targetFile}...` + ); + } } } -function pipAcceptsSystem(pythonBin) { +async function pipAcceptsSystem(pythonBin, pluginInstance) { // Check if pip has Debian's --system option and set it if so - const pipTestRes = spawnSync(pythonBin, ['-m', 'pip', 'help', 'install']); - if (pipTestRes.error) { - if (pipTestRes.error.code === 'ENOENT') { - throw new Error(`${pythonBin} not found! Try the pythonBin option.`); + try { + const pipTestRes = await spawn(pythonBin, ['-m', 'pip', 'help', 'install']); + return ( + pipTestRes.stdoutBuffer && + pipTestRes.stdoutBuffer.toString().indexOf('--system') >= 0 + ); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + throw new pluginInstance.serverless.classes.Error( + `${pythonBin} not found! Install it according to the poetry docs.`, + 'PYTHON_REQUIREMENTS_PYTHON_NOT_FOUND' + ); } - throw pipTestRes.error; + throw e; } - return pipTestRes.stdout.toString().indexOf('--system') >= 0; } /** @@ -115,225 +136,294 @@ function pipAcceptsSystem(pythonBin) { * @param {Object} options * @return {undefined} */ -function installRequirements(targetFolder, serverless, options) { +async function installRequirements(targetFolder, pluginInstance) { + const { options, serverless, log, progress } = pluginInstance; const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt'); - serverless.cli.log( - `Installing requirements from ${targetRequirementsTxt} ...` - ); - - const dockerCmd = []; - const pipCmd = [options.pythonBin, '-m', 'pip', 'install']; - - if ( - Array.isArray(options.pipCmdExtraArgs) && - options.pipCmdExtraArgs.length > 0 - ) { - options.pipCmdExtraArgs.forEach((cmd) => { - const parts = cmd.split(/\s+/, 2); - pipCmd.push(...parts); - }); + let installProgress; + if (progress) { + log.info(`Installing requirements from "${targetRequirementsTxt}"`); + installProgress = progress.get('python-install'); + installProgress.update('Installing requirements'); + } else { + serverless.cli.log( + `Installing requirements from ${targetRequirementsTxt} ...` + ); } - const pipCmds = [pipCmd]; - const postCmds = []; - // Check if we're using the legacy --cache-dir command... - if (options.pipCmdExtraArgs.indexOf('--cache-dir') > -1) { - if (options.dockerizePip) { - throw ( - 'Error: You can not use --cache-dir with Docker any more, please\n' + - ' use the new option useDownloadCache instead. Please see:\n' + - ' https://github.com/UnitedIncome/serverless-python-requirements#caching' - ); - } else { - serverless.cli.log('=================================================='); - serverless.cli.log( - 'Warning: You are using a deprecated --cache-dir inside\n' + - ' your pipCmdExtraArgs which may not work properly, please use the\n' + - ' useDownloadCache option instead. Please see: \n' + - ' https://github.com/UnitedIncome/serverless-python-requirements#caching' - ); - serverless.cli.log('=================================================='); - } - } + try { + const dockerCmd = []; + const pipCmd = [options.pythonBin, '-m', 'pip', 'install']; - if (!options.dockerizePip) { - // Push our local OS-specific paths for requirements and target directory - pipCmd.push( - '-t', - dockerPathForWin(targetFolder), - '-r', - dockerPathForWin(targetRequirementsTxt) - ); - // If we want a download cache... - if (options.useDownloadCache) { - const downloadCacheDir = path.join( - getUserCachePath(options), - 'downloadCacheslspyc' - ); - serverless.cli.log(`Using download cache directory ${downloadCacheDir}`); - fse.ensureDirSync(downloadCacheDir); - pipCmd.push('--cache-dir', downloadCacheDir); + if ( + Array.isArray(options.pipCmdExtraArgs) && + options.pipCmdExtraArgs.length > 0 + ) { + options.pipCmdExtraArgs.forEach((cmd) => { + const parts = cmd.split(/\s+/, 2); + pipCmd.push(...parts); + }); } - if (pipAcceptsSystem(options.pythonBin)) { - pipCmd.push('--system'); - } - } - // If we are dockerizing pip - if (options.dockerizePip) { - // Push docker-specific paths for requirements and target directory - pipCmd.push('-t', '/var/task/', '-r', '/var/task/requirements.txt'); - - // Build docker image if required - let dockerImage; - if (options.dockerFile) { - serverless.cli.log( - `Building custom docker image from ${options.dockerFile}...` - ); - dockerImage = buildImage( - options.dockerFile, - options.dockerBuildCmdExtraArgs - ); - } else { - dockerImage = options.dockerImage; + const pipCmds = [pipCmd]; + const postCmds = []; + // Check if we're using the legacy --cache-dir command... + if (options.pipCmdExtraArgs.indexOf('--cache-dir') > -1) { + if (options.dockerizePip) { + throw new pluginInstance.serverless.classes.Error( + 'You cannot use --cache-dir with Docker any more, please use the new option useDownloadCache instead. Please see: https://github.com/UnitedIncome/serverless-python-requirements#caching for more details.', + 'PYTHON_REQUIREMENTS_CACHE_DIR_DOCKER_INVALID' + ); + } else { + if (log) { + log.warning( + 'You are using a deprecated --cache-dir inside\n' + + ' your pipCmdExtraArgs which may not work properly, please use the\n' + + ' useDownloadCache option instead. Please see: \n' + + ' https://github.com/UnitedIncome/serverless-python-requirements#caching' + ); + } else { + serverless.cli.log( + '==================================================' + ); + serverless.cli.log( + 'Warning: You are using a deprecated --cache-dir inside\n' + + ' your pipCmdExtraArgs which may not work properly, please use the\n' + + ' useDownloadCache option instead. Please see: \n' + + ' https://github.com/UnitedIncome/serverless-python-requirements#caching' + ); + serverless.cli.log( + '==================================================' + ); + } + } } - serverless.cli.log(`Docker Image: ${dockerImage}`); - - // Prepare bind path depending on os platform - const bindPath = dockerPathForWin(getBindPath(serverless, targetFolder)); - - dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); - if (options.dockerSsh) { - // Mount necessary ssh files to work with private repos - dockerCmd.push( - '-v', - `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`, - '-v', - `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, - '-v', - `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`, - '-e', - 'SSH_AUTH_SOCK=/tmp/ssh_sock' + + if (!options.dockerizePip) { + // Push our local OS-specific paths for requirements and target directory + pipCmd.push( + '-t', + dockerPathForWin(targetFolder), + '-r', + dockerPathForWin(targetRequirementsTxt) ); + // If we want a download cache... + if (options.useDownloadCache) { + const downloadCacheDir = path.join( + getUserCachePath(options), + 'downloadCacheslspyc' + ); + if (log) { + log.info(`Using download cache directory ${downloadCacheDir}`); + } else { + serverless.cli.log( + `Using download cache directory ${downloadCacheDir}` + ); + } + fse.ensureDirSync(downloadCacheDir); + pipCmd.push('--cache-dir', downloadCacheDir); + } + + if (await pipAcceptsSystem(options.pythonBin, pluginInstance)) { + pipCmd.push('--system'); + } } - // If we want a download cache... - const dockerDownloadCacheDir = '/var/useDownloadCache'; - if (options.useDownloadCache) { - const downloadCacheDir = path.join( - getUserCachePath(options), - 'downloadCacheslspyc' - ); - serverless.cli.log(`Using download cache directory ${downloadCacheDir}`); - fse.ensureDirSync(downloadCacheDir); - // This little hack is necessary because getBindPath requires something inside of it to test... - // Ugh, this is so ugly, but someone has to fix getBindPath in some other way (eg: make it use - // its own temp file) - fse.closeSync( - fse.openSync(path.join(downloadCacheDir, 'requirements.txt'), 'w') + // If we are dockerizing pip + if (options.dockerizePip) { + // Push docker-specific paths for requirements and target directory + pipCmd.push('-t', '/var/task/', '-r', '/var/task/requirements.txt'); + + // Build docker image if required + let dockerImage; + if (options.dockerFile) { + let buildDockerImageProgress; + if (progress) { + buildDockerImageProgress = progress.get( + 'python-install-build-docker' + ); + buildDockerImageProgress.update( + `Building custom docker image from ${options.dockerFile}` + ); + } else { + serverless.cli.log( + `Building custom docker image from ${options.dockerFile}...` + ); + } + try { + dockerImage = await buildImage( + options.dockerFile, + options.dockerBuildCmdExtraArgs, + pluginInstance + ); + } finally { + buildDockerImageProgress && buildDockerImageProgress.remove(); + } + } else { + dockerImage = options.dockerImage; + } + if (log) { + log.info(`Docker Image: ${dockerImage}`); + } else { + serverless.cli.log(`Docker Image: ${dockerImage}`); + } + + // Prepare bind path depending on os platform + const bindPath = dockerPathForWin( + await getBindPath(targetFolder, pluginInstance) ); - const windowsized = getBindPath(serverless, downloadCacheDir); - // And now push it to a volume mount and to pip... - dockerCmd.push('-v', `${windowsized}:${dockerDownloadCacheDir}:z`); - pipCmd.push('--cache-dir', dockerDownloadCacheDir); - } - if (options.dockerEnv) { - // Add environment variables to docker run cmd - options.dockerEnv.forEach(function (item) { - dockerCmd.push('-e', item); - }); - } + dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); + if (options.dockerSsh) { + // Mount necessary ssh files to work with private repos + dockerCmd.push( + '-v', + `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`, + '-v', + `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, + '-v', + `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`, + '-e', + 'SSH_AUTH_SOCK=/tmp/ssh_sock' + ); + } - if (process.platform === 'linux') { - // Use same user so requirements folder is not root and so --cache-dir works + // If we want a download cache... + const dockerDownloadCacheDir = '/var/useDownloadCache'; if (options.useDownloadCache) { - // Set the ownership of the download cache dir to root - pipCmds.unshift(['chown', '-R', '0:0', dockerDownloadCacheDir]); + const downloadCacheDir = path.join( + getUserCachePath(options), + 'downloadCacheslspyc' + ); + if (log) { + log.info(`Using download cache directory ${downloadCacheDir}`); + } else { + serverless.cli.log( + `Using download cache directory ${downloadCacheDir}` + ); + } + fse.ensureDirSync(downloadCacheDir); + // This little hack is necessary because getBindPath requires something inside of it to test... + // Ugh, this is so ugly, but someone has to fix getBindPath in some other way (eg: make it use + // its own temp file) + fse.closeSync( + fse.openSync(path.join(downloadCacheDir, 'requirements.txt'), 'w') + ); + const windowsized = await getBindPath(downloadCacheDir, pluginInstance); + // And now push it to a volume mount and to pip... + dockerCmd.push('-v', `${windowsized}:${dockerDownloadCacheDir}:z`); + pipCmd.push('--cache-dir', dockerDownloadCacheDir); } - // Install requirements with pip - // Set the ownership of the current folder to user - pipCmds.push([ - 'chown', - '-R', - `${process.getuid()}:${process.getgid()}`, - '/var/task', - ]); - } else { - // Use same user so --cache-dir works - dockerCmd.push('-u', getDockerUid(bindPath)); - } - for (let path of options.dockerExtraFiles) { - pipCmds.push(['cp', path, '/var/task/']); - } + if (options.dockerEnv) { + // Add environment variables to docker run cmd + options.dockerEnv.forEach(function (item) { + dockerCmd.push('-e', item); + }); + } - if (process.platform === 'linux') { - if (options.useDownloadCache) { - // Set the ownership of the download cache dir back to user + if (process.platform === 'linux') { + // Use same user so requirements folder is not root and so --cache-dir works + if (options.useDownloadCache) { + // Set the ownership of the download cache dir to root + pipCmds.unshift(['chown', '-R', '0:0', dockerDownloadCacheDir]); + } + // Install requirements with pip + // Set the ownership of the current folder to user pipCmds.push([ 'chown', '-R', `${process.getuid()}:${process.getgid()}`, - dockerDownloadCacheDir, + '/var/task', ]); + } else { + // Use same user so --cache-dir works + dockerCmd.push('-u', await getDockerUid(bindPath, pluginInstance)); + } + + for (let path of options.dockerExtraFiles) { + pipCmds.push(['cp', path, '/var/task/']); + } + + if (process.platform === 'linux') { + if (options.useDownloadCache) { + // Set the ownership of the download cache dir back to user + pipCmds.push([ + 'chown', + '-R', + `${process.getuid()}:${process.getgid()}`, + dockerDownloadCacheDir, + ]); + } } + + if (Array.isArray(options.dockerRunCmdExtraArgs)) { + dockerCmd.push(...options.dockerRunCmdExtraArgs); + } else { + throw new pluginInstance.serverless.classes.Error( + 'dockerRunCmdExtraArgs option must be an array', + 'PYTHON_REQUIREMENTS_INVALID_DOCKER_EXTRA_ARGS' + ); + } + + dockerCmd.push(dockerImage); } - if (Array.isArray(options.dockerRunCmdExtraArgs)) { - dockerCmd.push(...options.dockerRunCmdExtraArgs); - } else { - throw new Error('dockerRunCmdExtraArgs option must be an array'); + // If enabled slimming, strip so files + switch (getStripMode(options)) { + case 'docker': + pipCmds.push(getStripCommand(options, '/var/task')); + break; + case 'direct': + postCmds.push(getStripCommand(options, dockerPathForWin(targetFolder))); + break; } - dockerCmd.push(dockerImage); - } + let spawnArgs = { shell: true }; + if (process.env.SLS_DEBUG) { + spawnArgs.stdio = 'inherit'; + } + let mainCmds = []; + if (dockerCmd.length) { + dockerCmd.push(...mergeCommands(pipCmds)); + mainCmds = [dockerCmd]; + } else { + mainCmds = pipCmds; + } + mainCmds.push(...postCmds); - // If enabled slimming, strip so files - switch (getStripMode(options)) { - case 'docker': - pipCmds.push(getStripCommand(options, '/var/task')); - break; - case 'direct': - postCmds.push(getStripCommand(options, dockerPathForWin(targetFolder))); - break; - } + if (log) { + log.info(`Running ${quote(dockerCmd)}...`); + } else { + serverless.cli.log(`Running ${quote(dockerCmd)}...`); + } - let spawnArgs = { shell: true }; - if (process.env.SLS_DEBUG) { - spawnArgs.stdio = 'inherit'; - } - let mainCmds = []; - if (dockerCmd.length) { - dockerCmd.push(...mergeCommands(pipCmds)); - mainCmds = [dockerCmd]; - } else { - mainCmds = pipCmds; - } - mainCmds.push(...postCmds); - - serverless.cli.log(`Running ${quote(dockerCmd)}...`); - - filterCommands(mainCmds).forEach(([cmd, ...args]) => { - const res = spawnSync(cmd, args); - if (res.error) { - if (res.error.code === 'ENOENT') { - const advice = - cmd.indexOf('python') > -1 - ? 'Try the pythonBin option' - : 'Please install it'; - throw new Error(`${cmd} not found! ${advice}`); + for (const [cmd, ...args] of mainCmds) { + try { + await spawn(cmd, args); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + const advice = + cmd.indexOf('python') > -1 + ? 'Try the pythonBin option' + : 'Please install it'; + throw new pluginInstance.serverless.classes.Error( + `${cmd} not found! ${advice}`, + 'PYTHON_REQUIREMENTS_COMMAND_NOT_FOUND' + ); + } + throw e; } - throw res.error; } - if (res.status !== 0) { - throw new Error(`STDOUT: ${res.stdout}\n\nSTDERR: ${res.stderr}`); + // If enabled slimming, delete files in slimPatterns + if (options.slim === true || options.slim === 'true') { + deleteFiles(options, targetFolder); } - }); - // If enabled slimming, delete files in slimPatterns - if (options.slim === true || options.slim === 'true') { - deleteFiles(options, targetFolder); + } finally { + installProgress && installProgress.remove(); } } @@ -382,7 +472,7 @@ function getRequirements(source) { * @param {string} target requirements where results are written * @param {Object} options */ -function filterRequirementsFile(source, target, options, serverless) { +function filterRequirementsFile(source, target, { options, serverless, log }) { const noDeploy = new Set(options.noDeploy || []); const requirements = getRequirements(source); var prepend = []; @@ -404,9 +494,13 @@ function filterRequirementsFile(source, target, options, serverless) { // not required inside final archive and avoids pip bugs // see https://github.com/UnitedIncome/serverless-python-requirements/issues/240 req = req.split('-e')[1].trim(); - serverless.cli.log( - `Warning: Stripping -e flag from requirement ${req}` - ); + if (log) { + log.warning(`Stripping -e flag from requirement ${req}`); + } else { + serverless.cli.log( + `Warning: Stripping -e flag from requirement ${req}` + ); + } } // Keep options for later @@ -434,13 +528,19 @@ function filterRequirementsFile(source, target, options, serverless) { * @param {Object} serverless * @return {undefined} */ -function copyVendors(vendorFolder, targetFolder, serverless) { +function copyVendors(vendorFolder, targetFolder, { serverless, log }) { // Create target folder if it does not exist fse.ensureDirSync(targetFolder); - serverless.cli.log( - `Copying vendor libraries from ${vendorFolder} to ${targetFolder}...` - ); + if (log) { + log.info( + `Copying vendor libraries from ${vendorFolder} to ${targetFolder}` + ); + } else { + serverless.cli.log( + `Copying vendor libraries from ${vendorFolder} to ${targetFolder}...` + ); + } fse.readdirSync(vendorFolder).map((file) => { let source = path.join(vendorFolder, file); @@ -489,13 +589,12 @@ function requirementsFileExists(servicePath, options, fileName) { * @param {Object} serverless * @return {string} */ -function installRequirementsIfNeeded( - servicePath, +async function installRequirementsIfNeeded( modulePath, - options, funcOptions, - serverless + pluginInstance ) { + const { servicePath, options, serverless } = pluginInstance; // Our source requirements, under our service path, and our module path (if specified) const fileName = path.join(servicePath, modulePath, options.fileName); @@ -518,19 +617,19 @@ function installRequirementsIfNeeded( fse.ensureDirSync(requirementsTxtDirectory); const slsReqsTxt = path.join(requirementsTxtDirectory, 'requirements.txt'); - generateRequirementsFile( - fileName, - slsReqsTxt, - serverless, - servicePath, - options - ); + generateRequirementsFile(fileName, slsReqsTxt, pluginInstance); // If no requirements file or an empty requirements file, then do nothing if (!fse.existsSync(slsReqsTxt) || fse.statSync(slsReqsTxt).size == 0) { - serverless.cli.log( - `Skipping empty output requirements.txt file from ${slsReqsTxt}` - ); + if (pluginInstance.log) { + pluginInstance.log.info( + `Skipping empty output requirements.txt file from ${slsReqsTxt}` + ); + } else { + serverless.cli.log( + `Skipping empty output requirements.txt file from ${slsReqsTxt}` + ); + } return false; } @@ -541,7 +640,8 @@ function installRequirementsIfNeeded( const workingReqsFolder = getRequirementsWorkingPath( reqChecksum, requirementsTxtDirectory, - options + options, + serverless ); // Check if our static cache is present and is valid @@ -550,9 +650,15 @@ function installRequirementsIfNeeded( fse.existsSync(path.join(workingReqsFolder, '.completed_requirements')) && workingReqsFolder.endsWith('_slspyc') ) { - serverless.cli.log( - `Using static cache of requirements found at ${workingReqsFolder} ...` - ); + if (pluginInstance.log) { + pluginInstance.log.info( + `Using static cache of requirements found at ${workingReqsFolder}` + ); + } else { + serverless.cli.log( + `Using static cache of requirements found at ${workingReqsFolder} ...` + ); + } // We'll "touch" the folder, as to bring it to the start of the FIFO cache fse.utimesSync(workingReqsFolder, new Date(), new Date()); return workingReqsFolder; @@ -573,14 +679,14 @@ function installRequirementsIfNeeded( fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt')); // Then install our requirements from this folder - installRequirements(workingReqsFolder, serverless, options); + await installRequirements(workingReqsFolder, pluginInstance); // Copy vendor libraries to requirements folder if (options.vendor) { - copyVendors(options.vendor, workingReqsFolder, serverless); + copyVendors(options.vendor, workingReqsFolder, pluginInstance); } if (funcOptions.vendor) { - copyVendors(funcOptions.vendor, workingReqsFolder, serverless); + copyVendors(funcOptions.vendor, workingReqsFolder, pluginInstance); } // Then touch our ".completed_requirements" file so we know we can use this for static cache @@ -596,69 +702,62 @@ function installRequirementsIfNeeded( * pip install the requirements to the requirements directory * @return {undefined} */ -function installAllRequirements() { +async function installAllRequirements() { // fse.ensureDirSync(path.join(this.servicePath, '.serverless')); // First, check and delete cache versions, if enabled - checkForAndDeleteMaxCacheVersions(this.options, this.serverless); + checkForAndDeleteMaxCacheVersions(this); // Then if we're going to package functions individually... if (this.serverless.service.package.individually) { let doneModules = []; - this.targetFuncs - .filter((func) => - (func.runtime || this.serverless.service.provider.runtime).match( - /^python.*/ - ) + const filteredFuncs = this.targetFuncs.filter((func) => + (func.runtime || this.serverless.service.provider.runtime).match( + /^python.*/ ) - .map((f) => { - if (!get(f, 'module')) { - set(f, ['module'], '.'); - } - // If we didn't already process a module (functions can re-use modules) - if (!doneModules.includes(f.module)) { - const reqsInstalledAt = installRequirementsIfNeeded( - this.servicePath, - f.module, - this.options, - f, - this.serverless - ); - // Add modulePath into .serverless for each module so it's easier for injecting and for users to see where reqs are - let modulePath = path.join( - this.servicePath, - '.serverless', - `${f.module}`, - 'requirements' - ); - // Only do if we didn't already do it - if ( - reqsInstalledAt && - !fse.existsSync(modulePath) && - reqsInstalledAt != modulePath - ) { - if (this.options.useStaticCache) { - // Windows can't symlink so we have to copy on Windows, - // it's not as fast, but at least it works - if (process.platform == 'win32') { - fse.copySync(reqsInstalledAt, modulePath); - } else { - fse.symlink(reqsInstalledAt, modulePath); - } + ); + + for (const f of filteredFuncs) { + if (!get(f, 'module')) { + set(f, ['module'], '.'); + } + + // If we didn't already process a module (functions can re-use modules) + if (!doneModules.includes(f.module)) { + const reqsInstalledAt = await installRequirementsIfNeeded( + f.module, + f, + this + ); + // Add modulePath into .serverless for each module so it's easier for injecting and for users to see where reqs are + let modulePath = path.join( + this.servicePath, + '.serverless', + `${f.module}`, + 'requirements' + ); + // Only do if we didn't already do it + if ( + reqsInstalledAt && + !fse.existsSync(modulePath) && + reqsInstalledAt != modulePath + ) { + if (this.options.useStaticCache) { + // Windows can't symlink so we have to copy on Windows, + // it's not as fast, but at least it works + if (process.platform == 'win32') { + fse.copySync(reqsInstalledAt, modulePath); } else { - fse.rename(reqsInstalledAt, modulePath); + fse.symlink(reqsInstalledAt, modulePath); } + } else { + fse.rename(reqsInstalledAt, modulePath); } - doneModules.push(f.module); } - }); + doneModules.push(f.module); + } + } } else { - const reqsInstalledAt = installRequirementsIfNeeded( - this.servicePath, - '', - this.options, - {}, - this.serverless - ); + const reqsInstalledAt = await installRequirementsIfNeeded('', {}, this); // Add symlinks into .serverless for so it's easier for injecting and for users to see where reqs are let symlinkPath = path.join( this.servicePath, diff --git a/lib/pipenv.js b/lib/pipenv.js index 063fb5d8..5856d47b 100644 --- a/lib/pipenv.js +++ b/lib/pipenv.js @@ -1,12 +1,12 @@ const fse = require('fs-extra'); const path = require('path'); -const { spawnSync } = require('child_process'); +const spawn = require('child-process-ext/spawn'); const { EOL } = require('os'); /** * pipenv install */ -function pipfileToRequirements() { +async function pipfileToRequirements() { if ( !this.options.usePipenv || !fse.existsSync(path.join(this.servicePath, 'Pipfile')) @@ -14,31 +14,49 @@ function pipfileToRequirements() { return; } - this.serverless.cli.log('Generating requirements.txt from Pipfile...'); + let generateRequirementsProgress; + if (this.progress && this.log) { + generateRequirementsProgress = this.progress.get( + 'python-generate-requirements-pipfile' + ); + generateRequirementsProgress.update( + 'Generating requirements.txt from Pipfile' + ); + this.log.info('Generating requirements.txt from Pipfile'); + } else { + this.serverless.cli.log('Generating requirements.txt from Pipfile...'); + } - const res = spawnSync( - 'pipenv', - ['lock', '--requirements', '--keep-outdated'], - { - cwd: this.servicePath, - } - ); - if (res.error) { - if (res.error.code === 'ENOENT') { - throw new Error( - `pipenv not found! Install it with 'pip install pipenv'.` + try { + let res; + try { + res = await spawn( + 'pipenv', + ['lock', '--requirements', '--keep-outdated'], + { + cwd: this.servicePath, + } ); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + throw new this.serverless.classes.Error( + `pipenv not found! Install it according to the poetry docs.`, + 'PYTHON_REQUIREMENTS_PIPENV_NOT_FOUND' + ); + } + throw e; } - throw new Error(res.error); - } - if (res.status !== 0) { - throw new Error(res.stderr); + fse.ensureDirSync(path.join(this.servicePath, '.serverless')); + fse.writeFileSync( + path.join(this.servicePath, '.serverless/requirements.txt'), + removeEditableFlagFromRequirementsString(res.stdoutBuffer) + ); + } finally { + generateRequirementsProgress && generateRequirementsProgress.remove(); } - fse.ensureDirSync(path.join(this.servicePath, '.serverless')); - fse.writeFileSync( - path.join(this.servicePath, '.serverless/requirements.txt'), - removeEditableFlagFromRequirementsString(res.stdout) - ); } /** diff --git a/lib/poetry.js b/lib/poetry.js index 553a1392..23f43dc0 100644 --- a/lib/poetry.js +++ b/lib/poetry.js @@ -1,68 +1,92 @@ const fs = require('fs'); const fse = require('fs-extra'); const path = require('path'); -const { spawnSync } = require('child_process'); + +const spawn = require('child-process-ext/spawn'); const tomlParse = require('@iarna/toml/parse-string'); /** * poetry install */ -function pyprojectTomlToRequirements() { +async function pyprojectTomlToRequirements() { if (!this.options.usePoetry || !isPoetryProject(this.servicePath)) { return; } - this.serverless.cli.log('Generating requirements.txt from pyproject.toml...'); + let generateRequirementsProgress; + if (this.progress && this.log) { + generateRequirementsProgress = this.progress.get( + 'python-generate-requirements-toml' + ); + generateRequirementsProgress.update( + 'Generating requirements.txt from "pyproject.toml"' + ); + this.log.info('Generating requirements.txt from "pyproject.toml"'); + } else { + this.serverless.cli.log( + 'Generating requirements.txt from pyproject.toml...' + ); + } - const res = spawnSync( - 'poetry', - [ - 'export', - '--without-hashes', - '-f', - 'requirements.txt', - '-o', - 'requirements.txt', - '--with-credentials', - ], - { - cwd: this.servicePath, - } - ); - if (res.error) { - if (res.error.code === 'ENOENT') { - throw new Error( - `poetry not found! Install it according to the poetry docs.` + try { + try { + await spawn( + 'poetry', + [ + 'export', + '--without-hashes', + '-f', + 'requirements.txt', + '-o', + 'requirements.txt', + '--with-credentials', + ], + { + cwd: this.servicePath, + } ); + } catch (e) { + if ( + e.stderrBuffer && + e.stderrBuffer.toString().includes('command not found') + ) { + throw new this.serverless.classes.Error( + `poetry not found! Install it according to the poetry docs.`, + 'PYTHON_REQUIREMENTS_POETRY_NOT_FOUND' + ); + } + throw e; } - throw new Error(res.error); - } - if (res.status !== 0) { - throw new Error(res.stderr); - } - const editableFlag = new RegExp(/^-e /gm); - const sourceRequirements = path.join(this.servicePath, 'requirements.txt'); - const requirementsContents = fse.readFileSync(sourceRequirements, { - encoding: 'utf-8', - }); + const editableFlag = new RegExp(/^-e /gm); + const sourceRequirements = path.join(this.servicePath, 'requirements.txt'); + const requirementsContents = fse.readFileSync(sourceRequirements, { + encoding: 'utf-8', + }); - if (requirementsContents.match(editableFlag)) { - this.serverless.cli.log( - 'The generated file contains -e flags, removing them...' - ); - fse.writeFileSync( + if (requirementsContents.match(editableFlag)) { + if (this.log) { + this.log.info('The generated file contains -e flags, removing them'); + } else { + this.serverless.cli.log( + 'The generated file contains -e flags, removing them...' + ); + } + fse.writeFileSync( + sourceRequirements, + requirementsContents.replace(editableFlag, '') + ); + } + + fse.ensureDirSync(path.join(this.servicePath, '.serverless')); + fse.moveSync( sourceRequirements, - requirementsContents.replace(editableFlag, '') + path.join(this.servicePath, '.serverless', 'requirements.txt'), + { overwrite: true } ); + } finally { + generateRequirementsProgress && generateRequirementsProgress.remove(); } - - fse.ensureDirSync(path.join(this.servicePath, '.serverless')); - fse.moveSync( - sourceRequirements, - path.join(this.servicePath, '.serverless', 'requirements.txt'), - { overwrite: true } - ); } /** diff --git a/lib/shared.js b/lib/shared.js index 79b60cef..426d6c50 100644 --- a/lib/shared.js +++ b/lib/shared.js @@ -12,7 +12,7 @@ const sha256File = require('sha256-file'); * @param {Object} serverless * @return {undefined} */ -function checkForAndDeleteMaxCacheVersions(options, serverless) { +function checkForAndDeleteMaxCacheVersions({ serverless, options, log }) { // If we're using the static cache, and we have static cache max versions enabled if ( options.useStaticCache && @@ -42,10 +42,17 @@ function checkForAndDeleteMaxCacheVersions(options, serverless) { rimraf.sync(files[i]); items++; } + // Log the number of cache files flushed - serverless.cli.log( - `Removed ${items} items from cache because of staticCacheMaxVersions` - ); + if (log) { + log.info( + `Removed ${items} items from cache because of staticCacheMaxVersions` + ); + } else { + serverless.cli.log( + `Removed ${items} items from cache because of staticCacheMaxVersions` + ); + } } } } @@ -55,17 +62,20 @@ function checkForAndDeleteMaxCacheVersions(options, serverless) { * @param {string} subfolder * @param {string} servicePath * @param {Object} options + * @param {Object} serverless * @return {string} */ function getRequirementsWorkingPath( subfolder, requirementsTxtDirectory, - options + options, + serverless ) { // If we want to use the static cache if (options && options.useStaticCache) { if (subfolder) { - subfolder = subfolder + '_slspyc'; + const architecture = serverless.service.provider.architecture || 'x86_64'; + subfolder = `${subfolder}_${architecture}_slspyc`; } // If we have max number of cache items... diff --git a/lib/zip.js b/lib/zip.js index 2e872aa9..cba29450 100644 --- a/lib/zip.js +++ b/lib/zip.js @@ -30,9 +30,13 @@ function addVendorHelper() { }) .then((functions) => uniqBy(functions, (func) => func.module)) .map((f) => { - this.serverless.cli.log( - `Adding Python requirements helper to ${f.module}...` - ); + if (this.log) { + this.log.info(`Adding Python requirements helper to ${f.module}`); + } else { + this.serverless.cli.log( + `Adding Python requirements helper to ${f.module}...` + ); + } return fse.copyAsync( path.resolve(__dirname, '../unzip_requirements.py'), @@ -40,7 +44,11 @@ function addVendorHelper() { ); }); } else { - this.serverless.cli.log('Adding Python requirements helper...'); + if (this.log) { + this.log.info('Adding Python requirements helper'); + } else { + this.serverless.cli.log('Adding Python requirements helper...'); + } if (!get(this.serverless.service, 'package.patterns')) { set(this.serverless.service, ['package', 'patterns'], []); @@ -72,15 +80,25 @@ function removeVendorHelper() { }) .then((funcs) => uniqBy(funcs, (f) => f.module)) .map((f) => { - this.serverless.cli.log( - `Removing Python requirements helper from ${f.module}...` - ); + if (this.log) { + this.log.info( + `Removing Python requirements helper from ${f.module}` + ); + } else { + this.serverless.cli.log( + `Removing Python requirements helper from ${f.module}...` + ); + } return fse.removeAsync( path.join(this.servicePath, f.module, 'unzip_requirements.py') ); }); } else { - this.serverless.cli.log('Removing Python requirements helper...'); + if (this.log) { + this.log.info('Removing Python requirements helper'); + } else { + this.serverless.cli.log('Removing Python requirements helper...'); + } return fse.removeAsync( path.join(this.servicePath, 'unzip_requirements.py') ); @@ -104,21 +122,38 @@ function packRequirements() { }) .then((funcs) => uniqBy(funcs, (f) => f.module)) .map((f) => { - this.serverless.cli.log( - `Zipping required Python packages for ${f.module}...` - ); + let packProgress; + if (this.progress && this.log) { + packProgress = this.progress.get( + `python-pack-requirements-${f.module}` + ); + packProgress.update( + `Zipping required Python packages for ${f.module}` + ); + this.log.info(`Zipping required Python packages for ${f.module}`); + } else { + this.serverless.cli.log( + `Zipping required Python packages for ${f.module}...` + ); + } f.package.patterns.push(`${f.module}/.requirements.zip`); - return addTree( - new JSZip(), - `.serverless/${f.module}/requirements` - ).then((zip) => writeZip(zip, `${f.module}/.requirements.zip`)); + return addTree(new JSZip(), `.serverless/${f.module}/requirements`) + .then((zip) => writeZip(zip, `${f.module}/.requirements.zip`)) + .finally(() => packProgress && packProgress.remove()); }); } else { - this.serverless.cli.log('Zipping required Python packages...'); + let packProgress; + if (this.progress) { + packProgress = this.progress.get(`python-pack-requirements`); + } else { + this.serverless.cli.log('Zipping required Python packages...'); + } this.serverless.service.package.patterns.push('.requirements.zip'); - return addTree(new JSZip(), '.serverless/requirements').then((zip) => - writeZip(zip, path.join(this.servicePath, '.requirements.zip')) - ); + return addTree(new JSZip(), '.serverless/requirements') + .then((zip) => + writeZip(zip, path.join(this.servicePath, '.requirements.zip')) + ) + .finally(() => packProgress && packProgress.remove()); } } } diff --git a/package.json b/package.json index 1fed4c39..c6a1f5e6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "serverless-python-requirements", - "version": "5.2.0", + "version": "5.2.1", "engines": { "node": ">=12.0" }, @@ -38,16 +38,24 @@ "main": "index.js", "bin": {}, "scripts": { - "ci:lint": "eslint *.js lib/*.js --format junit --output-file ~/reports/eslint.xml && prettier -c '{.,lib}/*.{js,md}'", - "test": "node test.js", - "lint": "eslint *.js lib/*.js && prettier -c '{.,lib}/*.{js,md}'", - "format": "prettier --write '{.,lib}/*.{js,md}'" + "commitlint": "commitlint -f HEAD@{15}", + "lint": "eslint .", + "lint:updated": "pipe-git-updated --ext=js -- eslint", + "prepare-release": "standard-version && prettier --write CHANGELOG.md", + "prettier-check": "prettier -c --ignore-path .gitignore \"**/*.{css,html,js,json,md,yaml,yml}\"", + "prettier-check:updated": "pipe-git-updated --ext=css --ext=html --ext=js --ext=json --ext=md --ext=yaml --ext=yml -- prettier -c", + "prettify": "prettier --write --ignore-path .gitignore \"**/*.{css,html,js,json,md,yaml,yml}\"", + "prettify:updated": "pipe-git-updated --ext=css --ext=html --ext=js --ext=json --ext=md --ext=yaml --ext=yml -- prettier --write", + "test": "node test.js" }, "devDependencies": { "cross-spawn": "*", "eslint": "^7.32.0", + "git-list-updated": "^1.2.1", + "github-release-from-cc-changelog": "^2.2.0", "lodash": "^4.17.21", "prettier": "^2", + "standard-version": "^9.3.2", "tape": "*", "tape-promise": "*" }, @@ -55,6 +63,7 @@ "@iarna/toml": "^2.2.5", "appdirectory": "^0.1.0", "bluebird": "^3.7.2", + "child-process-ext": "^2.1.1", "fs-extra": "^9.1.0", "glob-all": "^3.2.1", "is-wsl": "^2.2.0", @@ -70,6 +79,14 @@ "peerDependencies": { "serverless": "^2.32" }, + "lint-staged": { + "*.js": [ + "eslint" + ], + "*.{css,html,js,json,md,yaml,yml}": [ + "prettier -c" + ] + }, "eslintConfig": { "extends": "eslint:recommended", "env": { @@ -84,6 +101,30 @@ "no-console": "off" } }, + "standard-version": { + "skip": { + "commit": true, + "tag": true + }, + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "perf", + "section": "Performance Improvements" + }, + { + "type": "refactor", + "section": "Maintenance Improvements" + } + ] + }, "prettier": { "semi": true, "singleQuote": true diff --git a/test.js b/test.js index 0322ab91..ccd1920c 100644 --- a/test.js +++ b/test.js @@ -10,7 +10,7 @@ const { copySync, writeFileSync, statSync, - pathExistsSync + pathExistsSync, } = require('fs-extra'); const { quote } = require('shell-quote'); const { sep } = require('path'); @@ -19,30 +19,32 @@ const { getUserCachePath, sha256Path } = require('./lib/shared'); const initialWorkingDir = process.cwd(); -const mkCommand = cmd => (args, options = {}) => { - const { error, stdout, stderr, status } = crossSpawn.sync( - cmd, - args, - Object.assign( - { - env: Object.assign({}, process.env, { SLS_DEBUG: 't' }) - }, - options - ) - ); - if (error) { - console.error(`Error running: ${quote([cmd, ...args])}`); // eslint-disable-line no-console - throw error; - } - if (status) { - console.error('STDOUT: ', stdout.toString()); // eslint-disable-line no-console - console.error('STDERR: ', stderr.toString()); // eslint-disable-line no-console - throw new Error( - `${quote([cmd, ...args])} failed with status code ${status}` - ); - } - return stdout && stdout.toString().trim(); -}; +const mkCommand = + (cmd) => + (args, options = {}) => { + const { error, stdout, stderr, status } = crossSpawn.sync( + cmd, + args, + Object.assign( + { + env: Object.assign({}, process.env, { SLS_DEBUG: 't' }), + }, + options + ) + ); + if (error) { + console.error(`Error running: ${quote([cmd, ...args])}`); // eslint-disable-line no-console + throw error; + } + if (status) { + console.error('STDOUT: ', stdout.toString()); // eslint-disable-line no-console + console.error('STDERR: ', stderr.toString()); // eslint-disable-line no-console + throw new Error( + `${quote([cmd, ...args])} failed with status code ${status}` + ); + } + return stdout && stdout.toString().trim(); + }; const sls = mkCommand('sls'); const git = mkCommand('git'); const npm = mkCommand('npm'); @@ -73,8 +75,8 @@ const teardown = () => { 'serverless.yml.bak', 'module1/foobar', getUserCachePath(), - ...glob.sync('serverless-python-requirements-*.tgz') - ].map(path => removeSync(path)); + ...glob.sync('serverless-python-requirements-*.tgz'), + ].map((path) => removeSync(path)); if (!cwd.endsWith('base with a space')) { try { git(['checkout', 'serverless.yml']); @@ -93,15 +95,17 @@ const teardown = () => { const testFilter = (() => { const elems = process.argv.slice(2); // skip ['node', 'test.js'] if (elems.length) { - return desc => - elems.some(text => desc.search(text) != -1) ? tape.test : tape.test.skip; + return (desc) => + elems.some((text) => desc.search(text) != -1) + ? tape.test + : tape.test.skip; } else { return () => tape.test; } })(); const test = (desc, func, opts = {}) => - testFilter(desc)(desc, opts, async t => { + testFilter(desc)(desc, opts, async (t) => { setup(); let ended = false; try { @@ -124,7 +128,7 @@ const availablePythons = (() => { const mapping = {}; if (process.env.USE_PYTHON) { binaries.push( - ...process.env.USE_PYTHON.split(',').map(v => v.toString().trim()) + ...process.env.USE_PYTHON.split(',').map((v) => v.toString().trim()) ); } else { // For running outside of CI @@ -135,7 +139,7 @@ const availablePythons = (() => { const python = `${bin}${exe}`; const { stdout, status } = crossSpawn.sync(python, [ '-c', - 'import sys; sys.stdout.write(".".join(map(str, sys.version_info[:2])))' + 'import sys; sys.stdout.write(".".join(map(str, sys.version_info[:2])))', ]); const ver = stdout && stdout.toString().trim(); if (!status && ver) { @@ -152,29 +156,29 @@ const availablePythons = (() => { return mapping; })(); -const getPythonBin = version => { +const getPythonBin = (version) => { const bin = availablePythons[String(version)]; if (!bin) throw new Error(`No python version ${version} available`); return bin; }; -const hasPython = version => { +const hasPython = (version) => { return Boolean(availablePythons[String(version)]); }; -const listZipFiles = async function(filename) { +const listZipFiles = async function (filename) { const file = await readFile(filename); const zip = await new JSZip().loadAsync(file); return Object.keys(zip.files); }; -const listZipFilesWithMetaData = async function(filename) { +const listZipFilesWithMetaData = async function (filename) { const file = await readFile(filename); const zip = await new JSZip().loadAsync(file); return Object(zip.files); }; -const listRequirementsZipFiles = async function(filename) { +const listRequirementsZipFiles = async function (filename) { const file = await readFile(filename); const zip = await new JSZip().loadAsync(file); const reqsBuffer = await zip.file('.requirements.zip').async('nodebuffer'); @@ -197,7 +201,7 @@ const brokenOn = (...platforms) => platforms.indexOf(process.platform) != -1; test( 'default pythonBin can package flask with default options', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -212,7 +216,7 @@ test( test( 'py3.6 packages have the same hash', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -231,7 +235,7 @@ test( test( 'py3.6 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -246,14 +250,14 @@ test( test( 'py3.6 can package flask with hashes', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ `--pythonBin=${getPythonBin(3)}`, '--fileName=requirements-w-hashes.txt', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); @@ -264,14 +268,14 @@ test( test( 'py3.6 can package flask with nested', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ `--pythonBin=${getPythonBin(3)}`, '--fileName=requirements-w-nested.txt', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); @@ -283,7 +287,7 @@ test( test( 'py3.6 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -308,7 +312,7 @@ test( test( 'py3.6 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -316,12 +320,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -331,7 +336,7 @@ test( test( 'py3.6 can package flask with slim & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -340,12 +345,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -356,7 +361,7 @@ test( test( "py3.6 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -365,7 +370,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls([`--pythonBin=${getPythonBin(3)}`, 'package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -378,14 +383,14 @@ test( test( 'py3.6 can package boto3 with editable', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ `--pythonBin=${getPythonBin(3)}`, '--fileName=requirements-w-editable.txt', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged'); @@ -400,7 +405,7 @@ test( test( 'py3.6 can package flask with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -416,7 +421,7 @@ test( test( 'py3.6 can package flask with slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -424,12 +429,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are NOT packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -439,7 +445,7 @@ test( test( 'py3.6 can package flask with slim & dockerizePip & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -448,12 +454,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -464,7 +470,7 @@ test( test( 'py3.6 can package flask with zip & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -497,7 +503,7 @@ test( test( 'py3.6 can package flask with zip & slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -530,7 +536,7 @@ test( test( 'py2.7 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -545,7 +551,7 @@ test( test( 'py2.7 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -553,17 +559,18 @@ test( `--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -573,7 +580,7 @@ test( test( 'py2.7 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -581,7 +588,7 @@ test( `--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', '--zip=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( @@ -603,7 +610,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -614,17 +621,17 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -635,7 +642,7 @@ test( test( "py2.7 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -644,7 +651,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls([`--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', 'package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -657,7 +664,7 @@ test( test( 'py2.7 can package flask with zip & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -666,7 +673,7 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--zip=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -696,7 +703,7 @@ test( test( 'py2.7 can package flask with zip & slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -706,7 +713,7 @@ test( '--dockerizePip=true', '--zip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -736,7 +743,7 @@ test( test( 'py2.7 can package flask with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -744,7 +751,7 @@ test( `--pythonBin=${getPythonBin(2)}`, '--runtime=python2.7', '--dockerizePip=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -757,7 +764,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -766,17 +773,18 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are NOT packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -786,7 +794,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -797,17 +805,17 @@ test( '--runtime=python2.7', '--dockerizePip=true', '--slim=true', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], '*.pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -818,7 +826,7 @@ test( test( 'pipenv py3.6 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -837,7 +845,7 @@ test( test( 'pipenv py3.6 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -845,12 +853,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -860,7 +869,7 @@ test( test( 'pipenv py3.6 can package flask with slim & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/pipenv'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -870,12 +879,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -886,7 +895,7 @@ test( test( 'pipenv py3.6 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -911,7 +920,7 @@ test( test( "pipenv py3.6 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/pipenv'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -920,7 +929,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls(['package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -933,7 +942,7 @@ test( test( 'non build pyproject.toml uses requirements.txt', - async t => { + async (t) => { process.chdir('tests/non_build_pyproject'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -948,7 +957,7 @@ test( test( 'non poetry pyproject.toml without requirements.txt packages handler only', - async t => { + async (t) => { process.chdir('tests/non_poetry_pyproject'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -962,7 +971,7 @@ test( test( 'poetry py3.6 can package flask with default options', - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -978,7 +987,7 @@ test( test( 'poetry py3.6 can package flask with slim option', - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -986,12 +995,13 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.true( - zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0, + zipfiles.filter((filename) => filename.endsWith('__main__.py')).length > + 0, '__main__.py files are packaged' ); t.end(); @@ -1001,7 +1011,7 @@ test( test( 'poetry py3.6 can package flask with slim & slimPatterns options', - async t => { + async (t) => { process.chdir('tests/poetry'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); @@ -1011,12 +1021,12 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1027,7 +1037,7 @@ test( test( 'poetry py3.6 can package flask with zip option', - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1052,7 +1062,7 @@ test( test( "poetry py3.6 doesn't package bottle with noDeploy option", - async t => { + async (t) => { process.chdir('tests/poetry'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1061,7 +1071,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls(['package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -1074,7 +1084,7 @@ test( test( 'py3.6 can package flask with zip option and no explicit include', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1101,7 +1111,7 @@ test( test( 'py3.6 can package lambda-decorators using vendor option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1120,7 +1130,7 @@ test( test( "Don't nuke execute perms", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); const perm = '755'; @@ -1131,7 +1141,7 @@ test( '-i.bak', '-e', 's/(handler.py.*$)/$1\n - foobar/', - 'serverless.yml' + 'serverless.yml', ]); writeFileSync(`foobar`, ''); chmodSync(`foobar`, perm); @@ -1169,7 +1179,7 @@ test( test( 'py3.6 can package flask in a project with a space in it', - async t => { + async (t) => { copySync('tests/base', 'tests/base with a space'); process.chdir('tests/base with a space'); const path = npm(['pack', '../..']); @@ -1185,7 +1195,7 @@ test( test( 'py3.6 can package flask in a project with a space in it with docker', - async t => { + async (t) => { copySync('tests/base', 'tests/base with a space'); process.chdir('tests/base with a space'); const path = npm(['pack', '../..']); @@ -1201,7 +1211,7 @@ test( test( 'py3.6 supports custom file name with fileName option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); writeFileSync('puck', 'requests'); @@ -1227,7 +1237,7 @@ test( test( "py3.6 doesn't package bottle with zip option", - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1236,7 +1246,7 @@ test( '-i.bak', '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', - 'serverless.yml' + 'serverless.yml', ]); sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); @@ -1270,7 +1280,7 @@ test( test( 'py3.6 can package flask with slim, slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1280,11 +1290,11 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1295,7 +1305,7 @@ test( test( 'py3.6 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1304,17 +1314,17 @@ test( '--dockerizePip=true', '--slim=true', '--slimPatternsAppendDefaults=false', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1325,7 +1335,7 @@ test( test( 'py2.7 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1334,17 +1344,17 @@ test( '--runtime=python2.7', '--slim=true', '--slimPatternsAppendDefaults=false', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1355,7 +1365,7 @@ test( test( 'py2.7 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', - async t => { + async (t) => { process.chdir('tests/base'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1365,16 +1375,16 @@ test( '--runtime=python2.7', '--slim=true', '--slimPatternsAppendDefaults=false', - 'package' + 'package', ]); const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1385,7 +1395,7 @@ test( test( 'pipenv py3.6 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', - async t => { + async (t) => { process.chdir('tests/pipenv'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1395,11 +1405,11 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1410,7 +1420,7 @@ test( test( 'poetry py3.6 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', - async t => { + async (t) => { process.chdir('tests/poetry'); copySync('_slimPatterns.yml', 'slimPatterns.yml'); const path = npm(['pack', '../..']); @@ -1420,11 +1430,11 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.true( - zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1, + zipfiles.filter((filename) => filename.endsWith('.pyc')).length >= 1, 'pyc files are packaged' ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('__main__.py')), + zipfiles.filter((filename) => filename.endsWith('__main__.py')), [], '__main__.py files are NOT packaged' ); @@ -1435,7 +1445,7 @@ test( test( 'py3.6 can package flask with package individually option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1520,11 +1530,9 @@ test( { skip: !hasPython(3.6) } ); - - test( 'py3.6 can package flask with package individually & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1536,7 +1544,7 @@ test( 'handler.py is packaged in function hello' ); t.deepEqual( - zipfiles_hello.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello' ); @@ -1555,7 +1563,7 @@ test( 'handler.py is packaged in function hello2' ); t.deepEqual( - zipfiles_hello2.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello2.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello2' ); @@ -1574,7 +1582,7 @@ test( 'handler.py is packaged in function hello3' ); t.deepEqual( - zipfiles_hello3.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello3.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello3' ); @@ -1599,7 +1607,7 @@ test( 'flask is NOT packaged in function hello4' ); t.deepEqual( - zipfiles_hello4.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello4.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello4' ); @@ -1611,7 +1619,7 @@ test( test( 'py2.7 can package flask with package individually option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1682,7 +1690,7 @@ test( test( 'py2.7 can package flask with package individually & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1690,7 +1698,7 @@ test( '--individually=true', '--runtime=python2.7', '--slim=true', - 'package' + 'package', ]); const zipfiles_hello = await listZipFiles('.serverless/hello.zip'); @@ -1699,7 +1707,7 @@ test( 'handler.py is packaged in function hello' ); t.deepEqual( - zipfiles_hello.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello' ); @@ -1718,7 +1726,7 @@ test( 'handler.py is packaged in function hello2' ); t.deepEqual( - zipfiles_hello2.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello2.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello2' ); @@ -1737,7 +1745,7 @@ test( 'handler.py is packaged in function hello3' ); t.deepEqual( - zipfiles_hello3.filter(filename => filename.endsWith('.pyc')), + zipfiles_hello3.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files packaged in function hello3' ); @@ -1773,7 +1781,7 @@ test( test( 'py2.7 can ignore functions defined with `image`', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1807,7 +1815,7 @@ test( test( 'py3.6 can package only requirements of module', - async t => { + async (t) => { process.chdir('tests/individually'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1868,7 +1876,7 @@ test( test( 'py3.6 can package lambda-decorators using vendor and invidiually option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -1950,7 +1958,7 @@ test( test( "Don't nuke execute perms when using individually", - async t => { + async (t) => { process.chdir('tests/individually'); const path = npm(['pack', '../..']); const perm = '755'; @@ -1974,8 +1982,9 @@ test( const zipfiles_hello2 = await listZipFilesWithMetaData( '.serverless/module2-sls-py-req-test-indiv-dev-hello2.zip' ); - const flaskPerm = statSync('.serverless/module2/requirements/bin/flask') - .mode; + const flaskPerm = statSync( + '.serverless/module2/requirements/bin/flask' + ).mode; t.true( zipfiles_hello2['bin/flask'].unixPermissions === flaskPerm, @@ -1989,7 +1998,7 @@ test( test( "Don't nuke execute perms when using individually w/docker", - async t => { + async (t) => { process.chdir('tests/individually'); const path = npm(['pack', '../..']); const perm = '755'; @@ -2013,8 +2022,9 @@ test( const zipfiles_hello2 = await listZipFilesWithMetaData( '.serverless/module2-sls-py-req-test-indiv-dev-hello2.zip' ); - const flaskPerm = statSync('.serverless/module2/requirements/bin/flask') - .mode; + const flaskPerm = statSync( + '.serverless/module2/requirements/bin/flask' + ).mode; t.true( zipfiles_hello2['bin/flask'].unixPermissions === flaskPerm, @@ -2028,7 +2038,7 @@ test( test( 'py3.6 uses download cache by default option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2045,7 +2055,7 @@ test( test( 'py3.6 uses download cache by default', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2061,7 +2071,7 @@ test( test( 'py3.6 uses download cache with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2078,14 +2088,14 @@ test( test( 'py3.6 uses download cache with dockerizePip by default option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls([ '--dockerizePip=true', '--cacheLocation=.requirements-cache', - 'package' + 'package', ]); t.true( pathExistsSync(`.requirements-cache${sep}downloadCacheslspyc${sep}http`), @@ -2098,19 +2108,22 @@ test( test( 'py3.6 uses static and download cache', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls(['package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), 'http exists in download-cache' ); t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.end(); @@ -2120,19 +2133,22 @@ test( test( 'py3.6 uses static and download cache with dockerizePip option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls(['--dockerizePip=true', 'package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( pathExistsSync(`${cachepath}${sep}downloadCacheslspyc${sep}http`), 'http exists in download-cache' ); t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.end(); @@ -2142,27 +2158,30 @@ test( test( 'py3.6 uses static cache', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls(['package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.true( pathExistsSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}.completed_requirements` + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}.completed_requirements` ), '.completed_requirements exists in static-cache' ); // py3.6 checking that static cache actually pulls from cache (by poisoning it) writeFileSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}injected_file_is_bad_form`, + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' ); sls(['package']); @@ -2180,20 +2199,23 @@ test( test( 'py3.6 uses static cache with cacheLocation option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); const cachepath = '.requirements-cache'; sls([`--cacheLocation=${cachepath}`, 'package']); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.true( pathExistsSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}.completed_requirements` + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}.completed_requirements` ), '.completed_requirements exists in static-cache' ); @@ -2204,27 +2226,30 @@ test( test( 'py3.6 uses static cache with dockerizePip & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls(['--dockerizePip=true', '--slim=true', 'package']); const cachepath = getUserCachePath(); const cacheFolderHash = sha256Path('.serverless/requirements.txt'); + const arch = 'x86_64'; t.true( - pathExistsSync(`${cachepath}${sep}${cacheFolderHash}_slspyc${sep}flask`), + pathExistsSync( + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}flask` + ), 'flask exists in static-cache' ); t.true( pathExistsSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}.completed_requirements` + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}.completed_requirements` ), '.completed_requirements exists in static-cache' ); // py3.6 checking that static cache actually pulls from cache (by poisoning it) writeFileSync( - `${cachepath}${sep}${cacheFolderHash}_slspyc${sep}injected_file_is_bad_form`, + `${cachepath}${sep}${cacheFolderHash}_${arch}_slspyc${sep}injected_file_is_bad_form`, 'injected new file into static cache folder' ); sls(['--dockerizePip=true', '--slim=true', 'package']); @@ -2235,7 +2260,7 @@ test( "static cache is really used when running 'sls package' again" ); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files are packaged' ); @@ -2247,7 +2272,7 @@ test( test( 'py3.6 uses download cache with dockerizePip & slim option', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); @@ -2261,7 +2286,7 @@ test( const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged'); t.deepEqual( - zipfiles.filter(filename => filename.endsWith('.pyc')), + zipfiles.filter((filename) => filename.endsWith('.pyc')), [], 'no pyc files are packaged' ); @@ -2273,13 +2298,12 @@ test( test( 'py3.6 can ignore functions defined with `image`', - async t => { + async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); sls(['--individually=true', 'package']); - t.true( pathExistsSync('.serverless/hello.zip'), 'function hello is packaged' diff --git a/tests/base/_slimPatterns.yml b/tests/base/_slimPatterns.yml index 02c631b4..443af9a0 100644 --- a/tests/base/_slimPatterns.yml +++ b/tests/base/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "**/__main__.py" + - '**/__main__.py' diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index 6bb1f322..0b360e9b 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -50,6 +50,3 @@ functions: - 'fn2/**' hello5: image: 000000000000.dkr.ecr.sa-east-1.amazonaws.com/test-lambda-docker@sha256:6bb600b4d6e1d7cf521097177dd0c4e9ea373edb91984a505333be8ac9455d38 - - - diff --git a/tests/pipenv/_slimPatterns.yml b/tests/pipenv/_slimPatterns.yml index 02c631b4..443af9a0 100644 --- a/tests/pipenv/_slimPatterns.yml +++ b/tests/pipenv/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "**/__main__.py" + - '**/__main__.py' diff --git a/tests/poetry/_slimPatterns.yml b/tests/poetry/_slimPatterns.yml index 02c631b4..443af9a0 100644 --- a/tests/poetry/_slimPatterns.yml +++ b/tests/poetry/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "**/__main__.py" + - '**/__main__.py'