diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7c592a95..00000000 --- a/.coveragerc +++ /dev/null @@ -1,29 +0,0 @@ -[run] -branch = True -source = - . -omit = - .tox/* - /usr/* - setup.py - -[report] -show_missing = True -skip_covered = True -exclude_lines = - # Have to re-enable the standard pragma - \#\s*pragma: no cover - - # Don't complain if tests don't hit defensive assertion code: - ^\s*raise AssertionError\b - ^\s*raise NotImplementedError\b - ^\s*return NotImplemented\b - ^\s*raise$ - - # Don't complain if non-runnable code isn't run: - ^if __name__ == ['"]__main__['"]:$ - -[html] -directory = coverage-html - -# vim:ft=dosini diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..9408e44d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +github: asottile +open_collective: pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d4a3a1aa..2bd89e0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v4.0.1 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -12,33 +12,35 @@ repos: - id: name-tests-test - id: double-quote-string-fixer - id: requirements-txt-fixer -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.1 +- repo: https://github.com/PyCQA/flake8 + rev: 3.9.2 hooks: - id: flake8 + additional_dependencies: [flake8-typing-imports==1.7.0] - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.4.3 + rev: v1.5.7 hooks: - id: autopep8 -- repo: https://github.com/pre-commit/pre-commit - rev: v1.14.2 - hooks: - - id: validate_manifest - repo: https://github.com/asottile/reorder_python_imports - rev: v1.3.5 + rev: v2.5.0 hooks: - id: reorder-python-imports - language_version: python3 + args: [--py3-plus] - repo: https://github.com/asottile/pyupgrade - rev: v1.11.1 + rev: v2.15.0 hooks: - id: pyupgrade + args: [--py36-plus] - repo: https://github.com/asottile/add-trailing-comma - rev: v0.7.1 + rev: v2.1.0 hooks: - id: add-trailing-comma + args: [--py36-plus] +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.17.0 + hooks: + - id: setup-cfg-fmt - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.660 + rev: v0.812 hooks: - id: mypy - language_version: python3 diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 7a3b380a..91dbdf0b 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,10 +1,3 @@ -- id: autopep8-wrapper - name: autopep8 wrapper - description: This is deprecated, use pre-commit/mirrors-autopep8 instead. - entry: autopep8-wrapper - language: python - types: [python] - args: [-i] - id: check-added-large-files name: Check for added large files description: Prevent giant files from being committed @@ -17,8 +10,8 @@ language: python types: [python] - id: check-byte-order-marker - name: Check for byte-order marker - description: Forbid files which have a UTF-8 byte-order marker + name: 'check BOM - deprecated: use fix-byte-order-marker' + description: forbid files which have a UTF-8 byte-order marker entry: check-byte-order-marker language: python types: [text] @@ -52,6 +45,13 @@ entry: check-json language: python types: [json] +- id: check-shebang-scripts-are-executable + name: Check that scripts with shebangs are executable + description: Ensures that (non-binary) files with a shebang are executable. + entry: check-shebang-scripts-are-executable + language: python + types: [text] + stages: [commit, push, manual] - id: pretty-format-json name: Pretty format JSON description: This hook sets a standard for formatting JSON files. @@ -100,6 +100,12 @@ entry: debug-statement-hook language: python types: [python] +- id: destroyed-symlinks + name: Detect Destroyed Symlinks + description: Detects symlinks which are changed to regular files with a content of a path which that symlink was pointing to. + entry: destroyed-symlinks + language: python + types: [file] - id: detect-aws-credentials name: Detect AWS Credentials description: Detects *your* aws credentials from the aws cli credentials file @@ -131,19 +137,18 @@ entry: file-contents-sorter language: python files: '^$' +- id: fix-byte-order-marker + name: fix UTF-8 byte order marker + description: removes UTF-8 byte order marker + entry: fix-byte-order-marker + language: python + types: [text] - id: fix-encoding-pragma name: Fix python encoding pragma language: python entry: fix-encoding-pragma description: 'Add # -*- coding: utf-8 -*- to the top of python files' types: [python] -- id: flake8 - name: Flake8 - description: This hook runs flake8. - entry: flake8 - language: python - types: [python] - require_serial: true - id: forbid-new-submodules name: Forbid new submodules language: python @@ -167,12 +172,6 @@ language: python pass_filenames: false always_run: true -- id: pyflakes - name: Pyflakes (DEPRECATED, use flake8) - description: This hook runs pyflakes. (This is deprecated, use flake8). - entry: pyflakes - language: python - types: [python] - id: requirements-txt-fixer name: Fix requirements.txt description: Sorts entries in requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 4327dad3..530c4c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,192 @@ -2.3.0 -===== +4.0.1 - 2021-05-16 +================== + +### Fixes +- `check-shebang-scripts-are-executable` fix entry point. + - #602 issue by @Person-93. + - #603 PR by @scop. + +4.0.0 - 2021-05-14 +================== + +### Features +- `check-json`: report duplicate keys. + - #558 PR by @AdityaKhursale. + - #554 issue by @adamchainz. +- `no-commit-to-branch`: add `main` to default blocked branches. + - #565 PR by @ndevenish. +- `check-case-conflict`: check conflicts in directory names as well. + - #575 PR by @slsyy. + - #70 issue by @andyjack. +- `check-vcs-permalinks`: forbid other branch names. + - #582 PR by @jack1142. + - #581 issue by @jack1142. +- `check-shebang-scripts-are-executable`: new hook which ensures shebang'd + scripts are executable. + - #545 PR by @scop. + +### Fixes +- `check-executables-have-shebangs`: Short circuit shebang lookup on windows. + - #544 PR by @scop. +- `requirements-txt-fixer`: Fix comments which have indentation + - #549 PR by @greshilov. + - #548 issue by @greshilov. +- `pretty-format-json`: write to stdout using UTF-8 encoding. + - #571 PR by @jack1142. + - #570 issue by @jack1142. +- Use more inclusive language. + - #599 PR by @asottile. + +### Breaking changes +- Remove deprecated hooks: `flake8`, `pyflakes`, `autopep8-wrapper`. + - #597 PR by @asottile. + + +3.4.0 - 2020-12-15 +================== + +### Features +- `file-contents-sorter`: Add `--unique` argument + - #524 PR by @danielhoherd. +- `check-vcs-permalinks`: Add `--additional-github-domain` option + - #530 PR by @youngminz. +- New hook: `destroyed-symlinks` to detect unintentional symlink-breakages on + windows. + - #511 PR by @m-khvoinitsky. + +3.3.0 - 2020-10-20 +================== + +### Features +- `file-contents-sorter`: add `--ignore-case` option for case-insensitive + sorting + - #514 PR by @Julian. +- `check-added-large-files`: add `--enforce-all` option to check non-added + files as well + - #519 PR by @mshawcroft. + - #518 issue by @mshawcroft. +- `fix-byte-order-marker`: new hook which fixes UTF-8 byte-order marker. + - #522 PR by @jgowdy. + +### Deprecations +- `check-byte-order-marker` is now deprecated for `fix-byte-order-marker` + +3.2.0 - 2020-07-30 +================== + +### Features +- `debug-statements`: add support for `pydevd_pycharm` debugger + - #502 PR by @jgeerds. + +### Fixes +- `check-executables-have-shebangs`: fix git-quoted files on windows (spaces, + non-ascii, etc.) + - #509 PR by @pawamoy. + - #508 issue by @pawamoy. + +3.1.0 - 2020-05-20 +================== + +### Features +- `check-executables-have-shebangs`: on windows, validate the mode bits using + `git` + - #480 PR by @mxr. + - #435 issue by @dstandish. +- `requirements-txt-fixer`: support more operators + - #483 PR by @mxr. + - #331 issue by @hackedd. + +### Fixes +- `pre-commit-hooks-removed`: Fix when removed hooks used `args` + - #487 PR by @pedrocalleja. + - #485 issue by @pedrocalleja. + +3.0.1 - 2020-05-16 +================== + +### Fixes +- `check-toml`: use UTF-8 encoding to load toml files + - #479 PR by @mxr. + - #474 issue by @staticdev. + +3.0.0 - 2020-05-14 +================== + +### Features +- `detect-aws-credentials`: skip empty aws keys + - #450 PR by @begoon. + - #449 issue by @begoon. +- `debug-statements`: add detection `wdb` debugger + - #452 PR by @itsdkey. + - #451 issue by @itsdkey. +- `requirements-txt-fixer`: support line continuation for dependencies + - #469 PR by @aniketbhatnagar. + - #465 issue by @aniketbhatnagar. + +### Fixes +- `detect-aws-credentials`: fix `UnicodeDecodeError` when running on non-UTF8 + files. + - #453 PR by @asottile. + - #393 PR by @a7p + - #346 issue by @rpdelaney. + +### Updating +- pre-commit/pre-commit-hooks now requires python3.6.1+ + - #447 PR by @asottile. + - #455 PR by @asottile. +- `flake8` / `pyflakes` have been removed, use `flake8` from `pycqa/flake8` + instead: + + ```yaml + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.1 + hooks: + - id: flake8 + ``` + + - #476 PR by @asottile. + - #477 PR by @asottile. + - #344 issue by @asottile. + + +2.5.0 - 2020-02-04 +================== + +### Fixes +- Fix sorting of requirements which use `egg=...` + - #425 PR by @vinayinvicible. +- Fix over-eager regular expression for test filename matching + - #429 PR by @rrauenza. + +### Updating +- Use `flake8` from `pycqa/flake8` instead: + + ```yaml + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.9 + hooks: + - id: flake8 + ``` + +2.4.0 - 2019-10-28 +================== + +### Features +- Add diff output to `pretty-format-json` when run without `--autofix`. + - #408 PR by @joepin. +- Add `--chars` option to `trailing-whitespace` fixer to control which + characters are stripped instead of all whitespace. + - #421 PR by @iconmaster5326. + +### Fixes +- Fix `requirements-txt-fixer` when file does not end in a newline. + - #414 issue by @barakreif. + - #415 PR by @barakreif. +- Fix double printing of filename in `pretty-format-json`. + - #419 PR by @asottile. + +2.3.0 - 2019-08-05 +================== ### Features - Add `rpdb` to detected debuggers in `debug-statements` @@ -17,31 +204,31 @@ - Fix `git-lfs` tests in azure pipelines - #403 PR by @ssbarnea. -2.2.3 -===== +2.2.3 - 2019-05-16 +================== ### Fixes - Handle CRLF line endings in `double-quote-string-fixer` - #385 issue by @Trim21. - #386 PR by @asottile. -2.2.2 -===== +2.2.2 - 2019-05-15 +================== ### Fixes - Handle CRLF line endings in `fix-encoding-pragma` - #384 PR by @asottile. -2.2.1 -===== +2.2.1 - 2019-04-21 +================== ### Fixes - Use UTF-8 to load yaml files - #377 issue by @roottool. - #378 PR by @roottool. -2.2.0 -===== +2.2.0 - 2019-04-20 +================== ### Features - Switch from `pyyaml` to `ruamel.yaml` @@ -71,8 +258,8 @@ - `pre-commit-hooks` now is type checked with mypy. - #360 PR by @asottile. -2.1.0 -===== +2.1.0 - 2018-12-26 +================== ### Features - Detect PGP/GPG private keys in `detect-private-key` @@ -93,8 +280,8 @@ - #343 PR by @TheKevJames. -2.0.0 -===== +2.0.0 - 2018-10-12 +================== ### Breaking changes @@ -125,8 +312,8 @@ [mirrors-autopep8]: https://github.com/pre-commit/mirrors-autopep8 -1.4.0-1 -======= +1.4.0-1 - 2018-09-27 +==================== (Note: this is a tag-only release as no code changes occurred) @@ -135,8 +322,8 @@ - #315 issue by @revolter. - #317 PR by @revolter. -1.4.0 -===== +1.4.0 - 2018-07-22 +================== ### Features - `no-commit-to-branch`: allow `--branch` to be specified multiple times @@ -159,8 +346,8 @@ - Test against python3.7 - #304 PR by @expobrain. -1.3.0 -===== +1.3.0 - 2018-05-28 +================== ### Features - Add an `--unsafe` argument to `check-yaml` to allow custom yaml tags @@ -181,23 +368,23 @@ - #285 issue by @EgoWumpus. - #286 PR by @asottile. -1.2.3 -===== +1.2.3 - 2018-02-28 +================== ### Fixes - `trailing-whitespace` entrypoint was incorrect. - f6780b9 by @asottile. -1.2.2 -===== +1.2.2 - 2018-02-28 +================== ### Fixes - `trailing-whitespace` no longer adds a missing newline at end-of-file - #270 issue by @fractos. - #271 PR by @asottile. -1.2.1-1 -======= +1.2.1-1 - 2018-02-24 +==================== (Note: this is a tag-only release as no code changes occurred) @@ -206,8 +393,8 @@ - #268 issue by @dongyuzheng. - #269 PR by @asottile. -1.2.1 -===== +1.2.1 - 2018-02-19 +================== ### Fixes: - `detect-aws-credentials` false positive when key was empty - #258 issue by @PVSec. @@ -216,8 +403,8 @@ - #265 issue by @hectorv. - #266 PR by @asottile. -1.2.0 -===== +1.2.0 - 2018-01-13 +================== ### Features: - Add new `check-builtin-literals` hook. - #249 #251 PR by @benwebber. @@ -227,14 +414,14 @@ - #255 issue by @SaMnCo @nicain. - #256 PR by @nicain. -1.1.1 -===== +1.1.1 - 2017-10-19 +================== ### Fixes: - Fix output interleaving in `check-vcs-permalinks` under python3. - #245 PR by @asottile. -1.1.0 -===== +1.1.0 - 2017-10-12 +================== ### Features: - `check-yaml` gains a `--allow-multiple-documents` (`-m`) argument to allow linting of files using the @@ -242,8 +429,8 @@ - pre-commit/pre-commit#635 issue by @geekobi. - #244 PR by @asottile. -1.0.0 -===== +1.0.0 - 2017-10-09 +================== ### Features: - New hook: `check-vcs-permalinks` for ensuring permalinked github urls. - #241 PR by @asottile. @@ -254,20 +441,20 @@ - Fix `requirements-txt-fixer` for files ending in comments - #243 PR by @asottile. -0.9.5 -===== +0.9.5 - 2017-09-27 +================== - Fix mixed-line-endings `--fix=...` when whole file is a different ending -0.9.4 -===== +0.9.4 - 2017-09-19 +================== - Fix entry point for `mixed-line-ending` -0.9.3 -===== +0.9.3 - 2017-09-07 +================== - New hook: `mixed-line-ending` -0.9.2 -===== +0.9.2 - 2017-08-21 +================== - Report full python version in `check-ast`. - Apply a more strict regular expression for `name-tests-test` - Upgrade binding for `git-lfs` for `check-added-large-files`. The oldest @@ -275,12 +462,12 @@ files as "lfs" (false negative) and earlier versions will crash. - `debug-statements` now works for non-utf-8 files. -0.9.1 -===== +0.9.1 - 2017-07-02 +================== - Add `check-executables-have-shebangs` hook. -0.9.0 -===== +0.9.0 - 2017-07-02 +================== - Add `sort-simple-yaml` hook - Fix `requirements-txt-fixer` for empty files - Add `file-contents-sorter` hook for sorting flat files @@ -289,24 +476,24 @@ allows the text processing hooks to match *all* text files (and to match files which would only be classifiable by their shebangs). -0.8.0 -===== +0.8.0 - 2017-06-06 +================== - Add flag allowing missing keys to `detect-aws-credentials` - Handle django default `tests.py` in `name-tests-test` - Add `--no-ensure-ascii` option to `pretty-format-json` - Add `no-commit-to-branch` hook -0.7.1 -===== +0.7.1 - 2017-02-07 +================== - Don't false positive on files where trailing whitespace isn't changed. -0.7.0 -===== +0.7.0 - 2017-01-21 +================== - Improve search for detecting aws keys - Add .pre-commit-hooks.yaml for forward compatibility -0.6.1 -===== +0.6.1 - 2016-11-30 +================== - trailing-whitespace-hook: restore original file on catastrophic failure - trailing-whitespace-hook: support crlf - check-yaml: Use safe_load @@ -314,20 +501,20 @@ - check-json: display filename for non-utf8 files - New hook: forbid-new-submodules -0.6.0 -===== +0.6.0 - 2016-08-12 +================== - Merge conflict detection no longer crashes on binary files - Indentation in json may be an arbitrary separator - Editable requirements are properly sorted - Encoding pragma fixer pragma is configurable -0.5.1 -===== +0.5.1 - 2016-05-16 +================== - Add a --no-sort-keys to json pretty formatter - Add a --remove to fix-encoding-pragma -0.5.0 -===== +0.5.0 - 2016-04-05 +================== - Add check-byte-order-marker - Add check-synlinks - check-large-files-added understands git-lfs @@ -340,8 +527,8 @@ - Allow binary files to pass private key hook - Add pretty-format-json hook -0.4.2 -===== +0.4.2 - 2015-05-31 +================== - Add --django to test name checker - Add check-merge-conflict hook - Remove dependency on plumbum @@ -350,13 +537,13 @@ - Teach trailing-whitespace about markdown - Quickfix for pyflakes - flake8 version conflict -0.4.1 -===== +0.4.1 - 2015-03-08 +================== - Respect configuration when running autopep8 - Quickfix for pep8 version conflicts -0.4.0 -===== +0.4.0 - 2015-02-22 +================== - Fix trailing-whitespace on OS X - Add check-added-large-files hook - Add check-docstring-first hook @@ -367,18 +554,18 @@ - Fix end-of-file-fixer for windows - Add double-quote-string-fixer hook -0.3.0 -===== +0.3.0 - 2014-08-22 +================== - Add autopep8-wrapper hook -0.2.0 -===== +0.2.0 - 2014-08-19 +================== - Add check-json hook -0.1.1 -===== +0.1.1 - 2014-06-19 +================== - Don't crash on non-parseable files for debug-statement-hook -0.1.0 -===== +0.1.0 - 2014-06-07 +================== - Initial Release diff --git a/README.md b/README.md index 1950fa19..7486aba2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Build Status](https://asottile.visualstudio.com/asottile/_apis/build/status/pre-commit.pre-commit-hooks?branchName=master)](https://asottile.visualstudio.com/asottile/_build/latest?definitionId=17&branchName=master) [![Azure DevOps coverage](https://img.shields.io/azure-devops/coverage/asottile/asottile/17/master.svg)](https://dev.azure.com/asottile/asottile/_build/latest?definitionId=17&branchName=master) +[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pre-commit/pre-commit-hooks/master.svg)](https://results.pre-commit.ci/latest/github/pre-commit/pre-commit-hooks/master) pre-commit-hooks ================ @@ -13,103 +14,189 @@ See also: https://github.com/pre-commit/pre-commit Add this to your `.pre-commit-config.yaml` - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 # Use the ref you want to point at - hooks: - - id: trailing-whitespace - # - id: ... - +```yaml +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 # Use the ref you want to point at + hooks: + - id: trailing-whitespace + # - id: ... +``` ### Hooks available -- `check-added-large-files` - Prevent giant files from being committed. - - Specify what is "too large" with `args: ['--maxkb=123']` (default=500kB). - - If `git-lfs` is installed, lfs files will be skipped - (requires `git-lfs>=2.2.1`) -- `check-ast` - Simply check whether files parse as valid python. -- `check-builtin-literals` - Require literal syntax when initializing empty or zero Python builtin types. - - Allows calling constructors with positional arguments (e.g., `list('abc')`). - - Allows calling constructors from the `builtins` (`__builtin__`) namespace (`builtins.list()`). - - Ignore this requirement for specific builtin types with `--ignore=type1,type2,…`. - - Forbid `dict` keyword syntax with `--no-allow-dict-kwargs`. -- `check-byte-order-marker` - Forbid files which have a UTF-8 byte-order marker -- `check-case-conflict` - Check for files with names that would conflict on a - case-insensitive filesystem like MacOS HFS+ or Windows FAT. -- `check-docstring-first` - Checks for a common error of placing code before - the docstring. -- `check-executables-have-shebangs` - Checks that non-binary executables have a - proper shebang. -- `check-json` - Attempts to load all json files to verify syntax. -- `check-merge-conflict` - Check for files that contain merge conflict strings. -- `check-symlinks` - Checks for symlinks which do not point to anything. -- `check-toml` - Attempts to load all TOML files to verify syntax. -- `check-vcs-permalinks` - Ensures that links to vcs websites are permalinks. -- `check-xml` - Attempts to load all xml files to verify syntax. -- `check-yaml` - Attempts to load all yaml files to verify syntax. - - `--allow-multiple-documents` - allow yaml files which use the - [multi-document syntax](http://www.yaml.org/spec/1.2/spec.html#YAML) - - `--unsafe` - Instead of loading the files, simply parse them for syntax. - A syntax-only check enables extensions and unsafe constructs which would - otherwise be forbidden. Using this option removes all guarantees of - portability to other yaml implementations. - Implies `--allow-multiple-documents`. -- `debug-statements` - Check for debugger imports and py37+ `breakpoint()` - calls in python source. -- `detect-aws-credentials` - Checks for the existence of AWS secrets that you - have set up with the AWS CLI. - The following arguments are available: - - `--credentials-file CREDENTIALS_FILE` - additional AWS CLI style - configuration file in a non-standard location to fetch configured - credentials from. Can be repeated multiple times. - - `--allow-missing-credentials` - Allow hook to pass when no credentials are - detected. -- `detect-private-key` - Checks for the existence of private keys. -- `double-quote-string-fixer` - This hook replaces double quoted strings - with single quoted strings. -- `end-of-file-fixer` - Makes sure files end in a newline and only a newline. -- `fix-encoding-pragma` - Add `# -*- coding: utf-8 -*-` to the top of python files. - - To remove the coding pragma pass `--remove` (useful in a python3-only codebase) -- `file-contents-sorter` - Sort the lines in specified files (defaults to alphabetical). You must provide list of target files as input to it. Note that this hook WILL remove blank lines and does NOT respect any comments. -- `flake8` - Run flake8 on your python files. -- `forbid-new-submodules` - Prevent addition of new git submodules. -- `mixed-line-ending` - Replaces or checks mixed line ending. - - `--fix={auto,crlf,lf,no}` - - `auto` - Replaces automatically the most frequent line ending. This is the default argument. - - `crlf`, `lf` - Forces to replace line ending by respectively CRLF and LF. - - `no` - Checks if there is any mixed line ending without modifying any file. -- `name-tests-test` - Assert that files in tests/ end in `_test.py`. - - Use `args: ['--django']` to match `test*.py` instead. -- `no-commit-to-branch` - Protect specific branches from direct checkins. - - Use `args: [--branch, staging, --branch, master]` to set the branch. - `master` is the default if no branch argument is set. - - `-b` / `--branch` may be specified multiple times to protect multiple - branches. - - `-p` / `--pattern` can be used to protect branches that match a supplied regex - (e.g. `--pattern, release/.*`). May be specified multiple times. -- `pretty-format-json` - Checks that all your JSON files are pretty. "Pretty" - here means that keys are sorted and indented. You can configure this with - the following commandline options: - - `--autofix` - automatically format json files - - `--indent ...` - Control the indentation (either a number for a number of spaces or a string of whitespace). Defaults to 4 spaces. - - `--no-sort-keys` - when autofixing, retain the original key ordering (instead of sorting the keys) - - `--top-keys comma,separated,keys` - Keys to keep at the top of mappings. -- `requirements-txt-fixer` - Sorts entries in requirements.txt and removes incorrect entry for `pkg-resources==0.0.0` -- `sort-simple-yaml` - Sorts simple YAML files which consist only of top-level keys, preserving comments and blocks. -- `trailing-whitespace` - Trims trailing whitespace. - - To preserve Markdown [hard linebreaks](https://github.github.com/gfm/#hard-line-break) - use `args: [--markdown-linebreak-ext=md]` (or other extensions used - by your markdownfiles). If for some reason you want to treat all files - as markdown, use `--markdown-linebreak-ext=*`. +#### `check-added-large-files` +Prevent giant files from being committed. + - Specify what is "too large" with `args: ['--maxkb=123']` (default=500kB). + - Limits checked files to those indicated as staged for addition by git. + - If `git-lfs` is installed, lfs files will be skipped + (requires `git-lfs>=2.2.1`) + - `--enforce-all` - Check all listed files not just those staged for + addition. + +#### `check-ast` +Simply check whether files parse as valid python. + +#### `check-builtin-literals` +Require literal syntax when initializing empty or zero Python builtin types. + - Allows calling constructors with positional arguments (e.g., `list('abc')`). + - Allows calling constructors from the `builtins` (`__builtin__`) namespace (`builtins.list()`). + - Ignore this requirement for specific builtin types with `--ignore=type1,type2,…`. + - Forbid `dict` keyword syntax with `--no-allow-dict-kwargs`. + +#### `check-case-conflict` +Check for files with names that would conflict on a case-insensitive filesystem like MacOS HFS+ or Windows FAT. + +#### `check-docstring-first` +Checks for a common error of placing code before the docstring. + +#### `check-executables-have-shebangs` +Checks that non-binary executables have a proper shebang. + +#### `check-json` +Attempts to load all json files to verify syntax. + +#### `check-merge-conflict` +Check for files that contain merge conflict strings. + +#### `check-shebang-scripts-are-executable` +Checks that scripts with shebangs are executable. + +#### `check-symlinks` +Checks for symlinks which do not point to anything. + +#### `check-toml` +Attempts to load all TOML files to verify syntax. + +#### `check-vcs-permalinks` +Ensures that links to vcs websites are permalinks. + - `--additional-github-domain DOMAIN` - Add check for specified domain. + Can be repeated multiple times. for example, if your company uses + GitHub Enterprise you may use something like + `--additional-github-domain github.example.com` + +#### `check-xml` +Attempts to load all xml files to verify syntax. + +#### `check-yaml` +Attempts to load all yaml files to verify syntax. + - `--allow-multiple-documents` - allow yaml files which use the + [multi-document syntax](http://www.yaml.org/spec/1.2/spec.html#YAML) + - `--unsafe` - Instead of loading the files, simply parse them for syntax. + A syntax-only check enables extensions and unsafe constructs which would + otherwise be forbidden. Using this option removes all guarantees of + portability to other yaml implementations. + Implies `--allow-multiple-documents`. + +#### `debug-statements` +Check for debugger imports and py37+ `breakpoint()` calls in python source. + +#### `destroyed-symlinks` +Detects symlinks which are changed to regular files with a content of a path +which that symlink was pointing to. +This usually happens on Windows when a user clones a repository that has +symlinks but they do not have the permission to create symlinks. + +#### `detect-aws-credentials` +Checks for the existence of AWS secrets that you have set up with the AWS CLI. +The following arguments are available: +- `--credentials-file CREDENTIALS_FILE` - additional AWS CLI style + configuration file in a non-standard location to fetch configured + credentials from. Can be repeated multiple times. +- `--allow-missing-credentials` - Allow hook to pass when no credentials are detected. + +#### `detect-private-key` +Checks for the existence of private keys. + +#### `double-quote-string-fixer` +This hook replaces double quoted strings with single quoted strings. + +#### `end-of-file-fixer` +Makes sure files end in a newline and only a newline. + +#### `fix-byte-order-marker` +removes UTF-8 byte order marker + +#### `fix-encoding-pragma` +Add `# -*- coding: utf-8 -*-` to the top of python files. + - To remove the coding pragma pass `--remove` (useful in a python3-only codebase) + +#### `file-contents-sorter` +Sort the lines in specified files (defaults to alphabetical). +You must provide list of target files as input to it. +Note that this hook WILL remove blank lines and does NOT respect any comments. + +#### `forbid-new-submodules` +Prevent addition of new git submodules. + +#### `mixed-line-ending` +Replaces or checks mixed line ending. + - `--fix={auto,crlf,lf,no}` + - `auto` - Replaces automatically the most frequent line ending. This is the default argument. + - `crlf`, `lf` - Forces to replace line ending by respectively CRLF and LF. + - This option isn't compatible with git setup check-in LF check-out CRLF as git smudge this later than the hook is invoked. + - `no` - Checks if there is any mixed line ending without modifying any file. + +#### `name-tests-test` +Assert that files in tests/ end in `_test.py`. + - Use `args: ['--django']` to match `test*.py` instead. + +#### `no-commit-to-branch` +Protect specific branches from direct checkins. + - Use `args: [--branch, staging, --branch, master]` to set the branch. + Both `master` and `main` are protected by default if no branch argument is set. + - `-b` / `--branch` may be specified multiple times to protect multiple + branches. + - `-p` / `--pattern` can be used to protect branches that match a supplied regex + (e.g. `--pattern, release/.*`). May be specified multiple times. + +Note that `no-commit-to-branch` is configured by default to [`always_run`](https://pre-commit.com/#config-always_run). +As a result, it will ignore any setting of [`files`](https://pre-commit.com/#config-files), +[`exclude`](https://pre-commit.com/#config-exclude), [`types`](https://pre-commit.com/#config-types) +or [`exclude_types`](https://pre-commit.com/#config-exclude_types). +Set [`always_run: false`](https://pre-commit.com/#config-always_run) to allow this hook to be skipped according to these +file filters. Caveat: In this configuration, empty commits (`git commit --allow-empty`) would always be allowed by this hook. + +#### `pretty-format-json` +Checks that all your JSON files are pretty. "Pretty" +here means that keys are sorted and indented. You can configure this with +the following commandline options: + - `--autofix` - automatically format json files + - `--indent ...` - Control the indentation (either a number for a number of spaces or a string of whitespace). Defaults to 2 spaces. + - `--no-ensure-ascii` preserve unicode characters instead of converting to escape sequences + - `--no-sort-keys` - when autofixing, retain the original key ordering (instead of sorting the keys) + - `--top-keys comma,separated,keys` - Keys to keep at the top of mappings. + +#### `requirements-txt-fixer` +Sorts entries in requirements.txt and removes incorrect entry for `pkg-resources==0.0.0` + +#### `sort-simple-yaml` +Sorts simple YAML files which consist only of top-level +keys, preserving comments and blocks. + +Note that `sort-simple-yaml` by default matches no `files` as it enforces a +very specific format. You must opt in to this by setting [`files`](https://pre-commit.com/#config-files), for example: + +```yaml + - id: sort-simple-yaml + files: ^config/simple/ +``` + + +#### `trailing-whitespace` +Trims trailing whitespace. + - To preserve Markdown [hard linebreaks](https://github.github.com/gfm/#hard-line-break) + use `args: [--markdown-linebreak-ext=md]` (or other extensions used + by your markdownfiles). If for some reason you want to treat all files + as markdown, use `--markdown-linebreak-ext=*`. + - By default, this hook trims all whitespace from the ends of lines. + To specify a custom set of characters to trim instead, use `args: [--chars,""]`. ### Deprecated / replaced hooks -- `autopep8-wrapper`: instead use - [mirrors-autopep8](https://github.com/pre-commit/mirrors-autopep8) -- `pyflakes`: instead use `flake8` +- `check-byte-order-marker`: instead use fix-byte-order-marker ### As a standalone package -If you'd like to use these hooks, they're also available as a standalone -package. +If you'd like to use these hooks, they're also available as a standalone package. Simply `pip install pre-commit-hooks` diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3156958c..58dc61dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,15 +10,14 @@ resources: type: github endpoint: github name: asottile/azure-pipeline-templates - ref: refs/tags/v0.0.8 + ref: refs/tags/v2.1.0 jobs: -- template: job--pre-commit.yml@asottile - template: job--python-tox.yml@asottile parameters: - toxenvs: [py27, py37] + toxenvs: [py38] os: windows - template: job--python-tox.yml@asottile parameters: - toxenvs: [pypy, pypy3, py27, py36, py37] + toxenvs: [pypy3, py36, py37, py38] os: linux diff --git a/pre_commit_hooks/autopep8_wrapper.py b/pre_commit_hooks/autopep8_wrapper.py deleted file mode 100644 index 8b69a049..00000000 --- a/pre_commit_hooks/autopep8_wrapper.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - - -def main(): # type: () -> int - raise SystemExit( - 'autopep8-wrapper is deprecated. Instead use autopep8 directly via ' - 'https://github.com/pre-commit/mirrors-autopep8', - ) - - -if __name__ == '__main__': - exit(main()) diff --git a/pre_commit_hooks/check_added_large_files.py b/pre_commit_hooks/check_added_large_files.py index be394989..cb646d7b 100644 --- a/pre_commit_hooks/check_added_large_files.py +++ b/pre_commit_hooks/check_added_large_files.py @@ -1,13 +1,7 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - import argparse import json import math import os -from typing import Iterable from typing import Optional from typing import Sequence from typing import Set @@ -17,7 +11,7 @@ from pre_commit_hooks.util import cmd_output -def lfs_files(): # type: () -> Set[str] +def lfs_files() -> Set[str]: try: # Introduced in git-lfs 2.2.0, first working in 2.2.1 lfs_ret = cmd_output('git', 'lfs', 'status', '--json') @@ -27,35 +21,49 @@ def lfs_files(): # type: () -> Set[str] return set(json.loads(lfs_ret)['files']) -def find_large_added_files(filenames, maxkb): - # type: (Iterable[str], int) -> int +def find_large_added_files( + filenames: Sequence[str], + maxkb: int, + *, + enforce_all: bool = False, +) -> int: # Find all added files that are also in the list of files pre-commit tells # us about - filenames = (added_files() & set(filenames)) - lfs_files() - retv = 0 - for filename in filenames: + filenames_filtered = set(filenames) - lfs_files() + if not enforce_all: + filenames_filtered &= added_files() + + for filename in filenames_filtered: kb = int(math.ceil(os.stat(filename).st_size / 1024)) if kb > maxkb: - print('{} ({} KB) exceeds {} KB.'.format(filename, kb, maxkb)) + print(f'{filename} ({kb} KB) exceeds {maxkb} KB.') retv = 1 return retv -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( 'filenames', nargs='*', help='Filenames pre-commit believes are changed.', ) + parser.add_argument( + '--enforce-all', action='store_true', + help='Enforce all files are checked, not just staged files.', + ) parser.add_argument( '--maxkb', type=int, default=500, help='Maxmimum allowable KB for added files', ) - args = parser.parse_args(argv) - return find_large_added_files(args.filenames, args.maxkb) + + return find_large_added_files( + args.filenames, + args.maxkb, + enforce_all=args.enforce_all, + ) if __name__ == '__main__': diff --git a/pre_commit_hooks/check_ast.py b/pre_commit_hooks/check_ast.py index 0df35407..2be6e1af 100644 --- a/pre_commit_hooks/check_ast.py +++ b/pre_commit_hooks/check_ast.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse import ast import platform @@ -11,7 +7,7 @@ from typing import Sequence -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*') args = parser.parse_args(argv) @@ -23,14 +19,11 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int with open(filename, 'rb') as f: ast.parse(f.read(), filename=filename) except SyntaxError: - print('{}: failed parsing with {} {}:'.format( - filename, - platform.python_implementation(), - sys.version.partition(' ')[0], - )) - print('\n{}'.format( - ' ' + traceback.format_exc().replace('\n', '\n '), - )) + impl = platform.python_implementation() + version = sys.version.split()[0] + print(f'{filename}: failed parsing with {impl} {version}:') + tb = ' ' + traceback.format_exc().replace('\n', '\n ') + print(f'\n{tb}') retval = 1 return retval diff --git a/pre_commit_hooks/check_builtin_literals.py b/pre_commit_hooks/check_builtin_literals.py index 4ddaa8c6..6bcd8387 100644 --- a/pre_commit_hooks/check_builtin_literals.py +++ b/pre_commit_hooks/check_builtin_literals.py @@ -1,10 +1,7 @@ -from __future__ import unicode_literals - import argparse import ast -import collections -import sys from typing import List +from typing import NamedTuple from typing import Optional from typing import Sequence from typing import Set @@ -21,23 +18,26 @@ } -Call = collections.namedtuple('Call', ['name', 'line', 'column']) +class Call(NamedTuple): + name: str + line: int + column: int class Visitor(ast.NodeVisitor): - def __init__(self, ignore=None, allow_dict_kwargs=True): - # type: (Optional[Sequence[str]], bool) -> None - self.builtin_type_calls = [] # type: List[Call] + def __init__( + self, + ignore: Optional[Sequence[str]] = None, + allow_dict_kwargs: bool = True, + ) -> None: + self.builtin_type_calls: List[Call] = [] self.ignore = set(ignore) if ignore else set() self.allow_dict_kwargs = allow_dict_kwargs - def _check_dict_call(self, node): # type: (ast.Call) -> bool - return ( - self.allow_dict_kwargs and - (getattr(node, 'kwargs', None) or getattr(node, 'keywords', None)) - ) + def _check_dict_call(self, node: ast.Call) -> bool: + return self.allow_dict_kwargs and bool(node.keywords) - def visit_Call(self, node): # type: (ast.Call) -> None + def visit_Call(self, node: ast.Call) -> None: if not isinstance(node.func, ast.Name): # Ignore functions that are object attributes (`foo.bar()`). # Assume that if the user calls `builtins.list()`, they know what @@ -54,8 +54,11 @@ def visit_Call(self, node): # type: (ast.Call) -> None ) -def check_file(filename, ignore=None, allow_dict_kwargs=True): - # type: (str, Optional[Sequence[str]], bool) -> List[Call] +def check_file( + filename: str, + ignore: Optional[Sequence[str]] = None, + allow_dict_kwargs: bool = True, +) -> List[Call]: with open(filename, 'rb') as f: tree = ast.parse(f.read(), filename=filename) visitor = Visitor(ignore=ignore, allow_dict_kwargs=allow_dict_kwargs) @@ -63,11 +66,11 @@ def check_file(filename, ignore=None, allow_dict_kwargs=True): return visitor.builtin_type_calls -def parse_ignore(value): # type: (str) -> Set[str] +def parse_ignore(value: str) -> Set[str]: return set(value.split(',')) -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*') parser.add_argument('--ignore', type=parse_ignore, default=set()) @@ -93,15 +96,11 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int rc = rc or 1 for call in calls: print( - '{filename}:{call.line}:{call.column}: ' - 'replace {call.name}() with {replacement}'.format( - filename=filename, - call=call, - replacement=BUILTIN_TYPES[call.name], - ), + f'{filename}:{call.line}:{call.column}: ' + f'replace {call.name}() with {BUILTIN_TYPES[call.name]}', ) return rc if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/check_byte_order_marker.py b/pre_commit_hooks/check_byte_order_marker.py index 10667c33..c0c2969c 100644 --- a/pre_commit_hooks/check_byte_order_marker.py +++ b/pre_commit_hooks/check_byte_order_marker.py @@ -1,13 +1,9 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse from typing import Optional from typing import Sequence -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to check') args = parser.parse_args(argv) @@ -18,7 +14,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int with open(filename, 'rb') as f: if f.read(3) == b'\xef\xbb\xbf': retv = 1 - print('{}: Has a byte-order marker'.format(filename)) + print(f'{filename}: Has a byte-order marker') return retv diff --git a/pre_commit_hooks/check_case_conflict.py b/pre_commit_hooks/check_case_conflict.py index e343d61f..024c1c3c 100644 --- a/pre_commit_hooks/check_case_conflict.py +++ b/pre_commit_hooks/check_case_conflict.py @@ -1,9 +1,7 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse +import os.path from typing import Iterable +from typing import Iterator from typing import Optional from typing import Sequence from typing import Set @@ -12,13 +10,26 @@ from pre_commit_hooks.util import cmd_output -def lower_set(iterable): # type: (Iterable[str]) -> Set[str] +def lower_set(iterable: Iterable[str]) -> Set[str]: return {x.lower() for x in iterable} -def find_conflicting_filenames(filenames): # type: (Sequence[str]) -> int +def parents(file: str) -> Iterator[str]: + file = os.path.dirname(file) + while file: + yield file + file = os.path.dirname(file) + + +def directories_for(files: Set[str]) -> Set[str]: + return {parent for file in files for parent in parents(file)} + + +def find_conflicting_filenames(filenames: Sequence[str]) -> int: repo_files = set(cmd_output('git', 'ls-files').splitlines()) + repo_files |= directories_for(repo_files) relevant_files = set(filenames) | added_files() + relevant_files |= directories_for(relevant_files) repo_files -= relevant_files retv = 0 @@ -39,13 +50,13 @@ def find_conflicting_filenames(filenames): # type: (Sequence[str]) -> int if x.lower() in conflicts ] for filename in sorted(conflicting_files): - print('Case-insensitivity conflict found: {}'.format(filename)) + print(f'Case-insensitivity conflict found: {filename}') retv = 1 return retv -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( 'filenames', nargs='*', diff --git a/pre_commit_hooks/check_docstring_first.py b/pre_commit_hooks/check_docstring_first.py index 6c19381c..875c0fba 100644 --- a/pre_commit_hooks/check_docstring_first.py +++ b/pre_commit_hooks/check_docstring_first.py @@ -1,30 +1,17 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse import io import tokenize +from tokenize import tokenize as tokenize_tokenize from typing import Optional from typing import Sequence -import six - -if six.PY2: # pragma: no cover (PY2) - from tokenize import generate_tokens as tokenize_tokenize - OTHER_NON_CODE = () -else: # pragma: no cover (PY3) - from tokenize import tokenize as tokenize_tokenize - OTHER_NON_CODE = (tokenize.ENCODING,) - -NON_CODE_TOKENS = frozenset( - (tokenize.COMMENT, tokenize.ENDMARKER, tokenize.NEWLINE, tokenize.NL) + - OTHER_NON_CODE, -) +NON_CODE_TOKENS = frozenset(( + tokenize.COMMENT, tokenize.ENDMARKER, tokenize.NEWLINE, tokenize.NL, + tokenize.ENCODING, +)) -def check_docstring_first(src, filename=''): - # type: (bytes, str) -> int +def check_docstring_first(src: bytes, filename: str = '') -> int: """Returns nonzero if the source has what looks like a docstring that is not at the beginning of the source. @@ -40,18 +27,14 @@ def check_docstring_first(src, filename=''): if tok_type == tokenize.STRING and scol == 0: if found_docstring_line is not None: print( - '{}:{} Multiple module docstrings ' - '(first docstring on line {}).'.format( - filename, sline, found_docstring_line, - ), + f'{filename}:{sline} Multiple module docstrings ' + f'(first docstring on line {found_docstring_line}).', ) return 1 elif found_code_line is not None: print( - '{}:{} Module docstring appears after code ' - '(code seen on line {}).'.format( - filename, sline, found_code_line, - ), + f'{filename}:{sline} Module docstring appears after code ' + f'(code seen on line {found_code_line}).', ) return 1 else: @@ -62,7 +45,7 @@ def check_docstring_first(src, filename=''): return 0 -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*') args = parser.parse_args(argv) diff --git a/pre_commit_hooks/check_executables_have_shebangs.py b/pre_commit_hooks/check_executables_have_shebangs.py index 4db2f9fb..e271c662 100644 --- a/pre_commit_hooks/check_executables_have_shebangs.py +++ b/pre_commit_hooks/check_executables_have_shebangs.py @@ -1,46 +1,80 @@ """Check that executable text files have a shebang.""" -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse -import pipes +import shlex import sys +from typing import Generator +from typing import List +from typing import NamedTuple from typing import Optional from typing import Sequence +from typing import Set + +from pre_commit_hooks.util import cmd_output +from pre_commit_hooks.util import zsplit + +EXECUTABLE_VALUES = frozenset(('1', '3', '5', '7')) + + +def check_executables(paths: List[str]) -> int: + if sys.platform == 'win32': # pragma: win32 cover + return _check_git_filemode(paths) + else: # pragma: win32 no cover + retv = 0 + for path in paths: + if not has_shebang(path): + _message(path) + retv = 1 + + return retv + + +class GitLsFile(NamedTuple): + mode: str + filename: str + + +def git_ls_files(paths: Sequence[str]) -> Generator[GitLsFile, None, None]: + outs = cmd_output('git', 'ls-files', '-z', '--stage', '--', *paths) + for out in zsplit(outs): + metadata, filename = out.split('\t') + mode, _, _ = metadata.split() + yield GitLsFile(mode, filename) -def check_has_shebang(path): # type: (str) -> int +def _check_git_filemode(paths: Sequence[str]) -> int: + seen: Set[str] = set() + for ls_file in git_ls_files(paths): + is_executable = any(b in EXECUTABLE_VALUES for b in ls_file.mode[-3:]) + if is_executable and not has_shebang(ls_file.filename): + _message(ls_file.filename) + seen.add(ls_file.filename) + + return int(bool(seen)) + + +def has_shebang(path: str) -> int: with open(path, 'rb') as f: first_bytes = f.read(2) - if first_bytes != b'#!': - print( - '{path}: marked executable but has no (or invalid) shebang!\n' - " If it isn't supposed to be executable, try: chmod -x {quoted}\n" - ' If it is supposed to be executable, double-check its shebang.' - .format( - path=path, - quoted=pipes.quote(path), - ), - file=sys.stderr, - ) - return 1 - else: - return 0 - - -def main(argv=None): # type: (Optional[Sequence[str]]) -> int + return first_bytes == b'#!' + + +def _message(path: str) -> None: + print( + f'{path}: marked executable but has no (or invalid) shebang!\n' + f" If it isn't supposed to be executable, try: " + f'`chmod -x {shlex.quote(path)}`\n' + f' If it is supposed to be executable, double-check its shebang.', + file=sys.stderr, + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('filenames', nargs='*') args = parser.parse_args(argv) - retv = 0 - - for filename in args.filenames: - retv |= check_has_shebang(filename) - - return retv + return check_executables(args.filenames) if __name__ == '__main__': diff --git a/pre_commit_hooks/check_json.py b/pre_commit_hooks/check_json.py index f26e0a5b..db589d01 100644 --- a/pre_commit_hooks/check_json.py +++ b/pre_commit_hooks/check_json.py @@ -1,27 +1,40 @@ -from __future__ import print_function - import argparse -import io import json -import sys +from typing import Any +from typing import Dict +from typing import List from typing import Optional from typing import Sequence +from typing import Tuple + + +def raise_duplicate_keys( + ordered_pairs: List[Tuple[str, Any]], +) -> Dict[str, Any]: + d = {} + for key, val in ordered_pairs: + if key in d: + raise ValueError(f'Duplicate key: {key}') + else: + d[key] = val + return d -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to check.') args = parser.parse_args(argv) retval = 0 for filename in args.filenames: - try: - json.load(io.open(filename, encoding='UTF-8')) - except (ValueError, UnicodeDecodeError) as exc: - print('{}: Failed to json decode ({})'.format(filename, exc)) - retval = 1 + with open(filename, 'rb') as f: + try: + json.load(f, object_pairs_hook=raise_duplicate_keys) + except ValueError as exc: + print(f'{filename}: Failed to json decode ({exc})') + retval = 1 return retval if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/check_merge_conflict.py b/pre_commit_hooks/check_merge_conflict.py index 74e4ae17..c20a8af7 100644 --- a/pre_commit_hooks/check_merge_conflict.py +++ b/pre_commit_hooks/check_merge_conflict.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import argparse import os.path from typing import Optional @@ -12,10 +10,9 @@ b'=======\n', b'>>>>>>> ', ] -WARNING_MSG = 'Merge conflict string "{0}" found in {1}:{2}' -def is_in_merge(): # type: () -> int +def is_in_merge() -> int: return ( os.path.exists(os.path.join('.git', 'MERGE_MSG')) and ( @@ -26,7 +23,7 @@ def is_in_merge(): # type: () -> int ) -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*') parser.add_argument('--assume-in-merge', action='store_true') @@ -41,9 +38,10 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int for i, line in enumerate(inputfile): for pattern in CONFLICT_PATTERNS: if line.startswith(pattern): - print(WARNING_MSG.format( - pattern.decode(), filename, i + 1, - )) + print( + f'Merge conflict string "{pattern.decode()}" ' + f'found in {filename}:{i + 1}', + ) retcode = 1 return retcode diff --git a/pre_commit_hooks/check_shebang_scripts_are_executable.py b/pre_commit_hooks/check_shebang_scripts_are_executable.py new file mode 100644 index 00000000..dce8c59d --- /dev/null +++ b/pre_commit_hooks/check_shebang_scripts_are_executable.py @@ -0,0 +1,53 @@ +"""Check that text files with a shebang are executable.""" +import argparse +import shlex +import sys +from typing import List +from typing import Optional +from typing import Sequence +from typing import Set + +from pre_commit_hooks.check_executables_have_shebangs import EXECUTABLE_VALUES +from pre_commit_hooks.check_executables_have_shebangs import git_ls_files +from pre_commit_hooks.check_executables_have_shebangs import has_shebang + + +def check_shebangs(paths: List[str]) -> int: + # Cannot optimize on non-executability here if we intend this check to + # work on win32 -- and that's where problems caused by non-executability + # (elsewhere) are most likely to arise from. + return _check_git_filemode(paths) + + +def _check_git_filemode(paths: Sequence[str]) -> int: + seen: Set[str] = set() + for ls_file in git_ls_files(paths): + is_executable = any(b in EXECUTABLE_VALUES for b in ls_file.mode[-3:]) + if not is_executable and has_shebang(ls_file.filename): + _message(ls_file.filename) + seen.add(ls_file.filename) + + return int(bool(seen)) + + +def _message(path: str) -> None: + print( + f'{path}: has a shebang but is not marked executable!\n' + f' If it is supposed to be executable, try: ' + f'`chmod +x {shlex.quote(path)}`\n' + f' If it not supposed to be executable, double-check its shebang ' + f'is wanted.\n', + file=sys.stderr, + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('filenames', nargs='*') + args = parser.parse_args(argv) + + return check_shebangs(args.filenames) + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit_hooks/check_symlinks.py b/pre_commit_hooks/check_symlinks.py index 736bf99c..f014714a 100644 --- a/pre_commit_hooks/check_symlinks.py +++ b/pre_commit_hooks/check_symlinks.py @@ -1,14 +1,10 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse import os.path from typing import Optional from typing import Sequence -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description='Checks for broken symlinks.') parser.add_argument('filenames', nargs='*', help='Filenames to check') args = parser.parse_args(argv) @@ -20,7 +16,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int os.path.islink(filename) and not os.path.exists(filename) ): # pragma: no cover (symlink support required) - print('{}: Broken symlink'.format(filename)) + print(f'{filename}: Broken symlink') retv = 1 return retv diff --git a/pre_commit_hooks/check_toml.py b/pre_commit_hooks/check_toml.py index e16e17c9..87496753 100644 --- a/pre_commit_hooks/check_toml.py +++ b/pre_commit_hooks/check_toml.py @@ -1,14 +1,11 @@ -from __future__ import print_function - import argparse -import sys from typing import Optional from typing import Sequence import toml -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to check.') args = parser.parse_args(argv) @@ -16,13 +13,12 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int retval = 0 for filename in args.filenames: try: - with open(filename) as f: - toml.load(f) + toml.load(filename) except toml.TomlDecodeError as exc: - print('{}: {}'.format(filename, exc)) + print(f'{filename}: {exc}') retval = 1 return retval if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/check_vcs_permalinks.py b/pre_commit_hooks/check_vcs_permalinks.py index f6e2a7d5..5231d7af 100644 --- a/pre_commit_hooks/check_vcs_permalinks.py +++ b/pre_commit_hooks/check_vcs_permalinks.py @@ -1,39 +1,53 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse import re import sys +from typing import List from typing import Optional +from typing import Pattern from typing import Sequence -GITHUB_NON_PERMALINK = re.compile( - br'https://github.com/[^/ ]+/[^/ ]+/blob/master/[^# ]+#L\d+', -) +def _get_pattern(domain: str) -> Pattern[bytes]: + regex = ( + rf'https://{domain}/[^/ ]+/[^/ ]+/blob/' + r'(?![a-fA-F0-9]{4,64}/)([^/. ]+)/[^# ]+#L\d+' + ) + return re.compile(regex.encode()) -def _check_filename(filename): # type: (str) -> int +def _check_filename(filename: str, patterns: List[Pattern[bytes]]) -> int: retv = 0 with open(filename, 'rb') as f: for i, line in enumerate(f, 1): - if GITHUB_NON_PERMALINK.search(line): - sys.stdout.write('{}:{}:'.format(filename, i)) - sys.stdout.flush() - getattr(sys.stdout, 'buffer', sys.stdout).write(line) - retv = 1 + for pattern in patterns: + if pattern.search(line): + sys.stdout.write(f'{filename}:{i}:') + sys.stdout.flush() + sys.stdout.buffer.write(line) + retv = 1 return retv -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*') + parser.add_argument( + '--additional-github-domain', + dest='additional_github_domains', + action='append', + default=['github.com'], + ) args = parser.parse_args(argv) + patterns = [ + _get_pattern(domain) + for domain in args.additional_github_domains + ] + retv = 0 + for filename in args.filenames: - retv |= _check_filename(filename) + retv |= _check_filename(filename, patterns) if retv: print() diff --git a/pre_commit_hooks/check_xml.py b/pre_commit_hooks/check_xml.py index 66e10bac..59b4d59e 100644 --- a/pre_commit_hooks/check_xml.py +++ b/pre_commit_hooks/check_xml.py @@ -1,30 +1,25 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse -import io -import sys import xml.sax.handler from typing import Optional from typing import Sequence -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='XML filenames to check.') args = parser.parse_args(argv) retval = 0 + handler = xml.sax.handler.ContentHandler() for filename in args.filenames: try: - with io.open(filename, 'rb') as xml_file: - xml.sax.parse(xml_file, xml.sax.handler.ContentHandler()) + with open(filename, 'rb') as xml_file: + xml.sax.parse(xml_file, handler) except xml.sax.SAXException as exc: - print('{}: Failed to xml parse ({})'.format(filename, exc)) + print(f'{filename}: Failed to xml parse ({exc})') retval = 1 return retval if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/check_yaml.py b/pre_commit_hooks/check_yaml.py index 5b664854..7453f6fb 100644 --- a/pre_commit_hooks/check_yaml.py +++ b/pre_commit_hooks/check_yaml.py @@ -1,11 +1,7 @@ -from __future__ import print_function - import argparse -import collections -import io -import sys from typing import Any from typing import Generator +from typing import NamedTuple from typing import Optional from typing import Sequence @@ -14,20 +10,24 @@ yaml = ruamel.yaml.YAML(typ='safe') -def _exhaust(gen): # type: (Generator[str, None, None]) -> None +def _exhaust(gen: Generator[str, None, None]) -> None: for _ in gen: pass -def _parse_unsafe(*args, **kwargs): # type: (*Any, **Any) -> None +def _parse_unsafe(*args: Any, **kwargs: Any) -> None: _exhaust(yaml.parse(*args, **kwargs)) -def _load_all(*args, **kwargs): # type: (*Any, **Any) -> None +def _load_all(*args: Any, **kwargs: Any) -> None: _exhaust(yaml.load_all(*args, **kwargs)) -Key = collections.namedtuple('Key', ('multi', 'unsafe')) +class Key(NamedTuple): + multi: bool + unsafe: bool + + LOAD_FNS = { Key(multi=False, unsafe=False): yaml.load, Key(multi=False, unsafe=True): _parse_unsafe, @@ -36,7 +36,7 @@ def _load_all(*args, **kwargs): # type: (*Any, **Any) -> None } -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( '-m', '--multi', '--allow-multiple-documents', action='store_true', @@ -59,7 +59,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int retval = 0 for filename in args.filenames: try: - with io.open(filename, encoding='UTF-8') as f: + with open(filename, encoding='UTF-8') as f: load_fn(f) except ruamel.yaml.YAMLError as exc: print(exc) @@ -68,4 +68,4 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/debug_statement_hook.py b/pre_commit_hooks/debug_statement_hook.py index 48648734..794f7080 100644 --- a/pre_commit_hooks/debug_statement_hook.py +++ b/pre_commit_hooks/debug_statement_hook.py @@ -1,35 +1,47 @@ -from __future__ import print_function -from __future__ import unicode_literals - import argparse import ast -import collections import traceback from typing import List +from typing import NamedTuple from typing import Optional from typing import Sequence -DEBUG_STATEMENTS = {'pdb', 'ipdb', 'pudb', 'q', 'rdb', 'rpdb'} -Debug = collections.namedtuple('Debug', ('line', 'col', 'name', 'reason')) +DEBUG_STATEMENTS = { + 'ipdb', + 'pdb', + 'pudb', + 'pydevd_pycharm', + 'q', + 'rdb', + 'rpdb', + 'wdb', +} + + +class Debug(NamedTuple): + line: int + col: int + name: str + reason: str class DebugStatementParser(ast.NodeVisitor): - def __init__(self): # type: () -> None - self.breakpoints = [] # type: List[Debug] + def __init__(self) -> None: + self.breakpoints: List[Debug] = [] - def visit_Import(self, node): # type: (ast.Import) -> None + def visit_Import(self, node: ast.Import) -> None: for name in node.names: if name.name in DEBUG_STATEMENTS: st = Debug(node.lineno, node.col_offset, name.name, 'imported') self.breakpoints.append(st) - def visit_ImportFrom(self, node): # type: (ast.ImportFrom) -> None + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: if node.module in DEBUG_STATEMENTS: st = Debug(node.lineno, node.col_offset, node.module, 'imported') self.breakpoints.append(st) - def visit_Call(self, node): # type: (ast.Call) -> None + def visit_Call(self, node: ast.Call) -> None: """python3.7+ breakpoint()""" if isinstance(node.func, ast.Name) and node.func.id == 'breakpoint': st = Debug(node.lineno, node.col_offset, node.func.id, 'called') @@ -37,12 +49,12 @@ def visit_Call(self, node): # type: (ast.Call) -> None self.generic_visit(node) -def check_file(filename): # type: (str) -> int +def check_file(filename: str) -> int: try: with open(filename, 'rb') as f: ast_obj = ast.parse(f.read(), filename=filename) except SyntaxError: - print('{} - Could not parse ast'.format(filename)) + print(f'{filename} - Could not parse ast') print() print('\t' + traceback.format_exc().replace('\n', '\n\t')) print() @@ -52,16 +64,12 @@ def check_file(filename): # type: (str) -> int visitor.visit(ast_obj) for bp in visitor.breakpoints: - print( - '{}:{}:{} - {} {}'.format( - filename, bp.line, bp.col, bp.name, bp.reason, - ), - ) + print(f'{filename}:{bp.line}:{bp.col} - {bp.name} {bp.reason}') return int(bool(visitor.breakpoints)) -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to run') args = parser.parse_args(argv) diff --git a/pre_commit_hooks/destroyed_symlinks.py b/pre_commit_hooks/destroyed_symlinks.py new file mode 100755 index 00000000..cfaf4e53 --- /dev/null +++ b/pre_commit_hooks/destroyed_symlinks.py @@ -0,0 +1,96 @@ +import argparse +import shlex +import subprocess +from typing import List +from typing import Optional +from typing import Sequence + +from pre_commit_hooks.util import cmd_output +from pre_commit_hooks.util import zsplit + +ORDINARY_CHANGED_ENTRIES_MARKER = '1' +PERMS_LINK = '120000' +PERMS_NONEXIST = '000000' + + +def find_destroyed_symlinks(files: Sequence[str]) -> List[str]: + destroyed_links: List[str] = [] + if not files: + return destroyed_links + for line in zsplit( + cmd_output('git', 'status', '--porcelain=v2', '-z', '--', *files), + ): + splitted = line.split(' ') + if splitted and splitted[0] == ORDINARY_CHANGED_ENTRIES_MARKER: + # https://git-scm.com/docs/git-status#_changed_tracked_entries + ( + _, _, _, + mode_HEAD, + mode_index, + _, + hash_HEAD, + hash_index, + *path_splitted, + ) = splitted + path = ' '.join(path_splitted) + if ( + mode_HEAD == PERMS_LINK and + mode_index != PERMS_LINK and + mode_index != PERMS_NONEXIST + ): + if hash_HEAD == hash_index: + # if old and new hashes are equal, it's not needed to check + # anything more, we've found a destroyed symlink for sure + destroyed_links.append(path) + else: + # if old and new hashes are *not* equal, it doesn't mean + # that everything is OK - new file may be altered + # by something like trailing-whitespace and/or + # mixed-line-ending hooks so we need to go deeper + SIZE_CMD = ('git', 'cat-file', '-s') + size_index = int(cmd_output(*SIZE_CMD, hash_index).strip()) + size_HEAD = int(cmd_output(*SIZE_CMD, hash_HEAD).strip()) + + # in the worst case new file may have CRLF added + # so check content only if new file is bigger + # not more than 2 bytes compared to the old one + if size_index <= size_HEAD + 2: + head_content = subprocess.check_output( + ('git', 'cat-file', '-p', hash_HEAD), + ).rstrip() + index_content = subprocess.check_output( + ('git', 'cat-file', '-p', hash_index), + ).rstrip() + if head_content == index_content: + destroyed_links.append(path) + return destroyed_links + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument('filenames', nargs='*', help='Filenames to check.') + args = parser.parse_args(argv) + destroyed_links = find_destroyed_symlinks(files=args.filenames) + if destroyed_links: + print('Destroyed symlinks:') + for destroyed_link in destroyed_links: + print(f'- {destroyed_link}') + print('You should unstage affected files:') + print( + '\tgit reset HEAD -- {}'.format( + ' '.join(shlex.quote(link) for link in destroyed_links), + ), + ) + print( + 'And retry commit. As a long term solution ' + 'you may try to explicitly tell git that your ' + 'environment does not support symlinks:', + ) + print('\tgit config core.symlinks false') + return 1 + else: + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit_hooks/detect_aws_credentials.py b/pre_commit_hooks/detect_aws_credentials.py index da80ab4f..1663cfd6 100644 --- a/pre_commit_hooks/detect_aws_credentials.py +++ b/pre_commit_hooks/detect_aws_credentials.py @@ -1,18 +1,19 @@ -from __future__ import print_function -from __future__ import unicode_literals - import argparse +import configparser import os -from typing import Dict from typing import List +from typing import NamedTuple from typing import Optional from typing import Sequence from typing import Set -from six.moves import configparser + +class BadFile(NamedTuple): + filename: str + key: str -def get_aws_cred_files_from_env(): # type: () -> Set[str] +def get_aws_cred_files_from_env() -> Set[str]: """Extract credential file paths from environment variables.""" return { os.environ[env_var] @@ -24,18 +25,18 @@ def get_aws_cred_files_from_env(): # type: () -> Set[str] } -def get_aws_secrets_from_env(): # type: () -> Set[str] +def get_aws_secrets_from_env() -> Set[str]: """Extract AWS secrets from environment variables.""" keys = set() for env_var in ( 'AWS_SECRET_ACCESS_KEY', 'AWS_SECURITY_TOKEN', 'AWS_SESSION_TOKEN', ): - if env_var in os.environ: + if os.environ.get(env_var): keys.add(os.environ[env_var]) return keys -def get_aws_secrets_from_file(credentials_file): # type: (str) -> Set[str] +def get_aws_secrets_from_file(credentials_file: str) -> Set[str]: """Extract AWS secrets from configuration files. Read an ini-style configuration file and return a set with all found AWS @@ -66,8 +67,10 @@ def get_aws_secrets_from_file(credentials_file): # type: (str) -> Set[str] return keys -def check_file_for_aws_keys(filenames, keys): - # type: (Sequence[str], Set[str]) -> List[Dict[str, str]] +def check_file_for_aws_keys( + filenames: Sequence[str], + keys: Set[bytes], +) -> List[BadFile]: """Check if files contain AWS secrets. Return a list of all files containing AWS secrets and keys found, with all @@ -76,19 +79,18 @@ def check_file_for_aws_keys(filenames, keys): bad_files = [] for filename in filenames: - with open(filename, 'r') as content: + with open(filename, 'rb') as content: text_body = content.read() for key in keys: # naively match the entire file, low chance of incorrect # collision if key in text_body: - bad_files.append({ - 'filename': filename, 'key': key[:4] + '*' * 28, - }) + key_hidden = key.decode()[:4].ljust(28, '*') + bad_files.append(BadFile(filename, key_hidden)) return bad_files -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='+', help='Filenames to run') parser.add_argument( @@ -117,7 +119,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int # of files to to gather AWS secrets from. credential_files |= get_aws_cred_files_from_env() - keys = set() # type: Set[str] + keys: Set[str] = set() for credential_file in credential_files: keys |= get_aws_secrets_from_file(credential_file) @@ -136,10 +138,11 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int ) return 2 - bad_filenames = check_file_for_aws_keys(args.filenames, keys) + keys_b = {key.encode() for key in keys} + bad_filenames = check_file_for_aws_keys(args.filenames, keys_b) if bad_filenames: for bad_file in bad_filenames: - print('AWS secret found in {filename}: {key}'.format(**bad_file)) + print(f'AWS secret found in {bad_file.filename}: {bad_file.key}') return 1 else: return 0 diff --git a/pre_commit_hooks/detect_private_key.py b/pre_commit_hooks/detect_private_key.py index d31957de..7bbc2f91 100644 --- a/pre_commit_hooks/detect_private_key.py +++ b/pre_commit_hooks/detect_private_key.py @@ -1,7 +1,4 @@ -from __future__ import print_function - import argparse -import sys from typing import Optional from typing import Sequence @@ -17,7 +14,7 @@ ] -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to check') args = parser.parse_args(argv) @@ -32,11 +29,11 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int if private_key_files: for private_key_file in private_key_files: - print('Private key found: {}'.format(private_key_file)) + print(f'Private key found: {private_key_file}') return 1 else: return 0 if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/end_of_file_fixer.py b/pre_commit_hooks/end_of_file_fixer.py index 4e77c945..1c07379d 100644 --- a/pre_commit_hooks/end_of_file_fixer.py +++ b/pre_commit_hooks/end_of_file_fixer.py @@ -1,20 +1,16 @@ -from __future__ import print_function -from __future__ import unicode_literals - import argparse import os -import sys from typing import IO from typing import Optional from typing import Sequence -def fix_file(file_obj): # type: (IO[bytes]) -> int +def fix_file(file_obj: IO[bytes]) -> int: # Test for newline at end of file # Empty files will throw IOError here try: file_obj.seek(-1, os.SEEK_END) - except IOError: + except OSError: return 0 last_character = file_obj.read(1) # last_character will be '' for an empty file @@ -52,7 +48,7 @@ def fix_file(file_obj): # type: (IO[bytes]) -> int return 0 -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -64,11 +60,11 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int with open(filename, 'rb+') as file_obj: ret_for_file = fix_file(file_obj) if ret_for_file: - print('Fixing {}'.format(filename)) + print(f'Fixing {filename}') retv |= ret_for_file return retv if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/file_contents_sorter.py b/pre_commit_hooks/file_contents_sorter.py index 1598d2da..ebbcd206 100644 --- a/pre_commit_hooks/file_contents_sorter.py +++ b/pre_commit_hooks/file_contents_sorter.py @@ -2,18 +2,18 @@ A very simple pre-commit hook that, when passed one or more filenames as arguments, will sort the lines in those files. -An example use case for this: you have a deploy-whitelist.txt file +An example use case for this: you have a deploy-allowlist.txt file in a repo that contains a list of filenames that is used to specify files to be included in a docker container. This file has one filename per line. Various users are adding/removing lines from this file; using this hook on that file should reduce the instances of git merge conflicts and keep the file nicely ordered. """ -from __future__ import print_function - import argparse -import sys +from typing import Any +from typing import Callable from typing import IO +from typing import Iterable from typing import Optional from typing import Sequence @@ -21,9 +21,19 @@ FAIL = 1 -def sort_file_contents(f): # type: (IO[bytes]) -> int +def sort_file_contents( + f: IO[bytes], + key: Optional[Callable[[bytes], Any]], + *, + unique: bool = False, +) -> int: before = list(f) - after = sorted([line.strip(b'\n\r') for line in before if line.strip()]) + lines: Iterable[bytes] = ( + line.rstrip(b'\n\r') for line in before if line.strip() + ) + if unique: + lines = set(lines) + after = sorted(lines, key=key) before_string = b''.join(before) after_string = b'\n'.join(after) + b'\n' @@ -37,19 +47,33 @@ def sort_file_contents(f): # type: (IO[bytes]) -> int return FAIL -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='+', help='Files to sort') + parser.add_argument( + '--ignore-case', + action='store_const', + const=bytes.lower, + default=None, + help='fold lower case to upper case characters', + ) + parser.add_argument( + '--unique', + action='store_true', + help='ensure each line is unique', + ) args = parser.parse_args(argv) retv = PASS for arg in args.filenames: with open(arg, 'rb+') as file_obj: - ret_for_file = sort_file_contents(file_obj) + ret_for_file = sort_file_contents( + file_obj, key=args.ignore_case, unique=args.unique, + ) if ret_for_file: - print('Sorting {}'.format(arg)) + print(f'Sorting {arg}') retv |= ret_for_file @@ -57,4 +81,4 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/fix_byte_order_marker.py b/pre_commit_hooks/fix_byte_order_marker.py new file mode 100644 index 00000000..1ffe047d --- /dev/null +++ b/pre_commit_hooks/fix_byte_order_marker.py @@ -0,0 +1,30 @@ +import argparse +from typing import Optional +from typing import Sequence + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument('filenames', nargs='*', help='Filenames to check') + args = parser.parse_args(argv) + + retv = 0 + + for filename in args.filenames: + with open(filename, 'rb') as f_b: + bts = f_b.read(3) + + if bts == b'\xef\xbb\xbf': + with open(filename, newline='', encoding='utf-8-sig') as f: + contents = f.read() + with open(filename, 'w', newline='', encoding='utf-8') as f: + f.write(contents) + + print(f'{filename}: removed byte-order marker') + retv = 1 + + return retv + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit_hooks/fix_encoding_pragma.py b/pre_commit_hooks/fix_encoding_pragma.py index 23fc79fd..88d72ed7 100644 --- a/pre_commit_hooks/fix_encoding_pragma.py +++ b/pre_commit_hooks/fix_encoding_pragma.py @@ -1,18 +1,13 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse -import collections from typing import IO +from typing import NamedTuple from typing import Optional from typing import Sequence -from typing import Union DEFAULT_PRAGMA = b'# -*- coding: utf-8 -*-' -def has_coding(line): # type: (bytes) -> bool +def has_coding(line: bytes) -> bool: if not line.strip(): return False return ( @@ -25,28 +20,30 @@ def has_coding(line): # type: (bytes) -> bool ) -class ExpectedContents(collections.namedtuple( - 'ExpectedContents', ('shebang', 'rest', 'pragma_status', 'ending'), -)): - """ - pragma_status: - - True: has exactly the coding pragma expected - - False: missing coding pragma entirely - - None: has a coding pragma, but it does not match - """ - __slots__ = () +class ExpectedContents(NamedTuple): + shebang: bytes + rest: bytes + # True: has exactly the coding pragma expected + # False: missing coding pragma entirely + # None: has a coding pragma, but it does not match + pragma_status: Optional[bool] + ending: bytes @property - def has_any_pragma(self): # type: () -> bool + def has_any_pragma(self) -> bool: return self.pragma_status is not False - def is_expected_pragma(self, remove): # type: (bool) -> bool + def is_expected_pragma(self, remove: bool) -> bool: expected_pragma_status = not remove return self.pragma_status is expected_pragma_status -def _get_expected_contents(first_line, second_line, rest, expected_pragma): - # type: (bytes, bytes, bytes, bytes) -> ExpectedContents +def _get_expected_contents( + first_line: bytes, + second_line: bytes, + rest: bytes, + expected_pragma: bytes, +) -> ExpectedContents: ending = b'\r\n' if first_line.endswith(b'\r\n') else b'\n' if first_line.startswith(b'#!'): @@ -58,7 +55,7 @@ def _get_expected_contents(first_line, second_line, rest, expected_pragma): rest = second_line + rest if potential_coding.rstrip(b'\r\n') == expected_pragma: - pragma_status = True # type: Optional[bool] + pragma_status: Optional[bool] = True elif has_coding(potential_coding): pragma_status = None else: @@ -70,8 +67,11 @@ def _get_expected_contents(first_line, second_line, rest, expected_pragma): ) -def fix_encoding_pragma(f, remove=False, expected_pragma=DEFAULT_PRAGMA): - # type: (IO[bytes], bool, bytes) -> int +def fix_encoding_pragma( + f: IO[bytes], + remove: bool = False, + expected_pragma: bytes = DEFAULT_PRAGMA, +) -> int: expected = _get_expected_contents( f.readline(), f.readline(), f.read(), expected_pragma, ) @@ -101,21 +101,20 @@ def fix_encoding_pragma(f, remove=False, expected_pragma=DEFAULT_PRAGMA): return 1 -def _normalize_pragma(pragma): # type: (Union[bytes, str]) -> bytes - if not isinstance(pragma, bytes): - pragma = pragma.encode('UTF-8') - return pragma.rstrip() +def _normalize_pragma(pragma: str) -> bytes: + return pragma.encode().rstrip() -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser( 'Fixes the encoding pragma of python files', ) parser.add_argument('filenames', nargs='*', help='Filenames to fix') parser.add_argument( '--pragma', default=DEFAULT_PRAGMA, type=_normalize_pragma, - help='The encoding pragma to use. Default: {}'.format( - DEFAULT_PRAGMA.decode(), + help=( + f'The encoding pragma to use. ' + f'Default: {DEFAULT_PRAGMA.decode()}' ), ) parser.add_argument( @@ -138,9 +137,9 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int ) retv |= file_ret if file_ret: - print(fmt.format( - pragma=args.pragma.decode(), filename=filename, - )) + print( + fmt.format(pragma=args.pragma.decode(), filename=filename), + ) return retv diff --git a/pre_commit_hooks/forbid_new_submodules.py b/pre_commit_hooks/forbid_new_submodules.py index bdbd6f7f..c144d728 100644 --- a/pre_commit_hooks/forbid_new_submodules.py +++ b/pre_commit_hooks/forbid_new_submodules.py @@ -1,14 +1,10 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - from typing import Optional from typing import Sequence from pre_commit_hooks.util import cmd_output -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: # `argv` is ignored, pre-commit will send us a list of files that we # don't care about added_diff = cmd_output( @@ -19,7 +15,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int metadata, filename = line.split('\t', 1) new_mode = metadata.split(' ')[1] if new_mode == '160000': - print('{}: new submodule introduced'.format(filename)) + print(f'{filename}: new submodule introduced') retv = 1 if retv: diff --git a/pre_commit_hooks/mixed_line_ending.py b/pre_commit_hooks/mixed_line_ending.py index 90aef035..0ef8e2c0 100644 --- a/pre_commit_hooks/mixed_line_ending.py +++ b/pre_commit_hooks/mixed_line_ending.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse import collections from typing import Dict @@ -17,7 +13,7 @@ FIX_TO_LINE_ENDING = {'cr': CR, 'crlf': CRLF, 'lf': LF} -def _fix(filename, contents, ending): # type: (str, bytes, bytes) -> None +def _fix(filename: str, contents: bytes, ending: bytes) -> None: new_contents = b''.join( line.rstrip(b'\r\n') + ending for line in contents.splitlines(True) ) @@ -25,11 +21,11 @@ def _fix(filename, contents, ending): # type: (str, bytes, bytes) -> None f.write(new_contents) -def fix_filename(filename, fix): # type: (str, str) -> int +def fix_filename(filename: str, fix: str) -> int: with open(filename, 'rb') as f: contents = f.read() - counts = collections.defaultdict(int) # type: Dict[bytes, int] + counts: Dict[bytes, int] = collections.defaultdict(int) for line in contents.splitlines(True): for ending in ALL_ENDINGS: @@ -66,7 +62,7 @@ def fix_filename(filename, fix): # type: (str, str) -> int return other_endings -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( '-f', '--fix', @@ -81,9 +77,9 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int for filename in args.filenames: if fix_filename(filename, args.fix): if args.fix == 'no': - print('{}: mixed line endings'.format(filename)) + print(f'{filename}: mixed line endings') else: - print('{}: fixed mixed line endings'.format(filename)) + print(f'{filename}: fixed mixed line endings') retv = 1 return retv diff --git a/pre_commit_hooks/no_commit_to_branch.py b/pre_commit_hooks/no_commit_to_branch.py index 31310595..49ffecf7 100644 --- a/pre_commit_hooks/no_commit_to_branch.py +++ b/pre_commit_hooks/no_commit_to_branch.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import argparse import re from typing import AbstractSet @@ -10,8 +8,10 @@ from pre_commit_hooks.util import cmd_output -def is_on_branch(protected, patterns=frozenset()): - # type: (AbstractSet[str], AbstractSet[str]) -> bool +def is_on_branch( + protected: AbstractSet[str], + patterns: AbstractSet[str] = frozenset(), +) -> bool: try: ref_name = cmd_output('git', 'symbolic-ref', 'HEAD') except CalledProcessError: @@ -23,7 +23,7 @@ def is_on_branch(protected, patterns=frozenset()): ) -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( '-b', '--branch', action='append', @@ -38,7 +38,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int ) args = parser.parse_args(argv) - protected = frozenset(args.branch or ('master',)) + protected = frozenset(args.branch or ('master', 'main')) patterns = frozenset(args.pattern or ()) return int(is_on_branch(protected, patterns)) diff --git a/pre_commit_hooks/pretty_format_json.py b/pre_commit_hooks/pretty_format_json.py index e734ca81..61b01698 100644 --- a/pre_commit_hooks/pretty_format_json.py +++ b/pre_commit_hooks/pretty_format_json.py @@ -1,10 +1,7 @@ -from __future__ import print_function - import argparse -import io import json import sys -from collections import OrderedDict +from difflib import unified_diff from typing import List from typing import Mapping from typing import Optional @@ -12,38 +9,36 @@ from typing import Tuple from typing import Union -from six import text_type - def _get_pretty_format( - contents, indent, ensure_ascii=True, sort_keys=True, top_keys=(), -): # type: (str, str, bool, bool, Sequence[str]) -> str - def pairs_first(pairs): - # type: (Sequence[Tuple[str, str]]) -> Mapping[str, str] + contents: str, + indent: str, + ensure_ascii: bool = True, + sort_keys: bool = True, + top_keys: Sequence[str] = (), +) -> str: + def pairs_first(pairs: Sequence[Tuple[str, str]]) -> Mapping[str, str]: before = [pair for pair in pairs if pair[0] in top_keys] before = sorted(before, key=lambda x: top_keys.index(x[0])) after = [pair for pair in pairs if pair[0] not in top_keys] if sort_keys: - after = sorted(after, key=lambda x: x[0]) - return OrderedDict(before + after) + after.sort() + return dict(before + after) json_pretty = json.dumps( json.loads(contents, object_pairs_hook=pairs_first), indent=indent, ensure_ascii=ensure_ascii, - # Workaround for https://bugs.python.org/issue16333 - separators=(',', ': '), ) - # Ensure unicode (Py2) and add the newline that dumps does not end with. - return text_type(json_pretty) + '\n' + return f'{json_pretty}\n' -def _autofix(filename, new_contents): # type: (str, str) -> None - print('Fixing file {}'.format(filename)) - with io.open(filename, 'w', encoding='UTF-8') as f: +def _autofix(filename: str, new_contents: str) -> None: + print(f'Fixing file {filename}') + with open(filename, 'w', encoding='UTF-8') as f: f.write(new_contents) -def parse_num_to_int(s): # type: (str) -> Union[int, str] +def parse_num_to_int(s: str) -> Union[int, str]: """Convert string numbers to int, leaving strings as is.""" try: return int(s) @@ -51,11 +46,18 @@ def parse_num_to_int(s): # type: (str) -> Union[int, str] return s -def parse_topkeys(s): # type: (str) -> List[str] +def parse_topkeys(s: str) -> List[str]: return s.split(',') -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def get_diff(source: str, target: str, file: str) -> str: + source_lines = source.splitlines(True) + target_lines = target.splitlines(True) + diff = unified_diff(source_lines, target_lines, fromfile=file, tofile=file) + return ''.join(diff) + + +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( '--autofix', @@ -96,14 +98,13 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int default=[], help='Ordered list of keys to keep at the top of JSON hashes', ) - parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) status = 0 for json_file in args.filenames: - with io.open(json_file, encoding='UTF-8') as f: + with open(json_file, encoding='UTF-8') as f: contents = f.read() try: @@ -111,23 +112,24 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int contents, args.indent, ensure_ascii=not args.no_ensure_ascii, sort_keys=not args.no_sort_keys, top_keys=args.top_keys, ) - - if contents != pretty_contents: - print('File {} is not pretty-formatted'.format(json_file)) - - if args.autofix: - _autofix(json_file, pretty_contents) - - status = 1 except ValueError: print( - 'Input File {} is not a valid JSON, consider using check-json' - .format(json_file), + f'Input File {json_file} is not a valid JSON, consider using ' + f'check-json', ) return 1 + if contents != pretty_contents: + if args.autofix: + _autofix(json_file, pretty_contents) + else: + diff_output = get_diff(contents, pretty_contents, json_file) + sys.stdout.buffer.write(diff_output.encode()) + + status = 1 + return status if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/removed.py b/pre_commit_hooks/removed.py new file mode 100644 index 00000000..60df0963 --- /dev/null +++ b/pre_commit_hooks/removed.py @@ -0,0 +1,15 @@ +import sys +from typing import Optional +from typing import Sequence + + +def main(argv: Optional[Sequence[str]] = None) -> int: + argv = argv if argv is not None else sys.argv[1:] + hookid, new_hookid, url = argv[:3] + raise SystemExit( + f'`{hookid}` has been removed -- use `{new_hookid}` from {url}', + ) + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit_hooks/requirements_txt_fixer.py b/pre_commit_hooks/requirements_txt_fixer.py index 4575975a..351e5b15 100644 --- a/pre_commit_hooks/requirements_txt_fixer.py +++ b/pre_commit_hooks/requirements_txt_fixer.py @@ -1,6 +1,5 @@ -from __future__ import print_function - import argparse +import re from typing import IO from typing import List from typing import Optional @@ -11,22 +10,33 @@ FAIL = 1 -class Requirement(object): +class Requirement: + UNTIL_COMPARISON = re.compile(b'={2,3}|!=|~=|>=?|<=?') + UNTIL_SEP = re.compile(rb'[^;\s]+') - def __init__(self): # type: () -> None - super(Requirement, self).__init__() - self.value = None # type: Optional[bytes] - self.comments = [] # type: List[bytes] + def __init__(self) -> None: + self.value: Optional[bytes] = None + self.comments: List[bytes] = [] @property - def name(self): # type: () -> bytes + def name(self) -> bytes: assert self.value is not None, self.value - if self.value.startswith(b'-e '): - return self.value.lower().partition(b'=')[-1] + name = self.value.lower() + for egg in (b'#egg=', b'&egg='): + if egg in self.value: + return name.partition(egg)[-1] + + m = self.UNTIL_SEP.match(name) + assert m is not None - return self.value.lower().partition(b'==')[0] + name = m.group() + m = self.UNTIL_COMPARISON.search(name) + if not m: + return name - def __lt__(self, requirement): # type: (Requirement) -> int + return name[:m.start()] + + def __lt__(self, requirement: 'Requirement') -> bool: # \n means top of file comment, so always return True, # otherwise just do a string comparison with value. assert self.value is not None, self.value @@ -37,14 +47,31 @@ def __lt__(self, requirement): # type: (Requirement) -> int else: return self.name < requirement.name + def is_complete(self) -> bool: + return ( + self.value is not None and + not self.value.rstrip(b'\r\n').endswith(b'\\') + ) -def fix_requirements(f): # type: (IO[bytes]) -> int - requirements = [] # type: List[Requirement] - before = tuple(f) - after = [] # type: List[bytes] + def append_value(self, value: bytes) -> None: + if self.value is not None: + self.value += value + else: + self.value = value + + +def fix_requirements(f: IO[bytes]) -> int: + requirements: List[Requirement] = [] + before = list(f) + after: List[bytes] = [] before_string = b''.join(before) + # adds new line in case one is missing + # AND a change to the requirements file is needed regardless: + if before and not before[-1].endswith(b'\n'): + before[-1] += b'\n' + # If the file is empty (i.e. only whitespace/newlines) exit early if before_string.strip() == b'': return PASS @@ -53,7 +80,7 @@ def fix_requirements(f): # type: (IO[bytes]) -> int # If the most recent requirement object has a value, then it's # time to start building the next requirement object. - if not len(requirements) or requirements[-1].value is not None: + if not len(requirements) or requirements[-1].is_complete(): requirements.append(Requirement()) requirement = requirements[-1] @@ -68,10 +95,10 @@ def fix_requirements(f): # type: (IO[bytes]) -> int requirement.value = b'\n' else: requirement.comments.append(line) - elif line.startswith(b'#') or line.strip() == b'': + elif line.lstrip().startswith(b'#') or line.strip() == b'': requirement.comments.append(line) else: - requirement.value = line + requirement.append_value(line) # if a file ends in a comment, preserve it at the end if requirements[-1].value is None: @@ -103,7 +130,7 @@ def fix_requirements(f): # type: (IO[bytes]) -> int return FAIL -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -115,7 +142,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int ret_for_file = fix_requirements(file_obj) if ret_for_file: - print('Sorting {}'.format(arg)) + print(f'Sorting {arg}') retv |= ret_for_file diff --git a/pre_commit_hooks/sort_simple_yaml.py b/pre_commit_hooks/sort_simple_yaml.py index a381679f..8ebc84ff 100755 --- a/pre_commit_hooks/sort_simple_yaml.py +++ b/pre_commit_hooks/sort_simple_yaml.py @@ -18,8 +18,6 @@ In other words, we don't sort deeper than the top layer, and might corrupt complicated YAML files. """ -from __future__ import print_function - import argparse from typing import List from typing import Optional @@ -29,7 +27,7 @@ QUOTES = ["'", '"'] -def sort(lines): # type: (List[str]) -> List[str] +def sort(lines: List[str]) -> List[str]: """Sort a YAML file in alphabetical order, keeping blocks together. :param lines: array of strings (without newlines) @@ -47,7 +45,7 @@ def sort(lines): # type: (List[str]) -> List[str] return new_lines -def parse_block(lines, header=False): # type: (List[str], bool) -> List[str] +def parse_block(lines: List[str], header: bool = False) -> List[str]: """Parse and return a single block, popping off the start of `lines`. If parsing a header block, we stop after we reach a line that is not a @@ -63,7 +61,7 @@ def parse_block(lines, header=False): # type: (List[str], bool) -> List[str] return block_lines -def parse_blocks(lines): # type: (List[str]) -> List[List[str]] +def parse_blocks(lines: List[str]) -> List[List[str]]: """Parse and return all possible blocks, popping off the start of `lines`. :param lines: list of lines @@ -80,7 +78,7 @@ def parse_blocks(lines): # type: (List[str]) -> List[List[str]] return blocks -def first_key(lines): # type: (List[str]) -> str +def first_key(lines: List[str]) -> str: """Returns a string representing the sort key of a block. The sort key is the first YAML key we encounter, ignoring comments, and @@ -102,7 +100,7 @@ def first_key(lines): # type: (List[str]) -> str return '' # not actually reached in reality -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -115,7 +113,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int new_lines = sort(lines) if lines != new_lines: - print('Fixing file `{filename}`'.format(filename=filename)) + print(f'Fixing file `{filename}`') f.seek(0) f.write('\n'.join(new_lines) + '\n') f.truncate() diff --git a/pre_commit_hooks/string_fixer.py b/pre_commit_hooks/string_fixer.py index a41b7373..3fdb6e2f 100644 --- a/pre_commit_hooks/string_fixer.py +++ b/pre_commit_hooks/string_fixer.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import argparse import io import re @@ -13,7 +9,7 @@ START_QUOTE_RE = re.compile('^[a-zA-Z]*"') -def handle_match(token_text): # type: (str) -> str +def handle_match(token_text: str) -> str: if '"""' in token_text or "'''" in token_text: return token_text @@ -28,7 +24,7 @@ def handle_match(token_text): # type: (str) -> str return token_text -def get_line_offsets_by_line_no(src): # type: (str) -> List[int] +def get_line_offsets_by_line_no(src: str) -> List[int]: # Padded so we can index with line number offsets = [-1, 0] for line in src.splitlines(True): @@ -36,8 +32,8 @@ def get_line_offsets_by_line_no(src): # type: (str) -> List[int] return offsets -def fix_strings(filename): # type: (str) -> int - with io.open(filename, encoding='UTF-8', newline='') as f: +def fix_strings(filename: str) -> int: + with open(filename, encoding='UTF-8', newline='') as f: contents = f.read() line_offsets = get_line_offsets_by_line_no(contents) @@ -45,9 +41,8 @@ def fix_strings(filename): # type: (str) -> int splitcontents = list(contents) # Iterate in reverse so the offsets are always correct - tokens = reversed(list(tokenize.generate_tokens( - io.StringIO(contents).readline, - ))) + tokens_l = list(tokenize.generate_tokens(io.StringIO(contents).readline)) + tokens = reversed(tokens_l) for token_type, token_text, (srow, scol), (erow, ecol), _ in tokens: if token_type == tokenize.STRING: new_text = handle_match(token_text) @@ -58,14 +53,14 @@ def fix_strings(filename): # type: (str) -> int new_contents = ''.join(splitcontents) if contents != new_contents: - with io.open(filename, 'w', encoding='UTF-8', newline='') as f: + with open(filename, 'w', encoding='UTF-8', newline='') as f: f.write(new_contents) return 1 else: return 0 -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -75,7 +70,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int for filename in args.filenames: return_value = fix_strings(filename) if return_value != 0: - print('Fixing strings in {}'.format(filename)) + print(f'Fixing strings in {filename}') retv |= return_value return retv diff --git a/pre_commit_hooks/tests_should_end_in_test.py b/pre_commit_hooks/tests_should_end_in_test.py index 7a1e7c04..b8cf9152 100644 --- a/pre_commit_hooks/tests_should_end_in_test.py +++ b/pre_commit_hooks/tests_should_end_in_test.py @@ -1,14 +1,11 @@ -from __future__ import print_function - import argparse import os.path import re -import sys from typing import Optional from typing import Sequence -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*') parser.add_argument( @@ -18,7 +15,7 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int args = parser.parse_args(argv) retcode = 0 - test_name_pattern = 'test.*.py' if args.django else '.*_test.py' + test_name_pattern = r'test.*\.py' if args.django else r'.*_test\.py' for filename in args.filenames: base = os.path.basename(filename) if ( @@ -27,14 +24,10 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int not base == 'conftest.py' ): retcode = 1 - print( - '{} does not match pattern "{}"'.format( - filename, test_name_pattern, - ), - ) + print(f'{filename} does not match pattern "{test_name_pattern}"') return retcode if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/trailing_whitespace_fixer.py b/pre_commit_hooks/trailing_whitespace_fixer.py index 2ccc0038..05ed9994 100644 --- a/pre_commit_hooks/trailing_whitespace_fixer.py +++ b/pre_commit_hooks/trailing_whitespace_fixer.py @@ -1,16 +1,17 @@ -from __future__ import print_function - import argparse import os -import sys from typing import Optional from typing import Sequence -def _fix_file(filename, is_markdown): # type: (str, bool) -> bool +def _fix_file( + filename: str, + is_markdown: bool, + chars: Optional[bytes], +) -> bool: with open(filename, mode='rb') as file_processed: lines = file_processed.readlines() - newlines = [_process_line(line, is_markdown) for line in lines] + newlines = [_process_line(line, is_markdown, chars) for line in lines] if newlines != lines: with open(filename, mode='wb') as file_processed: for line in newlines: @@ -20,20 +21,26 @@ def _fix_file(filename, is_markdown): # type: (str, bool) -> bool return False -def _process_line(line, is_markdown): # type: (bytes, bool) -> bytes +def _process_line( + line: bytes, + is_markdown: bool, + chars: Optional[bytes], +) -> bytes: if line[-2:] == b'\r\n': eol = b'\r\n' + line = line[:-2] elif line[-1:] == b'\n': eol = b'\n' + line = line[:-1] else: eol = b'' # preserve trailing two-space for non-blank lines in markdown files - if is_markdown and (not line.isspace()) and line.endswith(b' ' + eol): - return line.rstrip() + b' ' + eol - return line.rstrip() + eol + if is_markdown and (not line.isspace()) and line.endswith(b' '): + return line[:-2].rstrip(chars) + b' ' + eol + return line.rstrip(chars) + eol -def main(argv=None): # type: (Optional[Sequence[str]]) -> int +def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( '--no-markdown-linebreak-ext', @@ -50,6 +57,13 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int 'default: %(default)s' ), ) + parser.add_argument( + '--chars', + help=( + 'The set of characters to strip from the end of lines. ' + 'Defaults to all whitespace characters.' + ), + ) parser.add_argument('filenames', nargs='*', help='Filenames to fix') args = parser.parse_args(argv) @@ -69,20 +83,20 @@ def main(argv=None): # type: (Optional[Sequence[str]]) -> int for ext in md_exts: if any(c in ext[1:] for c in r'./\:'): parser.error( - 'bad --markdown-linebreak-ext extension {!r} (has . / \\ :)\n' - " (probably filename; use '--markdown-linebreak-ext=EXT')" - .format(ext), + f'bad --markdown-linebreak-ext extension ' + f'{ext!r} (has . / \\ :)\n' + f" (probably filename; use '--markdown-linebreak-ext=EXT')", ) - + chars = None if args.chars is None else args.chars.encode() return_code = 0 for filename in args.filenames: _, extension = os.path.splitext(filename.lower()) md = all_markdown or extension in md_exts - if _fix_file(filename, md): - print('Fixing {}'.format(filename)) + if _fix_file(filename, md, chars): + print(f'Fixing {filename}') return_code = 1 return return_code if __name__ == '__main__': - sys.exit(main()) + exit(main()) diff --git a/pre_commit_hooks/util.py b/pre_commit_hooks/util.py index d68e7699..402e33e6 100644 --- a/pre_commit_hooks/util.py +++ b/pre_commit_hooks/util.py @@ -1,9 +1,7 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import subprocess from typing import Any +from typing import List +from typing import Optional from typing import Set @@ -11,19 +9,25 @@ class CalledProcessError(RuntimeError): pass -def added_files(): # type: () -> Set[str] - return set(cmd_output( - 'git', 'diff', '--staged', '--name-only', '--diff-filter=A', - ).splitlines()) +def added_files() -> Set[str]: + cmd = ('git', 'diff', '--staged', '--name-only', '--diff-filter=A') + return set(cmd_output(*cmd).splitlines()) -def cmd_output(*cmd, **kwargs): # type: (*str, **Any) -> str - retcode = kwargs.pop('retcode', 0) +def cmd_output(*cmd: str, retcode: Optional[int] = 0, **kwargs: Any) -> str: kwargs.setdefault('stdout', subprocess.PIPE) kwargs.setdefault('stderr', subprocess.PIPE) proc = subprocess.Popen(cmd, **kwargs) stdout, stderr = proc.communicate() - stdout = stdout.decode('UTF-8') + stdout = stdout.decode() if retcode is not None and proc.returncode != retcode: raise CalledProcessError(cmd, retcode, proc.returncode, stdout, stderr) return stdout + + +def zsplit(s: str) -> List[str]: + s = s.strip('\0') + if s: + return s.split('\0') + else: + return [] diff --git a/requirements-dev.txt b/requirements-dev.txt index 2922ef5f..0c5a37eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,3 @@ --e . - +covdefaults coverage -flake8 -mock -pre-commit pytest diff --git a/setup.cfg b/setup.cfg index 4038836f..fc579392 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pre_commit_hooks -version = 2.3.0 +version = 4.0.1 description = Some out-of-the-box hooks for pre-commit. long_description = file: README.md long_description_content_type = text/markdown @@ -11,29 +11,29 @@ license = MIT license_file = LICENSE classifiers = License :: OSI Approved :: MIT License - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.4 - Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy [options] packages = find: install_requires = - flake8 ruamel.yaml>=0.15 toml - six - typing; python_version<"3.5" -python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +python_requires = >=3.6.1 + +[options.packages.find] +exclude = + tests* + testing* [options.entry_points] console_scripts = - autopep8-wrapper = pre_commit_hooks.autopep8_wrapper:main check-added-large-files = pre_commit_hooks.check_added_large_files:main check-ast = pre_commit_hooks.check_ast:main check-builtin-literals = pre_commit_hooks.check_builtin_literals:main @@ -43,35 +43,37 @@ console_scripts = check-executables-have-shebangs = pre_commit_hooks.check_executables_have_shebangs:main check-json = pre_commit_hooks.check_json:main check-merge-conflict = pre_commit_hooks.check_merge_conflict:main + check-shebang-scripts-are-executable = pre_commit_hooks.check_shebang_scripts_are_executable:main check-symlinks = pre_commit_hooks.check_symlinks:main check-toml = pre_commit_hooks.check_toml:main check-vcs-permalinks = pre_commit_hooks.check_vcs_permalinks:main check-xml = pre_commit_hooks.check_xml:main check-yaml = pre_commit_hooks.check_yaml:main debug-statement-hook = pre_commit_hooks.debug_statement_hook:main + destroyed-symlinks = pre_commit_hooks.destroyed_symlinks:main detect-aws-credentials = pre_commit_hooks.detect_aws_credentials:main detect-private-key = pre_commit_hooks.detect_private_key:main double-quote-string-fixer = pre_commit_hooks.string_fixer:main end-of-file-fixer = pre_commit_hooks.end_of_file_fixer:main file-contents-sorter = pre_commit_hooks.file_contents_sorter:main + fix-byte-order-marker = pre_commit_hooks.fix_byte_order_marker:main fix-encoding-pragma = pre_commit_hooks.fix_encoding_pragma:main forbid-new-submodules = pre_commit_hooks.forbid_new_submodules:main mixed-line-ending = pre_commit_hooks.mixed_line_ending:main name-tests-test = pre_commit_hooks.tests_should_end_in_test:main no-commit-to-branch = pre_commit_hooks.no_commit_to_branch:main + pre-commit-hooks-removed = pre_commit_hooks.removed:main pretty-format-json = pre_commit_hooks.pretty_format_json:main requirements-txt-fixer = pre_commit_hooks.requirements_txt_fixer:main sort-simple-yaml = pre_commit_hooks.sort_simple_yaml:main trailing-whitespace-fixer = pre_commit_hooks.trailing_whitespace_fixer:main -[options.packages.find] -exclude = - tests* - testing* - [bdist_wheel] universal = True +[coverage:run] +plugins = covdefaults + [mypy] check_untyped_defs = true disallow_any_generics = true diff --git a/testing/resources/duplicate_key_json.notjson b/testing/resources/duplicate_key_json.notjson new file mode 100644 index 00000000..8a432623 --- /dev/null +++ b/testing/resources/duplicate_key_json.notjson @@ -0,0 +1,4 @@ +{ + "hello": "world", + "hello": "planet" +} diff --git a/testing/util.py b/testing/util.py index fac498c7..8e468d60 100644 --- a/testing/util.py +++ b/testing/util.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import os.path diff --git a/tests/autopep8_wrapper_test.py b/tests/autopep8_wrapper_test.py deleted file mode 100644 index 615ec25c..00000000 --- a/tests/autopep8_wrapper_test.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import pytest - -from pre_commit_hooks.autopep8_wrapper import main - - -def test_invariantly_fails(): - with pytest.raises(SystemExit) as excinfo: - main() - msg, = excinfo.value.args - assert 'https://github.com/pre-commit/mirrors-autopep8' in msg diff --git a/tests/check_added_large_files_test.py b/tests/check_added_large_files_test.py index 2f67d1b0..ff53b05b 100644 --- a/tests/check_added_large_files_test.py +++ b/tests/check_added_large_files_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import distutils.spawn import pytest @@ -43,6 +40,17 @@ def test_add_something_giant(temp_git_dir): assert find_large_added_files(['f.py'], 10) == 0 +def test_enforce_all(temp_git_dir): + with temp_git_dir.as_cwd(): + temp_git_dir.join('f.py').write('a' * 10000) + + # Should fail, when not staged with enforce_all + assert find_large_added_files(['f.py'], 0, enforce_all=True) == 1 + + # Should pass, when not staged without enforce_all + assert find_large_added_files(['f.py'], 0, enforce_all=False) == 0 + + def test_added_file_not_in_pre_commits_list(temp_git_dir): with temp_git_dir.as_cwd(): temp_git_dir.join('f.py').write("print('hello world')") @@ -78,7 +86,7 @@ def has_gitlfs(): @xfailif_no_gitlfs def test_allows_gitlfs(temp_git_dir, monkeypatch): # pragma: no cover with temp_git_dir.as_cwd(): - monkeypatch.setenv(str('HOME'), str(temp_git_dir.strpath)) + monkeypatch.setenv('HOME', str(temp_git_dir)) cmd_output('git', 'lfs', 'install') temp_git_dir.join('f.py').write('a' * 10000) cmd_output('git', 'lfs', 'track', 'f.py') @@ -90,7 +98,7 @@ def test_allows_gitlfs(temp_git_dir, monkeypatch): # pragma: no cover @xfailif_no_gitlfs def test_moves_with_gitlfs(temp_git_dir, monkeypatch): # pragma: no cover with temp_git_dir.as_cwd(): - monkeypatch.setenv(str('HOME'), str(temp_git_dir.strpath)) + monkeypatch.setenv('HOME', str(temp_git_dir)) cmd_output('git', 'lfs', 'install') cmd_output('git', 'lfs', 'track', 'a.bin', 'b.bin') # First add the file we're going to move @@ -100,3 +108,15 @@ def test_moves_with_gitlfs(temp_git_dir, monkeypatch): # pragma: no cover # Now move it and make sure the hook still succeeds cmd_output('git', 'mv', 'a.bin', 'b.bin') assert main(('--maxkb', '9', 'b.bin')) == 0 + + +@xfailif_no_gitlfs +def test_enforce_allows_gitlfs(temp_git_dir, monkeypatch): # pragma: no cover + with temp_git_dir.as_cwd(): + monkeypatch.setenv('HOME', str(temp_git_dir)) + cmd_output('git', 'lfs', 'install') + temp_git_dir.join('f.py').write('a' * 10000) + cmd_output('git', 'lfs', 'track', 'f.py') + cmd_output('git', 'add', '--', '.') + # With --enforce-all large files on git lfs should succeed + assert main(('--enforce-all', '--maxkb', '9', 'f.py')) == 0 diff --git a/tests/check_ast_test.py b/tests/check_ast_test.py index c16f5fcc..686fd116 100644 --- a/tests/check_ast_test.py +++ b/tests/check_ast_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - from pre_commit_hooks.check_ast import main from testing.util import get_resource_path diff --git a/tests/check_builtin_literals_test.py b/tests/check_builtin_literals_test.py index 19263b79..e9367989 100644 --- a/tests/check_builtin_literals_test.py +++ b/tests/check_builtin_literals_test.py @@ -7,7 +7,7 @@ from pre_commit_hooks.check_builtin_literals import Visitor BUILTIN_CONSTRUCTORS = '''\ -from six.moves import builtins +import builtins c1 = complex() d1 = dict() @@ -121,9 +121,9 @@ def test_dict_no_allow_kwargs_exprs(expression, calls): def test_ignore_constructors(): - visitor = Visitor(ignore=( - 'complex', 'dict', 'float', 'int', 'list', 'str', 'tuple', - )) + visitor = Visitor( + ignore=('complex', 'dict', 'float', 'int', 'list', 'str', 'tuple'), + ) visitor.visit(ast.parse(BUILTIN_CONSTRUCTORS)) assert visitor.builtin_type_calls == [] @@ -131,19 +131,19 @@ def test_ignore_constructors(): def test_failing_file(tmpdir): f = tmpdir.join('f.py') f.write(BUILTIN_CONSTRUCTORS) - rc = main([f.strpath]) + rc = main([str(f)]) assert rc == 1 def test_passing_file(tmpdir): f = tmpdir.join('f.py') f.write(BUILTIN_LITERALS) - rc = main([f.strpath]) + rc = main([str(f)]) assert rc == 0 def test_failing_file_ignore_all(tmpdir): f = tmpdir.join('f.py') f.write(BUILTIN_CONSTRUCTORS) - rc = main(['--ignore=complex,dict,float,int,list,str,tuple', f.strpath]) + rc = main(['--ignore=complex,dict,float,int,list,str,tuple', str(f)]) assert rc == 0 diff --git a/tests/check_byte_order_marker_test.py b/tests/check_byte_order_marker_test.py index 53cb4a1b..4c402476 100644 --- a/tests/check_byte_order_marker_test.py +++ b/tests/check_byte_order_marker_test.py @@ -1,16 +1,13 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - from pre_commit_hooks import check_byte_order_marker def test_failure(tmpdir): f = tmpdir.join('f.txt') f.write_text('ohai', encoding='utf-8-sig') - assert check_byte_order_marker.main((f.strpath,)) == 1 + assert check_byte_order_marker.main((str(f),)) == 1 def test_success(tmpdir): f = tmpdir.join('f.txt') f.write_text('ohai', encoding='utf-8') - assert check_byte_order_marker.main((f.strpath,)) == 0 + assert check_byte_order_marker.main((str(f),)) == 0 diff --git a/tests/check_case_conflict_test.py b/tests/check_case_conflict_test.py index 077b41b6..c8c9d122 100644 --- a/tests/check_case_conflict_test.py +++ b/tests/check_case_conflict_test.py @@ -1,10 +1,24 @@ -from __future__ import absolute_import -from __future__ import unicode_literals +import sys + +import pytest from pre_commit_hooks.check_case_conflict import find_conflicting_filenames from pre_commit_hooks.check_case_conflict import main +from pre_commit_hooks.check_case_conflict import parents from pre_commit_hooks.util import cmd_output +skip_win32 = pytest.mark.skipif( + sys.platform == 'win32', + reason='case conflicts between directories and files', +) + + +def test_parents(): + assert set(parents('a')) == set() + assert set(parents('a/b')) == {'a'} + assert set(parents('a/b/c')) == {'a/b', 'a'} + assert set(parents('a/b/c/d')) == {'a/b/c', 'a/b', 'a'} + def test_nothing_added(temp_git_dir): with temp_git_dir.as_cwd(): @@ -29,6 +43,36 @@ def test_adding_something_with_conflict(temp_git_dir): assert find_conflicting_filenames(['f.py', 'F.py']) == 1 +@skip_win32 # pragma: win32 no cover +def test_adding_files_with_conflicting_directories(temp_git_dir): + with temp_git_dir.as_cwd(): + temp_git_dir.mkdir('dir').join('x').write('foo') + temp_git_dir.mkdir('DIR').join('y').write('foo') + cmd_output('git', 'add', '-A') + + assert find_conflicting_filenames([]) == 1 + + +@skip_win32 # pragma: win32 no cover +def test_adding_files_with_conflicting_deep_directories(temp_git_dir): + with temp_git_dir.as_cwd(): + temp_git_dir.mkdir('x').mkdir('y').join('z').write('foo') + temp_git_dir.join('X').write('foo') + cmd_output('git', 'add', '-A') + + assert find_conflicting_filenames([]) == 1 + + +@skip_win32 # pragma: win32 no cover +def test_adding_file_with_conflicting_directory(temp_git_dir): + with temp_git_dir.as_cwd(): + temp_git_dir.mkdir('dir').join('x').write('foo') + temp_git_dir.join('DIR').write('foo') + cmd_output('git', 'add', '-A') + + assert find_conflicting_filenames([]) == 1 + + def test_added_file_not_in_pre_commits_list(temp_git_dir): with temp_git_dir.as_cwd(): temp_git_dir.join('f.py').write("print('hello world')") @@ -49,6 +93,19 @@ def test_file_conflicts_with_committed_file(temp_git_dir): assert find_conflicting_filenames(['F.py']) == 1 +@skip_win32 # pragma: win32 no cover +def test_file_conflicts_with_committed_dir(temp_git_dir): + with temp_git_dir.as_cwd(): + temp_git_dir.mkdir('dir').join('x').write('foo') + cmd_output('git', 'add', '-A') + cmd_output('git', 'commit', '--no-gpg-sign', '-n', '-m', 'Add f.py') + + temp_git_dir.join('DIR').write('foo') + cmd_output('git', 'add', '-A') + + assert find_conflicting_filenames([]) == 1 + + def test_integration(temp_git_dir): with temp_git_dir.as_cwd(): assert main(argv=[]) == 0 diff --git a/tests/check_docstring_first_test.py b/tests/check_docstring_first_test.py index 0973a58f..ed5c08ef 100644 --- a/tests/check_docstring_first_test.py +++ b/tests/check_docstring_first_test.py @@ -1,7 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from pre_commit_hooks.check_docstring_first import check_docstring_first @@ -60,12 +56,12 @@ def test_unit(capsys, contents, expected, expected_out): def test_integration(tmpdir, capsys, contents, expected, expected_out): f = tmpdir.join('test.py') f.write_binary(contents) - assert main([f.strpath]) == expected - assert capsys.readouterr()[0] == expected_out.format(filename=f.strpath) + assert main([str(f)]) == expected + assert capsys.readouterr()[0] == expected_out.format(filename=str(f)) def test_arbitrary_encoding(tmpdir): f = tmpdir.join('f.py') contents = '# -*- coding: cp1252\nx = "£"'.encode('cp1252') f.write_binary(contents) - assert main([f.strpath]) == 0 + assert main([str(f)]) == 0 diff --git a/tests/check_executables_have_shebangs_test.py b/tests/check_executables_have_shebangs_test.py index 0cb9dcfd..5703eded 100644 --- a/tests/check_executables_have_shebangs_test.py +++ b/tests/check_executables_have_shebangs_test.py @@ -1,39 +1,125 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import unicode_literals +import os +import sys import pytest +from pre_commit_hooks import check_executables_have_shebangs from pre_commit_hooks.check_executables_have_shebangs import main +from pre_commit_hooks.util import cmd_output + +skip_win32 = pytest.mark.skipif( + sys.platform == 'win32', + reason="non-git checks aren't relevant on windows", +) +@skip_win32 # pragma: win32 no cover @pytest.mark.parametrize( 'content', ( b'#!/bin/bash\nhello world\n', b'#!/usr/bin/env python3.6', b'#!python', - '#!☃'.encode('UTF-8'), + '#!☃'.encode(), ), ) def test_has_shebang(content, tmpdir): path = tmpdir.join('path') path.write(content, 'wb') - assert main((path.strpath,)) == 0 + assert main((str(path),)) == 0 +@skip_win32 # pragma: win32 no cover @pytest.mark.parametrize( 'content', ( b'', b' #!python\n', b'\n#!python\n', b'python\n', - '☃'.encode('UTF-8'), - + '☃'.encode(), ), ) def test_bad_shebang(content, tmpdir, capsys): path = tmpdir.join('path') path.write(content, 'wb') - assert main((path.strpath,)) == 1 + assert main((str(path),)) == 1 _, stderr = capsys.readouterr() - assert stderr.startswith('{}: marked executable but'.format(path.strpath)) + assert stderr.startswith(f'{path}: marked executable but') + + +def test_check_git_filemode_passing(tmpdir): + with tmpdir.as_cwd(): + cmd_output('git', 'init', '.') + + f = tmpdir.join('f') + f.write('#!/usr/bin/env bash') + f_path = str(f) + cmd_output('chmod', '+x', f_path) + cmd_output('git', 'add', f_path) + cmd_output('git', 'update-index', '--chmod=+x', f_path) + + g = tmpdir.join('g').ensure() + g_path = str(g) + cmd_output('git', 'add', g_path) + + # this is potentially a problem, but not something the script intends + # to check for -- we're only making sure that things that are + # executable have shebangs + h = tmpdir.join('h') + h.write('#!/usr/bin/env bash') + h_path = str(h) + cmd_output('git', 'add', h_path) + + files = (f_path, g_path, h_path) + assert check_executables_have_shebangs._check_git_filemode(files) == 0 + + +def test_check_git_filemode_passing_unusual_characters(tmpdir): + with tmpdir.as_cwd(): + cmd_output('git', 'init', '.') + + f = tmpdir.join('mañana.txt') + f.write('#!/usr/bin/env bash') + f_path = str(f) + cmd_output('chmod', '+x', f_path) + cmd_output('git', 'add', f_path) + cmd_output('git', 'update-index', '--chmod=+x', f_path) + + files = (f_path,) + assert check_executables_have_shebangs._check_git_filemode(files) == 0 + + +def test_check_git_filemode_failing(tmpdir): + with tmpdir.as_cwd(): + cmd_output('git', 'init', '.') + + f = tmpdir.join('f').ensure() + f_path = str(f) + cmd_output('chmod', '+x', f_path) + cmd_output('git', 'add', f_path) + cmd_output('git', 'update-index', '--chmod=+x', f_path) + + files = (f_path,) + assert check_executables_have_shebangs._check_git_filemode(files) == 1 + + +@pytest.mark.parametrize( + ('content', 'mode', 'expected'), + ( + pytest.param('#!python', '+x', 0, id='shebang with executable'), + pytest.param('#!python', '-x', 0, id='shebang without executable'), + pytest.param('', '+x', 1, id='no shebang with executable'), + pytest.param('', '-x', 0, id='no shebang without executable'), + ), +) +def test_git_executable_shebang(temp_git_dir, content, mode, expected): + with temp_git_dir.as_cwd(): + path = temp_git_dir.join('path') + path.write(content) + cmd_output('git', 'add', str(path)) + cmd_output('chmod', mode, str(path)) + cmd_output('git', 'update-index', f'--chmod={mode}', str(path)) + + # simulate how identify chooses that something is executable + filenames = [path for path in [str(path)] if os.access(path, os.X_OK)] + + assert main(filenames) == expected diff --git a/tests/check_json_test.py b/tests/check_json_test.py index 6654ed10..3ec67f1c 100644 --- a/tests/check_json_test.py +++ b/tests/check_json_test.py @@ -9,6 +9,7 @@ ('bad_json.notjson', 1), ('bad_json_latin1.nonjson', 1), ('ok_json.json', 0), + ('duplicate_key_json.notjson', 1), ), ) def test_main(capsys, filename, expected_retval): @@ -17,3 +18,9 @@ def test_main(capsys, filename, expected_retval): if expected_retval == 1: stdout, _ = capsys.readouterr() assert filename in stdout + + +def test_non_utf8_file(tmpdir): + f = tmpdir.join('t.json') + f.write_binary(b'\xa9\xfe\x12') + assert main((str(f),)) diff --git a/tests/check_merge_conflict_test.py b/tests/check_merge_conflict_test.py index af7cc43b..fccf41ff 100644 --- a/tests/check_merge_conflict_test.py +++ b/tests/check_merge_conflict_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import os import shutil @@ -19,13 +16,13 @@ def f1_is_a_conflict_file(tmpdir): repo2 = tmpdir.join('repo2') repo2_f1 = repo2.join('f1') - cmd_output('git', 'init', '--', repo1.strpath) + cmd_output('git', 'init', '--', str(repo1)) with repo1.as_cwd(): repo1_f1.ensure() cmd_output('git', 'add', '.') cmd_output('git', 'commit', '--no-gpg-sign', '-m', 'commit1') - cmd_output('git', 'clone', repo1.strpath, repo2.strpath) + cmd_output('git', 'clone', str(repo1), str(repo2)) # Commit in master with repo1.as_cwd(): @@ -74,13 +71,13 @@ def repository_pending_merge(tmpdir): repo2 = tmpdir.join('repo2') repo2_f1 = repo2.join('f1') repo2_f2 = repo2.join('f2') - cmd_output('git', 'init', repo1.strpath) + cmd_output('git', 'init', str(repo1)) with repo1.as_cwd(): repo1_f1.ensure() cmd_output('git', 'add', '.') cmd_output('git', 'commit', '--no-gpg-sign', '-m', 'commit1') - cmd_output('git', 'clone', repo1.strpath, repo2.strpath) + cmd_output('git', 'clone', str(repo1), str(repo2)) # Commit in master with repo1.as_cwd(): diff --git a/tests/check_shebang_scripts_are_executable_test.py b/tests/check_shebang_scripts_are_executable_test.py new file mode 100644 index 00000000..9e78b06c --- /dev/null +++ b/tests/check_shebang_scripts_are_executable_test.py @@ -0,0 +1,87 @@ +import os + +import pytest + +from pre_commit_hooks.check_shebang_scripts_are_executable import \ + _check_git_filemode +from pre_commit_hooks.check_shebang_scripts_are_executable import main +from pre_commit_hooks.util import cmd_output + + +def test_check_git_filemode_passing(tmpdir): + with tmpdir.as_cwd(): + cmd_output('git', 'init', '.') + + f = tmpdir.join('f') + f.write('#!/usr/bin/env bash') + f_path = str(f) + cmd_output('chmod', '+x', f_path) + cmd_output('git', 'add', f_path) + cmd_output('git', 'update-index', '--chmod=+x', f_path) + + g = tmpdir.join('g').ensure() + g_path = str(g) + cmd_output('git', 'add', g_path) + + files = [f_path, g_path] + assert _check_git_filemode(files) == 0 + + # this is the one we should trigger on + h = tmpdir.join('h') + h.write('#!/usr/bin/env bash') + h_path = str(h) + cmd_output('git', 'add', h_path) + + files = [h_path] + assert _check_git_filemode(files) == 1 + + +def test_check_git_filemode_passing_unusual_characters(tmpdir): + with tmpdir.as_cwd(): + cmd_output('git', 'init', '.') + + f = tmpdir.join('mañana.txt') + f.write('#!/usr/bin/env bash') + f_path = str(f) + cmd_output('chmod', '+x', f_path) + cmd_output('git', 'add', f_path) + cmd_output('git', 'update-index', '--chmod=+x', f_path) + + files = (f_path,) + assert _check_git_filemode(files) == 0 + + +def test_check_git_filemode_failing(tmpdir): + with tmpdir.as_cwd(): + cmd_output('git', 'init', '.') + + f = tmpdir.join('f').ensure() + f.write('#!/usr/bin/env bash') + f_path = str(f) + cmd_output('git', 'add', f_path) + + files = (f_path,) + assert _check_git_filemode(files) == 1 + + +@pytest.mark.parametrize( + ('content', 'mode', 'expected'), + ( + pytest.param('#!python', '+x', 0, id='shebang with executable'), + pytest.param('#!python', '-x', 1, id='shebang without executable'), + pytest.param('', '+x', 0, id='no shebang with executable'), + pytest.param('', '-x', 0, id='no shebang without executable'), + ), +) +def test_git_executable_shebang(temp_git_dir, content, mode, expected): + with temp_git_dir.as_cwd(): + path = temp_git_dir.join('path') + path.write(content) + cmd_output('git', 'add', str(path)) + cmd_output('chmod', mode, str(path)) + cmd_output('git', 'update-index', f'--chmod={mode}', str(path)) + + # simulate how identify chooses that something is executable + filenames = [path for path in [str(path)] if os.access(path, os.X_OK)] + + assert main(filenames) == expected diff --git a/tests/check_symlinks_test.py b/tests/check_symlinks_test.py index ecbc7aec..07c11687 100644 --- a/tests/check_symlinks_test.py +++ b/tests/check_symlinks_test.py @@ -16,8 +16,8 @@ def test_main(tmpdir, dest, expected): # pragma: no cover (symlinks) tmpdir.join('exists').ensure() symlink = tmpdir.join('symlink') symlink.mksymlinkto(tmpdir.join(dest)) - assert main((symlink.strpath,)) == expected + assert main((str(symlink),)) == expected def test_main_normal_file(tmpdir): - assert main((tmpdir.join('f').ensure().strpath,)) == 0 + assert main((str(tmpdir.join('f').ensure()),)) == 0 diff --git a/tests/check_toml_test.py b/tests/check_toml_test.py index 1172c40a..c7251eb0 100644 --- a/tests/check_toml_test.py +++ b/tests/check_toml_test.py @@ -1,21 +1,18 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - from pre_commit_hooks.check_toml import main -def test_toml_good(tmpdir): +def test_toml_bad(tmpdir): filename = tmpdir.join('f') filename.write(""" key = # INVALID = "no key name" # INVALID """) - ret = main((filename.strpath,)) + ret = main((str(filename),)) assert ret == 1 -def test_toml_bad(tmpdir): +def test_toml_good(tmpdir): filename = tmpdir.join('f') filename.write( """ @@ -28,5 +25,12 @@ def test_toml_bad(tmpdir): dob = 1979-05-27T07:32:00-08:00 # First class dates """, ) - ret = main((filename.strpath,)) + ret = main((str(filename),)) + assert ret == 0 + + +def test_toml_good_unicode(tmpdir): + filename = tmpdir.join('f') + filename.write_binary('letter = "\N{SNOWMAN}"\n'.encode()) + ret = main((str(filename),)) assert ret == 0 diff --git a/tests/check_vcs_permalinks_test.py b/tests/check_vcs_permalinks_test.py index 00e5396d..ad591515 100644 --- a/tests/check_vcs_permalinks_test.py +++ b/tests/check_vcs_permalinks_test.py @@ -1,12 +1,9 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - from pre_commit_hooks.check_vcs_permalinks import main def test_trivial(tmpdir): f = tmpdir.join('f.txt').ensure() - assert not main((f.strpath,)) + assert not main((str(f),)) def test_passing(tmpdir): @@ -14,24 +11,30 @@ def test_passing(tmpdir): f.write_binary( # permalinks are ok b'https://github.com/asottile/test/blob/649e6/foo%20bar#L1\n' + # tags are ok + b'https://github.com/asottile/test/blob/1.0.0/foo%20bar#L1\n' # links to files but not line numbers are ok b'https://github.com/asottile/test/blob/master/foo%20bar\n' # regression test for overly-greedy regex b'https://github.com/ yes / no ? /blob/master/foo#L1\n', ) - assert not main((f.strpath,)) + assert not main((str(f),)) def test_failing(tmpdir, capsys): with tmpdir.as_cwd(): tmpdir.join('f.txt').write_binary( - b'https://github.com/asottile/test/blob/master/foo#L1\n', + b'https://github.com/asottile/test/blob/master/foo#L1\n' + b'https://example.com/asottile/test/blob/master/foo#L1\n' + b'https://example.com/asottile/test/blob/main/foo#L1\n', ) - assert main(('f.txt',)) + assert main(('f.txt', '--additional-github-domain', 'example.com')) out, _ = capsys.readouterr() assert out == ( 'f.txt:1:https://github.com/asottile/test/blob/master/foo#L1\n' + 'f.txt:2:https://example.com/asottile/test/blob/master/foo#L1\n' + 'f.txt:3:https://example.com/asottile/test/blob/main/foo#L1\n' '\n' 'Non-permanent github link detected.\n' 'On any page on github press [y] to load a permalink.\n' diff --git a/tests/check_yaml_test.py b/tests/check_yaml_test.py index d267150a..1a017a12 100644 --- a/tests/check_yaml_test.py +++ b/tests/check_yaml_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from pre_commit_hooks.check_yaml import main @@ -23,16 +20,16 @@ def test_main_allow_multiple_documents(tmpdir): f.write('---\nfoo\n---\nbar\n') # should fail without the setting - assert main((f.strpath,)) + assert main((str(f),)) # should pass when we allow multiple documents - assert not main(('--allow-multiple-documents', f.strpath)) + assert not main(('--allow-multiple-documents', str(f))) def test_fails_even_with_allow_multiple_documents(tmpdir): f = tmpdir.join('test.yaml') f.write('[') - assert main(('--allow-multiple-documents', f.strpath)) + assert main(('--allow-multiple-documents', str(f))) def test_main_unsafe(tmpdir): @@ -43,12 +40,12 @@ def test_main_unsafe(tmpdir): ' deadbeefdeadbeefdeadbeef\n', ) # should fail "safe" check - assert main((f.strpath,)) + assert main((str(f),)) # should pass when we allow unsafe documents - assert not main(('--unsafe', f.strpath)) + assert not main(('--unsafe', str(f))) def test_main_unsafe_still_fails_on_syntax_errors(tmpdir): f = tmpdir.join('test.yaml') f.write('[') - assert main(('--unsafe', f.strpath)) + assert main(('--unsafe', str(f))) diff --git a/tests/conftest.py b/tests/conftest.py index da206cb9..f92cfc18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import pytest from pre_commit_hooks.util import cmd_output @@ -10,5 +6,5 @@ @pytest.fixture def temp_git_dir(tmpdir): git_dir = tmpdir.join('gits') - cmd_output('git', 'init', '--', git_dir.strpath) + cmd_output('git', 'init', '--', str(git_dir)) yield git_dir diff --git a/tests/debug_statement_hook_test.py b/tests/debug_statement_hook_test.py index d15f5f7c..428421a0 100644 --- a/tests/debug_statement_hook_test.py +++ b/tests/debug_statement_hook_test.py @@ -1,7 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import unicode_literals - import ast from pre_commit_hooks.debug_statement_hook import Debug @@ -37,7 +33,7 @@ def test_finds_breakpoint(): def test_returns_one_for_failing_file(tmpdir): f_py = tmpdir.join('f.py') f_py.write('def f():\n import pdb; pdb.set_trace()') - ret = main([f_py.strpath]) + ret = main([str(f_py)]) assert ret == 1 @@ -54,10 +50,10 @@ def test_syntaxerror_file(): def test_non_utf8_file(tmpdir): f_py = tmpdir.join('f.py') f_py.write_binary('# -*- coding: cp1252 -*-\nx = "€"\n'.encode('cp1252')) - assert main((f_py.strpath,)) == 0 + assert main((str(f_py),)) == 0 def test_py37_breakpoint(tmpdir): f_py = tmpdir.join('f.py') f_py.write('def f():\n breakpoint()\n') - assert main((f_py.strpath,)) == 1 + assert main((str(f_py),)) == 1 diff --git a/tests/destroyed_symlinks_test.py b/tests/destroyed_symlinks_test.py new file mode 100644 index 00000000..d2c90310 --- /dev/null +++ b/tests/destroyed_symlinks_test.py @@ -0,0 +1,74 @@ +import os +import subprocess + +import pytest + +from pre_commit_hooks.destroyed_symlinks import find_destroyed_symlinks +from pre_commit_hooks.destroyed_symlinks import main + +TEST_SYMLINK = 'test_symlink' +TEST_SYMLINK_TARGET = '/doesnt/really/matters' +TEST_FILE = 'test_file' +TEST_FILE_RENAMED = f'{TEST_FILE}_renamed' + + +@pytest.fixture +def repo_with_destroyed_symlink(tmpdir): + source_repo = tmpdir.join('src') + os.makedirs(source_repo, exist_ok=True) + test_repo = tmpdir.join('test') + with source_repo.as_cwd(): + subprocess.check_call(('git', 'init')) + os.symlink(TEST_SYMLINK_TARGET, TEST_SYMLINK) + with open(TEST_FILE, 'w') as f: + print('some random content', file=f) + subprocess.check_call(('git', 'add', '.')) + subprocess.check_call( + ('git', 'commit', '--no-gpg-sign', '-m', 'initial'), + ) + assert b'120000 ' in subprocess.check_output( + ('git', 'cat-file', '-p', 'HEAD^{tree}'), + ) + subprocess.check_call( + ('git', '-c', 'core.symlinks=false', 'clone', source_repo, test_repo), + ) + with test_repo.as_cwd(): + subprocess.check_call( + ('git', 'config', '--local', 'core.symlinks', 'true'), + ) + subprocess.check_call(('git', 'mv', TEST_FILE, TEST_FILE_RENAMED)) + assert not os.path.islink(test_repo.join(TEST_SYMLINK)) + yield test_repo + + +def test_find_destroyed_symlinks(repo_with_destroyed_symlink): + with repo_with_destroyed_symlink.as_cwd(): + assert find_destroyed_symlinks([]) == [] + assert main([]) == 0 + + subprocess.check_call(('git', 'add', TEST_SYMLINK)) + assert find_destroyed_symlinks([TEST_SYMLINK]) == [TEST_SYMLINK] + assert find_destroyed_symlinks([]) == [] + assert main([]) == 0 + assert find_destroyed_symlinks([TEST_FILE_RENAMED, TEST_FILE]) == [] + ALL_STAGED = [TEST_SYMLINK, TEST_FILE_RENAMED] + assert find_destroyed_symlinks(ALL_STAGED) == [TEST_SYMLINK] + assert main(ALL_STAGED) != 0 + + with open(TEST_SYMLINK, 'a') as f: + print(file=f) # add trailing newline + subprocess.check_call(['git', 'add', TEST_SYMLINK]) + assert find_destroyed_symlinks(ALL_STAGED) == [TEST_SYMLINK] + assert main(ALL_STAGED) != 0 + + with open(TEST_SYMLINK, 'w') as f: + print('0' * len(TEST_SYMLINK_TARGET), file=f) + subprocess.check_call(('git', 'add', TEST_SYMLINK)) + assert find_destroyed_symlinks(ALL_STAGED) == [] + assert main(ALL_STAGED) == 0 + + with open(TEST_SYMLINK, 'w') as f: + print('0' * (len(TEST_SYMLINK_TARGET) + 3), file=f) + subprocess.check_call(('git', 'add', TEST_SYMLINK)) + assert find_destroyed_symlinks(ALL_STAGED) == [] + assert main(ALL_STAGED) == 0 diff --git a/tests/detect_aws_credentials_test.py b/tests/detect_aws_credentials_test.py index 777fb480..72125099 100644 --- a/tests/detect_aws_credentials_test.py +++ b/tests/detect_aws_credentials_test.py @@ -1,5 +1,6 @@ +from unittest.mock import patch + import pytest -from mock import patch from pre_commit_hooks.detect_aws_credentials import get_aws_cred_files_from_env from pre_commit_hooks.detect_aws_credentials import get_aws_secrets_from_env @@ -12,15 +13,15 @@ ('env_vars', 'values'), ( ({}, set()), - ({'AWS_DUMMY_KEY': '/foo'}, set()), + ({'AWS_PLACEHOLDER_KEY': '/foo'}, set()), ({'AWS_CONFIG_FILE': '/foo'}, {'/foo'}), ({'AWS_CREDENTIAL_FILE': '/foo'}, {'/foo'}), ({'AWS_SHARED_CREDENTIALS_FILE': '/foo'}, {'/foo'}), ({'BOTO_CONFIG': '/foo'}, {'/foo'}), - ({'AWS_DUMMY_KEY': '/foo', 'AWS_CONFIG_FILE': '/bar'}, {'/bar'}), + ({'AWS_PLACEHOLDER_KEY': '/foo', 'AWS_CONFIG_FILE': '/bar'}, {'/bar'}), ( { - 'AWS_DUMMY_KEY': '/foo', 'AWS_CONFIG_FILE': '/bar', + 'AWS_PLACEHOLDER_KEY': '/foo', 'AWS_CONFIG_FILE': '/bar', 'AWS_CREDENTIAL_FILE': '/baz', }, {'/bar', '/baz'}, @@ -43,11 +44,16 @@ def test_get_aws_credentials_file_from_env(env_vars, values): ('env_vars', 'values'), ( ({}, set()), - ({'AWS_DUMMY_KEY': 'foo'}, set()), + ({'AWS_PLACEHOLDER_KEY': 'foo'}, set()), ({'AWS_SECRET_ACCESS_KEY': 'foo'}, {'foo'}), ({'AWS_SECURITY_TOKEN': 'foo'}, {'foo'}), ({'AWS_SESSION_TOKEN': 'foo'}, {'foo'}), - ({'AWS_DUMMY_KEY': 'foo', 'AWS_SECRET_ACCESS_KEY': 'bar'}, {'bar'}), + ({'AWS_SESSION_TOKEN': ''}, set()), + ({'AWS_SESSION_TOKEN': 'foo', 'AWS_SECURITY_TOKEN': ''}, {'foo'}), + ( + {'AWS_PLACEHOLDER_KEY': 'foo', 'AWS_SECRET_ACCESS_KEY': 'bar'}, + {'bar'}, + ), ( {'AWS_SECRET_ACCESS_KEY': 'foo', 'AWS_SECURITY_TOKEN': 'bar'}, {'foo', 'bar'}, @@ -115,6 +121,19 @@ def test_detect_aws_credentials(filename, expected_retval): assert ret == expected_retval +def test_allows_arbitrarily_encoded_files(tmpdir): + src_ini = tmpdir.join('src.ini') + src_ini.write( + '[default]\n' + 'aws_access_key_id=AKIASDFASDF\n' + 'aws_secret_Access_key=9018asdf23908190238123\n', + ) + arbitrary_encoding = tmpdir.join('f') + arbitrary_encoding.write_binary(b'\x12\x9a\xe2\xf2') + ret = main((str(arbitrary_encoding), '--credentials-file', str(src_ini))) + assert ret == 0 + + @patch('pre_commit_hooks.detect_aws_credentials.get_aws_secrets_from_file') @patch('pre_commit_hooks.detect_aws_credentials.get_aws_secrets_from_env') def test_non_existent_credentials(mock_secrets_env, mock_secrets_file, capsys): diff --git a/tests/detect_private_key_test.py b/tests/detect_private_key_test.py index 9266f2b0..72810008 100644 --- a/tests/detect_private_key_test.py +++ b/tests/detect_private_key_test.py @@ -21,4 +21,4 @@ def test_main(input_s, expected_retval, tmpdir): path = tmpdir.join('file.txt') path.write_binary(input_s) - assert main([path.strpath]) == expected_retval + assert main([str(path)]) == expected_retval diff --git a/tests/end_of_file_fixer_test.py b/tests/end_of_file_fixer_test.py index 7f644e76..60b9e82f 100644 --- a/tests/end_of_file_fixer_test.py +++ b/tests/end_of_file_fixer_test.py @@ -35,7 +35,7 @@ def test_integration(input_s, expected_retval, output, tmpdir): path = tmpdir.join('file.txt') path.write_binary(input_s) - ret = main([path.strpath]) + ret = main([str(path)]) file_output = path.read_binary() assert file_output == output diff --git a/tests/file_contents_sorter_test.py b/tests/file_contents_sorter_test.py index 1f9a14b4..15f11342 100644 --- a/tests/file_contents_sorter_test.py +++ b/tests/file_contents_sorter_test.py @@ -6,28 +6,82 @@ @pytest.mark.parametrize( - ('input_s', 'expected_retval', 'output'), + ('input_s', 'argv', 'expected_retval', 'output'), ( - (b'', FAIL, b'\n'), - (b'lonesome\n', PASS, b'lonesome\n'), - (b'missing_newline', FAIL, b'missing_newline\n'), - (b'newline\nmissing', FAIL, b'missing\nnewline\n'), - (b'missing\nnewline', FAIL, b'missing\nnewline\n'), - (b'alpha\nbeta\n', PASS, b'alpha\nbeta\n'), - (b'beta\nalpha\n', FAIL, b'alpha\nbeta\n'), - (b'C\nc\n', PASS, b'C\nc\n'), - (b'c\nC\n', FAIL, b'C\nc\n'), - (b'mag ical \n tre vor\n', FAIL, b' tre vor\nmag ical \n'), - (b'@\n-\n_\n#\n', FAIL, b'#\n-\n@\n_\n'), - (b'extra\n\n\nwhitespace\n', FAIL, b'extra\nwhitespace\n'), - (b'whitespace\n\n\nextra\n', FAIL, b'extra\nwhitespace\n'), + (b'', [], FAIL, b'\n'), + (b'lonesome\n', [], PASS, b'lonesome\n'), + (b'missing_newline', [], FAIL, b'missing_newline\n'), + (b'newline\nmissing', [], FAIL, b'missing\nnewline\n'), + (b'missing\nnewline', [], FAIL, b'missing\nnewline\n'), + (b'alpha\nbeta\n', [], PASS, b'alpha\nbeta\n'), + (b'beta\nalpha\n', [], FAIL, b'alpha\nbeta\n'), + (b'C\nc\n', [], PASS, b'C\nc\n'), + (b'c\nC\n', [], FAIL, b'C\nc\n'), + (b'mag ical \n tre vor\n', [], FAIL, b' tre vor\nmag ical \n'), + (b'@\n-\n_\n#\n', [], FAIL, b'#\n-\n@\n_\n'), + (b'extra\n\n\nwhitespace\n', [], FAIL, b'extra\nwhitespace\n'), + (b'whitespace\n\n\nextra\n', [], FAIL, b'extra\nwhitespace\n'), + ( + b'fee\nFie\nFoe\nfum\n', + [], + FAIL, + b'Fie\nFoe\nfee\nfum\n', + ), + ( + b'Fie\nFoe\nfee\nfum\n', + [], + PASS, + b'Fie\nFoe\nfee\nfum\n', + ), + ( + b'fee\nFie\nFoe\nfum\n', + ['--ignore-case'], + PASS, + b'fee\nFie\nFoe\nfum\n', + ), + ( + b'Fie\nFoe\nfee\nfum\n', + ['--ignore-case'], + FAIL, + b'fee\nFie\nFoe\nfum\n', + ), + ( + b'Fie\nFoe\nfee\nfee\nfum\n', + ['--ignore-case'], + FAIL, + b'fee\nfee\nFie\nFoe\nfum\n', + ), + ( + b'Fie\nFoe\nfee\nfum\n', + ['--unique'], + PASS, + b'Fie\nFoe\nfee\nfum\n', + ), + ( + b'Fie\nFie\nFoe\nfee\nfum\n', + ['--unique'], + FAIL, + b'Fie\nFoe\nfee\nfum\n', + ), + ( + b'fee\nFie\nFoe\nfum\n', + ['--unique', '--ignore-case'], + PASS, + b'fee\nFie\nFoe\nfum\n', + ), + ( + b'fee\nfee\nFie\nFoe\nfum\n', + ['--unique', '--ignore-case'], + FAIL, + b'fee\nFie\nFoe\nfum\n', + ), ), ) -def test_integration(input_s, expected_retval, output, tmpdir): +def test_integration(input_s, argv, expected_retval, output, tmpdir): path = tmpdir.join('file.txt') path.write_binary(input_s) - output_retval = main([path.strpath]) + output_retval = main([str(path)] + argv) assert path.read_binary() == output assert output_retval == expected_retval diff --git a/tests/fix_byte_order_marker_test.py b/tests/fix_byte_order_marker_test.py new file mode 100644 index 00000000..da150e37 --- /dev/null +++ b/tests/fix_byte_order_marker_test.py @@ -0,0 +1,13 @@ +from pre_commit_hooks import fix_byte_order_marker + + +def test_failure(tmpdir): + f = tmpdir.join('f.txt') + f.write_text('ohai', encoding='utf-8-sig') + assert fix_byte_order_marker.main((str(f),)) == 1 + + +def test_success(tmpdir): + f = tmpdir.join('f.txt') + f.write_text('ohai', encoding='utf-8') + assert fix_byte_order_marker.main((str(f),)) == 0 diff --git a/tests/fix_encoding_pragma_test.py b/tests/fix_encoding_pragma_test.py index d94b7256..f3afa094 100644 --- a/tests/fix_encoding_pragma_test.py +++ b/tests/fix_encoding_pragma_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import io import pytest @@ -14,7 +11,7 @@ def test_integration_inserting_pragma(tmpdir): path = tmpdir.join('foo.py') path.write_binary(b'import httplib\n') - assert main((path.strpath,)) == 1 + assert main((str(path),)) == 1 assert path.read_binary() == ( b'# -*- coding: utf-8 -*-\n' @@ -25,14 +22,14 @@ def test_integration_inserting_pragma(tmpdir): def test_integration_ok(tmpdir): path = tmpdir.join('foo.py') path.write_binary(b'# -*- coding: utf-8 -*-\nx = 1\n') - assert main((path.strpath,)) == 0 + assert main((str(path),)) == 0 def test_integration_remove(tmpdir): path = tmpdir.join('foo.py') path.write_binary(b'# -*- coding: utf-8 -*-\nx = 1\n') - assert main((path.strpath, '--remove')) == 1 + assert main((str(path), '--remove')) == 1 assert path.read_binary() == b'x = 1\n' @@ -40,7 +37,7 @@ def test_integration_remove(tmpdir): def test_integration_remove_ok(tmpdir): path = tmpdir.join('foo.py') path.write_binary(b'x = 1\n') - assert main((path.strpath, '--remove')) == 0 + assert main((str(path), '--remove')) == 0 @pytest.mark.parametrize( @@ -129,9 +126,6 @@ def test_not_ok_input_alternate_pragma(): @pytest.mark.parametrize( ('input_s', 'expected'), ( - # Python 2 cli parameters are bytes - (b'# coding: utf-8', b'# coding: utf-8'), - # Python 3 cli parameters are text ('# coding: utf-8', b'# coding: utf-8'), # trailing whitespace ('# coding: utf-8\n', b'# coding: utf-8'), @@ -146,20 +140,20 @@ def test_integration_alternate_pragma(tmpdir, capsys): f.write('x = 1\n') pragma = '# coding: utf-8' - assert main((f.strpath, '--pragma', pragma)) == 1 + assert main((str(f), '--pragma', pragma)) == 1 assert f.read() == '# coding: utf-8\nx = 1\n' out, _ = capsys.readouterr() - assert out == 'Added `# coding: utf-8` to {}\n'.format(f.strpath) + assert out == f'Added `# coding: utf-8` to {str(f)}\n' def test_crlf_ok(tmpdir): f = tmpdir.join('f.py') f.write_binary(b'# -*- coding: utf-8 -*-\r\nx = 1\r\n') - assert not main((f.strpath,)) + assert not main((str(f),)) def test_crfl_adds(tmpdir): f = tmpdir.join('f.py') f.write_binary(b'x = 1\r\n') - assert main((f.strpath,)) + assert main((str(f),)) assert f.read_binary() == b'# -*- coding: utf-8 -*-\r\nx = 1\r\n' diff --git a/tests/forbid_new_submodules_test.py b/tests/forbid_new_submodules_test.py index 523628d6..4871ae7f 100644 --- a/tests/forbid_new_submodules_test.py +++ b/tests/forbid_new_submodules_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import subprocess import pytest @@ -17,7 +15,7 @@ def git_dir_with_git_dir(tmpdir): subprocess.check_call(('git', 'init', 'foo')) subprocess.check_call( ('git', 'commit', '-m', 'init', '--allow-empty', '--no-gpg-sign'), - cwd=tmpdir.join('foo').strpath, + cwd=str(tmpdir.join('foo')), ) yield diff --git a/tests/mixed_line_ending_test.py b/tests/mixed_line_ending_test.py index 8ae93547..f1c26418 100644 --- a/tests/mixed_line_ending_test.py +++ b/tests/mixed_line_ending_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from pre_commit_hooks.mixed_line_ending import main @@ -28,7 +25,7 @@ def test_mixed_line_ending_fixes_auto(input_s, output, tmpdir): path = tmpdir.join('file.txt') path.write_binary(input_s) - ret = main((path.strpath,)) + ret = main((str(path),)) assert ret == 1 assert path.read_binary() == output @@ -37,7 +34,7 @@ def test_mixed_line_ending_fixes_auto(input_s, output, tmpdir): def test_non_mixed_no_newline_end_of_file(tmpdir): path = tmpdir.join('f.txt') path.write_binary(b'foo\nbar\nbaz') - assert not main((path.strpath,)) + assert not main((str(path),)) # the hook *could* fix the end of the file, but leaves it alone # this is mostly to document the current behaviour assert path.read_binary() == b'foo\nbar\nbaz' @@ -46,7 +43,7 @@ def test_non_mixed_no_newline_end_of_file(tmpdir): def test_mixed_no_newline_end_of_file(tmpdir): path = tmpdir.join('f.txt') path.write_binary(b'foo\r\nbar\nbaz') - assert main((path.strpath,)) + assert main((str(path),)) # the hook rewrites the end of the file, this is slightly inconsistent # with the non-mixed case but I think this is the better behaviour # this is mostly to document the current behaviour @@ -69,7 +66,7 @@ def test_mixed_no_newline_end_of_file(tmpdir): def test_line_endings_ok(fix_option, input_s, tmpdir, capsys): path = tmpdir.join('input.txt') path.write_binary(input_s) - ret = main((fix_option, path.strpath)) + ret = main((fix_option, str(path))) assert ret == 0 assert path.read_binary() == input_s @@ -81,29 +78,29 @@ def test_no_fix_does_not_modify(tmpdir, capsys): path = tmpdir.join('input.txt') contents = b'foo\r\nbar\rbaz\nwomp\n' path.write_binary(contents) - ret = main(('--fix=no', path.strpath)) + ret = main(('--fix=no', str(path))) assert ret == 1 assert path.read_binary() == contents out, _ = capsys.readouterr() - assert out == '{}: mixed line endings\n'.format(path) + assert out == f'{path}: mixed line endings\n' def test_fix_lf(tmpdir, capsys): path = tmpdir.join('input.txt') path.write_binary(b'foo\r\nbar\rbaz\n') - ret = main(('--fix=lf', path.strpath)) + ret = main(('--fix=lf', str(path))) assert ret == 1 assert path.read_binary() == b'foo\nbar\nbaz\n' out, _ = capsys.readouterr() - assert out == '{}: fixed mixed line endings\n'.format(path) + assert out == f'{path}: fixed mixed line endings\n' def test_fix_crlf(tmpdir): path = tmpdir.join('input.txt') path.write_binary(b'foo\r\nbar\rbaz\n') - ret = main(('--fix=crlf', path.strpath)) + ret = main(('--fix=crlf', str(path))) assert ret == 1 assert path.read_binary() == b'foo\r\nbar\r\nbaz\r\n' @@ -113,7 +110,7 @@ def test_fix_lf_all_crlf(tmpdir): """Regression test for #239""" path = tmpdir.join('input.txt') path.write_binary(b'foo\r\nbar\r\n') - ret = main(('--fix=lf', path.strpath)) + ret = main(('--fix=lf', str(path))) assert ret == 1 assert path.read_binary() == b'foo\nbar\n' diff --git a/tests/no_commit_to_branch_test.py b/tests/no_commit_to_branch_test.py index a2ab1f1a..610e660e 100644 --- a/tests/no_commit_to_branch_test.py +++ b/tests/no_commit_to_branch_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from pre_commit_hooks.no_commit_to_branch import is_on_branch @@ -70,3 +67,10 @@ def test_not_on_a_branch(temp_git_dir): cmd_output('git', 'checkout', head) # we're not on a branch! assert main(()) == 0 + + +@pytest.mark.parametrize('branch_name', ('master', 'main')) +def test_default_branch_names(temp_git_dir, branch_name): + with temp_git_dir.as_cwd(): + cmd_output('git', 'checkout', '-b', branch_name) + assert main(()) == 1 diff --git a/tests/pretty_format_json_test.py b/tests/pretty_format_json_test.py index 3b7b9a24..7fda23b3 100644 --- a/tests/pretty_format_json_test.py +++ b/tests/pretty_format_json_test.py @@ -1,7 +1,7 @@ +import os import shutil import pytest -from six import PY2 from pre_commit_hooks.pretty_format_json import main from pre_commit_hooks.pretty_format_json import parse_num_to_int @@ -41,7 +41,6 @@ def test_unsorted_main(filename, expected_retval): assert ret == expected_retval -@pytest.mark.skipif(PY2, reason='Requires Python3') @pytest.mark.parametrize( ('filename', 'expected_retval'), ( ('not_pretty_formatted_json.json', 1), @@ -51,7 +50,7 @@ def test_unsorted_main(filename, expected_retval): ('tab_pretty_formatted_json.json', 0), ), ) -def test_tab_main(filename, expected_retval): # pragma: no cover +def test_tab_main(filename, expected_retval): ret = main(['--indent', '\t', get_resource_path(filename)]) assert ret == expected_retval @@ -68,16 +67,16 @@ def test_autofix_main(tmpdir): srcfile = tmpdir.join('to_be_json_formatted.json') shutil.copyfile( get_resource_path('not_pretty_formatted_json.json'), - srcfile.strpath, + str(srcfile), ) # now launch the autofix on that file - ret = main(['--autofix', srcfile.strpath]) + ret = main(['--autofix', str(srcfile)]) # it should have formatted it assert ret == 1 # file was formatted (shouldn't trigger linter again) - ret = main([srcfile.strpath]) + ret = main([str(srcfile)]) assert ret == 0 @@ -105,3 +104,34 @@ def test_top_sorted_get_pretty_format(): def test_badfile_main(): ret = main([get_resource_path('ok_yaml.yaml')]) assert ret == 1 + + +def test_diffing_output(capsys): + resource_path = get_resource_path('not_pretty_formatted_json.json') + expected_retval = 1 + a = os.path.join('a', resource_path) + b = os.path.join('b', resource_path) + expected_out = f'''\ +--- {a} ++++ {b} +@@ -1,6 +1,9 @@ + {{ +- "foo": +- "bar", +- "alist": [2, 34, 234], +- "blah": null ++ "alist": [ ++ 2, ++ 34, ++ 234 ++ ], ++ "blah": null, ++ "foo": "bar" + }} +''' + actual_retval = main([resource_path]) + actual_out, actual_err = capsys.readouterr() + + assert actual_retval == expected_retval + assert actual_out == expected_out + assert actual_err == '' diff --git a/tests/readme_test.py b/tests/readme_test.py index fd6d2654..7df7fcfe 100644 --- a/tests/readme_test.py +++ b/tests/readme_test.py @@ -1,15 +1,10 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import io - from pre_commit_hooks.check_yaml import yaml def test_readme_contains_all_hooks(): - with io.open('README.md', encoding='UTF-8') as f: + with open('README.md', encoding='UTF-8') as f: readme_contents = f.read() - with io.open('.pre-commit-hooks.yaml', encoding='UTF-8') as f: + with open('.pre-commit-hooks.yaml', encoding='UTF-8') as f: hooks = yaml.load(f) for hook in hooks: - assert '`{}`'.format(hook['id']) in readme_contents + assert f'`{hook["id"]}`' in readme_contents diff --git a/tests/removed_test.py b/tests/removed_test.py new file mode 100644 index 00000000..d635eb1e --- /dev/null +++ b/tests/removed_test.py @@ -0,0 +1,17 @@ +import pytest + +from pre_commit_hooks.removed import main + + +def test_always_fails(): + with pytest.raises(SystemExit) as excinfo: + main(( + 'autopep8-wrapper', 'autopep8', + 'https://github.com/pre-commit/mirrors-autopep8', + '--foo', 'bar', + )) + msg, = excinfo.value.args + assert msg == ( + '`autopep8-wrapper` has been removed -- ' + 'use `autopep8` from https://github.com/pre-commit/mirrors-autopep8' + ) diff --git a/tests/requirements_txt_fixer_test.py b/tests/requirements_txt_fixer_test.py index c7c6e47a..e3c6ed50 100644 --- a/tests/requirements_txt_fixer_test.py +++ b/tests/requirements_txt_fixer_test.py @@ -15,6 +15,9 @@ (b'foo\n# comment at end\n', PASS, b'foo\n# comment at end\n'), (b'foo\nbar\n', FAIL, b'bar\nfoo\n'), (b'bar\nfoo\n', PASS, b'bar\nfoo\n'), + (b'a\nc\nb\n', FAIL, b'a\nb\nc\n'), + (b'a\nc\nb', FAIL, b'a\nb\nc\n'), + (b'a\nb\nc', FAIL, b'a\nb\nc\n'), ( b'#comment1\nfoo\n#comment2\nbar\n', FAIL, @@ -27,12 +30,41 @@ ), (b'#comment\n\nfoo\nbar\n', FAIL, b'#comment\n\nbar\nfoo\n'), (b'#comment\n\nbar\nfoo\n', PASS, b'#comment\n\nbar\nfoo\n'), + ( + b'foo\n\t#comment with indent\nbar\n', + FAIL, + b'\t#comment with indent\nbar\nfoo\n', + ), + ( + b'bar\n\t#comment with indent\nfoo\n', + PASS, + b'bar\n\t#comment with indent\nfoo\n', + ), (b'\nfoo\nbar\n', FAIL, b'bar\n\nfoo\n'), (b'\nbar\nfoo\n', PASS, b'\nbar\nfoo\n'), ( - b'pyramid==1\npyramid-foo==2\n', - PASS, - b'pyramid==1\npyramid-foo==2\n', + b'pyramid-foo==1\npyramid>=2\n', + FAIL, + b'pyramid>=2\npyramid-foo==1\n', + ), + ( + b'a==1\n' + b'c>=1\n' + b'bbbb!=1\n' + b'c-a>=1;python_version>="3.6"\n' + b'e>=2\n' + b'd>2\n' + b'g<2\n' + b'f<=2\n', + FAIL, + b'a==1\n' + b'bbbb!=1\n' + b'c>=1\n' + b'c-a>=1;python_version>="3.6"\n' + b'd>2\n' + b'e>=2\n' + b'f<=2\n' + b'g<2\n', ), (b'ocflib\nDjango\nPyMySQL\n', FAIL, b'Django\nocflib\nPyMySQL\n'), ( @@ -42,13 +74,36 @@ ), (b'bar\npkg-resources==0.0.0\nfoo\n', FAIL, b'bar\nfoo\n'), (b'foo\npkg-resources==0.0.0\nbar\n', FAIL, b'bar\nfoo\n'), + ( + b'git+ssh://git_url@tag#egg=ocflib\nDjango\nijk\n', + FAIL, + b'Django\nijk\ngit+ssh://git_url@tag#egg=ocflib\n', + ), + ( + b'b==1.0.0\n' + b'c=2.0.0 \\\n' + b' --hash=sha256:abcd\n' + b'a=3.0.0 \\\n' + b' --hash=sha256:a1b1c1d1', + FAIL, + b'a=3.0.0 \\\n' + b' --hash=sha256:a1b1c1d1\n' + b'b==1.0.0\n' + b'c=2.0.0 \\\n' + b' --hash=sha256:abcd\n', + ), + ( + b'a=2.0.0 \\\n --hash=sha256:abcd\nb==1.0.0\n', + PASS, + b'a=2.0.0 \\\n --hash=sha256:abcd\nb==1.0.0\n', + ), ), ) def test_integration(input_s, expected_retval, output, tmpdir): path = tmpdir.join('file.txt') path.write_binary(input_s) - output_retval = main([path.strpath]) + output_retval = main([str(path)]) assert path.read_binary() == output assert output_retval == expected_retval diff --git a/tests/sort_simple_yaml_test.py b/tests/sort_simple_yaml_test.py index 4261d5d4..a682c158 100644 --- a/tests/sort_simple_yaml_test.py +++ b/tests/sort_simple_yaml_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import os import pytest @@ -42,14 +39,14 @@ @pytest.mark.parametrize('bad_lines,good_lines,retval', TEST_SORTS) def test_integration_good_bad_lines(tmpdir, bad_lines, good_lines, retval): - file_path = os.path.join(tmpdir.strpath, 'foo.yaml') + file_path = os.path.join(str(tmpdir), 'foo.yaml') with open(file_path, 'w') as f: f.write('\n'.join(bad_lines) + '\n') assert main([file_path]) == retval - with open(file_path, 'r') as f: + with open(file_path) as f: assert [line.rstrip() for line in f.readlines()] == good_lines diff --git a/tests/string_fixer_test.py b/tests/string_fixer_test.py index 4adca4a5..6ddb0ac8 100644 --- a/tests/string_fixer_test.py +++ b/tests/string_fixer_test.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import textwrap import pytest @@ -46,7 +42,7 @@ def test_rewrite(input_s, output, expected_retval, tmpdir): path = tmpdir.join('file.py') path.write(input_s) - retval = main([path.strpath]) + retval = main([str(path)]) assert path.read() == output assert retval == expected_retval @@ -54,5 +50,5 @@ def test_rewrite(input_s, output, expected_retval, tmpdir): def test_rewrite_crlf(tmpdir): f = tmpdir.join('f.py') f.write_binary(b'"foo"\r\n"bar"\r\n') - assert main((f.strpath,)) + assert main((str(f),)) assert f.read_binary() == b"'foo'\r\n'bar'\r\n" diff --git a/tests/tests_should_end_in_test_test.py b/tests/tests_should_end_in_test_test.py index 2acfa177..4df2963f 100644 --- a/tests/tests_should_end_in_test_test.py +++ b/tests/tests_should_end_in_test_test.py @@ -11,6 +11,10 @@ def test_main_one_fails(): assert ret == 1 +def test_regex(): + assert main(('foo_test_py',)) == 1 + + def test_main_django_all_pass(): ret = main(( '--django', 'tests.py', 'test_foo.py', 'test_bar.py', diff --git a/tests/trailing_whitespace_fixer_test.py b/tests/trailing_whitespace_fixer_test.py index 82c9b6db..bb3b62d4 100644 --- a/tests/trailing_whitespace_fixer_test.py +++ b/tests/trailing_whitespace_fixer_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from pre_commit_hooks.trailing_whitespace_fixer import main @@ -16,14 +13,14 @@ def test_fixes_trailing_whitespace(input_s, expected, tmpdir): path = tmpdir.join('file.md') path.write(input_s) - assert main((path.strpath,)) == 1 + assert main((str(path),)) == 1 assert path.read() == expected def test_ok_no_newline_end_of_file(tmpdir): filename = tmpdir.join('f') filename.write_binary(b'foo\nbar') - ret = main((filename.strpath,)) + ret = main((str(filename),)) assert filename.read_binary() == b'foo\nbar' assert ret == 0 @@ -31,7 +28,7 @@ def test_ok_no_newline_end_of_file(tmpdir): def test_ok_with_dos_line_endings(tmpdir): filename = tmpdir.join('f') filename.write_binary(b'foo\r\nbar\r\nbaz\r\n') - ret = main((filename.strpath,)) + ret = main((str(filename),)) assert filename.read_binary() == b'foo\r\nbar\r\nbaz\r\n' assert ret == 0 @@ -46,7 +43,7 @@ def test_fixes_markdown_files(tmpdir, ext): '\t\n' # trailing tabs are stripped anyway '\n ', # whitespace at the end of the file is removed ) - ret = main((path.strpath, '--markdown-linebreak-ext={}'.format(ext))) + ret = main((str(path), f'--markdown-linebreak-ext={ext}')) assert ret == 1 assert path.read() == ( 'foo \n' @@ -66,7 +63,7 @@ def test_markdown_linebreak_ext_badopt(arg): def test_prints_warning_with_no_markdown_ext(capsys, tmpdir): f = tmpdir.join('f').ensure() - assert main((f.strpath, '--no-markdown-linebreak-ext')) == 0 + assert main((str(f), '--no-markdown-linebreak-ext')) == 0 out, _ = capsys.readouterr() assert out == '--no-markdown-linebreak-ext now does nothing!\n' @@ -75,6 +72,30 @@ def test_preserve_non_utf8_file(tmpdir): non_utf8_bytes_content = b'\xe9 \n\n' path = tmpdir.join('file.txt') path.write_binary(non_utf8_bytes_content) - ret = main([path.strpath]) + ret = main([str(path)]) assert ret == 1 assert path.size() == (len(non_utf8_bytes_content) - 1) + + +def test_custom_charset_change(tmpdir): + # strip spaces only, no tabs + path = tmpdir.join('file.txt') + path.write('\ta \t \n') + ret = main([str(path), '--chars', ' ']) + assert ret == 1 + assert path.read() == '\ta \t\n' + + +def test_custom_charset_no_change(tmpdir): + path = tmpdir.join('file.txt') + path.write('\ta \t\n') + ret = main([str(path), '--chars', ' ']) + assert ret == 0 + + +def test_markdown_with_custom_charset(tmpdir): + path = tmpdir.join('file.md') + path.write('\ta \t \n') + ret = main([str(path), '--chars', ' ', '--markdown-linebreak-ext', '*']) + assert ret == 1 + assert path.read() == '\ta \t \n' diff --git a/tests/util_test.py b/tests/util_test.py index 9b2d7236..7f488161 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -1,10 +1,8 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from pre_commit_hooks.util import CalledProcessError from pre_commit_hooks.util import cmd_output +from pre_commit_hooks.util import zsplit def test_raises_on_error(): @@ -15,3 +13,13 @@ def test_raises_on_error(): def test_output(): ret = cmd_output('sh', '-c', 'echo hi') assert ret == 'hi\n' + + +@pytest.mark.parametrize('out', ('\0f1\0f2\0', '\0f1\0f2', 'f1\0f2\0')) +def test_check_zsplits_str_correctly(out): + assert zsplit(out) == ['f1', 'f2'] + + +@pytest.mark.parametrize('out', ('\0\0', '\0', '')) +def test_check_zsplit_returns_empty(out): + assert zsplit(out) == [] diff --git a/tox.ini b/tox.ini index a6b3fb4d..965eba92 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py36,py37,pypy,pypy3,pre-commit +envlist = py36,py37,py38,pypy3,pre-commit [testenv] deps = -rrequirements-dev.txt @@ -12,7 +12,6 @@ commands = coverage erase coverage run -m pytest {posargs:tests} coverage report --fail-under 100 - pre-commit install [testenv:pre-commit] skip_install = true