diff --git a/.github/actions/pre-test/action.yml b/.github/actions/pre-test/action.yml index 9d1eb2de6..b70c942fe 100644 --- a/.github/actions/pre-test/action.yml +++ b/.github/actions/pre-test/action.yml @@ -6,4 +6,4 @@ runs: using: composite steps: - uses: asottile/workflows/.github/actions/latest-git@v1.4.0 - if: inputs.env == 'py38' && runner.os == 'Linux' + if: inputs.env == 'py39' && runner.os == 'Linux' diff --git a/.github/workflows/languages.yaml b/.github/workflows/languages.yaml index 5a6ae9cd7..fccf29892 100644 --- a/.github/workflows/languages.yaml +++ b/.github/workflows/languages.yaml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - name: install deps run: python -mpip install -e . -r requirements-dev.txt - name: vars @@ -36,10 +36,10 @@ jobs: matrix: include: ${{ fromJSON(needs.vars.outputs.languages) }} steps: - - uses: asottile/workflows/.github/actions/fast-checkout@v1.4.0 + - uses: asottile/workflows/.github/actions/fast-checkout@v1.8.1 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - run: echo "$CONDA\Scripts" >> "$GITHUB_PATH" shell: bash @@ -65,6 +65,8 @@ jobs: if: matrix.os == 'windows-latest' && matrix.language == 'perl' - uses: haskell/actions/setup@v2 if: matrix.language == 'haskell' + - uses: r-lib/actions/setup-r@v2 + if: matrix.os == 'ubuntu-latest' && matrix.language == 'r' - name: install deps run: python -mpip install -e . -r requirements-dev.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 903d24780..7fda646ff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,12 +12,12 @@ concurrency: jobs: main-windows: - uses: asottile/workflows/.github/workflows/tox.yml@v1.4.0 + uses: asottile/workflows/.github/workflows/tox.yml@v1.8.1 with: - env: '["py38"]' + env: '["py39"]' os: windows-latest main-linux: - uses: asottile/workflows/.github/workflows/tox.yml@v1.4.0 + uses: asottile/workflows/.github/workflows/tox.yml@v1.8.1 with: - env: '["py38", "py39", "py310"]' + env: '["py39", "py310", "py311", "py312"]' os: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5381cd611..b216fbd69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -10,35 +10,35 @@ repos: - id: name-tests-test - id: requirements-txt-fixer - repo: https://github.com/asottile/setup-cfg-fmt - rev: v2.5.0 + rev: v2.7.0 hooks: - id: setup-cfg-fmt - repo: https://github.com/asottile/reorder-python-imports - rev: v3.12.0 + rev: v3.14.0 hooks: - id: reorder-python-imports exclude: ^(pre_commit/resources/|testing/resources/python3_hooks_repo/) - args: [--py38-plus, --add-import, 'from __future__ import annotations'] + args: [--py39-plus, --add-import, 'from __future__ import annotations'] - repo: https://github.com/asottile/add-trailing-comma rev: v3.1.0 hooks: - id: add-trailing-comma - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + rev: v3.19.1 hooks: - id: pyupgrade - args: [--py38-plus] + args: [--py39-plus] - repo: https://github.com/hhatto/autopep8 - rev: v2.0.4 + rev: v2.3.2 hooks: - id: autopep8 - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.1.2 hooks: - id: flake8 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.15.0 hooks: - id: mypy - additional_dependencies: [types-all] + additional_dependencies: [types-pyyaml] exclude: ^testing/resources/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a1b61a49..b63f44317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,139 @@ +4.2.0 - 2025-03-18 +================== + +### Features +- For `language: python` first attempt a versioned python executable for + the default language version before consulting a potentially unversioned + `sys.executable`. + - #3430 PR by @asottile. + +### Fixes +- Handle error during conflict detection when a file is named "HEAD" + - #3425 PR by @tusharsadhwani. + +4.1.0 - 2025-01-20 +================== + +### Features +- Add `language: julia`. + - #3348 PR by @fredrikekre. + - #2689 issue @jmuchovej. + +### Fixes +- Disable automatic toolchain switching for `language: golang`. + - #3304 PR by @AleksaC. + - #3300 issue by @AleksaC. + - #3149 issue by @nijel. +- Fix `language: r` installation when initiated by RStudio. + - #3389 PR by @lorenzwalthert. + - #3385 issue by @lorenzwalthert. + + +4.0.1 - 2024-10-08 +================== + +### Fixes +- Fix `pre-commit migrate-config` for unquoted deprecated stages names with + purelib `pyyaml`. + - #3324 PR by @asottile. + - pre-commit-ci/issues#234 issue by @lorenzwalthert. + +4.0.0 - 2024-10-05 +================== + +### Features +- Improve `pre-commit migrate-config` to handle more yaml formats. + - #3301 PR by @asottile. +- Handle `stages` deprecation in `pre-commit migrate-config`. + - #3302 PR by @asottile. + - #2732 issue by @asottile. +- Upgrade `ruby-build`. + - #3199 PR by @ThisGuyCodes. +- Add "sensible regex" warnings to `repo: meta`. + - #3311 PR by @asottile. +- Add warnings for deprecated `stages` (`commit` -> `pre-commit`, `push` -> + `pre-push`, `merge-commit` -> `pre-merge-commit`). + - #3312 PR by @asottile. + - #3313 PR by @asottile. + - #3315 PR by @asottile. + - #2732 issue by @asottile. + +### Migrating +- `language: python_venv` has been removed -- use `language: python` instead. + - #3320 PR by @asottile. + - #2734 issue by @asottile. + +3.8.0 - 2024-07-28 +================== + +### Features +- Implement health checks for `language: r` so environments are recreated if + the system version of R changes. + - #3206 issue by @lorenzwalthert. + - #3265 PR by @lorenzwalthert. + +3.7.1 - 2024-05-10 +================== + +### Fixes +- Fix `language: rust` default language version check when `rust-toolchain.toml` + is present. + - issue by @gaborbernat. + - #3201 PR by @asottile. + +3.7.0 - 2024-03-24 +================== + +### Features +- Use a tty for `docker` and `docker_image` hooks when `--color` is specified. + - #3122 PR by @glehmann. + +### Fixes +- Fix `fail_fast` for individual hooks stopping when previous hooks had failed. + - #3167 issue by @tp832944. + - #3168 PR by @asottile. + +### Updating +- The per-hook behaviour of `fail_fast` was fixed. If you want the pre-3.7.0 + behaviour, add `fail_fast: true` to all hooks before the last `fail_fast` + hook. + +3.6.2 - 2024-02-18 +================== + +### Fixes +- Fix building golang hooks during `git commit --all`. + - #3130 PR by @asottile. + - #2722 issue by @pestanko and @matthewhughes934. + +3.6.1 - 2024-02-10 +================== + +### Fixes +- Remove `PYTHONEXECUTABLE` from environment when running. + - #3110 PR by @untitaker. +- Handle staged-files-only with only a crlf diff. + - #3126 PR by @asottile. + - issue by @tyyrok. + +3.6.0 - 2023-12-09 +================== + +### Features +- Check `minimum_pre_commit_version` first when parsing configs. + - #3092 PR by @asottile. + +### Fixes +- Fix deprecation warnings for `importlib.resources`. + - #3043 PR by @asottile. +- Fix deprecation warnings for rmtree. + - #3079 PR by @edgarrmondragon. + +### Updating +- Drop support for python<3.9. + - #3042 PR by @asottile. + - #3093 PR by @asottile. + 3.5.0 - 2023-10-13 ================== diff --git a/pre_commit/all_languages.py b/pre_commit/all_languages.py index 476bad9da..ba569c377 100644 --- a/pre_commit/all_languages.py +++ b/pre_commit/all_languages.py @@ -10,6 +10,7 @@ from pre_commit.languages import fail from pre_commit.languages import golang from pre_commit.languages import haskell +from pre_commit.languages import julia from pre_commit.languages import lua from pre_commit.languages import node from pre_commit.languages import perl @@ -33,6 +34,7 @@ 'fail': fail, 'golang': golang, 'haskell': haskell, + 'julia': julia, 'lua': lua, 'node': node, 'perl': perl, @@ -44,7 +46,5 @@ 'script': script, 'swift': swift, 'system': system, - # TODO: fully deprecate `python_venv` - 'python_venv': python, } language_names = sorted(languages) diff --git a/pre_commit/clientlib.py b/pre_commit/clientlib.py index d0651cae2..c0f736d92 100644 --- a/pre_commit/clientlib.py +++ b/pre_commit/clientlib.py @@ -2,12 +2,13 @@ import functools import logging +import os.path import re import shlex import sys +from collections.abc import Sequence from typing import Any from typing import NamedTuple -from typing import Sequence import cfgv from identify.identify import ALL_TAGS @@ -70,6 +71,43 @@ def transform_stage(stage: str) -> str: return _STAGES.get(stage, stage) +MINIMAL_MANIFEST_SCHEMA = cfgv.Array( + cfgv.Map( + 'Hook', 'id', + cfgv.Required('id', cfgv.check_string), + cfgv.Optional('stages', cfgv.check_array(cfgv.check_string), []), + ), +) + + +def warn_for_stages_on_repo_init(repo: str, directory: str) -> None: + try: + manifest = cfgv.load_from_filename( + os.path.join(directory, C.MANIFEST_FILE), + schema=MINIMAL_MANIFEST_SCHEMA, + load_strategy=yaml_load, + exc_tp=InvalidManifestError, + ) + except InvalidManifestError: + return # they'll get a better error message when it actually loads! + + legacy_stages = {} # sorted set + for hook in manifest: + for stage in hook.get('stages', ()): + if stage in _STAGES: + legacy_stages[stage] = True + + if legacy_stages: + logger.warning( + f'repo `{repo}` uses deprecated stage names ' + f'({", ".join(legacy_stages)}) which will be removed in a ' + f'future version. ' + f'Hint: often `pre-commit autoupdate --repo {shlex.quote(repo)}` ' + f'will fix this. ' + f'if it does not -- consider reporting an issue to that repo.', + ) + + class StagesMigrationNoDefault(NamedTuple): key: str default: Sequence[str] @@ -99,9 +137,68 @@ def apply_default(self, dct: dict[str, Any]) -> None: super().apply_default(dct) +class DeprecatedStagesWarning(NamedTuple): + key: str + + def check(self, dct: dict[str, Any]) -> None: + if self.key not in dct: + return + + val = dct[self.key] + cfgv.check_array(cfgv.check_any)(val) + + legacy_stages = [stage for stage in val if stage in _STAGES] + if legacy_stages: + logger.warning( + f'hook id `{dct["id"]}` uses deprecated stage names ' + f'({", ".join(legacy_stages)}) which will be removed in a ' + f'future version. ' + f'run: `pre-commit migrate-config` to automatically fix this.', + ) + + def apply_default(self, dct: dict[str, Any]) -> None: + pass + + def remove_default(self, dct: dict[str, Any]) -> None: + raise NotImplementedError + + +class DeprecatedDefaultStagesWarning(NamedTuple): + key: str + + def check(self, dct: dict[str, Any]) -> None: + if self.key not in dct: + return + + val = dct[self.key] + cfgv.check_array(cfgv.check_any)(val) + + legacy_stages = [stage for stage in val if stage in _STAGES] + if legacy_stages: + logger.warning( + f'top-level `default_stages` uses deprecated stage names ' + f'({", ".join(legacy_stages)}) which will be removed in a ' + f'future version. ' + f'run: `pre-commit migrate-config` to automatically fix this.', + ) + + def apply_default(self, dct: dict[str, Any]) -> None: + pass + + def remove_default(self, dct: dict[str, Any]) -> None: + raise NotImplementedError + + MANIFEST_HOOK_DICT = cfgv.Map( 'Hook', 'id', + # check first in case it uses some newer, incompatible feature + cfgv.Optional( + 'minimum_pre_commit_version', + cfgv.check_and(cfgv.check_string, check_min_version), + '0', + ), + cfgv.Required('id', cfgv.check_string), cfgv.Required('name', cfgv.check_string), cfgv.Required('entry', cfgv.check_string), @@ -124,7 +221,6 @@ def apply_default(self, dct: dict[str, Any]) -> None: cfgv.Optional('description', cfgv.check_string, ''), cfgv.Optional('language_version', cfgv.check_string, C.DEFAULT), cfgv.Optional('log_file', cfgv.check_string, ''), - cfgv.Optional('minimum_pre_commit_version', cfgv.check_string, '0'), cfgv.Optional('require_serial', cfgv.check_bool, False), StagesMigration('stages', []), cfgv.Optional('verbose', cfgv.check_bool, False), @@ -261,6 +357,12 @@ def check(self, dct: dict[str, Any]) -> None: raise cfgv.ValidationError(f'{self.key!r} cannot be overridden') +_COMMON_HOOK_WARNINGS = ( + OptionalSensibleRegexAtHook('files', cfgv.check_string), + OptionalSensibleRegexAtHook('exclude', cfgv.check_string), + DeprecatedStagesWarning('stages'), +) + META_HOOK_DICT = cfgv.Map( 'Hook', 'id', cfgv.Required('id', cfgv.check_string), @@ -283,6 +385,7 @@ def check(self, dct: dict[str, Any]) -> None: item for item in MANIFEST_HOOK_DICT.items ), + *_COMMON_HOOK_WARNINGS, ) CONFIG_HOOK_DICT = cfgv.Map( 'Hook', 'id', @@ -300,16 +403,13 @@ def check(self, dct: dict[str, Any]) -> None: if item.key != 'stages' ), StagesMigrationNoDefault('stages', []), - OptionalSensibleRegexAtHook('files', cfgv.check_string), - OptionalSensibleRegexAtHook('exclude', cfgv.check_string), + *_COMMON_HOOK_WARNINGS, ) LOCAL_HOOK_DICT = cfgv.Map( 'Hook', 'id', *MANIFEST_HOOK_DICT.items, - - OptionalSensibleRegexAtHook('files', cfgv.check_string), - OptionalSensibleRegexAtHook('exclude', cfgv.check_string), + *_COMMON_HOOK_WARNINGS, ) CONFIG_REPO_DICT = cfgv.Map( 'Repository', 'repo', @@ -345,6 +445,13 @@ def check(self, dct: dict[str, Any]) -> None: CONFIG_SCHEMA = cfgv.Map( 'Config', None, + # check first in case it uses some newer, incompatible feature + cfgv.Optional( + 'minimum_pre_commit_version', + cfgv.check_and(cfgv.check_string, check_min_version), + '0', + ), + cfgv.RequiredRecurse('repos', cfgv.Array(CONFIG_REPO_DICT)), cfgv.Optional( 'default_install_hook_types', @@ -355,14 +462,10 @@ def check(self, dct: dict[str, Any]) -> None: 'default_language_version', DEFAULT_LANGUAGE_VERSION, {}, ), StagesMigration('default_stages', STAGES), + DeprecatedDefaultStagesWarning('default_stages'), cfgv.Optional('files', check_string_regex, ''), cfgv.Optional('exclude', check_string_regex, '^$'), cfgv.Optional('fail_fast', cfgv.check_bool, False), - cfgv.Optional( - 'minimum_pre_commit_version', - cfgv.check_and(cfgv.check_string, check_min_version), - '0', - ), cfgv.WarnAdditionalKeys( ( 'repos', diff --git a/pre_commit/commands/autoupdate.py b/pre_commit/commands/autoupdate.py index e7725fdc4..aa0c5e25e 100644 --- a/pre_commit/commands/autoupdate.py +++ b/pre_commit/commands/autoupdate.py @@ -4,9 +4,9 @@ import os.path import re import tempfile +from collections.abc import Sequence from typing import Any from typing import NamedTuple -from typing import Sequence import pre_commit.constants as C from pre_commit import git diff --git a/pre_commit/commands/hook_impl.py b/pre_commit/commands/hook_impl.py index dab2135d4..49a80b7b3 100644 --- a/pre_commit/commands/hook_impl.py +++ b/pre_commit/commands/hook_impl.py @@ -4,7 +4,7 @@ import os.path import subprocess import sys -from typing import Sequence +from collections.abc import Sequence from pre_commit.commands.run import run from pre_commit.envcontext import envcontext diff --git a/pre_commit/commands/migrate_config.py b/pre_commit/commands/migrate_config.py index 842fb3a7b..c5d47a08e 100644 --- a/pre_commit/commands/migrate_config.py +++ b/pre_commit/commands/migrate_config.py @@ -1,13 +1,21 @@ from __future__ import annotations -import re +import functools +import itertools import textwrap +from typing import Callable import cfgv import yaml +from yaml.nodes import ScalarNode from pre_commit.clientlib import InvalidConfigError +from pre_commit.yaml import yaml_compose from pre_commit.yaml import yaml_load +from pre_commit.yaml_rewrite import MappingKey +from pre_commit.yaml_rewrite import MappingValue +from pre_commit.yaml_rewrite import match +from pre_commit.yaml_rewrite import SequenceItem def _is_header_line(line: str) -> bool: @@ -38,16 +46,69 @@ def _migrate_map(contents: str) -> str: return contents -def _migrate_sha_to_rev(contents: str) -> str: - return re.sub(r'(\n\s+)sha:', r'\1rev:', contents) +def _preserve_style(n: ScalarNode, *, s: str) -> str: + style = n.style or '' + return f'{style}{s}{style}' -def _migrate_python_venv(contents: str) -> str: - return re.sub( - r'(\n\s+)language: python_venv\b', - r'\1language: python', - contents, +def _fix_stage(n: ScalarNode) -> str: + return _preserve_style(n, s=f'pre-{n.value}') + + +def _migrate_composed(contents: str) -> str: + tree = yaml_compose(contents) + rewrites: list[tuple[ScalarNode, Callable[[ScalarNode], str]]] = [] + + # sha -> rev + sha_to_rev_replace = functools.partial(_preserve_style, s='rev') + sha_to_rev_matcher = ( + MappingValue('repos'), + SequenceItem(), + MappingKey('sha'), + ) + for node in match(tree, sha_to_rev_matcher): + rewrites.append((node, sha_to_rev_replace)) + + # python_venv -> python + language_matcher = ( + MappingValue('repos'), + SequenceItem(), + MappingValue('hooks'), + SequenceItem(), + MappingValue('language'), ) + python_venv_replace = functools.partial(_preserve_style, s='python') + for node in match(tree, language_matcher): + if node.value == 'python_venv': + rewrites.append((node, python_venv_replace)) + + # stages rewrites + default_stages_matcher = (MappingValue('default_stages'), SequenceItem()) + default_stages_match = match(tree, default_stages_matcher) + hook_stages_matcher = ( + MappingValue('repos'), + SequenceItem(), + MappingValue('hooks'), + SequenceItem(), + MappingValue('stages'), + SequenceItem(), + ) + hook_stages_match = match(tree, hook_stages_matcher) + for node in itertools.chain(default_stages_match, hook_stages_match): + if node.value in {'commit', 'push', 'merge-commit'}: + rewrites.append((node, _fix_stage)) + + rewrites.sort(reverse=True, key=lambda nf: nf[0].start_mark.index) + + src_parts = [] + end: int | None = None + for node, func in rewrites: + src_parts.append(contents[node.end_mark.index:end]) + src_parts.append(func(node)) + end = node.start_mark.index + src_parts.append(contents[:end]) + src_parts.reverse() + return ''.join(src_parts) def migrate_config(config_file: str, quiet: bool = False) -> int: @@ -62,8 +123,7 @@ def migrate_config(config_file: str, quiet: bool = False) -> int: raise cfgv.ValidationError(str(e)) contents = _migrate_map(contents) - contents = _migrate_sha_to_rev(contents) - contents = _migrate_python_venv(contents) + contents = _migrate_composed(contents) if contents != orig_contents: with open(config_file, 'w') as f: diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py index 41ba4ecf0..793adbdb2 100644 --- a/pre_commit/commands/run.py +++ b/pre_commit/commands/run.py @@ -9,11 +9,11 @@ import subprocess import time import unicodedata +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import MutableMapping +from collections.abc import Sequence from typing import Any -from typing import Generator -from typing import Iterable -from typing import MutableMapping -from typing import Sequence from identify.identify import tags_from_path @@ -61,7 +61,7 @@ def filter_by_include_exclude( names: Iterable[str], include: str, exclude: str, -) -> Generator[str, None, None]: +) -> Generator[str]: include_re, exclude_re = re.compile(include), re.compile(exclude) return ( filename for filename in names @@ -74,7 +74,7 @@ class Classifier: def __init__(self, filenames: Iterable[str]) -> None: self.filenames = [f for f in filenames if os.path.lexists(f)] - @functools.lru_cache(maxsize=None) + @functools.cache def _types_for_file(self, filename: str) -> set[str]: return tags_from_path(filename) @@ -84,7 +84,7 @@ def by_types( types: Iterable[str], types_or: Iterable[str], exclude_types: Iterable[str], - ) -> Generator[str, None, None]: + ) -> Generator[str]: types = frozenset(types) types_or = frozenset(types_or) exclude_types = frozenset(exclude_types) @@ -97,7 +97,7 @@ def by_types( ): yield filename - def filenames_for_hook(self, hook: Hook) -> Generator[str, None, None]: + def filenames_for_hook(self, hook: Hook) -> Generator[str]: return self.by_types( filter_by_include_exclude( self.filenames, @@ -298,7 +298,7 @@ def _run_hooks( verbose=args.verbose, use_color=args.color, ) retval |= current_retval - if retval and (config['fail_fast'] or hook.fail_fast): + if current_retval and (config['fail_fast'] or hook.fail_fast): break if retval and args.show_diff_on_failure and prior_diff: if args.all_files: diff --git a/pre_commit/commands/validate_config.py b/pre_commit/commands/validate_config.py index 24bd3135e..b3de635b1 100644 --- a/pre_commit/commands/validate_config.py +++ b/pre_commit/commands/validate_config.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from pre_commit import clientlib diff --git a/pre_commit/commands/validate_manifest.py b/pre_commit/commands/validate_manifest.py index 419031a9b..8493c6e1e 100644 --- a/pre_commit/commands/validate_manifest.py +++ b/pre_commit/commands/validate_manifest.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from pre_commit import clientlib diff --git a/pre_commit/envcontext.py b/pre_commit/envcontext.py index 4f5956016..d4d241184 100644 --- a/pre_commit/envcontext.py +++ b/pre_commit/envcontext.py @@ -3,10 +3,9 @@ import contextlib import enum import os -from typing import Generator -from typing import MutableMapping +from collections.abc import Generator +from collections.abc import MutableMapping from typing import NamedTuple -from typing import Tuple from typing import Union _Unset = enum.Enum('_Unset', 'UNSET') @@ -18,9 +17,9 @@ class Var(NamedTuple): default: str = '' -SubstitutionT = Tuple[Union[str, Var], ...] +SubstitutionT = tuple[Union[str, Var], ...] ValueT = Union[str, _Unset, SubstitutionT] -PatchesT = Tuple[Tuple[str, ValueT], ...] +PatchesT = tuple[tuple[str, ValueT], ...] def format_env(parts: SubstitutionT, env: MutableMapping[str, str]) -> str: @@ -34,7 +33,7 @@ def format_env(parts: SubstitutionT, env: MutableMapping[str, str]) -> str: def envcontext( patch: PatchesT, _env: MutableMapping[str, str] | None = None, -) -> Generator[None, None, None]: +) -> Generator[None]: """In this context, `os.environ` is modified according to `patch`. `patch` is an iterable of 2-tuples (key, value): diff --git a/pre_commit/error_handler.py b/pre_commit/error_handler.py index d740ee3e4..4f0e05733 100644 --- a/pre_commit/error_handler.py +++ b/pre_commit/error_handler.py @@ -5,7 +5,7 @@ import os.path import sys import traceback -from typing import Generator +from collections.abc import Generator from typing import IO import pre_commit.constants as C @@ -68,7 +68,7 @@ def _log_and_exit( @contextlib.contextmanager -def error_handler() -> Generator[None, None, None]: +def error_handler() -> Generator[None]: try: yield except (Exception, KeyboardInterrupt) as e: diff --git a/pre_commit/file_lock.py b/pre_commit/file_lock.py index f67a58644..c840ad8b5 100644 --- a/pre_commit/file_lock.py +++ b/pre_commit/file_lock.py @@ -3,8 +3,8 @@ import contextlib import errno import sys +from collections.abc import Generator from typing import Callable -from typing import Generator if sys.platform == 'win32': # pragma: no cover (windows) @@ -20,7 +20,7 @@ def _locked( fileno: int, blocked_cb: Callable[[], None], - ) -> Generator[None, None, None]: + ) -> Generator[None]: try: msvcrt.locking(fileno, msvcrt.LK_NBLCK, _region) except OSError: @@ -53,7 +53,7 @@ def _locked( def _locked( fileno: int, blocked_cb: Callable[[], None], - ) -> Generator[None, None, None]: + ) -> Generator[None]: try: fcntl.flock(fileno, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: # pragma: no cover (tests are single-threaded) @@ -69,7 +69,7 @@ def _locked( def lock( path: str, blocked_cb: Callable[[], None], -) -> Generator[None, None, None]: +) -> Generator[None]: with open(path, 'a+') as f: with _locked(f.fileno(), blocked_cb): yield diff --git a/pre_commit/git.py b/pre_commit/git.py index 333dc7ba3..2f424f89e 100644 --- a/pre_commit/git.py +++ b/pre_commit/git.py @@ -3,7 +3,7 @@ import logging import os.path import sys -from typing import Mapping +from collections.abc import Mapping from pre_commit.errors import FatalError from pre_commit.util import CalledProcessError @@ -126,7 +126,7 @@ def get_conflicted_files() -> set[str]: merge_diff_filenames = zsplit( cmd_output( 'git', 'diff', '--name-only', '--no-ext-diff', '-z', - '-m', tree_hash, 'HEAD', 'MERGE_HEAD', + '-m', tree_hash, 'HEAD', 'MERGE_HEAD', '--', )[1], ) return set(merge_conflict_filenames) | set(merge_diff_filenames) diff --git a/pre_commit/hook.py b/pre_commit/hook.py index 6d436ca30..309cd5be3 100644 --- a/pre_commit/hook.py +++ b/pre_commit/hook.py @@ -1,9 +1,9 @@ from __future__ import annotations import logging +from collections.abc import Sequence from typing import Any from typing import NamedTuple -from typing import Sequence from pre_commit.prefix import Prefix diff --git a/pre_commit/lang_base.py b/pre_commit/lang_base.py index 4a993eaa4..95be7b9b3 100644 --- a/pre_commit/lang_base.py +++ b/pre_commit/lang_base.py @@ -5,12 +5,12 @@ import random import re import shlex +from collections.abc import Generator +from collections.abc import Sequence from typing import Any from typing import ContextManager -from typing import Generator from typing import NoReturn from typing import Protocol -from typing import Sequence import pre_commit.constants as C from pre_commit import parse_shebang @@ -127,7 +127,7 @@ def no_install( @contextlib.contextmanager -def no_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def no_env(prefix: Prefix, version: str) -> Generator[None]: yield diff --git a/pre_commit/languages/conda.py b/pre_commit/languages/conda.py index 41c355e77..d397ebeb7 100644 --- a/pre_commit/languages/conda.py +++ b/pre_commit/languages/conda.py @@ -3,8 +3,8 @@ import contextlib import os import sys -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -41,7 +41,7 @@ def get_env_patch(env: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/coursier.py b/pre_commit/languages/coursier.py index 9c5fbfe24..08f9a958f 100644 --- a/pre_commit/languages/coursier.py +++ b/pre_commit/languages/coursier.py @@ -2,8 +2,8 @@ import contextlib import os.path -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -70,7 +70,7 @@ def get_env_patch(target_dir: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/dart.py b/pre_commit/languages/dart.py index e8539caa2..52a229eef 100644 --- a/pre_commit/languages/dart.py +++ b/pre_commit/languages/dart.py @@ -4,8 +4,8 @@ import os.path import shutil import tempfile -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -29,7 +29,7 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/docker.py b/pre_commit/languages/docker.py index 8e53ca9e3..4de1d5824 100644 --- a/pre_commit/languages/docker.py +++ b/pre_commit/languages/docker.py @@ -3,7 +3,7 @@ import hashlib import json import os -from typing import Sequence +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.prefix import Prefix @@ -108,10 +108,15 @@ def get_docker_user() -> tuple[str, ...]: # pragma: win32 no cover return () -def docker_cmd() -> tuple[str, ...]: # pragma: win32 no cover +def get_docker_tty(*, color: bool) -> tuple[str, ...]: # pragma: win32 no cover # noqa: E501 + return (('--tty',) if color else ()) + + +def docker_cmd(*, color: bool) -> tuple[str, ...]: # pragma: win32 no cover return ( 'docker', 'run', '--rm', + *get_docker_tty(color=color), *get_docker_user(), # https://docs.docker.com/engine/reference/commandline/run/#mount-volumes-from-container-volumes-from # The `Z` option tells Docker to label the content with a private @@ -139,7 +144,7 @@ def run_hook( entry_tag = ('--entrypoint', entry_exe, docker_tag(prefix)) return lang_base.run_xargs( - (*docker_cmd(), *entry_tag, *cmd_rest), + (*docker_cmd(color=color), *entry_tag, *cmd_rest), file_args, require_serial=require_serial, color=color, diff --git a/pre_commit/languages/docker_image.py b/pre_commit/languages/docker_image.py index 26f006e4a..60caa101d 100644 --- a/pre_commit/languages/docker_image.py +++ b/pre_commit/languages/docker_image.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.languages.docker import docker_cmd @@ -23,7 +23,7 @@ def run_hook( require_serial: bool, color: bool, ) -> tuple[int, bytes]: # pragma: win32 no cover - cmd = docker_cmd() + lang_base.hook_cmd(entry, args) + cmd = docker_cmd(color=color) + lang_base.hook_cmd(entry, args) return lang_base.run_xargs( cmd, file_args, diff --git a/pre_commit/languages/dotnet.py b/pre_commit/languages/dotnet.py index e9568f222..ffc65d1e8 100644 --- a/pre_commit/languages/dotnet.py +++ b/pre_commit/languages/dotnet.py @@ -6,8 +6,8 @@ import tempfile import xml.etree.ElementTree import zipfile -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -30,14 +30,14 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield @contextlib.contextmanager -def _nuget_config_no_sources() -> Generator[str, None, None]: +def _nuget_config_no_sources() -> Generator[str]: with tempfile.TemporaryDirectory() as tmpdir: nuget_config = os.path.join(tmpdir, 'nuget.config') with open(nuget_config, 'w') as f: diff --git a/pre_commit/languages/fail.py b/pre_commit/languages/fail.py index a8ec6a53d..6ac4d7675 100644 --- a/pre_commit/languages/fail.py +++ b/pre_commit/languages/fail.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.prefix import Prefix diff --git a/pre_commit/languages/golang.py b/pre_commit/languages/golang.py index bea91e9bd..678c04b14 100644 --- a/pre_commit/languages/golang.py +++ b/pre_commit/languages/golang.py @@ -12,17 +12,18 @@ import urllib.error import urllib.request import zipfile +from collections.abc import Generator +from collections.abc import Sequence from typing import ContextManager -from typing import Generator from typing import IO from typing import Protocol -from typing import Sequence import pre_commit.constants as C from pre_commit import lang_base from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var +from pre_commit.git import no_git_env from pre_commit.prefix import Prefix from pre_commit.util import cmd_output from pre_commit.util import rmtree @@ -74,6 +75,7 @@ def get_env_patch(venv: str, version: str) -> PatchesT: return ( ('GOROOT', os.path.join(venv, '.go')), + ('GOTOOLCHAIN', 'local'), ( 'PATH', ( os.path.join(venv, 'bin'), os.pathsep, @@ -120,7 +122,7 @@ def _install_go(version: str, dest: str) -> None: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir, version)): yield @@ -141,9 +143,10 @@ def install_environment( else: gopath = env_dir - env = dict(os.environ, GOPATH=gopath) + env = no_git_env(dict(os.environ, GOPATH=gopath)) env.pop('GOBIN', None) if version != 'system': + env['GOTOOLCHAIN'] = 'local' env['GOROOT'] = os.path.join(env_dir, '.go') env['PATH'] = os.pathsep.join(( os.path.join(env_dir, '.go', 'bin'), os.environ['PATH'], diff --git a/pre_commit/languages/haskell.py b/pre_commit/languages/haskell.py index 76442eb02..28bca08cc 100644 --- a/pre_commit/languages/haskell.py +++ b/pre_commit/languages/haskell.py @@ -2,8 +2,8 @@ import contextlib import os.path -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -24,7 +24,7 @@ def get_env_patch(target_dir: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/julia.py b/pre_commit/languages/julia.py new file mode 100644 index 000000000..df91c0697 --- /dev/null +++ b/pre_commit/languages/julia.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import contextlib +import os +import shutil +from collections.abc import Generator +from collections.abc import Sequence + +from pre_commit import lang_base +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import UNSET +from pre_commit.prefix import Prefix +from pre_commit.util import cmd_output_b + +ENVIRONMENT_DIR = 'juliaenv' +health_check = lang_base.basic_health_check +get_default_version = lang_base.basic_get_default_version + + +def run_hook( + prefix: Prefix, + entry: str, + args: Sequence[str], + file_args: Sequence[str], + *, + is_local: bool, + require_serial: bool, + color: bool, +) -> tuple[int, bytes]: + # `entry` is a (hook-repo relative) file followed by (optional) args, e.g. + # `bin/id.jl` or `bin/hook.jl --arg1 --arg2` so we + # 1) shell parse it and join with args with hook_cmd + # 2) prepend the hooks prefix path to the first argument (the file), unless + # it is a local script + # 3) prepend `julia` as the interpreter + + cmd = lang_base.hook_cmd(entry, args) + script = cmd[0] if is_local else prefix.path(cmd[0]) + cmd = ('julia', script, *cmd[1:]) + return lang_base.run_xargs( + cmd, + file_args, + require_serial=require_serial, + color=color, + ) + + +def get_env_patch(target_dir: str, version: str) -> PatchesT: + return ( + ('JULIA_LOAD_PATH', target_dir), + # May be set, remove it to not interfer with LOAD_PATH + ('JULIA_PROJECT', UNSET), + ) + + +@contextlib.contextmanager +def in_env(prefix: Prefix, version: str) -> Generator[None]: + envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) + with envcontext(get_env_patch(envdir, version)): + yield + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) + with in_env(prefix, version): + # TODO: Support language_version with juliaup similar to rust via + # rustup + # if version != 'system': + # ... + + # Copy Project.toml to hook env if it exist + os.makedirs(envdir, exist_ok=True) + project_names = ('JuliaProject.toml', 'Project.toml') + project_found = False + for project_name in project_names: + project_file = prefix.path(project_name) + if not os.path.isfile(project_file): + continue + shutil.copy(project_file, envdir) + project_found = True + break + + # If no project file was found we create an empty one so that the + # package manager doesn't error + if not project_found: + open(os.path.join(envdir, 'Project.toml'), 'a').close() + + # Copy Manifest.toml to hook env if it exists + manifest_names = ('JuliaManifest.toml', 'Manifest.toml') + for manifest_name in manifest_names: + manifest_file = prefix.path(manifest_name) + if not os.path.isfile(manifest_file): + continue + shutil.copy(manifest_file, envdir) + break + + # Julia code to instantiate the hook environment + julia_code = """ + @assert length(ARGS) > 0 + hook_env = ARGS[1] + deps = join(ARGS[2:end], " ") + + # We prepend @stdlib here so that we can load the package manager even + # though `get_env_patch` limits `JULIA_LOAD_PATH` to just the hook env. + pushfirst!(LOAD_PATH, "@stdlib") + using Pkg + popfirst!(LOAD_PATH) + + # Instantiate the environment shipped with the hook repo. If we have + # additional dependencies we disable precompilation in this step to + # avoid double work. + precompile = isempty(deps) ? "1" : "0" + withenv("JULIA_PKG_PRECOMPILE_AUTO" => precompile) do + Pkg.instantiate() + end + + # Add additional dependencies (with precompilation) + if !isempty(deps) + withenv("JULIA_PKG_PRECOMPILE_AUTO" => "1") do + Pkg.REPLMode.pkgstr("add " * deps) + end + end + """ + cmd_output_b( + 'julia', '-e', julia_code, '--', envdir, *additional_dependencies, + cwd=prefix.prefix_dir, + ) diff --git a/pre_commit/languages/lua.py b/pre_commit/languages/lua.py index 12d066140..15ac1a2ec 100644 --- a/pre_commit/languages/lua.py +++ b/pre_commit/languages/lua.py @@ -3,8 +3,8 @@ import contextlib import os import sys -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -44,7 +44,7 @@ def get_env_patch(d: str) -> PatchesT: # pragma: win32 no cover @contextlib.contextmanager # pragma: win32 no cover -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py index 3e22dc78e..af7dc6f87 100644 --- a/pre_commit/languages/node.py +++ b/pre_commit/languages/node.py @@ -4,8 +4,8 @@ import functools import os import sys -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence import pre_commit.constants as C from pre_commit import lang_base @@ -59,7 +59,7 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/perl.py b/pre_commit/languages/perl.py index 2a7f16290..a07d442ac 100644 --- a/pre_commit/languages/perl.py +++ b/pre_commit/languages/perl.py @@ -3,8 +3,8 @@ import contextlib import os import shlex -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -33,7 +33,7 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/pygrep.py b/pre_commit/languages/pygrep.py index ec55560b0..72a9345fa 100644 --- a/pre_commit/languages/pygrep.py +++ b/pre_commit/languages/pygrep.py @@ -3,9 +3,9 @@ import argparse import re import sys +from collections.abc import Sequence +from re import Pattern from typing import NamedTuple -from typing import Pattern -from typing import Sequence from pre_commit import lang_base from pre_commit import output diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 3ef343608..88ececce6 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -4,8 +4,8 @@ import functools import os import sys -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence import pre_commit.constants as C from pre_commit import lang_base @@ -24,7 +24,7 @@ run_hook = lang_base.basic_run_hook -@functools.lru_cache(maxsize=None) +@functools.cache def _version_info(exe: str) -> str: prog = 'import sys;print(".".join(str(p) for p in sys.version_info))' try: @@ -65,7 +65,7 @@ def _find_by_py_launcher( version: str, ) -> str | None: # pragma: no cover (windows only) if version.startswith('python'): - num = version[len('python'):] + num = version.removeprefix('python') cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)') env = dict(os.environ, PYTHONIOENCODING='UTF-8') try: @@ -75,6 +75,13 @@ def _find_by_py_launcher( return None +def _impl_exe_name() -> str: + if sys.implementation.name == 'cpython': # pragma: cpython cover + return 'python' + else: # pragma: cpython no cover + return sys.implementation.name # pypy mostly + + def _find_by_sys_executable() -> str | None: def _norm(path: str) -> str | None: _, exe = os.path.split(path.lower()) @@ -100,18 +107,25 @@ def _norm(path: str) -> str | None: @functools.lru_cache(maxsize=1) def get_default_version() -> str: # pragma: no cover (platform dependent) - # First attempt from `sys.executable` (or the realpath) - exe = _find_by_sys_executable() - if exe: - return exe + v_major = f'{sys.version_info[0]}' + v_minor = f'{sys.version_info[0]}.{sys.version_info[1]}' - # Next try the `pythonX.X` executable - exe = f'python{sys.version_info[0]}.{sys.version_info[1]}' - if find_executable(exe): - return exe + # attempt the likely implementation exe + for potential in (v_minor, v_major): + exe = f'{_impl_exe_name()}{potential}' + if find_executable(exe): + return exe - if _find_by_py_launcher(exe): - return exe + # next try `sys.executable` (or the realpath) + maybe_exe = _find_by_sys_executable() + if maybe_exe: + return maybe_exe + + # maybe on windows we can find it via py launcher? + if sys.platform == 'win32': # pragma: win32 cover + exe = f'python{v_minor}' + if _find_by_py_launcher(exe): + return exe # We tried! return C.DEFAULT @@ -124,7 +138,7 @@ def _sys_executable_matches(version: str) -> bool: return False try: - info = tuple(int(p) for p in version[len('python'):].split('.')) + info = tuple(int(p) for p in version.removeprefix('python').split('.')) except ValueError: return False @@ -152,7 +166,7 @@ def norm_version(version: str) -> str | None: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/languages/r.py b/pre_commit/languages/r.py index 6feb06523..f70d2fdca 100644 --- a/pre_commit/languages/r.py +++ b/pre_commit/languages/r.py @@ -6,25 +6,109 @@ import shutil import tempfile import textwrap -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET from pre_commit.prefix import Prefix -from pre_commit.util import cmd_output_b +from pre_commit.util import cmd_output from pre_commit.util import win_exe ENVIRONMENT_DIR = 'renv' -RSCRIPT_OPTS = ('--no-save', '--no-restore', '--no-site-file', '--no-environ') get_default_version = lang_base.basic_get_default_version -health_check = lang_base.basic_health_check + +_RENV_ACTIVATED_OPTS = ( + '--no-save', '--no-restore', '--no-site-file', '--no-environ', +) + + +def _execute_r( + code: str, *, + prefix: Prefix, version: str, args: Sequence[str] = (), cwd: str, + cli_opts: Sequence[str], +) -> str: + with in_env(prefix, version), _r_code_in_tempfile(code) as f: + _, out, _ = cmd_output( + _rscript_exec(), *cli_opts, f, *args, cwd=cwd, + ) + return out.rstrip('\n') + + +def _execute_r_in_renv( + code: str, *, + prefix: Prefix, version: str, args: Sequence[str] = (), cwd: str, +) -> str: + return _execute_r( + code=code, prefix=prefix, version=version, args=args, cwd=cwd, + cli_opts=_RENV_ACTIVATED_OPTS, + ) + + +def _execute_vanilla_r( + code: str, *, + prefix: Prefix, version: str, args: Sequence[str] = (), cwd: str, +) -> str: + return _execute_r( + code=code, prefix=prefix, version=version, args=args, cwd=cwd, + cli_opts=('--vanilla',), + ) + + +def _read_installed_version(envdir: str, prefix: Prefix, version: str) -> str: + return _execute_r_in_renv( + 'cat(renv::settings$r.version())', + prefix=prefix, version=version, + cwd=envdir, + ) + + +def _read_executable_version(envdir: str, prefix: Prefix, version: str) -> str: + return _execute_r_in_renv( + 'cat(as.character(getRversion()))', + prefix=prefix, version=version, + cwd=envdir, + ) + + +def _write_current_r_version( + envdir: str, prefix: Prefix, version: str, +) -> None: + _execute_r_in_renv( + 'renv::settings$r.version(as.character(getRversion()))', + prefix=prefix, version=version, + cwd=envdir, + ) + + +def health_check(prefix: Prefix, version: str) -> str | None: + envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) + + r_version_installation = _read_installed_version( + envdir=envdir, prefix=prefix, version=version, + ) + r_version_current_executable = _read_executable_version( + envdir=envdir, prefix=prefix, version=version, + ) + if r_version_installation in {'NULL', ''}: + return ( + f'Hooks were installed with an unknown R version. R version for ' + f'hook repo now set to {r_version_current_executable}' + ) + elif r_version_installation != r_version_current_executable: + return ( + f'Hooks were installed for R version {r_version_installation}, ' + f'but current R executable has version ' + f'{r_version_current_executable}' + ) + + return None @contextlib.contextmanager -def _r_code_in_tempfile(code: str) -> Generator[str, None, None]: +def _r_code_in_tempfile(code: str) -> Generator[str]: """ To avoid quoting and escaping issues, avoid `Rscript [options] -e {expr}` but use `Rscript [options] path/to/file_with_expr.R` @@ -44,7 +128,7 @@ def get_env_patch(venv: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield @@ -100,7 +184,7 @@ def _cmd_from_hook( _entry_validate(cmd) cmd_part = _prefix_if_file_entry(cmd, prefix, is_local=is_local) - return (cmd[0], *RSCRIPT_OPTS, *cmd_part, *args) + return (cmd[0], *_RENV_ACTIVATED_OPTS, *cmd_part, *args) def install_environment( @@ -143,20 +227,19 @@ def install_environment( renv::install(prefix_dir) }} """ + _execute_vanilla_r( + r_code_inst_environment, + prefix=prefix, version=version, cwd=env_dir, + ) - with _r_code_in_tempfile(r_code_inst_environment) as f: - cmd_output_b(_rscript_exec(), '--vanilla', f, cwd=env_dir) - + _write_current_r_version(envdir=env_dir, prefix=prefix, version=version) if additional_dependencies: r_code_inst_add = 'renv::install(commandArgs(trailingOnly = TRUE))' - with in_env(prefix, version): - with _r_code_in_tempfile(r_code_inst_add) as f: - cmd_output_b( - _rscript_exec(), *RSCRIPT_OPTS, - f, - *additional_dependencies, - cwd=env_dir, - ) + _execute_r_in_renv( + code=r_code_inst_add, prefix=prefix, version=version, + args=additional_dependencies, + cwd=env_dir, + ) def _inline_r_setup(code: str) -> str: diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py index c88269f24..f32fea3fa 100644 --- a/pre_commit/languages/ruby.py +++ b/pre_commit/languages/ruby.py @@ -6,9 +6,9 @@ import os.path import shutil import tarfile -from typing import Generator +from collections.abc import Generator +from collections.abc import Sequence from typing import IO -from typing import Sequence import pre_commit.constants as C from pre_commit import lang_base @@ -25,7 +25,8 @@ def _resource_bytesio(filename: str) -> IO[bytes]: - return importlib.resources.open_binary('pre_commit.resources', filename) + files = importlib.resources.files('pre_commit.resources') + return files.joinpath(filename).open('rb') @functools.lru_cache(maxsize=1) @@ -72,7 +73,7 @@ def get_env_patch( @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir, version)): yield diff --git a/pre_commit/languages/rust.py b/pre_commit/languages/rust.py index 7eec0e7d6..fd77a9d29 100644 --- a/pre_commit/languages/rust.py +++ b/pre_commit/languages/rust.py @@ -7,8 +7,8 @@ import sys import tempfile import urllib.request -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence import pre_commit.constants as C from pre_commit import lang_base @@ -34,7 +34,7 @@ def get_default_version() -> str: # Just detecting the executable does not suffice, because if rustup is # installed but no toolchain is available, then `cargo` exists but # cannot be used without installing a toolchain first. - if cmd_output_b('cargo', '--version', check=False)[0] == 0: + if cmd_output_b('cargo', '--version', check=False, cwd='/')[0] == 0: return 'system' else: return C.DEFAULT @@ -61,7 +61,7 @@ def get_env_patch(target_dir: str, version: str) -> PatchesT: @contextlib.contextmanager -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir, version)): yield @@ -134,7 +134,7 @@ def install_environment( packages_to_install: set[tuple[str, ...]] = {('--path', '.')} for cli_dep in cli_deps: - cli_dep = cli_dep[len('cli:'):] + cli_dep = cli_dep.removeprefix('cli:') package, _, crate_version = cli_dep.partition(':') if crate_version != '': packages_to_install.add((package, '--version', crate_version)) diff --git a/pre_commit/languages/script.py b/pre_commit/languages/script.py index 89a3ab2d6..1eaa1e270 100644 --- a/pre_commit/languages/script.py +++ b/pre_commit/languages/script.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.prefix import Prefix diff --git a/pre_commit/languages/swift.py b/pre_commit/languages/swift.py index f16bb0451..08a9c39a8 100644 --- a/pre_commit/languages/swift.py +++ b/pre_commit/languages/swift.py @@ -2,8 +2,8 @@ import contextlib import os -from typing import Generator -from typing import Sequence +from collections.abc import Generator +from collections.abc import Sequence from pre_commit import lang_base from pre_commit.envcontext import envcontext @@ -27,7 +27,7 @@ def get_env_patch(venv: str) -> PatchesT: # pragma: win32 no cover @contextlib.contextmanager # pragma: win32 no cover -def in_env(prefix: Prefix, version: str) -> Generator[None, None, None]: +def in_env(prefix: Prefix, version: str) -> Generator[None]: envdir = lang_base.environment_dir(prefix, ENVIRONMENT_DIR, version) with envcontext(get_env_patch(envdir)): yield diff --git a/pre_commit/logging_handler.py b/pre_commit/logging_handler.py index 1b68fc7d6..74772beee 100644 --- a/pre_commit/logging_handler.py +++ b/pre_commit/logging_handler.py @@ -2,7 +2,7 @@ import contextlib import logging -from typing import Generator +from collections.abc import Generator from pre_commit import color from pre_commit import output @@ -32,7 +32,7 @@ def emit(self, record: logging.LogRecord) -> None: @contextlib.contextmanager -def logging_handler(use_color: bool) -> Generator[None, None, None]: +def logging_handler(use_color: bool) -> Generator[None]: handler = LoggingHandler(use_color) logger.addHandler(handler) logger.setLevel(logging.INFO) diff --git a/pre_commit/main.py b/pre_commit/main.py index 9dfce2c25..559c927c9 100644 --- a/pre_commit/main.py +++ b/pre_commit/main.py @@ -4,7 +4,7 @@ import logging import os import sys -from typing import Sequence +from collections.abc import Sequence import pre_commit.constants as C from pre_commit import clientlib @@ -37,6 +37,9 @@ # pyvenv os.environ.pop('__PYVENV_LAUNCHER__', None) +# https://github.com/getsentry/snuba/pull/5388 +os.environ.pop('PYTHONEXECUTABLE', None) + COMMANDS_NO_GIT = { 'clean', 'gc', 'init-templatedir', 'sample-config', 'validate-config', 'validate-manifest', diff --git a/pre_commit/meta_hooks/check_hooks_apply.py b/pre_commit/meta_hooks/check_hooks_apply.py index 7f491a209..84c142b45 100644 --- a/pre_commit/meta_hooks/check_hooks_apply.py +++ b/pre_commit/meta_hooks/check_hooks_apply.py @@ -1,7 +1,7 @@ from __future__ import annotations import argparse -from typing import Sequence +from collections.abc import Sequence import pre_commit.constants as C from pre_commit import git diff --git a/pre_commit/meta_hooks/check_useless_excludes.py b/pre_commit/meta_hooks/check_useless_excludes.py index 8b0c106a3..664251a44 100644 --- a/pre_commit/meta_hooks/check_useless_excludes.py +++ b/pre_commit/meta_hooks/check_useless_excludes.py @@ -2,8 +2,8 @@ import argparse import re -from typing import Iterable -from typing import Sequence +from collections.abc import Iterable +from collections.abc import Sequence from cfgv import apply_defaults diff --git a/pre_commit/meta_hooks/identity.py b/pre_commit/meta_hooks/identity.py index 72ee440bc..3e20bbc68 100644 --- a/pre_commit/meta_hooks/identity.py +++ b/pre_commit/meta_hooks/identity.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from typing import Sequence +from collections.abc import Sequence from pre_commit import output diff --git a/pre_commit/parse_shebang.py b/pre_commit/parse_shebang.py index 3ee04e8d7..043a9b5d7 100644 --- a/pre_commit/parse_shebang.py +++ b/pre_commit/parse_shebang.py @@ -1,7 +1,7 @@ from __future__ import annotations import os.path -from typing import Mapping +from collections.abc import Mapping from typing import NoReturn from identify.identify import parse_shebang_from_file diff --git a/pre_commit/repository.py b/pre_commit/repository.py index 040f238f0..a9461ab63 100644 --- a/pre_commit/repository.py +++ b/pre_commit/repository.py @@ -3,16 +3,14 @@ import json import logging import os -import shlex +from collections.abc import Sequence from typing import Any -from typing import Sequence import pre_commit.constants as C from pre_commit.all_languages import languages from pre_commit.clientlib import load_manifest from pre_commit.clientlib import LOCAL from pre_commit.clientlib import META -from pre_commit.clientlib import parse_version from pre_commit.hook import Hook from pre_commit.lang_base import environment_dir from pre_commit.prefix import Prefix @@ -69,14 +67,6 @@ def _hook_install(hook: Hook) -> None: logger.info('Once installed this environment will be reused.') logger.info('This may take a few minutes...') - if hook.language == 'python_venv': - logger.warning( - f'`repo: {hook.src}` uses deprecated `language: python_venv`. ' - f'This is an alias for `language: python`. ' - f'Often `pre-commit autoupdate --repo {shlex.quote(hook.src)}` ' - f'will fix this.', - ) - lang = languages[hook.language] assert lang.ENVIRONMENT_DIR is not None @@ -124,15 +114,6 @@ def _hook( for dct in rest: ret.update(dct) - version = ret['minimum_pre_commit_version'] - if parse_version(version) > parse_version(C.VERSION): - logger.error( - f'The hook `{ret["id"]}` requires pre-commit version {version} ' - f'but version {C.VERSION} is installed. ' - f'Perhaps run `pip install --upgrade pre-commit`.', - ) - exit(1) - lang = ret['language'] if ret['language_version'] == C.DEFAULT: ret['language_version'] = root_config['default_language_version'][lang] diff --git a/pre_commit/resources/rbenv.tar.gz b/pre_commit/resources/rbenv.tar.gz index da2514e71..111546e3d 100644 Binary files a/pre_commit/resources/rbenv.tar.gz and b/pre_commit/resources/rbenv.tar.gz differ diff --git a/pre_commit/resources/ruby-build.tar.gz b/pre_commit/resources/ruby-build.tar.gz index 19d467fdd..a4f7eb24f 100644 Binary files a/pre_commit/resources/ruby-build.tar.gz and b/pre_commit/resources/ruby-build.tar.gz differ diff --git a/pre_commit/resources/ruby-download.tar.gz b/pre_commit/resources/ruby-download.tar.gz index 92502a77e..f7cb0b421 100644 Binary files a/pre_commit/resources/ruby-download.tar.gz and b/pre_commit/resources/ruby-download.tar.gz differ diff --git a/pre_commit/staged_files_only.py b/pre_commit/staged_files_only.py index 881235656..99ea0979b 100644 --- a/pre_commit/staged_files_only.py +++ b/pre_commit/staged_files_only.py @@ -4,7 +4,7 @@ import logging import os.path import time -from typing import Generator +from collections.abc import Generator from pre_commit import git from pre_commit.errors import FatalError @@ -33,7 +33,7 @@ def _git_apply(patch: str) -> None: @contextlib.contextmanager -def _intent_to_add_cleared() -> Generator[None, None, None]: +def _intent_to_add_cleared() -> Generator[None]: intent_to_add = git.intent_to_add_files() if intent_to_add: logger.warning('Unstaged intent-to-add files detected.') @@ -48,7 +48,7 @@ def _intent_to_add_cleared() -> Generator[None, None, None]: @contextlib.contextmanager -def _unstaged_changes_cleared(patch_dir: str) -> Generator[None, None, None]: +def _unstaged_changes_cleared(patch_dir: str) -> Generator[None]: tree = cmd_output('git', 'write-tree')[1].strip() diff_cmd = ( 'git', 'diff-index', '--ignore-submodules', '--binary', @@ -59,6 +59,11 @@ def _unstaged_changes_cleared(patch_dir: str) -> Generator[None, None, None]: # There weren't any staged files so we don't need to do anything # special yield + elif retcode == 1 and not diff_stdout.strip(): + # due to behaviour (probably a bug?) in git with crlf endings and + # autocrlf set to either `true` or `input` sometimes git will refuse + # to show a crlf-only diff to us :( + yield elif retcode == 1 and diff_stdout.strip(): patch_filename = f'patch{int(time.time())}-{os.getpid()}' patch_filename = os.path.join(patch_dir, patch_filename) @@ -100,7 +105,7 @@ def _unstaged_changes_cleared(patch_dir: str) -> Generator[None, None, None]: @contextlib.contextmanager -def staged_files_only(patch_dir: str) -> Generator[None, None, None]: +def staged_files_only(patch_dir: str) -> Generator[None]: """Clear any unstaged changes from the git working directory inside this context. """ diff --git a/pre_commit/store.py b/pre_commit/store.py index 487e3e798..1235942c5 100644 --- a/pre_commit/store.py +++ b/pre_commit/store.py @@ -5,11 +5,12 @@ import os.path import sqlite3 import tempfile +from collections.abc import Generator +from collections.abc import Sequence from typing import Callable -from typing import Generator -from typing import Sequence import pre_commit.constants as C +from pre_commit import clientlib from pre_commit import file_lock from pre_commit import git from pre_commit.util import CalledProcessError @@ -101,7 +102,7 @@ def __init__(self, directory: str | None = None) -> None: os.replace(tmpfile, self.db_path) @contextlib.contextmanager - def exclusive_lock(self) -> Generator[None, None, None]: + def exclusive_lock(self) -> Generator[None]: def blocked_cb() -> None: # pragma: no cover (tests are in-process) logger.info('Locking pre-commit directory') @@ -112,7 +113,7 @@ def blocked_cb() -> None: # pragma: no cover (tests are in-process) def connect( self, db_path: str | None = None, - ) -> Generator[sqlite3.Connection, None, None]: + ) -> Generator[sqlite3.Connection]: db_path = db_path or self.db_path # sqlite doesn't close its fd with its contextmanager >.< # contextlib.closing fixes this. @@ -136,6 +137,7 @@ def _new_repo( deps: Sequence[str], make_strategy: Callable[[str], None], ) -> str: + original_repo = repo repo = self.db_repo_name(repo, deps) def _get_result() -> str | None: @@ -168,6 +170,9 @@ def _get_result() -> str | None: 'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)', [repo, ref, directory], ) + + clientlib.warn_for_stages_on_repo_init(original_repo, directory) + return directory def _complete_clone(self, ref: str, git_cmd: Callable[..., None]) -> None: diff --git a/pre_commit/util.py b/pre_commit/util.py index 4f8e8357d..e199d0807 100644 --- a/pre_commit/util.py +++ b/pre_commit/util.py @@ -8,10 +8,10 @@ import stat import subprocess import sys +from collections.abc import Generator from types import TracebackType from typing import Any from typing import Callable -from typing import Generator from pre_commit import parse_shebang @@ -25,7 +25,7 @@ def force_bytes(exc: Any) -> bytes: @contextlib.contextmanager -def clean_path_on_failure(path: str) -> Generator[None, None, None]: +def clean_path_on_failure(path: str) -> Generator[None]: """Cleans up the directory on an exceptional failure.""" try: yield @@ -36,7 +36,8 @@ def clean_path_on_failure(path: str) -> Generator[None, None, None]: def resource_text(filename: str) -> str: - return importlib.resources.read_text('pre_commit.resources', filename) + files = importlib.resources.files('pre_commit.resources') + return files.joinpath(filename).read_text() def make_executable(filename: str) -> None: @@ -201,24 +202,37 @@ def cmd_output_p( cmd_output_p = cmd_output_b -def rmtree(path: str) -> None: - """On windows, rmtree fails for readonly dirs.""" - def handle_remove_readonly( - func: Callable[..., Any], - path: str, - exc: tuple[type[OSError], OSError, TracebackType], +def _handle_readonly( + func: Callable[[str], object], + path: str, + exc: BaseException, +) -> None: + if ( + func in (os.rmdir, os.remove, os.unlink) and + isinstance(exc, OSError) and + exc.errno in {errno.EACCES, errno.EPERM} + ): + for p in (path, os.path.dirname(path)): + os.chmod(p, os.stat(p).st_mode | stat.S_IWUSR) + func(path) + else: + raise + + +if sys.version_info < (3, 12): # pragma: <3.12 cover + def _handle_readonly_old( + func: Callable[[str], object], + path: str, + excinfo: tuple[type[BaseException], BaseException, TracebackType], ) -> None: - excvalue = exc[1] - if ( - func in (os.rmdir, os.remove, os.unlink) and - excvalue.errno in {errno.EACCES, errno.EPERM} - ): - for p in (path, os.path.dirname(path)): - os.chmod(p, os.stat(p).st_mode | stat.S_IWUSR) - func(path) - else: - raise - shutil.rmtree(path, ignore_errors=False, onerror=handle_remove_readonly) + return _handle_readonly(func, path, excinfo[1]) + + def rmtree(path: str) -> None: + shutil.rmtree(path, ignore_errors=False, onerror=_handle_readonly_old) +else: # pragma: >=3.12 cover + def rmtree(path: str) -> None: + """On windows, rmtree fails for readonly dirs.""" + shutil.rmtree(path, ignore_errors=False, onexc=_handle_readonly) def win_exe(s: str) -> str: diff --git a/pre_commit/xargs.py b/pre_commit/xargs.py index a7493c01d..a1345b583 100644 --- a/pre_commit/xargs.py +++ b/pre_commit/xargs.py @@ -7,12 +7,12 @@ import os import subprocess import sys +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import MutableMapping +from collections.abc import Sequence from typing import Any from typing import Callable -from typing import Generator -from typing import Iterable -from typing import MutableMapping -from typing import Sequence from typing import TypeVar from pre_commit import parse_shebang @@ -120,7 +120,6 @@ def partition( @contextlib.contextmanager def _thread_mapper(maxsize: int) -> Generator[ Callable[[Callable[[TArg], TRet], Iterable[TArg]], Iterable[TRet]], - None, None, ]: if maxsize == 1: yield map diff --git a/pre_commit/yaml.py b/pre_commit/yaml.py index bdf4ec47d..a5bbbc999 100644 --- a/pre_commit/yaml.py +++ b/pre_commit/yaml.py @@ -6,6 +6,7 @@ import yaml Loader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader) +yaml_compose = functools.partial(yaml.compose, Loader=Loader) yaml_load = functools.partial(yaml.load, Loader=Loader) Dumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper) diff --git a/pre_commit/yaml_rewrite.py b/pre_commit/yaml_rewrite.py new file mode 100644 index 000000000..8d0e8fdb2 --- /dev/null +++ b/pre_commit/yaml_rewrite.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from collections.abc import Generator +from collections.abc import Iterable +from typing import NamedTuple +from typing import Protocol + +from yaml.nodes import MappingNode +from yaml.nodes import Node +from yaml.nodes import ScalarNode +from yaml.nodes import SequenceNode + + +class _Matcher(Protocol): + def match(self, n: Node) -> Generator[Node]: ... + + +class MappingKey(NamedTuple): + k: str + + def match(self, n: Node) -> Generator[Node]: + if isinstance(n, MappingNode): + for k, _ in n.value: + if k.value == self.k: + yield k + + +class MappingValue(NamedTuple): + k: str + + def match(self, n: Node) -> Generator[Node]: + if isinstance(n, MappingNode): + for k, v in n.value: + if k.value == self.k: + yield v + + +class SequenceItem(NamedTuple): + def match(self, n: Node) -> Generator[Node]: + if isinstance(n, SequenceNode): + yield from n.value + + +def _match(gen: Iterable[Node], m: _Matcher) -> Iterable[Node]: + return (n for src in gen for n in m.match(src)) + + +def match(n: Node, matcher: tuple[_Matcher, ...]) -> Generator[ScalarNode]: + gen: Iterable[Node] = (n,) + for m in matcher: + gen = _match(gen, m) + return (n for n in gen if isinstance(n, ScalarNode)) diff --git a/setup.cfg b/setup.cfg index 7543835d7..af34452df 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pre_commit -version = 3.5.0 +version = 4.2.0 description = A framework for managing and maintaining multi-language pre-commit hooks. long_description = file: README.md long_description_content_type = text/markdown @@ -24,7 +24,7 @@ install_requires = nodeenv>=0.11.1 pyyaml>=5.1 virtualenv>=20.10.0 -python_requires = >=3.8 +python_requires = >=3.9 [options.packages.find] exclude = diff --git a/testing/language_helpers.py b/testing/language_helpers.py index ead8dae27..05c94ebca 100644 --- a/testing/language_helpers.py +++ b/testing/language_helpers.py @@ -1,7 +1,7 @@ from __future__ import annotations import os -from typing import Sequence +from collections.abc import Sequence from pre_commit.lang_base import Language from pre_commit.prefix import Prefix diff --git a/testing/make-archives b/testing/make-archives index 8ec05e2de..eb3f3af85 100755 --- a/testing/make-archives +++ b/testing/make-archives @@ -8,7 +8,7 @@ import shutil import subprocess import tarfile import tempfile -from typing import Sequence +from collections.abc import Sequence # This is a script for generating the tarred resources for git repo @@ -17,7 +17,7 @@ from typing import Sequence REPOS = ( ('rbenv', 'https://github.com/rbenv/rbenv', '38e1fbb'), - ('ruby-build', 'https://github.com/rbenv/ruby-build', '855b963'), + ('ruby-build', 'https://github.com/rbenv/ruby-build', 'ed384c8'), ( 'ruby-download', 'https://github.com/garnieretienne/rvm-download', @@ -57,8 +57,7 @@ def make_archive(name: str, repo: str, ref: str, destdir: str) -> str: arcs.sort() with gzip.GzipFile(output_path, 'wb', mtime=0) as gzipf: - # https://github.com/python/typeshed/issues/5491 - with tarfile.open(fileobj=gzipf, mode='w') as tf: # type: ignore + with tarfile.open(fileobj=gzipf, mode='w') as tf: for arcname, abspath in arcs: tf.add( abspath, diff --git a/tests/all_languages_test.py b/tests/all_languages_test.py deleted file mode 100644 index 98c912150..000000000 --- a/tests/all_languages_test.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from pre_commit.all_languages import languages - - -def test_python_venv_is_an_alias_to_python(): - assert languages['python_venv'] is languages['python'] diff --git a/tests/clientlib_test.py b/tests/clientlib_test.py index 568b2e974..7aa84af0e 100644 --- a/tests/clientlib_test.py +++ b/tests/clientlib_test.py @@ -40,56 +40,51 @@ def test_check_type_tag_success(): @pytest.mark.parametrize( - ('config_obj', 'expected'), ( - ( - { - 'repos': [{ - 'repo': 'git@github.com:pre-commit/pre-commit-hooks', - 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', - 'hooks': [{'id': 'pyflakes', 'files': '\\.py$'}], - }], - }, - True, - ), - ( - { - 'repos': [{ - 'repo': 'git@github.com:pre-commit/pre-commit-hooks', - 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', - 'hooks': [ - { - 'id': 'pyflakes', - 'files': '\\.py$', - 'args': ['foo', 'bar', 'baz'], - }, - ], - }], - }, - True, - ), - ( - { - 'repos': [{ - 'repo': 'git@github.com:pre-commit/pre-commit-hooks', - 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', - 'hooks': [ - { - 'id': 'pyflakes', - 'files': '\\.py$', - # Exclude pattern must be a string - 'exclude': 0, - 'args': ['foo', 'bar', 'baz'], - }, - ], - }], - }, - False, - ), + 'cfg', + ( + { + 'repos': [{ + 'repo': 'git@github.com:pre-commit/pre-commit-hooks', + 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', + 'hooks': [{'id': 'pyflakes', 'files': '\\.py$'}], + }], + }, + { + 'repos': [{ + 'repo': 'git@github.com:pre-commit/pre-commit-hooks', + 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', + 'hooks': [ + { + 'id': 'pyflakes', + 'files': '\\.py$', + 'args': ['foo', 'bar', 'baz'], + }, + ], + }], + }, ), ) -def test_config_valid(config_obj, expected): - ret = is_valid_according_to_schema(config_obj, CONFIG_SCHEMA) - assert ret is expected +def test_config_valid(cfg): + assert is_valid_according_to_schema(cfg, CONFIG_SCHEMA) + + +def test_invalid_config_wrong_type(): + cfg = { + 'repos': [{ + 'repo': 'git@github.com:pre-commit/pre-commit-hooks', + 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', + 'hooks': [ + { + 'id': 'pyflakes', + 'files': '\\.py$', + # Exclude pattern must be a string + 'exclude': 0, + 'args': ['foo', 'bar', 'baz'], + }, + ], + }], + } + assert not is_valid_according_to_schema(cfg, CONFIG_SCHEMA) def test_local_hooks_with_rev_fails(): @@ -198,14 +193,13 @@ def test_warn_mutable_rev_conditional(): ), ) def test_sensible_regex_validators_dont_pass_none(validator_cls): - key = 'files' + validator = validator_cls('files', cfgv.check_string) with pytest.raises(cfgv.ValidationError) as excinfo: - validator = validator_cls(key, cfgv.check_string) - validator.check({key: None}) + validator.check({'files': None}) assert str(excinfo.value) == ( '\n' - f'==> At key: {key}' + '==> At key: files' '\n' '=====> Expected string got NoneType' ) @@ -262,6 +256,24 @@ def test_validate_optional_sensible_regex_at_local_hook(caplog): ] +def test_validate_optional_sensible_regex_at_meta_hook(caplog): + config_obj = { + 'repo': 'meta', + 'hooks': [{'id': 'identity', 'files': 'dir/*.py'}], + } + + cfgv.validate(config_obj, CONFIG_REPO_DICT) + + assert caplog.record_tuples == [ + ( + 'pre_commit', + logging.WARNING, + "The 'files' field in hook 'identity' is a regex, not a glob " + "-- matching '/*' probably isn't what you want here", + ), + ] + + @pytest.mark.parametrize( ('regex', 'warning'), ( @@ -297,47 +309,87 @@ def test_validate_optional_sensible_regex_at_top_level(caplog, regex, warning): assert caplog.record_tuples == [('pre_commit', logging.WARNING, warning)] -@pytest.mark.parametrize( - ('manifest_obj', 'expected'), - ( - ( - [{ - 'id': 'a', - 'name': 'b', - 'entry': 'c', - 'language': 'python', - 'files': r'\.py$', - }], - True, - ), +def test_warning_for_deprecated_stages(caplog): + config_obj = sample_local_config() + config_obj['hooks'][0]['stages'] = ['commit', 'push'] + + cfgv.validate(config_obj, CONFIG_REPO_DICT) + + assert caplog.record_tuples == [ ( - [{ - 'id': 'a', - 'name': 'b', - 'entry': 'c', - 'language': 'python', - 'language_version': 'python3.4', - 'files': r'\.py$', - }], - True, + 'pre_commit', + logging.WARNING, + 'hook id `do_not_commit` uses deprecated stage names ' + '(commit, push) which will be removed in a future version. ' + 'run: `pre-commit migrate-config` to automatically fix this.', ), + ] + + +def test_no_warning_for_non_deprecated_stages(caplog): + config_obj = sample_local_config() + config_obj['hooks'][0]['stages'] = ['pre-commit', 'pre-push'] + + cfgv.validate(config_obj, CONFIG_REPO_DICT) + + assert caplog.record_tuples == [] + + +def test_warning_for_deprecated_default_stages(caplog): + cfg = {'default_stages': ['commit', 'push'], 'repos': []} + + cfgv.validate(cfg, CONFIG_SCHEMA) + + assert caplog.record_tuples == [ ( - # A regression in 0.13.5: always_run and files are permissible - [{ - 'id': 'a', - 'name': 'b', - 'entry': 'c', - 'language': 'python', - 'files': '', - 'always_run': True, - }], - True, + 'pre_commit', + logging.WARNING, + 'top-level `default_stages` uses deprecated stage names ' + '(commit, push) which will be removed in a future version. ' + 'run: `pre-commit migrate-config` to automatically fix this.', ), + ] + + +def test_no_warning_for_non_deprecated_default_stages(caplog): + cfg = {'default_stages': ['pre-commit', 'pre-push'], 'repos': []} + + cfgv.validate(cfg, CONFIG_SCHEMA) + + assert caplog.record_tuples == [] + + +@pytest.mark.parametrize( + 'manifest_obj', + ( + [{ + 'id': 'a', + 'name': 'b', + 'entry': 'c', + 'language': 'python', + 'files': r'\.py$', + }], + [{ + 'id': 'a', + 'name': 'b', + 'entry': 'c', + 'language': 'python', + 'language_version': 'python3.4', + 'files': r'\.py$', + }], + # A regression in 0.13.5: always_run and files are permissible + [{ + 'id': 'a', + 'name': 'b', + 'entry': 'c', + 'language': 'python', + 'files': '', + 'always_run': True, + }], ), ) -def test_valid_manifests(manifest_obj, expected): - ret = is_valid_according_to_schema(manifest_obj, MANIFEST_SCHEMA) - assert ret is expected +def test_valid_manifests(manifest_obj): + assert is_valid_according_to_schema(manifest_obj, MANIFEST_SCHEMA) @pytest.mark.parametrize( @@ -393,8 +445,39 @@ def test_parse_version(): def test_minimum_pre_commit_version_failing(): + cfg = {'repos': [], 'minimum_pre_commit_version': '999'} + with pytest.raises(cfgv.ValidationError) as excinfo: + cfgv.validate(cfg, CONFIG_SCHEMA) + assert str(excinfo.value) == ( + f'\n' + f'==> At Config()\n' + f'==> At key: minimum_pre_commit_version\n' + f'=====> pre-commit version 999 is required but version {C.VERSION} ' + f'is installed. Perhaps run `pip install --upgrade pre-commit`.' + ) + + +def test_minimum_pre_commit_version_failing_in_config(): + cfg = {'repos': [sample_local_config()]} + cfg['repos'][0]['hooks'][0]['minimum_pre_commit_version'] = '999' + with pytest.raises(cfgv.ValidationError) as excinfo: + cfgv.validate(cfg, CONFIG_SCHEMA) + assert str(excinfo.value) == ( + f'\n' + f'==> At Config()\n' + f'==> At key: repos\n' + f"==> At Repository(repo='local')\n" + f'==> At key: hooks\n' + f"==> At Hook(id='do_not_commit')\n" + f'==> At key: minimum_pre_commit_version\n' + f'=====> pre-commit version 999 is required but version {C.VERSION} ' + f'is installed. Perhaps run `pip install --upgrade pre-commit`.' + ) + + +def test_minimum_pre_commit_version_failing_before_other_error(): + cfg = {'repos': 5, 'minimum_pre_commit_version': '999'} with pytest.raises(cfgv.ValidationError) as excinfo: - cfg = {'repos': [], 'minimum_pre_commit_version': '999'} cfgv.validate(cfg, CONFIG_SCHEMA) assert str(excinfo.value) == ( f'\n' diff --git a/tests/commands/install_uninstall_test.py b/tests/commands/install_uninstall_test.py index 8b0d3ece4..9eb0e741a 100644 --- a/tests/commands/install_uninstall_test.py +++ b/tests/commands/install_uninstall_test.py @@ -349,8 +349,9 @@ def test_install_existing_hooks_no_overwrite(tempdir_factory, store): # We should run both the legacy and pre-commit hooks ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert output.startswith('legacy hook\n') - NORMAL_PRE_COMMIT_RUN.assert_matches(output[len('legacy hook\n'):]) + legacy = 'legacy hook\n' + assert output.startswith(legacy) + NORMAL_PRE_COMMIT_RUN.assert_matches(output.removeprefix(legacy)) def test_legacy_overwriting_legacy_hook(tempdir_factory, store): @@ -375,8 +376,9 @@ def test_install_existing_hook_no_overwrite_idempotent(tempdir_factory, store): # We should run both the legacy and pre-commit hooks ret, output = _get_commit_output(tempdir_factory) assert ret == 0 - assert output.startswith('legacy hook\n') - NORMAL_PRE_COMMIT_RUN.assert_matches(output[len('legacy hook\n'):]) + legacy = 'legacy hook\n' + assert output.startswith(legacy) + NORMAL_PRE_COMMIT_RUN.assert_matches(output.removeprefix(legacy)) def test_install_with_existing_non_utf8_script(tmpdir, store): diff --git a/tests/commands/migrate_config_test.py b/tests/commands/migrate_config_test.py index ba1846360..a517d2f44 100644 --- a/tests/commands/migrate_config_test.py +++ b/tests/commands/migrate_config_test.py @@ -1,10 +1,26 @@ from __future__ import annotations +from unittest import mock + import pytest +import yaml import pre_commit.constants as C from pre_commit.clientlib import InvalidConfigError from pre_commit.commands.migrate_config import migrate_config +from pre_commit.yaml import yaml_compose + + +@pytest.fixture(autouse=True, params=['c', 'pure']) +def switch_pyyaml_impl(request): + if request.param == 'c': + yield + else: + with mock.patch.dict( + yaml_compose.keywords, + {'Loader': yaml.SafeLoader}, + ): + yield def test_migrate_config_normal_format(tmpdir, capsys): @@ -134,6 +150,27 @@ def test_migrate_config_sha_to_rev(tmpdir): ) +def test_migrate_config_sha_to_rev_json(tmp_path): + contents = """\ +{"repos": [{ + "repo": "https://github.com/pre-commit/pre-commit-hooks", + "sha": "v1.2.0", + "hooks": [] +}]} +""" + expected = """\ +{"repos": [{ + "repo": "https://github.com/pre-commit/pre-commit-hooks", + "rev": "v1.2.0", + "hooks": [] +}]} +""" + cfg = tmp_path.joinpath('cfg.yaml') + cfg.write_text(contents) + assert not migrate_config(str(cfg)) + assert cfg.read_text() == expected + + def test_migrate_config_language_python_venv(tmp_path): src = '''\ repos: @@ -167,6 +204,73 @@ def test_migrate_config_language_python_venv(tmp_path): assert cfg.read_text() == expected +def test_migrate_config_quoted_python_venv(tmp_path): + src = '''\ +repos: +- repo: local + hooks: + - id: example + name: example + entry: example + language: "python_venv" +''' + expected = '''\ +repos: +- repo: local + hooks: + - id: example + name: example + entry: example + language: "python" +''' + cfg = tmp_path.joinpath('cfg.yaml') + cfg.write_text(src) + assert migrate_config(str(cfg)) == 0 + assert cfg.read_text() == expected + + +def test_migrate_config_default_stages(tmp_path): + src = '''\ +default_stages: [commit, push, merge-commit, commit-msg] +repos: [] +''' + expected = '''\ +default_stages: [pre-commit, pre-push, pre-merge-commit, commit-msg] +repos: [] +''' + cfg = tmp_path.joinpath('cfg.yaml') + cfg.write_text(src) + assert migrate_config(str(cfg)) == 0 + assert cfg.read_text() == expected + + +def test_migrate_config_hook_stages(tmp_path): + src = '''\ +repos: +- repo: local + hooks: + - id: example + name: example + entry: example + language: system + stages: ["commit", "push", "merge-commit", "commit-msg"] +''' + expected = '''\ +repos: +- repo: local + hooks: + - id: example + name: example + entry: example + language: system + stages: ["pre-commit", "pre-push", "pre-merge-commit", "commit-msg"] +''' + cfg = tmp_path.joinpath('cfg.yaml') + cfg.write_text(src) + assert migrate_config(str(cfg)) == 0 + assert cfg.read_text() == expected + + def test_migrate_config_invalid_yaml(tmpdir): contents = '[' cfg = tmpdir.join(C.CONFIG_FILE) diff --git a/tests/commands/run_test.py b/tests/commands/run_test.py index 6a0cd8556..50a20f377 100644 --- a/tests/commands/run_test.py +++ b/tests/commands/run_test.py @@ -4,7 +4,7 @@ import shlex import sys import time -from typing import MutableMapping +from collections.abc import MutableMapping from unittest import mock import pytest @@ -1088,6 +1088,22 @@ def test_fail_fast_per_hook(cap_out, store, repo_with_failing_hook): assert printed.count(b'Failing hook') == 1 +def test_fail_fast_not_prev_failures(cap_out, store, repo_with_failing_hook): + with modify_config() as config: + config['repos'].append({ + 'repo': 'meta', + 'hooks': [ + {'id': 'identity', 'fail_fast': True}, + {'id': 'identity', 'name': 'run me!'}, + ], + }) + stage_a_file() + + ret, printed = _do_run(cap_out, store, repo_with_failing_hook, run_opts()) + # should still run the last hook since the `fail_fast` one didn't fail + assert printed.count(b'run me!') == 1 + + def test_classifier_removes_dne(): classifier = Classifier(('this_file_does_not_exist',)) assert classifier.filenames == [] diff --git a/tests/conftest.py b/tests/conftest.py index 30761715b..8c9cd14db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,6 @@ import functools import io -import logging import os.path from unittest import mock @@ -203,42 +202,25 @@ def store(tempdir_factory): yield Store(os.path.join(tempdir_factory.get(), '.pre-commit')) -@pytest.fixture -def log_info_mock(): - with mock.patch.object(logging.getLogger('pre_commit'), 'info') as mck: - yield mck - - -class FakeStream: - def __init__(self): - self.data = io.BytesIO() - - def write(self, s): - self.data.write(s) - - def flush(self): - pass - - class Fixture: - def __init__(self, stream): + def __init__(self, stream: io.BytesIO) -> None: self._stream = stream - def get_bytes(self): + def get_bytes(self) -> bytes: """Get the output as-if no encoding occurred""" - data = self._stream.data.getvalue() - self._stream.data.seek(0) - self._stream.data.truncate() + data = self._stream.getvalue() + self._stream.seek(0) + self._stream.truncate() return data.replace(b'\r\n', b'\n') - def get(self): + def get(self) -> str: """Get the output assuming it was written as UTF-8 bytes""" return self.get_bytes().decode() @pytest.fixture def cap_out(): - stream = FakeStream() + stream = io.BytesIO() write = functools.partial(output.write, stream=stream) write_line_b = functools.partial(output.write_line_b, stream=stream) with mock.patch.multiple(output, write=write, write_line_b=write_line_b): diff --git a/tests/git_test.py b/tests/git_test.py index 93f5a1c6e..02b6ce3ae 100644 --- a/tests/git_test.py +++ b/tests/git_test.py @@ -141,6 +141,15 @@ def test_get_conflicted_files_unstaged_files(in_merge_conflict): assert ret == {'conflict_file'} +def test_get_conflicted_files_with_file_named_head(in_merge_conflict): + resolve_conflict() + open('HEAD', 'w').close() + cmd_output('git', 'add', 'HEAD') + + ret = set(git.get_conflicted_files()) + assert ret == {'conflict_file', 'HEAD'} + + MERGE_MSG = b"Merge branch 'foo' into bar\n\nConflicts:\n\tconflict_file\n" OTHER_MERGE_MSG = MERGE_MSG + b'\tother_conflict_file\n' diff --git a/tests/languages/docker_image_test.py b/tests/languages/docker_image_test.py index 7993c11a8..4f720600b 100644 --- a/tests/languages/docker_image_test.py +++ b/tests/languages/docker_image_test.py @@ -1,10 +1,18 @@ from __future__ import annotations +import pytest + from pre_commit.languages import docker_image +from pre_commit.util import cmd_output_b from testing.language_helpers import run_language from testing.util import xfailif_windows +@pytest.fixture(autouse=True, scope='module') +def _ensure_image_available(): + cmd_output_b('docker', 'run', '--rm', 'ubuntu:22.04', 'echo') + + @xfailif_windows # pragma: win32 no cover def test_docker_image_hook_via_entrypoint(tmp_path): ret = run_language( @@ -25,3 +33,27 @@ def test_docker_image_hook_via_args(tmp_path): args=('hello hello world',), ) assert ret == (0, b'hello hello world\n') + + +@xfailif_windows # pragma: win32 no cover +def test_docker_image_color_tty(tmp_path): + ret = run_language( + tmp_path, + docker_image, + 'ubuntu:22.04', + args=('grep', '--color', 'root', '/etc/group'), + color=True, + ) + assert ret == (0, b'\x1b[01;31m\x1b[Kroot\x1b[m\x1b[K:x:0:\n') + + +@xfailif_windows # pragma: win32 no cover +def test_docker_image_no_color_no_tty(tmp_path): + ret = run_language( + tmp_path, + docker_image, + 'ubuntu:22.04', + args=('grep', '--color', 'root', '/etc/group'), + color=False, + ) + assert ret == (0, b'root:x:0:\n') diff --git a/tests/languages/dotnet_test.py b/tests/languages/dotnet_test.py index 470c03b22..ee4082568 100644 --- a/tests/languages/dotnet_test.py +++ b/tests/languages/dotnet_test.py @@ -27,7 +27,7 @@ def _csproj(tool_name): Exe - net6 + net8 true {tool_name} ./nupkg diff --git a/tests/languages/golang_test.py b/tests/languages/golang_test.py index 19e9f62f6..7fb6ab18b 100644 --- a/tests/languages/golang_test.py +++ b/tests/languages/golang_test.py @@ -7,10 +7,18 @@ import pre_commit.constants as C from pre_commit import lang_base +from pre_commit.commands.install_uninstall import install from pre_commit.envcontext import envcontext from pre_commit.languages import golang from pre_commit.store import _make_local_repo +from pre_commit.util import CalledProcessError +from pre_commit.util import cmd_output +from testing.fixtures import add_config_to_repo +from testing.fixtures import make_config_from_repo from testing.language_helpers import run_language +from testing.util import cmd_output_mocked_pre_commit_home +from testing.util import cwd +from testing.util import git_commit ACTUAL_GET_DEFAULT_VERSION = golang.get_default_version.__wrapped__ @@ -134,3 +142,95 @@ def test_local_golang_additional_deps(tmp_path): def test_golang_hook_still_works_when_gobin_is_set(tmp_path): with envcontext((('GOBIN', str(tmp_path.joinpath('gobin'))),)): test_golang_system(tmp_path) + + +def test_during_commit_all(tmp_path, tempdir_factory, store, in_git_dir): + hook_dir = tmp_path.joinpath('hook') + hook_dir.mkdir() + _make_hello_world(hook_dir) + hook_dir.joinpath('.pre-commit-hooks.yaml').write_text( + '- id: hello-world\n' + ' name: hello world\n' + ' entry: golang-hello-world\n' + ' language: golang\n' + ' always_run: true\n', + ) + cmd_output('git', 'init', hook_dir) + cmd_output('git', 'add', '.', cwd=hook_dir) + git_commit(cwd=hook_dir) + + add_config_to_repo(in_git_dir, make_config_from_repo(hook_dir)) + + assert not install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + + git_commit( + fn=cmd_output_mocked_pre_commit_home, + tempdir_factory=tempdir_factory, + ) + + +def test_automatic_toolchain_switching(tmp_path): + go_mod = '''\ +module toolchain-version-test + +go 1.23.1 +''' + main_go = '''\ +package main + +func main() {} +''' + tmp_path.joinpath('go.mod').write_text(go_mod) + mod_dir = tmp_path.joinpath('toolchain-version-test') + mod_dir.mkdir() + main_file = mod_dir.joinpath('main.go') + main_file.write_text(main_go) + + with pytest.raises(CalledProcessError) as excinfo: + run_language( + path=tmp_path, + language=golang, + version='1.22.0', + exe='golang-version-test', + ) + + assert 'go.mod requires go >= 1.23.1' in excinfo.value.stderr.decode() + + +def test_automatic_toolchain_switching_go_fmt(tmp_path, monkeypatch): + go_mod_hook = '''\ +module toolchain-version-test + +go 1.22.0 +''' + go_mod = '''\ +module toolchain-version-test + +go 1.23.1 +''' + main_go = '''\ +package main + +func main() {} +''' + hook_dir = tmp_path.joinpath('hook') + hook_dir.mkdir() + hook_dir.joinpath('go.mod').write_text(go_mod_hook) + + test_dir = tmp_path.joinpath('test') + test_dir.mkdir() + test_dir.joinpath('go.mod').write_text(go_mod) + main_file = test_dir.joinpath('main.go') + main_file.write_text(main_go) + + with cwd(test_dir): + ret, out = run_language( + path=hook_dir, + language=golang, + version='1.22.0', + exe='go fmt', + file_args=(str(main_file),), + ) + + assert ret == 1 + assert 'go.mod requires go >= 1.23.1' in out.decode() diff --git a/tests/languages/julia_test.py b/tests/languages/julia_test.py new file mode 100644 index 000000000..4ea3c25b2 --- /dev/null +++ b/tests/languages/julia_test.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from pre_commit.languages import julia +from testing.language_helpers import run_language +from testing.util import cwd + + +def _make_hook(tmp_path, julia_code): + src_dir = tmp_path.joinpath('src') + src_dir.mkdir() + src_dir.joinpath('main.jl').write_text(julia_code) + tmp_path.joinpath('Project.toml').write_text( + '[deps]\n' + 'Example = "7876af07-990d-54b4-ab0e-23690620f79a"\n', + ) + + +def test_julia_hook(tmp_path): + code = """ + using Example + function main() + println("Hello, world!") + end + main() + """ + _make_hook(tmp_path, code) + expected = (0, b'Hello, world!\n') + assert run_language(tmp_path, julia, 'src/main.jl') == expected + + +def test_julia_hook_manifest(tmp_path): + code = """ + using Example + println(pkgversion(Example)) + """ + _make_hook(tmp_path, code) + + tmp_path.joinpath('Manifest.toml').write_text( + 'manifest_format = "2.0"\n\n' + '[[deps.Example]]\n' + 'git-tree-sha1 = "11820aa9c229fd3833d4bd69e5e75ef4e7273bf1"\n' + 'uuid = "7876af07-990d-54b4-ab0e-23690620f79a"\n' + 'version = "0.5.4"\n', + ) + expected = (0, b'0.5.4\n') + assert run_language(tmp_path, julia, 'src/main.jl') == expected + + +def test_julia_hook_args(tmp_path): + code = """ + function main(argv) + foreach(println, argv) + end + main(ARGS) + """ + _make_hook(tmp_path, code) + expected = (0, b'--arg1\n--arg2\n') + assert run_language( + tmp_path, julia, 'src/main.jl --arg1 --arg2', + ) == expected + + +def test_julia_hook_additional_deps(tmp_path): + code = """ + using TOML + function main() + project_file = Base.active_project() + dict = TOML.parsefile(project_file) + for (k, v) in dict["deps"] + println(k, " = ", v) + end + end + main() + """ + _make_hook(tmp_path, code) + deps = ('TOML=fa267f1f-6049-4f14-aa54-33bafae1ed76',) + ret, out = run_language(tmp_path, julia, 'src/main.jl', deps=deps) + assert ret == 0 + assert b'Example = 7876af07-990d-54b4-ab0e-23690620f79a' in out + assert b'TOML = fa267f1f-6049-4f14-aa54-33bafae1ed76' in out + + +def test_julia_repo_local(tmp_path): + env_dir = tmp_path.joinpath('envdir') + env_dir.mkdir() + local_dir = tmp_path.joinpath('local') + local_dir.mkdir() + local_dir.joinpath('local.jl').write_text( + 'using TOML; foreach(println, ARGS)', + ) + with cwd(local_dir): + deps = ('TOML=fa267f1f-6049-4f14-aa54-33bafae1ed76',) + expected = (0, b'--local-arg1\n--local-arg2\n') + assert run_language( + env_dir, julia, 'local.jl --local-arg1 --local-arg2', + deps=deps, is_local=True, + ) == expected diff --git a/tests/languages/python_test.py b/tests/languages/python_test.py index ab26e14e7..565525a40 100644 --- a/tests/languages/python_test.py +++ b/tests/languages/python_test.py @@ -12,6 +12,7 @@ from pre_commit.prefix import Prefix from pre_commit.util import make_executable from pre_commit.util import win_exe +from testing.auto_namedtuple import auto_namedtuple from testing.language_helpers import run_language @@ -34,6 +35,72 @@ def test_read_pyvenv_cfg_non_utf8(tmpdir): assert python._read_pyvenv_cfg(pyvenv_cfg) == expected +def _get_default_version( + *, + impl: str, + exe: str, + found: set[str], + version: tuple[int, int], +) -> str: + sys_exe = f'/fake/path/{exe}' + sys_impl = auto_namedtuple(name=impl) + sys_ver = auto_namedtuple(major=version[0], minor=version[1]) + + def find_exe(s): + if s in found: + return f'/fake/path/found/{exe}' + else: + return None + + with ( + mock.patch.object(sys, 'implementation', sys_impl), + mock.patch.object(sys, 'executable', sys_exe), + mock.patch.object(sys, 'version_info', sys_ver), + mock.patch.object(python, 'find_executable', find_exe), + ): + return python.get_default_version.__wrapped__() + + +def test_default_version_sys_executable_found(): + ret = _get_default_version( + impl='cpython', + exe='python3.12', + found={'python3.12'}, + version=(3, 12), + ) + assert ret == 'python3.12' + + +def test_default_version_picks_specific_when_found(): + ret = _get_default_version( + impl='cpython', + exe='python3', + found={'python3', 'python3.12'}, + version=(3, 12), + ) + assert ret == 'python3.12' + + +def test_default_version_picks_pypy_versioned_exe(): + ret = _get_default_version( + impl='pypy', + exe='python', + found={'pypy3.12', 'python3'}, + version=(3, 12), + ) + assert ret == 'pypy3.12' + + +def test_default_version_picks_pypy_unversioned_exe(): + ret = _get_default_version( + impl='pypy', + exe='python', + found={'pypy3', 'python3'}, + version=(3, 12), + ) + assert ret == 'pypy3' + + def test_norm_version_expanduser(): home = os.path.expanduser('~') if sys.platform == 'win32': # pragma: win32 cover diff --git a/tests/languages/r_test.py b/tests/languages/r_test.py index 02c559cb4..9e73129e1 100644 --- a/tests/languages/r_test.py +++ b/tests/languages/r_test.py @@ -1,14 +1,17 @@ from __future__ import annotations import os.path -import shutil +from unittest import mock import pytest +import pre_commit.constants as C from pre_commit import envcontext +from pre_commit import lang_base from pre_commit.languages import r from pre_commit.prefix import Prefix from pre_commit.store import _make_local_repo +from pre_commit.util import resource_text from pre_commit.util import win_exe from testing.language_helpers import run_language @@ -127,7 +130,8 @@ def test_path_rscript_exec_no_r_home_set(): assert r._rscript_exec() == 'Rscript' -def test_r_hook(tmp_path): +@pytest.fixture +def renv_lock_file(tmp_path): renv_lock = '''\ { "R": { @@ -157,6 +161,12 @@ def test_r_hook(tmp_path): } } ''' + tmp_path.joinpath('renv.lock').write_text(renv_lock) + yield + + +@pytest.fixture +def description_file(tmp_path): description = '''\ Package: gli.clu Title: What the Package Does (One Line, Title Case) @@ -178,27 +188,39 @@ def test_r_hook(tmp_path): Imports: rprojroot ''' - hello_world_r = '''\ + tmp_path.joinpath('DESCRIPTION').write_text(description) + yield + + +@pytest.fixture +def hello_world_file(tmp_path): + hello_world = '''\ stopifnot( packageVersion('rprojroot') == '1.0', packageVersion('gli.clu') == '0.0.0.9000' ) cat("Hello, World, from R!\n") ''' + tmp_path.joinpath('hello-world.R').write_text(hello_world) + yield - tmp_path.joinpath('renv.lock').write_text(renv_lock) - tmp_path.joinpath('DESCRIPTION').write_text(description) - tmp_path.joinpath('hello-world.R').write_text(hello_world_r) + +@pytest.fixture +def renv_folder(tmp_path): renv_dir = tmp_path.joinpath('renv') renv_dir.mkdir() - shutil.copy( - os.path.join( - os.path.dirname(__file__), - '../../pre_commit/resources/empty_template_activate.R', - ), - renv_dir.joinpath('activate.R'), - ) + activate_r = resource_text('empty_template_activate.R') + renv_dir.joinpath('activate.R').write_text(activate_r) + yield + +def test_r_hook( + tmp_path, + renv_lock_file, + description_file, + hello_world_file, + renv_folder, +): expected = (0, b'Hello, World, from R!\n') assert run_language(tmp_path, r, 'Rscript hello-world.R') == expected @@ -221,3 +243,55 @@ def test_r_inline(tmp_path): args=('hi', 'hello'), ) assert ret == (0, b'hi, hello, from R!\n') + + +@pytest.fixture +def prefix(tmpdir): + yield Prefix(str(tmpdir)) + + +@pytest.fixture +def installed_environment( + renv_lock_file, + hello_world_file, + renv_folder, + prefix, +): + env_dir = lang_base.environment_dir( + prefix, r.ENVIRONMENT_DIR, r.get_default_version(), + ) + r.install_environment(prefix, C.DEFAULT, ()) + yield prefix, env_dir + + +def test_health_check_healthy(installed_environment): + # should be healthy right after creation + prefix, _ = installed_environment + assert r.health_check(prefix, C.DEFAULT) is None + + +def test_health_check_after_downgrade(installed_environment): + prefix, _ = installed_environment + + # pretend the saved installed version is old + with mock.patch.object(r, '_read_installed_version', return_value='1.0.0'): + output = r.health_check(prefix, C.DEFAULT) + + assert output is not None + assert output.startswith('Hooks were installed for R version') + + +@pytest.mark.parametrize('version', ('NULL', 'NA', "''")) +def test_health_check_without_version(prefix, installed_environment, version): + prefix, env_dir = installed_environment + + # simulate old pre-commit install by unsetting the installed version + r._execute_r_in_renv( + f'renv::settings$r.version({version})', + prefix=prefix, version=C.DEFAULT, cwd=env_dir, + ) + + # no R version specified fails as unhealty + msg = 'Hooks were installed with an unknown R version' + check_output = r.health_check(prefix, C.DEFAULT) + assert check_output is not None and check_output.startswith(msg) diff --git a/tests/languages/ruby_test.py b/tests/languages/ruby_test.py index 6397a4347..5d767b25d 100644 --- a/tests/languages/ruby_test.py +++ b/tests/languages/ruby_test.py @@ -91,8 +91,8 @@ def test_ruby_additional_deps(tmp_path): tmp_path, ruby, 'ruby -e', - args=('require "tins"',), - deps=('tins',), + args=('require "jmespath"',), + deps=('jmespath',), ) assert ret == (0, b'') diff --git a/tests/languages/rust_test.py b/tests/languages/rust_test.py index 5c17f5b69..52e356134 100644 --- a/tests/languages/rust_test.py +++ b/tests/languages/rust_test.py @@ -9,6 +9,7 @@ from pre_commit.languages import rust from pre_commit.store import _make_local_repo from testing.language_helpers import run_language +from testing.util import cwd ACTUAL_GET_DEFAULT_VERSION = rust.get_default_version.__wrapped__ @@ -29,6 +30,14 @@ def test_uses_default_when_rust_is_not_available(cmd_output_b_mck): assert ACTUAL_GET_DEFAULT_VERSION() == C.DEFAULT +def test_selects_system_even_if_rust_toolchain_toml(tmp_path): + toolchain_toml = '[toolchain]\nchannel = "wtf"\n' + tmp_path.joinpath('rust-toolchain.toml').write_text(toolchain_toml) + + with cwd(tmp_path): + assert ACTUAL_GET_DEFAULT_VERSION() == 'system' + + def _make_hello_world(tmp_path): src_dir = tmp_path.joinpath('src') src_dir.mkdir() diff --git a/tests/repository_test.py b/tests/repository_test.py index b8dde99b4..b54c910d3 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -9,7 +9,6 @@ import cfgv import pytest -import re_assert import pre_commit.constants as C from pre_commit import lang_base @@ -27,7 +26,6 @@ from pre_commit.util import cmd_output_b from testing.fixtures import make_config_from_repo from testing.fixtures import make_repo -from testing.fixtures import modify_manifest from testing.language_helpers import run_language from testing.util import cwd from testing.util import get_resource_path @@ -82,24 +80,6 @@ def _test_hook_repo( assert out == expected -def test_python_venv_deprecation(store, caplog): - config = { - 'repo': 'local', - 'hooks': [{ - 'id': 'example', - 'name': 'example', - 'language': 'python_venv', - 'entry': 'echo hi', - }], - } - _get_hook(config, store, 'example') - assert caplog.messages[-1] == ( - '`repo: local` uses deprecated `language: python_venv`. ' - 'This is an alias for `language: python`. ' - 'Often `pre-commit autoupdate --repo local` will fix this.' - ) - - def test_system_hook_with_spaces(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'system_hook_with_spaces_repo', @@ -242,16 +222,16 @@ def test_unknown_keys(store, caplog): assert msg == 'Unexpected key(s) present on local => too-much: foo, hello' -def test_reinstall(tempdir_factory, store, log_info_mock): +def test_reinstall(tempdir_factory, store, caplog): path = make_repo(tempdir_factory, 'python_hooks_repo') config = make_config_from_repo(path) _get_hook(config, store, 'foo') # We print some logging during clone (1) + install (3) - assert log_info_mock.call_count == 4 - log_info_mock.reset_mock() + assert len(caplog.record_tuples) == 4 + caplog.clear() # Reinstall on another run should not trigger another install _get_hook(config, store, 'foo') - assert log_info_mock.call_count == 0 + assert len(caplog.record_tuples) == 0 def test_control_c_control_c_on_install(tempdir_factory, store): @@ -433,32 +413,6 @@ def test_hook_id_not_present(tempdir_factory, store, caplog): ) -def test_too_new_version(tempdir_factory, store, caplog): - path = make_repo(tempdir_factory, 'script_hooks_repo') - with modify_manifest(path) as manifest: - manifest[0]['minimum_pre_commit_version'] = '999.0.0' - config = make_config_from_repo(path) - with pytest.raises(SystemExit): - _get_hook(config, store, 'bash_hook') - _, msg = caplog.messages - pattern = re_assert.Matches( - r'^The hook `bash_hook` requires pre-commit version 999\.0\.0 but ' - r'version \d+\.\d+\.\d+ is installed. ' - r'Perhaps run `pip install --upgrade pre-commit`\.$', - ) - pattern.assert_matches(msg) - - -@pytest.mark.parametrize('version', ('0.1.0', C.VERSION)) -def test_versions_ok(tempdir_factory, store, version): - path = make_repo(tempdir_factory, 'script_hooks_repo') - with modify_manifest(path) as manifest: - manifest[0]['minimum_pre_commit_version'] = version - config = make_config_from_repo(path) - # Should succeed - _get_hook(config, store, 'bash_hook') - - def test_manifest_hooks(tempdir_factory, store): path = make_repo(tempdir_factory, 'script_hooks_repo') config = make_config_from_repo(path) diff --git a/tests/staged_files_only_test.py b/tests/staged_files_only_test.py index 58dbe5ac6..cd2f63870 100644 --- a/tests/staged_files_only_test.py +++ b/tests/staged_files_only_test.py @@ -358,6 +358,21 @@ def test_crlf(in_git_dir, patch_dir, crlf_before, crlf_after, autocrlf): assert_no_diff() +@pytest.mark.parametrize('autocrlf', ('true', 'input')) +def test_crlf_diff_only(in_git_dir, patch_dir, autocrlf): + # due to a quirk (?) in git -- a diff only in crlf does not show but + # still results in an exit code of `1` + # we treat this as "no diff" -- though ideally it would discard the diff + # while committing + cmd_output('git', 'config', '--local', 'core.autocrlf', autocrlf) + + _write(b'1\r\n2\r\n3\r\n') + cmd_output('git', 'add', 'foo') + _write(b'1\n2\n3\n') + with staged_files_only(patch_dir): + pass + + def test_whitespace_errors(in_git_dir, patch_dir): cmd_output('git', 'config', '--local', 'apply.whitespace', 'error') test_crlf(in_git_dir, patch_dir, True, True, 'true') diff --git a/tests/store_test.py b/tests/store_test.py index eaab94000..7d4dffb09 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -1,12 +1,15 @@ from __future__ import annotations +import logging import os.path +import shlex import sqlite3 import stat from unittest import mock import pytest +import pre_commit.constants as C from pre_commit import git from pre_commit.store import _get_default_directory from pre_commit.store import _LOCAL_RESOURCES @@ -65,7 +68,7 @@ def test_store_init(store): assert text_line in readme_contents -def test_clone(store, tempdir_factory, log_info_mock): +def test_clone(store, tempdir_factory, caplog): path = git_dir(tempdir_factory) with cwd(path): git_commit() @@ -74,7 +77,7 @@ def test_clone(store, tempdir_factory, log_info_mock): ret = store.clone(path, rev) # Should have printed some stuff - assert log_info_mock.call_args_list[0][0][0].startswith( + assert caplog.record_tuples[0][-1].startswith( 'Initializing environment for ', ) @@ -91,6 +94,72 @@ def test_clone(store, tempdir_factory, log_info_mock): assert store.select_all_repos() == [(path, rev, ret)] +def test_warning_for_deprecated_stages_on_init(store, tempdir_factory, caplog): + manifest = '''\ +- id: hook1 + name: hook1 + language: system + entry: echo hook1 + stages: [commit, push] +- id: hook2 + name: hook2 + language: system + entry: echo hook2 + stages: [push, merge-commit] +''' + + path = git_dir(tempdir_factory) + with open(os.path.join(path, C.MANIFEST_FILE), 'w') as f: + f.write(manifest) + cmd_output('git', 'add', '.', cwd=path) + git_commit(cwd=path) + rev = git.head_rev(path) + + store.clone(path, rev) + assert caplog.record_tuples[1] == ( + 'pre_commit', + logging.WARNING, + f'repo `{path}` uses deprecated stage names ' + f'(commit, push, merge-commit) which will be removed in a future ' + f'version. ' + f'Hint: often `pre-commit autoupdate --repo {shlex.quote(path)}` ' + f'will fix this. ' + f'if it does not -- consider reporting an issue to that repo.', + ) + + # should not re-warn + caplog.clear() + store.clone(path, rev) + assert caplog.record_tuples == [] + + +def test_no_warning_for_non_deprecated_stages_on_init( + store, tempdir_factory, caplog, +): + manifest = '''\ +- id: hook1 + name: hook1 + language: system + entry: echo hook1 + stages: [pre-commit, pre-push] +- id: hook2 + name: hook2 + language: system + entry: echo hook2 + stages: [pre-push, pre-merge-commit] +''' + + path = git_dir(tempdir_factory) + with open(os.path.join(path, C.MANIFEST_FILE), 'w') as f: + f.write(manifest) + cmd_output('git', 'add', '.', cwd=path) + git_commit(cwd=path) + rev = git.head_rev(path) + + store.clone(path, rev) + assert logging.WARNING not in {tup[1] for tup in caplog.record_tuples} + + def test_clone_cleans_up_on_checkout_failure(store): with pytest.raises(Exception) as excinfo: # This raises an exception because you can't clone something that @@ -118,7 +187,7 @@ def test_clone_when_repo_already_exists(store): def test_clone_shallow_failure_fallback_to_complete( store, tempdir_factory, - log_info_mock, + caplog, ): path = git_dir(tempdir_factory) with cwd(path): @@ -134,7 +203,7 @@ def fake_shallow_clone(self, *args, **kwargs): ret = store.clone(path, rev) # Should have printed some stuff - assert log_info_mock.call_args_list[0][0][0].startswith( + assert caplog.record_tuples[0][-1].startswith( 'Initializing environment for ', ) @@ -185,7 +254,7 @@ def test_db_repo_name(store): def test_local_resources_reflects_reality(): on_disk = { - res[len('empty_template_'):] + res.removeprefix('empty_template_') for res in os.listdir('pre_commit/resources') if res.startswith('empty_template_') } diff --git a/tests/yaml_rewrite_test.py b/tests/yaml_rewrite_test.py new file mode 100644 index 000000000..d0f6841cf --- /dev/null +++ b/tests/yaml_rewrite_test.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import pytest + +from pre_commit.yaml import yaml_compose +from pre_commit.yaml_rewrite import MappingKey +from pre_commit.yaml_rewrite import MappingValue +from pre_commit.yaml_rewrite import match +from pre_commit.yaml_rewrite import SequenceItem + + +def test_match_produces_scalar_values_only(): + src = '''\ +- name: foo +- name: [not, foo] # not a scalar: should be skipped! +- name: bar +''' + matcher = (SequenceItem(), MappingValue('name')) + ret = [n.value for n in match(yaml_compose(src), matcher)] + assert ret == ['foo', 'bar'] + + +@pytest.mark.parametrize('cls', (MappingKey, MappingValue)) +def test_mapping_not_a_map(cls): + m = cls('s') + assert list(m.match(yaml_compose('[foo]'))) == [] + + +def test_sequence_item_not_a_sequence(): + assert list(SequenceItem().match(yaml_compose('s: val'))) == [] + + +def test_mapping_key(): + m = MappingKey('s') + ret = [n.value for n in m.match(yaml_compose('s: val\nt: val2'))] + assert ret == ['s'] + + +def test_mapping_value(): + m = MappingValue('s') + ret = [n.value for n in m.match(yaml_compose('s: val\nt: val2'))] + assert ret == ['val'] + + +def test_sequence_item(): + ret = [n.value for n in SequenceItem().match(yaml_compose('[a, b, c]'))] + assert ret == ['a', 'b', 'c']