From 8534317f0cc47be3d02aa79dcc35a012d250076b Mon Sep 17 00:00:00 2001 From: tomruk <41700170+tomruk@users.noreply.github.com> Date: Fri, 21 Apr 2023 15:31:58 +0300 Subject: [PATCH 01/42] Add edge case: patterns that end with an escaped space --- pathspec/patterns/gitwildmatch.py | 9 ++++++++- tests/test_pathspec.py | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pathspec/patterns/gitwildmatch.py b/pathspec/patterns/gitwildmatch.py index 94d3115..3f35f2d 100644 --- a/pathspec/patterns/gitwildmatch.py +++ b/pathspec/patterns/gitwildmatch.py @@ -68,7 +68,14 @@ def pattern_to_regex( raise TypeError(f"pattern:{pattern!r} is not a unicode or byte string.") original_pattern = pattern - pattern = pattern.strip() + + if pattern.endswith('\\ '): + # EDGE CASE: Spaces can be escaped with backslash. + # If a pattern that ends with backslash followed by a space, + # only strip from left. + pattern = pattern.lstrip() + else: + pattern = pattern.strip() if pattern.startswith('#'): # A pattern starting with a hash ('#') serves as a comment diff --git a/tests/test_pathspec.py b/tests/test_pathspec.py index 1b900b0..20cbc6b 100644 --- a/tests/test_pathspec.py +++ b/tests/test_pathspec.py @@ -15,6 +15,7 @@ PathSpec) from pathspec.util import ( iter_tree_entries) +from pathspec.patterns.gitwildmatch import GitWildMatchPatternError from tests.util import ( make_dirs, make_files, @@ -119,6 +120,32 @@ def test_01_current_dir_paths(self): './src/test2/c/c.txt', }) + def test_01_empty_path(self): + """ + Tests that patterns that end with an escaped space will be treated properly. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + '\\ ', + 'abc\\ ' + ]) + test_files = [ + ' ', + ' ', + 'abc ', + 'somefile', + ] + results = list(filter(spec.match_file, test_files)) + self.assertEqual(results, [ + ' ', + 'abc ' + ]) + + # An escape with double spaces is invalid. + # Disallow it. Better to be safe than sorry. + self.assertRaises(GitWildMatchPatternError, lambda: PathSpec.from_lines('gitwildmatch', [ + '\\ ' + ])) + def test_01_match_files(self): """ Tests that matching files one at a time yields the same results as From 9302492483b06ca1613aa8b11cf08cef93930b12 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sun, 23 Apr 2023 15:25:43 -0400 Subject: [PATCH 02/42] Fix issue 77 --- CHANGES.rst | 11 ++++++++++- pathspec/_meta.py | 3 ++- pathspec/patterns/gitwildmatch.py | 9 +++++---- tests/test_gitwildmatch.py | 25 +++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f37c491..825dac8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,16 @@ Change History ============== +0.11.2 (TBD) +------------ + +Bug fixes: + +- `Issue #77`_: On bracket expression negation. + +.. _`Issue #77`: https://github.com/cpburnz/python-pathspec/issues/77 + + 0.11.1 (2023-03-14) ------------------- @@ -19,7 +29,6 @@ Improvements: .. _`Pull #75`: https://github.com/cpburnz/python-pathspec/pull/75 - 0.11.0 (2023-01-24) ------------------- diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 80942a5..3f417db 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -50,6 +50,7 @@ "Avasam ", "yschroeder ", "axesider ", + "tomruk ", ] __license__ = "MPL 2.0" -__version__ = "0.11.1" +__version__ = "0.11.2.dev1" diff --git a/pathspec/patterns/gitwildmatch.py b/pathspec/patterns/gitwildmatch.py index 94d3115..8593872 100644 --- a/pathspec/patterns/gitwildmatch.py +++ b/pathspec/patterns/gitwildmatch.py @@ -306,16 +306,17 @@ def _translate_segment_glob(pattern: str) -> str: expr = '[' if pattern[i] == '!': - # Braket expression needs to be negated. + # Bracket expression needs to be negated. expr += '^' i += 1 elif pattern[i] == '^': # POSIX declares that the regex bracket expression negation # "[^...]" is undefined in a glob pattern. Python's # `fnmatch.translate()` escapes the caret ('^') as a - # literal. To maintain consistency with undefined behavior, - # I am escaping the '^' as well. - expr += '\\^' + # literal. Git supports the using a caret for negation. + # Maintain consistency with Git because that is the expected + # behavior. + expr += '^' i += 1 # Build regex bracket expression. Escape slashes so they are diff --git a/tests/test_gitwildmatch.py b/tests/test_gitwildmatch.py index 7dccaee..6a51683 100644 --- a/tests/test_gitwildmatch.py +++ b/tests/test_gitwildmatch.py @@ -773,3 +773,28 @@ def test_12_asterisk_4_descendant(self): self.assertEqual(results, { 'anydir/file.txt', }) + + def test_13_issue_77_1_regex(self): + """ + Test the resulting regex for regex bracket expression negation. + """ + regex, include = GitWildMatchPattern.pattern_to_regex('a[^b]c') + self.assertTrue(include) + + equiv_regex, include = GitWildMatchPattern.pattern_to_regex('a[!b]c') + self.assertTrue(include) + + self.assertEqual(regex, equiv_regex) + + def test_13_issue_77_2_results(self): + """ + Test that regex bracket expression negation works. + """ + pattern = GitWildMatchPattern('a[^b]c') + results = set(filter(pattern.match_file, [ + 'abc', + 'azc', + ])) + self.assertEqual(results, { + 'azc', + }) From b9a014e560af033591d8cfe7734deb929cc52f67 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sun, 23 Apr 2023 16:19:57 -0400 Subject: [PATCH 03/42] Update CHANGES --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 825dac8..f5a473b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,8 +8,11 @@ Change History Bug fixes: +- `Pull #76`_: Add edge case: patterns that end with an escaped space - `Issue #77`_: On bracket expression negation. + +.. _`Pull #76`: https://github.com/cpburnz/python-pathspec/pull/76 .. _`Issue #77`: https://github.com/cpburnz/python-pathspec/issues/77 @@ -25,6 +28,7 @@ Improvements: - `Pull #75`_: Fix partially unknown PathLike type. - Convert `os.PathLike` to a string properly using `os.fspath`. + .. _`Issue #74`: https://github.com/cpburnz/python-pathspec/issues/74 .. _`Pull #75`: https://github.com/cpburnz/python-pathspec/pull/75 From 6b58e23b6038051fb2d690691912bca3dffcde97 Mon Sep 17 00:00:00 2001 From: tomruk <41700170+tomruk@users.noreply.github.com> Date: Mon, 24 Apr 2023 10:58:41 +0300 Subject: [PATCH 04/42] Negate with caret symbol as with the exclamation mark --- pathspec/patterns/gitwildmatch.py | 12 ++---------- tests/test_gitwildmatch.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/pathspec/patterns/gitwildmatch.py b/pathspec/patterns/gitwildmatch.py index 3f35f2d..8701762 100644 --- a/pathspec/patterns/gitwildmatch.py +++ b/pathspec/patterns/gitwildmatch.py @@ -312,18 +312,10 @@ def _translate_segment_glob(pattern: str) -> str: j += 1 expr = '[' - if pattern[i] == '!': - # Braket expression needs to be negated. + if pattern[i] == '!' or pattern[i] == '^': + # Bracket expression needs to be negated. expr += '^' i += 1 - elif pattern[i] == '^': - # POSIX declares that the regex bracket expression negation - # "[^...]" is undefined in a glob pattern. Python's - # `fnmatch.translate()` escapes the caret ('^') as a - # literal. To maintain consistency with undefined behavior, - # I am escaping the '^' as well. - expr += '\\^' - i += 1 # Build regex bracket expression. Escape slashes so they are # treated as literal slashes by regex as defined by POSIX. diff --git a/tests/test_gitwildmatch.py b/tests/test_gitwildmatch.py index 7dccaee..2eaab6a 100644 --- a/tests/test_gitwildmatch.py +++ b/tests/test_gitwildmatch.py @@ -773,3 +773,29 @@ def test_12_asterisk_4_descendant(self): self.assertEqual(results, { 'anydir/file.txt', }) + + def test_13_negate_with_caret(self): + """ + Test negation using the caret symbol (^) + """ + pattern = GitWildMatchPattern("a[^gy]c") + results = set(filter(pattern.match_file, [ + "agc", + "ayc", + "abc", + "adc", + ])) + self.assertEqual(results, {"abc", "adc"}) + + def test_13_negate_with_exclamation_mark(self): + """ + Test negation using the exclamation mark (!) + """ + pattern = GitWildMatchPattern("a[!gy]c") + results = set(filter(pattern.match_file, [ + "agc", + "ayc", + "abc", + "adc", + ])) + self.assertEqual(results, {"abc", "adc"}) From 57fbd3ed69597d35a50c56446d60e2c17ab04baf Mon Sep 17 00:00:00 2001 From: tomruk <41700170+tomruk@users.noreply.github.com> Date: Mon, 24 Apr 2023 11:22:18 +0300 Subject: [PATCH 05/42] Pass caret negation --- pathspec/patterns/gitwildmatch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pathspec/patterns/gitwildmatch.py b/pathspec/patterns/gitwildmatch.py index 8701762..8276b00 100644 --- a/pathspec/patterns/gitwildmatch.py +++ b/pathspec/patterns/gitwildmatch.py @@ -290,8 +290,8 @@ def _translate_segment_glob(pattern: str) -> str: # - "[]-]" matches ']' and '-'. # - "[!]a-]" matches any character except ']', 'a' and '-'. j = i - # Pass brack expression negation. - if j < end and pattern[j] == '!': + # Pass bracket expression negation. + if j < end and (pattern[j] == '!' or pattern[j] == '^'): j += 1 # Pass first closing bracket if it is at the beginning of the # expression. From dfb630b7bd42956bdd6834c651806d010e01e5f6 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Mon, 24 Apr 2023 19:24:13 -0400 Subject: [PATCH 06/42] Update CHANGES --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f5a473b..922889c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,11 +9,12 @@ Change History Bug fixes: - `Pull #76`_: Add edge case: patterns that end with an escaped space -- `Issue #77`_: On bracket expression negation. +- `Issue #77`_/`Pull #78`_: On bracket expression negation. .. _`Pull #76`: https://github.com/cpburnz/python-pathspec/pull/76 .. _`Issue #77`: https://github.com/cpburnz/python-pathspec/issues/77 +.. _`Pull #78`: https://github.com/cpburnz/python-pathspec/pull/78/ 0.11.1 (2023-03-14) From 933dd7da982551300a584c98570993402a56bc27 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Mon, 24 Apr 2023 19:27:54 -0400 Subject: [PATCH 07/42] Update CHANGES --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 922889c..f8ff9c5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,7 +9,7 @@ Change History Bug fixes: - `Pull #76`_: Add edge case: patterns that end with an escaped space -- `Issue #77`_/`Pull #78`_: On bracket expression negation. +- `Issue #77`_/`Pull #78`_: Negate with caret symbol as with the exclamation mark. .. _`Pull #76`: https://github.com/cpburnz/python-pathspec/pull/76 From fb2246c0b50b95803b8abee2b2bfbdca39850918 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:02:27 -0400 Subject: [PATCH 08/42] Implement issue 80 --- CHANGES.rst | 5 ++ pathspec/_meta.py | 1 + pathspec/pathspec.py | 116 ++++++++++++++++++++++++++--------------- tests/test_pathspec.py | 50 ++++++++++++++++++ tox.ini | 2 +- 5 files changed, 131 insertions(+), 43 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f8ff9c5..9db9535 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,10 @@ Change History 0.11.2 (TBD) ------------ +New features: + +- `Issue #80`_: match_files with negated path spec. `pathspec.PathSpec.match_*()` now have a `negate` parameter to make using *.gitignore* logic easier and more efficient. + Bug fixes: - `Pull #76`_: Add edge case: patterns that end with an escaped space @@ -15,6 +19,7 @@ Bug fixes: .. _`Pull #76`: https://github.com/cpburnz/python-pathspec/pull/76 .. _`Issue #77`: https://github.com/cpburnz/python-pathspec/issues/77 .. _`Pull #78`: https://github.com/cpburnz/python-pathspec/pull/78/ +.. _`Issue #80`: https://github.com/cpburnz/python-pathspec/issues/80 0.11.1 (2023-03-14) diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 3f417db..4e45b5a 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -51,6 +51,7 @@ "yschroeder ", "axesider ", "tomruk ", + "oprypin ", ] __license__ = "MPL 2.0" __version__ = "0.11.2.dev1" diff --git a/pathspec/pathspec.py b/pathspec/pathspec.py index e66331b..93f2f60 100644 --- a/pathspec/pathspec.py +++ b/pathspec/pathspec.py @@ -1,15 +1,11 @@ """ -This module provides an object oriented interface for pattern matching -of files. +This module provides an object oriented interface for pattern matching of files. """ -import sys from collections.abc import ( Collection as CollectionType) from itertools import ( zip_longest) -from os import ( - PathLike) from typing import ( AnyStr, Callable, @@ -107,15 +103,15 @@ def from_lines( """ Compiles the pattern lines. - *pattern_factory* can be either the name of a registered pattern - factory (:class:`str`), or a :class:`~collections.abc.Callable` used - to compile patterns. It must accept an uncompiled pattern (:class:`str`) - and return the compiled pattern (:class:`.Pattern`). + *pattern_factory* can be either the name of a registered pattern factory + (:class:`str`), or a :class:`~collections.abc.Callable` used to compile + patterns. It must accept an uncompiled pattern (:class:`str`) and return the + compiled pattern (:class:`.Pattern`). - *lines* (:class:`~collections.abc.Iterable`) yields each uncompiled - pattern (:class:`str`). This simply has to yield each line so it can - be a :class:`io.TextIOBase` (e.g., from :func:`open` or - :class:`io.StringIO`) or the result from :meth:`str.splitlines`. + *lines* (:class:`~collections.abc.Iterable`) yields each uncompiled pattern + (:class:`str`). This simply has to yield each line so that it can be a + :class:`io.TextIOBase` (e.g., from :func:`open` or :class:`io.StringIO`) or + the result from :meth:`str.splitlines`. Returns the :class:`PathSpec` instance. """ @@ -135,6 +131,8 @@ def match_entries( self, entries: Iterable[TreeEntry], separators: Optional[Collection[str]] = None, + *, + negate: Optional[bool] = None, ) -> Iterator[TreeEntry]: """ Matches the entries to this path-spec. @@ -142,10 +140,14 @@ def match_entries( *entries* (:class:`~collections.abc.Iterable` of :class:`~util.TreeEntry`) contains the entries to be matched against :attr:`self.patterns `. - *separators* (:class:`~collections.abc.Collection` of :class:`str`; - or :data:`None`) optionally contains the path separators to - normalize. See :func:`~pathspec.util.normalize_file` for more - information. + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. Returns the matched entries (:class:`~collections.abc.Iterator` of :class:`~util.TreeEntry`). @@ -156,12 +158,17 @@ def match_entries( use_patterns = _filter_patterns(self.patterns) for entry in entries: norm_file = normalize_file(entry.path, separators) - if self._match_file(use_patterns, norm_file): + is_match = self._match_file(use_patterns, norm_file) + + if negate: + is_match = not is_match + + if is_match: yield entry - # Match files using the `match_file()` utility function. Subclasses - # may override this method as an instance method. It does not have to - # be a static method. + # Match files using the `match_file()` utility function. Subclasses may + # override this method as an instance method. It does not have to be a static + # method. _match_file = staticmethod(match_file) def match_file( @@ -188,6 +195,8 @@ def match_files( self, files: Iterable[StrPath], separators: Optional[Collection[str]] = None, + *, + negate: Optional[bool] = None, ) -> Iterator[StrPath]: """ Matches the files to this path-spec. @@ -196,10 +205,14 @@ def match_files( :class:`os.PathLike[str]`) contains the file paths to be matched against :attr:`self.patterns `. - *separators* (:class:`~collections.abc.Collection` of :class:`str`; - or :data:`None`) optionally contains the path separators to - normalize. See :func:`~pathspec.util.normalize_file` for more - information. + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. Returns the matched files (:class:`~collections.abc.Iterator` of :class:`str` or :class:`os.PathLike[str]`). @@ -210,7 +223,12 @@ def match_files( use_patterns = _filter_patterns(self.patterns) for orig_file in files: norm_file = normalize_file(orig_file, separators) - if self._match_file(use_patterns, norm_file): + is_match = self._match_file(use_patterns, norm_file) + + if negate: + is_match = not is_match + + if is_match: yield orig_file def match_tree_entries( @@ -218,55 +236,69 @@ def match_tree_entries( root: StrPath, on_error: Optional[Callable] = None, follow_links: Optional[bool] = None, + *, + negate: Optional[bool] = None, ) -> Iterator[TreeEntry]: """ Walks the specified root path for all files and matches them to this path-spec. - *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory - to search. + *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory to + search. - *on_error* (:class:`~collections.abc.Callable` or :data:`None`) - optionally is the error handler for file-system exceptions. See + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally + is the error handler for file-system exceptions. See :func:`~pathspec.util.iter_tree_entries` for more information. - *follow_links* (:class:`bool` or :data:`None`) optionally is whether - to walk symbolic links that resolve to directories. See + *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk + symbolic links that resolve to directories. See :func:`~pathspec.util.iter_tree_files` for more information. + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + Returns the matched files (:class:`~collections.abc.Iterator` of :class:`.TreeEntry`). """ entries = util.iter_tree_entries(root, on_error=on_error, follow_links=follow_links) - yield from self.match_entries(entries) + yield from self.match_entries(entries, negate=negate) def match_tree_files( self, root: StrPath, on_error: Optional[Callable] = None, follow_links: Optional[bool] = None, + *, + negate: Optional[bool] = None, ) -> Iterator[str]: """ Walks the specified root path for all files and matches them to this path-spec. - *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory - to search for files. + *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory to + search for files. - *on_error* (:class:`~collections.abc.Callable` or :data:`None`) - optionally is the error handler for file-system exceptions. See + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally + is the error handler for file-system exceptions. See :func:`~pathspec.util.iter_tree_files` for more information. - *follow_links* (:class:`bool` or :data:`None`) optionally is whether - to walk symbolic links that resolve to directories. See + *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk + symbolic links that resolve to directories. See :func:`~pathspec.util.iter_tree_files` for more information. + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + Returns the matched files (:class:`~collections.abc.Iterable` of :class:`str`). """ files = util.iter_tree_files(root, on_error=on_error, follow_links=follow_links) - yield from self.match_files(files) + yield from self.match_files(files, negate=negate) - # Alias `match_tree_files()` as `match_tree()` for backward - # compatibility before v0.3.2. + # Alias `match_tree_files()` as `match_tree()` for backward compatibility + # before v0.3.2. match_tree = match_tree_files diff --git a/tests/test_pathspec.py b/tests/test_pathspec.py index 20cbc6b..2e7c8f0 100644 --- a/tests/test_pathspec.py +++ b/tests/test_pathspec.py @@ -579,3 +579,53 @@ def test_08_issue_39(self): 'important/d.log', 'important/e.txt', }) + + def test_09_issue_80_a(self): + """ + Test negating patterns. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + 'build', + '*.log', + '.*', + '!.gitignore', + ]) + files = { + '.c-tmp', + '.gitignore', + 'a.log', + 'b.txt', + 'build/d.log', + 'build/trace.bin', + 'trace.c', + } + keeps = set(spec.match_files(files, negate=True)) + self.assertEqual(keeps, { + '.gitignore', + 'b.txt', + 'trace.c', + }) + + def test_09_issue_80_b(self): + """ + Test negating patterns. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + 'build', + '*.log', + '.*', + '!.gitignore', + ]) + files = { + '.c-tmp', + '.gitignore', + 'a.log', + 'b.txt', + 'build/d.log', + 'build/trace.bin', + 'trace.c', + } + keeps = set(spec.match_files(files, negate=True)) + ignores = set(spec.match_files(files)) + self.assertEqual(files - ignores, keeps) + self.assertEqual(files - keeps, ignores) diff --git a/tox.ini b/tox.ini index 9b820e3..5d039ae 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, py311, pypy3 +envlist = py37, py38, py39, py310, py311, py312, pypy3 isolated_build = True [testenv] From c0aca9fa0fe4aaa7967beb717f726ed36d9ab2fe Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:04:01 -0400 Subject: [PATCH 09/42] Release v0.11.2 --- CHANGES.rst | 4 ++-- pathspec/_meta.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9db9535..6ab1ca1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,8 +3,8 @@ Change History ============== -0.11.2 (TBD) ------------- +0.11.2 (2023-07-28) +------------------- New features: diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 4e45b5a..6cba91d 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -54,4 +54,4 @@ "oprypin ", ] __license__ = "MPL 2.0" -__version__ = "0.11.2.dev1" +__version__ = "0.11.2" From 878be226c5324a4c5470c2ff86034d27c0734d70 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:04:37 -0400 Subject: [PATCH 10/42] Release v0.11.2 --- README-dist.rst | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README-dist.rst b/README-dist.rst index bfe0e13..5eca0da 100644 --- a/README-dist.rst +++ b/README-dist.rst @@ -170,6 +170,25 @@ Change History ============== +0.11.2 (2023-07-28) +------------------- + +New features: + +- `Issue #80`_: match_files with negated path spec. `pathspec.PathSpec.match_*()` now have a `negate` parameter to make using *.gitignore* logic easier and more efficient. + +Bug fixes: + +- `Pull #76`_: Add edge case: patterns that end with an escaped space +- `Issue #77`_/`Pull #78`_: Negate with caret symbol as with the exclamation mark. + + +.. _`Pull #76`: https://github.com/cpburnz/python-pathspec/pull/76 +.. _`Issue #77`: https://github.com/cpburnz/python-pathspec/issues/77 +.. _`Pull #78`: https://github.com/cpburnz/python-pathspec/pull/78/ +.. _`Issue #80`: https://github.com/cpburnz/python-pathspec/issues/80 + + 0.11.1 (2023-03-14) ------------------- @@ -182,11 +201,11 @@ Improvements: - `Pull #75`_: Fix partially unknown PathLike type. - Convert `os.PathLike` to a string properly using `os.fspath`. + .. _`Issue #74`: https://github.com/cpburnz/python-pathspec/issues/74 .. _`Pull #75`: https://github.com/cpburnz/python-pathspec/pull/75 - 0.11.0 (2023-01-24) ------------------- From 1af9902b55f8e100ceceeaa43ad8dcaed2d3b363 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 4 Sep 2023 23:24:03 +0300 Subject: [PATCH 11/42] Add support for Python 3.12 --- .github/workflows/ci.yaml | 5 +++-- pyproject.toml | 1 + setup.cfg | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a31480c..199874d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,16 +16,17 @@ jobs: fail-fast: false matrix: os: [ubuntu, macos, windows] - python: ["3.7", "3.8", "3.9", "3.10", "3.11.0-rc - 3.11", + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.7", "pypy-3.8", "pypy-3.9"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Install tox run: python -m pip install tox diff --git a/pyproject.toml b/pyproject.toml index 81ee2be..75f51d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", diff --git a/setup.cfg b/setup.cfg index 6e83c10..863d85f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Topic :: Software Development :: Libraries :: Python Modules From e4d509c7e2e1f05685c48d0bd462c5e6efd6afb8 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 4 Sep 2023 23:26:17 +0300 Subject: [PATCH 12/42] Drop support for EOL Python 3.7 --- .github/workflows/ci.yaml | 4 ++-- pyproject.toml | 3 +-- setup.cfg | 3 +-- tox.ini | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 199874d..3d8bbff 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,8 +16,8 @@ jobs: fail-fast: false matrix: os: [ubuntu, macos, windows] - python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", - "pypy-3.7", "pypy-3.8", "pypy-3.9"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12", + "pypy-3.8", "pypy-3.9"] steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index 75f51d2..1d65322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -29,7 +28,7 @@ dynamic = ["version"] license = {text = "MPL 2.0"} name = "pathspec" readme = "README-dist.rst" -requires-python = ">=3.7" +requires-python = ">=3.8" [project.urls] "Source Code" = "https://github.com/cpburnz/python-pathspec" diff --git a/setup.cfg b/setup.cfg index 863d85f..34abfd1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,6 @@ classifiers = Operating System :: OS Independent Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 @@ -28,7 +27,7 @@ version = attr: pathspec._meta.__version__ [options] packages = find: -python_requires = >=3.7 +python_requires = >=3.8 setup_requires = setuptools>=40.8.0 test_suite = tests diff --git a/tox.ini b/tox.ini index 5d039ae..89f20bd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, py311, py312, pypy3 +envlist = py38, py39, py310, py311, py312, pypy3 isolated_build = True [testenv] From 1ce7369c866c252df5423e321bea65da693abe4a Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Thu, 7 Sep 2023 00:22:53 -0400 Subject: [PATCH 13/42] Notes --- DEV.md | 122 ++++++++++++++++++++++++++++++++++++++++++++++ pathspec/_meta.py | 2 +- 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 DEV.md diff --git a/DEV.md b/DEV.md new file mode 100644 index 0000000..51727a0 --- /dev/null +++ b/DEV.md @@ -0,0 +1,122 @@ + +Development Notes +================= + +Python Versions +--------------- + +These are notes to myself for things to review before decommissioning EoL versions of Python. + + +### Python + +**Python 3.7:** + +- EoL as of 2023-06-27. + +**Python 3.8:** + +- Becomes EoL in 2024-10. + +References: + +- [Status of Python Versions](https://devguide.python.org/versions/) + + +### Linux + +Review the following Linux distributions. + +**CentOS:** + +- TODO + +**Debian:** + +- Goal: + - Support stable release. +- Debian 12 "Bookworm": + - Current stable release as of 2023-09-06. + - EoL date TBD. + - Uses Python 3.11. +- References: + - [Debian Releases](https://wiki.debian.org/DebianReleases) + - Package: [python3](https://packages.debian.org/stable/python3) + - Package: [python3-pathspec](https://packages.debian.org/stable/python3-pathspec) + +**Fedora:** + +- Goal: + - Support oldest supported release. +- Fedora 37: + - Oldest supported release as of 2023-09-06. + - Becomes EoL on 2023-11-14. + - Uses Python 3.11. +- References: + - [End of Life Releases +](https://docs.fedoraproject.org/en-US/releases/eol/) + - [Fedora Linux 39 Schedule: Key +](https://fedorapeople.org/groups/schedule/f-39/f-39-key-tasks.html) + - [Python](https://docs.fedoraproject.org/en-US/fedora/f37/release-notes/developers/Development_Python/) + - Package: [python-pathspec](https://src.fedoraproject.org/rpms/python-pathspec) + +**Gentoo:** + +- Uses Python 3.10+. +- References: + - Package: [pathspec](https://packages.gentoo.org/packages/dev-python/pathspec) + +**RHEL via Fedora EPEL:** + +- RHEL 9: + - Uses Python 3.9. +- References: + - [Chapter 1. Introduction to Python](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/installing_and_using_dynamic_programming_languages/assembly_introduction-to-python_installing-and-using-dynamic-programming-languages#con_python-versions_assembly_introduction-to-python) + - Package: [python-pathspec](https://src.fedoraproject.org/rpms/python-pathspec) + +**Ubuntu:** + +- Goal: + - Support oldest LTS release in standard support. +- Ubuntu 20.04 "Focal Fossa": + - Oldest LTS release in standard support as of 2023-09-06. + - Ends standard support in 2025-04. + - Package is outdated (v0.7.0 from 2019-12-27; as of 2023-09-06). + - Uses Python 3.8. +- Ubuntu 22.04 "Jammy Jellyfish": + - Latest LTS release as of 2023-09-06. + - Ends standard support in 2027-04. + - Package is outdated (v0.9.0 from 2021-07-17; as of 2023-09-06). + - Uses Python 3.10. +- References: + - [Releases](https://wiki.ubuntu.com/Releases) + - Package: [python3](https://packages.ubuntu.com/focal/python3) (focal) + - Package: [python3](https://packages.ubuntu.com/jammy/python3) (jammy) + - Package: [python3-pathspec](https://packages.ubuntu.com/focal/python3-pathspec) (flocal) + - Package: [python3-pathspec](https://packages.ubuntu.com/jammy/python3-pathspec) (jammy) + + +### PyPI + +Review the following PyPI packages. + +[ansible-lint](https://pypi.org/project/ansible-lint/) + +- v6.19.0 (latest as of 2023-09-06) requires Python 3.9+. +- [ansible-lint on Wheelodex](https://www.wheelodex.org/projects/ansible-lint/). + +[black](https://pypi.org/project/black/) + +- v23.7.0 (latest as of 2023-09-06) requires Python 3.8+. +- [black on Wheelodex](https://www.wheelodex.org/projects/black/). + + +[hatchling](https://pypi.org/project/hatchling/) + +- v1.18.0 (latest as of 2023-09-06) requires Python 3.8+. +- [hatchling on Wheelodex](https://www.wheelodex.org/projects/hatchling/). + +[yamllint](https://pypi.org/project/yamllint/) + +- v1.32.0 (latest as of 2023-09-06) requires Python 3.7+. +- [yamllint on Wheelodex](https://www.wheelodex.org/projects/yamllint/). diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 6cba91d..3aa1890 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -54,4 +54,4 @@ "oprypin ", ] __license__ = "MPL 2.0" -__version__ = "0.11.2" +__version__ = "0.11.3.dev1" From 072788fe559f3264353b541cb512725dec4a4b13 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Thu, 7 Sep 2023 00:28:11 -0400 Subject: [PATCH 14/42] Notes --- DEV.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DEV.md b/DEV.md index 51727a0..fb646b4 100644 --- a/DEV.md +++ b/DEV.md @@ -68,7 +68,11 @@ Review the following Linux distributions. **RHEL via Fedora EPEL:** +- Goal: + - Support oldest release with recent version of *python-pathspec* package. - RHEL 9: + - Oldest release with recent version of *python-pathspec* package (v0.10.1 from 2022-09-02; as of 2023-09-07). + - Ends full support on 2027-05-31. - Uses Python 3.9. - References: - [Chapter 1. Introduction to Python](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9/html/installing_and_using_dynamic_programming_languages/assembly_introduction-to-python_installing-and-using-dynamic-programming-languages#con_python-versions_assembly_introduction-to-python) From 59aa2bed4c5a9d2fb4debc491a355acaf6bdbfea Mon Sep 17 00:00:00 2001 From: "Caleb P. Burns" <2126043+cpburnz@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:52:31 -0400 Subject: [PATCH 15/42] Update ci.yaml Add PyPy 3.10 to CI --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3d8bbff..43f3f97 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,8 +16,7 @@ jobs: fail-fast: false matrix: os: [ubuntu, macos, windows] - python: ["3.8", "3.9", "3.10", "3.11", "3.12", - "pypy-3.8", "pypy-3.9"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.8", "pypy-3.9", "pypy-3.10"] steps: - uses: actions/checkout@v4 From e549c727f010e1c1df7051481897dce23a97e5a3 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Thu, 7 Sep 2023 22:54:14 -0400 Subject: [PATCH 16/42] Test fix for PyPy failure on Windows --- tests/test_util.py | 2 +- tests/util.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index 5a036a0..033c745 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -205,7 +205,7 @@ def test_2_2_links(self): ('bx', 'b'), ('Dir/cx', 'Dir/c'), ('Dir/dx', 'Dir/d'), - ('DirX', 'Dir'), + ('DirX', 'Dir', True), ]) results = set(iter_tree_files(self.temp_dir)) self.assertEqual(results, set(map(ospath, [ diff --git a/tests/util.py b/tests/util.py index 301427b..29b95b6 100644 --- a/tests/util.py +++ b/tests/util.py @@ -48,10 +48,11 @@ def make_links(temp_dir: pathlib.Path, links: Iterable[Tuple[str, str]]) -> None the destination link path (:class:`str`) and source node path (:class:`str`). """ - for link, node in links: + for link, node, *opt in links: src = temp_dir / ospath(node) dest = temp_dir / ospath(link) - os.symlink(src, dest) + is_dir = opt[0] if opt else False + os.symlink(src, dest, target_is_directory=is_dir) def mkfile(file: pathlib.Path) -> None: From 280c43422b4bf5c70cecb0ddd7f0a98b30f1f5aa Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Thu, 7 Sep 2023 23:01:51 -0400 Subject: [PATCH 17/42] Revert "Test fix for PyPy failure on Windows" This reverts commit e549c727f010e1c1df7051481897dce23a97e5a3. --- tests/test_util.py | 2 +- tests/util.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index 033c745..5a036a0 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -205,7 +205,7 @@ def test_2_2_links(self): ('bx', 'b'), ('Dir/cx', 'Dir/c'), ('Dir/dx', 'Dir/d'), - ('DirX', 'Dir', True), + ('DirX', 'Dir'), ]) results = set(iter_tree_files(self.temp_dir)) self.assertEqual(results, set(map(ospath, [ diff --git a/tests/util.py b/tests/util.py index 29b95b6..301427b 100644 --- a/tests/util.py +++ b/tests/util.py @@ -48,11 +48,10 @@ def make_links(temp_dir: pathlib.Path, links: Iterable[Tuple[str, str]]) -> None the destination link path (:class:`str`) and source node path (:class:`str`). """ - for link, node, *opt in links: + for link, node in links: src = temp_dir / ospath(node) dest = temp_dir / ospath(link) - is_dir = opt[0] if opt else False - os.symlink(src, dest, target_is_directory=is_dir) + os.symlink(src, dest) def mkfile(file: pathlib.Path) -> None: From 312bcc61ced39f06b070f5ed4128b1c05c395914 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Fri, 8 Sep 2023 00:11:15 -0400 Subject: [PATCH 18/42] Improve testing --- tests/{test_util.py => test_01_util.py} | 133 ++++++++++++------ ...itwildmatch.py => test_02_gitwildmatch.py} | 0 .../{test_pathspec.py => test_03_pathspec.py} | 0 ...test_gitignore.py => test_04_gitignore.py} | 0 4 files changed, 89 insertions(+), 44 deletions(-) rename tests/{test_util.py => test_01_util.py} (81%) rename tests/{test_gitwildmatch.py => test_02_gitwildmatch.py} (100%) rename tests/{test_pathspec.py => test_03_pathspec.py} (100%) rename tests/{test_gitignore.py => test_04_gitignore.py} (100%) diff --git a/tests/test_util.py b/tests/test_01_util.py similarity index 81% rename from tests/test_util.py rename to tests/test_01_util.py index 5a036a0..74cc4f8 100644 --- a/tests/test_util.py +++ b/tests/test_01_util.py @@ -13,6 +13,7 @@ partial) from typing import ( Iterable, + Optional, Tuple) from pathspec.patterns.gitwildmatch import ( @@ -31,36 +32,6 @@ ospath) -class MatchFileTest(unittest.TestCase): - """ - The :class:`MatchFileTest` class tests the :meth:`.match_file` - function. - """ - - def test_1_match_file(self): - """ - Test matching files individually. - """ - patterns = list(map(GitWildMatchPattern, [ - '*.txt', - '!b.txt', - ])) - results = set(filter(partial(match_file, patterns), [ - 'X/a.txt', - 'X/b.txt', - 'X/Z/c.txt', - 'Y/a.txt', - 'Y/b.txt', - 'Y/Z/c.txt', - ])) - self.assertEqual(results, { - 'X/a.txt', - 'X/Z/c.txt', - 'Y/a.txt', - 'Y/Z/c.txt', - }) - - class IterTreeTest(unittest.TestCase): """ The :class:`IterTreeTest` class tests :meth:`.iter_tree_entries` and @@ -85,6 +56,14 @@ def make_links(self, links: Iterable[Tuple[str, str]]) -> None: """ make_links(self.temp_dir, links) + def require_islink_dir(self) -> None: + """ + Skips the test if `os.path.islink` does not properly support symlinks to + directories. + """ + if self.broken_islink_dir: + raise unittest.SkipTest("`os.path.islink` is broken for directories.") + def require_realpath(self) -> None: """ Skips the test if `os.path.realpath` does not properly support @@ -112,7 +91,7 @@ def tearDown(self) -> None: """ shutil.rmtree(self.temp_dir) - def test_1_files(self): + def test_01_files(self): """ Tests to make sure all files are found. """ @@ -139,13 +118,13 @@ def test_1_files(self): 'Dir/Inner/f', ]))) - def test_2_0_check_symlink(self): + def test_02_link_1_check_1_symlink(self): """ Tests whether links can be created. """ # NOTE: Windows Vista and greater supports `os.symlink` for Python # 3.2+. - no_symlink = None + no_symlink: Optional[bool] = None try: file = self.temp_dir / 'file' link = self.temp_dir / 'link' @@ -161,14 +140,14 @@ def test_2_0_check_symlink(self): finally: self.__class__.no_symlink = no_symlink - def test_2_1_check_realpath(self): + def test_02_link_1_check_2_realpath(self): """ Tests whether `os.path.realpath` works properly with symlinks. """ # NOTE: Windows does not follow symlinks with `os.path.realpath` # which is what we use to detect recursion. See # for details. - broken_realpath = None + broken_realpath: Optional[bool] = None try: self.require_symlink() file = self.temp_dir / 'file' @@ -186,11 +165,36 @@ def test_2_1_check_realpath(self): finally: self.__class__.broken_realpath = broken_realpath - def test_2_2_links(self): + def test_02_link_1_check_3_islink(self): + """ + Tests whether `os.path.islink` works properly with symlinks to directories. + """ + # NOTE: PyPy on Windows does not detect symlinks to directories. + # - See + broken_islink_dir: Optional[bool] = None + try: + self.require_symlink() + folder = self.temp_dir / 'folder' + link = self.temp_dir / 'link' + os.mkdir(folder) + os.symlink(folder, link) + + try: + self.assertTrue(os.path.islink(link)) + except AssertionError: + broken_islink_dir = True + else: + broken_islink_dir = False + + finally: + self.__class__.broken_islink_dir = broken_islink_dir + + def test_02_link_2_links(self): """ Tests to make sure links to directories and files work. """ self.require_symlink() + self.require_islink_dir() self.make_dirs([ 'Dir', ]) @@ -223,12 +227,13 @@ def test_2_2_links(self): 'DirX/dx', ]))) - def test_2_3_sideways_links(self): + def test_02_link_3_sideways_links(self): """ Tests to make sure the same directory can be encountered multiple times via links. """ self.require_symlink() + self.require_islink_dir() self.make_dirs([ 'Dir', 'Dir/Target', @@ -259,12 +264,13 @@ def test_2_3_sideways_links(self): 'Dir/Target/file', ]))) - def test_2_4_recursive_links(self): + def test_02_link_4_recursive_links(self): """ Tests detection of recursive links. """ self.require_symlink() self.require_realpath() + self.require_islink_dir() self.make_dirs([ 'Dir', ]) @@ -280,12 +286,13 @@ def test_2_4_recursive_links(self): self.assertEqual(context.exception.first_path, 'Dir') self.assertEqual(context.exception.second_path, ospath('Dir/Self')) - def test_2_5_recursive_circular_links(self): + def test_02_link_5_recursive_circular_links(self): """ Tests detection of recursion through circular links. """ self.require_symlink() self.require_realpath() + self.require_islink_dir() self.make_dirs([ 'A', 'B', @@ -311,7 +318,7 @@ def test_2_5_recursive_circular_links(self): 'C': ospath('C/Ax/Bx/Cx'), }[context.exception.first_path]) - def test_2_6_detect_broken_links(self): + def test_02_link_6_detect_broken_links(self): """ Tests that broken links are detected. """ @@ -327,7 +334,7 @@ def reraise(e): self.assertEqual(context.exception.errno, errno.ENOENT) - def test_2_7_ignore_broken_links(self): + def test_02_link_7_ignore_broken_links(self): """ Tests that broken links are ignored. """ @@ -338,11 +345,12 @@ def test_2_7_ignore_broken_links(self): results = set(iter_tree_files(self.temp_dir)) self.assertEqual(results, set()) - def test_2_8_no_follow_links(self): + def test_02_link_8_no_follow_links(self): """ Tests to make sure directory links can be ignored. """ self.require_symlink() + self.require_islink_dir() self.make_dirs([ 'Dir', ]) @@ -372,7 +380,7 @@ def test_2_8_no_follow_links(self): 'DirX', ]))) - def test_3_entries(self): + def test_03_entries(self): """ Tests to make sure all files are found. """ @@ -402,7 +410,44 @@ def test_3_entries(self): 'Empty', ]))) - def test_4_normalizing_pathlib_path(self): + +class MatchFileTest(unittest.TestCase): + """ + The :class:`MatchFileTest` class tests the :meth:`.match_file` + function. + """ + + def test_01_match_file(self): + """ + Test matching files individually. + """ + patterns = list(map(GitWildMatchPattern, [ + '*.txt', + '!b.txt', + ])) + results = set(filter(partial(match_file, patterns), [ + 'X/a.txt', + 'X/b.txt', + 'X/Z/c.txt', + 'Y/a.txt', + 'Y/b.txt', + 'Y/Z/c.txt', + ])) + self.assertEqual(results, { + 'X/a.txt', + 'X/Z/c.txt', + 'Y/a.txt', + 'Y/Z/c.txt', + }) + + +class NormalizeFileTest(unittest.TestCase): + """ + The :class:`NormalizeFileTest` class tests the :meth:`.normalize_file` + function. + """ + + def test_01_purepath(self): """ Tests normalizing a :class:`pathlib.PurePath` as argument. """ diff --git a/tests/test_gitwildmatch.py b/tests/test_02_gitwildmatch.py similarity index 100% rename from tests/test_gitwildmatch.py rename to tests/test_02_gitwildmatch.py diff --git a/tests/test_pathspec.py b/tests/test_03_pathspec.py similarity index 100% rename from tests/test_pathspec.py rename to tests/test_03_pathspec.py diff --git a/tests/test_gitignore.py b/tests/test_04_gitignore.py similarity index 100% rename from tests/test_gitignore.py rename to tests/test_04_gitignore.py From 0efbb037d2d1c10178448bac2cb2b9ae8b9effd4 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Mon, 11 Sep 2023 19:35:34 -0400 Subject: [PATCH 19/42] realpath bug fixed in Python 3.8 --- tests/test_01_util.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/tests/test_01_util.py b/tests/test_01_util.py index 74cc4f8..ee77ee5 100644 --- a/tests/test_01_util.py +++ b/tests/test_01_util.py @@ -64,14 +64,6 @@ def require_islink_dir(self) -> None: if self.broken_islink_dir: raise unittest.SkipTest("`os.path.islink` is broken for directories.") - def require_realpath(self) -> None: - """ - Skips the test if `os.path.realpath` does not properly support - symlinks. - """ - if self.broken_realpath: - raise unittest.SkipTest("`os.path.realpath` is broken.") - def require_symlink(self) -> None: """ Skips the test if `os.symlink` is not supported. @@ -140,31 +132,6 @@ def test_02_link_1_check_1_symlink(self): finally: self.__class__.no_symlink = no_symlink - def test_02_link_1_check_2_realpath(self): - """ - Tests whether `os.path.realpath` works properly with symlinks. - """ - # NOTE: Windows does not follow symlinks with `os.path.realpath` - # which is what we use to detect recursion. See - # for details. - broken_realpath: Optional[bool] = None - try: - self.require_symlink() - file = self.temp_dir / 'file' - link = self.temp_dir / 'link' - mkfile(file) - os.symlink(file, link) - - try: - self.assertEqual(os.path.realpath(file), os.path.realpath(link)) - except AssertionError: - broken_realpath = True - else: - broken_realpath = False - - finally: - self.__class__.broken_realpath = broken_realpath - def test_02_link_1_check_3_islink(self): """ Tests whether `os.path.islink` works properly with symlinks to directories. @@ -269,7 +236,6 @@ def test_02_link_4_recursive_links(self): Tests detection of recursive links. """ self.require_symlink() - self.require_realpath() self.require_islink_dir() self.make_dirs([ 'Dir', @@ -291,7 +257,6 @@ def test_02_link_5_recursive_circular_links(self): Tests detection of recursion through circular links. """ self.require_symlink() - self.require_realpath() self.require_islink_dir() self.make_dirs([ 'A', From 64ea07e5dfc39d9c52213b91008df353a1af63ef Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Mon, 11 Sep 2023 20:38:42 -0400 Subject: [PATCH 20/42] islink works with pypy on windows in my testing --- tests/test_01_util.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/tests/test_01_util.py b/tests/test_01_util.py index ee77ee5..67c9b41 100644 --- a/tests/test_01_util.py +++ b/tests/test_01_util.py @@ -56,14 +56,6 @@ def make_links(self, links: Iterable[Tuple[str, str]]) -> None: """ make_links(self.temp_dir, links) - def require_islink_dir(self) -> None: - """ - Skips the test if `os.path.islink` does not properly support symlinks to - directories. - """ - if self.broken_islink_dir: - raise unittest.SkipTest("`os.path.islink` is broken for directories.") - def require_symlink(self) -> None: """ Skips the test if `os.symlink` is not supported. @@ -132,36 +124,11 @@ def test_02_link_1_check_1_symlink(self): finally: self.__class__.no_symlink = no_symlink - def test_02_link_1_check_3_islink(self): - """ - Tests whether `os.path.islink` works properly with symlinks to directories. - """ - # NOTE: PyPy on Windows does not detect symlinks to directories. - # - See - broken_islink_dir: Optional[bool] = None - try: - self.require_symlink() - folder = self.temp_dir / 'folder' - link = self.temp_dir / 'link' - os.mkdir(folder) - os.symlink(folder, link) - - try: - self.assertTrue(os.path.islink(link)) - except AssertionError: - broken_islink_dir = True - else: - broken_islink_dir = False - - finally: - self.__class__.broken_islink_dir = broken_islink_dir - def test_02_link_2_links(self): """ Tests to make sure links to directories and files work. """ self.require_symlink() - self.require_islink_dir() self.make_dirs([ 'Dir', ]) @@ -200,7 +167,6 @@ def test_02_link_3_sideways_links(self): times via links. """ self.require_symlink() - self.require_islink_dir() self.make_dirs([ 'Dir', 'Dir/Target', @@ -236,7 +202,6 @@ def test_02_link_4_recursive_links(self): Tests detection of recursive links. """ self.require_symlink() - self.require_islink_dir() self.make_dirs([ 'Dir', ]) @@ -257,7 +222,6 @@ def test_02_link_5_recursive_circular_links(self): Tests detection of recursion through circular links. """ self.require_symlink() - self.require_islink_dir() self.make_dirs([ 'A', 'B', @@ -315,7 +279,6 @@ def test_02_link_8_no_follow_links(self): Tests to make sure directory links can be ignored. """ self.require_symlink() - self.require_islink_dir() self.make_dirs([ 'Dir', ]) From 14fc3d258881e3c34e3c83dc90cbfe35cfb229d5 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 30 Sep 2023 11:49:48 -0400 Subject: [PATCH 21/42] Testing for issue 81 --- DEV.md | 4 +++ tests/test_02_gitwildmatch.py | 55 +++++++++++++++++++++++------------ tests/test_04_gitignore.py | 54 ++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/DEV.md b/DEV.md index fb646b4..ee4d876 100644 --- a/DEV.md +++ b/DEV.md @@ -114,6 +114,10 @@ Review the following PyPI packages. - v23.7.0 (latest as of 2023-09-06) requires Python 3.8+. - [black on Wheelodex](https://www.wheelodex.org/projects/black/). +[dvc](https://github.com/iterative/dvc) + +- v3.23.0 (latest as of 2023-09-30) requires Python 3.8+. +- [dvc on Wheelodex](https://www.wheelodex.org/projects/dvc/). [hatchling](https://pypi.org/project/hatchling/) diff --git a/tests/test_02_gitwildmatch.py b/tests/test_02_gitwildmatch.py index da176bd..b933082 100644 --- a/tests/test_02_gitwildmatch.py +++ b/tests/test_02_gitwildmatch.py @@ -669,10 +669,11 @@ def test_10_escape_pound_start(self): ])) self.assertEqual(results, {"#sign"}) - def test_11_match_directory_1(self): + def test_11_issue_19_directory_a(self): """ - Test matching a directory. + Test a directory discrepancy, scenario A. """ + # NOTE: The result from GitWildMatchPattern will differ from GitIgnoreSpec. pattern = GitWildMatchPattern("dirG/") results = set(filter(pattern.match_file, [ 'fileA', @@ -689,10 +690,11 @@ def test_11_match_directory_1(self): 'dirG/fileO', }) - def test_11_match_directory_2(self): + def test_11_issue_19_directory_b(self): """ - Test matching a directory. + Test a directory discrepancy, scenario B. """ + # NOTE: The result from GitWildMatchPattern will differ from GitIgnoreSpec. pattern = GitWildMatchPattern("dirG/*") results = set(filter(pattern.match_file, [ 'fileA', @@ -709,10 +711,11 @@ def test_11_match_directory_2(self): 'dirG/fileO', }) - def test_11_match_sub_directory_3(self): + def test_11_issue_19_directory_c(self): """ - Test matching a directory. + Test a directory discrepancy, scenario C. """ + # NOTE: The result from GitWildMatchPattern will differ from GitIgnoreSpec. pattern = GitWildMatchPattern("dirG/**") results = set(filter(pattern.match_file, [ 'fileA', @@ -774,21 +777,23 @@ def test_12_asterisk_4_descendant(self): 'anydir/file.txt', }) - def test_13_issue_77_regex(self): + def test_12_issue_62(self): """ - Test the resulting regex for regex bracket expression negation. + Test including all files, scenario A. """ - regex, include = GitWildMatchPattern.pattern_to_regex('a[^b]c') - self.assertTrue(include) - - equiv_regex, include = GitWildMatchPattern.pattern_to_regex('a[!b]c') - self.assertTrue(include) - - self.assertEqual(regex, equiv_regex) + pattern = GitWildMatchPattern('*') + results = set(filter(pattern.match_file, [ + 'file.txt', + 'anydir/file.txt', + ])) + self.assertEqual(results, { + 'file.txt', + 'anydir/file.txt', + }) - def test_13_negate_with_caret(self): + def test_13_issue_77_1_negate_with_caret(self): """ - Test negation using the caret symbol (^) + Test negation using the caret symbol ("^"). """ pattern = GitWildMatchPattern("a[^gy]c") results = set(filter(pattern.match_file, [ @@ -799,9 +804,9 @@ def test_13_negate_with_caret(self): ])) self.assertEqual(results, {"abc", "adc"}) - def test_13_negate_with_exclamation_mark(self): + def test_13_issue_77_1_negate_with_exclamation_mark(self): """ - Test negation using the exclamation mark (!) + Test negation using the exclamation mark ("!"). """ pattern = GitWildMatchPattern("a[!gy]c") results = set(filter(pattern.match_file, [ @@ -811,3 +816,15 @@ def test_13_negate_with_exclamation_mark(self): "adc", ])) self.assertEqual(results, {"abc", "adc"}) + + def test_13_issue_77_2_regex(self): + """ + Test the resulting regex for regex bracket expression negation. + """ + regex, include = GitWildMatchPattern.pattern_to_regex('a[^b]c') + self.assertTrue(include) + + equiv_regex, include = GitWildMatchPattern.pattern_to_regex('a[!b]c') + self.assertTrue(include) + + self.assertEqual(regex, equiv_regex) diff --git a/tests/test_04_gitignore.py b/tests/test_04_gitignore.py index abb5b6a..9478800 100644 --- a/tests/test_04_gitignore.py +++ b/tests/test_04_gitignore.py @@ -388,3 +388,57 @@ def test_07_issue_74(self): 'test2/a.txt', 'test2/c/c.txt', }) + + def test_08_issue_81_a(self): + """ + Test issue 81. + """ + spec = GitIgnoreSpec.from_lines([ + "*", + "!libfoo", + "!libfoo/**", + ]) + files = { + "./libfoo/__init__.py", + } + ignores = set(spec.match_files(files)) + self.assertEqual(ignores, set()) + self.assertEqual(files - ignores, { + "./libfoo/__init__.py", + }) + + def test_08_issue_81_b(self): + """ + Test issue 81. + """ + spec = GitIgnoreSpec.from_lines([ + "*", + "!libfoo", + "!libfoo/*", + ]) + files = { + "./libfoo/__init__.py", + } + ignores = set(spec.match_files(files)) + self.assertEqual(ignores, set()) + self.assertEqual(files - ignores, { + "./libfoo/__init__.py", + }) + + def test_08_issue_81_c(self): + """ + Test issue 81. + """ + spec = GitIgnoreSpec.from_lines([ + "*", + "!libfoo", + "!libfoo/", + ]) + files = { + "./libfoo/__init__.py", + } + ignores = set(spec.match_files(files)) + self.assertEqual(ignores, { + "./libfoo/__init__.py", + }) + self.assertEqual(files - ignores, set()) From 695a6b0bb7afdb3fa1013bdfecb2fc7dc7a09f61 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Fri, 1 Dec 2023 08:10:42 -0600 Subject: [PATCH 22/42] Add a ReadTheDocs configuration file and a docs test environment --- .gitignore | 1 + .readthedocs.yaml | 14 ++++++++++++++ doc/requirements.txt | 1 + tox.ini | 10 +++++++++- 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 .readthedocs.yaml create mode 100644 doc/requirements.txt diff --git a/.gitignore b/.gitignore index 8fe10f2..61f7a75 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ MANIFEST # Hidden files. .* +!.readthedocs.yaml !.gitignore diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..40aafd3 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,14 @@ +version: 2 + +build: + os: 'ubuntu-22.04' + tools: + python: '3.12' + +sphinx: + configuration: 'doc/source/conf.py' + fail_on_warning: true + +python: + install: + - requirements: 'doc/requirements.txt' diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..6fc994d --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1 @@ +Sphinx==7.2.6 diff --git a/tox.ini b/tox.ini index 89f20bd..7d7a17c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,14 @@ [tox] -envlist = py38, py39, py310, py311, py312, pypy3 +envlist = + py{38, 39, 310, 311, 312} + pypy3 + docs isolated_build = True [testenv] commands = python -m unittest {posargs} + +[testenv:docs] +base_path = py312 +deps = -rdoc/requirements.txt +commands = sphinx-build -aWEnqb html doc/source doc/build From 903027cb11631dc3aeb961ce085de7783493695d Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Fri, 1 Dec 2023 08:13:58 -0600 Subject: [PATCH 23/42] Resolve issues identified by the docs tests Most issues boil down to "bad reference or typo". Some were rendering differences between Sphinx 7.2 and the ancient Sphinx 1.8 version ReadTheDocs defaults to when there isn't a `.readthedocs.yaml` file. --- doc/source/api.rst | 1 + doc/source/conf.py | 14 +++++++------- doc/source/index.rst | 1 + pathspec/gitignore.py | 9 +++++---- pathspec/pathspec.py | 14 +++++++------- pathspec/pattern.py | 4 ++-- pathspec/util.py | 26 +++++++++++++------------- 7 files changed, 36 insertions(+), 33 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index 0fb44b8..c62961b 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -1,3 +1,4 @@ +:tocdepth: 2 API === diff --git a/doc/source/conf.py b/doc/source/conf.py index e8c0b53..3e5ef46 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -24,15 +24,15 @@ # -- General configuration ------------------------------------------------ -# If your documentation needs a minimal Sphinx version, state it here. -# -needs_sphinx = '1.2' - # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode'] +# The autodoc extension doesn't understand the `Self` typehint. +# To avoid documentation build errors, autodoc typehints must be disabled. +autodoc_typehints = "none" + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -64,7 +64,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -102,7 +102,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -165,4 +165,4 @@ ] # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} diff --git a/doc/source/index.rst b/doc/source/index.rst index 8437bec..6649eab 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,6 +9,7 @@ Welcome to pathspec's documentation! .. toctree:: :caption: Contents: + :maxdepth: 2 readme api diff --git a/pathspec/gitignore.py b/pathspec/gitignore.py index a939225..fd79223 100644 --- a/pathspec/gitignore.py +++ b/pathspec/gitignore.py @@ -32,14 +32,14 @@ class GitIgnoreSpec(PathSpec): """ - The :class:`GitIgnoreSpec` class extends :class:`PathSpec` to + The :class:`GitIgnoreSpec` class extends :class:`pathspec.pathspec.PathSpec` to replicate *.gitignore* behavior. """ def __eq__(self, other: object) -> bool: """ - Tests the equality of this gitignore-spec with *other* - (:class:`GitIgnoreSpec`) by comparing their :attr:`~PathSpec.patterns` + Tests the equality of this gitignore-spec with *other* (:class:`GitIgnoreSpec`) + by comparing their :attr:`~pathspec.pattern.Pattern` attributes. A non-:class:`GitIgnoreSpec` will not compare equal. """ if isinstance(other, GitIgnoreSpec): @@ -66,7 +66,8 @@ def from_lines( *pattern_factory* can be :data:`None`, the name of a registered pattern factory (:class:`str`), or a :class:`~collections.abc.Callable` used to compile patterns. The callable must accept an uncompiled - pattern (:class:`str`) and return the compiled pattern (:class:`.Pattern`). + pattern (:class:`str`) and return the compiled pattern + (:class:`pathspec.pattern.Pattern`). Default is :data:`None` for :class:`.GitWildMatchPattern`). Returns the :class:`GitIgnoreSpec` instance. diff --git a/pathspec/pathspec.py b/pathspec/pathspec.py index 93f2f60..9153baa 100644 --- a/pathspec/pathspec.py +++ b/pathspec/pathspec.py @@ -137,7 +137,7 @@ def match_entries( """ Matches the entries to this path-spec. - *entries* (:class:`~collections.abc.Iterable` of :class:`~util.TreeEntry`) + *entries* (:class:`~collections.abc.Iterable` of :class:`~pathspec.util.TreeEntry`) contains the entries to be matched against :attr:`self.patterns `. *separators* (:class:`~collections.abc.Collection` of :class:`str`; or @@ -150,7 +150,7 @@ def match_entries( :data:`False`. Returns the matched entries (:class:`~collections.abc.Iterator` of - :class:`~util.TreeEntry`). + :class:`~pathspec.util.TreeEntry`). """ if not _is_iterable(entries): raise TypeError(f"entries:{entries!r} is not an iterable.") @@ -179,7 +179,7 @@ def match_file( """ Matches the file to this path-spec. - *file* (:class:`str` or :class:`os.PathLike[str]`) is the file path to be + *file* (:class:`str` or :class:`os.PathLike`) is the file path to be matched against :attr:`self.patterns `. *separators* (:class:`~collections.abc.Collection` of :class:`str`) @@ -202,7 +202,7 @@ def match_files( Matches the files to this path-spec. *files* (:class:`~collections.abc.Iterable` of :class:`str` or - :class:`os.PathLike[str]`) contains the file paths to be matched against + :class:`os.PathLike`) contains the file paths to be matched against :attr:`self.patterns `. *separators* (:class:`~collections.abc.Collection` of :class:`str`; or @@ -215,7 +215,7 @@ def match_files( :data:`False`. Returns the matched files (:class:`~collections.abc.Iterator` of - :class:`str` or :class:`os.PathLike[str]`). + :class:`str` or :class:`os.PathLike`). """ if not _is_iterable(files): raise TypeError(f"files:{files!r} is not an iterable.") @@ -243,7 +243,7 @@ def match_tree_entries( Walks the specified root path for all files and matches them to this path-spec. - *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory to + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to search. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally @@ -277,7 +277,7 @@ def match_tree_files( Walks the specified root path for all files and matches them to this path-spec. - *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory to + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to search for files. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally diff --git a/pathspec/pattern.py b/pathspec/pattern.py index 5222ec0..8f20e73 100644 --- a/pathspec/pattern.py +++ b/pathspec/pattern.py @@ -51,7 +51,7 @@ def match(self, files: Iterable[str]) -> Iterator[str]: *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains each file relative to the root directory (e.g., - :data:`"relative/path/to/file"`). + ``"relative/path/to/file"``). Returns an :class:`~collections.abc.Iterable` yielding each matched file path (:class:`str`). @@ -160,7 +160,7 @@ def match_file(self, file: str) -> Optional['RegexMatchResult']: *file* (:class:`str`) contains each file relative to the root directory (e.g., "relative/path/to/file"). - Returns the match result (:class:`RegexMatchResult`) if *file* + Returns the match result (:class:`.RegexMatchResult`) if *file* matched; otherwise, :data:`None`. """ if self.include is not None: diff --git a/pathspec/util.py b/pathspec/util.py index 969e3bc..9408f25 100644 --- a/pathspec/util.py +++ b/pathspec/util.py @@ -62,7 +62,7 @@ def append_dir_sep(path: pathlib.Path) -> str: files on the file-system by relying on the presence of a trailing path separator. - *path* (:class:`pathlib.path`) is the path to use. + *path* (:class:`pathlib.Path`) is the path to use. Returns the path (:class:`str`). """ @@ -88,7 +88,7 @@ def detailed_match_files( *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains the normalized file paths to be matched against *patterns*. - *all_matches* (:class:`boot` or :data:`None`) is whether to return all + *all_matches* (:class:`bool` or :data:`None`) is whether to return all matches patterns (:data:`True`), or only the last matched pattern (:data:`False`). Default is :data:`None` for :data:`False`. @@ -154,7 +154,7 @@ def iter_tree_entries( """ Walks the specified directory for all files and directories. - *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory to + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to search. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) @@ -270,7 +270,7 @@ def iter_tree_files( """ Walks the specified directory for all files. - *root* (:class:`str` or :class:`os.PathLike[str]`) is the root directory to + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to search for files. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) @@ -376,16 +376,16 @@ def normalize_file( ) -> str: """ Normalizes the file path to use the POSIX path separator (i.e., - :data:`'/'`), and make the paths relative (remove leading :data:`'/'`). + ``"/"``), and make the paths relative (remove leading ``"/"``). - *file* (:class:`str` or :class:`os.PathLike[str]`) is the file path. + *file* (:class:`str` or :class:`os.PathLike`) is the file path. *separators* (:class:`~collections.abc.Collection` of :class:`str`; or - :data:`None`) optionally contains the path separators to normalize. - This does not need to include the POSIX path separator (:data:`'/'`), - but including it will not affect the results. Default is :data:`None` - for :data:`NORMALIZE_PATH_SEPS`. To prevent normalization, pass an - empty container (e.g., an empty tuple :data:`()`). + ``None``) optionally contains the path separators to normalize. + This does not need to include the POSIX path separator (``"/"``), + but including it will not affect the results. Default is ``None`` + for ``NORMALIZE_PATH_SEPS``. To prevent normalization, pass an + empty container (e.g., an empty tuple ``()``). Returns the normalized file path (:class:`str`). """ @@ -421,7 +421,7 @@ def normalize_files( Normalizes the file paths to use the POSIX path separator. *files* (:class:`~collections.abc.Iterable` of :class:`str` or - :class:`os.PathLike[str]`) contains the file paths to be normalized. + :class:`os.PathLike`) contains the file paths to be normalized. *separators* (:class:`~collections.abc.Collection` of :class:`str`; or :data:`None`) optionally contains the path separators to normalize. @@ -429,7 +429,7 @@ def normalize_files( Returns a :class:`dict` mapping each normalized file path (:class:`str`) to the original file paths (:class:`list` of :class:`str` or - :class:`os.PathLike[str]`). + :class:`os.PathLike`). """ warnings.warn(( "util.normalize_files() is deprecated. Use util.normalize_file() " From 51b0c01dfe4b0082c5f8dbc17296239ebad89fa1 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Fri, 1 Dec 2023 08:17:31 -0600 Subject: [PATCH 24/42] Test doc builds in CI --- .github/workflows/ci.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 43f3f97..b2e2a39 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,3 +32,22 @@ jobs: - name: Run tests run: python -m tox -e py -- --verbose + + docs: + # Test documentation builds. + # This environment mirrors the ReadTheDocs build environment. + name: Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install tox + run: python -m pip install tox + + - name: Run tests + run: python -m tox -e docs From b988d1c0206c5b3dfeedcfd73a896af744943ce9 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Tue, 5 Dec 2023 22:55:09 -0500 Subject: [PATCH 25/42] Improve tests --- CHANGES.rst | 10 +++++ pathspec/_meta.py | 1 + pathspec/gitignore.py | 12 +----- tests/test_02_gitwildmatch.py | 44 +++++++++----------- tests/test_04_gitignore.py | 78 +++++++++++++++++++++-------------- 5 files changed, 78 insertions(+), 67 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6ab1ca1..16cf8d8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,16 @@ Change History ============== +0.11.3 (TDB) +------------ + +Bug fixes: + +- `Pull #83`_: Fix ReadTheDocs builds. + + +.. _`Pull #83`: https://github.com/cpburnz/python-pathspec/pull/83 + 0.11.2 (2023-07-28) ------------------- diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 3aa1890..ab5405a 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -52,6 +52,7 @@ "axesider ", "tomruk ", "oprypin ", + "kurtmckee ", ] __license__ = "MPL 2.0" __version__ = "0.11.3.dev1" diff --git a/pathspec/gitignore.py b/pathspec/gitignore.py index a939225..81bf303 100644 --- a/pathspec/gitignore.py +++ b/pathspec/gitignore.py @@ -18,7 +18,6 @@ Pattern) from .patterns.gitwildmatch import ( GitWildMatchPattern, - GitWildMatchPatternError, _DIR_MARK) from .util import ( _is_iterable) @@ -110,16 +109,7 @@ def _match_file( # Pattern matched. # Check for directory marker. - try: - dir_mark = match.match.group(_DIR_MARK) - except IndexError as e: - # NOTICE: The exact content of this error message is subject - # to change. - raise GitWildMatchPatternError(( - f"Invalid git pattern: directory marker regex group is missing. " - f"Debug: file={file!r} regex={pattern.regex!r} " - f"group={_DIR_MARK!r} match={match.match!r}." - )) from e + dir_mark = match.match.groupdict().get(_DIR_MARK) if dir_mark: # Pattern matched by a directory pattern. diff --git a/tests/test_02_gitwildmatch.py b/tests/test_02_gitwildmatch.py index b933082..be915cf 100644 --- a/tests/test_02_gitwildmatch.py +++ b/tests/test_02_gitwildmatch.py @@ -78,7 +78,7 @@ def test_01_absolute_ignore(self): Tests an ignore absolute path pattern. """ regex, include = GitWildMatchPattern.pattern_to_regex('!/foo/build') - self.assertFalse(include) + self.assertIs(include, False) self.assertEqual(regex, f'^foo/build{RE_SUB}$') # NOTE: The pattern match is backwards because the pattern itself @@ -180,8 +180,7 @@ def test_02_ignore(self): temp/foo """ regex, include = GitWildMatchPattern.pattern_to_regex('!temp') - self.assertIsNotNone(include) - self.assertFalse(include) + self.assertIs(include, False) self.assertEqual(regex, f'^(?:.+/)?temp{RE_SUB}$') # NOTE: The pattern match is backwards because the pattern itself @@ -259,6 +258,7 @@ def test_03_only_double_asterisk(self): regex, include = GitWildMatchPattern.pattern_to_regex('**') self.assertTrue(include) self.assertEqual(regex, f'^[^/]+{RE_SUB}$') + pattern = GitWildMatchPattern(re.compile(regex), include) results = set(filter(pattern.match_file, [ 'x', @@ -757,38 +757,28 @@ def test_12_asterisk_3_child(self): """ Test a relative asterisk path pattern matching a direct child path. """ - pattern = GitWildMatchPattern('*') - results = set(filter(pattern.match_file, [ - 'file.txt', - ])) - self.assertEqual(results, { - 'file.txt', - }) + pattern = GitWildMatchPattern("*") + self.assertTrue(pattern.match_file("file.txt")) def test_12_asterisk_4_descendant(self): """ Test a relative asterisk path pattern matching a descendant path. """ - pattern = GitWildMatchPattern('*') - results = set(filter(pattern.match_file, [ - 'anydir/file.txt', - ])) - self.assertEqual(results, { - 'anydir/file.txt', - }) + pattern = GitWildMatchPattern("*") + self.assertTrue(pattern.match_file("anydir/file.txt")) def test_12_issue_62(self): """ Test including all files, scenario A. """ - pattern = GitWildMatchPattern('*') + pattern = GitWildMatchPattern("*") results = set(filter(pattern.match_file, [ - 'file.txt', - 'anydir/file.txt', + "file.txt", + "anydir/file.txt", ])) self.assertEqual(results, { - 'file.txt', - 'anydir/file.txt', + "file.txt", + "anydir/file.txt", }) def test_13_issue_77_1_negate_with_caret(self): @@ -802,7 +792,10 @@ def test_13_issue_77_1_negate_with_caret(self): "abc", "adc", ])) - self.assertEqual(results, {"abc", "adc"}) + self.assertEqual(results, { + "abc", + "adc", + }) def test_13_issue_77_1_negate_with_exclamation_mark(self): """ @@ -815,7 +808,10 @@ def test_13_issue_77_1_negate_with_exclamation_mark(self): "abc", "adc", ])) - self.assertEqual(results, {"abc", "adc"}) + self.assertEqual(results, { + "abc", + "adc", + }) def test_13_issue_77_2_regex(self): """ diff --git a/tests/test_04_gitignore.py b/tests/test_04_gitignore.py index 9478800..aa3c231 100644 --- a/tests/test_04_gitignore.py +++ b/tests/test_04_gitignore.py @@ -88,18 +88,19 @@ def test_02_issue_41_a(self): Test including a file and excluding a directory with the same name pattern, scenario A. """ + # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ '*.yaml', '!*.yaml/', ]) files = { - 'dir.yaml/file.sql', - 'dir.yaml/file.yaml', - 'dir.yaml/index.txt', - 'dir/file.sql', - 'dir/file.yaml', - 'dir/index.txt', - 'file.yaml', + 'dir.yaml/file.sql', # - + 'dir.yaml/file.yaml', # 1:*.yaml + 'dir.yaml/index.txt', # - + 'dir/file.sql', # - + 'dir/file.yaml', # 1:*.yaml + 'dir/index.txt', # - + 'file.yaml', # 1:*.yaml } ignores = set(spec.match_files(files)) self.assertEqual(ignores, { @@ -119,18 +120,19 @@ def test_02_issue_41_b(self): Test including a file and excluding a directory with the same name pattern, scenario B. """ + # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ '!*.yaml/', '*.yaml', ]) files = { - 'dir.yaml/file.sql', - 'dir.yaml/file.yaml', - 'dir.yaml/index.txt', - 'dir/file.sql', - 'dir/file.yaml', - 'dir/index.txt', - 'file.yaml', + 'dir.yaml/file.sql', # 2:*.yaml + 'dir.yaml/file.yaml', # 2:*.yaml + 'dir.yaml/index.txt', # 2:*.yaml + 'dir/file.sql', # - + 'dir/file.yaml', # 2:*.yaml + 'dir/index.txt', # - + 'file.yaml', # 2:*.yaml } ignores = set(spec.match_files(files)) self.assertEqual(ignores, { @@ -150,18 +152,19 @@ def test_02_issue_41_c(self): Test including a file and excluding a directory with the same name pattern, scenario C. """ + # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ '*.yaml', '!dir.yaml', ]) files = { - 'dir.yaml/file.sql', - 'dir.yaml/file.yaml', - 'dir.yaml/index.txt', - 'dir/file.sql', - 'dir/file.yaml', - 'dir/index.txt', - 'file.yaml', + 'dir.yaml/file.sql', # - + 'dir.yaml/file.yaml', # 1:*.yaml + 'dir.yaml/index.txt', # - + 'dir/file.sql', # - + 'dir/file.yaml', # 1:*.yaml + 'dir/index.txt', # - + 'file.yaml', # 1:*.yaml } ignores = set(spec.match_files(files)) self.assertEqual(ignores, { @@ -391,54 +394,65 @@ def test_07_issue_74(self): def test_08_issue_81_a(self): """ - Test issue 81. + Test issue 81, scenario A. """ + # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ "*", "!libfoo", "!libfoo/**", ]) files = { - "./libfoo/__init__.py", + "ignore.txt", # 1:* + "libfoo/__init__.py", # 3:!libfoo/** } ignores = set(spec.match_files(files)) - self.assertEqual(ignores, set()) + self.assertEqual(ignores, { + "ignore.txt", + }) self.assertEqual(files - ignores, { - "./libfoo/__init__.py", + "libfoo/__init__.py", }) def test_08_issue_81_b(self): """ - Test issue 81. + Test issue 81, scenario B. """ + # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ "*", "!libfoo", "!libfoo/*", ]) files = { - "./libfoo/__init__.py", + "ignore.txt", # 1:* + "libfoo/__init__.py", # 3:!libfoo/* } ignores = set(spec.match_files(files)) - self.assertEqual(ignores, set()) + self.assertEqual(ignores, { + "ignore.txt", + }) self.assertEqual(files - ignores, { - "./libfoo/__init__.py", + "libfoo/__init__.py", }) def test_08_issue_81_c(self): """ - Test issue 81. + Test issue 81, scenario C. """ + # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ "*", "!libfoo", "!libfoo/", ]) files = { - "./libfoo/__init__.py", + "ignore.txt", # 1:* + "libfoo/__init__.py", # 1:* } ignores = set(spec.match_files(files)) self.assertEqual(ignores, { - "./libfoo/__init__.py", + "ignore.txt", + "libfoo/__init__.py", }) self.assertEqual(files - ignores, set()) From 92a9066197f26f95a0438a70b2c8f0373c37125c Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 15:25:09 -0500 Subject: [PATCH 26/42] Improve debugging --- CHANGES.rst | 18 ++- DEV.md | 2 +- pathspec/_meta.py | 2 +- pathspec/gitignore.py | 75 ++++++++---- pathspec/pathspec.py | 158 ++++++++++++++++++------ pathspec/pattern.py | 111 +++++++++-------- pathspec/util.py | 120 +++++++++++++++---- tests/test_04_gitignore.py | 240 ++++++++++++++++++++++++------------- tests/util.py | 69 ++++++++++- 9 files changed, 570 insertions(+), 225 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 16cf8d8..9afe2d2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,13 +2,29 @@ Change History ============== -0.11.3 (TDB) +0.12.0 (TDB) ------------ +API changes: + +- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. + +New features: + +- Added `pathspec.pathspec.PathSpec.check_*()` methods. These methods behave similarly to `.match_*()` but return additional information in the `pathspec.util.CheckResult` objects (e.g., `CheckResult.index` indicates the index of the last pattern that matched the file). +- Added `pathspec.pattern.RegexPattern.pattern` attribute which stores the original, uncompiled pattern. + + Bug fixes: - `Pull #83`_: Fix ReadTheDocs builds. +Improvements: + +- Improve test debugging. +- Improve type hint on *on_error* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. +- Improve type hint on *on_error* parameter on `pathspec.util.iter_tree_entries()`. + .. _`Pull #83`: https://github.com/cpburnz/python-pathspec/pull/83 diff --git a/DEV.md b/DEV.md index ee4d876..365a37f 100644 --- a/DEV.md +++ b/DEV.md @@ -96,7 +96,7 @@ Review the following Linux distributions. - [Releases](https://wiki.ubuntu.com/Releases) - Package: [python3](https://packages.ubuntu.com/focal/python3) (focal) - Package: [python3](https://packages.ubuntu.com/jammy/python3) (jammy) - - Package: [python3-pathspec](https://packages.ubuntu.com/focal/python3-pathspec) (flocal) + - Package: [python3-pathspec](https://packages.ubuntu.com/focal/python3-pathspec) (focal) - Package: [python3-pathspec](https://packages.ubuntu.com/jammy/python3-pathspec) (jammy) diff --git a/pathspec/_meta.py b/pathspec/_meta.py index ab5405a..193fa9d 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -55,4 +55,4 @@ "kurtmckee ", ] __license__ = "MPL 2.0" -__version__ = "0.11.3.dev1" +__version__ = "0.12.0.dev1" diff --git a/pathspec/gitignore.py b/pathspec/gitignore.py index e5f7d48..d5eac8e 100644 --- a/pathspec/gitignore.py +++ b/pathspec/gitignore.py @@ -5,12 +5,15 @@ from typing import ( AnyStr, - Callable, - Collection, - Iterable, - Type, + Callable, # Replaced by `collections.abc.Callable` in 3.9. + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Tuple, # Replaced by `tuple` in 3.9. + Type, # Replaced by `type` in 3.9. TypeVar, - Union) + Union, # Replaced by `X | Y` in 3.10. + cast, + overload) from .pathspec import ( PathSpec) @@ -48,6 +51,25 @@ def __eq__(self, other: object) -> bool: else: return NotImplemented + # Support reversed order of arguments from PathSpec. + @overload + @classmethod + def from_lines( + cls: Type[Self], + pattern_factory: Union[str, Callable[[AnyStr], Pattern]], + lines: Iterable[AnyStr], + ) -> Self: + ... + + @overload + @classmethod + def from_lines( + cls: Type[Self], + lines: Iterable[AnyStr], + pattern_factory: Union[str, Callable[[AnyStr], Pattern], None] = None, + ) -> Self: + ... + @classmethod def from_lines( cls: Type[Self], @@ -74,36 +96,40 @@ def from_lines( if pattern_factory is None: pattern_factory = GitWildMatchPattern - elif (isinstance(lines, str) or callable(lines)) and _is_iterable(pattern_factory): + elif (isinstance(lines, (str, bytes)) or callable(lines)) and _is_iterable(pattern_factory): # Support reversed order of arguments from PathSpec. pattern_factory, lines = lines, pattern_factory self = super().from_lines(pattern_factory, lines) - return self # type: ignore + return cast(Self, self) @staticmethod def _match_file( - patterns: Collection[GitWildMatchPattern], + patterns: Iterable[Tuple[int, GitWildMatchPattern]], file: str, - ) -> bool: + ) -> Tuple[Optional[bool], Optional[int]]: """ - Matches the file to the patterns. + Check the file against the patterns. - .. NOTE:: Subclasses of :class:`.PathSpec` may override this - method as an instance method. It does not have to be a static - method. + .. NOTE:: Subclasses of :class:`~pathspec.pathspec.PathSpec` may override + this method as an instance method. It does not have to be a static + method. The signature for this method is subject to change. - *patterns* (:class:`~collections.abc.Iterable` of :class:`~pathspec.pattern.Pattern`) - contains the patterns to use. + *patterns* (:class:`~collections.abc.Iterable`) yields each indexed pattern + (:class:`tuple`) which contains the pattern index (:class:`int`) and actual + pattern (:class:`~pathspec.pattern.Pattern`). - *file* (:class:`str`) is the normalized file path to be matched - against *patterns*. + *file* (:class:`str`) is the normalized file path to be matched against + *patterns*. - Returns :data:`True` if *file* matched; otherwise, :data:`False`. + Returns a :class:`tuple` containing whether to include *file* (:class:`bool` + or :data:`None`), and the index of the last matched pattern (:class:`int` or + :data:`None`). """ - out_matched = False + out_include: Optional[bool] = None + out_index: Optional[int] = None out_priority = 0 - for pattern in patterns: + for index, pattern in patterns: if pattern.include is not None: match = pattern.match_file(file) if match is not None: @@ -112,6 +138,9 @@ def _match_file( # Check for directory marker. dir_mark = match.match.groupdict().get(_DIR_MARK) + # TODO: A exclude (whitelist) dir pattern here needs to deprioritize + # for 81-c. + if dir_mark: # Pattern matched by a directory pattern. priority = 1 @@ -120,10 +149,10 @@ def _match_file( priority = 2 if pattern.include and dir_mark: - out_matched = pattern.include + out_include = pattern.include out_priority = priority elif priority >= out_priority: - out_matched = pattern.include + out_include = pattern.include out_priority = priority - return out_matched + return out_include, out_index diff --git a/pathspec/pathspec.py b/pathspec/pathspec.py index 9153baa..ebffede 100644 --- a/pathspec/pathspec.py +++ b/pathspec/pathspec.py @@ -8,24 +8,25 @@ zip_longest) from typing import ( AnyStr, - Callable, - Collection, - Iterable, - Iterator, - Optional, - Type, + Callable, # Replaced by `collections.abc.Callable` in 3.9. + Collection, # Replaced by `collections.abc.Collection` in 3.9. + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Iterator, # Replaced by `collections.abc.Iterator` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Type, # Replaced by `type` in 3.9. TypeVar, - Union) + Union) # Replaced by `X | Y` in 3.10. from . import util from .pattern import ( Pattern) from .util import ( + CheckResult, StrPath, + TStrPath, TreeEntry, - _filter_patterns, + _filter_check_patterns, _is_iterable, - match_file, normalize_file) Self = TypeVar("Self", bound="PathSpec") @@ -48,8 +49,10 @@ def __init__(self, patterns: Iterable[Pattern]) -> None: *patterns* (:class:`~collections.abc.Collection` or :class:`~collections.abc.Iterable`) yields each compiled pattern (:class:`.Pattern`). """ + if not isinstance(patterns, CollectionType): + patterns = list(patterns) - self.patterns = patterns if isinstance(patterns, CollectionType) else list(patterns) + self.patterns: Collection[Pattern] = patterns """ *patterns* (:class:`~collections.abc.Collection` of :class:`.Pattern`) contains the compiled patterns. @@ -94,6 +97,88 @@ def __iadd__(self: Self, other: "PathSpec") -> Self: else: return NotImplemented + def check_file( + self, + file: TStrPath, + separators: Optional[Collection[str]] = None, + ) -> CheckResult[TStrPath]: + """ + Check the files against this path-spec. + + *file* (:class:`str` or :class:`os.PathLike`) is the file path to be + matched against :attr:`self.patterns `. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + Returns the file check result (:class:`CheckResult`). + """ + norm_file = normalize_file(file, separators) + include, index = self._match_file(enumerate(self.patterns), norm_file) + return CheckResult(file, include, index) + + def check_files( + self, + files: Iterable[TStrPath], + separators: Optional[Collection[str]] = None, + ) -> Iterator[CheckResult[TStrPath]]: + """ + Check the files against this path-spec. + + *files* (:class:`~collections.abc.Iterable` of :class:`str` or + :class:`os.PathLike`) contains the file paths to be checked against + :attr:`self.patterns `. + + *separators* (:class:`~collections.abc.Collection` of :class:`str`; or + :data:`None`) optionally contains the path separators to normalize. See + :func:`~pathspec.util.normalize_file` for more information. + + Returns an :class:`~collections.abc.Iterator` yielding each file check + result (:class:`CheckResult`). + """ + if not _is_iterable(files): + raise TypeError(f"files:{files!r} is not an iterable.") + + use_patterns = _filter_check_patterns(self.patterns) + for orig_file in files: + norm_file = normalize_file(orig_file, separators) + include, index = self._match_file(use_patterns, norm_file) + yield CheckResult(orig_file, include, index) + + def check_tree_files( + self, + root: StrPath, + on_error: Optional[Callable[[OSError], None]] = None, + follow_links: Optional[bool] = None, + ) -> Iterator[CheckResult[str]]: + """ + Walks the specified root path for all files and checks them against this + path-spec. + + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search for files. + + *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally + is the error handler for file-system exceptions. It will be called with the + exception (:exc:`OSError`). Reraise the exception to abort the walk. Default + is :data:`None` to ignore file-system exceptions. + + *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk + symbolic links that resolve to directories. Default is :data:`None` for + :data:`True`. + + *negate* (:class:`bool` or :data:`None`) is whether to negate the match + results of the patterns. If :data:`True`, a pattern matching a file will + exclude the file rather than include it. Default is :data:`None` for + :data:`False`. + + Returns an :class:`~collections.abc.Iterator` yielding each file check + result (:class:`CheckResult`). + """ + files = util.iter_tree_files(root, on_error=on_error, follow_links=follow_links) + yield from self.check_files(files) + @classmethod def from_lines( cls: Type[Self], @@ -155,21 +240,23 @@ def match_entries( if not _is_iterable(entries): raise TypeError(f"entries:{entries!r} is not an iterable.") - use_patterns = _filter_patterns(self.patterns) + use_patterns = _filter_check_patterns(self.patterns) for entry in entries: norm_file = normalize_file(entry.path, separators) - is_match = self._match_file(use_patterns, norm_file) + include, _index = self._match_file(use_patterns, norm_file) if negate: - is_match = not is_match + include = not include - if is_match: + if include: yield entry - # Match files using the `match_file()` utility function. Subclasses may - # override this method as an instance method. It does not have to be a static - # method. - _match_file = staticmethod(match_file) + _match_file = staticmethod(util.check_match_file) + """ + Match files using the `check_match_file()` utility function. Subclasses may + override this method as an instance method. It does not have to be a static + method. The signature for this method is subject to change. + """ def match_file( self, @@ -188,8 +275,9 @@ def match_file( Returns :data:`True` if *file* matched; otherwise, :data:`False`. """ - norm_file = util.normalize_file(file, separators=separators) - return self._match_file(self.patterns, norm_file) + norm_file = normalize_file(file, separators) + include, _index = self._match_file(enumerate(self.patterns), norm_file) + return include def match_files( self, @@ -220,21 +308,21 @@ def match_files( if not _is_iterable(files): raise TypeError(f"files:{files!r} is not an iterable.") - use_patterns = _filter_patterns(self.patterns) + use_patterns = _filter_check_patterns(self.patterns) for orig_file in files: norm_file = normalize_file(orig_file, separators) - is_match = self._match_file(use_patterns, norm_file) + include, _index = self._match_file(use_patterns, norm_file) if negate: - is_match = not is_match + include = not include - if is_match: + if include: yield orig_file def match_tree_entries( self, root: StrPath, - on_error: Optional[Callable] = None, + on_error: Optional[Callable[[OSError], None]] = None, follow_links: Optional[bool] = None, *, negate: Optional[bool] = None, @@ -247,12 +335,13 @@ def match_tree_entries( search. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally - is the error handler for file-system exceptions. See - :func:`~pathspec.util.iter_tree_entries` for more information. + is the error handler for file-system exceptions. It will be called with the + exception (:exc:`OSError`). Reraise the exception to abort the walk. Default + is :data:`None` to ignore file-system exceptions. *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk - symbolic links that resolve to directories. See - :func:`~pathspec.util.iter_tree_files` for more information. + symbolic links that resolve to directories. Default is :data:`None` for + :data:`True`. *negate* (:class:`bool` or :data:`None`) is whether to negate the match results of the patterns. If :data:`True`, a pattern matching a file will @@ -268,7 +357,7 @@ def match_tree_entries( def match_tree_files( self, root: StrPath, - on_error: Optional[Callable] = None, + on_error: Optional[Callable[[OSError], None]] = None, follow_links: Optional[bool] = None, *, negate: Optional[bool] = None, @@ -281,12 +370,13 @@ def match_tree_files( search for files. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally - is the error handler for file-system exceptions. See - :func:`~pathspec.util.iter_tree_files` for more information. + is the error handler for file-system exceptions. It will be called with the + exception (:exc:`OSError`). Reraise the exception to abort the walk. Default + is :data:`None` to ignore file-system exceptions. *follow_links* (:class:`bool` or :data:`None`) optionally is whether to walk - symbolic links that resolve to directories. See - :func:`~pathspec.util.iter_tree_files` for more information. + symbolic links that resolve to directories. Default is :data:`None` for + :data:`True`. *negate* (:class:`bool` or :data:`None`) is whether to negate the match results of the patterns. If :data:`True`, a pattern matching a file will diff --git a/pathspec/pattern.py b/pathspec/pattern.py index 8f20e73..d081557 100644 --- a/pathspec/pattern.py +++ b/pathspec/pattern.py @@ -8,13 +8,13 @@ from typing import ( Any, AnyStr, - Iterable, - Iterator, - Match as MatchHint, - Optional, - Pattern as PatternHint, - Tuple, - Union) + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Iterator, # Replaced by `collections.abc.Iterator` in 3.9. + Match as MatchHint, # Replaced by `re.Match` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Pattern as PatternHint, # Replaced by `re.Pattern` in 3.9. + Tuple, # Replaced by `tuple` in 3.9. + Union) # Replaced by `X | Y` in 3.10. class Pattern(object): @@ -23,44 +23,45 @@ class Pattern(object): """ # Make the class dict-less. - __slots__ = ('include',) + __slots__ = ( + 'include', + ) def __init__(self, include: Optional[bool]) -> None: """ Initializes the :class:`Pattern` instance. - *include* (:class:`bool` or :data:`None`) is whether the matched - files should be included (:data:`True`), excluded (:data:`False`), - or is a null-operation (:data:`None`). + *include* (:class:`bool` or :data:`None`) is whether the matched files + should be included (:data:`True`), excluded (:data:`False`), or is a + null-operation (:data:`None`). """ self.include = include """ - *include* (:class:`bool` or :data:`None`) is whether the matched - files should be included (:data:`True`), excluded (:data:`False`), - or is a null-operation (:data:`None`). + *include* (:class:`bool` or :data:`None`) is whether the matched files + should be included (:data:`True`), excluded (:data:`False`), or is a + null-operation (:data:`None`). """ def match(self, files: Iterable[str]) -> Iterator[str]: """ DEPRECATED: This method is no longer used and has been replaced by - :meth:`.match_file`. Use the :meth:`.match_file` method with a loop - for similar results. + :meth:`.match_file`. Use the :meth:`.match_file` method with a loop for + similar results. Matches this pattern against the specified files. - *files* (:class:`~collections.abc.Iterable` of :class:`str`) - contains each file relative to the root directory (e.g., - ``"relative/path/to/file"``). + *files* (:class:`~collections.abc.Iterable` of :class:`str`) contains each + file relative to the root directory (e.g., ``"relative/path/to/file"``). - Returns an :class:`~collections.abc.Iterable` yielding each matched - file path (:class:`str`). + Returns an :class:`~collections.abc.Iterable` yielding each matched file + path (:class:`str`). """ warnings.warn(( - "{0.__module__}.{0.__qualname__}.match() is deprecated. Use " - "{0.__module__}.{0.__qualname__}.match_file() with a loop for " + "{cls.__module__}.{cls.__qualname__}.match() is deprecated. Use " + "{cls.__module__}.{cls.__qualname__}.match_file() with a loop for " "similar results." - ).format(self.__class__), DeprecationWarning, stacklevel=2) + ).format(cls=self.__class__), DeprecationWarning, stacklevel=2) for file in files: if self.match_file(file) is not None: @@ -75,22 +76,25 @@ def match_file(self, file: str) -> Optional[Any]: Returns the match result if *file* matched; otherwise, :data:`None`. """ raise NotImplementedError(( - "{0.__module__}.{0.__qualname__} must override match_file()." - ).format(self.__class__)) + "{cls.__module__}.{cls.__qualname__} must override match_file()." + ).format(cls=self.__class__)) class RegexPattern(Pattern): """ - The :class:`RegexPattern` class is an implementation of a pattern - using regular expressions. + The :class:`RegexPattern` class is an implementation of a pattern using + regular expressions. """ # Keep the class dict-less. - __slots__ = ('regex',) + __slots__ = ( + 'pattern', + 'regex', + ) def __init__( self, - pattern: Union[AnyStr, PatternHint], + pattern: Union[AnyStr, PatternHint, None], include: Optional[bool] = None, ) -> None: """ @@ -99,20 +103,18 @@ def __init__( *pattern* (:class:`str`, :class:`bytes`, :class:`re.Pattern`, or :data:`None`) is the pattern to compile into a regular expression. - *include* (:class:`bool` or :data:`None`) must be :data:`None` - unless *pattern* is a precompiled regular expression (:class:`re.Pattern`) - in which case it is whether matched files should be included - (:data:`True`), excluded (:data:`False`), or is a null operation - (:data:`None`). + *include* (:class:`bool` or :data:`None`) must be :data:`None` unless + *pattern* is a precompiled regular expression (:class:`re.Pattern`) in which + case it is whether matched files should be included (:data:`True`), excluded + (:data:`False`), or is a null operation (:data:`None`). - .. NOTE:: Subclasses do not need to support the *include* - parameter. + .. NOTE:: Subclasses do not need to support the *include* parameter. """ if isinstance(pattern, (str, bytes)): assert include is None, ( - "include:{!r} must be null when pattern:{!r} is a string." - ).format(include, pattern) + f"include:{include!r} must be null when pattern:{pattern!r} is a string." + ) regex, include = self.pattern_to_regex(pattern) # NOTE: Make sure to allow a null regular expression to be # returned for a null-operation. @@ -128,18 +130,23 @@ def __init__( # NOTE: Make sure to allow a null pattern to be passed for a # null-operation. assert include is None, ( - "include:{!r} must be null when pattern:{!r} is null." - ).format(include, pattern) + f"include:{include!r} must be null when pattern:{pattern!r} is null." + ) else: - raise TypeError("pattern:{!r} is not a string, re.Pattern, or None.".format(pattern)) + raise TypeError(f"pattern:{pattern!r} is not a string, re.Pattern, or None.") super(RegexPattern, self).__init__(include) + self.pattern: Union[AnyStr, PatternHint, None] = pattern + """ + *pattern* (:class:`str`, :class:`bytes`, :class:`re.Pattern`, or + :data:`None`) is the uncompiled, input pattern. This is for reference. + """ + self.regex: PatternHint = regex """ - *regex* (:class:`re.Pattern`) is the regular expression for the - pattern. + *regex* (:class:`re.Pattern`) is the regular expression for the pattern. """ def __eq__(self, other: 'RegexPattern') -> bool: @@ -157,11 +164,11 @@ def match_file(self, file: str) -> Optional['RegexMatchResult']: """ Matches this pattern against the specified file. - *file* (:class:`str`) - contains each file relative to the root directory (e.g., "relative/path/to/file"). + *file* (:class:`str`) contains each file relative to the root directory + (e.g., "relative/path/to/file"). - Returns the match result (:class:`.RegexMatchResult`) if *file* - matched; otherwise, :data:`None`. + Returns the match result (:class:`.RegexMatchResult`) if *file* matched; + otherwise, :data:`None`. """ if self.include is not None: match = self.regex.match(file) @@ -179,8 +186,8 @@ def pattern_to_regex(cls, pattern: str) -> Tuple[str, bool]: expression. Returns the uncompiled regular expression (:class:`str` or :data:`None`), - and whether matched files should be included (:data:`True`), - excluded (:data:`False`), or is a null-operation (:data:`None`). + and whether matched files should be included (:data:`True`), excluded + (:data:`False`), or is a null-operation (:data:`None`). .. NOTE:: The default implementation simply returns *pattern* and :data:`True`. @@ -191,8 +198,8 @@ def pattern_to_regex(cls, pattern: str) -> Tuple[str, bool]: @dataclasses.dataclass() class RegexMatchResult(object): """ - The :class:`RegexMatchResult` data class is used to return information - about the matched regular expression. + The :class:`RegexMatchResult` data class is used to return information about + the matched regular expression. """ # Keep the class dict-less. diff --git a/pathspec/util.py b/pathspec/util.py index 9408f25..54e1b2c 100644 --- a/pathspec/util.py +++ b/pathspec/util.py @@ -12,21 +12,26 @@ from collections.abc import ( Collection as CollectionType, Iterable as IterableType) +from dataclasses import ( + dataclass) from os import ( PathLike) from typing import ( Any, AnyStr, - Callable, - Collection, - Dict, - Iterable, - Iterator, - List, - Optional, - Sequence, - Set, - Union) + Callable, # Replaced by `collections.abc.Callable` in 3.9. + Collection, # Replaced by `collections.abc.Collection` in 3.9. + Dict, # Replaced by `dict` in 3.9. + Generic, + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Iterator, # Replaced by `collections.abc.Iterator` in 3.9. + List, # Replaced by `list` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Sequence, # Replaced by `collections.abc.Sequence` in 3.9. + Set, # Replaced by `set` in 3.9. + Tuple, # Replaced by `tuple` in 3.9. + TypeVar, + Union) # Replaced by `X | Y` in 3.10. from .pattern import ( Pattern) @@ -36,6 +41,8 @@ else: StrPath = Union[str, PathLike] +TStrPath = TypeVar("TStrPath", bound=StrPath) + NORMALIZE_PATH_SEPS = [ __sep for __sep in [os.sep, os.altsep] @@ -73,6 +80,34 @@ def append_dir_sep(path: pathlib.Path) -> str: return str_path +def check_match_file( + patterns: Iterable[Tuple[int, Pattern]], + file: str, +) -> Tuple[Optional[bool], Optional[int]]: + """ + Check the file against the patterns. + + *patterns* (:class:`~collections.abc.Iterable`) yields each indexed pattern + (:class:`tuple`) which contains the pattern index (:class:`int`) and actual + pattern (:class:`~pathspec.pattern.Pattern`). + + *file* (:class:`str`) is the normalized file path to be matched + against *patterns*. + + Returns a :class:`tuple` containing whether to include *file* (:class:`bool` + or :data:`None`), and the index of the last matched pattern (:class:`int` or + :data:`None`). + """ + out_include: Optional[bool] = None + out_index: Optional[int] = None + for index, pattern in patterns: + if pattern.include is not None and pattern.match_file(file) is not None: + out_include = pattern.include + out_index = index + + return out_include, out_index + + def detailed_match_files( patterns: Iterable[Pattern], files: Iterable[str], @@ -119,18 +154,22 @@ def detailed_match_files( return return_files -def _filter_patterns(patterns: Iterable[Pattern]) -> List[Pattern]: +def _filter_check_patterns( + patterns: Iterable[Pattern], +) -> List[Tuple[int, Pattern]]: """ Filters out null-patterns. *patterns* (:class:`Iterable` of :class:`.Pattern`) contains the patterns. - Returns the patterns (:class:`list` of :class:`.Pattern`). + Returns a :class:`list` containing each indexed pattern (:class:`tuple`) which + contains the pattern index (:class:`int`) and the actual pattern + (:class:`~pathspec.pattern.Pattern`). """ return [ - __pat - for __pat in patterns + (__index, __pat) + for __index, __pat in enumerate(patterns) if __pat.include is not None ] @@ -148,7 +187,7 @@ def _is_iterable(value: Any) -> bool: def iter_tree_entries( root: StrPath, - on_error: Optional[Callable] = None, + on_error: Optional[Callable[[OSError], None]] = None, follow_links: Optional[bool] = None, ) -> Iterator['TreeEntry']: """ @@ -185,7 +224,7 @@ def _iter_tree_entries_next( root_full: str, dir_rel: str, memo: Dict[str, str], - on_error: Callable, + on_error: Callable[[OSError], None], follow_links: bool, ) -> Iterator['TreeEntry']: """ @@ -264,7 +303,7 @@ def _iter_tree_entries_next( def iter_tree_files( root: StrPath, - on_error: Optional[Callable] = None, + on_error: Optional[Callable[[OSError], None]] = None, follow_links: Optional[bool] = None, ) -> Iterator[str]: """ @@ -330,9 +369,8 @@ def match_file(patterns: Iterable[Pattern], file: str) -> bool: """ matched = False for pattern in patterns: - if pattern.include is not None: - if pattern.match_file(file) is not None: - matched = pattern.include + if pattern.include is not None and pattern.match_file(file) is not None: + matched = pattern.include return matched @@ -342,8 +380,8 @@ def match_files( files: Iterable[str], ) -> Set[str]: """ - DEPRECATED: This is an old function no longer used. Use the :func:`.match_file` - function with a loop for better results. + DEPRECATED: This is an old function no longer used. Use the + :func:`~pathspec.util.match_file` function with a loop for better results. Matches the files to the patterns. @@ -356,11 +394,11 @@ def match_files( Returns the matched files (:class:`set` of :class:`str`). """ warnings.warn(( - "util.match_files() is deprecated. Use util.match_file() with a " - "loop for better results." + f"{__name__}.match_files() is deprecated. Use {__name__}.match_file() with " + f"a loop for better results." ), DeprecationWarning, stacklevel=2) - use_patterns = _filter_patterns(patterns) + use_patterns = [__pat for __pat in patterns if __pat.include is not None] return_files = set() for file in files: @@ -588,6 +626,38 @@ def second_path(self) -> str: return self.args[2] +@dataclass(frozen=True) +class CheckResult(Generic[TStrPath]): + """ + The :class:`CheckResult` class contains information about the file and which + pattern matched it. + """ + + # Make the class dict-less. + __slots__ = ( + 'file', + 'include', + 'index', + ) + + file: TStrPath + """ + *file* (:class:`str` or :class:`os.PathLike`) is the file path. + """ + + include: Optional[bool] + """ + *include* (:class:`bool` or :data:`None`) is whether to include or exclude the + file. If :data:`None`, no pattern matched. + """ + + index: Optional[int] + """ + *index* (:class:`int` or :data:`None`) is the index of the last pattern that + matched. If :data:`None`, no pattern matched. + """ + + class MatchDetail(object): """ The :class:`.MatchDetail` class contains information about diff --git a/tests/test_04_gitignore.py b/tests/test_04_gitignore.py index aa3c231..479bd75 100644 --- a/tests/test_04_gitignore.py +++ b/tests/test_04_gitignore.py @@ -7,11 +7,14 @@ from pathspec.gitignore import ( GitIgnoreSpec) +from .util import ( + debug_results, + get_includes) + class GitIgnoreSpecTest(unittest.TestCase): """ - The :class:`GitIgnoreSpecTest` class tests the :class:`.GitIgnoreSpec` - class. + The :class:`GitIgnoreSpecTest` class tests the :class:`.GitIgnoreSpec` class. """ def test_01_reversed_args(self): @@ -19,13 +22,18 @@ def test_01_reversed_args(self): Test reversed args for `.from_lines()`. """ spec = GitIgnoreSpec.from_lines('gitwildmatch', ['*.txt']) - results = set(spec.match_files([ + files = { 'a.txt', 'b.bin', - ])) - self.assertEqual(results, { + } + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(ignores, { 'a.txt', - }) + }, debug) def test_02_dir_exclusions(self): """ @@ -43,17 +51,21 @@ def test_02_dir_exclusions(self): 'test2/b.bin', 'test2/c/c.txt', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'test1/a.txt', 'test1/c/c.txt', 'test2/a.txt', 'test2/c/c.txt', - }) + }, debug) self.assertEqual(files - ignores, { 'test1/b.bin', 'test2/b.bin', - }) + }, debug) def test_02_file_exclusions(self): """ @@ -71,22 +83,26 @@ def test_02_file_exclusions(self): 'Y/b.txt', 'Y/Z/c.txt', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', 'Y/Z/c.txt', - }) + }, debug) self.assertEqual(files - ignores, { 'X/b.txt', 'Y/b.txt', - }) + }, debug) def test_02_issue_41_a(self): """ - Test including a file and excluding a directory with the same name - pattern, scenario A. + Test including a file and excluding a directory with the same name pattern, + scenario A. """ # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ @@ -94,31 +110,35 @@ def test_02_issue_41_a(self): '!*.yaml/', ]) files = { - 'dir.yaml/file.sql', # - + 'dir.yaml/file.sql', # - 'dir.yaml/file.yaml', # 1:*.yaml 'dir.yaml/index.txt', # - - 'dir/file.sql', # - - 'dir/file.yaml', # 1:*.yaml - 'dir/index.txt', # - - 'file.yaml', # 1:*.yaml + 'dir/file.sql', # - + 'dir/file.yaml', # 1:*.yaml + 'dir/index.txt', # - + 'file.yaml', # 1:*.yaml } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dir.yaml/file.yaml', 'dir/file.yaml', 'file.yaml', - }) + }, debug) self.assertEqual(files - ignores, { 'dir.yaml/file.sql', 'dir.yaml/index.txt', 'dir/file.sql', 'dir/index.txt', - }) + }, debug) def test_02_issue_41_b(self): """ - Test including a file and excluding a directory with the same name - pattern, scenario B. + Test including a file and excluding a directory with the same name pattern, + scenario B. """ # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ @@ -126,31 +146,35 @@ def test_02_issue_41_b(self): '*.yaml', ]) files = { - 'dir.yaml/file.sql', # 2:*.yaml + 'dir.yaml/file.sql', # 2:*.yaml 'dir.yaml/file.yaml', # 2:*.yaml 'dir.yaml/index.txt', # 2:*.yaml - 'dir/file.sql', # - - 'dir/file.yaml', # 2:*.yaml - 'dir/index.txt', # - - 'file.yaml', # 2:*.yaml + 'dir/file.sql', # - + 'dir/file.yaml', # 2:*.yaml + 'dir/index.txt', # - + 'file.yaml', # 2:*.yaml } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dir.yaml/file.sql', 'dir.yaml/file.yaml', 'dir.yaml/index.txt', 'dir/file.yaml', 'file.yaml', - }) + }, debug) self.assertEqual(files - ignores, { 'dir/file.sql', 'dir/index.txt', - }) + }, debug) def test_02_issue_41_c(self): """ - Test including a file and excluding a directory with the same name - pattern, scenario C. + Test including a file and excluding a directory with the same name pattern, + scenario C. """ # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ @@ -158,26 +182,30 @@ def test_02_issue_41_c(self): '!dir.yaml', ]) files = { - 'dir.yaml/file.sql', # - + 'dir.yaml/file.sql', # - 'dir.yaml/file.yaml', # 1:*.yaml 'dir.yaml/index.txt', # - - 'dir/file.sql', # - - 'dir/file.yaml', # 1:*.yaml - 'dir/index.txt', # - - 'file.yaml', # 1:*.yaml + 'dir/file.sql', # - + 'dir/file.yaml', # 1:*.yaml + 'dir/index.txt', # - + 'file.yaml', # 1:*.yaml } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dir.yaml/file.yaml', 'dir/file.yaml', 'file.yaml', - }) + }, debug) self.assertEqual(files - ignores, { 'dir.yaml/file.sql', 'dir.yaml/index.txt', 'dir/file.sql', 'dir/index.txt', - }) + }, debug) def test_03_subdir(self): """ @@ -195,23 +223,26 @@ def test_03_subdir(self): 'dirG/dirH/fileJ', 'dirG/fileO', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dirG/dirH/fileI', 'dirG/dirH/fileJ', 'dirG/fileO', - }) + }, debug) self.assertEqual(files - ignores, { 'fileA', 'fileB', 'dirD/fileE', 'dirD/fileF', - }) + }, debug) def test_03_issue_19_a(self): """ - Test matching files in a subdirectory of an included directory, - scenario A. + Test matching files in a subdirectory of an included directory, scenario A. """ spec = GitIgnoreSpec.from_lines([ "dirG/", @@ -225,23 +256,26 @@ def test_03_issue_19_a(self): 'dirG/dirH/fileJ', 'dirG/fileO', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dirG/dirH/fileI', 'dirG/dirH/fileJ', 'dirG/fileO', - }) + }, debug) self.assertEqual(files - ignores, { 'fileA', 'fileB', 'dirD/fileE', 'dirD/fileF', - }) + }, debug) def test_03_issue_19_b(self): """ - Test matching files in a subdirectory of an included directory, - scenario B. + Test matching files in a subdirectory of an included directory, scenario B. """ spec = GitIgnoreSpec.from_lines([ "dirG/*", @@ -255,23 +289,26 @@ def test_03_issue_19_b(self): 'dirG/dirH/fileJ', 'dirG/fileO', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dirG/dirH/fileI', 'dirG/dirH/fileJ', 'dirG/fileO', - }) + }, debug) self.assertEqual(files - ignores, { 'fileA', 'fileB', 'dirD/fileE', 'dirD/fileF', - }) + }, debug) def test_03_issue_19_c(self): """ - Test matching files in a subdirectory of an included directory, - scenario C. + Test matching files in a subdirectory of an included directory, scenario C. """ spec = GitIgnoreSpec.from_lines([ "dirG/**", @@ -285,18 +322,22 @@ def test_03_issue_19_c(self): 'dirG/dirH/fileJ', 'dirG/fileO', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dirG/dirH/fileI', 'dirG/dirH/fileJ', 'dirG/fileO', - }) + }, debug) self.assertEqual(files - ignores, { 'fileA', 'fileB', 'dirD/fileE', 'dirD/fileF', - }) + }, debug) def test_04_issue_62(self): """ @@ -306,14 +347,19 @@ def test_04_issue_62(self): '*', '!product_dir/', ]) - results = set(spec.match_files([ + files = { 'anydir/file.txt', 'product_dir/file.txt', - ])) - self.assertEqual(results, { + } + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(ignores, { 'anydir/file.txt', 'product_dir/file.txt', - }) + }, debug) def test_05_issue_39(self): """ @@ -331,16 +377,20 @@ def test_05_issue_39(self): 'important/e.txt', 'trace.c', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'a.log', 'trace.c', - }) + }, debug) self.assertEqual(files - ignores, { 'b.txt', 'important/d.log', 'important/e.txt', - }) + }, debug) def test_06_issue_64(self): """ @@ -359,8 +409,12 @@ def test_06_issue_64(self): 'A/B/C/x', 'A/B/C/y.py', } - ignores = set(spec.match_files(files)) - self.assertEqual(ignores, files) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(ignores, files, debug) def test_07_issue_74(self): """ @@ -380,21 +434,25 @@ def test_07_issue_74(self): 'test2/b.bin', 'test2/c/c.txt', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'test1/b.bin', 'test1/a.txt', 'test1/c/c.txt', 'test2/b.bin', - }) + }, debug) self.assertEqual(files - ignores, { 'test2/a.txt', 'test2/c/c.txt', - }) + }, debug) def test_08_issue_81_a(self): """ - Test issue 81, scenario A. + Test issue 81 whitelist, scenario A. """ # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ @@ -403,20 +461,24 @@ def test_08_issue_81_a(self): "!libfoo/**", ]) files = { - "ignore.txt", # 1:* + "ignore.txt", # 1:* "libfoo/__init__.py", # 3:!libfoo/** } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { "ignore.txt", - }) + }, debug) self.assertEqual(files - ignores, { "libfoo/__init__.py", - }) + }, debug) def test_08_issue_81_b(self): """ - Test issue 81, scenario B. + Test issue 81 whitelist, scenario B. """ # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ @@ -425,20 +487,24 @@ def test_08_issue_81_b(self): "!libfoo/*", ]) files = { - "ignore.txt", # 1:* + "ignore.txt", # 1:* "libfoo/__init__.py", # 3:!libfoo/* } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { "ignore.txt", - }) + }, debug) self.assertEqual(files - ignores, { "libfoo/__init__.py", - }) + }, debug) def test_08_issue_81_c(self): """ - Test issue 81, scenario C. + Test issue 81 whitelist, scenario C. """ # Confirmed results with git (v2.42.0). spec = GitIgnoreSpec.from_lines([ @@ -447,12 +513,14 @@ def test_08_issue_81_c(self): "!libfoo/", ]) files = { - "ignore.txt", # 1:* + "ignore.txt", # 1:* "libfoo/__init__.py", # 1:* } - ignores = set(spec.match_files(files)) + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) self.assertEqual(ignores, { "ignore.txt", "libfoo/__init__.py", - }) + }, debug) self.assertEqual(files - ignores, set()) diff --git a/tests/util.py b/tests/util.py index 301427b..fa22870 100644 --- a/tests/util.py +++ b/tests/util.py @@ -7,8 +7,73 @@ import pathlib from typing import ( - Iterable, - Tuple) + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Tuple, # Replaced by `collections.abc.Tuple` in 3.9. + cast) + +from pathspec import ( + PathSpec, + RegexPattern) +from pathspec.util import ( + CheckResult, + TStrPath) + + +def debug_results(spec: PathSpec, results: Iterable[CheckResult[str]]) -> str: + """ + Format the check results message. + + *spec* (:class:`~pathspec.PathSpec`) is the path-spec. + + *results* (:class:`~collections.abc.Iterable` or :class:`~pathspec.util.CheckResult`) + yields each file check result. + + Returns the message (:class:`str`). + """ + patterns = cast(list[RegexPattern], spec.patterns) + + result_table = [] + for result in results: + if result.index is not None: + pattern = patterns[result.index] + result_table.append((f"{result.index + 1}:{pattern.pattern}", result.file)) + else: + result_table.append(("-", result.file)) + + result_table.sort(key=lambda r: r[1]) + + first_max_len = max((len(__row[0]) for __row in result_table), default=0) + first_width = min(first_max_len, 20) + + result_lines = [] + for row in result_table: + result_lines.append(f" {row[0]:<{first_width}} {row[1]}") + + pattern_lines = [] + for index, pattern in enumerate(patterns, 1): + first_col = f"{index}:{pattern.pattern}" + pattern_lines.append(f" {first_col:<{first_width}} {pattern.regex.pattern!r}") + + return "\n".join([ + "\n", + " DEBUG ".center(32, "-"), + *pattern_lines, + "-"*32, + *result_lines, + "-"*32, + ]) + + +def get_includes(results: Iterable[CheckResult[TStrPath]]) -> set[TStrPath]: + """ + Get the included files from the check results. + + *results* (:class:`~collections.abc.Iterable` or :class:`~pathspec.util.CheckResult`) + yields each file check result. + + Returns the included files (:class:`set` of :class:`str`). + """ + return {__res.file for __res in results if __res.include} def make_dirs(temp_dir: pathlib.Path, dirs: Iterable[str]) -> None: From f568711ed00790c2d8e202947b438c86b6bbc5f4 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 15:38:42 -0500 Subject: [PATCH 27/42] CHANGES --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 9afe2d2..0610f84 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,11 @@ Change History 0.12.0 (TDB) ------------ +Major changes: + +- Dropped support of EOL Python 3.7. See `Pull #82`_. + + API changes: - Signature of protected method `pathspec.pathspec.PathSpec._match_file()` has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. @@ -21,11 +26,13 @@ Bug fixes: Improvements: +- Mark Python 3.12 as supported. See `Pull #82`_. - Improve test debugging. - Improve type hint on *on_error* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. - Improve type hint on *on_error* parameter on `pathspec.util.iter_tree_entries()`. +.. _`Pull #82`: https://github.com/cpburnz/python-pathspec/pull/82 .. _`Pull #83`: https://github.com/cpburnz/python-pathspec/pull/83 From 67ddd73f790b77445868e05ee0b9305fc87c6db9 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 15:41:05 -0500 Subject: [PATCH 28/42] DEV --- DEV.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEV.md b/DEV.md index 365a37f..fcd9b7f 100644 --- a/DEV.md +++ b/DEV.md @@ -126,5 +126,5 @@ Review the following PyPI packages. [yamllint](https://pypi.org/project/yamllint/) -- v1.32.0 (latest as of 2023-09-06) requires Python 3.7+. +- v1.33.0 (latest as of 2023-12-09) requires Python 3.8+. - [yamllint on Wheelodex](https://www.wheelodex.org/projects/yamllint/). From f092e3248bdc7c80bfdfce20f0a0c660f8caf2e6 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 16:50:12 -0500 Subject: [PATCH 29/42] Improve debugging --- pathspec/gitignore.py | 2 ++ tests/util.py | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pathspec/gitignore.py b/pathspec/gitignore.py index d5eac8e..a8d3c43 100644 --- a/pathspec/gitignore.py +++ b/pathspec/gitignore.py @@ -150,9 +150,11 @@ def _match_file( if pattern.include and dir_mark: out_include = pattern.include + out_index = index out_priority = priority elif priority >= out_priority: out_include = pattern.include + out_index = index out_priority = priority return out_include, out_index diff --git a/tests/util.py b/tests/util.py index fa22870..b845a3e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -2,6 +2,7 @@ This module provides utility functions shared by tests. """ +import itertools import os import os.path import pathlib @@ -32,6 +33,10 @@ def debug_results(spec: PathSpec, results: Iterable[CheckResult[str]]) -> str: """ patterns = cast(list[RegexPattern], spec.patterns) + pattern_table = [] + for index, pattern in enumerate(patterns, 1): + pattern_table.append((f"{index}:{pattern.pattern}", repr(pattern.regex.pattern))) + result_table = [] for result in results: if result.index is not None: @@ -42,18 +47,19 @@ def debug_results(spec: PathSpec, results: Iterable[CheckResult[str]]) -> str: result_table.sort(key=lambda r: r[1]) - first_max_len = max((len(__row[0]) for __row in result_table), default=0) + first_max_len = max(( + len(__row[0]) for __row in itertools.chain(pattern_table, result_table) + ), default=0) first_width = min(first_max_len, 20) + pattern_lines = [] + for row in pattern_table: + pattern_lines.append(f" {row[0]:<{first_width}} {row[1]}") + result_lines = [] for row in result_table: result_lines.append(f" {row[0]:<{first_width}} {row[1]}") - pattern_lines = [] - for index, pattern in enumerate(patterns, 1): - first_col = f"{index}:{pattern.pattern}" - pattern_lines.append(f" {first_col:<{first_width}} {pattern.regex.pattern!r}") - return "\n".join([ "\n", " DEBUG ".center(32, "-"), From 5bd2db7bd2125475a3396a4547d0c0eb418b88a2 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 16:51:49 -0500 Subject: [PATCH 30/42] Improve debugging --- tests/test_03_pathspec.py | 333 ++++++++++++++++++++++++++------------ 1 file changed, 231 insertions(+), 102 deletions(-) diff --git a/tests/test_03_pathspec.py b/tests/test_03_pathspec.py index 2e7c8f0..f77d18d 100644 --- a/tests/test_03_pathspec.py +++ b/tests/test_03_pathspec.py @@ -2,8 +2,6 @@ This script tests :class:`.PathSpec`. """ -import os -import os.path import pathlib import shutil import tempfile @@ -13,10 +11,14 @@ from pathspec import ( PathSpec) +from pathspec.patterns.gitwildmatch import ( + GitWildMatchPatternError) from pathspec.util import ( iter_tree_entries) -from pathspec.patterns.gitwildmatch import GitWildMatchPatternError -from tests.util import ( + +from .util import ( + debug_results, + get_includes, make_dirs, make_files, ospath) @@ -58,7 +60,7 @@ def test_01_absolute_dir_paths_1(self): spec = PathSpec.from_lines('gitwildmatch', [ 'foo', ]) - results = set(spec.match_files([ + files = { '/a.py', '/foo/a.py', '/x/a.py', @@ -67,13 +69,18 @@ def test_01_absolute_dir_paths_1(self): 'foo/a.py', 'x/a.py', 'x/foo/a.py', - ])) - self.assertEqual(results, { + } + + results = list(spec.check_files(files)) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { '/foo/a.py', '/x/foo/a.py', 'foo/a.py', 'x/foo/a.py', - }) + }, debug) def test_01_absolute_dir_paths_2(self): """ @@ -82,7 +89,7 @@ def test_01_absolute_dir_paths_2(self): spec = PathSpec.from_lines('gitwildmatch', [ '/foo', ]) - results = set(spec.match_files([ + files = { '/a.py', '/foo/a.py', '/x/a.py', @@ -91,11 +98,62 @@ def test_01_absolute_dir_paths_2(self): 'foo/a.py', 'x/a.py', 'x/foo/a.py', - ])) - self.assertEqual(results, { + } + + results = list(spec.check_files(files)) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { '/foo/a.py', 'foo/a.py', - }) + }, debug) + + def test_01_check_files(self): + """ + Test that checking files one at a time yields the same results as checking + multiples files at once. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + '*.txt', + '!test1/**', + ]) + files = { + 'src/test1/a.txt', + 'src/test1/b.txt', + 'src/test1/c/c.txt', + 'src/test2/a.txt', + 'src/test2/b.txt', + 'src/test2/c/c.txt', + } + + single_results = set(map(spec.check_file, files)) + multi_results = set(spec.check_files(files)) + + self.assertEqual(single_results, multi_results) + + def test_01_check_match_files(self): + """ + Test that checking files and matching files yield the same results. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + '*.txt', + '!test1/**', + ]) + files = { + 'src/test1/a.txt', + 'src/test1/b.txt', + 'src/test1/c/c.txt', + 'src/test2/a.txt', + 'src/test2/b.txt', + 'src/test2/c/c.txt', + } + + check_results = set(spec.check_files(files)) + check_includes = get_includes(check_results) + match_files = set(spec.match_files(files)) + + self.assertEqual(check_includes, match_files) def test_01_current_dir_paths(self): """ @@ -106,21 +164,26 @@ def test_01_current_dir_paths(self): '*.txt', '!test1/', ]) - results = set(spec.match_files([ + files = { './src/test1/a.txt', './src/test1/b.txt', './src/test1/c/c.txt', './src/test2/a.txt', './src/test2/b.txt', './src/test2/c/c.txt', - ])) - self.assertEqual(results, { + } + + results = list(spec.check_files(files)) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { './src/test2/a.txt', './src/test2/b.txt', './src/test2/c/c.txt', - }) + }, debug) - def test_01_empty_path(self): + def test_01_empty_path_1(self): """ Tests that patterns that end with an escaped space will be treated properly. """ @@ -128,44 +191,55 @@ def test_01_empty_path(self): '\\ ', 'abc\\ ' ]) - test_files = [ + files = { ' ', ' ', 'abc ', 'somefile', - ] - results = list(filter(spec.match_file, test_files)) - self.assertEqual(results, [ + } + + results = list(spec.check_files(files)) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { ' ', 'abc ' - ]) + }, debug) - # An escape with double spaces is invalid. - # Disallow it. Better to be safe than sorry. - self.assertRaises(GitWildMatchPatternError, lambda: PathSpec.from_lines('gitwildmatch', [ - '\\ ' - ])) + def test_01_empty_path_2(self): + """ + Tests that patterns that end with an escaped space will be treated properly. + """ + with self.assertRaises(GitWildMatchPatternError): + # An escape with double spaces is invalid. Disallow it. Better to be safe + # than sorry. + PathSpec.from_lines('gitwildmatch', [ + '\\ ', + ]) def test_01_match_files(self): """ - Tests that matching files one at a time yields the same results as - matching multiples files at once. + Test that matching files one at a time yields the same results as matching + multiples files at once. """ spec = PathSpec.from_lines('gitwildmatch', [ '*.txt', - '!test1/', + '!test1/**', ]) - test_files = [ + files = { 'src/test1/a.txt', 'src/test1/b.txt', 'src/test1/c/c.txt', 'src/test2/a.txt', 'src/test2/b.txt', 'src/test2/c/c.txt', - ] - single_results = set(filter(spec.match_file, test_files)) - multi_results = set(spec.match_files(test_files)) - self.assertEqual(single_results, multi_results) + } + + single_files = set(filter(spec.match_file, files)) + multi_files = set(spec.match_files(files)) + + self.assertEqual(single_files, multi_files) def test_01_windows_current_dir_paths(self): """ @@ -176,19 +250,24 @@ def test_01_windows_current_dir_paths(self): '*.txt', '!test1/', ]) - results = set(spec.match_files([ + files = { '.\\src\\test1\\a.txt', '.\\src\\test1\\b.txt', '.\\src\\test1\\c\\c.txt', '.\\src\\test2\\a.txt', '.\\src\\test2\\b.txt', '.\\src\\test2\\c\\c.txt', - ], separators=('\\',))) - self.assertEqual(results, { + } + + results = list(spec.check_files(files, separators=['\\'])) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { '.\\src\\test2\\a.txt', '.\\src\\test2\\b.txt', '.\\src\\test2\\c\\c.txt', - }) + }, debug) def test_01_windows_paths(self): """ @@ -198,19 +277,24 @@ def test_01_windows_paths(self): '*.txt', '!test1/', ]) - results = set(spec.match_files([ + files = { 'src\\test1\\a.txt', 'src\\test1\\b.txt', 'src\\test1\\c\\c.txt', 'src\\test2\\a.txt', 'src\\test2\\b.txt', 'src\\test2\\c\\c.txt', - ], separators=('\\',))) - self.assertEqual(results, { + } + + results = list(spec.check_files(files, separators=['\\'])) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { 'src\\test2\\a.txt', 'src\\test2\\b.txt', 'src\\test2\\c\\c.txt', - }) + }, debug) def test_02_eq(self): """ @@ -218,11 +302,11 @@ def test_02_eq(self): """ first_spec = PathSpec.from_lines('gitwildmatch', [ '*.txt', - '!test1/', + '!test1/**', ]) second_spec = PathSpec.from_lines('gitwildmatch', [ '*.txt', - '!test1/', + '!test1/**', ]) self.assertEqual(first_spec, second_spec) @@ -243,51 +327,61 @@ def test_03_add(self): Test spec addition using :data:`+` operator. """ first_spec = PathSpec.from_lines('gitwildmatch', [ + 'test.png', 'test.txt', - 'test.png' ]) second_spec = PathSpec.from_lines('gitwildmatch', [ 'test.html', - 'test.jpg' + 'test.jpg', ]) combined_spec = first_spec + second_spec - results = set(combined_spec.match_files([ - 'test.txt', - 'test.png', + files = { 'test.html', - 'test.jpg' - ])) - self.assertEqual(results, { - 'test.txt', + 'test.jpg', 'test.png', + 'test.txt', + } + + results = list(combined_spec.check_files(files)) + includes = get_includes(results) + debug = debug_results(combined_spec, results) + + self.assertEqual(includes, { 'test.html', - 'test.jpg' - }) + 'test.jpg', + 'test.png', + 'test.txt', + }, debug) def test_03_iadd(self): """ Test spec addition using :data:`+=` operator. """ spec = PathSpec.from_lines('gitwildmatch', [ + 'test.png', 'test.txt', - 'test.png' ]) spec += PathSpec.from_lines('gitwildmatch', [ 'test.html', - 'test.jpg' + 'test.jpg', ]) - results = set(spec.match_files([ - 'test.txt', - 'test.png', + files = { 'test.html', - 'test.jpg' - ])) - self.assertEqual(results, { - 'test.txt', + 'test.jpg', 'test.png', + 'test.txt', + } + + results = list(spec.check_files(files)) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { 'test.html', - 'test.jpg' - }) + 'test.jpg', + 'test.png', + 'test.txt', + }, debug) def test_04_len(self): """ @@ -321,12 +415,13 @@ def test_05_match_entries(self): 'Y/b.txt', 'Y/Z/c.txt', ]) + entries = iter_tree_entries(self.temp_dir) - results = { - __entry.path - for __entry in spec.match_entries(entries) + includes = { + __entry.path for __entry in spec.match_entries(entries) } - self.assertEqual(results, set(map(ospath, [ + + self.assertEqual(includes, set(map(ospath, [ 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', @@ -341,15 +436,18 @@ def test_05_match_file(self): '*.txt', '!b.txt', ]) - results = set(filter(spec.match_file, [ + files = { 'X/a.txt', 'X/b.txt', 'X/Z/c.txt', 'Y/a.txt', 'Y/b.txt', 'Y/Z/c.txt', - ])) - self.assertEqual(results, { + } + + includes = set(filter(spec.match_file, files)) + + self.assertEqual(includes, { 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', @@ -364,15 +462,18 @@ def test_05_match_files(self): '*.txt', '!b.txt', ]) - results = set(spec.match_files([ + files = { 'X/a.txt', 'X/b.txt', 'X/Z/c.txt', 'Y/a.txt', 'Y/b.txt', 'Y/Z/c.txt', - ])) - self.assertEqual(results, { + } + + includes = set(spec.match_files(files)) + + self.assertEqual(includes, { 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', @@ -401,11 +502,12 @@ def test_05_match_tree_entries(self): 'Y/b.txt', 'Y/Z/c.txt', ]) - results = { - __entry.path - for __entry in spec.match_tree_entries(self.temp_dir) + + includes = { + __entry.path for __entry in spec.match_tree_entries(self.temp_dir) } - self.assertEqual(results, set(map(ospath, [ + + self.assertEqual(includes, set(map(ospath, [ 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', @@ -434,8 +536,10 @@ def test_05_match_tree_files(self): 'Y/b.txt', 'Y/Z/c.txt', ]) - results = set(spec.match_tree_files(self.temp_dir)) - self.assertEqual(results, set(map(ospath, [ + + includes = set(spec.match_tree_files(self.temp_dir)) + + self.assertEqual(includes, set(map(ospath, [ 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', @@ -444,8 +548,8 @@ def test_05_match_tree_files(self): def test_06_issue_41_a(self): """ - Test including a file and excluding a directory with the same name - pattern, scenario A. + Test including a file and excluding a directory with the same name pattern, + scenario A. """ spec = PathSpec.from_lines('gitwildmatch', [ '*.yaml', @@ -460,19 +564,23 @@ def test_06_issue_41_a(self): 'dir/index.txt', 'file.yaml', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { #'dir.yaml/file.yaml', # Discrepancy with Git. 'dir/file.yaml', 'file.yaml', - }) + }, debug) self.assertEqual(files - ignores, { 'dir.yaml/file.sql', 'dir.yaml/file.yaml', # Discrepancy with Git. 'dir.yaml/index.txt', 'dir/file.sql', 'dir/index.txt', - }) + }, debug) def test_06_issue_41_b(self): """ @@ -492,18 +600,22 @@ def test_06_issue_41_b(self): 'dir/index.txt', 'file.yaml', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'dir.yaml/file.sql', 'dir.yaml/file.yaml', 'dir.yaml/index.txt', 'dir/file.yaml', 'file.yaml', - }) + }, debug) self.assertEqual(files - ignores, { 'dir/file.sql', 'dir/index.txt', - }) + }, debug) def test_06_issue_41_c(self): """ @@ -523,19 +635,23 @@ def test_06_issue_41_c(self): 'dir/index.txt', 'file.yaml', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { #'dir.yaml/file.yaml', # Discrepancy with Git. 'dir/file.yaml', 'file.yaml', - }) + }, debug) self.assertEqual(files - ignores, { 'dir.yaml/file.sql', 'dir.yaml/file.yaml', # Discrepancy with Git. 'dir.yaml/index.txt', 'dir/file.sql', 'dir/index.txt', - }) + }, debug) def test_07_issue_62(self): """ @@ -545,13 +661,18 @@ def test_07_issue_62(self): '*', '!product_dir/', ]) - results = set(spec.match_files([ + files = { 'anydir/file.txt', - 'product_dir/file.txt' - ])) - self.assertEqual(results, { + 'product_dir/file.txt', + } + + results = list(spec.check_files(files)) + includes = get_includes(results) + debug = debug_results(spec, results) + + self.assertEqual(includes, { 'anydir/file.txt', - }) + }, debug) def test_08_issue_39(self): """ @@ -569,16 +690,20 @@ def test_08_issue_39(self): 'important/e.txt', 'trace.c', } - ignores = set(spec.match_files(files)) + + results = list(spec.check_files(files)) + ignores = get_includes(results) + debug = debug_results(spec, results) + self.assertEqual(ignores, { 'a.log', 'trace.c', - }) + }, debug) self.assertEqual(files - ignores, { 'b.txt', 'important/d.log', 'important/e.txt', - }) + }, debug) def test_09_issue_80_a(self): """ @@ -599,7 +724,9 @@ def test_09_issue_80_a(self): 'build/trace.bin', 'trace.c', } + keeps = set(spec.match_files(files, negate=True)) + self.assertEqual(keeps, { '.gitignore', 'b.txt', @@ -625,7 +752,9 @@ def test_09_issue_80_b(self): 'build/trace.bin', 'trace.c', } + keeps = set(spec.match_files(files, negate=True)) ignores = set(spec.match_files(files)) + self.assertEqual(files - ignores, keeps) self.assertEqual(files - keeps, ignores) From 5fda810ef51900b7d46a728d247a7f70cb6543cd Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:13:23 -0500 Subject: [PATCH 31/42] Fix issue 81 --- CHANGES.rst | 2 + pathspec/gitignore.py | 3 - pathspec/patterns/gitwildmatch.py | 208 +++++++++++++++--------------- tests/test_02_gitwildmatch.py | 47 ++++++- 4 files changed, 147 insertions(+), 113 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0610f84..9916547 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,7 @@ New features: Bug fixes: +- `Issue #81`_: GitIgnoreSpec behaviors differ from git. - `Pull #83`_: Fix ReadTheDocs builds. Improvements: @@ -32,6 +33,7 @@ Improvements: - Improve type hint on *on_error* parameter on `pathspec.util.iter_tree_entries()`. +.. _`Issue #81`: https://github.com/cpburnz/python-pathspec/issues/81 .. _`Pull #82`: https://github.com/cpburnz/python-pathspec/pull/82 .. _`Pull #83`: https://github.com/cpburnz/python-pathspec/pull/83 diff --git a/pathspec/gitignore.py b/pathspec/gitignore.py index a8d3c43..994a2c7 100644 --- a/pathspec/gitignore.py +++ b/pathspec/gitignore.py @@ -138,9 +138,6 @@ def _match_file( # Check for directory marker. dir_mark = match.match.groupdict().get(_DIR_MARK) - # TODO: A exclude (whitelist) dir pattern here needs to deprioritize - # for 81-c. - if dir_mark: # Pattern matched by a directory pattern. priority = 1 diff --git a/pathspec/patterns/gitwildmatch.py b/pathspec/patterns/gitwildmatch.py index 5c00086..6a3d6d5 100644 --- a/pathspec/patterns/gitwildmatch.py +++ b/pathspec/patterns/gitwildmatch.py @@ -1,15 +1,14 @@ """ -This module implements Git's wildmatch pattern matching which itself is -derived from Rsync's wildmatch. Git uses wildmatch for its ".gitignore" -files. +This module implements Git's wildmatch pattern matching which itself is derived +from Rsync's wildmatch. Git uses wildmatch for its ".gitignore" files. """ import re import warnings from typing import ( AnyStr, - Optional, - Tuple) + Optional, # Replaced by `X | None` in 3.10. + Tuple) # Replaced by `tuple` in 3.9. from .. import util from ..pattern import RegexPattern @@ -36,8 +35,8 @@ class GitWildMatchPatternError(ValueError): class GitWildMatchPattern(RegexPattern): """ - The :class:`GitWildMatchPattern` class represents a compiled Git - wildmatch pattern. + The :class:`GitWildMatchPattern` class represents a compiled Git wildmatch + pattern. """ # Keep the dict-less class hierarchy. @@ -51,13 +50,12 @@ def pattern_to_regex( """ Convert the pattern into a regular expression. - *pattern* (:class:`str` or :class:`bytes`) is the pattern to convert - into a regular expression. + *pattern* (:class:`str` or :class:`bytes`) is the pattern to convert into a + regular expression. - Returns the uncompiled regular expression (:class:`str`, :class:`bytes`, - or :data:`None`); and whether matched files should be included - (:data:`True`), excluded (:data:`False`), or if it is a - null-operation (:data:`None`). + Returns the uncompiled regular expression (:class:`str`, :class:`bytes`, or + :data:`None`); and whether matched files should be included (:data:`True`), + excluded (:data:`False`), or if it is a null-operation (:data:`None`). """ if isinstance(pattern, str): return_type = str @@ -70,51 +68,52 @@ def pattern_to_regex( original_pattern = pattern if pattern.endswith('\\ '): - # EDGE CASE: Spaces can be escaped with backslash. - # If a pattern that ends with backslash followed by a space, - # only strip from left. + # EDGE CASE: Spaces can be escaped with backslash. If a pattern that ends + # with backslash followed by a space, only strip from left. pattern = pattern.lstrip() else: pattern = pattern.strip() if pattern.startswith('#'): - # A pattern starting with a hash ('#') serves as a comment - # (neither includes nor excludes files). Escape the hash with a - # back-slash to match a literal hash (i.e., '\#'). + # A pattern starting with a hash ('#') serves as a comment (neither + # includes nor excludes files). Escape the hash with a back-slash to match + # a literal hash (i.e., '\#'). regex = None include = None elif pattern == '/': - # EDGE CASE: According to `git check-ignore` (v2.4.1), a single - # '/' does not match any file. + # EDGE CASE: According to `git check-ignore` (v2.4.1), a single '/' does + # not match any file. regex = None include = None elif pattern: if pattern.startswith('!'): - # A pattern starting with an exclamation mark ('!') negates the - # pattern (exclude instead of include). Escape the exclamation - # mark with a back-slash to match a literal exclamation mark - # (i.e., '\!'). + # A pattern starting with an exclamation mark ('!') negates the pattern + # (exclude instead of include). Escape the exclamation mark with a + # back-slash to match a literal exclamation mark (i.e., '\!'). include = False # Remove leading exclamation mark. pattern = pattern[1:] else: include = True - # Allow a regex override for edge cases that cannot be handled - # through normalization. + # Allow a regex override for edge cases that cannot be handled through + # normalization. override_regex = None # Split pattern into segments. pattern_segs = pattern.split('/') + # Check whether the pattern is specifically a directory pattern before + # normalization. + is_dir_pattern = not pattern_segs[-1] + # Normalize pattern to make processing easier. - # EDGE CASE: Deal with duplicate double-asterisk sequences. - # Collapse each sequence down to one double-asterisk. Iterate over - # the segments in reverse and remove the duplicate double - # asterisks as we go. + # EDGE CASE: Deal with duplicate double-asterisk sequences. Collapse each + # sequence down to one double-asterisk. Iterate over the segments in + # reverse and remove the duplicate double asterisks as we go. for i in range(len(pattern_segs) - 1, 0, -1): prev = pattern_segs[i-1] seg = pattern_segs[i] @@ -122,45 +121,42 @@ def pattern_to_regex( del pattern_segs[i] if len(pattern_segs) == 2 and pattern_segs[0] == '**' and not pattern_segs[1]: - # EDGE CASE: The '**/' pattern should match everything except - # individual files in the root directory. This case cannot be - # adequately handled through normalization. Use the override. + # EDGE CASE: The '**/' pattern should match everything except individual + # files in the root directory. This case cannot be adequately handled + # through normalization. Use the override. override_regex = f'^.+(?P<{_DIR_MARK}>/).*$' if not pattern_segs[0]: - # A pattern beginning with a slash ('/') will only match paths - # directly on the root directory instead of any descendant - # paths. So, remove empty first segment to make pattern relative - # to root. + # A pattern beginning with a slash ('/') will only match paths directly + # on the root directory instead of any descendant paths. So, remove + # empty first segment to make pattern relative to root. del pattern_segs[0] elif len(pattern_segs) == 1 or (len(pattern_segs) == 2 and not pattern_segs[1]): - # A single pattern without a beginning slash ('/') will match - # any descendant path. This is equivalent to "**/{pattern}". So, - # prepend with double-asterisks to make pattern relative to - # root. - # EDGE CASE: This also holds for a single pattern with a - # trailing slash (e.g. dir/). + # A single pattern without a beginning slash ('/') will match any + # descendant path. This is equivalent to "**/{pattern}". So, prepend + # with double-asterisks to make pattern relative to root. + # - EDGE CASE: This also holds for a single pattern with a trailing + # slash (e.g. dir/). if pattern_segs[0] != '**': pattern_segs.insert(0, '**') else: - # EDGE CASE: A pattern without a beginning slash ('/') but - # contains at least one prepended directory (e.g. - # "dir/{pattern}") should not match "**/dir/{pattern}", - # according to `git check-ignore` (v2.4.1). + # EDGE CASE: A pattern without a beginning slash ('/') but contains at + # least one prepended directory (e.g. "dir/{pattern}") should not match + # "**/dir/{pattern}", according to `git check-ignore` (v2.4.1). pass if not pattern_segs: - # After resolving the edge cases, we end up with no pattern at - # all. This must be because the pattern is invalid. + # After resolving the edge cases, we end up with no pattern at all. This + # must be because the pattern is invalid. raise GitWildMatchPatternError(f"Invalid git pattern: {original_pattern!r}") if not pattern_segs[-1] and len(pattern_segs) > 1: - # A pattern ending with a slash ('/') will match all descendant - # paths if it is a directory but not if it is a regular file. - # This is equivalent to "{pattern}/**". So, set last segment to - # a double-asterisk to include all descendants. + # A pattern ending with a slash ('/') will match all descendant paths if + # it is a directory but not if it is a regular file. This is equivalent + # to "{pattern}/**". So, set last segment to a double-asterisk to + # include all descendants. pattern_segs[-1] = '**' if override_regex is None: @@ -171,21 +167,27 @@ def pattern_to_regex( for i, seg in enumerate(pattern_segs): if seg == '**': if i == 0 and i == end: - # A pattern consisting solely of double-asterisks ('**') - # will match every path. - output.append(f'[^/]+(?:(?P<{_DIR_MARK}>/).*)?') + # A pattern consisting solely of double-asterisks ('**') will + # match every path. + output.append(f'[^/]+(?:/.*)?') + elif i == 0: # A normalized pattern beginning with double-asterisks # ('**') will match any leading path segments. output.append('(?:.+/)?') need_slash = False + elif i == end: - # A normalized pattern ending with double-asterisks ('**') - # will match any trailing path segments. - output.append(f'(?P<{_DIR_MARK}>/).*') + # A normalized pattern ending with double-asterisks ('**') will + # match any trailing path segments. + if is_dir_pattern: + output.append(f'(?P<{_DIR_MARK}>/).*') + else: + output.append(f'/.*') + else: - # A pattern with inner double-asterisks ('**') will match - # multiple (or zero) inner path segments. + # A pattern with inner double-asterisks ('**') will match multiple + # (or zero) inner path segments. output.append('(?:/.+)?') need_slash = True @@ -197,9 +199,9 @@ def pattern_to_regex( output.append('[^/]+') if i == end: - # A pattern ending without a slash ('/') will match a file - # or a directory (with paths underneath it). E.g., "foo" - # matches "foo", "foo/bar", "foo/bar/baz", etc. + # A pattern ending without a slash ('/') will match a file or a + # directory (with paths underneath it). E.g., "foo" matches "foo", + # "foo/bar", "foo/bar/baz", etc. output.append(f'(?:(?P<{_DIR_MARK}>/).*)?') need_slash = True @@ -215,9 +217,9 @@ def pattern_to_regex( raise GitWildMatchPatternError(f"Invalid git pattern: {original_pattern!r}") from e if i == end: - # A pattern ending without a slash ('/') will match a file - # or a directory (with paths underneath it). E.g., "foo" - # matches "foo", "foo/bar", "foo/bar/baz", etc. + # A pattern ending without a slash ('/') will match a file or a + # directory (with paths underneath it). E.g., "foo" matches "foo", + # "foo/bar", "foo/bar/baz", etc. output.append(f'(?:(?P<{_DIR_MARK}>/).*)?') need_slash = True @@ -230,8 +232,8 @@ def pattern_to_regex( regex = override_regex else: - # A blank pattern is a null-operation (neither includes nor - # excludes files). + # A blank pattern is a null-operation (neither includes nor excludes + # files). regex = None include = None @@ -243,16 +245,16 @@ def pattern_to_regex( @staticmethod def _translate_segment_glob(pattern: str) -> str: """ - Translates the glob pattern to a regular expression. This is used in - the constructor to translate a path segment glob pattern to its - corresponding regular expression. + Translates the glob pattern to a regular expression. This is used in the + constructor to translate a path segment glob pattern to its corresponding + regular expression. *pattern* (:class:`str`) is the glob pattern. Returns the regular expression (:class:`str`). """ - # NOTE: This is derived from `fnmatch.translate()` and is similar to - # the POSIX function `fnmatch()` with the `FNM_PATHNAME` flag set. + # NOTE: This is derived from `fnmatch.translate()` and is similar to the + # POSIX function `fnmatch()` with the `FNM_PATHNAME` flag set. escape = False regex = '' @@ -272,41 +274,40 @@ def _translate_segment_glob(pattern: str) -> str: escape = True elif char == '*': - # Multi-character wildcard. Match any string (except slashes), - # including an empty string. + # Multi-character wildcard. Match any string (except slashes), including + # an empty string. regex += '[^/]*' elif char == '?': - # Single-character wildcard. Match any single character (except - # a slash). + # Single-character wildcard. Match any single character (except a + # slash). regex += '[^/]' elif char == '[': - # Bracket expression wildcard. Except for the beginning - # exclamation mark, the whole bracket expression can be used - # directly as regex but we have to find where the expression - # ends. + # Bracket expression wildcard. Except for the beginning exclamation + # mark, the whole bracket expression can be used directly as regex, but + # we have to find where the expression ends. # - "[][!]" matches ']', '[' and '!'. # - "[]-]" matches ']' and '-'. # - "[!]a-]" matches any character except ']', 'a' and '-'. j = i - + # Pass bracket expression negation. if j < end and (pattern[j] == '!' or pattern[j] == '^'): j += 1 - + # Pass first closing bracket if it is at the beginning of the # expression. if j < end and pattern[j] == ']': j += 1 - + # Find closing bracket. Stop once we reach the end or find it. while j < end and pattern[j] != ']': j += 1 if j < end: - # Found end of bracket expression. Increment j to be one past - # the closing bracket: + # Found end of bracket expression. Increment j to be one past the + # closing bracket: # # [...] # ^ ^ @@ -320,17 +321,16 @@ def _translate_segment_glob(pattern: str) -> str: expr += '^' i += 1 elif pattern[i] == '^': - # POSIX declares that the regex bracket expression negation - # "[^...]" is undefined in a glob pattern. Python's - # `fnmatch.translate()` escapes the caret ('^') as a - # literal. Git supports the using a caret for negation. - # Maintain consistency with Git because that is the expected - # behavior. + # POSIX declares that the regex bracket expression negation "[^...]" + # is undefined in a glob pattern. Python's `fnmatch.translate()` + # escapes the caret ('^') as a literal. Git supports the using a + # caret for negation. Maintain consistency with Git because that is + # the expected behavior. expr += '^' i += 1 - # Build regex bracket expression. Escape slashes so they are - # treated as literal slashes by regex as defined by POSIX. + # Build regex bracket expression. Escape slashes so they are treated + # as literal slashes by regex as defined by POSIX. expr += pattern[i:j].replace('\\', '\\\\') # Add regex bracket expression to regex result. @@ -340,8 +340,8 @@ def _translate_segment_glob(pattern: str) -> str: i = j else: - # Failed to find closing bracket, treat opening bracket as a - # bracket literal instead of as an expression. + # Failed to find closing bracket, treat opening bracket as a bracket + # literal instead of as an expression. regex += '\\[' else: @@ -358,8 +358,8 @@ def escape(s: AnyStr) -> AnyStr: """ Escape special characters in the given string. - *s* (:class:`str` or :class:`bytes`) a filename or a string that you - want to escape, usually before adding it to a ".gitignore". + *s* (:class:`str` or :class:`bytes`) a filename or a string that you want to + escape, usually before adding it to a ".gitignore". Returns the escaped string (:class:`str` or :class:`bytes`). """ @@ -404,8 +404,8 @@ def _deprecated() -> None: Warn about deprecation. """ warnings.warn(( - "GitIgnorePattern ('gitignore') is deprecated. Use " - "GitWildMatchPattern ('gitwildmatch') instead." + "GitIgnorePattern ('gitignore') is deprecated. Use GitWildMatchPattern " + "('gitwildmatch') instead." ), DeprecationWarning, stacklevel=3) @classmethod @@ -416,6 +416,6 @@ def pattern_to_regex(cls, *args, **kw): cls._deprecated() return super(GitIgnorePattern, cls).pattern_to_regex(*args, **kw) -# Register `GitIgnorePattern` as "gitignore" for backward compatibility -# with v0.4. +# Register `GitIgnorePattern` as "gitignore" for backward compatibility with +# v0.4. util.register_pattern('gitignore', GitIgnorePattern) diff --git a/tests/test_02_gitwildmatch.py b/tests/test_02_gitwildmatch.py index be915cf..3d272a4 100644 --- a/tests/test_02_gitwildmatch.py +++ b/tests/test_02_gitwildmatch.py @@ -208,7 +208,7 @@ def test_03_child_double_asterisk(self): """ regex, include = GitWildMatchPattern.pattern_to_regex('spam/**') self.assertTrue(include) - self.assertEqual(regex, f'^spam{RE_DIR}.*$') + self.assertEqual(regex, "^spam/.*$") pattern = GitWildMatchPattern(re.compile(regex), include) results = set(filter(pattern.match_file, [ @@ -257,7 +257,7 @@ def test_03_only_double_asterisk(self): """ regex, include = GitWildMatchPattern.pattern_to_regex('**') self.assertTrue(include) - self.assertEqual(regex, f'^[^/]+{RE_SUB}$') + self.assertEqual(regex, f'^[^/]+(?:/.*)?$') pattern = GitWildMatchPattern(re.compile(regex), include) results = set(filter(pattern.match_file, [ @@ -314,7 +314,7 @@ def test_03_duplicate_leading_double_asterisk_edge_case(self): """ regex, include = GitWildMatchPattern.pattern_to_regex('**') self.assertTrue(include) - self.assertEqual(regex, f'^[^/]+{RE_SUB}$') + self.assertEqual(regex, "^[^/]+(?:/.*)?$") equivalent_regex, include = GitWildMatchPattern.pattern_to_regex('**/**') self.assertTrue(include) @@ -336,10 +336,14 @@ def test_03_duplicate_leading_double_asterisk_edge_case(self): self.assertTrue(include) self.assertEqual(regex, f'^(?:.+/)?api{RE_DIR}.*$') - equivalent_regex, include = GitWildMatchPattern.pattern_to_regex('**/api/**') + equivalent_regex, include = GitWildMatchPattern.pattern_to_regex(f'**/**/api/') self.assertTrue(include) self.assertEqual(equivalent_regex, regex) + regex, include = GitWildMatchPattern.pattern_to_regex('**/api/**') + self.assertTrue(include) + self.assertEqual(regex, "^(?:.+/)?api/.*$") + equivalent_regex, include = GitWildMatchPattern.pattern_to_regex('**/**/api/**/**') self.assertTrue(include) self.assertEqual(equivalent_regex, regex) @@ -817,10 +821,41 @@ def test_13_issue_77_2_regex(self): """ Test the resulting regex for regex bracket expression negation. """ - regex, include = GitWildMatchPattern.pattern_to_regex('a[^b]c') + regex, include = GitWildMatchPattern.pattern_to_regex("a[^b]c") self.assertTrue(include) - equiv_regex, include = GitWildMatchPattern.pattern_to_regex('a[!b]c') + equiv_regex, include = GitWildMatchPattern.pattern_to_regex("a[!b]c") self.assertTrue(include) self.assertEqual(regex, equiv_regex) + + def test_14_issue_81_a(self): + """ + Test ignoring files in a directory, scenario A. + """ + pattern = GitWildMatchPattern("!libfoo/**") + + self.assertEqual(pattern.regex.pattern, "^libfoo/.*$") + self.assertIs(pattern.include, False) + self.assertTrue(pattern.match_file("libfoo/__init__.py")) + + def test_14_issue_81_b(self): + """ + Test ignoring files in a directory, scenario B. + """ + pattern = GitWildMatchPattern("!libfoo/*") + + self.assertEqual(pattern.regex.pattern, f"^libfoo/[^/]+{RE_SUB}$") + self.assertIs(pattern.include, False) + self.assertTrue(pattern.match_file("libfoo/__init__.py")) + + def test_14_issue_81_c(self): + """ + Test ignoring files in a directory, scenario C. + """ + # GitWildMatchPattern will match the file, but GitIgnoreSpec should not. + pattern = GitWildMatchPattern("!libfoo/") + + self.assertEqual(pattern.regex.pattern, f"^(?:.+/)?libfoo{RE_DIR}.*$") + self.assertIs(pattern.include, False) + self.assertTrue(pattern.match_file("libfoo/__init__.py")) From 42bee7699ceaac630dd67fbb07a7a015dc53e8d1 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 18:02:30 -0500 Subject: [PATCH 32/42] Fix Python 3.8 regression --- tests/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/util.py b/tests/util.py index b845a3e..e16bfd3 100644 --- a/tests/util.py +++ b/tests/util.py @@ -9,7 +9,9 @@ from typing import ( Iterable, # Replaced by `collections.abc.Iterable` in 3.9. - Tuple, # Replaced by `collections.abc.Tuple` in 3.9. + List, # Replaced by `set` in 3.9. + Set, # Replaced by `set` in 3.9. + Tuple, # Replaced by `tuple` in 3.9. cast) from pathspec import ( @@ -31,7 +33,7 @@ def debug_results(spec: PathSpec, results: Iterable[CheckResult[str]]) -> str: Returns the message (:class:`str`). """ - patterns = cast(list[RegexPattern], spec.patterns) + patterns = cast(List[RegexPattern], spec.patterns) pattern_table = [] for index, pattern in enumerate(patterns, 1): @@ -70,7 +72,7 @@ def debug_results(spec: PathSpec, results: Iterable[CheckResult[str]]) -> str: ]) -def get_includes(results: Iterable[CheckResult[TStrPath]]) -> set[TStrPath]: +def get_includes(results: Iterable[CheckResult[TStrPath]]) -> Set[TStrPath]: """ Get the included files from the check results. From 101e6284b77a1a38cf1fff70075cc660d4119e0f Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 18:02:43 -0500 Subject: [PATCH 33/42] Fix doc build --- Makefile | 10 ++++++++-- pathspec/pathspec.py | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 5fd8928..d783ed8 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ # Updated: 2022-09-01 # -.PHONY: build create-venv help prebuild publish test test-all update-venv +.PHONY: build create-venv help prebuild publish test test-all test-doc update-venv help: @echo "Usage: make []" @@ -22,6 +22,7 @@ help: @echo " create-venv Create the development Python virtual environment." @echo " test Run tests using the development virtual environment." @echo " test-all Run tests using Tox for all virtual environments." + @echo " test-doc Run tests using Tox for just documentation." @echo " update-venv Update the development Python virtual environment." build: dist-build @@ -36,6 +37,8 @@ test: dev-test-primary test-all: dev-test-all +test-doc: dev-test-doc + update-venv: dev-venv-install @@ -49,11 +52,14 @@ VENV_DIR := ./dev/venv PYTHON := python3 VENV := ./dev/venv.sh "${VENV_DIR}" -.PHONY: dev-test-all dev-test-primary dev-venv-base dev-venv-create dev-venv-install +.PHONY: dev-test-all dev-test-doc dev-test-primary dev-venv-base dev-venv-create dev-venv-install dev-test-all: ${VENV} tox +dev-test-doc: + ${VENV} tox -e doc + dev-test-primary: ${VENV} python -m unittest -v diff --git a/pathspec/pathspec.py b/pathspec/pathspec.py index ebffede..377f159 100644 --- a/pathspec/pathspec.py +++ b/pathspec/pathspec.py @@ -112,7 +112,7 @@ def check_file( :data:`None`) optionally contains the path separators to normalize. See :func:`~pathspec.util.normalize_file` for more information. - Returns the file check result (:class:`CheckResult`). + Returns the file check result (:class:`~pathspec.util.CheckResult`). """ norm_file = normalize_file(file, separators) include, index = self._match_file(enumerate(self.patterns), norm_file) @@ -135,7 +135,7 @@ def check_files( :func:`~pathspec.util.normalize_file` for more information. Returns an :class:`~collections.abc.Iterator` yielding each file check - result (:class:`CheckResult`). + result (:class:`~pathspec.util.CheckResult`). """ if not _is_iterable(files): raise TypeError(f"files:{files!r} is not an iterable.") @@ -174,7 +174,7 @@ def check_tree_files( :data:`False`. Returns an :class:`~collections.abc.Iterator` yielding each file check - result (:class:`CheckResult`). + result (:class:`~pathspec.util.CheckResult`). """ files = util.iter_tree_files(root, on_error=on_error, follow_links=follow_links) yield from self.check_files(files) From f6bfc89d56734589c0a6e9737413b849b49705ac Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 18:14:50 -0500 Subject: [PATCH 34/42] Fix docs build --- Makefile | 12 ++++++------ pathspec/util.py | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index d783ed8..f88a705 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ # Updated: 2022-09-01 # -.PHONY: build create-venv help prebuild publish test test-all test-doc update-venv +.PHONY: build create-venv help prebuild publish test test-all test-docs update-venv help: @echo "Usage: make []" @@ -22,7 +22,7 @@ help: @echo " create-venv Create the development Python virtual environment." @echo " test Run tests using the development virtual environment." @echo " test-all Run tests using Tox for all virtual environments." - @echo " test-doc Run tests using Tox for just documentation." + @echo " test-docs Run tests using Tox for just documentation." @echo " update-venv Update the development Python virtual environment." build: dist-build @@ -37,7 +37,7 @@ test: dev-test-primary test-all: dev-test-all -test-doc: dev-test-doc +test-docs: dev-test-docs update-venv: dev-venv-install @@ -52,13 +52,13 @@ VENV_DIR := ./dev/venv PYTHON := python3 VENV := ./dev/venv.sh "${VENV_DIR}" -.PHONY: dev-test-all dev-test-doc dev-test-primary dev-venv-base dev-venv-create dev-venv-install +.PHONY: dev-test-all dev-test-docs dev-test-primary dev-venv-base dev-venv-create dev-venv-install dev-test-all: ${VENV} tox -dev-test-doc: - ${VENV} tox -e doc +dev-test-docs: + ${VENV} tox -e docs dev-test-primary: ${VENV} python -m unittest -v diff --git a/pathspec/util.py b/pathspec/util.py index 54e1b2c..5883951 100644 --- a/pathspec/util.py +++ b/pathspec/util.py @@ -42,6 +42,9 @@ StrPath = Union[str, PathLike] TStrPath = TypeVar("TStrPath", bound=StrPath) +""" +Type variable for :class:`str` or :class:`os.PathLike`. +""" NORMALIZE_PATH_SEPS = [ __sep From 37e289515fbae7e92077a2438996046e79cae5bf Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sat, 9 Dec 2023 18:21:33 -0500 Subject: [PATCH 35/42] Release v0.12.0 --- CHANGES.rst | 5 ++--- README-dist.rst | 34 ++++++++++++++++++++++++++++++++++ pathspec/_meta.py | 2 +- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9916547..b11b4a2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,8 +2,8 @@ Change History ============== -0.12.0 (TDB) ------------- +0.12.0 (2023-12-09) +------------------- Major changes: @@ -19,7 +19,6 @@ New features: - Added `pathspec.pathspec.PathSpec.check_*()` methods. These methods behave similarly to `.match_*()` but return additional information in the `pathspec.util.CheckResult` objects (e.g., `CheckResult.index` indicates the index of the last pattern that matched the file). - Added `pathspec.pattern.RegexPattern.pattern` attribute which stores the original, uncompiled pattern. - Bug fixes: - `Issue #81`_: GitIgnoreSpec behaviors differ from git. diff --git a/README-dist.rst b/README-dist.rst index 5eca0da..74d29d8 100644 --- a/README-dist.rst +++ b/README-dist.rst @@ -169,6 +169,40 @@ a `Ruby gem`_. Change History ============== +0.12.0 (2023-12-09) +------------------- + +Major changes: + +- Dropped support of EOL Python 3.7. See `Pull #82`_. + + +API changes: + +- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. + +New features: + +- Added `pathspec.pathspec.PathSpec.check_*()` methods. These methods behave similarly to `.match_*()` but return additional information in the `pathspec.util.CheckResult` objects (e.g., `CheckResult.index` indicates the index of the last pattern that matched the file). +- Added `pathspec.pattern.RegexPattern.pattern` attribute which stores the original, uncompiled pattern. + +Bug fixes: + +- `Issue #81`_: GitIgnoreSpec behaviors differ from git. +- `Pull #83`_: Fix ReadTheDocs builds. + +Improvements: + +- Mark Python 3.12 as supported. See `Pull #82`_. +- Improve test debugging. +- Improve type hint on *on_error* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. +- Improve type hint on *on_error* parameter on `pathspec.util.iter_tree_entries()`. + + +.. _`Issue #81`: https://github.com/cpburnz/python-pathspec/issues/81 +.. _`Pull #82`: https://github.com/cpburnz/python-pathspec/pull/82 +.. _`Pull #83`: https://github.com/cpburnz/python-pathspec/pull/83 + 0.11.2 (2023-07-28) ------------------- diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 193fa9d..8483e71 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -55,4 +55,4 @@ "kurtmckee ", ] __license__ = "MPL 2.0" -__version__ = "0.12.0.dev1" +__version__ = "0.12.0" From 81368adefcf18b97457958cb50bb04785e63cc6e Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sun, 10 Dec 2023 17:25:57 -0500 Subject: [PATCH 36/42] Fix issue #84 --- CHANGES.rst | 14 +++- Makefile | 3 - pathspec/_meta.py | 2 +- pathspec/pathspec.py | 2 +- tests/test_01_util.py | 133 ++++++++++++++++++++++++++++++++-- tests/test_03_pathspec.py | 145 +++++++++++++++++++++++++++++--------- 6 files changed, 253 insertions(+), 46 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b11b4a2..bbae426 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,18 @@ Change History ============== + +0.12.1 (TBD) +------------------- + +- Bug fixes: + +- `Issue #84`_: PathSpec.match_file() returns None since 0.12.0. + + +.. _`Issue #84`: https://github.com/cpburnz/python-pathspec/issues/84 + + 0.12.0 (2023-12-09) ------------------- @@ -12,7 +24,7 @@ Major changes: API changes: -- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. +- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` (with a leading underscore) has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. New features: diff --git a/Makefile b/Makefile index f88a705..559e09d 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,6 @@ # # This Makefile is used to manage development and distribution. # -# Created: 2022-08-11 -# Updated: 2022-09-01 -# .PHONY: build create-venv help prebuild publish test test-all test-docs update-venv diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 8483e71..173c4a4 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -55,4 +55,4 @@ "kurtmckee ", ] __license__ = "MPL 2.0" -__version__ = "0.12.0" +__version__ = "0.12.1.dev1" diff --git a/pathspec/pathspec.py b/pathspec/pathspec.py index 377f159..bdfaccd 100644 --- a/pathspec/pathspec.py +++ b/pathspec/pathspec.py @@ -277,7 +277,7 @@ def match_file( """ norm_file = normalize_file(file, separators) include, _index = self._match_file(enumerate(self.patterns), norm_file) - return include + return bool(include) def match_files( self, diff --git a/tests/test_01_util.py b/tests/test_01_util.py index 67c9b41..1385fe7 100644 --- a/tests/test_01_util.py +++ b/tests/test_01_util.py @@ -12,14 +12,15 @@ from functools import ( partial) from typing import ( - Iterable, - Optional, - Tuple) + Iterable, # Replaced by `collections.abc.Iterable` in 3.9. + Optional, # Replaced by `X | None` in 3.10. + Tuple) # Replaced by `tuple` in 3.9. from pathspec.patterns.gitwildmatch import ( GitWildMatchPattern) from pathspec.util import ( RecursionError, + check_match_file, iter_tree_entries, iter_tree_files, match_file, @@ -32,6 +33,82 @@ ospath) +class CheckMatchFileTest(unittest.TestCase): + """ + The :class:`CheckMatchFileTest` class tests the :meth:`.check_match_file` + function. + """ + + def test_01_single_1_include(self): + """ + Test checking a single file that is included. + """ + patterns = list(enumerate(map(GitWildMatchPattern, [ + "*.txt", + "!test/", + ]))) + + include_index = check_match_file(patterns, "include.txt") + + self.assertEqual(include_index, (True, 0)) + + def test_01_single_2_exclude(self): + """ + Test checking a single file that is excluded. + """ + patterns = list(enumerate(map(GitWildMatchPattern, [ + "*.txt", + "!test/", + ]))) + + include_index = check_match_file(patterns, "test/exclude.txt") + + self.assertEqual(include_index, (False, 1)) + + def test_01_single_3_unmatch(self): + """ + Test checking a single file that is ignored. + """ + patterns = list(enumerate(map(GitWildMatchPattern, [ + "*.txt", + "!test/", + ]))) + + include_index = check_match_file(patterns, "unmatch.bin") + + self.assertEqual(include_index, (None, None)) + + def test_02_many(self): + """ + Test matching files individually. + """ + patterns = list(enumerate(map(GitWildMatchPattern, [ + '*.txt', + '!b.txt', + ]))) + files = { + 'X/a.txt', + 'X/b.txt', + 'X/Z/c.txt', + 'Y/a.txt', + 'Y/b.txt', + 'Y/Z/c.txt', + } + + includes = { + __file + for __file in files + if check_match_file(patterns, __file)[0] + } + + self.assertEqual(includes, { + 'X/a.txt', + 'X/Z/c.txt', + 'Y/a.txt', + 'Y/Z/c.txt', + }) + + class IterTreeTest(unittest.TestCase): """ The :class:`IterTreeTest` class tests :meth:`.iter_tree_entries` and @@ -345,7 +422,46 @@ class MatchFileTest(unittest.TestCase): function. """ - def test_01_match_file(self): + def test_01_single_1_include(self): + """ + Test checking a single file that is included. + """ + patterns = list(map(GitWildMatchPattern, [ + "*.txt", + "!test/", + ])) + + include = match_file(patterns, "include.txt") + + self.assertIs(include, True) + + def test_01_single_2_exclude(self): + """ + Test checking a single file that is excluded. + """ + patterns = list(map(GitWildMatchPattern, [ + "*.txt", + "!test/", + ])) + + include = match_file(patterns, "test/exclude.txt") + + self.assertIs(include, False) + + def test_01_single_3_unmatch(self): + """ + Test checking a single file that is ignored. + """ + patterns = list(map(GitWildMatchPattern, [ + "*.txt", + "!test/", + ])) + + include = match_file(patterns, "unmatch.bin") + + self.assertIs(include, False) + + def test_02_many(self): """ Test matching files individually. """ @@ -353,15 +469,18 @@ def test_01_match_file(self): '*.txt', '!b.txt', ])) - results = set(filter(partial(match_file, patterns), [ + files = { 'X/a.txt', 'X/b.txt', 'X/Z/c.txt', 'Y/a.txt', 'Y/b.txt', 'Y/Z/c.txt', - ])) - self.assertEqual(results, { + } + + includes = set(filter(partial(match_file, patterns), files)) + + self.assertEqual(includes, { 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', diff --git a/tests/test_03_pathspec.py b/tests/test_03_pathspec.py index f77d18d..d1b18cd 100644 --- a/tests/test_03_pathspec.py +++ b/tests/test_03_pathspec.py @@ -17,6 +17,7 @@ iter_tree_entries) from .util import ( + CheckResult, debug_results, get_includes, make_dirs, @@ -109,22 +110,61 @@ def test_01_absolute_dir_paths_2(self): 'foo/a.py', }, debug) - def test_01_check_files(self): + def test_01_check_file_1_include(self): + """ + Test checking a single file that is included. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + "*.txt", + "!test/", + ]) + + result = spec.check_file("include.txt") + + self.assertEqual(result, CheckResult("include.txt", True, 0)) + + def test_01_check_file_2_exclude(self): + """ + Test checking a single file that is excluded. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + "*.txt", + "!test/", + ]) + + result = spec.check_file("test/exclude.txt") + + self.assertEqual(result, CheckResult("test/exclude.txt", False, 1)) + + def test_01_check_file_3_unmatch(self): + """ + Test checking a single file that is unmatched. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + "*.txt", + "!test/", + ]) + + result = spec.check_file("unmatch.bin") + + self.assertEqual(result, CheckResult("unmatch.bin", None, None)) + + def test_01_check_file_4_many(self): """ Test that checking files one at a time yields the same results as checking multiples files at once. """ spec = PathSpec.from_lines('gitwildmatch', [ '*.txt', - '!test1/**', + '!test1/', ]) files = { - 'src/test1/a.txt', - 'src/test1/b.txt', - 'src/test1/c/c.txt', - 'src/test2/a.txt', - 'src/test2/b.txt', - 'src/test2/c/c.txt', + 'test1/a.txt', + 'test1/b.txt', + 'test1/c/c.txt', + 'test2/a.txt', + 'test2/b.txt', + 'test2/c/c.txt', } single_results = set(map(spec.check_file, files)) @@ -218,6 +258,45 @@ def test_01_empty_path_2(self): '\\ ', ]) + def test_01_match_file_1_include(self): + """ + Test matching a single file that is included. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + "*.txt", + "!test/", + ]) + + include = spec.match_file("include.txt") + + self.assertIs(include, True) + + def test_01_match_file_2_exclude(self): + """ + Test matching a single file that is excluded. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + "*.txt", + "!test/", + ]) + + include = spec.match_file("test/exclude.txt") + + self.assertIs(include, False) + + def test_01_match_file_3_unmatch(self): + """ + Test match a single file that is unmatched. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + "*.txt", + "!test/", + ]) + + include = spec.match_file("unmatch.bin") + + self.assertIs(include, False) + def test_01_match_files(self): """ Test that matching files one at a time yields the same results as matching @@ -225,15 +304,15 @@ def test_01_match_files(self): """ spec = PathSpec.from_lines('gitwildmatch', [ '*.txt', - '!test1/**', + '!test1/', ]) files = { - 'src/test1/a.txt', - 'src/test1/b.txt', - 'src/test1/c/c.txt', - 'src/test2/a.txt', - 'src/test2/b.txt', - 'src/test2/c/c.txt', + 'test1/a.txt', + 'test1/b.txt', + 'test1/c/c.txt', + 'test2/a.txt', + 'test2/b.txt', + 'test2/c/c.txt', } single_files = set(filter(spec.match_file, files)) @@ -251,12 +330,12 @@ def test_01_windows_current_dir_paths(self): '!test1/', ]) files = { - '.\\src\\test1\\a.txt', - '.\\src\\test1\\b.txt', - '.\\src\\test1\\c\\c.txt', - '.\\src\\test2\\a.txt', - '.\\src\\test2\\b.txt', - '.\\src\\test2\\c\\c.txt', + '.\\test1\\a.txt', + '.\\test1\\b.txt', + '.\\test1\\c\\c.txt', + '.\\test2\\a.txt', + '.\\test2\\b.txt', + '.\\test2\\c\\c.txt', } results = list(spec.check_files(files, separators=['\\'])) @@ -264,9 +343,9 @@ def test_01_windows_current_dir_paths(self): debug = debug_results(spec, results) self.assertEqual(includes, { - '.\\src\\test2\\a.txt', - '.\\src\\test2\\b.txt', - '.\\src\\test2\\c\\c.txt', + '.\\test2\\a.txt', + '.\\test2\\b.txt', + '.\\test2\\c\\c.txt', }, debug) def test_01_windows_paths(self): @@ -278,12 +357,12 @@ def test_01_windows_paths(self): '!test1/', ]) files = { - 'src\\test1\\a.txt', - 'src\\test1\\b.txt', - 'src\\test1\\c\\c.txt', - 'src\\test2\\a.txt', - 'src\\test2\\b.txt', - 'src\\test2\\c\\c.txt', + 'test1\\a.txt', + 'test1\\b.txt', + 'test1\\c\\c.txt', + 'test2\\a.txt', + 'test2\\b.txt', + 'test2\\c\\c.txt', } results = list(spec.check_files(files, separators=['\\'])) @@ -291,9 +370,9 @@ def test_01_windows_paths(self): debug = debug_results(spec, results) self.assertEqual(includes, { - 'src\\test2\\a.txt', - 'src\\test2\\b.txt', - 'src\\test2\\c\\c.txt', + 'test2\\a.txt', + 'test2\\b.txt', + 'test2\\c\\c.txt', }, debug) def test_02_eq(self): From 6485791e1b5cf2ef4e756ae392fa80f2c5045d4c Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sun, 10 Dec 2023 17:28:25 -0500 Subject: [PATCH 37/42] Release v0.12.1 --- CHANGES.rst | 2 +- README-dist.rst | 14 +++++++++++++- pathspec/_meta.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index bbae426..e9c6da1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,7 @@ Change History ============== -0.12.1 (TBD) +0.12.1 (2023-12-10) ------------------- - Bug fixes: diff --git a/README-dist.rst b/README-dist.rst index 74d29d8..01bf7d6 100644 --- a/README-dist.rst +++ b/README-dist.rst @@ -169,6 +169,18 @@ a `Ruby gem`_. Change History ============== + +0.12.1 (2023-12-10) +------------------- + +- Bug fixes: + +- `Issue #84`_: PathSpec.match_file() returns None since 0.12.0. + + +.. _`Issue #84`: https://github.com/cpburnz/python-pathspec/issues/84 + + 0.12.0 (2023-12-09) ------------------- @@ -179,7 +191,7 @@ Major changes: API changes: -- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. +- Signature of protected method `pathspec.pathspec.PathSpec._match_file()` (with a leading underscore) has been changed from `def _match_file(patterns: Iterable[Pattern], file: str) -> bool` to `def _match_file(patterns: Iterable[Tuple[int, Pattern]], file: str) -> Tuple[Optional[bool], Optional[int]]`. New features: diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 173c4a4..4d8c89d 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -55,4 +55,4 @@ "kurtmckee ", ] __license__ = "MPL 2.0" -__version__ = "0.12.1.dev1" +__version__ = "0.12.1" From 8634368a07bd3bf13c30b67ae394b43ba33e2197 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Sun, 10 Dec 2023 17:30:00 -0500 Subject: [PATCH 38/42] Release v0.12.1 --- CHANGES.rst | 2 +- README-dist.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e9c6da1..eb9e11c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,7 @@ Change History 0.12.1 (2023-12-10) ------------------- -- Bug fixes: +Bug fixes: - `Issue #84`_: PathSpec.match_file() returns None since 0.12.0. diff --git a/README-dist.rst b/README-dist.rst index 01bf7d6..e7752fc 100644 --- a/README-dist.rst +++ b/README-dist.rst @@ -173,7 +173,7 @@ Change History 0.12.1 (2023-12-10) ------------------- -- Bug fixes: +Bug fixes: - `Issue #84`_: PathSpec.match_file() returns None since 0.12.0. From 2eddc56099b0272eb53aaef56dcf8e4b4110e200 Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Mon, 10 Feb 2025 21:45:27 -0500 Subject: [PATCH 39/42] Misc --- DEV.md | 6 +++++- pathspec/patterns/gitwildmatch.py | 5 ++++- pyproject.toml | 2 +- tox.ini | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/DEV.md b/DEV.md index fcd9b7f..3d08ada 100644 --- a/DEV.md +++ b/DEV.md @@ -16,7 +16,11 @@ These are notes to myself for things to review before decommissioning EoL versio **Python 3.8:** -- Becomes EoL in 2024-10. +- EoL as of 2024-10-07. + +**Python 3.9:** + +- Becomes EoL in 2025-10. References: diff --git a/pathspec/patterns/gitwildmatch.py b/pathspec/patterns/gitwildmatch.py index 6a3d6d5..d680bdd 100644 --- a/pathspec/patterns/gitwildmatch.py +++ b/pathspec/patterns/gitwildmatch.py @@ -74,6 +74,9 @@ def pattern_to_regex( else: pattern = pattern.strip() + regex: Optional[str] + include: Optional[bool] + if pattern.startswith('#'): # A pattern starting with a hash ('#') serves as a comment (neither # includes nor excludes files). Escape the hash with a back-slash to match @@ -100,7 +103,7 @@ def pattern_to_regex( # Allow a regex override for edge cases that cannot be handled through # normalization. - override_regex = None + override_regex: Optional[str] = None # Split pattern into segments. pattern_segs = pattern.split('/') diff --git a/pyproject.toml b/pyproject.toml index 1d65322..007f7d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", @@ -35,7 +36,6 @@ requires-python = ">=3.8" "Documentation" = "https://python-path-specification.readthedocs.io/en/latest/index.html" "Issue Tracker" = "https://github.com/cpburnz/python-pathspec/issues" - [tool.flit.sdist] include = [ "*.cfg", diff --git a/tox.ini b/tox.ini index 7d7a17c..0cc17e1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{38, 39, 310, 311, 312} + py{38, 39, 310, 311, 312, 313} pypy3 docs isolated_build = True From 033301c2ed2047de0d6b527e80b2de88583887c7 Mon Sep 17 00:00:00 2001 From: "Caleb P. Burns" <2126043+cpburnz@users.noreply.github.com> Date: Mon, 10 Feb 2025 21:51:28 -0500 Subject: [PATCH 40/42] Update ci.yaml --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b2e2a39..0f41361 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [ubuntu, macos, windows] - python: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.8", "pypy-3.9", "pypy-3.10"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "pypy-3.8", "pypy-3.9", "pypy-3.10", "pypy-3.11"] steps: - uses: actions/checkout@v4 From 0e34fa78b832fcfee998ec5c6067d3832f44b7cd Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Tue, 1 Apr 2025 22:10:38 -0400 Subject: [PATCH 41/42] Update DEV --- DEV.md | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/DEV.md b/DEV.md index 3d08ada..68137ff 100644 --- a/DEV.md +++ b/DEV.md @@ -52,15 +52,15 @@ Review the following Linux distributions. - Goal: - Support oldest supported release. -- Fedora 37: - - Oldest supported release as of 2023-09-06. - - Becomes EoL on 2023-11-14. +- Fedora 40: + - Oldest supported release as of 2025-04-01. + - Becomes EoL on 2025-05-13. - Uses Python 3.11. - References: - [End of Life Releases ](https://docs.fedoraproject.org/en-US/releases/eol/) - - [Fedora Linux 39 Schedule: Key -](https://fedorapeople.org/groups/schedule/f-39/f-39-key-tasks.html) + - [Fedora Linux 40 Schedule: Key +](https://fedorapeople.org/groups/schedule/f-40/f-40-key-tasks.html) - [Python](https://docs.fedoraproject.org/en-US/fedora/f37/release-notes/developers/Development_Python/) - Package: [python-pathspec](https://src.fedoraproject.org/rpms/python-pathspec) @@ -87,21 +87,28 @@ Review the following Linux distributions. - Goal: - Support oldest LTS release in standard support. - Ubuntu 20.04 "Focal Fossa": - - Oldest LTS release in standard support as of 2023-09-06. + - Oldest LTS release in standard support as of 2025-04-01. - Ends standard support in 2025-04. - - Package is outdated (v0.7.0 from 2019-12-27; as of 2023-09-06). + - Package is outdated (v0.7.0 from 2019-12-27; as of 2025-04-01). - Uses Python 3.8. - Ubuntu 22.04 "Jammy Jellyfish": - - Latest LTS release as of 2023-09-06. + - Active LTS release as of 2025-04-01. - Ends standard support in 2027-04. - - Package is outdated (v0.9.0 from 2021-07-17; as of 2023-09-06). + - Package is outdated (v0.9.0 from 2021-07-17; as of 2025-04-01). - Uses Python 3.10. +- Ubuntu 24.04 "Noble Numbat": + - Latest LTS release as of 2025-04-01. + - Ends standard support in 2029-04. + - Package is update-to-date (v0.12.1 from 2023-12-10; as of 2025-04-01). + - Uses Python 3.12. - References: - [Releases](https://wiki.ubuntu.com/Releases) - Package: [python3](https://packages.ubuntu.com/focal/python3) (focal) - Package: [python3](https://packages.ubuntu.com/jammy/python3) (jammy) + - Package: [python3](https://packages.ubuntu.com/noble/python3) (noble) - Package: [python3-pathspec](https://packages.ubuntu.com/focal/python3-pathspec) (focal) - Package: [python3-pathspec](https://packages.ubuntu.com/jammy/python3-pathspec) (jammy) + - Package: [python3-pathspec](https://packages.ubuntu.com/noble/python3-pathspec) (noble) ### PyPI @@ -110,25 +117,25 @@ Review the following PyPI packages. [ansible-lint](https://pypi.org/project/ansible-lint/) -- v6.19.0 (latest as of 2023-09-06) requires Python 3.9+. +- v25.1.3 (latest as of 2025-04-01) requires Python 3.10+. - [ansible-lint on Wheelodex](https://www.wheelodex.org/projects/ansible-lint/). [black](https://pypi.org/project/black/) -- v23.7.0 (latest as of 2023-09-06) requires Python 3.8+. +- v25.1.0 (latest as of 2025-04-01) requires Python 3.9+. - [black on Wheelodex](https://www.wheelodex.org/projects/black/). -[dvc](https://github.com/iterative/dvc) +[dvc](https://pypi.org/project/dvc/) -- v3.23.0 (latest as of 2023-09-30) requires Python 3.8+. +- v3.59.1 (latest as of 2025-04-01) requires Python 3.9+. - [dvc on Wheelodex](https://www.wheelodex.org/projects/dvc/). [hatchling](https://pypi.org/project/hatchling/) -- v1.18.0 (latest as of 2023-09-06) requires Python 3.8+. +- v1.27.0 (latest as of 2025-04-01) requires Python 3.8+. - [hatchling on Wheelodex](https://www.wheelodex.org/projects/hatchling/). [yamllint](https://pypi.org/project/yamllint/) -- v1.33.0 (latest as of 2023-12-09) requires Python 3.8+. +- v1.37.0 (latest as of 2025-04-01) requires Python 3.9+. - [yamllint on Wheelodex](https://www.wheelodex.org/projects/yamllint/). From e4977fc41a36d6cba9529a377ff48bf67d6371af Mon Sep 17 00:00:00 2001 From: cpburnz <2126043+cpburnz@users.noreply.github.com> Date: Thu, 26 Jun 2025 22:19:19 -0400 Subject: [PATCH 42/42] Misc --- CHANGES.rst | 8 ++++++ DEV.md | 70 ++++++++++++++++++++++++++--------------------- README-dist.rst | 8 ++++++ pathspec/_meta.py | 4 +-- pyproject.toml | 1 + setup.cfg | 2 ++ tox.ini | 2 +- 7 files changed, 61 insertions(+), 34 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index eb9e11c..537ca90 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,14 @@ Change History ============== +0.12.2 (TBD) +------------------- + +Improvements: + +- Support Python 3.13, 3.14. + + 0.12.1 (2023-12-10) ------------------- diff --git a/DEV.md b/DEV.md index 68137ff..0c4d6bb 100644 --- a/DEV.md +++ b/DEV.md @@ -10,17 +10,32 @@ These are notes to myself for things to review before decommissioning EoL versio ### Python -**Python 3.7:** - -- EoL as of 2023-06-27. - **Python 3.8:** - EoL as of 2024-10-07. +- Cannot remove support until Hatchling stops supporting Python 3.8. **Python 3.9:** - Becomes EoL in 2025-10. +- Cannot remove support until RHEL 9 ends support in 2027-05-31. +- Cannot remove support until all major dependents stop supporting Python 3.9. + +**Python 3.10:** + +- Becomes EoL in 2026-10. + +**Python 3.11:** + +- Becomes EoL in 2027-10. + +**Python 3.12:** + +- Becomes EoL in 2028-10. + +**Python 3.13:** + +- Becomes EoL in 2029-10. References: @@ -40,8 +55,8 @@ Review the following Linux distributions. - Goal: - Support stable release. - Debian 12 "Bookworm": - - Current stable release as of 2023-09-06. - - EoL date TBD. + - Current stable release as of 2025-06-26. + - Becomes EoL on 2028-06-30. - Uses Python 3.11. - References: - [Debian Releases](https://wiki.debian.org/DebianReleases) @@ -52,21 +67,21 @@ Review the following Linux distributions. - Goal: - Support oldest supported release. -- Fedora 40: - - Oldest supported release as of 2025-04-01. - - Becomes EoL on 2025-05-13. - - Uses Python 3.11. +- Fedora 41: + - Oldest supported release as of 2025-06-26. + - Becomes EoL on 2025-11-19. + - Uses Python 3.13. - References: - [End of Life Releases ](https://docs.fedoraproject.org/en-US/releases/eol/) - - [Fedora Linux 40 Schedule: Key -](https://fedorapeople.org/groups/schedule/f-40/f-40-key-tasks.html) - - [Python](https://docs.fedoraproject.org/en-US/fedora/f37/release-notes/developers/Development_Python/) + - [Fedora Linux 41 Schedule: Key +](https://fedorapeople.org/groups/schedule/f-41/f-41-key-tasks.html) + - [Multiple Pythons](https://developer.fedoraproject.org/tech/languages/python/multiple-pythons.html) - Package: [python-pathspec](https://src.fedoraproject.org/rpms/python-pathspec) **Gentoo:** -- Uses Python 3.10+. +- Uses Python 3.11+ (as of 2025-06-26). - References: - Package: [pathspec](https://packages.gentoo.org/packages/dev-python/pathspec) @@ -75,7 +90,7 @@ Review the following Linux distributions. - Goal: - Support oldest release with recent version of *python-pathspec* package. - RHEL 9: - - Oldest release with recent version of *python-pathspec* package (v0.10.1 from 2022-09-02; as of 2023-09-07). + - Oldest release with recent version of *python-pathspec* package (v0.12.1/latest from 2023-12-01; as of 2025-06-26). - Ends full support on 2027-05-31. - Uses Python 3.9. - References: @@ -86,27 +101,20 @@ Review the following Linux distributions. - Goal: - Support oldest LTS release in standard support. -- Ubuntu 20.04 "Focal Fossa": - - Oldest LTS release in standard support as of 2025-04-01. - - Ends standard support in 2025-04. - - Package is outdated (v0.7.0 from 2019-12-27; as of 2025-04-01). - - Uses Python 3.8. - Ubuntu 22.04 "Jammy Jellyfish": - - Active LTS release as of 2025-04-01. + - Active LTS release as of 2025-06-26. - Ends standard support in 2027-04. - - Package is outdated (v0.9.0 from 2021-07-17; as of 2025-04-01). + - Package is outdated (v0.9.0 from 2021-07-17; as of 2025-06-26). - Uses Python 3.10. - Ubuntu 24.04 "Noble Numbat": - - Latest LTS release as of 2025-04-01. + - Latest LTS release as of 2025-06-26. - Ends standard support in 2029-04. - - Package is update-to-date (v0.12.1 from 2023-12-10; as of 2025-04-01). + - Package is update-to-date (v0.12.1 from 2023-12-10; as of 2025-06-26). - Uses Python 3.12. - References: - [Releases](https://wiki.ubuntu.com/Releases) - - Package: [python3](https://packages.ubuntu.com/focal/python3) (focal) - Package: [python3](https://packages.ubuntu.com/jammy/python3) (jammy) - Package: [python3](https://packages.ubuntu.com/noble/python3) (noble) - - Package: [python3-pathspec](https://packages.ubuntu.com/focal/python3-pathspec) (focal) - Package: [python3-pathspec](https://packages.ubuntu.com/jammy/python3-pathspec) (jammy) - Package: [python3-pathspec](https://packages.ubuntu.com/noble/python3-pathspec) (noble) @@ -117,25 +125,25 @@ Review the following PyPI packages. [ansible-lint](https://pypi.org/project/ansible-lint/) -- v25.1.3 (latest as of 2025-04-01) requires Python 3.10+. +- v25.6.1 (latest as of 2025-06-26) requires Python 3.10+. - [ansible-lint on Wheelodex](https://www.wheelodex.org/projects/ansible-lint/). [black](https://pypi.org/project/black/) -- v25.1.0 (latest as of 2025-04-01) requires Python 3.9+. +- v25.1.0 (latest as of 2025-06-26) requires Python 3.9+. - [black on Wheelodex](https://www.wheelodex.org/projects/black/). [dvc](https://pypi.org/project/dvc/) -- v3.59.1 (latest as of 2025-04-01) requires Python 3.9+. +- v3.60.1 (latest as of 2025-06-26) requires Python 3.9+. - [dvc on Wheelodex](https://www.wheelodex.org/projects/dvc/). [hatchling](https://pypi.org/project/hatchling/) -- v1.27.0 (latest as of 2025-04-01) requires Python 3.8+. +- v1.27.0 (latest as of 2025-06-26) requires Python 3.8+. - [hatchling on Wheelodex](https://www.wheelodex.org/projects/hatchling/). [yamllint](https://pypi.org/project/yamllint/) -- v1.37.0 (latest as of 2025-04-01) requires Python 3.9+. +- v1.37.1 (latest as of 2025-06-26) requires Python 3.9+. - [yamllint on Wheelodex](https://www.wheelodex.org/projects/yamllint/). diff --git a/README-dist.rst b/README-dist.rst index e7752fc..20592a7 100644 --- a/README-dist.rst +++ b/README-dist.rst @@ -170,6 +170,14 @@ Change History ============== +0.12.2 (TBD) +------------------- + +Improvements: + +- Support Python 3.13, 3.14. + + 0.12.1 (2023-12-10) ------------------- diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 4d8c89d..ed3d887 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -3,7 +3,7 @@ """ __author__ = "Caleb P. Burns" -__copyright__ = "Copyright © 2013-2023 Caleb P. Burns" +__copyright__ = "Copyright © 2013-2025 Caleb P. Burns" __credits__ = [ "dahlia ", "highb ", @@ -55,4 +55,4 @@ "kurtmckee ", ] __license__ = "MPL 2.0" -__version__ = "0.12.1" +__version__ = "0.12.2.dev1" diff --git a/pyproject.toml b/pyproject.toml index 007f7d5..29fc8f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", diff --git a/setup.cfg b/setup.cfg index 34abfd1..2d13930 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,8 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 + Programming Language :: Python :: 3.14 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Topic :: Software Development :: Libraries :: Python Modules diff --git a/tox.ini b/tox.ini index 0cc17e1..e24264a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{38, 39, 310, 311, 312, 313} + py{38, 39, 310, 311, 312, 313, 314} pypy3 docs isolated_build = True