diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..a31480c --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,34 @@ +name: CI + +permissions: + contents: read + +on: + push: + branches: [master] + pull_request: + +jobs: + test: + name: Test / ${{ matrix.python }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu, macos, windows] + python: ["3.7", "3.8", "3.9", "3.10", "3.11.0-rc - 3.11", + "pypy-3.7", "pypy-3.8", "pypy-3.9"] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install tox + run: python -m pip install tox + + - name: Run tests + run: python -m tox -e py -- --verbose diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 817661c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: python -cache: pip - -# Supported CPython versions: -# https://en.wikipedia.org/wiki/CPython#Version_history -matrix: - fast_finish: true - include: - - python: 3.11 - - python: 3.10 - - python: 3.9 - - python: 3.8 - - python: 3.7 - - python: pypy3 - -install: - - pip install -U pip - - pip install -U tox-travis - -script: - - tox diff --git a/CHANGES.rst b/CHANGES.rst index 2d99b97..b18e24b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,27 @@ Change History ============== +0.10.2 (2022-11-12) +------------------- + +Bug fixes: + +- Fix failing tests on Windows. +- Type hint on *root* parameter on `pathspec.pathspec.PathSpec.match_tree_entries()`. +- Type hint on *root* parameter on `pathspec.pathspec.PathSpec.match_tree_files()`. +- Type hint on *root* parameter on `pathspec.util.iter_tree_entries()`. +- Type hint on *root* parameter on `pathspec.util.iter_tree_files()`. +- `Issue #64`_: IndexError with my .gitignore file when trying to build a Python package. + +Improvements: + +- `Issue #58`_: CI: add GitHub Actions test workflow. + + +.. _`Issue #58`: https://github.com/cpburnz/python-pathspec/pull/58 +.. _`Issue #64`: https://github.com/cpburnz/python-pathspec/issues/64 + + 0.10.1 (2022-09-02) ------------------- diff --git a/pathspec/_meta.py b/pathspec/_meta.py index 9f39532..6bd1ccf 100644 --- a/pathspec/_meta.py +++ b/pathspec/_meta.py @@ -46,6 +46,7 @@ "jack1142 ", "mgorny ", "bzakdd ", + "haimat ", ] __license__ = "MPL 2.0" -__version__ = "0.10.1" +__version__ = "0.10.2" diff --git a/pathspec/gitignore.py b/pathspec/gitignore.py index b0ddc92..908a9bd 100644 --- a/pathspec/gitignore.py +++ b/pathspec/gitignore.py @@ -16,7 +16,9 @@ from .pattern import ( Pattern) from .patterns.gitwildmatch import ( - GitWildMatchPattern) + GitWildMatchPattern, + GitWildMatchPatternError, + _DIR_MARK) from .util import ( _is_iterable) @@ -101,7 +103,17 @@ def _match_file( # Pattern matched. # Check for directory marker. - dir_mark = match.match.group('ps_d') + try: + dir_mark = match.match.group(_DIR_MARK) + except IndexError as e: + # NOTICE: The exact content of this error message is subject + # to change. + raise GitWildMatchPatternError(( + f"Invalid git pattern: directory marker regex group is missing. " + f"Debug: file={file!r} regex={pattern.regex!r} " + f"group={_DIR_MARK!r} match={match.match!r}." + )) from e + if dir_mark: # Pattern matched by a directory pattern. priority = 1 diff --git a/pathspec/pathspec.py b/pathspec/pathspec.py index f520328..d740aa8 100644 --- a/pathspec/pathspec.py +++ b/pathspec/pathspec.py @@ -206,7 +206,7 @@ def match_files( def match_tree_entries( self, - root: str, + root: Union[str, PathLike], on_error: Optional[Callable] = None, follow_links: Optional[bool] = None, ) -> Iterator[TreeEntry]: @@ -233,7 +233,7 @@ def match_tree_entries( def match_tree_files( self, - root: str, + root: Union[str, PathLike], on_error: Optional[Callable] = None, follow_links: Optional[bool] = None, ) -> Iterator[str]: diff --git a/pathspec/patterns/gitwildmatch.py b/pathspec/patterns/gitwildmatch.py index c6b6fb7..94d3115 100644 --- a/pathspec/patterns/gitwildmatch.py +++ b/pathspec/patterns/gitwildmatch.py @@ -19,6 +19,12 @@ The encoding to use when parsing a byte string pattern. """ +_DIR_MARK = 'ps_d' +""" +The regex group name for the directory marker. This is only used by +:class:`GitIgnoreSpec`. +""" + class GitWildMatchPatternError(ValueError): """ @@ -112,7 +118,7 @@ def pattern_to_regex( # EDGE CASE: The '**/' pattern should match everything except # individual files in the root directory. This case cannot be # adequately handled through normalization. Use the override. - override_regex = '^.+(?P/).*$' + override_regex = f'^.+(?P<{_DIR_MARK}>/).*$' if not pattern_segs[0]: # A pattern beginning with a slash ('/') will only match paths @@ -160,7 +166,7 @@ def pattern_to_regex( if i == 0 and i == end: # A pattern consisting solely of double-asterisks ('**') # will match every path. - output.append('.+') + output.append(f'[^/]+(?:(?P<{_DIR_MARK}>/).*)?') elif i == 0: # A normalized pattern beginning with double-asterisks # ('**') will match any leading path segments. @@ -169,7 +175,7 @@ def pattern_to_regex( elif i == end: # A normalized pattern ending with double-asterisks ('**') # will match any trailing path segments. - output.append('(?P/).*') + output.append(f'(?P<{_DIR_MARK}>/).*') else: # A pattern with inner double-asterisks ('**') will match # multiple (or zero) inner path segments. @@ -187,7 +193,7 @@ def pattern_to_regex( # A pattern ending without a slash ('/') will match a file # or a directory (with paths underneath it). E.g., "foo" # matches "foo", "foo/bar", "foo/bar/baz", etc. - output.append('(?:(?P/).*)?') + output.append(f'(?:(?P<{_DIR_MARK}>/).*)?') need_slash = True @@ -205,7 +211,7 @@ def pattern_to_regex( # A pattern ending without a slash ('/') will match a file # or a directory (with paths underneath it). E.g., "foo" # matches "foo", "foo/bar", "foo/bar/baz", etc. - output.append('(?:(?P/).*)?') + output.append(f'(?:(?P<{_DIR_MARK}>/).*)?') need_slash = True diff --git a/pathspec/util.py b/pathspec/util.py index b78e838..38cc9e2 100644 --- a/pathspec/util.py +++ b/pathspec/util.py @@ -122,14 +122,15 @@ def _is_iterable(value: Any) -> bool: def iter_tree_entries( - root: str, + root: Union[str, PathLike], on_error: Optional[Callable] = None, follow_links: Optional[bool] = None, ) -> Iterator['TreeEntry']: """ Walks the specified directory for all files and directories. - *root* (:class:`str`) is the root directory to search. + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally is the error handler for file-system exceptions. It will be @@ -237,14 +238,15 @@ def _iter_tree_entries_next( def iter_tree_files( - root: str, + root: Union[str, PathLike], on_error: Optional[Callable] = None, follow_links: Optional[bool] = None, ) -> Iterator[str]: """ Walks the specified directory for all files. - *root* (:class:`str`) is the root directory to search for files. + *root* (:class:`str` or :class:`os.PathLike`) is the root directory to + search for files. *on_error* (:class:`~collections.abc.Callable` or :data:`None`) optionally is the error handler for file-system exceptions. It will be diff --git a/tests/test_gitignore.py b/tests/test_gitignore.py index 04902d2..7d261ec 100644 --- a/tests/test_gitignore.py +++ b/tests/test_gitignore.py @@ -311,3 +311,50 @@ def test_04_issue_62(self): 'anydir/file.txt', 'product_dir/file.txt', }) + + def test_05_issue_39(self): + """ + Test excluding files in a directory. + """ + spec = GitIgnoreSpec.from_lines([ + '*.log', + '!important/*.log', + 'trace.*', + ]) + files = { + 'a.log', + 'b.txt', + 'important/d.log', + 'important/e.txt', + 'trace.c', + } + ignores = set(spec.match_files(files)) + self.assertEqual(ignores, { + 'a.log', + 'trace.c', + }) + self.assertEqual(files - ignores, { + 'b.txt', + 'important/d.log', + 'important/e.txt', + }) + + def test_06_issue_64(self): + """ + Test using a double asterisk pattern. + """ + spec = GitIgnoreSpec.from_lines([ + "**", + ]) + files = { + 'x', + 'y.py', + 'A/x', + 'A/y.py', + 'A/B/x', + 'A/B/y.py', + 'A/B/C/x', + 'A/B/C/y.py', + } + ignores = set(spec.match_files(files)) + self.assertEqual(ignores, files) diff --git a/tests/test_gitwildmatch.py b/tests/test_gitwildmatch.py index 8563d95..7dccaee 100644 --- a/tests/test_gitwildmatch.py +++ b/tests/test_gitwildmatch.py @@ -9,17 +9,18 @@ from pathspec.patterns.gitwildmatch import ( GitWildMatchPattern, GitWildMatchPatternError, - _BYTES_ENCODING) + _BYTES_ENCODING, + _DIR_MARK) from pathspec.util import ( lookup_pattern) -RE_DIR = "(?P/)" +RE_DIR = f"(?P<{_DIR_MARK}>/)" """ This regular expression matches the directory marker. """ -RE_SUB = "(?:(?P/).*)?" +RE_SUB = f"(?:{RE_DIR}.*)?" """ This regular expression matches an optional sub-path. """ @@ -257,7 +258,29 @@ def test_03_only_double_asterisk(self): """ regex, include = GitWildMatchPattern.pattern_to_regex('**') self.assertTrue(include) - self.assertEqual(regex, '^.+$') + self.assertEqual(regex, f'^[^/]+{RE_SUB}$') + pattern = GitWildMatchPattern(re.compile(regex), include) + results = set(filter(pattern.match_file, [ + 'x', + 'y.py', + 'A/x', + 'A/y.py', + 'A/B/x', + 'A/B/y.py', + 'A/B/C/x', + 'A/B/C/y.py', + ])) + + self.assertEqual(results, { + 'x', + 'y.py', + 'A/x', + 'A/y.py', + 'A/B/x', + 'A/B/y.py', + 'A/B/C/x', + 'A/B/C/y.py', + }) def test_03_parent_double_asterisk(self): """ @@ -291,7 +314,7 @@ def test_03_duplicate_leading_double_asterisk_edge_case(self): """ regex, include = GitWildMatchPattern.pattern_to_regex('**') self.assertTrue(include) - self.assertEqual(regex, '^.+$') + self.assertEqual(regex, f'^[^/]+{RE_SUB}$') equivalent_regex, include = GitWildMatchPattern.pattern_to_regex('**/**') self.assertTrue(include) diff --git a/tests/test_pathspec.py b/tests/test_pathspec.py index cb6c476..1b900b0 100644 --- a/tests/test_pathspec.py +++ b/tests/test_pathspec.py @@ -4,6 +4,7 @@ import os import os.path +import pathlib import shutil import tempfile import unittest @@ -14,6 +15,10 @@ PathSpec) from pathspec.util import ( iter_tree_entries) +from tests.util import ( + make_dirs, + make_files, + ospath) class PathSpecTest(unittest.TestCase): @@ -25,36 +30,19 @@ def make_dirs(self, dirs: Iterable[str]) -> None: """ Create the specified directories. """ - for dir in dirs: - os.mkdir(os.path.join(self.temp_dir, self.ospath(dir))) + make_dirs(self.temp_dir, dirs) def make_files(self, files: Iterable[str]) -> None: """ Create the specified files. """ - for file in files: - self.mkfile(os.path.join(self.temp_dir, self.ospath(file))) - - @staticmethod - def mkfile(file: str) -> None: - """ - Creates an empty file. - """ - with open(file, 'wb'): - pass - - @staticmethod - def ospath(path: str) -> str: - """ - Convert the POSIX path to a native OS path. - """ - return os.path.join(*path.split('/')) + return make_files(self.temp_dir, files) def setUp(self) -> None: """ Called before each test. """ - self.temp_dir = tempfile.mkdtemp() + self.temp_dir = pathlib.Path(tempfile.mkdtemp()) def tearDown(self) -> None: """ @@ -311,12 +299,12 @@ def test_05_match_entries(self): __entry.path for __entry in spec.match_entries(entries) } - self.assertEqual(results, { + self.assertEqual(results, set(map(ospath, [ 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', 'Y/Z/c.txt', - }) + ]))) def test_05_match_file(self): """ @@ -390,12 +378,12 @@ def test_05_match_tree_entries(self): __entry.path for __entry in spec.match_tree_entries(self.temp_dir) } - self.assertEqual(results, { + self.assertEqual(results, set(map(ospath, [ 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', 'Y/Z/c.txt', - }) + ]))) def test_05_match_tree_files(self): """ @@ -420,12 +408,12 @@ def test_05_match_tree_files(self): 'Y/Z/c.txt', ]) results = set(spec.match_tree_files(self.temp_dir)) - self.assertEqual(results, { + self.assertEqual(results, set(map(ospath, [ 'X/a.txt', 'X/Z/c.txt', 'Y/a.txt', 'Y/Z/c.txt', - }) + ]))) def test_06_issue_41_a(self): """ @@ -537,3 +525,30 @@ def test_07_issue_62(self): self.assertEqual(results, { 'anydir/file.txt', }) + + def test_08_issue_39(self): + """ + Test excluding files in a directory. + """ + spec = PathSpec.from_lines('gitwildmatch', [ + '*.log', + '!important/*.log', + 'trace.*', + ]) + files = { + 'a.log', + 'b.txt', + 'important/d.log', + 'important/e.txt', + 'trace.c', + } + ignores = set(spec.match_files(files)) + self.assertEqual(ignores, { + 'a.log', + 'trace.c', + }) + self.assertEqual(files - ignores, { + 'b.txt', + 'important/d.log', + 'important/e.txt', + }) diff --git a/tests/test_util.py b/tests/test_util.py index 7a7b10c..5a036a0 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -23,6 +23,12 @@ iter_tree_files, match_file, normalize_file) +from tests.util import ( + make_dirs, + make_files, + make_links, + mkfile, + ospath) class MatchFileTest(unittest.TestCase): @@ -65,39 +71,19 @@ def make_dirs(self, dirs: Iterable[str]) -> None: """ Create the specified directories. """ - for dir in dirs: - os.mkdir(os.path.join(self.temp_dir, self.ospath(dir))) + make_dirs(self.temp_dir, dirs) def make_files(self, files: Iterable[str]) -> None: """ Create the specified files. """ - for file in files: - self.mkfile(os.path.join(self.temp_dir, self.ospath(file))) + make_files(self.temp_dir, files) def make_links(self, links: Iterable[Tuple[str, str]]) -> None: """ Create the specified links. """ - for link, node in links: - src = os.path.join(self.temp_dir, self.ospath(node)) - dest = os.path.join(self.temp_dir, self.ospath(link)) - os.symlink(src, dest) - - @staticmethod - def mkfile(file: str) -> None: - """ - Creates an empty file. - """ - with open(file, 'wb'): - pass - - @staticmethod - def ospath(path: str) -> str: - """ - Convert the POSIX path to a native OS path. - """ - return os.path.join(*path.split('/')) + make_links(self.temp_dir, links) def require_realpath(self) -> None: """ @@ -118,7 +104,7 @@ def setUp(self) -> None: """ Called before each test. """ - self.temp_dir = tempfile.mkdtemp() + self.temp_dir = pathlib.Path(tempfile.mkdtemp()) def tearDown(self) -> None: """ @@ -144,7 +130,7 @@ def test_1_files(self): 'Dir/Inner/f', ]) results = set(iter_tree_files(self.temp_dir)) - self.assertEqual(results, set(map(self.ospath, [ + self.assertEqual(results, set(map(ospath, [ 'a', 'b', 'Dir/c', @@ -161,16 +147,16 @@ def test_2_0_check_symlink(self): # 3.2+. no_symlink = None try: - file = os.path.join(self.temp_dir, 'file') - link = os.path.join(self.temp_dir, 'link') - self.mkfile(file) + file = self.temp_dir / 'file' + link = self.temp_dir / 'link' + mkfile(file) try: os.symlink(file, link) except (AttributeError, NotImplementedError, OSError): no_symlink = True - raise - no_symlink = False + else: + no_symlink = False finally: self.__class__.no_symlink = no_symlink @@ -185,18 +171,17 @@ def test_2_1_check_realpath(self): broken_realpath = None try: self.require_symlink() - file = os.path.join(self.temp_dir, 'file') - link = os.path.join(self.temp_dir, 'link') - self.mkfile(file) + file = self.temp_dir / 'file' + link = self.temp_dir / 'link' + mkfile(file) os.symlink(file, link) try: self.assertEqual(os.path.realpath(file), os.path.realpath(link)) except AssertionError: broken_realpath = True - raise - - broken_realpath = False + else: + broken_realpath = False finally: self.__class__.broken_realpath = broken_realpath @@ -223,7 +208,7 @@ def test_2_2_links(self): ('DirX', 'Dir'), ]) results = set(iter_tree_files(self.temp_dir)) - self.assertEqual(results, set(map(self.ospath, [ + self.assertEqual(results, set(map(ospath, [ 'a', 'ax', 'b', @@ -260,7 +245,7 @@ def test_2_3_sideways_links(self): ('Dir/Fx', 'Dir/Target'), ]) results = set(iter_tree_files(self.temp_dir)) - self.assertEqual(results, set(map(self.ospath, [ + self.assertEqual(results, set(map(ospath, [ 'Ax/Ex/file', 'Ax/Fx/file', 'Ax/Target/file', @@ -293,7 +278,7 @@ def test_2_4_recursive_links(self): set(iter_tree_files(self.temp_dir)) self.assertEqual(context.exception.first_path, 'Dir') - self.assertEqual(context.exception.second_path, self.ospath('Dir/Self')) + self.assertEqual(context.exception.second_path, ospath('Dir/Self')) def test_2_5_recursive_circular_links(self): """ @@ -321,9 +306,9 @@ def test_2_5_recursive_circular_links(self): self.assertIn(context.exception.first_path, ('A', 'B', 'C')) self.assertEqual(context.exception.second_path, { - 'A': self.ospath('A/Bx/Cx/Ax'), - 'B': self.ospath('B/Cx/Ax/Bx'), - 'C': self.ospath('C/Ax/Bx/Cx'), + 'A': ospath('A/Bx/Cx/Ax'), + 'B': ospath('B/Cx/Ax/Bx'), + 'C': ospath('C/Ax/Bx/Cx'), }[context.exception.first_path]) def test_2_6_detect_broken_links(self): @@ -375,7 +360,7 @@ def test_2_8_no_follow_links(self): ('DirX', 'Dir'), ]) results = set(iter_tree_files(self.temp_dir, follow_links=False)) - self.assertEqual(results, set(map(self.ospath, [ + self.assertEqual(results, set(map(ospath, [ 'A', 'Ax', 'B', @@ -405,7 +390,7 @@ def test_3_entries(self): 'Dir/Inner/f', ]) results = {entry.path for entry in iter_tree_entries(self.temp_dir)} - self.assertEqual(results, set(map(self.ospath, [ + self.assertEqual(results, set(map(ospath, [ 'a', 'b', 'Dir', diff --git a/tests/util.py b/tests/util.py new file mode 100644 index 0000000..301427b --- /dev/null +++ b/tests/util.py @@ -0,0 +1,75 @@ +""" +This module provides utility functions shared by tests. +""" + +import os +import os.path +import pathlib + +from typing import ( + Iterable, + Tuple) + + +def make_dirs(temp_dir: pathlib.Path, dirs: Iterable[str]) -> None: + """ + Create the specified directories. + + *temp_dir* (:class:`pathlib.Path`) is the temporary directory to use. + + *dirs* (:class:`Iterable` of :class:`str`) is the POSIX directory + paths (relative to *temp_dir*) to create. + """ + for dir in dirs: + os.mkdir(temp_dir / ospath(dir)) + + +def make_files(temp_dir: pathlib.Path, files: Iterable[str]) -> None: + """ + Create the specified files. + + *temp_dir* (:class:`pathlib.Path`) is the temporary directory to use. + + *files* (:class:`Iterable` of :class:`str`) is the POSIX file paths + (relative to *temp_dir*) to create. + """ + for file in files: + mkfile(temp_dir / ospath(file)) + + +def make_links(temp_dir: pathlib.Path, links: Iterable[Tuple[str, str]]) -> None: + """ + Create the specified links. + + *temp_dir* (:class:`pathlib.Path`) is the temporary directory to use. + + *links* (:class:`Iterable` of :class:`tuple`) contains the POSIX links + to create relative to *temp_dir*. Each link (:class:`tuple`) contains + the destination link path (:class:`str`) and source node path + (:class:`str`). + """ + for link, node in links: + src = temp_dir / ospath(node) + dest = temp_dir / ospath(link) + os.symlink(src, dest) + + +def mkfile(file: pathlib.Path) -> None: + """ + Creates an empty file. + + *file* (:class:`pathlib.Path`) is the native file path to create. + """ + with open(file, 'wb'): + pass + + +def ospath(path: str) -> str: + """ + Convert the POSIX path to a native OS path. + + *path* (:class:`str`) is the POSIX path. + + Returns the native path (:class:`str`). + """ + return os.path.join(*path.split('/')) diff --git a/tox.ini b/tox.ini index f9d53c7..9b820e3 100644 --- a/tox.ini +++ b/tox.ini @@ -3,4 +3,4 @@ envlist = py37, py38, py39, py310, py311, pypy3 isolated_build = True [testenv] -commands = python -m unittest +commands = python -m unittest {posargs}