Skip to content

GH-116380: Support dir_fd argument in glob._Globber. #117737

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

GH-116380: Support dir_fd argument in glob._Globber. #117737

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
151511d
GH-116380: Support `dir_fd` argument in `glob._Globber`.
barneygale Apr 10, 2024
335820e
Merge branch 'main' into gh-116380-dir-fd
barneygale Apr 12, 2024
edd3a2d
Swap condition order in select_exists(), use os.lstat() explicitly
barneygale Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 64 additions & 15 deletions Lib/glob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,22 @@ def _compile_pattern(pat, sep, case_sensitive, recursive=True):
return re.compile(regex, flags=flags).match


def _open_dir(path, dir_fd=None, rel_path=None):
"""Prepares the directory for scanning. Returns a 3-tuple with parts:

1. A path or fd to supply to `os.scandir()`.
2. The file descriptor for the directory, or None.
3. Whether the caller should close the fd (bool).
"""
if dir_fd is None:
return path, None, False
elif rel_path == './':
return dir_fd, dir_fd, False
else:
fd = os.open(rel_path, _dir_open_flags, dir_fd=dir_fd)
return fd, fd, True


class _Globber:
"""Class providing shell-style pattern matching and globbing.
"""
Expand Down Expand Up @@ -385,9 +401,11 @@ def special_selector(self, part, parts):
"""
select_next = self.selector(parts)

def select_special(path, exists=False):
def select_special(path, dir_fd=None, rel_path=None, exists=False):
path = self.concat_path(self.add_slash(path), part)
return select_next(path, exists)
if dir_fd is not None:
rel_path = self.concat_path(self.add_slash(rel_path), part)
return select_next(path, dir_fd, rel_path, exists)
return select_special

def literal_selector(self, part, parts):
Expand All @@ -402,9 +420,11 @@ def literal_selector(self, part, parts):

select_next = self.selector(parts)

def select_literal(path, exists=False):
def select_literal(path, dir_fd=None, rel_path=None, exists=False):
path = self.concat_path(self.add_slash(path), part)
return select_next(path, exists=False)
if dir_fd is not None:
rel_path = self.concat_path(self.add_slash(rel_path), part)
return select_next(path, dir_fd, rel_path, exists=False)
return select_literal

def wildcard_selector(self, part, parts):
Expand All @@ -417,11 +437,15 @@ def wildcard_selector(self, part, parts):
if dir_only:
select_next = self.selector(parts)

def select_wildcard(path, exists=False):
def select_wildcard(path, dir_fd=None, rel_path=None, exists=False):
close_fd = False
try:
arg, fd, close_fd = _open_dir(path, dir_fd, rel_path)
if fd is not None:
prefix = self.add_slash(path)
# We must close the scandir() object before proceeding to
# avoid exhausting file descriptors when globbing deep trees.
with self.scandir(path) as scandir_it:
with self.scandir(arg) as scandir_it:
entries = list(scandir_it)
except OSError:
pass
Expand All @@ -435,10 +459,16 @@ def select_wildcard(path, exists=False):
except OSError:
continue
entry_path = self.parse_entry(entry)
if fd is not None:
entry_path = self.concat_path(prefix, entry_path)
if dir_only:
yield from select_next(entry_path, exists=True)
yield from select_next(
entry_path, fd, entry.name, exists=True)
else:
yield entry_path
finally:
if close_fd:
os.close(fd)
return select_wildcard

def recursive_selector(self, part, parts):
Expand All @@ -464,21 +494,31 @@ def recursive_selector(self, part, parts):
dir_only = bool(parts)
select_next = self.selector(parts)

def select_recursive(path, exists=False):
def select_recursive(path, dir_fd=None, rel_path=None, exists=False):
path = self.add_slash(path)
if dir_fd is not None:
rel_path = self.add_slash(rel_path)
match_pos = len(str(path))
if match is None or match(str(path), match_pos):
yield from select_next(path, exists)
stack = [path]
yield from select_next(path, dir_fd, rel_path, exists)
stack = [(path, dir_fd, rel_path)]
while stack:
yield from select_recursive_step(stack, match_pos)

def select_recursive_step(stack, match_pos):
path = stack.pop()
path, dir_fd, rel_path = stack.pop()
try:
if path is None:
os.close(dir_fd)
return
arg, fd, close_fd = _open_dir(path, dir_fd, rel_path)
if fd is not None:
prefix = self.add_slash(path)
if close_fd:
stack.append((None, fd, None))
# We must close the scandir() object before proceeding to
# avoid exhausting file descriptors when globbing deep trees.
with self.scandir(path) as scandir_it:
with self.scandir(arg) as scandir_it:
entries = list(scandir_it)
except OSError:
pass
Expand All @@ -493,25 +533,34 @@ def select_recursive_step(stack, match_pos):

if is_dir or not dir_only:
entry_path = self.parse_entry(entry)
if fd is not None:
entry_path = self.concat_path(prefix, entry_path)
if match is None or match(str(entry_path), match_pos):
if dir_only:
yield from select_next(entry_path, exists=True)
yield from select_next(
entry_path, fd, entry.name, exists=True)
else:
# Optimization: directly yield the path if this is
# last pattern part.
yield entry_path
if is_dir:
stack.append(entry_path)
stack.append((entry_path, fd, entry.name))

return select_recursive

def select_exists(self, path, exists=False):
def select_exists(self, path, dir_fd=None, rel_path=None, exists=False):
"""Yields the given path, if it exists.
"""
if exists:
# Optimization: this path is already known to exist, e.g. because
# it was returned from os.scandir(), so we skip calling lstat().
yield path
elif dir_fd is not None:
try:
os.lstat(rel_path, dir_fd=dir_fd)
yield path
except OSError:
pass
else:
try:
self.lstat(path)
Expand Down