bpo-37935: Added tests for os.walk(), glob.iglob() and Path.glob() (GH-15956) (GH-16043)

miss-islington · serhiy-storchaka · gpshead · commit 98a4a713d001 · 2019-09-12T16:07:47.000+01:00
Test that they do not keep too many file descriptors open for the host OS in a reasonable test scenario. See [bpo-37935](https://bugs.python.org/issue37935). (cherry picked from commit f9dc2ad) Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py
@@ -264,6 +264,23 @@ def test_recursive_glob(self):
                 expect += [join('sym3', 'EF')]
             eq(glob.glob(join('**', 'EF'), recursive=True), expect)
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        base = os.path.join(self.tempdir, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+        pattern = os.path.join(base, *(['*']*depth))
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        pattern = os.path.join(base, '**', 'd')
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = os.path.join(p, 'd')
+            for it in iters:
+                self.assertEqual(next(it), p)
+
 
 @skip_unless_symlink
 class SymlinkLoopGlobTests(unittest.TestCase):
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
@@ -1178,6 +1178,27 @@ def test_walk_bad_dir(self):
         finally:
             os.rename(path1new, path1)
 
+    def test_walk_many_open_files(self):
+        depth = 30
+        base = os.path.join(support.TESTFN, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+
+        iters = [self.walk(base, topdown=False) for j in range(100)]
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.dirname(p)
+
+        iters = [self.walk(base, topdown=True) for j in range(100)]
+        p = base
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i < depth else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.join(p, 'd')
+
 
 @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
 class FwalkTests(WalkTests):
@@ -1247,6 +1268,10 @@ def test_fd_leak(self):
         self.addCleanup(os.close, newfd)
         self.assertEqual(newfd, minfd)
 
+    # fwalk() keeps file descriptors open
+    test_walk_many_open_files = None
+
+
 class BytesWalkTests(WalkTests):
     """Tests for os.walk() with bytes."""
     def walk(self, top, **kwargs):
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
@@ -1475,6 +1475,23 @@ def test_rglob_symlink_loop(self):
                   }
         self.assertEqual(given, {p / x for x in expect})
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        P = self.cls
+        base = P(BASE) / 'deep'
+        p = P(base, *(['d']*depth))
+        p.mkdir(parents=True)
+        pattern = '/'.join(['*'] * depth)
+        iters = [base.glob(pattern) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        iters = [base.rglob('d') for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = p / 'd'
+            for it in iters:
+                self.assertEqual(next(it), p)
+
     def test_glob_dotdot(self):
         # ".." is not special in globs.
         P = self.cls
