From dec587134cf88340a8e11948541ab39eea261537 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 6 Sep 2019 14:02:38 +0200 Subject: [PATCH 01/13] Raise the limit of maximum path depth to actual recursion limit --- Lib/compileall.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/compileall.py b/Lib/compileall.py index 49306d9dabbc51..37556c2b6a004e 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -18,9 +18,11 @@ from functools import partial +RECURSION_LIMIT = sys.getrecursionlimit() + __all__ = ["compile_dir","compile_file","compile_path"] -def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): +def _walk_dir(dir, ddir=None, maxlevels=RECURSION_LIMIT, quiet=0): if quiet < 2 and isinstance(dir, os.PathLike): dir = os.fspath(dir) if not quiet: @@ -47,15 +49,15 @@ def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): yield from _walk_dir(fullname, ddir=dfile, maxlevels=maxlevels - 1, quiet=quiet) -def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, - quiet=0, legacy=False, optimize=-1, workers=1, +def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, + rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): dir: the directory to byte-compile - maxlevels: maximum recursion level (default 10) + maxlevels: maximum recursion level (default `sys.getrecursionlimit()`) ddir: the directory that will be prepended to the path to the file as it is compiled into each byte-code file. force: if True, force compilation, even if timestamps are up-to-date @@ -225,7 +227,7 @@ def main(): parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.') parser.add_argument('-l', action='store_const', const=0, - default=10, dest='maxlevels', + default=RECURSION_LIMIT, dest='maxlevels', help="don't recurse into subdirectories") parser.add_argument('-r', type=int, dest='recursion', help=('control the maximum recursion level. ' From 268acef2475b743604e7aed3a916bfe461cda6d4 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 6 Sep 2019 14:07:46 +0200 Subject: [PATCH 02/13] [refactoring] ddir arg is useless in _walk_dir function --- Lib/compileall.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/compileall.py b/Lib/compileall.py index 37556c2b6a004e..4b2a7de39b9d69 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -22,7 +22,7 @@ __all__ = ["compile_dir","compile_file","compile_path"] -def _walk_dir(dir, ddir=None, maxlevels=RECURSION_LIMIT, quiet=0): +def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0): if quiet < 2 and isinstance(dir, os.PathLike): dir = os.fspath(dir) if not quiet: @@ -38,16 +38,12 @@ def _walk_dir(dir, ddir=None, maxlevels=RECURSION_LIMIT, quiet=0): if name == '__pycache__': continue fullname = os.path.join(dir, name) - if ddir is not None: - dfile = os.path.join(ddir, name) - else: - dfile = None if not os.path.isdir(fullname): yield fullname elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): - yield from _walk_dir(fullname, ddir=dfile, - maxlevels=maxlevels - 1, quiet=quiet) + yield from _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet) def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, @@ -78,8 +74,7 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, from concurrent.futures import ProcessPoolExecutor except ImportError: workers = 1 - files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, - ddir=ddir) + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True if workers != 1 and ProcessPoolExecutor is not None: # If workers == 0, let ProcessPoolExecutor choose From dae2c6a96ea58d093091b423f8d74174c6ca2597 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 6 Sep 2019 14:15:38 +0200 Subject: [PATCH 03/13] Add posibilities to adjust a path compiled in .pyc file. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now, you can: - Strip a part of path from a beggining of path into compiled file example "-s /test /test/build/real/test.py" → "build/real/test.py" - Append some new path to a beggining of path into compiled file example "-p /boo real/test.py" → "/boo/real/test.py" You can also use both options in the same time. In that case, striping is done before appending. --- Lib/compileall.py | 67 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/Lib/compileall.py b/Lib/compileall.py index 4b2a7de39b9d69..919529e8567492 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -47,7 +47,8 @@ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0): def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, - invalidation_mode=None): + invalidation_mode=None, stripdir=None, + prependdir=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -63,6 +64,9 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, optimize: optimization level or -1 for level of the interpreter workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked + stripdir: part of path to left-strip from source file path + prependdir: path to prepend to beggining of original file path, applied + after stripdir """ ProcessPoolExecutor = None if workers < 0: @@ -85,19 +89,22 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, rx=rx, quiet=quiet, legacy=legacy, optimize=optimize, - invalidation_mode=invalidation_mode), + invalidation_mode=invalidation_mode, + stripdir=stripdir, + prependdir=prependdir), files) success = min(results, default=True) else: for file in files: if not compile_file(file, ddir, force, rx, quiet, - legacy, optimize, invalidation_mode): + legacy, optimize, invalidation_mode, + stripdir=stripdir, prependdir=prependdir): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, - invalidation_mode=None): + invalidation_mode=None, stripdir=None, prependdir=None): """Byte-compile one file. Arguments (only fullname is required): @@ -111,15 +118,41 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter invalidation_mode: how the up-to-dateness of the pyc will be checked + stripdir: part of path to left-strip from source file path + prependdir: path to prepend to beggining of original file path, applied + after stripdir """ + + if ddir is not None and (stripdir is not None or prependdir is not None): + raise ValueError(("Destination dir (ddir) cannot be used " + "in combination with stripdir or prependdir")) + success = True if quiet < 2 and isinstance(fullname, os.PathLike): fullname = os.fspath(fullname) name = os.path.basename(fullname) + + dfile = None + if ddir is not None: dfile = os.path.join(ddir, name) - else: - dfile = None + + if stripdir is not None: + fullname_parts = fullname.split(os.path.sep) + stripdir_parts = stripdir.split(os.path.sep) + ddir_parts = list(fullname_parts) + + for spart, opart in zip(stripdir_parts, fullname_parts): + if spart == opart: + ddir_parts.remove(spart) + + dfile = os.path.join(*ddir_parts) + + if prependdir is not None: + if dfile is None: + dfile = os.path.join(prependdir, fullname) + else: + dfile = os.path.join(prependdir, dfile) if rx is not None: mo = rx.search(fullname) if mo: @@ -240,6 +273,20 @@ def main(): 'compile-time tracebacks and in runtime ' 'tracebacks in cases where the source file is ' 'unavailable')) + parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir', + default=None, + help=('part of path to left-strip from path ' + 'to source file - for example buildroot. ' + '`-d` and `-s` options cannot be ' + 'specified together.')) + parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir', + default=None, + help=('path to add as prefix to path ' + 'to source file - for example / to make ' + 'it absolute when some part is removed ' + 'by `-s` option. ' + '`-d` and `-p` options cannot be ' + 'specified together.')) parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, help=('skip files matching the regular expression; ' 'the regexp is searched for in the full path ' @@ -300,13 +347,17 @@ def main(): if os.path.isfile(dest): if not compile_file(dest, args.ddir, args.force, args.rx, args.quiet, args.legacy, - invalidation_mode=invalidation_mode): + invalidation_mode=invalidation_mode, + stripdir=args.stripdir, + prependdir=args.prependdir): success = False else: if not compile_dir(dest, maxlevels, args.ddir, args.force, args.rx, args.quiet, args.legacy, workers=args.workers, - invalidation_mode=invalidation_mode): + invalidation_mode=invalidation_mode, + stripdir=args.stripdir, + prependdir=args.prependdir): success = False return success else: From 3ce38d74cab69d1741d5ccfd04fa5db2e3c540dd Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 6 Sep 2019 14:34:56 +0200 Subject: [PATCH 04/13] Add a possibility to specify multiple optimization levels Each optimization level then leads to separated compiled file. Use `action='append'` instead of `nargs='+'` for the -o option. Instead of `-o 0 1 2`, specify `-o 0 -o 1 -o 2`. It's more to type, but much more explicit. --- Lib/compileall.py | 65 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/Lib/compileall.py b/Lib/compileall.py index 919529e8567492..4325e4bf9935a2 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -61,7 +61,9 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, quiet: full output with False or 0, errors only with 1, no output with 2 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths - optimize: optimization level or -1 for level of the interpreter + optimize: int or list of optimization levels or -1 for level of + the interpreter. Multiple levels leads to multiple compiled + files each with one optimization level. workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path @@ -116,7 +118,9 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, quiet: full output with False or 0, errors only with 1, no output with 2 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths - optimize: optimization level or -1 for level of the interpreter + optimize: int or list of optimization levels or -1 for level of + the interpreter. Multiple levels leads to multiple compiled + files each with one optimization level. invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path prependdir: path to prepend to beggining of original file path, applied @@ -153,21 +157,31 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, dfile = os.path.join(prependdir, fullname) else: dfile = os.path.join(prependdir, dfile) + + if isinstance(optimize, int): + optimize = [optimize] + if rx is not None: mo = rx.search(fullname) if mo: return success + + opt_cfiles = {} + if os.path.isfile(fullname): - if legacy: - cfile = fullname + 'c' - else: - if optimize >= 0: - opt = optimize if optimize >= 1 else '' - cfile = importlib.util.cache_from_source( - fullname, optimization=opt) + for opt_level in optimize: + if legacy: + opt_cfiles[opt_level] = fullname + 'c' else: - cfile = importlib.util.cache_from_source(fullname) - cache_dir = os.path.dirname(cfile) + if opt_level >= 0: + opt = opt_level if opt_level >= 1 else '' + cfile = (importlib.util.cache_from_source( + fullname, optimization=opt)) + opt_cfiles[opt_level] = cfile + else: + cfile = importlib.util.cache_from_source(fullname) + opt_cfiles[opt_level] = cfile + head, tail = name[:-3], name[-3:] if tail == '.py': if not force: @@ -175,18 +189,22 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, mtime = int(os.stat(fullname).st_mtime) expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, 0, mtime) - with open(cfile, 'rb') as chandle: - actual = chandle.read(12) - if expect == actual: + for cfile in opt_cfiles.values(): + with open(cfile, 'rb') as chandle: + actual = chandle.read(12) + if expect != actual: + break + else: return success except OSError: pass if not quiet: print('Compiling {!r}...'.format(fullname)) try: - ok = py_compile.compile(fullname, cfile, dfile, True, - optimize=optimize, - invalidation_mode=invalidation_mode) + for opt_level, cfile in opt_cfiles.items(): + ok = py_compile.compile(fullname, cfile, dfile, True, + optimize=opt_level, + invalidation_mode=invalidation_mode) except py_compile.PyCompileError as err: success = False if quiet >= 2: @@ -309,6 +327,10 @@ def main(): '"checked-hash" if the SOURCE_DATE_EPOCH ' 'environment variable is set, and ' '"timestamp" otherwise.')) + parser.add_argument('-o', action='append', type=int, dest='opt_levels', + help=('Optimization levels to run compilation with.' + 'Default is -1 which uses optimization level of' + 'Python interpreter itself (specified by -O).')) args = parser.parse_args() compile_dests = args.compile_dest @@ -323,6 +345,9 @@ def main(): else: maxlevels = args.maxlevels + if args.opt_levels is None: + args.opt_levels = [-1] + # if flist is provided then load it if args.flist: try: @@ -349,7 +374,8 @@ def main(): args.quiet, args.legacy, invalidation_mode=invalidation_mode, stripdir=args.stripdir, - prependdir=args.prependdir): + prependdir=args.prependdir, + optimize=args.opt_levels): success = False else: if not compile_dir(dest, maxlevels, args.ddir, @@ -357,7 +383,8 @@ def main(): args.legacy, workers=args.workers, invalidation_mode=invalidation_mode, stripdir=args.stripdir, - prependdir=args.prependdir): + prependdir=args.prependdir, + optimize=args.opt_levels): success = False return success else: From 23ef6f95ad82ea4d6992c9bb778099db3e7412fc Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Sat, 7 Sep 2019 08:32:53 +0200 Subject: [PATCH 05/13] Add a symlinks limitation feature This feature allows us to limit byte-compilation of symbolic links if they are pointing outside specified dir (build root for example). --- Lib/compileall.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/Lib/compileall.py b/Lib/compileall.py index 4325e4bf9935a2..11e7d793c649bb 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -17,6 +17,7 @@ import struct from functools import partial +from pathlib import Path RECURSION_LIMIT = sys.getrecursionlimit() @@ -48,7 +49,7 @@ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0): def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, - prependdir=None): + prependdir=None, limit_sl_dest=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -69,6 +70,8 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, stripdir: part of path to left-strip from source file path prependdir: path to prepend to beggining of original file path, applied after stripdir + limit_sl_dest: ignore symlinks if they are pointing outside of + the defined path """ ProcessPoolExecutor = None if workers < 0: @@ -93,20 +96,23 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, optimize=optimize, invalidation_mode=invalidation_mode, stripdir=stripdir, - prependdir=prependdir), + prependdir=prependdir, + limit_sl_dest=limit_sl_dest), files) success = min(results, default=True) else: for file in files: if not compile_file(file, ddir, force, rx, quiet, legacy, optimize, invalidation_mode, - stripdir=stripdir, prependdir=prependdir): + stripdir=stripdir, prependdir=prependdir, + limit_sl_dest=limit_sl_dest): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, - invalidation_mode=None, stripdir=None, prependdir=None): + invalidation_mode=None, stripdir=None, prependdir=None, + limit_sl_dest=None): """Byte-compile one file. Arguments (only fullname is required): @@ -125,6 +131,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, stripdir: part of path to left-strip from source file path prependdir: path to prepend to beggining of original file path, applied after stripdir + limit_sl_dest: ignore symlinks if they are pointing outside of + the defined path. """ if ddir is not None and (stripdir is not None or prependdir is not None): @@ -166,6 +174,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if mo: return success + if limit_sl_dest is not None and os.path.islink(fullname): + if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents: + return success + opt_cfiles = {} if os.path.isfile(fullname): @@ -331,6 +343,8 @@ def main(): help=('Optimization levels to run compilation with.' 'Default is -1 which uses optimization level of' 'Python interpreter itself (specified by -O).')) + parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', + help='Ignore symlinks pointing outsite of the DIR') args = parser.parse_args() compile_dests = args.compile_dest @@ -339,6 +353,8 @@ def main(): import re args.rx = re.compile(args.rx) + if args.limit_sl_dest == "": + args.limit_sl_dest = None if args.recursion is not None: maxlevels = args.recursion @@ -375,7 +391,8 @@ def main(): invalidation_mode=invalidation_mode, stripdir=args.stripdir, prependdir=args.prependdir, - optimize=args.opt_levels): + optimize=args.opt_levels, + limit_sl_dest=args.limit_sl_dest): success = False else: if not compile_dir(dest, maxlevels, args.ddir, @@ -384,7 +401,8 @@ def main(): invalidation_mode=invalidation_mode, stripdir=args.stripdir, prependdir=args.prependdir, - optimize=args.opt_levels): + optimize=args.opt_levels, + limit_sl_dest=args.limit_sl_dest): success = False return success else: From ba1a47c1cfd6fdb53a18ba9b6e9726455c72870c Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Sat, 7 Sep 2019 09:23:11 +0200 Subject: [PATCH 06/13] Add test of maxlevels attribute of compile_dir function --- Lib/test/test_compileall.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 99d843704fc73b..e46e1864b497d5 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -41,6 +41,16 @@ def setUp(self): os.mkdir(self.subdirectory) self.source_path3 = os.path.join(self.subdirectory, '_test3.py') shutil.copyfile(self.source_path, self.source_path3) + many_directories = [str(number) for number in range(1, 100)] + self.long_path = os.path.join(self.directory, + "long", + *many_directories) + os.makedirs(self.long_path) + self.source_path_long = os.path.join(self.long_path, '_test4.py') + shutil.copyfile(self.source_path, self.source_path_long) + self.bc_path_long = importlib.util.cache_from_source( + self.source_path_long + ) def tearDown(self): shutil.rmtree(self.directory) @@ -194,6 +204,15 @@ def test_compile_missing_multiprocessing(self, compile_file_mock): compileall.compile_dir(self.directory, quiet=True, workers=5) self.assertTrue(compile_file_mock.called) + def text_compile_dir_maxlevels(self): + # Test the actual impact of maxlevels attr + compileall.compile_dir(os.path.join(self.directory, "long"), + maxlevels=10, quiet=True) + self.assertFalse(os.path.isfile(self.bc_path_long)) + compileall.compile_dir(os.path.join(self.directory, "long"), + maxlevels=110, quiet=True) + self.assertTrue(os.path.isfile(self.bc_path_long)) + class CompileallTestsWithSourceEpoch(CompileallTestsBase, unittest.TestCase, From a5118d2321eb0db1fd239c9d788eab32f9c683dc Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Sat, 7 Sep 2019 09:33:07 +0200 Subject: [PATCH 07/13] Add tests for new features strip/prepend --- Lib/test/test_compileall.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index e46e1864b497d5..f34026b3a69a1b 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -213,6 +213,70 @@ def text_compile_dir_maxlevels(self): maxlevels=110, quiet=True) self.assertTrue(os.path.isfile(self.bc_path_long)) + def test_strip_only(self): + fullpath = ["test", "build", "real", "path"] + path = os.path.join(self.directory, *fullpath) + os.makedirs(path) + script = script_helper.make_script(path, "test", "1 / 0") + bc = importlib.util.cache_from_source(script) + stripdir = os.path.join(self.directory, *fullpath[:2]) + compileall.compile_dir(path, quiet=True, stripdir=stripdir) + rc, out, err = script_helper.assert_python_failure(bc) + expected_in = os.path.join(*fullpath[2:]) + self.assertIn( + expected_in, + str(err, encoding=sys.getdefaultencoding()) + ) + self.assertNotIn( + stripdir, + str(err, encoding=sys.getdefaultencoding()) + ) + + def test_prepend_only(self): + fullpath = ["test", "build", "real", "path"] + path = os.path.join(self.directory, *fullpath) + os.makedirs(path) + script = script_helper.make_script(path, "test", "1 / 0") + bc = importlib.util.cache_from_source(script) + prependdir = "/foo" + compileall.compile_dir(path, quiet=True, prependdir=prependdir) + rc, out, err = script_helper.assert_python_failure(bc) + expected_in = os.path.join(prependdir, self.directory, *fullpath) + self.assertIn( + expected_in, + str(err, encoding=sys.getdefaultencoding()) + ) + + def test_strip_and_prepend(self): + fullpath = ["test", "build", "real", "path"] + path = os.path.join(self.directory, *fullpath) + os.makedirs(path) + script = script_helper.make_script(path, "test", "1 / 0") + bc = importlib.util.cache_from_source(script) + stripdir = os.path.join(self.directory, *fullpath[:2]) + prependdir = "/foo" + compileall.compile_dir(path, quiet=True, + stripdir=stripdir, prependdir=prependdir) + rc, out, err = script_helper.assert_python_failure(bc) + expected_in = os.path.join(prependdir, *fullpath[2:]) + self.assertIn( + expected_in, + str(err, encoding=sys.getdefaultencoding()) + ) + self.assertNotIn( + stripdir, + str(err, encoding=sys.getdefaultencoding()) + ) + + def test_strip_prepend_and_ddir(self): + fullpath = ["test", "build", "real", "path", "ddir"] + path = os.path.join(self.directory, *fullpath) + os.makedirs(path) + script_helper.make_script(path, "test", "1 / 0") + with self.assertRaises(ValueError): + compileall.compile_dir(path, quiet=True, ddir="/bar", + stripdir="/foo", prependdir="/bar") + class CompileallTestsWithSourceEpoch(CompileallTestsBase, unittest.TestCase, @@ -596,6 +660,26 @@ def test_workers_available_cores(self, compile_dir): self.assertTrue(compile_dir.called) self.assertEqual(compile_dir.call_args[-1]['workers'], 0) + def test_strip_and_prepend(self): + fullpath = ["test", "build", "real", "path"] + path = os.path.join(self.directory, *fullpath) + os.makedirs(path) + script = script_helper.make_script(path, "test", "1 / 0") + bc = importlib.util.cache_from_source(script) + stripdir = os.path.join(self.directory, *fullpath[:2]) + prependdir = "/foo" + self.assertRunOK("-s", stripdir, "-p", prependdir, path) + rc, out, err = script_helper.assert_python_failure(bc) + expected_in = os.path.join(prependdir, *fullpath[2:]) + self.assertIn( + expected_in, + str(err, encoding=sys.getdefaultencoding()) + ) + self.assertNotIn( + stripdir, + str(err, encoding=sys.getdefaultencoding()) + ) + class CommandLineTestsWithSourceEpoch(CommandLineTestsBase, unittest.TestCase, From 21acf0d17e0c7e529e1c5696b85a8ef7e4b67ca6 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Sat, 7 Sep 2019 09:36:55 +0200 Subject: [PATCH 08/13] Add tests for multiple optimization levels --- Lib/test/test_compileall.py | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index f34026b3a69a1b..f2c9223ba9fa43 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -277,6 +277,25 @@ def test_strip_prepend_and_ddir(self): compileall.compile_dir(path, quiet=True, ddir="/bar", stripdir="/foo", prependdir="/bar") + def test_multiple_optimization_levels(self): + script = script_helper.make_script(self.directory, + "test_optimization", + "a = 0") + bc = [] + for opt_level in "", 1, 2, 3: + bc.append(importlib.util.cache_from_source(script, + optimization=opt_level)) + test_combinations = [[0, 1], [1, 2], [0, 2], [0, 1, 2]] + for opt_combination in test_combinations: + compileall.compile_file(script, quiet=True, + optimize=opt_combination) + for opt_level in opt_combination: + self.assertTrue(os.path.isfile(bc[opt_level])) + try: + os.unlink(bc[opt_level]) + except Exception: + pass + class CompileallTestsWithSourceEpoch(CompileallTestsBase, unittest.TestCase, @@ -680,6 +699,29 @@ def test_strip_and_prepend(self): str(err, encoding=sys.getdefaultencoding()) ) + def test_multiple_optimization_levels(self): + path = os.path.join(self.directory, "optimizations") + os.makedirs(path) + script = script_helper.make_script(path, + "test_optimization", + "a = 0") + bc = [] + for opt_level in "", 1, 2, 3: + bc.append(importlib.util.cache_from_source(script, + optimization=opt_level)) + test_combinations = [["0", "1"], + ["1", "2"], + ["0", "2"], + ["0", "1", "2"]] + for opt_combination in test_combinations: + self.assertRunOK(path, *("-o" + str(n) for n in opt_combination)) + for opt_level in opt_combination: + self.assertTrue(os.path.isfile(bc[int(opt_level)])) + try: + os.unlink(bc[opt_level]) + except Exception: + pass + class CommandLineTestsWithSourceEpoch(CommandLineTestsBase, unittest.TestCase, From 6ecce95aa2c00a3603e80953f67986fd92c26949 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Sat, 7 Sep 2019 09:40:42 +0200 Subject: [PATCH 09/13] Add tests for symlinks limit feature --- Lib/test/test_compileall.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index f2c9223ba9fa43..8210f4b49e4f3c 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -296,6 +296,31 @@ def test_multiple_optimization_levels(self): except Exception: pass + @support.skip_unless_symlink + def test_ignore_symlink_destination(self): + # Create folders for allowed files, symlinks and prohibited area + allowed_path = os.path.join(self.directory, "test", "dir", "allowed") + symlinks_path = os.path.join(self.directory, "test", "dir", "symlinks") + prohibited_path = os.path.join(self.directory, "test", "dir", "prohibited") + os.makedirs(allowed_path) + os.makedirs(symlinks_path) + os.makedirs(prohibited_path) + + # Create scripts and symlinks and remember their byte-compiled versions + allowed_script = script_helper.make_script(allowed_path, "test_allowed", "a = 0") + prohibited_script = script_helper.make_script(prohibited_path, "test_prohibited", "a = 0") + allowed_symlink = os.path.join(symlinks_path, "test_allowed.py") + prohibited_symlink = os.path.join(symlinks_path, "test_prohibited.py") + os.symlink(allowed_script, allowed_symlink) + os.symlink(prohibited_script, prohibited_symlink) + allowed_bc = importlib.util.cache_from_source(allowed_symlink) + prohibited_bc = importlib.util.cache_from_source(prohibited_symlink) + + compileall.compile_dir(symlinks_path, quiet=True, limit_sl_dest=allowed_path) + + self.assertTrue(os.path.isfile(allowed_bc)) + self.assertFalse(os.path.isfile(prohibited_bc)) + class CompileallTestsWithSourceEpoch(CompileallTestsBase, unittest.TestCase, @@ -722,6 +747,31 @@ def test_multiple_optimization_levels(self): except Exception: pass + @support.skip_unless_symlink + def test_ignore_symlink_destination(self): + # Create folders for allowed files, symlinks and prohibited area + allowed_path = os.path.join(self.directory, "test", "dir", "allowed") + symlinks_path = os.path.join(self.directory, "test", "dir", "symlinks") + prohibited_path = os.path.join(self.directory, "test", "dir", "prohibited") + os.makedirs(allowed_path) + os.makedirs(symlinks_path) + os.makedirs(prohibited_path) + + # Create scripts and symlinks and remember their byte-compiled versions + allowed_script = script_helper.make_script(allowed_path, "test_allowed", "a = 0") + prohibited_script = script_helper.make_script(prohibited_path, "test_prohibited", "a = 0") + allowed_symlink = os.path.join(symlinks_path, "test_allowed.py") + prohibited_symlink = os.path.join(symlinks_path, "test_prohibited.py") + os.symlink(allowed_script, allowed_symlink) + os.symlink(prohibited_script, prohibited_symlink) + allowed_bc = importlib.util.cache_from_source(allowed_symlink) + prohibited_bc = importlib.util.cache_from_source(prohibited_symlink) + + self.assertRunOK(symlinks_path, "-e", allowed_path) + + self.assertTrue(os.path.isfile(allowed_bc)) + self.assertFalse(os.path.isfile(prohibited_bc)) + class CommandLineTestsWithSourceEpoch(CommandLineTestsBase, unittest.TestCase, From 1009e79204305e344d905a96d658001f12c64632 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 12 Sep 2019 13:39:25 +0100 Subject: [PATCH 10/13] Test behavior of compileall with a symlink loop --- Lib/test/test_compileall.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 8210f4b49e4f3c..af885c51224064 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -563,6 +563,20 @@ def test_recursion_limit(self): self.assertCompiled(spamfn) self.assertCompiled(eggfn) + @support.skip_unless_symlink + def test_symlink_loop(self): + # Currently, compileall ignores symlinks to directories. + # If that limitation is ever lifted, it should protect against + # recursion in symlink loops. + pkg = os.path.join(self.pkgdir, 'spam') + script_helper.make_pkg(pkg) + os.symlink('.', os.path.join(pkg, 'evil')) + os.symlink('.', os.path.join(pkg, 'evil2')) + self.assertRunOK('-q', self.pkgdir) + self.assertCompiled(os.path.join( + self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py' + )) + def test_quiet(self): noisy = self.assertRunOK(self.pkgdir) quiet = self.assertRunOK('-q', self.pkgdir) From 6c2469465da9ca4ae599656c3f5109ff862ea8da Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 12 Sep 2019 14:16:53 +0100 Subject: [PATCH 11/13] Adjust documentation --- Doc/library/compileall.rst | 39 +++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 9ce5ca819c68a1..4512eb6e70acd6 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -52,6 +52,13 @@ compile Python sources. cases where the source file does not exist at the time the byte-code file is executed. +.. cmdoption:: -s strip_prefix +.. cmdoption:: -p prepend_prefix + + Remove (``-s``) or append (``-p``) the given prefix of paths + recorded in the ``.pyc`` files. + Raises :exc:`ValueError` if combined with ``-d``. + .. cmdoption:: -x regex regex is used to search the full path to each file considered for @@ -96,6 +103,16 @@ compile Python sources. variable is not set, and ``checked-hash`` if the ``SOURCE_DATE_EPOCH`` environment variable is set. +.. cmdoption:: -o level + + Compile with the given optimization level. May be used multiple times + to compile for multiple levels at a time (for example, + ``compileall -o 1 -o 2``). + +.. cmdoption:: -e dir + + Ignore symlinks pointing outside the given directory. + .. versionchanged:: 3.2 Added the ``-i``, ``-b`` and ``-h`` options. @@ -107,6 +124,12 @@ compile Python sources. .. versionchanged:: 3.7 Added the ``--invalidation-mode`` option. +.. versionchanged:: 3.9 + Added the ``-s``, ``-p``, ``-e`` options. + Raised the default default recursion limit from 10 to + :py:func:`sys.getrecursionlimit()`. + Added the possibility to specify the ``-o`` option multiple times. + There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already @@ -120,7 +143,7 @@ runtime. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None) +.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, prependdir=None, limit_sl_dest=None) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. Return a true value if all the files compiled successfully, @@ -166,6 +189,10 @@ Public functions :class:`py_compile.PycInvalidationMode` enum and controls how the generated pycs are invalidated at runtime. + The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to + the ``-s``, ``-p`` and ``-e`` options described above. + They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. @@ -191,6 +218,9 @@ Public functions .. versionchanged:: 3.8 Setting *workers* to 0 now chooses the optimal number of cores. + .. versionchanged:: 3.9 + Added *stripdir*, *prependdir* and *limit_sl_dest* arguments. + .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) Compile the file with path *fullname*. Return a true value if the file @@ -223,6 +253,10 @@ Public functions :class:`py_compile.PycInvalidationMode` enum and controls how the generated pycs are invalidated at runtime. + The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to + the ``-s``, ``-p`` and ``-e`` options described above. + They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + .. versionadded:: 3.2 .. versionchanged:: 3.5 @@ -238,6 +272,9 @@ Public functions .. versionchanged:: 3.7.2 The *invalidation_mode* parameter's default value is updated to None. + .. versionchanged:: 3.9 + Added *stripdir*, *prependdir* and *limit_sl_dest* arguments. + .. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) Byte-compile all the :file:`.py` files found along ``sys.path``. Return a From 0783a298b1bc4c22015ed4cdca58b7b7c3e85139 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 24 Sep 2019 10:55:58 +0200 Subject: [PATCH 12/13] Add NEWS entry for compileall module changes --- .../next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst diff --git a/Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst b/Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst new file mode 100644 index 00000000000000..ea49898de8d8e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-24-10-55-01.bpo-38112.2EinX9.rst @@ -0,0 +1,3 @@ +:mod:`compileall` has a higher default recursion limit and new command-line +arguments for path manipulation, symlinks handling, and multiple +optimization levels. From 7e1ed0f3756719fdcd6c0fe3f91c967e257d8987 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 25 Sep 2019 13:18:23 +0200 Subject: [PATCH 13/13] Raise an agrparse error for bad arguments; update documentation --- Doc/library/compileall.rst | 4 ++-- Lib/compileall.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 4512eb6e70acd6..394d60634f1e0d 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -57,7 +57,7 @@ compile Python sources. Remove (``-s``) or append (``-p``) the given prefix of paths recorded in the ``.pyc`` files. - Raises :exc:`ValueError` if combined with ``-d``. + Cannot be combined with ``-d``. .. cmdoption:: -x regex @@ -126,7 +126,7 @@ compile Python sources. .. versionchanged:: 3.9 Added the ``-s``, ``-p``, ``-e`` options. - Raised the default default recursion limit from 10 to + Raised the default recursion limit from 10 to :py:func:`sys.getrecursionlimit()`. Added the possibility to specify the ``-o`` option multiple times. diff --git a/Lib/compileall.py b/Lib/compileall.py index 11e7d793c649bb..26caf34baec0ae 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -364,6 +364,11 @@ def main(): if args.opt_levels is None: args.opt_levels = [-1] + if args.ddir is not None and ( + args.stripdir is not None or args.prependdir is not None + ): + parser.error("-d cannot be used in combination with -s or -p") + # if flist is provided then load it if args.flist: try: