Skip to content

Commit ce720d3

Browse files
authored
bpo-39769: Fix compileall ddir for subpkgs. (GH-18676) (GH-18718)
Fix compileall.compile_dir() ddir= behavior on sub-packages. Fixes compileall.compile_dir's ddir parameter and compileall command line flag `-d` to no longer write the wrong pathname to the generated pyc file for submodules beneath the root of the directory tree being compiled. This fixes a regression introduced with Python 3.5. Tests backported from GH 0267335, the implementation is different due to intervening code changes. But still quiet simple. Why was the bug ever introduced? The refactoring to add parallel execution kept the ddir -> dfile computations but discarded the results instead of sending them to compile_file(). This fixes that. Lack of tests meant this went unnoticed.
1 parent fec6681 commit ce720d3

File tree

4 files changed

+73
-12
lines changed

4 files changed

+73
-12
lines changed

Lib/compileall.py

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
4141
else:
4242
dfile = None
4343
if not os.path.isdir(fullname):
44-
yield fullname
44+
yield fullname, ddir
4545
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
4646
os.path.isdir(fullname) and not os.path.islink(fullname)):
4747
yield from _walk_dir(fullname, ddir=dfile,
@@ -76,28 +76,33 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
7676
from concurrent.futures import ProcessPoolExecutor
7777
except ImportError:
7878
workers = 1
79-
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
80-
ddir=ddir)
79+
files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
80+
ddir=ddir)
8181
success = True
8282
if workers != 1 and ProcessPoolExecutor is not None:
8383
# If workers == 0, let ProcessPoolExecutor choose
8484
workers = workers or None
8585
with ProcessPoolExecutor(max_workers=workers) as executor:
86-
results = executor.map(partial(compile_file,
87-
ddir=ddir, force=force,
88-
rx=rx, quiet=quiet,
89-
legacy=legacy,
90-
optimize=optimize,
91-
invalidation_mode=invalidation_mode),
92-
files)
86+
results = executor.map(
87+
partial(_compile_file_tuple,
88+
force=force, rx=rx, quiet=quiet,
89+
legacy=legacy, optimize=optimize,
90+
invalidation_mode=invalidation_mode,
91+
),
92+
files_and_ddirs)
9393
success = min(results, default=True)
9494
else:
95-
for file in files:
96-
if not compile_file(file, ddir, force, rx, quiet,
95+
for file, dfile in files_and_ddirs:
96+
if not compile_file(file, dfile, force, rx, quiet,
9797
legacy, optimize, invalidation_mode):
9898
success = False
9999
return success
100100

101+
def _compile_file_tuple(file_and_dfile, **kwargs):
102+
"""Needs to be toplevel for ProcessPoolExecutor."""
103+
file, dfile = file_and_dfile
104+
return compile_file(file, dfile, **kwargs)
105+
101106
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
102107
legacy=False, optimize=-1,
103108
invalidation_mode=None):

Lib/test/test_compileall.py

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,47 @@ def test_workers_available_cores(self, compile_dir):
577577
self.assertTrue(compile_dir.called)
578578
self.assertEqual(compile_dir.call_args[-1]['workers'], 0)
579579

580+
def _test_ddir_only(self, *, ddir, parallel=True):
581+
"""Recursive compile_dir ddir must contain package paths; bpo39769."""
582+
fullpath = ["test", "foo"]
583+
path = self.directory
584+
mods = []
585+
for subdir in fullpath:
586+
path = os.path.join(path, subdir)
587+
os.mkdir(path)
588+
script_helper.make_script(path, "__init__", "")
589+
mods.append(script_helper.make_script(path, "mod",
590+
"def fn(): 1/0\nfn()\n"))
591+
compileall.compile_dir(
592+
self.directory, quiet=True, ddir=ddir,
593+
workers=2 if parallel else 1)
594+
self.assertTrue(mods)
595+
for mod in mods:
596+
self.assertTrue(mod.startswith(self.directory), mod)
597+
modcode = importlib.util.cache_from_source(mod)
598+
modpath = mod[len(self.directory+os.sep):]
599+
_, _, err = script_helper.assert_python_failure(modcode)
600+
expected_in = os.path.join(ddir, modpath)
601+
mod_code_obj = test.test_importlib.util._get_code_from_pyc(modcode)
602+
self.assertEqual(mod_code_obj.co_filename, expected_in)
603+
self.assertIn(f'"{expected_in}"', os.fsdecode(err))
604+
605+
def test_ddir_only_one_worker(self):
606+
"""Recursive compile_dir ddir= contains package paths; bpo39769."""
607+
return self._test_ddir_only(ddir="<a prefix>", parallel=False)
608+
609+
def test_ddir_multiple_workers(self):
610+
"""Recursive compile_dir ddir= contains package paths; bpo39769."""
611+
return self._test_ddir_only(ddir="<a prefix>", parallel=True)
612+
613+
def test_ddir_empty_only_one_worker(self):
614+
"""Recursive compile_dir ddir='' contains package paths; bpo39769."""
615+
return self._test_ddir_only(ddir="", parallel=False)
616+
617+
def test_ddir_empty_multiple_workers(self):
618+
"""Recursive compile_dir ddir='' contains package paths; bpo39769."""
619+
return self._test_ddir_only(ddir="", parallel=True)
620+
580621

581622
class CommmandLineTestsWithSourceEpoch(CommandLineTestsBase,
582623
unittest.TestCase,

Lib/test/test_importlib/util.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from importlib import machinery, util, invalidate_caches
88
from importlib.abc import ResourceReader
99
import io
10+
import marshal
1011
import os
1112
import os.path
1213
from pathlib import Path, PurePath
@@ -118,6 +119,16 @@ def submodule(parent, name, pkg_dir, content=''):
118119
return '{}.{}'.format(parent, name), path
119120

120121

122+
def _get_code_from_pyc(pyc_path):
123+
"""Reads a pyc file and returns the unmarshalled code object within.
124+
125+
No header validation is performed.
126+
"""
127+
with open(pyc_path, 'rb') as pyc_f:
128+
pyc_f.seek(16)
129+
return marshal.load(pyc_f)
130+
131+
121132
@contextlib.contextmanager
122133
def uncache(*names):
123134
"""Uncache a module from sys.modules.
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The :func:`compileall.compile_dir` function's *ddir* parameter and the
2+
compileall command line flag `-d` no longer write the wrong pathname to the
3+
generated pyc file for submodules beneath the root of the directory tree
4+
being compiled. This fixes a regression introduced with Python 3.5.

0 commit comments

Comments
 (0)