diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py new file mode 100644 index 0000000000..cb455f40c4 --- /dev/null +++ b/Lib/modulefinder.py @@ -0,0 +1,685 @@ +"""Find modules used by a script, using introspection.""" + +import dis +import importlib._bootstrap_external +import importlib.machinery +import marshal +import os +import io +import sys + + +LOAD_CONST = dis.opmap['LOAD_CONST'] +IMPORT_NAME = dis.opmap['IMPORT_NAME'] +STORE_NAME = dis.opmap['STORE_NAME'] +STORE_GLOBAL = dis.opmap['STORE_GLOBAL'] +STORE_OPS = STORE_NAME, STORE_GLOBAL +EXTENDED_ARG = dis.EXTENDED_ARG + +# Old imp constants: + +_SEARCH_ERROR = 0 +_PY_SOURCE = 1 +_PY_COMPILED = 2 +_C_EXTENSION = 3 +_PKG_DIRECTORY = 5 +_C_BUILTIN = 6 +_PY_FROZEN = 7 + +# Modulefinder does a good job at simulating Python's, but it can not +# handle __path__ modifications packages make at runtime. Therefore there +# is a mechanism whereby you can register extra paths in this map for a +# package, and it will be honored. + +# Note this is a mapping is lists of paths. +packagePathMap = {} + +# A Public interface +def AddPackagePath(packagename, path): + packagePathMap.setdefault(packagename, []).append(path) + +replacePackageMap = {} + +# This ReplacePackage mechanism allows modulefinder to work around +# situations in which a package injects itself under the name +# of another package into sys.modules at runtime by calling +# ReplacePackage("real_package_name", "faked_package_name") +# before running ModuleFinder. + +def ReplacePackage(oldname, newname): + replacePackageMap[oldname] = newname + + +def _find_module(name, path=None): + """An importlib reimplementation of imp.find_module (for our purposes).""" + + # It's necessary to clear the caches for our Finder first, in case any + # modules are being added/deleted/modified at runtime. In particular, + # test_modulefinder.py changes file tree contents in a cache-breaking way: + + importlib.machinery.PathFinder.invalidate_caches() + + spec = importlib.machinery.PathFinder.find_spec(name, path) + + if spec is None: + raise ImportError("No module named {name!r}".format(name=name), name=name) + + # Some special cases: + + if spec.loader is importlib.machinery.BuiltinImporter: + return None, None, ("", "", _C_BUILTIN) + + if spec.loader is importlib.machinery.FrozenImporter: + return None, None, ("", "", _PY_FROZEN) + + file_path = spec.origin + + if spec.loader.is_package(name): + return None, os.path.dirname(file_path), ("", "", _PKG_DIRECTORY) + + if isinstance(spec.loader, importlib.machinery.SourceFileLoader): + kind = _PY_SOURCE + + elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): + kind = _C_EXTENSION + + elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): + kind = _PY_COMPILED + + else: # Should never happen. + return None, None, ("", "", _SEARCH_ERROR) + + file = io.open_code(file_path) + suffix = os.path.splitext(file_path)[-1] + + return file, file_path, (suffix, "rb", kind) + + +class Module: + + def __init__(self, name, file=None, path=None): + self.__name__ = name + self.__file__ = file + self.__path__ = path + self.__code__ = None + # The set of global names that are assigned to in the module. + # This includes those names imported through starimports of + # Python modules. + self.globalnames = {} + # The set of starimports this module did that could not be + # resolved, ie. a starimport from a non-Python module. + self.starimports = {} + + def __repr__(self): + s = "Module(%r" % (self.__name__,) + if self.__file__ is not None: + s = s + ", %r" % (self.__file__,) + if self.__path__ is not None: + s = s + ", %r" % (self.__path__,) + s = s + ")" + return s + +class ModuleFinder: + + def __init__(self, path=None, debug=0, excludes=None, replace_paths=None): + if path is None: + path = sys.path + self.path = path + self.modules = {} + self.badmodules = {} + self.debug = debug + self.indent = 0 + self.excludes = excludes if excludes is not None else [] + self.replace_paths = replace_paths if replace_paths is not None else [] + self.processed_paths = [] # Used in debugging only + + def msg(self, level, str, *args): + if level <= self.debug: + for i in range(self.indent): + print(" ", end=' ') + print(str, end=' ') + for arg in args: + print(repr(arg), end=' ') + print() + + def msgin(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent + 1 + self.msg(*args) + + def msgout(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent - 1 + self.msg(*args) + + def run_script(self, pathname): + self.msg(2, "run_script", pathname) + with io.open_code(pathname) as fp: + stuff = ("", "rb", _PY_SOURCE) + self.load_module('__main__', fp, pathname, stuff) + + def load_file(self, pathname): + dir, name = os.path.split(pathname) + name, ext = os.path.splitext(name) + with io.open_code(pathname) as fp: + stuff = (ext, "rb", _PY_SOURCE) + self.load_module(name, fp, pathname, stuff) + + def import_hook(self, name, caller=None, fromlist=None, level=-1): + self.msg(3, "import_hook", name, caller, fromlist, level) + parent = self.determine_parent(caller, level=level) + q, tail = self.find_head_package(parent, name) + m = self.load_tail(q, tail) + if not fromlist: + return q + if m.__path__: + self.ensure_fromlist(m, fromlist) + return None + + def determine_parent(self, caller, level=-1): + self.msgin(4, "determine_parent", caller, level) + if not caller or level == 0: + self.msgout(4, "determine_parent -> None") + return None + pname = caller.__name__ + if level >= 1: # relative import + if caller.__path__: + level -= 1 + if level == 0: + parent = self.modules[pname] + assert parent is caller + self.msgout(4, "determine_parent ->", parent) + return parent + if pname.count(".") < level: + raise ImportError("relative importpath too deep") + pname = ".".join(pname.split(".")[:-level]) + parent = self.modules[pname] + self.msgout(4, "determine_parent ->", parent) + return parent + if caller.__path__: + parent = self.modules[pname] + assert caller is parent + self.msgout(4, "determine_parent ->", parent) + return parent + if '.' in pname: + i = pname.rfind('.') + pname = pname[:i] + parent = self.modules[pname] + assert parent.__name__ == pname + self.msgout(4, "determine_parent ->", parent) + return parent + self.msgout(4, "determine_parent -> None") + return None + + def find_head_package(self, parent, name): + self.msgin(4, "find_head_package", parent, name) + if '.' in name: + i = name.find('.') + head = name[:i] + tail = name[i+1:] + else: + head = name + tail = "" + if parent: + qname = "%s.%s" % (parent.__name__, head) + else: + qname = head + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + if parent: + qname = head + parent = None + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + self.msgout(4, "raise ImportError: No module named", qname) + raise ImportError("No module named " + qname) + + def load_tail(self, q, tail): + self.msgin(4, "load_tail", q, tail) + m = q + while tail: + i = tail.find('.') + if i < 0: i = len(tail) + head, tail = tail[:i], tail[i+1:] + mname = "%s.%s" % (m.__name__, head) + m = self.import_module(head, mname, m) + if not m: + self.msgout(4, "raise ImportError: No module named", mname) + raise ImportError("No module named " + mname) + self.msgout(4, "load_tail ->", m) + return m + + def ensure_fromlist(self, m, fromlist, recursive=0): + self.msg(4, "ensure_fromlist", m, fromlist, recursive) + for sub in fromlist: + if sub == "*": + if not recursive: + all = self.find_all_submodules(m) + if all: + self.ensure_fromlist(m, all, 1) + elif not hasattr(m, sub): + subname = "%s.%s" % (m.__name__, sub) + submod = self.import_module(sub, subname, m) + if not submod: + raise ImportError("No module named " + subname) + + def find_all_submodules(self, m): + if not m.__path__: + return + modules = {} + # 'suffixes' used to be a list hardcoded to [".py", ".pyc"]. + # But we must also collect Python extension modules - although + # we cannot separate normal dlls from Python extensions. + suffixes = [] + suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] + for dir in m.__path__: + try: + names = os.listdir(dir) + except OSError: + self.msg(2, "can't list directory", dir) + continue + for name in names: + mod = None + for suff in suffixes: + n = len(suff) + if name[-n:] == suff: + mod = name[:-n] + break + if mod and mod != "__init__": + modules[mod] = mod + return modules.keys() + + def import_module(self, partname, fqname, parent): + self.msgin(3, "import_module", partname, fqname, parent) + try: + m = self.modules[fqname] + except KeyError: + pass + else: + self.msgout(3, "import_module ->", m) + return m + if fqname in self.badmodules: + self.msgout(3, "import_module -> None") + return None + if parent and parent.__path__ is None: + self.msgout(3, "import_module -> None") + return None + try: + fp, pathname, stuff = self.find_module(partname, + parent and parent.__path__, parent) + except ImportError: + self.msgout(3, "import_module ->", None) + return None + + try: + m = self.load_module(fqname, fp, pathname, stuff) + finally: + if fp: + fp.close() + if parent: + setattr(parent, partname, m) + self.msgout(3, "import_module ->", m) + return m + + def load_module(self, fqname, fp, pathname, file_info): + suffix, mode, type = file_info + self.msgin(2, "load_module", fqname, fp and "fp", pathname) + if type == _PKG_DIRECTORY: + m = self.load_package(fqname, pathname) + self.msgout(2, "load_module ->", m) + return m + if type == _PY_SOURCE: + co = compile(fp.read(), pathname, 'exec') + elif type == _PY_COMPILED: + try: + data = fp.read() + importlib._bootstrap_external._classify_pyc(data, fqname, {}) + except ImportError as exc: + self.msgout(2, "raise ImportError: " + str(exc), pathname) + raise + co = marshal.loads(memoryview(data)[16:]) + else: + co = None + m = self.add_module(fqname) + m.__file__ = pathname + if co: + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) + self.msgout(2, "load_module ->", m) + return m + + def _add_badmodule(self, name, caller): + if name not in self.badmodules: + self.badmodules[name] = {} + if caller: + self.badmodules[name][caller.__name__] = 1 + else: + self.badmodules[name]["-"] = 1 + + def _safe_import_hook(self, name, caller, fromlist, level=-1): + # wrapper for self.import_hook() that won't raise ImportError + if name in self.badmodules: + self._add_badmodule(name, caller) + return + try: + self.import_hook(name, caller, level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(name, caller) + except SyntaxError as msg: + self.msg(2, "SyntaxError:", str(msg)) + self._add_badmodule(name, caller) + else: + if fromlist: + for sub in fromlist: + fullname = name + "." + sub + if fullname in self.badmodules: + self._add_badmodule(fullname, caller) + continue + try: + self.import_hook(name, caller, [sub], level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(fullname, caller) + + def scan_opcodes(self, co): + # Scan the code, and yield 'interesting' opcode combinations + code = co.co_code + names = co.co_names + consts = co.co_consts + opargs = [(op, arg) for _, op, arg in dis._unpack_opargs(code) + if op != EXTENDED_ARG] + for i, (op, oparg) in enumerate(opargs): + if op in STORE_OPS: + yield "store", (names[oparg],) + continue + if (op == IMPORT_NAME and i >= 2 + and opargs[i-1][0] == opargs[i-2][0] == LOAD_CONST): + level = consts[opargs[i-2][1]] + fromlist = consts[opargs[i-1][1]] + if level == 0: # absolute import + yield "absolute_import", (fromlist, names[oparg]) + else: # relative import + yield "relative_import", (level, fromlist, names[oparg]) + continue + + def scan_code(self, co, m): + code = co.co_code + scanner = self.scan_opcodes + for what, args in scanner(co): + if what == "store": + name, = args + m.globalnames[name] = 1 + elif what == "absolute_import": + fromlist, name = args + have_star = 0 + if fromlist is not None: + if "*" in fromlist: + have_star = 1 + fromlist = [f for f in fromlist if f != "*"] + self._safe_import_hook(name, m, fromlist, level=0) + if have_star: + # We've encountered an "import *". If it is a Python module, + # the code has already been parsed and we can suck out the + # global names. + mm = None + if m.__path__: + # At this point we don't know whether 'name' is a + # submodule of 'm' or a global module. Let's just try + # the full name first. + mm = self.modules.get(m.__name__ + "." + name) + if mm is None: + mm = self.modules.get(name) + if mm is not None: + m.globalnames.update(mm.globalnames) + m.starimports.update(mm.starimports) + if mm.__code__ is None: + m.starimports[name] = 1 + else: + m.starimports[name] = 1 + elif what == "relative_import": + level, fromlist, name = args + if name: + self._safe_import_hook(name, m, fromlist, level=level) + else: + parent = self.determine_parent(m, level=level) + self._safe_import_hook(parent.__name__, None, fromlist, level=0) + else: + # We don't expect anything else from the generator. + raise RuntimeError(what) + + for c in co.co_consts: + if isinstance(c, type(co)): + self.scan_code(c, m) + + def load_package(self, fqname, pathname): + self.msgin(2, "load_package", fqname, pathname) + newname = replacePackageMap.get(fqname) + if newname: + fqname = newname + m = self.add_module(fqname) + m.__file__ = pathname + m.__path__ = [pathname] + + # As per comment at top of file, simulate runtime __path__ additions. + m.__path__ = m.__path__ + packagePathMap.get(fqname, []) + + fp, buf, stuff = self.find_module("__init__", m.__path__) + try: + self.load_module(fqname, fp, buf, stuff) + self.msgout(2, "load_package ->", m) + return m + finally: + if fp: + fp.close() + + def add_module(self, fqname): + if fqname in self.modules: + return self.modules[fqname] + self.modules[fqname] = m = Module(fqname) + return m + + def find_module(self, name, path, parent=None): + if parent is not None: + # assert path is not None + fullname = parent.__name__+'.'+name + else: + fullname = name + if fullname in self.excludes: + self.msgout(3, "find_module -> Excluded", fullname) + raise ImportError(name) + + if path is None: + if name in sys.builtin_module_names: + return (None, None, ("", "", _C_BUILTIN)) + + path = self.path + + return _find_module(name, path) + + def report(self): + """Print a report to stdout, listing the found modules with their + paths, as well as modules that are missing, or seem to be missing. + """ + print() + print(" %-25s %s" % ("Name", "File")) + print(" %-25s %s" % ("----", "----")) + # Print modules found + keys = sorted(self.modules.keys()) + for key in keys: + m = self.modules[key] + if m.__path__: + print("P", end=' ') + else: + print("m", end=' ') + print("%-25s" % key, m.__file__ or "") + + # Print missing modules + missing, maybe = self.any_missing_maybe() + if missing: + print() + print("Missing modules:") + for name in missing: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + # Print modules that may be missing, but then again, maybe not... + if maybe: + print() + print("Submodules that appear to be missing, but could also be", end=' ') + print("global names in the parent package:") + for name in maybe: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + + def any_missing(self): + """Return a list of modules that appear to be missing. Use + any_missing_maybe() if you want to know which modules are + certain to be missing, and which *may* be missing. + """ + missing, maybe = self.any_missing_maybe() + return missing + maybe + + def any_missing_maybe(self): + """Return two lists, one with modules that are certainly missing + and one with modules that *may* be missing. The latter names could + either be submodules *or* just global names in the package. + + The reason it can't always be determined is that it's impossible to + tell which names are imported when "from module import *" is done + with an extension module, short of actually importing it. + """ + missing = [] + maybe = [] + for name in self.badmodules: + if name in self.excludes: + continue + i = name.rfind(".") + if i < 0: + missing.append(name) + continue + subname = name[i+1:] + pkgname = name[:i] + pkg = self.modules.get(pkgname) + if pkg is not None: + if pkgname in self.badmodules[name]: + # The package tried to import this module itself and + # failed. It's definitely missing. + missing.append(name) + elif subname in pkg.globalnames: + # It's a global in the package: definitely not missing. + pass + elif pkg.starimports: + # It could be missing, but the package did an "import *" + # from a non-Python module, so we simply can't be sure. + maybe.append(name) + else: + # It's not a global in the package, the package didn't + # do funny star imports, it's very likely to be missing. + # The symbol could be inserted into the package from the + # outside, but since that's not good style we simply list + # it missing. + missing.append(name) + else: + missing.append(name) + missing.sort() + maybe.sort() + return missing, maybe + + def replace_paths_in_code(self, co): + new_filename = original_filename = os.path.normpath(co.co_filename) + for f, r in self.replace_paths: + if original_filename.startswith(f): + new_filename = r + original_filename[len(f):] + break + + if self.debug and original_filename not in self.processed_paths: + if new_filename != original_filename: + self.msgout(2, "co_filename %r changed to %r" \ + % (original_filename,new_filename,)) + else: + self.msgout(2, "co_filename %r remains unchanged" \ + % (original_filename,)) + self.processed_paths.append(original_filename) + + consts = list(co.co_consts) + for i in range(len(consts)): + if isinstance(consts[i], type(co)): + consts[i] = self.replace_paths_in_code(consts[i]) + + return co.replace(co_consts=tuple(consts), co_filename=new_filename) + + +def test(): + # Parse command line + import getopt + try: + opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") + except getopt.error as msg: + print(msg) + return + + # Process options + debug = 1 + domods = 0 + addpath = [] + exclude = [] + for o, a in opts: + if o == '-d': + debug = debug + 1 + if o == '-m': + domods = 1 + if o == '-p': + addpath = addpath + a.split(os.pathsep) + if o == '-q': + debug = 0 + if o == '-x': + exclude.append(a) + + # Provide default arguments + if not args: + script = "hello.py" + else: + script = args[0] + + # Set the path based on sys.path and the script directory + path = sys.path[:] + path[0] = os.path.dirname(script) + path = addpath + path + if debug > 1: + print("path:") + for item in path: + print(" ", repr(item)) + + # Create the module finder and turn its crank + mf = ModuleFinder(path, debug, exclude) + for arg in args[1:]: + if arg == '-m': + domods = 1 + continue + if domods: + if arg[-2:] == '.*': + mf.import_hook(arg[:-2], None, ["*"]) + else: + mf.import_hook(arg) + else: + mf.load_file(arg) + mf.run_script(script) + mf.report() + return mf # for -i debugging + + +if __name__ == '__main__': + try: + mf = test() + except KeyboardInterrupt: + print("\n[interrupted]") diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py new file mode 100644 index 0000000000..ca1058b8d4 --- /dev/null +++ b/Lib/test/test_modulefinder.py @@ -0,0 +1,437 @@ +import os +import errno +import importlib.machinery +import py_compile +import shutil +import unittest +import tempfile + +from test import support + +import modulefinder + +TEST_DIR = tempfile.mkdtemp() +TEST_PATH = [TEST_DIR, os.path.dirname(tempfile.__file__)] + +# Each test description is a list of 5 items: +# +# 1. a module name that will be imported by modulefinder +# 2. a list of module names that modulefinder is required to find +# 3. a list of module names that modulefinder should complain +# about because they are not found +# 4. a list of module names that modulefinder should complain +# about because they MAY be not found +# 5. a string specifying packages to create; the format is obvious imo. +# +# Each package will be created in TEST_DIR, and TEST_DIR will be +# removed after the tests again. +# Modulefinder searches in a path that contains TEST_DIR, plus +# the standard Lib directory. + +maybe_test = [ + "a.module", + ["a", "a.module", "sys", + "b"], + ["c"], ["b.something"], + """\ +a/__init__.py +a/module.py + from b import something + from c import something +b/__init__.py + from sys import * +""", +] + +maybe_test_new = [ + "a.module", + ["a", "a.module", "sys", + "b", "__future__"], + ["c"], ["b.something"], + """\ +a/__init__.py +a/module.py + from b import something + from c import something +b/__init__.py + from __future__ import absolute_import + from sys import * +"""] + +package_test = [ + "a.module", + ["a", "a.b", "a.c", "a.module", "mymodule", "sys"], + ["blahblah", "c"], [], + """\ +mymodule.py +a/__init__.py + import blahblah + from a import b + import c +a/module.py + import sys + from a import b as x + from a.c import sillyname +a/b.py +a/c.py + from a.module import x + import mymodule as sillyname + from sys import version_info +"""] + +absolute_import_test = [ + "a.module", + ["a", "a.module", + "b", "b.x", "b.y", "b.z", + "__future__", "sys", "gc"], + ["blahblah", "z"], [], + """\ +mymodule.py +a/__init__.py +a/module.py + from __future__ import absolute_import + import sys # sys + import blahblah # fails + import gc # gc + import b.x # b.x + from b import y # b.y + from b.z import * # b.z.* +a/gc.py +a/sys.py + import mymodule +a/b/__init__.py +a/b/x.py +a/b/y.py +a/b/z.py +b/__init__.py + import z +b/unused.py +b/x.py +b/y.py +b/z.py +"""] + +relative_import_test = [ + "a.module", + ["__future__", + "a", "a.module", + "a.b", "a.b.y", "a.b.z", + "a.b.c", "a.b.c.moduleC", + "a.b.c.d", "a.b.c.e", + "a.b.x", + "gc"], + [], [], + """\ +mymodule.py +a/__init__.py + from .b import y, z # a.b.y, a.b.z +a/module.py + from __future__ import absolute_import # __future__ + import gc # gc +a/gc.py +a/sys.py +a/b/__init__.py + from ..b import x # a.b.x + #from a.b.c import moduleC + from .c import moduleC # a.b.moduleC +a/b/x.py +a/b/y.py +a/b/z.py +a/b/g.py +a/b/c/__init__.py + from ..c import e # a.b.c.e +a/b/c/moduleC.py + from ..c import d # a.b.c.d +a/b/c/d.py +a/b/c/e.py +a/b/c/x.py +"""] + +relative_import_test_2 = [ + "a.module", + ["a", "a.module", + "a.sys", + "a.b", "a.b.y", "a.b.z", + "a.b.c", "a.b.c.d", + "a.b.c.e", + "a.b.c.moduleC", + "a.b.c.f", + "a.b.x", + "a.another"], + [], [], + """\ +mymodule.py +a/__init__.py + from . import sys # a.sys +a/another.py +a/module.py + from .b import y, z # a.b.y, a.b.z +a/gc.py +a/sys.py +a/b/__init__.py + from .c import moduleC # a.b.c.moduleC + from .c import d # a.b.c.d +a/b/x.py +a/b/y.py +a/b/z.py +a/b/c/__init__.py + from . import e # a.b.c.e +a/b/c/moduleC.py + # + from . import f # a.b.c.f + from .. import x # a.b.x + from ... import another # a.another +a/b/c/d.py +a/b/c/e.py +a/b/c/f.py +"""] + +relative_import_test_3 = [ + "a.module", + ["a", "a.module"], + ["a.bar"], + [], + """\ +a/__init__.py + def foo(): pass +a/module.py + from . import foo + from . import bar +"""] + +relative_import_test_4 = [ + "a.module", + ["a", "a.module"], + [], + [], + """\ +a/__init__.py + def foo(): pass +a/module.py + from . import * +"""] + +bytecode_test = [ + "a", + ["a"], + [], + [], + "" +] + +syntax_error_test = [ + "a.module", + ["a", "a.module", "b"], + ["b.module"], [], + """\ +a/__init__.py +a/module.py + import b.module +b/__init__.py +b/module.py + ? # SyntaxError: invalid syntax +"""] + + +same_name_as_bad_test = [ + "a.module", + ["a", "a.module", "b", "b.c"], + ["c"], [], + """\ +a/__init__.py +a/module.py + import c + from b import c +b/__init__.py +b/c.py +"""] + +coding_default_utf8_test = [ + "a_utf8", + ["a_utf8", "b_utf8"], + [], [], + """\ +a_utf8.py + # use the default of utf8 + print('Unicode test A code point 2090 \u2090 that is not valid in cp1252') + import b_utf8 +b_utf8.py + # use the default of utf8 + print('Unicode test B code point 2090 \u2090 that is not valid in cp1252') +"""] + +coding_explicit_utf8_test = [ + "a_utf8", + ["a_utf8", "b_utf8"], + [], [], + """\ +a_utf8.py + # coding=utf8 + print('Unicode test A code point 2090 \u2090 that is not valid in cp1252') + import b_utf8 +b_utf8.py + # use the default of utf8 + print('Unicode test B code point 2090 \u2090 that is not valid in cp1252') +"""] + +coding_explicit_cp1252_test = [ + "a_cp1252", + ["a_cp1252", "b_utf8"], + [], [], + b"""\ +a_cp1252.py + # coding=cp1252 + # 0xe2 is not allowed in utf8 + print('CP1252 test P\xe2t\xe9') + import b_utf8 +""" + """\ +b_utf8.py + # use the default of utf8 + print('Unicode test A code point 2090 \u2090 that is not valid in cp1252') +""".encode('utf-8')] + +def open_file(path): + dirname = os.path.dirname(path) + try: + os.makedirs(dirname) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return open(path, 'wb') + + +def create_package(source): + ofi = None + try: + for line in source.splitlines(): + if type(line) != bytes: + line = line.encode('utf-8') + if line.startswith(b' ') or line.startswith(b'\t'): + ofi.write(line.strip() + b'\n') + else: + if ofi: + ofi.close() + if type(line) == bytes: + line = line.decode('utf-8') + ofi = open_file(os.path.join(TEST_DIR, line.strip())) + finally: + if ofi: + ofi.close() + +class ModuleFinderTest(unittest.TestCase): + def _do_test(self, info, report=False, debug=0, replace_paths=[], modulefinder_class=modulefinder.ModuleFinder): + import_this, modules, missing, maybe_missing, source = info + create_package(source) + try: + mf = modulefinder_class(path=TEST_PATH, debug=debug, + replace_paths=replace_paths) + mf.import_hook(import_this) + if report: + mf.report() +## # This wouldn't work in general when executed several times: +## opath = sys.path[:] +## sys.path = TEST_PATH +## try: +## __import__(import_this) +## except: +## import traceback; traceback.print_exc() +## sys.path = opath +## return + modules = sorted(set(modules)) + found = sorted(mf.modules) + # check if we found what we expected, not more, not less + self.assertEqual(found, modules) + + # check for missing and maybe missing modules + bad, maybe = mf.any_missing_maybe() + self.assertEqual(bad, missing) + self.assertEqual(maybe, maybe_missing) + finally: + shutil.rmtree(TEST_DIR) + + def test_package(self): + self._do_test(package_test) + + def test_maybe(self): + self._do_test(maybe_test) + + def test_maybe_new(self): + self._do_test(maybe_test_new) + + def test_absolute_imports(self): + self._do_test(absolute_import_test) + + def test_relative_imports(self): + self._do_test(relative_import_test) + + def test_relative_imports_2(self): + self._do_test(relative_import_test_2) + + def test_relative_imports_3(self): + self._do_test(relative_import_test_3) + + def test_relative_imports_4(self): + self._do_test(relative_import_test_4) + + def test_syntax_error(self): + self._do_test(syntax_error_test) + + def test_same_name_as_bad(self): + self._do_test(same_name_as_bad_test) + + def test_bytecode(self): + base_path = os.path.join(TEST_DIR, 'a') + source_path = base_path + importlib.machinery.SOURCE_SUFFIXES[0] + bytecode_path = base_path + importlib.machinery.BYTECODE_SUFFIXES[0] + with open_file(source_path) as file: + file.write('testing_modulefinder = True\n'.encode('utf-8')) + py_compile.compile(source_path, cfile=bytecode_path) + os.remove(source_path) + self._do_test(bytecode_test) + + def test_replace_paths(self): + old_path = os.path.join(TEST_DIR, 'a', 'module.py') + new_path = os.path.join(TEST_DIR, 'a', 'spam.py') + with support.captured_stdout() as output: + self._do_test(maybe_test, debug=2, + replace_paths=[(old_path, new_path)]) + output = output.getvalue() + expected = "co_filename %r changed to %r" % (old_path, new_path) + self.assertIn(expected, output) + + def test_extended_opargs(self): + extended_opargs_test = [ + "a", + ["a", "b"], + [], [], + """\ +a.py + %r + import b +b.py +""" % list(range(2**16))] # 2**16 constants + self._do_test(extended_opargs_test) + + def test_coding_default_utf8(self): + self._do_test(coding_default_utf8_test) + + def test_coding_explicit_utf8(self): + self._do_test(coding_explicit_utf8_test) + + def test_coding_explicit_cp1252(self): + self._do_test(coding_explicit_cp1252_test) + + def test_load_module_api(self): + class CheckLoadModuleApi(modulefinder.ModuleFinder): + def __init__(self, *args, **kwds): + super().__init__(*args, **kwds) + + def load_module(self, fqname, fp, pathname, file_info): + # confirm that the fileinfo is a tuple of 3 elements + suffix, mode, type = file_info + return super().load_module(fqname, fp, pathname, file_info) + + self._do_test(absolute_import_test, modulefinder_class=CheckLoadModuleApi) + +if __name__ == "__main__": + unittest.main()