From c14f999abc59d270aaebab59e585d5c8067d29c4 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 14 Sep 2018 14:56:51 -0700 Subject: [PATCH 1/4] bpo-34632: Port importlib_metadata to Python 3.8 --- Lib/importlib/metadata.py | 269 ++++++++++++++++++ Lib/test/test_importlib/data00/__init__.py | 0 .../data00/example-21.12-py3-none-any.whl | Bin 0 -> 1453 bytes Lib/test/test_importlib/test_metadata.py | 91 ++++++ 4 files changed, 360 insertions(+) create mode 100644 Lib/importlib/metadata.py create mode 100644 Lib/test/test_importlib/data00/__init__.py create mode 100644 Lib/test/test_importlib/data00/example-21.12-py3-none-any.whl create mode 100644 Lib/test/test_importlib/test_metadata.py diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py new file mode 100644 index 00000000000000..af1d42125d0025 --- /dev/null +++ b/Lib/importlib/metadata.py @@ -0,0 +1,269 @@ +import os +import re +import abc +import sys +import email +import itertools +import contextlib + +from configparser import ConfigParser +from importlib import import_module +from pathlib import Path +from types import ModuleType +from zipfile import ZipFile + + +__all__ = [ + 'Distribution', + 'PackageNotFoundError', + 'distribution', + 'entry_points', + 'resolve', + 'version', + ] + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + +def distribution(package): + """Get the ``Distribution`` instance for the given package. + + :param package: The module object for the package or the name of the + package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + if isinstance(package, ModuleType): + return Distribution.from_module(package) + else: + return Distribution.from_name(package) + + +def entry_points(name): + """Return the entry points for the named distribution package. + + :param name: The name of the distribution package to query. + :return: A ConfigParser instance where the sections and keys are taken + from the entry_points.txt ini-style contents. + """ + as_string = distribution(name).load_metadata('entry_points.txt') + # 2018-09-10(barry): Should we provide any options here, or let the caller + # send options to the underlying ConfigParser? For now, YAGNI. + config = ConfigParser() + config.read_string(as_string) + return config + + +def resolve(entry_point): + """Resolve an entry point string into the named callable. + + :param entry_point: An entry point string of the form + `path.to.module:callable`. + :return: The actual callable object `path.to.module.callable` + :raises ValueError: When `entry_point` doesn't have the proper format. + """ + path, colon, name = entry_point.rpartition(':') + if colon != ':': + raise ValueError('Not an entry point: {}'.format(entry_point)) + module = import_module(path) + return getattr(module, name) + + +def version(package): + """Get the version string for the named package. + + :param package: The module object for the package or the name of the + package as a string. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(package).version + + +class Distribution: + """A Python distribution package.""" + + @abc.abstractmethod + def load_metadata(self, name): + """Attempt to load metadata given by the name. + + :param name: The name of the distribution package. + :return: The metadata string if found, otherwise None. + """ + + @classmethod + def from_name(cls, name): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + """ + for resolver in cls._discover_resolvers(): + resolved = resolver(name) + if resolved is not None: + return resolved + else: + raise PackageNotFoundError(name) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distribution', None) + for finder in sys.meta_path + ) + return filter(None, declared) + + @classmethod + def from_module(cls, module): + """Discover the Distribution package for a module.""" + return cls.from_name(cls.name_for_module(module)) + + @classmethod + def from_named_module(cls, mod_name): + return cls.from_module(import_module(mod_name)) + + @staticmethod + def name_for_module(module): + """Given an imported module, infer the distribution package name.""" + return getattr(module, '__dist_name__', module.__name__) + + @property + def metadata(self): + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + return email.message_from_string( + self.load_metadata('METADATA') or self.load_metadata('PKG-INFO') + ) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + +class MetadataPathFinder: + """A degenerate finder for distribution packages on the file system. + + This finder supplies only a find_distribution() method for versions + of Python that do not have a PathFinder find_distribution(). + """ + @staticmethod + def find_spec(*args, **kwargs): + return None + + @classmethod + def find_distribution(cls, name): + paths = cls._search_paths(name) + dists = map(PathDistribution, paths) + return next(dists, None) + + @classmethod + def _search_paths(cls, name): + """ + Find metadata directories in sys.path heuristically. + """ + return itertools.chain.from_iterable( + cls._search_path(path, name) + for path in map(Path, sys.path) + ) + + @classmethod + def _search_path(cls, root, name): + if not root.is_dir(): + return () + return ( + item + for item in root.iterdir() + if item.is_dir() + and str(item.name).startswith(name) + and re.match(rf'{name}(-.*)?\.(dist|egg)-info', str(item.name)) + ) + + +class PathDistribution(Distribution): + def __init__(self, path): + """Construct a distribution from a path to the metadata directory.""" + self._path = path + + def load_metadata(self, name): + """Attempt to load metadata given by the name. + + :param name: The name of the distribution package. + :return: The metadata string if found, otherwise None. + """ + filename = os.path.join(self._path, name) + with contextlib.suppress(FileNotFoundError): + with open(filename, encoding='utf-8') as fp: + return fp.read() + return None + + +class WheelMetadataFinder: + """A degenerate finder for distribution packages in wheels. + + This finder supplies only a find_distribution() method for versions + of Python that do not have a PathFinder find_distribution(). + """ + @staticmethod + def find_spec(*args, **kwargs): + return None + + @classmethod + def find_distribution(cls, name): + try: + module = import_module(name) + except ImportError: + return None + archive = getattr(module.__loader__, 'archive', None) + if archive is None: + return None + try: + name, version = Path(archive).name.split('-')[0:2] + except ValueError: + return None + dist_info = '{}-{}.dist-info'.format(name, version) + with ZipFile(archive) as zf: + # Since we're opening the zip file anyway to see if there's a + # METADATA file in the .dist-info directory, we might as well + # read it and cache it here. + zi = zf.getinfo('{}/{}'.format(dist_info, 'METADATA')) + metadata = zf.read(zi).decode('utf-8') + return WheelDistribution(archive, dist_info, metadata) + + +class WheelDistribution(Distribution): + def __init__(self, archive, dist_info, metadata): + self._archive = archive + self._dist_info = dist_info + self._metadata = metadata + + def load_metadata(self, name): + """Attempt to load metadata given by the name. + + :param name: The name of the distribution package. + :return: The metadata string if found, otherwise None. + """ + if name == 'METADATA': + return self._metadata + with ZipFile(self._archive) as zf: + with contextlib.suppress(KeyError): + as_bytes = zf.read('{}/{}'.format(self._dist_info, name)) + return as_bytes.decode('utf-8') + return None + + +def _install(): # pragma: nocover + """Install the appropriate sys.meta_path finder for the Python version.""" + sys.meta_path.append(MetadataPathFinder) + sys.meta_path.append(WheelMetadataFinder) + + +_install() diff --git a/Lib/test/test_importlib/data00/__init__.py b/Lib/test/test_importlib/data00/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data00/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/data00/example-21.12-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..f92f7716e3e613a2a9703be785135fd8b35cfb1c GIT binary patch literal 1453 zcmWIWW@Zs#U|`^2$X%l2E2Vj`LJ`OVVPPOntw_u*$Vt_YkI&4@EQycTE2#AL^bJ1Y zd*;mL3tJuqF*Gf@GU?JH8`iH^x{lmwniEp0#}EKF@#|6@-~Vw}E~^1e(gI=)go(OF zhI)oZdMTO3CAyh;Y5Dr8c_l@a@df#rc_qbqB^4#ze&^0>pF8i_tM8|GN=HMp@2S^X zk2AU_JVQ5xHf&l`By9Y7#||{R?xt`CaRKe%0Af`#eJG?#%hkK?YZh9~AkY_15*$IjO%X$iwTT zj$Wre`^vxz1{aLYt{7i=!gcDr{>864*LXE_z0RKW*%YLqspb2W%hP9jkj4s=YiCcN z_rB_TX7!Ut=+0vKml077bk1%dR>0#dU)K;v7sn9C*zS#7hYUnqzke6~*(h?!FxuR) z*JA$j=D}6nJuk63>$g|^aHQ)0StmYywxSwA=est9~!zT(kqV|fK2BE1^mxK0q zD(qh)b?4sPb$w-l5$bn#F3Qs`KJ!0*^MQ6R{<p(1r$KgS)&i+9zwP$x1H90UiT)ZdwKjq+uV@j;~jI4*9 zFSbRdFl*>>tZs;(uk_eQ>hgU{@!k)&u4PPICc4gJ)}^;Bzl1h5{m``K-uhy9=9lTe z-NnTxRmGRCybveUZnWu;{lQCXJ}s6};eD{ z#=6?Q%UABG&=n2Q`J~J@GtS}8)q{&~{}TM{1aZ}u05ukm*h zl5ERPr=_}oV=S9Bfwe1=@2=aOxF-kSE;PEja34gdfE literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/test_metadata.py b/Lib/test/test_importlib/test_metadata.py new file mode 100644 index 00000000000000..d5f8696707d920 --- /dev/null +++ b/Lib/test/test_importlib/test_metadata.py @@ -0,0 +1,91 @@ +import re +import sys +import unittest +import importlib + +from contextlib import ExitStack +from importlib import metadata +from importlib.resources import path +from test.support import DirsOnSysPath, unload +from types import ModuleType + + +class BespokeLoader: + archive = 'bespoke' + + +class TestMetadata(unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def setUp(self): + # Find the path to the example.*.whl so we can add it to the front of + # sys.path, where we'll then try to find the metadata thereof. + self.resources = ExitStack() + self.addCleanup(self.resources.close) + wheel = self.resources.enter_context( + path('test.test_importlib.data00', + 'example-21.12-py3-none-any.whl')) + self.resources.enter_context(DirsOnSysPath(str(wheel))) + self.resources.callback(unload, 'example') + + def test_retrieves_version(self): + self.assertEqual(metadata.version('example'), '21.12') + + def test_retrieves_version_from_distribution(self): + example = importlib.import_module('example') + dist = metadata.Distribution.from_module(example) + assert isinstance(dist.version, str) + assert re.match(self.version_pattern, dist.version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(metadata.PackageNotFoundError): + metadata.distribution('does-not-exist') + + def test_for_name_does_not_exist_from_distribution(self): + with self.assertRaises(metadata.PackageNotFoundError): + metadata.Distribution.from_name('does-not-exist') + + def test_entry_points(self): + parser = metadata.entry_points('example') + entry_point = parser.get('console_scripts', 'example') + self.assertEqual(entry_point, 'example:main') + + def test_not_a_zip(self): + # For coverage purposes, this module is importable, but has neither a + # location on the file system, nor a .archive attribute. + sys.modules['bespoke'] = ModuleType('bespoke') + self.resources.callback(unload, 'bespoke') + self.assertRaises(ImportError, metadata.version, 'bespoke') + + def test_unversioned_dist_info(self): + # For coverage purposes, give the module an unversioned .archive + # attribute. + bespoke = sys.modules['bespoke'] = ModuleType('bespoke') + bespoke.__loader__ = BespokeLoader() + self.resources.callback(unload, 'bespoke') + self.assertRaises(ImportError, metadata.version, 'bespoke') + + def test_missing_metadata(self): + distribution = metadata.distribution('example') + self.assertIsNone(distribution.load_metadata('does not exist')) + + def test_for_module_by_name(self): + name = 'example' + distribution = metadata.distribution(name) + self.assertEqual( + distribution.load_metadata('top_level.txt').strip(), + 'example') + + def test_for_module_distribution_by_name(self): + name = 'example' + metadata.Distribution.from_named_module(name) + + def test_resolve(self): + entry_points = metadata.entry_points('example') + main = metadata.resolve( + entry_points.get('console_scripts', 'example')) + import example + self.assertEqual(main, example.main) + + def test_resolve_invalid(self): + self.assertRaises(ValueError, metadata.resolve, 'bogus.ep') From 220ce6fbe40203c98a81de71a82e2c01b1b11b88 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 14 Sep 2018 16:39:58 -0700 Subject: [PATCH 2/4] Documentation for importlib.metadata --- Doc/library/importlib.rst | 90 ++++++++++++++++++++++++++++++ Lib/importlib/metadata.py | 112 +++++++++++++++++++------------------- 2 files changed, 147 insertions(+), 55 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 6f4da11989551e..064c41425e6500 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -792,6 +792,96 @@ ABC hierarchy:: itself does not end in ``__init__``. +:mod:`importlib.metadata` -- Metadata +------------------------------------- + +.. module:: importlib.metadata + :synopsis: Installed package metadata reading + +**Source code:** :source:`Lib/importlib/metadata.py` + +-------------- + +.. versionadded:: 3.8 + +This module leverages Python's import system to provide access to installed +package metadata, such as the package's version, entry points, and more. + +.. note:: + + This module provides functionality similar to `pkg_resources + `_ `entry + point API + `_ + and `metadata API + `_. + + The standalone backport of this module provides more information + on `using importlib.metadata + `_. + +By *installed package* we generally mean a third party package installed into +Python's ``site-packages`` directory via tools such as ``pip``. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by :PEP:`566` or its older specifications. By +default, package metadata can live on the file system or in wheels on +``sys.path``. Through an extension mechanism, the metadata can live almost +anywhere. + +Support is provided by special finders on :attr:`sys.meta_path` (see below). +By default, packages installed on the file system and in :PEP:`491` wheel +files are supported. + +The following exception is defined. + +.. exception:: PackageNotFoundError + + Raised when the named package's metadata could not be found. + +The following functions are available. + +.. function:: entry_points(package): + + Returns the `entry points + `_ for the + given package, as a :class:`configparser.ConfigParser` instance. The + sections in this ``ConfigParser`` exactly match the sections in the + distribution package's `entry_points.txt + ` + file. For example, you can get the console script entry points by + extracting the the ``console_scripts`` section. + +.. function:: resolve(entry_point): + + This is a convenience function for turning an entry point specification + (e.g. ``foo.bar:main``) into the ``foo.bar.main`` object. Raises + :exc:`ValueError` if *entry_point* is not of the proper format. + +.. function:: version(package): + + Returns the distribution package's version as a string. This is a + convenience function for returning the ``Version`` metadata. + +In order to extend the types of distribution packages for which metadata can +be returned, you can implement a special finder that implements a method +called ``find_distribution()``. This function takes a single string argument +which names the distribution package to find, and it returns an instance of as +described below. ``find_distribution()`` can use any algorithm it wants to +find the distribution package's ``dist-info`` or ``egg-info`` directory. + +You can add this method to an existing finder, or you can create a simple +finder that onky finds distribution package metadata. In the latter case, you +must add a ``@staticmethod`` called ``find_spec()`` that accepts any arguments +and returns ``None``. Your finder must be added to :attr:`sys.meta_path`. + +``find_distribution()`` should return an instance of a class derived from the +abstract base class ``importlib.metadata.Distribution``. The only +additional method you need to implement on your subclass is +``load_metadata()``, which takes the name of the distribution package and +returns the contents of the package's ``dist-info/METADATA`` file as a +``str``, or ``None`` if the file does not exist. + + :mod:`importlib.resources` -- Resources --------------------------------------- diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py index af1d42125d0025..6e1117a9392a15 100644 --- a/Lib/importlib/metadata.py +++ b/Lib/importlib/metadata.py @@ -8,8 +8,10 @@ from configparser import ConfigParser from importlib import import_module +from importlib.resources import Package from pathlib import Path from types import ModuleType +from typing import Callable from zipfile import ZipFile @@ -24,61 +26,7 @@ class PackageNotFoundError(ModuleNotFoundError): - """The package was not found.""" - - -def distribution(package): - """Get the ``Distribution`` instance for the given package. - - :param package: The module object for the package or the name of the - package as a string. - :return: A ``Distribution`` instance (or subclass thereof). - """ - if isinstance(package, ModuleType): - return Distribution.from_module(package) - else: - return Distribution.from_name(package) - - -def entry_points(name): - """Return the entry points for the named distribution package. - - :param name: The name of the distribution package to query. - :return: A ConfigParser instance where the sections and keys are taken - from the entry_points.txt ini-style contents. - """ - as_string = distribution(name).load_metadata('entry_points.txt') - # 2018-09-10(barry): Should we provide any options here, or let the caller - # send options to the underlying ConfigParser? For now, YAGNI. - config = ConfigParser() - config.read_string(as_string) - return config - - -def resolve(entry_point): - """Resolve an entry point string into the named callable. - - :param entry_point: An entry point string of the form - `path.to.module:callable`. - :return: The actual callable object `path.to.module.callable` - :raises ValueError: When `entry_point` doesn't have the proper format. - """ - path, colon, name = entry_point.rpartition(':') - if colon != ':': - raise ValueError('Not an entry point: {}'.format(entry_point)) - module = import_module(path) - return getattr(module, name) - - -def version(package): - """Get the version string for the named package. - - :param package: The module object for the package or the name of the - package as a string. - :return: The version string for the package as defined in the package's - "Version" metadata key. - """ - return distribution(package).version + """The package's metadata was not found.""" class Distribution: @@ -149,6 +97,60 @@ def version(self): return self.metadata['Version'] +def distribution(package: Package) -> Distribution: + """Get the ``Distribution`` instance for the given package. + + :param package: The module object for the package or the name of the + package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + if isinstance(package, ModuleType): + return Distribution.from_module(package) + else: + return Distribution.from_name(package) + + +def entry_points(package: Package) -> ConfigParser: + """Return the entry points for the named distribution package. + + :param name: The name of the distribution package to query. + :return: A ConfigParser instance where the sections and keys are taken + from the entry_points.txt ini-style contents. + """ + as_string = distribution(package).load_metadata('entry_points.txt') + # 2018-09-10(barry): Should we provide any options here, or let the caller + # send options to the underlying ConfigParser? For now, YAGNI. + config = ConfigParser() + config.read_string(as_string) + return config + + +def resolve(entry_point: str) -> Callable: + """Resolve an entry point string into the named callable. + + :param entry_point: An entry point string of the form + `path.to.module:callable`. + :return: The actual callable object `path.to.module.callable` + :raises ValueError: When `entry_point` doesn't have the proper format. + """ + path, colon, name = entry_point.rpartition(':') + if colon != ':': + raise ValueError('Not an entry point: {}'.format(entry_point)) + module = import_module(path) + return getattr(module, name) + + +def version(package: Package) -> str: + """Get the version string for the named package. + + :param package: The module object for the package or the name of the + package as a string. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(package).version + + class MetadataPathFinder: """A degenerate finder for distribution packages on the file system. From fe6025b7404f957c155671bde5e3b18700d70aa1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 14 Sep 2018 16:42:03 -0700 Subject: [PATCH 3/4] Add a blurb --- .../NEWS.d/next/Library/2018-09-14-16-41-57.bpo-34632.7DX4Ks.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2018-09-14-16-41-57.bpo-34632.7DX4Ks.rst diff --git a/Misc/NEWS.d/next/Library/2018-09-14-16-41-57.bpo-34632.7DX4Ks.rst b/Misc/NEWS.d/next/Library/2018-09-14-16-41-57.bpo-34632.7DX4Ks.rst new file mode 100644 index 00000000000000..207890e16566b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-09-14-16-41-57.bpo-34632.7DX4Ks.rst @@ -0,0 +1 @@ +Add the :mod:`importlib.metadata` module. From 8a28679eb9c8fdf1d650806200977b3f97e005be Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 15 Sep 2018 18:14:13 -0700 Subject: [PATCH 4/4] Does this fix the doc issue? --- Doc/tools/susp-ignored.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 33cd48f3d81ef4..8a7e4a03eee05b 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -135,6 +135,7 @@ library/http.client,,:port,host:port library/http.cookies,,`,!#$%&'*+-.^_`|~: library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS" library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS" +library/importlib,,:main,(e.g. ``foo.bar:main``) into the ``foo.bar.main`` object. Raises library/inspect,,:int,">>> def foo(a, *, b:int, **kwargs):" library/inspect,,:int,"'(a, *, b:int, **kwargs)'" library/inspect,,:int,'b:int'