From bde4e157b4ac1924a953897c92ed0b2638cdb229 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 15 Jan 2021 12:34:13 -0500 Subject: [PATCH 01/77] Fix typo in docs. Ref bpo-42728. --- docs/using.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index efa40f86..00409867 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -200,9 +200,9 @@ Thus, an alternative way to get the version number is through the There are all kinds of additional metadata available on the ``Distribution`` instance:: - >>> d.metadata['Requires-Python'] + >>> dist.metadata['Requires-Python'] '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> d.metadata['License'] + >>> dist.metadata['License'] 'MIT' The full set of available metadata is not described here. See :pep:`566` From 0df40810ec54590c888ae0e4073d73f731c91f4a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 15 Jan 2021 19:16:28 -0500 Subject: [PATCH 02/77] Add support for namespace packages. Closes jaraco/skeleton#40. --- setup.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 88bc263a..106763e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,12 +15,18 @@ classifiers = Programming Language :: Python :: 3 :: Only [options] -packages = find: +packages = find_namespace: include_package_data = true python_requires = >=3.6 install_requires = setup_requires = setuptools_scm[toml] >= 3.4.1 +[options.packages.find] +exclude = + build* + docs* + tests* + [options.extras_require] testing = # upstream From 9950845a8b3f35843057d3708ed75ef30dd62659 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 21 Jan 2021 09:34:23 -0500 Subject: [PATCH 03/77] Add doctest illustrating the usage of constructing dict from EntryPoints. --- importlib_metadata/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 079cc0f9..fac3063b 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -138,7 +138,11 @@ def _for(self, dist): def __iter__(self): """ - Supply iter so one may construct dicts of EntryPoints easily. + Supply iter so one may construct dicts of EntryPoints by name. + + >>> eps = [EntryPoint('a', 'b', 'c'), EntryPoint('d', 'e', 'f')] + >>> dict(eps)['a'] + EntryPoint(name='a', value='b', group='c') """ return iter((self.name, self)) From a49252b513cf1e25e3885c60f046269ea293f13e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:21:37 -0500 Subject: [PATCH 04/77] Add explicit interfaces for loaded entrypoints, resolvable first by group then by name. --- importlib_metadata/__init__.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fac3063b..7ca3e938 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -153,6 +153,27 @@ def __reduce__(self): ) +class EntryPoints(tuple): + """ + A collection of EntryPoint objects, retrievable by name. + """ + + def __getitem__(self, name) -> EntryPoint: + try: + return next(ep for ep in self if ep.name == name) + except Exception: + raise KeyError(name) + + +class GroupedEntryPoints(tuple): + """ + A collection of EntryPoint objects, retrievable by group. + """ + + def __getitem__(self, group) -> EntryPoints: + return EntryPoints(ep for ep in self if ep.group == group) + + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" @@ -308,7 +329,8 @@ def version(self): @property def entry_points(self): - return list(EntryPoint._from_text_for(self.read_text('entry_points.txt'), self)) + eps = EntryPoint._from_text_for(self.read_text('entry_points.txt'), self) + return GroupedEntryPoints(eps) @property def files(self): @@ -647,10 +669,7 @@ def entry_points(): :return: EntryPoint objects for all installed packages. """ eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions()) - by_group = operator.attrgetter('group') - ordered = sorted(eps, key=by_group) - grouped = itertools.groupby(ordered, by_group) - return {group: tuple(eps) for group, eps in grouped} + return GroupedEntryPoints(eps) def files(distribution_name): From 6596183f79a3973698c4b2b825b12682ac6e7d96 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:29:26 -0500 Subject: [PATCH 05/77] Update tests to use new preferred API. --- tests/test_api.py | 12 +++++++++--- tests/test_main.py | 6 ++---- tests/test_zip.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index a386551f..e6a5adeb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -64,18 +64,24 @@ def test_read_text(self): self.assertEqual(top_level.read_text(), 'mod\n') def test_entry_points(self): - entries = dict(entry_points()['entries']) - ep = entries['main'] + ep = entry_points()['entries']['main'] self.assertEqual(ep.value, 'mod:main') self.assertEqual(ep.extras, []) def test_entry_points_distribution(self): - entries = dict(entry_points()['entries']) + entries = entry_points()['entries'] for entry in ("main", "ns:sub"): ep = entries[entry] self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) self.assertEqual(ep.dist.version, "1.0.0") + def test_entry_points_missing_name(self): + with self.assertRaises(KeyError): + entry_points()['entries']['missing'] + + def test_entry_points_missing_group(self): + assert entry_points()['missing'] == () + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' diff --git a/tests/test_main.py b/tests/test_main.py index 74979be8..566262f6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -57,13 +57,11 @@ def test_import_nonexistent_module(self): importlib.import_module('does_not_exist') def test_resolve(self): - entries = dict(entry_points()['entries']) - ep = entries['main'] + ep = entry_points()['entries']['main'] self.assertEqual(ep.load().__name__, "main") def test_entrypoint_with_colon_in_name(self): - entries = dict(entry_points()['entries']) - ep = entries['ns:sub'] + ep = entry_points()['entries']['ns:sub'] self.assertEqual(ep.value, 'mod:main') def test_resolve_without_attr(self): diff --git a/tests/test_zip.py b/tests/test_zip.py index 67311da2..5a63465f 100644 --- a/tests/test_zip.py +++ b/tests/test_zip.py @@ -45,7 +45,7 @@ def test_zip_version_does_not_match(self): version('definitely-not-installed') def test_zip_entry_points(self): - scripts = dict(entry_points()['console_scripts']) + scripts = entry_points()['console_scripts'] entry_point = scripts['example'] self.assertEqual(entry_point.value, 'example:main') entry_point = scripts['Example'] From b5081fa78358a9bb7c47eedfe084b3f96c024c63 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:33:37 -0500 Subject: [PATCH 06/77] Capture the legacy expectation. --- tests/test_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index e6a5adeb..c3e8b532 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -82,6 +82,16 @@ def test_entry_points_missing_name(self): def test_entry_points_missing_group(self): assert entry_points()['missing'] == () + def test_entry_points_dict_construction(self): + """ + Prior versions of entry_points() returned simple lists and + allowed casting those lists into maps by name using ``dict()``. + Capture this now deprecated use-case. + """ + eps = dict(entry_points()['entries']) + assert 'main' in eps + assert eps['main'] == entry_points()['entries']['main'] + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 99dd2242ab9c8c0b4a082e135f6bbda11c19540a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:50:02 -0500 Subject: [PATCH 07/77] Deprecate dict construction from EntryPoint items. --- importlib_metadata/__init__.py | 10 ++++++---- tests/test_api.py | 10 +++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 7ca3e938..681743dc 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -7,6 +7,7 @@ import email import pathlib import operator +import warnings import functools import itertools import posixpath @@ -139,11 +140,12 @@ def _for(self, dist): def __iter__(self): """ Supply iter so one may construct dicts of EntryPoints by name. - - >>> eps = [EntryPoint('a', 'b', 'c'), EntryPoint('d', 'e', 'f')] - >>> dict(eps)['a'] - EntryPoint(name='a', value='b', group='c') """ + msg = ( + "Construction of dict of EntryPoints is deprecated in " + "favor of EntryPoints." + ) + warnings.warn(msg, DeprecationWarning) return iter((self.name, self)) def __reduce__(self): diff --git a/tests/test_api.py b/tests/test_api.py index c3e8b532..8ce2e468 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ import re import textwrap import unittest +import warnings from . import fixtures from importlib_metadata import ( @@ -88,10 +89,17 @@ def test_entry_points_dict_construction(self): allowed casting those lists into maps by name using ``dict()``. Capture this now deprecated use-case. """ - eps = dict(entry_points()['entries']) + with warnings.catch_warnings(record=True) as caught: + eps = dict(entry_points()['entries']) + assert 'main' in eps assert eps['main'] == entry_points()['entries']['main'] + # check warning + expected = next(iter(caught)) + assert expected.category is DeprecationWarning + assert "Construction of dict of EntryPoints is deprecated" in str(expected) + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 2eeb629021d7218516f5ee43de51b8d93d32828a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:54:11 -0500 Subject: [PATCH 08/77] Suppress warning in test_json_dump. --- tests/test_main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 566262f6..b778572c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,6 +3,7 @@ import pickle import textwrap import unittest +import warnings import importlib import importlib_metadata import pyfakefs.fake_filesystem_unittest as ffs @@ -247,7 +248,8 @@ def test_json_dump(self): json should not expect to be able to dump an EntryPoint """ with self.assertRaises(Exception): - json.dumps(self.ep) + with warnings.catch_warnings(record=True): + json.dumps(self.ep) def test_module(self): assert self.ep.module == 'value' From eedd810b90083fd5a2b0bb398478527011c474eb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:06:29 -0500 Subject: [PATCH 09/77] Add 'groups' and 'names' to EntryPoints collections. --- importlib_metadata/__init__.py | 8 ++++++++ tests/test_api.py | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 681743dc..60967cd4 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -166,6 +166,10 @@ def __getitem__(self, name) -> EntryPoint: except Exception: raise KeyError(name) + @property + def names(self): + return set(ep.name for ep in self) + class GroupedEntryPoints(tuple): """ @@ -175,6 +179,10 @@ class GroupedEntryPoints(tuple): def __getitem__(self, group) -> EntryPoints: return EntryPoints(ep for ep in self if ep.group == group) + @property + def groups(self): + return set(ep.group for ep in self) + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" diff --git a/tests/test_api.py b/tests/test_api.py index 8ce2e468..7672556a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -65,7 +65,11 @@ def test_read_text(self): self.assertEqual(top_level.read_text(), 'mod\n') def test_entry_points(self): - ep = entry_points()['entries']['main'] + eps = entry_points() + assert 'entries' in eps.groups + entries = eps['entries'] + assert 'main' in entries.names + ep = entries['main'] self.assertEqual(ep.value, 'mod:main') self.assertEqual(ep.extras, []) From 720362fe25dd0211432784de02dd483b53ee7be8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:10:34 -0500 Subject: [PATCH 10/77] Update documentation on EntryPoints to reflect the new, preferred accessors. --- docs/using.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 00409867..534c1dea 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,7 +67,7 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a dictionary of all entry points, +The ``entry_points()`` function returns a sequence of all entry points, keyed by group. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, @@ -75,10 +75,12 @@ a ``.load()`` method to resolve the value. There are also ``.module``, ``.value`` attribute:: >>> eps = entry_points() - >>> list(eps) + >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] >>> scripts = eps['console_scripts'] - >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] + >>> 'wheel' in scripts.names + True + >>> wheel = scripts['wheel'] >>> wheel EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') >>> wheel.module From 28adeb8f84ac3e5052ea24c93b4fa3816e1fe4e6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:13:52 -0500 Subject: [PATCH 11/77] Update changelog. --- CHANGES.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 57901f23..02900674 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,16 @@ +v3.5.0 +====== + +* ``entry_points()`` now returns an ``GroupedEntryPoints`` + object, a tuple of all entry points but with a convenience + property ``groups`` and ``__getitem__`` accessor. Further, + accessing a group returns an ``EntryPoints`` object, + another tuple of entry points in the group, accessible by + name. Construction of entry points using + ``dict([EntryPoint, ...])`` is now deprecated and raises + an appropriate DeprecationWarning and will be removed in + a future version. + v3.4.0 ====== From 342a94ba5c373b01f3c5b827da1d4bd76ff2b04f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:54:23 -0500 Subject: [PATCH 12/77] Add deprecated .get to GroupedEntryPoints and test to capture expectation. --- importlib_metadata/__init__.py | 8 ++++++++ tests/test_api.py | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 60967cd4..edcf2691 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -183,6 +183,14 @@ def __getitem__(self, group) -> EntryPoints: def groups(self): return set(ep.group for ep in self) + def get(self, group, default=None): + """ + For backward compatibility, supply .get + """ + msg = "GroupedEntryPoints.get is deprecated. Just use __getitem__." + warnings.warn(msg, DeprecationWarning) + return self[group] or default + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" diff --git a/tests/test_api.py b/tests/test_api.py index 7672556a..dc0c7870 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -104,6 +104,17 @@ def test_entry_points_dict_construction(self): assert expected.category is DeprecationWarning assert "Construction of dict of EntryPoints is deprecated" in str(expected) + def test_entry_points_groups_get(self): + """ + Prior versions of entry_points() returned a dict. Ensure + that callers using '.get()' are supported but warned to + migrate. + """ + with warnings.catch_warnings(record=True): + entry_points().get('missing', 'default') == 'default' + entry_points().get('entries', 'default') == entry_points()['entries'] + entry_points().get('missing', ()) == entry_points()['missing'] + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 4e2603f5bd0b6c86a365e98e3e891fdcdfa9bc13 Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Sat, 23 Jan 2021 23:30:33 +0100 Subject: [PATCH 13/77] Separately profile cached and uncached lookup performance. ... in preparation of adding a lookup cache. --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 11f52d7a..f1586632 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,10 @@ use_develop = False deps = ipython commands = - python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' + python -c 'print("Cached lookup performance")' + python -m timeit -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' -- 'importlib_metadata.distribution("ipython")' + python -c 'print("Uncached lookup performance")' + python -m timeit -s 'import importlib, importlib_metadata' -- 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")' [testenv:release] skip_install = True From 35bc40379340e2b23d56a57c5c782ffe93a53396 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 15:38:00 -0500 Subject: [PATCH 14/77] When resolving entry points globally, only expose entry points for unique distributions. Fixes #280. --- .coveragerc | 1 + importlib_metadata/__init__.py | 7 ++++++- importlib_metadata/_itertools.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 importlib_metadata/_itertools.py diff --git a/.coveragerc b/.coveragerc index 66d32472..e91bbd6b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ omit = */.tox/* tests/* prepare/* + */_itertools.py [report] show_missing = True diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fac3063b..4c420a55 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -19,6 +19,8 @@ Protocol, ) +from ._itertools import unique_everseen + from configparser import ConfigParser from contextlib import suppress from importlib import import_module @@ -646,7 +648,10 @@ def entry_points(): :return: EntryPoint objects for all installed packages. """ - eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions()) + unique = functools.partial(unique_everseen, key=operator.attrgetter('name')) + eps = itertools.chain.from_iterable( + dist.entry_points for dist in unique(distributions()) + ) by_group = operator.attrgetter('group') ordered = sorted(eps, key=by_group) grouped = itertools.groupby(ordered, by_group) diff --git a/importlib_metadata/_itertools.py b/importlib_metadata/_itertools.py new file mode 100644 index 00000000..dd45f2f0 --- /dev/null +++ b/importlib_metadata/_itertools.py @@ -0,0 +1,19 @@ +from itertools import filterfalse + + +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element From 8935288354ba3b843a13cdb5577c3cdb7a672e0b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 18:49:51 -0500 Subject: [PATCH 15/77] Add test capturing failed expectation. Ref #280. --- tests/test_api.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index a386551f..04a5b9d3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -76,6 +76,34 @@ def test_entry_points_distribution(self): self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) self.assertEqual(ep.dist.version, "1.0.0") + def test_entry_points_unique_packages(self): + """ + Entry points should only be exposed for the first package + on sys.path with a given name. + """ + alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "distinfo_pkg-1.1.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Version: 1.1.0 + """, + "entry_points.txt": """ + [entries] + main = mod:altmain + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + entries = dict(entry_points()['entries']) + assert not any( + ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' + for ep in entries.values() + ) + # ns:sub doesn't exist in alt_pkg + assert 'ns:sub' not in entries + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 6cb27d381c0330c65fb07ce5a68d4525dea94600 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 20:28:36 -0500 Subject: [PATCH 16/77] Update changelog --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 57901f23..2bc3a356 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.5.0 +====== + +* #280: ``entry_points`` now only returns entry points for + unique distributions (by name). + v3.4.0 ====== From 2b64fa218dcb902baa7bcc2da42e834eec6bdab8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 20:33:10 -0500 Subject: [PATCH 17/77] Add performance test for entry_points --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 11f52d7a..32796a47 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ deps = ipython commands = python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' + python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.entry_points()' [testenv:release] skip_install = True From 51298a2cc4faa7253e9fe41d7a9574cf9aac997c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 9 Feb 2021 23:08:58 -0500 Subject: [PATCH 18/77] Normalize indentation --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 106763e3..8df8d273 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,9 +23,9 @@ setup_requires = setuptools_scm[toml] >= 3.4.1 [options.packages.find] exclude = - build* - docs* - tests* + build* + docs* + tests* [options.extras_require] testing = From 9448e13a10648ae5a086247dea8a17efff31b816 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 15 Feb 2021 18:05:18 -0500 Subject: [PATCH 19/77] Make entry point collections (more) immutable. --- importlib_metadata/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index edcf2691..4c8188ae 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -157,9 +157,11 @@ def __reduce__(self): class EntryPoints(tuple): """ - A collection of EntryPoint objects, retrievable by name. + An immutable collection of EntryPoint objects, retrievable by name. """ + __slots__ = () + def __getitem__(self, name) -> EntryPoint: try: return next(ep for ep in self if ep.name == name) @@ -173,9 +175,11 @@ def names(self): class GroupedEntryPoints(tuple): """ - A collection of EntryPoint objects, retrievable by group. + An immutable collection of EntryPoint objects, retrievable by group. """ + __slots__ = () + def __getitem__(self, group) -> EntryPoints: return EntryPoints(ep for ep in self if ep.group == group) From 71fd4a7b6a8141becd431edf51dac590493d61c2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 15 Feb 2021 21:28:16 -0500 Subject: [PATCH 20/77] Hide the deprecation warning from flake8 users --- importlib_metadata/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4c8188ae..f9af7824 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,6 +5,7 @@ import sys import zipp import email +import inspect import pathlib import operator import warnings @@ -191,8 +192,9 @@ def get(self, group, default=None): """ For backward compatibility, supply .get """ + is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) msg = "GroupedEntryPoints.get is deprecated. Just use __getitem__." - warnings.warn(msg, DeprecationWarning) + is_flake8 or warnings.warn(msg, DeprecationWarning) return self[group] or default From 8320adef797d5f14d9fff7b58ebc2a31a2a6a437 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 16 Feb 2021 22:01:49 -0500 Subject: [PATCH 21/77] Instead of presenting separate contexts for EntryPoints, unify into a single collection that can select on 'name' or 'group' or possibly other attributes. Expose that selection in the 'entry_points' function. --- docs/using.rst | 2 +- importlib_metadata/__init__.py | 56 +++++++++++++++++++--------------- tests/test_api.py | 26 +++++++++++----- tests/test_main.py | 4 +-- tests/test_zip.py | 2 +- 5 files changed, 54 insertions(+), 36 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 534c1dea..bdfe3e82 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -77,7 +77,7 @@ a ``.load()`` method to resolve the value. There are also ``.module``, >>> eps = entry_points() >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] - >>> scripts = eps['console_scripts'] + >>> scripts = eps.select(group='console_scripts') >>> 'wheel' in scripts.names True >>> wheel = scripts['wheel'] diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index f9af7824..77057703 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -130,10 +130,6 @@ def _from_text(cls, text): config.read_string(text) return cls._from_config(config) - @classmethod - def _from_text_for(cls, text, dist): - return (ep._for(dist) for ep in cls._from_text(text)) - def _for(self, dist): self.dist = dist return self @@ -155,35 +151,42 @@ def __reduce__(self): (self.name, self.value, self.group), ) + def matches(self, **params): + attrs = (getattr(self, param) for param in params) + return all(map(operator.eq, params.values(), attrs)) + class EntryPoints(tuple): """ - An immutable collection of EntryPoint objects, retrievable by name. + An immutable collection of selectable EntryPoint objects. """ __slots__ = () - def __getitem__(self, name) -> EntryPoint: + def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: try: - return next(ep for ep in self if ep.name == name) - except Exception: + match = next(iter(self.select(name=name))) + return match + except StopIteration: + if name in self.groups: + return self._group_getitem(name) raise KeyError(name) + def _group_getitem(self, name): + """ + For backward compatability, supply .__getitem__ for groups. + """ + msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." + warnings.warn(msg, DeprecationWarning) + return self.select(group=name) + + def select(self, **params): + return EntryPoints(ep for ep in self if ep.matches(**params)) + @property def names(self): return set(ep.name for ep in self) - -class GroupedEntryPoints(tuple): - """ - An immutable collection of EntryPoint objects, retrievable by group. - """ - - __slots__ = () - - def __getitem__(self, group) -> EntryPoints: - return EntryPoints(ep for ep in self if ep.group == group) - @property def groups(self): return set(ep.group for ep in self) @@ -193,9 +196,13 @@ def get(self, group, default=None): For backward compatibility, supply .get """ is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - msg = "GroupedEntryPoints.get is deprecated. Just use __getitem__." + msg = "GroupedEntryPoints.get is deprecated. Use select." is_flake8 or warnings.warn(msg, DeprecationWarning) - return self[group] or default + return self.select(group=group) or default + + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) class PackagePath(pathlib.PurePosixPath): @@ -353,8 +360,7 @@ def version(self): @property def entry_points(self): - eps = EntryPoint._from_text_for(self.read_text('entry_points.txt'), self) - return GroupedEntryPoints(eps) + return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property def files(self): @@ -687,13 +693,13 @@ def version(distribution_name): return distribution(distribution_name).version -def entry_points(): +def entry_points(**params): """Return EntryPoint objects for all installed packages. :return: EntryPoint objects for all installed packages. """ eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions()) - return GroupedEntryPoints(eps) + return EntryPoints(eps).select(**params) def files(distribution_name): diff --git a/tests/test_api.py b/tests/test_api.py index dc0c7870..a6466309 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -67,14 +67,14 @@ def test_read_text(self): def test_entry_points(self): eps = entry_points() assert 'entries' in eps.groups - entries = eps['entries'] + entries = eps.select(group='entries') assert 'main' in entries.names ep = entries['main'] self.assertEqual(ep.value, 'mod:main') self.assertEqual(ep.extras, []) def test_entry_points_distribution(self): - entries = entry_points()['entries'] + entries = entry_points(group='entries') for entry in ("main", "ns:sub"): ep = entries[entry] self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) @@ -82,10 +82,10 @@ def test_entry_points_distribution(self): def test_entry_points_missing_name(self): with self.assertRaises(KeyError): - entry_points()['entries']['missing'] + entry_points(group='entries')['missing'] def test_entry_points_missing_group(self): - assert entry_points()['missing'] == () + assert entry_points(group='missing') == () def test_entry_points_dict_construction(self): """ @@ -94,16 +94,28 @@ def test_entry_points_dict_construction(self): Capture this now deprecated use-case. """ with warnings.catch_warnings(record=True) as caught: - eps = dict(entry_points()['entries']) + eps = dict(entry_points(group='entries')) assert 'main' in eps - assert eps['main'] == entry_points()['entries']['main'] + assert eps['main'] == entry_points(group='entries')['main'] # check warning expected = next(iter(caught)) assert expected.category is DeprecationWarning assert "Construction of dict of EntryPoints is deprecated" in str(expected) + def test_entry_points_groups_getitem(self): + """ + Prior versions of entry_points() returned a dict. Ensure + that callers using '.__getitem__()' are supported but warned to + migrate. + """ + with warnings.catch_warnings(record=True): + entry_points()['entries'] == entry_points(group='entries') + + with self.assertRaises(KeyError): + entry_points()['missing'] + def test_entry_points_groups_get(self): """ Prior versions of entry_points() returned a dict. Ensure @@ -113,7 +125,7 @@ def test_entry_points_groups_get(self): with warnings.catch_warnings(record=True): entry_points().get('missing', 'default') == 'default' entry_points().get('entries', 'default') == entry_points()['entries'] - entry_points().get('missing', ()) == entry_points()['missing'] + entry_points().get('missing', ()) == () def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') diff --git a/tests/test_main.py b/tests/test_main.py index b778572c..e8a66c0d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -58,11 +58,11 @@ def test_import_nonexistent_module(self): importlib.import_module('does_not_exist') def test_resolve(self): - ep = entry_points()['entries']['main'] + ep = entry_points(group='entries')['main'] self.assertEqual(ep.load().__name__, "main") def test_entrypoint_with_colon_in_name(self): - ep = entry_points()['entries']['ns:sub'] + ep = entry_points(group='entries')['ns:sub'] self.assertEqual(ep.value, 'mod:main') def test_resolve_without_attr(self): diff --git a/tests/test_zip.py b/tests/test_zip.py index 5a63465f..4279046d 100644 --- a/tests/test_zip.py +++ b/tests/test_zip.py @@ -45,7 +45,7 @@ def test_zip_version_does_not_match(self): version('definitely-not-installed') def test_zip_entry_points(self): - scripts = entry_points()['console_scripts'] + scripts = entry_points(group='console_scripts') entry_point = scripts['example'] self.assertEqual(entry_point.value, 'example:main') entry_point = scripts['Example'] From 61a265c18bc481c1e49fded0476a04ba5f75b750 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 12:32:34 -0500 Subject: [PATCH 22/77] Fix perf tests --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a7476d12..5466077e 100644 --- a/tox.ini +++ b/tox.ini @@ -38,9 +38,9 @@ use_develop = False deps = ipython commands = - python -m 'print("Simple discovery performance")' + python -c 'print("Simple discovery performance")' python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' - python -m 'print("Entry point discovery performance")' + python -c 'print("Entry point discovery performance")' python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.entry_points()' python -c 'print("Cached lookup performance")' python -m timeit -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' -- 'importlib_metadata.distribution("ipython")' From 743af7249d56e55a7c2c5f3111958ceee008d8ea Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 13:04:46 -0500 Subject: [PATCH 23/77] Exclude dist from discovered packages. Fixes jaraco/skeleton#46. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 8df8d273..af246415 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ setup_requires = setuptools_scm[toml] >= 3.4.1 [options.packages.find] exclude = build* + dist* docs* tests* From 0da7579828f00c267915303481567174a4ae00be Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Sun, 10 Jan 2021 23:20:15 +0100 Subject: [PATCH 24/77] More speedup via mtime-base caching. Caching based on mtime is similar to the one done on importlib's FileFinder. Locally, on a large-ish environment, this speeds up repeated calls to `distribution("pip")` ~10x. --- importlib_metadata/__init__.py | 90 +++++++++++++++++++--------------- tests/test_api.py | 7 +++ 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4c420a55..fb91957c 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -464,9 +464,15 @@ class FastPath: children. """ - def __init__(self, root): + @functools.lru_cache() # type: ignore + def __new__(cls, root): + self = object().__new__(cls) self.root = str(root) self.base = os.path.basename(self.root).lower() + self.last_mtime = -1 + self.infos = {} + self.eggs = {} + return self def joinpath(self, child): return pathlib.Path(self.root, child) @@ -482,15 +488,47 @@ def zip_children(self): zip_path = zipp.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath - return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - def search(self, name): - return ( - self.joinpath(child) - for child in self.children() - if name.matches(child, self.base) - ) + def update_cache(self): + root = self.root or "." + try: + mtime = os.stat(root).st_mtime + except OSError: + self.infos.clear() + self.eggs.clear() + self.last_mtime = -1 + return + if mtime == self.last_mtime: + return + self.infos.clear() + self.eggs.clear() + base_is_egg = self.base.endswith(".egg") + for child in self.children(): + low = child.lower() + if low.endswith((".dist-info", ".egg-info")): + # rpartition is faster than splitext and suitable for this purpose. + name = low.rpartition(".")[0].partition("-")[0] + normalized = Prepared.normalize(name) + self.infos.setdefault(normalized, []).append(child) + elif base_is_egg and low == "egg-info": + name = self.base.rpartition(".")[0].partition("-")[0] + legacy_normalized = Prepared.legacy_normalize(name) + self.eggs.setdefault(legacy_normalized, []).append(child) + self.last_mtime = mtime + + def search(self, prepared): + self.update_cache() + if prepared.name: + infos = self.infos.get(prepared.normalized, []) + yield from map(self.joinpath, infos) + eggs = self.eggs.get(prepared.legacy_normalized, []) + yield from map(self.joinpath, eggs) + else: + for infos in self.infos.values(): + yield from map(self.joinpath, infos) + for eggs in self.eggs.values(): + yield from map(self.joinpath, eggs) class Prepared: @@ -499,22 +537,14 @@ class Prepared: """ normalized = None - suffixes = 'dist-info', 'egg-info' - exact_matches = [''][:0] - egg_prefix = '' - versionless_egg_name = '' + legacy_normalized = None def __init__(self, name): self.name = name if name is None: return self.normalized = self.normalize(name) - self.exact_matches = [ - self.normalized + '.' + suffix for suffix in self.suffixes - ] - legacy_normalized = self.legacy_normalize(self.name) - self.egg_prefix = legacy_normalized + '-' - self.versionless_egg_name = legacy_normalized + '.egg' + self.legacy_normalized = self.legacy_normalize(name) @staticmethod def normalize(name): @@ -531,27 +561,6 @@ def legacy_normalize(name): """ return name.lower().replace('-', '_') - def matches(self, cand, base): - low = cand.lower() - # rpartition is faster than splitext and suitable for this purpose. - pre, _, ext = low.rpartition('.') - name, _, rest = pre.partition('-') - return ( - low in self.exact_matches - or ext in self.suffixes - and (not self.normalized or name.replace('.', '_') == self.normalized) - # legacy case: - or self.is_egg(base) - and low == 'egg-info' - ) - - def is_egg(self, base): - return ( - base == self.versionless_egg_name - or base.startswith(self.egg_prefix) - and base.endswith('.egg') - ) - @install class MetadataPathFinder(NullFinder, DistributionFinder): @@ -581,6 +590,9 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + def invalidate_caches(cls): + FastPath.__new__.cache_clear() + class PathDistribution(Distribution): def __init__(self, path): diff --git a/tests/test_api.py b/tests/test_api.py index 04a5b9d3..134acf60 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ import re import textwrap import unittest +import importlib from . import fixtures from importlib_metadata import ( @@ -224,3 +225,9 @@ def test_distribution_at_str(self): dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' dist = Distribution.at(str(dist_info_path)) assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() From 38fff62edb5e282f144dc77cc1bf5555367336d9 Mon Sep 17 00:00:00 2001 From: KOLANICH Date: Sat, 6 Feb 2021 23:03:13 +0300 Subject: [PATCH 25/77] Added an .editorconfig. Pull request jaraco/skeleton#43. --- .editorconfig | 15 +++++++++++++++ pytest.ini | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..6385b573 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.py] +indent_style = space + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/pytest.ini b/pytest.ini index d7f0b115..016063b5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,5 @@ doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 filterwarnings= - # https://github.com/pytest-dev/pytest/issues/6928 - ignore:direct construction of .*Item has been deprecated:DeprecationWarning + # https://github.com/pytest-dev/pytest/issues/6928 + ignore:direct construction of .*Item has been deprecated:DeprecationWarning From 5e416793c008c5ef285c37828072fbea5ced6d08 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 21:34:35 -0500 Subject: [PATCH 26/77] It's no longer necessary to filter this warning and it's not a warning anymore. --- pytest.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 016063b5..6bf69af1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,3 @@ doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 filterwarnings= - # https://github.com/pytest-dev/pytest/issues/6928 - ignore:direct construction of .*Item has been deprecated:DeprecationWarning From d9a13c77ce2a3efea70c97d219ca4335c0f03c40 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 21:36:53 -0500 Subject: [PATCH 27/77] Bump minimum pytest --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index af246415..81f70eea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ exclude = [options.extras_require] testing = # upstream - pytest >= 3.5, !=3.7.3 + pytest >= 4.6 pytest-checkdocs >= 1.2.3 pytest-flake8 pytest-black >= 0.3.7; python_implementation != "PyPy" From e3d1b935b3a2185461aadca34192b93bfdeaa9ca Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 12:27:41 -0500 Subject: [PATCH 28/77] Update changelog. --- CHANGES.rst | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 368723d4..2f5b1ec5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,18 +1,34 @@ -v3.5.0 +v4.0.0 ====== -* #280: ``entry_points`` now only returns entry points for - unique distributions (by name). -* ``entry_points()`` now returns an ``GroupedEntryPoints`` - object, a tuple of all entry points but with a convenience - property ``groups`` and ``__getitem__`` accessor. Further, - accessing a group returns an ``EntryPoints`` object, - another tuple of entry points in the group, accessible by - name. Construction of entry points using +* #284: Introduces new ``EntryPoints`` object, a tuple of + ``EntryPoint`` objects but with convenience properties for + selecting and inspecting the results: + + - ``.select()`` accepts ``group`` or ``name`` keyword + parameters and returns a new ``EntryPoints`` tuple + with only those that match the selection. + - ``.groups`` property presents all of the group names. + - ``.names`` property presents the names of the entry points. + - Item access (e.g. ``eps[name]``) retrieves a single + entry point by name. + + ``entry_points()`` now returns an ``EntryPoints`` + object, but provides for backward compatibility with + a ``__getitem__`` accessor by group and a ``get()`` + method. + + Construction of entry points using ``dict([EntryPoint, ...])`` is now deprecated and raises an appropriate DeprecationWarning and will be removed in a future version. +v3.5.0 +====== + +* #280: ``entry_points`` now only returns entry points for + unique distributions (by name). + v3.4.0 ====== From 9d55a331c7d77025054e85f23bc23c614fab6856 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 22 Feb 2021 08:47:11 -0500 Subject: [PATCH 29/77] Separate compatibility shim from canonical EntryPoints container. --- importlib_metadata/__init__.py | 41 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 313beca2..94a82ffe 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -165,23 +165,12 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: + def __getitem__(self, name): # -> EntryPoint: try: - match = next(iter(self.select(name=name))) - return match + return next(iter(self.select(name=name))) except StopIteration: - if name in self.groups: - return self._group_getitem(name) raise KeyError(name) - def _group_getitem(self, name): - """ - For backward compatability, supply .__getitem__ for groups. - """ - msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." - warnings.warn(msg, DeprecationWarning) - return self.select(group=name) - def select(self, **params): return EntryPoints(ep for ep in self if ep.matches(**params)) @@ -193,6 +182,23 @@ def names(self): def groups(self): return set(ep.group for ep in self) + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) + + +class LegacyGroupedEntryPoints(EntryPoints): + def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: + try: + return super().__getitem__(name) + except KeyError: + if name not in self.groups: + raise + + msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." + warnings.warn(msg, DeprecationWarning) + return self.select(group=name) + def get(self, group, default=None): """ For backward compatibility, supply .get @@ -202,9 +208,10 @@ def get(self, group, default=None): is_flake8 or warnings.warn(msg, DeprecationWarning) return self.select(group=group) or default - @classmethod - def _from_text_for(cls, text, dist): - return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) + def select(self, **params): + if not params: + return self + return super().select(**params) class PackagePath(pathlib.PurePosixPath): @@ -704,7 +711,7 @@ def entry_points(**params): eps = itertools.chain.from_iterable( dist.entry_points for dist in unique(distributions()) ) - return EntryPoints(eps).select(**params) + return LegacyGroupedEntryPoints(eps).select(**params) def files(distribution_name): From d6f7c201b15c79bce7c4e27784a2bd61bdc43555 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 22 Feb 2021 20:49:20 -0500 Subject: [PATCH 30/77] Add docstrings to the compatibility shim. Give primacy to group lookup in compatibility shim. --- importlib_metadata/__init__.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 94a82ffe..bd7d4d7e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -188,27 +188,38 @@ def _from_text_for(cls, text, dist): class LegacyGroupedEntryPoints(EntryPoints): + """ + Compatibility wrapper around EntryPoints to provide + much of the 'dict' interface previously returned by + entry_points. + """ + def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: - try: - return super().__getitem__(name) - except KeyError: - if name not in self.groups: - raise + """ + When accessed by name that matches a group, return the group. + """ + group = self.select(group=name) + if group: + msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." + warnings.warn(msg, DeprecationWarning, stacklevel=2) + return group - msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." - warnings.warn(msg, DeprecationWarning) - return self.select(group=name) + return super().__getitem__(name) def get(self, group, default=None): """ - For backward compatibility, supply .get + For backward compatibility, supply .get. """ is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) msg = "GroupedEntryPoints.get is deprecated. Use select." - is_flake8 or warnings.warn(msg, DeprecationWarning) + is_flake8 or warnings.warn(msg, DeprecationWarning, stacklevel=2) return self.select(group=group) or default def select(self, **params): + """ + Prevent transform to EntryPoints during call to entry_points if + no selection parameters were passed. + """ if not params: return self return super().select(**params) From 4e288685ed6edee15b43629ec26897c4aeeab21f Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Tue, 23 Feb 2021 10:48:20 +0100 Subject: [PATCH 31/77] Use a hand-written parser for entry points. This speeds up the `entry_points()` tox perf check by ~30%, while being both shorter and easier to follow. --- importlib_metadata/__init__.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4c420a55..ae1d9e37 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -21,7 +21,6 @@ from ._itertools import unique_everseen -from configparser import ConfigParser from contextlib import suppress from importlib import import_module from importlib.abc import MetaPathFinder @@ -114,21 +113,18 @@ def extras(self): match = self.pattern.match(self.value) return list(re.finditer(r'\w+', match.group('extras') or '')) - @classmethod - def _from_config(cls, config): - return ( - cls(name, value, group) - for group in config.sections() - for name, value in config.items(group) - ) - @classmethod def _from_text(cls, text): - config = ConfigParser(delimiters='=') - # case sensitive: https://stackoverflow.com/q/1611799/812183 - config.optionxform = str - config.read_string(text) - return cls._from_config(config) + # A hand-rolled parser is much faster than ConfigParser. + if not text: + return + group = None + for line in filter(None, map(str.strip, text.splitlines())): + if line.startswith("["): + group = line[1:-1] + else: + name, value = map(str.strip, line.split("=", 1)) + yield cls(name, value, group) @classmethod def _from_text_for(cls, text, dist): From 2db4dada379822b4767809a5c4e2436f32908658 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 23 Feb 2021 09:58:19 -0500 Subject: [PATCH 32/77] Introduce SelectableGroups, created for the 3.x line to provide forward compatibilty to the new interfaces without sacrificing backward compatibility. --- CHANGES.rst | 21 ++++++++++---- docs/using.rst | 4 +-- importlib_metadata/__init__.py | 50 ++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2f5b1ec5..f8df681d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -v4.0.0 +v3.6.0 ====== * #284: Introduces new ``EntryPoints`` object, a tuple of @@ -13,10 +13,21 @@ v4.0.0 - Item access (e.g. ``eps[name]``) retrieves a single entry point by name. - ``entry_points()`` now returns an ``EntryPoints`` - object, but provides for backward compatibility with - a ``__getitem__`` accessor by group and a ``get()`` - method. + ``entry_points`` now accepts "selection parameters", + same as ``EntryPoint.select()``. + + ``entry_points()`` now provides a future-compatible + ``SelectableGroups`` object that supplies the above interface + but remains a dict for compatibility. + + In the future, ``entry_points()`` will return an + ``EntryPoints`` object, but provide for backward + compatibility with a deprecated ``__getitem__`` + accessor by group and a ``get()`` method. + + If passing selection parameters to ``entry_points``, the + future behavior is invoked and an ``EntryPoints`` is the + result. Construction of entry points using ``dict([EntryPoint, ...])`` is now deprecated and raises diff --git a/docs/using.rst b/docs/using.rst index bdfe3e82..97941452 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,8 +67,8 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a sequence of all entry points, -keyed by group. Entry points are represented by ``EntryPoint`` instances; +The ``entry_points()`` function returns a collection of entry points. +Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index bd7d4d7e..f2dc9c07 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -187,6 +187,37 @@ def _from_text_for(cls, text, dist): return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) +class SelectableGroups(dict): + """ + A backward- and forward-compatible result from + entry_points that fully implements the dict interface. + """ + + @classmethod + def load(cls, eps): + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return cls((group, EntryPoints(eps)) for group, eps in grouped) + + @property + def groups(self): + return self.keys() + + @property + def names(self): + return (ep.name for ep in self._all) + + @property + def _all(self): + return itertools.chain.from_iterable(self.values()) + + def select(self, **params): + if not params: + return self + return EntryPoints(self._all).select(**params) + + class LegacyGroupedEntryPoints(EntryPoints): """ Compatibility wrapper around EntryPoints to provide @@ -713,16 +744,29 @@ def version(distribution_name): return distribution(distribution_name).version -def entry_points(**params): +def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: """Return EntryPoint objects for all installed packages. - :return: EntryPoint objects for all installed packages. + Pass selection parameters (group or name) to filter the + result to entry points matching those properties (see + EntryPoints.select()). + + For compatibility, returns ``SelectableGroups`` object unless + selection parameters are supplied. In the future, this function + will return ``LegacyGroupedEntryPoints`` instead of + ``SelectableGroups`` and eventually will only return + ``EntryPoints``. + + For maximum future compatibility, pass selection parameters + or invoke ``.select`` with parameters on the result. + + :return: EntryPoints or SelectableGroups for all installed packages. """ unique = functools.partial(unique_everseen, key=operator.attrgetter('name')) eps = itertools.chain.from_iterable( dist.entry_points for dist in unique(distributions()) ) - return LegacyGroupedEntryPoints(eps).select(**params) + return SelectableGroups.load(eps).select(**params) def files(distribution_name): From 2def046c694cddfbd1967575f8ce7da95680c9c3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 23 Feb 2021 13:07:55 -0500 Subject: [PATCH 33/77] Address coverage misses, ignored for LegacyGroupedEntryPoints. --- importlib_metadata/__init__.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index f2dc9c07..5f156ae6 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -180,6 +180,11 @@ def names(self): @property def groups(self): + """ + For coverage while SelectableGroups is present. + >>> EntryPoints().groups + set() + """ return set(ep.group for ep in self) @classmethod @@ -202,11 +207,16 @@ def load(cls, eps): @property def groups(self): - return self.keys() + return set(self.keys()) @property def names(self): - return (ep.name for ep in self._all) + """ + for coverage: + >>> SelectableGroups().names + set() + """ + return set(ep.name for ep in self._all) @property def _all(self): @@ -218,7 +228,7 @@ def select(self, **params): return EntryPoints(self._all).select(**params) -class LegacyGroupedEntryPoints(EntryPoints): +class LegacyGroupedEntryPoints(EntryPoints): # pragma: nocover """ Compatibility wrapper around EntryPoints to provide much of the 'dict' interface previously returned by From dd8da47fdf97d4420cca557742f8f075da2123e4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 23 Feb 2021 13:12:42 -0500 Subject: [PATCH 34/77] Leverage EntryPoints interfaces in SelectableGroups --- importlib_metadata/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5f156ae6..b0b1ae0e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -205,9 +205,13 @@ def load(cls, eps): grouped = itertools.groupby(ordered, by_group) return cls((group, EntryPoints(eps)) for group, eps in grouped) + @property + def _all(self): + return EntryPoints(itertools.chain.from_iterable(self.values())) + @property def groups(self): - return set(self.keys()) + return self._all.groups @property def names(self): @@ -216,16 +220,12 @@ def names(self): >>> SelectableGroups().names set() """ - return set(ep.name for ep in self._all) - - @property - def _all(self): - return itertools.chain.from_iterable(self.values()) + return self._all.names def select(self, **params): if not params: return self - return EntryPoints(self._all).select(**params) + return self._all.select(**params) class LegacyGroupedEntryPoints(EntryPoints): # pragma: nocover From 466cd3c8e6036cbd16584629fa0e54d6c0d6b027 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 24 Feb 2021 11:42:19 -0500 Subject: [PATCH 35/77] Add 'packages_distributions'. Fixes #131. --- CHANGES.rst | 6 ++++++ docs/using.rst | 11 +++++++++++ importlib_metadata/__init__.py | 20 ++++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f8df681d..a4e468c0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.7.0 +====== + +* #131: Added ``packages_distributions`` to conveniently + resolve a top-level package or module to its distribution(s). + v3.6.0 ====== diff --git a/docs/using.rst b/docs/using.rst index 97941452..18aa2fda 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -182,6 +182,17 @@ function:: ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] +Package distributions +--------------------- + +A convience method to resolve the distribution or +distributions (in the case of a namespace package) for top-level +Python packages or modules:: + + >>> packages_distributions() + {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...} + + Distributions ============= diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index b0b1ae0e..5c19c237 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,7 +12,7 @@ import functools import itertools import posixpath -import collections +import collections.abc from ._compat import ( NullFinder, @@ -28,7 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Any, List, Optional, TypeVar, Union +from typing import Any, List, Mapping, Optional, TypeVar, Union __all__ = [ @@ -796,3 +796,19 @@ def requires(distribution_name): packaging.requirement.Requirement. """ return distribution(distribution_name).requires + + +def packages_distributions() -> Mapping[str, List[str]]: + """ + Return a mapping of top-level packages to their + distributions. + + >>> pkgs = packages_distributions() + >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) + True + """ + pkg_to_dist = collections.defaultdict(list) + for dist in distributions(): + for pkg in (dist.read_text('top_level.txt') or '').split(): + pkg_to_dist[pkg].append(dist.metadata['Name']) + return dict(pkg_to_dist) From 8c4cff1a2ffea6b4fa59d4a86c6608bb19861a92 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:14:02 -0500 Subject: [PATCH 36/77] Convert LegacyGroupedEntryPoints into simpler dict interface deprecation. --- importlib_metadata/__init__.py | 38 ++++++++-------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5c19c237..162656a8 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -228,42 +228,22 @@ def select(self, **params): return self._all.select(**params) -class LegacyGroupedEntryPoints(EntryPoints): # pragma: nocover +class DeprecatedDict(dict): # pragma: nocover """ - Compatibility wrapper around EntryPoints to provide - much of the 'dict' interface previously returned by - entry_points. + Compatibility wrapper around dict to indicate that + Mapping behavior is deprecated. """ - def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: - """ - When accessed by name that matches a group, return the group. - """ - group = self.select(group=name) - if group: - msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." - warnings.warn(msg, DeprecationWarning, stacklevel=2) - return group - + def __getitem__(self, name): + msg = "SelectableGroups.__getitem__ is deprecated. Use select." + warnings.warn(msg, DeprecationWarning, stacklevel=2) return super().__getitem__(name) - def get(self, group, default=None): - """ - For backward compatibility, supply .get. - """ + def get(self, name, default=None): is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - msg = "GroupedEntryPoints.get is deprecated. Use select." + msg = "SelectableGroups.get is deprecated. Use select." is_flake8 or warnings.warn(msg, DeprecationWarning, stacklevel=2) - return self.select(group=group) or default - - def select(self, **params): - """ - Prevent transform to EntryPoints during call to entry_points if - no selection parameters were passed. - """ - if not params: - return self - return super().select(**params) + return super().get(name, default) class PackagePath(pathlib.PurePosixPath): From b022ae991389755055cec67f5112c283b2413fe1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:20:28 -0500 Subject: [PATCH 37/77] Extract warning as a method. --- importlib_metadata/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 162656a8..5d85be62 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -234,15 +234,17 @@ class DeprecatedDict(dict): # pragma: nocover Mapping behavior is deprecated. """ + def _warn(self): + msg = "SelectableGroups dict interface is deprecated. Use select." + warnings.warn(msg, DeprecationWarning, stacklevel=3) + def __getitem__(self, name): - msg = "SelectableGroups.__getitem__ is deprecated. Use select." - warnings.warn(msg, DeprecationWarning, stacklevel=2) + self._warn() return super().__getitem__(name) def get(self, name, default=None): is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - msg = "SelectableGroups.get is deprecated. Use select." - is_flake8 or warnings.warn(msg, DeprecationWarning, stacklevel=2) + is_flake8 or self._warn() return super().get(name, default) From 1f463549a246ec9f855bea04b20080f3236a9cdc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:39:04 -0500 Subject: [PATCH 38/77] Remove flake8 bypass and implement warning as a partial. --- importlib_metadata/__init__.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5d85be62..26d11c2b 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,7 +5,6 @@ import sys import zipp import email -import inspect import pathlib import operator import warnings @@ -228,23 +227,33 @@ def select(self, **params): return self._all.select(**params) -class DeprecatedDict(dict): # pragma: nocover +class DeprecatedDict(dict): """ Compatibility wrapper around dict to indicate that Mapping behavior is deprecated. + + >>> recwarn = getfixture('recwarn') + >>> dd = DeprecatedDict(foo='bar') + >>> dd.get('baz', None) + >>> dd['foo'] + 'bar' + >>> len(recwarn) + 1 """ - def _warn(self): - msg = "SelectableGroups dict interface is deprecated. Use select." - warnings.warn(msg, DeprecationWarning, stacklevel=3) + _warn = functools.partial( + warnings.warn, + "SelectableGroups dict interface is deprecated. Use select.", + DeprecationWarning, + stacklevel=3, + ) def __getitem__(self, name): self._warn() return super().__getitem__(name) def get(self, name, default=None): - is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - is_flake8 or self._warn() + self._warn() return super().get(name, default) From 537c55da0cf507b1e800e0d7785ae983f8b1f1fb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:50:12 -0500 Subject: [PATCH 39/77] Reimplement flake8 bypass as a decorator. --- importlib_metadata/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 26d11c2b..d100d960 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,12 +5,14 @@ import sys import zipp import email +import inspect import pathlib import operator import warnings import functools import itertools import posixpath +import contextlib import collections.abc from ._compat import ( @@ -227,6 +229,13 @@ def select(self, **params): return self._all.select(**params) +class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): + def __enter__(self): + super().__enter__() + is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) + is_flake8 and warnings.simplefilter('ignore', DeprecationWarning) + + class DeprecatedDict(dict): """ Compatibility wrapper around dict to indicate that @@ -252,6 +261,7 @@ def __getitem__(self, name): self._warn() return super().__getitem__(name) + @Flake8Bypass() def get(self, name, default=None): self._warn() return super().get(name, default) From 1f2a89cf9a6fe71dbcdacdfd040a2abbd4ece842 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:54:48 -0500 Subject: [PATCH 40/77] Also deprecate iter, contains, keys, and values --- importlib_metadata/__init__.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index d100d960..00f758a3 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -246,6 +246,14 @@ class DeprecatedDict(dict): >>> dd.get('baz', None) >>> dd['foo'] 'bar' + >>> list(dd) + ['foo'] + >>> list(dd.keys()) + ['foo'] + >>> 'foo' in dd + True + >>> list(dd.values()) + ['bar'] >>> len(recwarn) 1 """ @@ -266,6 +274,22 @@ def get(self, name, default=None): self._warn() return super().get(name, default) + def __iter__(self): + self._warn() + return super().__iter__() + + def __contains__(self, *args): + self._warn() + return super().__contains__(*args) + + def keys(self): + self._warn() + return super().keys() + + def values(self): + self._warn() + return super().values() + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" From 401c041d15772869a6337aa7c3ddfde77cadcfb4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:57:43 -0500 Subject: [PATCH 41/77] Move selectable groups after DeprecatedDict --- importlib_metadata/__init__.py | 72 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 00f758a3..fb30e2cb 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -193,42 +193,6 @@ def _from_text_for(cls, text, dist): return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) -class SelectableGroups(dict): - """ - A backward- and forward-compatible result from - entry_points that fully implements the dict interface. - """ - - @classmethod - def load(cls, eps): - by_group = operator.attrgetter('group') - ordered = sorted(eps, key=by_group) - grouped = itertools.groupby(ordered, by_group) - return cls((group, EntryPoints(eps)) for group, eps in grouped) - - @property - def _all(self): - return EntryPoints(itertools.chain.from_iterable(self.values())) - - @property - def groups(self): - return self._all.groups - - @property - def names(self): - """ - for coverage: - >>> SelectableGroups().names - set() - """ - return self._all.names - - def select(self, **params): - if not params: - return self - return self._all.select(**params) - - class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): def __enter__(self): super().__enter__() @@ -291,6 +255,42 @@ def values(self): return super().values() +class SelectableGroups(dict): + """ + A backward- and forward-compatible result from + entry_points that fully implements the dict interface. + """ + + @classmethod + def load(cls, eps): + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return cls((group, EntryPoints(eps)) for group, eps in grouped) + + @property + def _all(self): + return EntryPoints(itertools.chain.from_iterable(self.values())) + + @property + def groups(self): + return self._all.groups + + @property + def names(self): + """ + for coverage: + >>> SelectableGroups().names + set() + """ + return self._all.names + + def select(self, **params): + if not params: + return self + return self._all.select(**params) + + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" From 7e7fc8c8f379df4a3d47258015de8e7ae4cd54c5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:22:38 -0500 Subject: [PATCH 42/77] Just check the filename in the frame. Otherwise, it'll match on the current line. --- importlib_metadata/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fb30e2cb..5dc51d3e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -196,7 +196,9 @@ def _from_text_for(cls, text, dist): class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): def __enter__(self): super().__enter__() - is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) + is_flake8 = any( + 'flake8' in str(frame.filename) for frame in inspect.stack()[:5] + ) is_flake8 and warnings.simplefilter('ignore', DeprecationWarning) @@ -219,7 +221,7 @@ class DeprecatedDict(dict): >>> list(dd.values()) ['bar'] >>> len(recwarn) - 1 + 2 """ _warn = functools.partial( From ed33213268c4cda0079649a410cfbfc679a90313 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:26:12 -0500 Subject: [PATCH 43/77] Wrap function rather than decorating method. Avoids varying stack depths. --- importlib_metadata/__init__.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5dc51d3e..44160596 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,7 +12,6 @@ import functools import itertools import posixpath -import contextlib import collections.abc from ._compat import ( @@ -193,13 +192,9 @@ def _from_text_for(cls, text, dist): return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) -class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): - def __enter__(self): - super().__enter__() - is_flake8 = any( - 'flake8' in str(frame.filename) for frame in inspect.stack()[:5] - ) - is_flake8 and warnings.simplefilter('ignore', DeprecationWarning) +def flake8_bypass(func): + is_flake8 = any('flake8' in str(frame.filename) for frame in inspect.stack()[:5]) + return func if not is_flake8 else lambda: None class DeprecatedDict(dict): @@ -221,7 +216,7 @@ class DeprecatedDict(dict): >>> list(dd.values()) ['bar'] >>> len(recwarn) - 2 + 1 """ _warn = functools.partial( @@ -235,9 +230,8 @@ def __getitem__(self, name): self._warn() return super().__getitem__(name) - @Flake8Bypass() def get(self, name, default=None): - self._warn() + flake8_bypass(self._warn)() return super().get(name, default) def __iter__(self): From d940e63e562e66ab2aaf9bdf8444fc40a5637966 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:28:30 -0500 Subject: [PATCH 44/77] Instead of subclassing dict, make Deprecated a mix-in. --- importlib_metadata/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 44160596..8ab38328 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -197,12 +197,13 @@ def flake8_bypass(func): return func if not is_flake8 else lambda: None -class DeprecatedDict(dict): +class Deprecated: """ - Compatibility wrapper around dict to indicate that - Mapping behavior is deprecated. + Compatibility add-in for mapping to indicate that + mapping behavior is deprecated. >>> recwarn = getfixture('recwarn') + >>> class DeprecatedDict(Deprecated, dict): pass >>> dd = DeprecatedDict(foo='bar') >>> dd.get('baz', None) >>> dd['foo'] From 0460524e44896b9e5c746a21e1f06efe9b5ed475 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:35:44 -0500 Subject: [PATCH 45/77] stacklevel of 2 is the right place --- importlib_metadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 8ab38328..b02dc159 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -224,7 +224,7 @@ class Deprecated: warnings.warn, "SelectableGroups dict interface is deprecated. Use select.", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) def __getitem__(self, name): From a54488dca687fbd4e3d35bcddadc26fba836183c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:38:51 -0500 Subject: [PATCH 46/77] Querying missing key will also be deprecated. --- tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index a2810acc..8c8d9abb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -141,8 +141,8 @@ def test_entry_points_groups_getitem(self): with warnings.catch_warnings(record=True): entry_points()['entries'] == entry_points(group='entries') - with self.assertRaises(KeyError): - entry_points()['missing'] + with self.assertRaises(KeyError): + entry_points()['missing'] def test_entry_points_groups_get(self): """ From cc40cd56bfd2ced7e90616149d5450e06877dbde Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:39:46 -0500 Subject: [PATCH 47/77] Update docstring --- importlib_metadata/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index b02dc159..e06c70b6 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -267,6 +267,9 @@ def load(cls, eps): @property def _all(self): + """ + Reconstruct a list of all entrypoints from the groups. + """ return EntryPoints(itertools.chain.from_iterable(self.values())) @property From 3a36cbd993d2eb1a4fd3c691d5b41fb1a46f7639 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:41:09 -0500 Subject: [PATCH 48/77] Deprecate dict usage of SelectableGroups. --- importlib_metadata/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index e06c70b6..da57dede 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -252,7 +252,7 @@ def values(self): return super().values() -class SelectableGroups(dict): +class SelectableGroups(Deprecated, dict): """ A backward- and forward-compatible result from entry_points that fully implements the dict interface. @@ -270,7 +270,8 @@ def _all(self): """ Reconstruct a list of all entrypoints from the groups. """ - return EntryPoints(itertools.chain.from_iterable(self.values())) + groups = super(Deprecated, self).values() + return EntryPoints(itertools.chain.from_iterable(groups)) @property def groups(self): From ae14c73354dbbeeb77038c74a5f2d2c15bc8cd25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:48:14 -0500 Subject: [PATCH 49/77] Update changelog. --- CHANGES.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a4e468c0..1fd456a3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +v3.8.0 +====== + +* Use of Mapping (dict) interfaces on ``SelectableGroups`` + is now flagged as deprecated. Instead, users are advised + to use the select interface for future compatibility. + v3.7.0 ====== @@ -24,12 +31,10 @@ v3.6.0 ``entry_points()`` now provides a future-compatible ``SelectableGroups`` object that supplies the above interface - but remains a dict for compatibility. + (except item access) but remains a dict for compatibility. In the future, ``entry_points()`` will return an - ``EntryPoints`` object, but provide for backward - compatibility with a deprecated ``__getitem__`` - accessor by group and a ``get()`` method. + ``EntryPoints`` object for all entry points. If passing selection parameters to ``entry_points``, the future behavior is invoked and an ``EntryPoints`` is the From bf9fae2c0df316dc837d56ae68880620733d5ff6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 6 Mar 2021 09:57:43 -0500 Subject: [PATCH 50/77] Require twine 3 with keyring unconditionally required. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 249f97c2..a9a50b01 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ commands = skip_install = True deps = build - twine[keyring]>=1.13 + twine>=3 path jaraco.develop>=7.1 passenv = From 7bdab57872da46ef6a5a7f5ea9099a197bdc3131 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:23:48 -0500 Subject: [PATCH 51/77] Add comments indicating why the exclusions are present --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 81f70eea..dd215c65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,8 +34,10 @@ testing = pytest >= 4.6 pytest-checkdocs >= 1.2.3 pytest-flake8 + # python_implementation: workaround for jaraco/skeleton#22 pytest-black >= 0.3.7; python_implementation != "PyPy" pytest-cov + # python_implementation: workaround for jaraco/skeleton#22 pytest-mypy; python_implementation != "PyPy" pytest-enabler From 14312a5bd75d3313ffd3e14fc7fbbc2a9b05cee5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:24:21 -0500 Subject: [PATCH 52/77] Exclude mypy on Python 3.10 as workaround for python/typed_ast#156. --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index dd215c65..55497f8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,8 @@ testing = pytest-black >= 0.3.7; python_implementation != "PyPy" pytest-cov # python_implementation: workaround for jaraco/skeleton#22 - pytest-mypy; python_implementation != "PyPy" + # python_version: workaround for python/typed_ast#156 + pytest-mypy; python_implementation != "PyPy" and python_version < "3.10" pytest-enabler # local From af5445115af0cb68e671a678538a0207389586be Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:30:25 -0500 Subject: [PATCH 53/77] Bump minimums on pytest-checkdocs and pytest-enabler as found on Setuptools. --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 55497f8e..3f6610be 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ exclude = testing = # upstream pytest >= 4.6 - pytest-checkdocs >= 1.2.3 + pytest-checkdocs >= 2.4 pytest-flake8 # python_implementation: workaround for jaraco/skeleton#22 pytest-black >= 0.3.7; python_implementation != "PyPy" @@ -40,7 +40,7 @@ testing = # python_implementation: workaround for jaraco/skeleton#22 # python_version: workaround for python/typed_ast#156 pytest-mypy; python_implementation != "PyPy" and python_version < "3.10" - pytest-enabler + pytest-enabler >= 1.0.1 # local From 86efb884f805a9e1f64661ec758f3bd084fed515 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:53:54 -0500 Subject: [PATCH 54/77] Also deny black on Python 3.10 as workaround for python/typed_ast#156. --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3f6610be..52876d55 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,8 @@ testing = pytest-checkdocs >= 2.4 pytest-flake8 # python_implementation: workaround for jaraco/skeleton#22 - pytest-black >= 0.3.7; python_implementation != "PyPy" + # python_version: workaround for python/typed_ast#156 + pytest-black >= 0.3.7; python_implementation != "PyPy" and python_version < "3.10" pytest-cov # python_implementation: workaround for jaraco/skeleton#22 # python_version: workaround for python/typed_ast#156 From 56d312b95217ece0191d4b587b3da81e6d9a71db Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 17:10:53 -0500 Subject: [PATCH 55/77] Update changelog. --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index a4e468c0..ba99d6d5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.7.1 +====== + +* Internal refactoring to facilitate ``entry_points() -> dict`` + deprecation. + v3.7.0 ====== From 1e2381fe101fd70742a0171e51c1be82aedf519b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 18:48:21 -0500 Subject: [PATCH 56/77] Remove latent reference to LegacyGroupedEntryPoints. --- CHANGES.rst | 5 +++++ importlib_metadata/__init__.py | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ba99d6d5..2792caf8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v3.7.2 +====== + +* Cleaned up cruft in entry_points docstring. + v3.7.1 ====== diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index e06c70b6..95087b55 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -788,9 +788,8 @@ def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: For compatibility, returns ``SelectableGroups`` object unless selection parameters are supplied. In the future, this function - will return ``LegacyGroupedEntryPoints`` instead of - ``SelectableGroups`` and eventually will only return - ``EntryPoints``. + will return ``EntryPoints`` instead of ``SelectableGroups`` + even when no selection parameters are supplied. For maximum future compatibility, pass selection parameters or invoke ``.select`` with parameters on the result. From 12e33bb37bc8517ff89f3e7b56a1b3bd585482ed Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 21:17:36 -0500 Subject: [PATCH 57/77] Extract Lookup class out of directory lookup behavior. Separate it from mtime caching behavior. --- importlib_metadata/__init__.py | 80 +++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 67c925d4..349de2e0 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,6 +12,7 @@ import functools import itertools import posixpath +import contextlib import collections.abc from ._compat import ( @@ -602,13 +603,10 @@ class FastPath: @functools.lru_cache() # type: ignore def __new__(cls, root): - self = object().__new__(cls) + return super().__new__(cls) + + def __init__(self, root): self.root = str(root) - self.base = os.path.basename(self.root).lower() - self.last_mtime = -1 - self.infos = {} - self.eggs = {} - return self def joinpath(self, child): return pathlib.Path(self.root, child) @@ -624,47 +622,54 @@ def zip_children(self): zip_path = zipp.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath + return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - def update_cache(self): - root = self.root or "." - try: - mtime = os.stat(root).st_mtime - except OSError: - self.infos.clear() - self.eggs.clear() - self.last_mtime = -1 - return - if mtime == self.last_mtime: - return - self.infos.clear() - self.eggs.clear() - base_is_egg = self.base.endswith(".egg") - for child in self.children(): + def search(self, name): + return self.lookup(self.mtime).search(name) + + @property + def mtime(self): + with contextlib.suppress(OSError): + return os.stat(self.root).st_mtime + FastPath.lookup.cache_clear() + + @functools.lru_cache() + def lookup(self, mtime): + return Lookup(self) + + +class Lookup: + def __init__(self, path: FastPath): + base = os.path.basename(path.root).lower() + base_is_egg = base.endswith(".egg") + self.infos = collections.defaultdict(list) + self.eggs = collections.defaultdict(list) + + for child in path.children(): low = child.lower() if low.endswith((".dist-info", ".egg-info")): # rpartition is faster than splitext and suitable for this purpose. name = low.rpartition(".")[0].partition("-")[0] normalized = Prepared.normalize(name) - self.infos.setdefault(normalized, []).append(child) + self.infos[normalized].append(path.joinpath(child)) elif base_is_egg and low == "egg-info": - name = self.base.rpartition(".")[0].partition("-")[0] + name = base.rpartition(".")[0].partition("-")[0] legacy_normalized = Prepared.legacy_normalize(name) - self.eggs.setdefault(legacy_normalized, []).append(child) - self.last_mtime = mtime + self.eggs[legacy_normalized].append(path.joinpath(child)) def search(self, prepared): - self.update_cache() - if prepared.name: - infos = self.infos.get(prepared.normalized, []) - yield from map(self.joinpath, infos) - eggs = self.eggs.get(prepared.legacy_normalized, []) - yield from map(self.joinpath, eggs) - else: - for infos in self.infos.values(): - yield from map(self.joinpath, infos) - for eggs in self.eggs.values(): - yield from map(self.joinpath, eggs) + infos = ( + self.infos[prepared.normalized] + if prepared + else itertools.chain.from_iterable(self.infos.values()) + ) + eggs = ( + self.eggs[prepared.legacy_normalized] + if prepared + else itertools.chain.from_iterable(self.eggs.values()) + ) + return itertools.chain(infos, eggs) class Prepared: @@ -697,6 +702,9 @@ def legacy_normalize(name): """ return name.lower().replace('-', '_') + def __bool__(self): + return bool(self.name) + @install class MetadataPathFinder(NullFinder, DistributionFinder): From 7bdeaa45e3710e33735a9632c65ab7916c06c410 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 09:24:04 -0400 Subject: [PATCH 58/77] Add packages_distributions to __all__ --- importlib_metadata/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 95087b55..f583dc91 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -40,6 +40,7 @@ 'entry_points', 'files', 'metadata', + 'packages_distributions', 'requires', 'version', ] From bb2437066473ed10dff16564d731917345251e88 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 09:57:53 -0400 Subject: [PATCH 59/77] Expand docs to explain more about the interfaces and to include a compatibility note. --- docs/using.rst | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 18aa2fda..53d83959 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -72,15 +72,43 @@ Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute:: +``.value`` attribute. + +Query all entry points:: >>> eps = entry_points() + +The ``entry_points()`` function returns an ``EntryPoints`` object, +a sequence of all ``EntryPoint`` objects with ``names`` and ``groups`` +attributes for convenience. + >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + +``EntryPoints`` has a ``select`` method to select entry points +matching specific properties. Select entry points in the +``console_scripts`` group:: + >>> scripts = eps.select(group='console_scripts') + +Equivalently, since ``entry_points`` passes keyword arguments +through to select:: + + >>> scripts = entry_points(group='console_scripts') + +Pick out a specific script named "wheel" (found in the wheel project):: + >>> 'wheel' in scripts.names True >>> wheel = scripts['wheel'] + +Equivalently, query for that entry point during selection:: + + >>> (wheel,) = entry_points(group='console_scripts', name='wheel') + >>> (wheel,) = entry_points().select(group='console_scripts', name='wheel') + +Inspect the resolved entry point:: + >>> wheel EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') >>> wheel.module @@ -99,6 +127,17 @@ group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. +*Compatibility Note* + +The "selectable" entry points were introduced in ``importlib_metadata`` +3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted +no parameters and always returned a dictionary of entry points, keyed +by group. For compatibility, if no parameters are passed to entry_points, +a ``SelectableGroups`` object is returned, implementing that dict +interface. In the future, calling ``entry_points`` with no parameters +will return an ``EntryPoints`` object. Users should rely on the selection +interface to retrieve entry points by group. + .. _metadata: From bf777ae8030fd5fe778778ed159a484129083ece Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:10:20 -0400 Subject: [PATCH 60/77] Expand docs on EntryPoints --- importlib_metadata/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index f583dc91..46203448 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -167,21 +167,33 @@ class EntryPoints(tuple): __slots__ = () def __getitem__(self, name): # -> EntryPoint: + """ + Get the EntryPoint in self matching name. + """ try: return next(iter(self.select(name=name))) except StopIteration: raise KeyError(name) def select(self, **params): + """ + Select entry points from self that match the + given parameters (typically group and/or name). + """ return EntryPoints(ep for ep in self if ep.matches(**params)) @property def names(self): + """ + Return the set of all names of all entry points. + """ return set(ep.name for ep in self) @property def groups(self): """ + Return the set of all groups of all entry points. + For coverage while SelectableGroups is present. >>> EntryPoints().groups set() From 60f2791a2f0aa0007cb39c46b28ad687cef8bd8a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:13:19 -0400 Subject: [PATCH 61/77] Importing inspect is expensive. Defer it unless needed. --- importlib_metadata/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 46203448..832a7116 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,7 +5,6 @@ import sys import zipp import email -import inspect import pathlib import operator import warnings @@ -206,6 +205,9 @@ def _from_text_for(cls, text, dist): def flake8_bypass(func): + # defer inspect import as performance optimization. + import inspect + is_flake8 = any('flake8' in str(frame.filename) for frame in inspect.stack()[:5]) return func if not is_flake8 else lambda: None From d176331ff44cb2636a7b285cfc70371890b6d95b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:15:18 -0400 Subject: [PATCH 62/77] Only import collections.abc in doctests. --- importlib_metadata/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 832a7116..112373e5 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -11,7 +11,7 @@ import functools import itertools import posixpath -import collections.abc +import collections from ._compat import ( NullFinder, @@ -842,6 +842,7 @@ def packages_distributions() -> Mapping[str, List[str]]: Return a mapping of top-level packages to their distributions. + >>> import collections.abc >>> pkgs = packages_distributions() >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) True From da0bc8969757cbe4d1e38aeac837450ff54816f4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:20:41 -0400 Subject: [PATCH 63/77] Update changelog. Ref python/cpython#24782. --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 2792caf8..5777e56e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.7.3 +====== + +* Docs enhancements and cleanup following review in + `GH-24782 `_. + v3.7.2 ====== From 5c9198cd8212d56db8ee9c49ee39a0980d11f5ff Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:32:55 -0400 Subject: [PATCH 64/77] Indicate code block consistently. --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 53d83959..17d6f590 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -80,7 +80,7 @@ Query all entry points:: The ``entry_points()`` function returns an ``EntryPoints`` object, a sequence of all ``EntryPoint`` objects with ``names`` and ``groups`` -attributes for convenience. +attributes for convenience:: >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] From 7fe4ab8294a843622d20face7f9f6ccddb2d0a14 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 15 Mar 2021 18:31:04 -0400 Subject: [PATCH 65/77] Add leading */ to coverage.run.omit. Workaround for pytest-dev/pytest-cov#456. --- .coveragerc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 45823064..6a34e662 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,7 @@ [run] -omit = .tox/* +omit = + # leading `*/` for pytest-dev/pytest-cov#456 + */.tox/* [report] show_missing = True From 6e2740ec6e1166f2b60719b47920eb66e87019ff Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:10:18 -0400 Subject: [PATCH 66/77] Remove 'cache clear' operation. Unlocks caching. --- importlib_metadata/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 349de2e0..5d34bbd2 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -632,7 +632,6 @@ def search(self, name): def mtime(self): with contextlib.suppress(OSError): return os.stat(self.root).st_mtime - FastPath.lookup.cache_clear() @functools.lru_cache() def lookup(self, mtime): From db55bc647bf1778c17339d57aefd2a90f074f264 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:41:32 -0400 Subject: [PATCH 67/77] Restore cache-clear behavior on a per-path basis. --- importlib_metadata/__init__.py | 4 +- importlib_metadata/_functools.py | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 importlib_metadata/_functools.py diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5d34bbd2..42259f6f 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -22,6 +22,7 @@ Protocol, ) +from ._functools import method_cache from ._itertools import unique_everseen from configparser import ConfigParser @@ -632,8 +633,9 @@ def search(self, name): def mtime(self): with contextlib.suppress(OSError): return os.stat(self.root).st_mtime + self.lookup.cache_clear() - @functools.lru_cache() + @method_cache def lookup(self, mtime): return Lookup(self) diff --git a/importlib_metadata/_functools.py b/importlib_metadata/_functools.py new file mode 100644 index 00000000..73f50d00 --- /dev/null +++ b/importlib_metadata/_functools.py @@ -0,0 +1,85 @@ +import types +import functools + + +# from jaraco.functools 3.3 +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Same for a method that hasn't yet been called. + + >>> c = MyClass() + >>> c.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = functools.lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or functools.lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = types.MethodType(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + # Support cache clear even before cache has been created. + wrapper.cache_clear = lambda: None + + return wrapper From 5ef4f529308db4e0778e71fb20522e004d057dcb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:54:45 -0400 Subject: [PATCH 68/77] Update changelog. --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 2792caf8..1bc91e5c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +v3.8.0 +====== + +* #290: Add mtime-based caching for ``FastPath`` and its + lookups, dramatically increasing performance for repeated + distribution lookups. + v3.7.2 ====== From 01cf7b4591671b5a44c28cb054e07074ce0ee760 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 10:47:14 -0400 Subject: [PATCH 69/77] Extract Sectioned for parsing entry points --- importlib_metadata/__init__.py | 74 +++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index ae1d9e37..e2abca6d 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -7,6 +7,7 @@ import email import pathlib import operator +import textwrap import functools import itertools import posixpath @@ -55,6 +56,60 @@ def name(self): return name +class Sectioned: + """ + A simple entry point config parser for performance + + >>> res = Sectioned.get_sections(Sectioned._sample) + >>> sec, values = next(res) + >>> sec + 'sec1' + >>> [(key, value) for key, value in values] + [('a', '1'), ('b', '2')] + >>> sec, values = next(res) + >>> sec + 'sec2' + >>> [(key, value) for key, value in values] + [('a', '2')] + >>> list(res) + [] + """ + + _sample = textwrap.dedent( + """ + [sec1] + a = 1 + b = 2 + + [sec2] + a = 2 + """ + ).lstrip() + + def __init__(self): + self.section = None + + def __call__(self, line): + if line.startswith('[') and line.endswith(']'): + # new section + self.section = line.strip('[]') + return + return self.section + + @classmethod + def get_sections(cls, text): + lines = filter(None, map(str.strip, text.splitlines())) + return ( + (section, map(cls.parse_value, values)) + for section, values in itertools.groupby(lines, cls()) + if section is not None + ) + + @staticmethod + def parse_value(line): + return map(str.strip, line.split("=", 1)) + + class EntryPoint( PyPy_repr, collections.namedtuple('EntryPointBase', 'name value group') ): @@ -115,16 +170,15 @@ def extras(self): @classmethod def _from_text(cls, text): - # A hand-rolled parser is much faster than ConfigParser. - if not text: - return - group = None - for line in filter(None, map(str.strip, text.splitlines())): - if line.startswith("["): - group = line[1:-1] - else: - name, value = map(str.strip, line.split("=", 1)) - yield cls(name, value, group) + return itertools.starmap(cls, cls._parse_groups(text or '')) + + @staticmethod + def _parse_groups(text): + return ( + (name, value, section) + for section, values in Sectioned.get_sections(text) + for name, value in values + ) @classmethod def _from_text_for(cls, text, dist): From 90355b35a2289f79e15bdaddac24e780b488bac4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 10:53:59 -0400 Subject: [PATCH 70/77] Move entry point parsing to EntryPoints class. --- importlib_metadata/__init__.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index cef4f85f..80d6c195 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -172,18 +172,6 @@ def extras(self): match = self.pattern.match(self.value) return list(re.finditer(r'\w+', match.group('extras') or '')) - @classmethod - def _from_text(cls, text): - return itertools.starmap(cls, cls._parse_groups(text or '')) - - @staticmethod - def _parse_groups(text): - return ( - (name, value, section) - for section, values in Sectioned.get_sections(text) - for name, value in values - ) - def _for(self, dist): self.dist = dist return self @@ -253,7 +241,19 @@ def groups(self): @classmethod def _from_text_for(cls, text, dist): - return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) + return cls(ep._for(dist) for ep in cls._from_text(text)) + + @classmethod + def _from_text(cls, text): + return itertools.starmap(EntryPoint, cls._parse_groups(text or '')) + + @staticmethod + def _parse_groups(text): + return ( + (name, value, section) + for section, values in Sectioned.get_sections(text) + for name, value in values + ) def flake8_bypass(func): From a0f0ba62eb6d740b4350bf7fd4067ee4591b30fd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 11:03:30 -0400 Subject: [PATCH 71/77] Update changelog. --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 9b7ca025..49af4a9d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,11 @@ v3.9.0 Preferably, switch to the ``select`` interface introduced in 3.7.0. +* #283: Entry point parsing no longer relies on ConfigParser + and instead uses a custom, one-pass parser to load the + config, resulting in a ~20% performance improvement when + loading entry points. + v3.8.0 ====== From d91dd253f2ba2c22aab4c11a9c03f1da5cb26a62 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 12:48:56 -0400 Subject: [PATCH 72/77] Eagerly consume infos and eggs. Workaround for #293. Still needs a test. --- CHANGES.rst | 5 +++++ importlib_metadata/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c5743ac6..b4d3effc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v3.8.1 +====== + +* #293: Workaround for error in distribution search. + v3.8.0 ====== diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 11b9aac8..fe81c778 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -685,7 +685,7 @@ def search(self, prepared): if prepared else itertools.chain.from_iterable(self.eggs.values()) ) - return itertools.chain(infos, eggs) + return list(itertools.chain(infos, eggs)) class Prepared: From f545a8f197ea90af685bd1d3218fb273968cd008 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 13:15:53 -0400 Subject: [PATCH 73/77] Add test capturing failure. Ref #293. --- tests/test_integration.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index 11835135..00e9021a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -7,6 +7,7 @@ Distribution, MetadataPathFinder, _compat, + distributions, version, ) @@ -59,3 +60,16 @@ def test_search_dist_dirs(self): """ res = MetadataPathFinder._search_paths('any-name', []) assert list(res) == [] + + def test_interleaved_discovery(self): + """ + When the search is cached, it is + possible for searches to be interleaved, so make sure + those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('importlib_metadata') + next(dists) From 8e3e4af3d0b3462d1a143f9544ce8777209b5908 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 13:47:10 -0400 Subject: [PATCH 74/77] Add freezable dict --- importlib_metadata/_collections.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 importlib_metadata/_collections.py diff --git a/importlib_metadata/_collections.py b/importlib_metadata/_collections.py new file mode 100644 index 00000000..f0357989 --- /dev/null +++ b/importlib_metadata/_collections.py @@ -0,0 +1,21 @@ +import collections + + +class freezable_defaultdict(collections.defaultdict): + """ + Mix-in to freeze a defaultdict. + + >>> dd = freezable_defaultdict(list) + >>> dd[0].append('1') + >>> dd.freeze() + >>> dd[1] + [] + >>> len(dd) + 1 + """ + + def __missing__(self, key): + return getattr(self, '_frozen', super().__missing__)(key) + + def freeze(self): + self._frozen = lambda key: self.default_factory() From d84930cf41ba3c1580310d93cb0748ba246b1c47 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 13:55:10 -0400 Subject: [PATCH 75/77] Freeze the defaultdict after construction and re-enable lazy evaluation of the search results. --- importlib_metadata/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fe81c778..0dd0ec55 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -14,13 +14,13 @@ import contextlib import collections +from ._collections import freezable_defaultdict from ._compat import ( NullFinder, + Protocol, PyPy_repr, install, - Protocol, ) - from ._functools import method_cache from ._itertools import unique_everseen @@ -659,8 +659,8 @@ class Lookup: def __init__(self, path: FastPath): base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") - self.infos = collections.defaultdict(list) - self.eggs = collections.defaultdict(list) + self.infos = freezable_defaultdict(list) + self.eggs = freezable_defaultdict(list) for child in path.children(): low = child.lower() @@ -674,6 +674,9 @@ def __init__(self, path: FastPath): legacy_normalized = Prepared.legacy_normalize(name) self.eggs[legacy_normalized].append(path.joinpath(child)) + self.infos.freeze() + self.eggs.freeze() + def search(self, prepared): infos = ( self.infos[prepared.normalized] @@ -685,7 +688,7 @@ def search(self, prepared): if prepared else itertools.chain.from_iterable(self.eggs.values()) ) - return list(itertools.chain(infos, eggs)) + return itertools.chain(infos, eggs) class Prepared: From a6bff7e9a637bb837d283fc5b0c9bb8d5ae73697 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 28 Mar 2021 03:28:53 -0400 Subject: [PATCH 76/77] Presented FreezableDefaultDict from jaraco.collections. Keep inline to minimize dependencies. --- importlib_metadata/__init__.py | 6 +++--- importlib_metadata/_collections.py | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 0dd0ec55..d705251e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -14,7 +14,7 @@ import contextlib import collections -from ._collections import freezable_defaultdict +from ._collections import FreezableDefaultDict from ._compat import ( NullFinder, Protocol, @@ -659,8 +659,8 @@ class Lookup: def __init__(self, path: FastPath): base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") - self.infos = freezable_defaultdict(list) - self.eggs = freezable_defaultdict(list) + self.infos = FreezableDefaultDict(list) + self.eggs = FreezableDefaultDict(list) for child in path.children(): low = child.lower() diff --git a/importlib_metadata/_collections.py b/importlib_metadata/_collections.py index f0357989..6aa17c84 100644 --- a/importlib_metadata/_collections.py +++ b/importlib_metadata/_collections.py @@ -1,11 +1,14 @@ import collections -class freezable_defaultdict(collections.defaultdict): +# from jaraco.collections 3.3 +class FreezableDefaultDict(collections.defaultdict): """ - Mix-in to freeze a defaultdict. + Often it is desirable to prevent the mutation of + a default dict after its initial construction, such + as to prevent mutation during iteration. - >>> dd = freezable_defaultdict(list) + >>> dd = FreezableDefaultDict(list) >>> dd[0].append('1') >>> dd.freeze() >>> dd[1] From d17d6e4ae667c086abf5189d4ffb782a46984584 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 28 Mar 2021 03:30:28 -0400 Subject: [PATCH 77/77] Update changelog. --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index b4d3effc..c30df1b8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.8.2 +====== + +* #293: Re-enabled lazy evaluation of path lookup through + a FreezableDefaultDict. + v3.8.1 ======