From b4ce0ff90ec730866885a08134ece217b69949d2 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 29 Jan 2024 01:30:22 -0800 Subject: [PATCH 01/15] gh-109653: Improve import time of importlib.metadata / email.utils (python/cpython#114664) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My criterion for delayed imports is that they're only worth it if the majority of users of the module would benefit from it, otherwise you're just moving latency around unpredictably. mktime_tz is not used anywhere in the standard library and grep.app indicates it's not got much use in the ecosystem either. Distribution.files is not nearly as widely used as other importlib.metadata APIs, so we defer the csv import. Before: ``` λ hyperfine -w 8 './python -c "import importlib.metadata"' Benchmark 1: ./python -c "import importlib.metadata" Time (mean ± σ): 65.1 ms ± 0.5 ms [User: 55.3 ms, System: 9.8 ms] Range (min … max): 64.4 ms … 66.4 ms 44 runs ``` After: ``` λ hyperfine -w 8 './python -c "import importlib.metadata"' Benchmark 1: ./python -c "import importlib.metadata" Time (mean ± σ): 62.0 ms ± 0.3 ms [User: 52.5 ms, System: 9.6 ms] Range (min … max): 61.3 ms … 62.8 ms 46 runs ``` for about a 3ms saving with warm disk cache, maybe 7-11ms with cold disk cache. --- importlib_metadata/__init__.py | 5 ++++- newsfragments/+.feature.rst | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 newsfragments/+.feature.rst diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index c68d8ad8..5cd15ab5 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -3,7 +3,6 @@ import os import re import abc -import csv import sys import json import zipp @@ -522,6 +521,10 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none diff --git a/newsfragments/+.feature.rst b/newsfragments/+.feature.rst new file mode 100644 index 00000000..865acfc1 --- /dev/null +++ b/newsfragments/+.feature.rst @@ -0,0 +1 @@ +Improve import time (python/cpython#114664). From fb7465c6f81847574b736e6dfa1bb93187e025cc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 14 Mar 2024 17:59:00 -0400 Subject: [PATCH 02/15] gh-116811: Ensure MetadataPathFinder.invalidate_caches is reachable when delegated through PathFinder. (python/cpython#116812) * Make MetadataPathFinder a proper classmethod. --- importlib_metadata/__init__.py | 1 + newsfragments/+.bugfix.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 newsfragments/+.bugfix.rst diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5cd15ab5..86e68a23 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -898,6 +898,7 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + @classmethod def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() diff --git a/newsfragments/+.bugfix.rst b/newsfragments/+.bugfix.rst new file mode 100644 index 00000000..a8dd3dde --- /dev/null +++ b/newsfragments/+.bugfix.rst @@ -0,0 +1 @@ +Allow ``MetadataPathFinder.invalidate_caches`` to be called as a classmethod. From 9a878d6b263b7e9dedc10fc0df1a7323e9ee7b99 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 16 Jan 2024 16:10:03 +0100 Subject: [PATCH 03/15] gh-114107: Fix symlink test if symlinks aren't supported (python/cpython#114108) --- tests/test_main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 6e0b1da7..45d74534 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,6 +5,8 @@ import importlib import importlib_metadata import contextlib +from test.support import os_helper + import pyfakefs.fake_filesystem_unittest as ffs from . import fixtures @@ -396,6 +398,7 @@ def test_packages_distributions_all_module_types(self): assert not any(name.endswith('.dist-info') for name in distributions) + @os_helper.skip_unless_symlink def test_packages_distributions_symlinked_top_level(self) -> None: """ Distribution is resolvable from a simple top-level symlink in RECORD. From f16e114956ea2970d4b391ca031dfe7f2d65d46e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 04:53:22 -0400 Subject: [PATCH 04/15] Add support for python/cpython references --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 134d7534..90a1da2a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ url='https://peps.python.org/pep-{pep_number:0>4}/', ), dict( - pattern=r'(Python #|py-)(?P\d+)', + pattern=r'(python/cpython#|Python #|py-)(?P\d+)', url='https://github.com/python/cpython/issues/{python}', ), ], From 3531507c7a275cfcbdefe02e0aed4f92ff589b09 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 05:11:25 -0400 Subject: [PATCH 05/15] Fix test failures on older Pythons with os_helper shim. Copied 'from_test_support' from importlib_resources. --- setup.cfg | 1 + tests/fixtures.py | 6 ++++-- tests/py39compat.py | 18 ++++++++++++++++-- tests/test_main.py | 2 +- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 71b66b39..f67df574 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ testing = pyfakefs flufl.flake8 pytest-perf >= 0.9.2 + jaraco.collections docs = # upstream diff --git a/tests/fixtures.py b/tests/fixtures.py index 7daae16a..88478050 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -9,7 +9,7 @@ import functools import contextlib -from .py39compat import FS_NONASCII +from .py39compat import os_helper from . import _path from ._path import FilesSpec @@ -335,7 +335,9 @@ def record_names(file_defs): class FileBuilder: def unicode_filename(self): - return FS_NONASCII or self.skip("File system does not support non-ascii.") + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) def DALS(str): diff --git a/tests/py39compat.py b/tests/py39compat.py index 926dcad9..b01ecad8 100644 --- a/tests/py39compat.py +++ b/tests/py39compat.py @@ -1,4 +1,18 @@ +import types + +from jaraco.collections import Projection + + +def from_test_support(*names): + """ + Return a SimpleNamespace of names from test.support. + """ + import test.support + + return types.SimpleNamespace(**Projection(names, vars(test.support))) + + try: - from test.support.os_helper import FS_NONASCII + from test.support import os_helper # type: ignore except ImportError: - from test.support import FS_NONASCII # noqa + os_helper = from_test_support('FS_NONASCII', 'skip_unless_symlink') diff --git a/tests/test_main.py b/tests/test_main.py index 45d74534..c94ab0f4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,7 +5,7 @@ import importlib import importlib_metadata import contextlib -from test.support import os_helper +from .py39compat import os_helper import pyfakefs.fake_filesystem_unittest as ffs From 856541b5f26c099ef7629b95533ca285fe6c102c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 05:12:18 -0400 Subject: [PATCH 06/15] Moved compatibility module to compat package. --- importlib_metadata/__init__.py | 7 ++++--- importlib_metadata/compat/__init__.py | 0 importlib_metadata/{_py39compat.py => compat/py39.py} | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 importlib_metadata/compat/__init__.py rename importlib_metadata/{_py39compat.py => compat/py39.py} (82%) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 86e68a23..c85ffa76 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -18,7 +18,8 @@ import posixpath import collections -from . import _adapters, _meta, _py39compat +from . import _adapters, _meta +from .compat import py39 from ._collections import FreezableDefaultDict, Pair from ._compat import ( NullFinder, @@ -279,7 +280,7 @@ def select(self, **params) -> EntryPoints: Select entry points from self that match the given parameters (typically group and/or name). """ - return EntryPoints(ep for ep in self if _py39compat.ep_matches(ep, **params)) + return EntryPoints(ep for ep in self if py39.ep_matches(ep, **params)) @property def names(self) -> Set[str]: @@ -996,7 +997,7 @@ def version(distribution_name: str) -> str: _unique = functools.partial( unique_everseen, - key=_py39compat.normalized_name, + key=py39.normalized_name, ) """ Wrapper for ``distributions`` to return unique distributions by name. diff --git a/importlib_metadata/compat/__init__.py b/importlib_metadata/compat/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/importlib_metadata/_py39compat.py b/importlib_metadata/compat/py39.py similarity index 82% rename from importlib_metadata/_py39compat.py rename to importlib_metadata/compat/py39.py index fc6b8221..1f15bd97 100644 --- a/importlib_metadata/_py39compat.py +++ b/importlib_metadata/compat/py39.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: # pragma: no cover # Prevent circular imports on runtime. - from . import Distribution, EntryPoint + from .. import Distribution, EntryPoint else: Distribution = EntryPoint = Any @@ -18,7 +18,7 @@ def normalized_name(dist: Distribution) -> Optional[str]: try: return dist._normalized_name except AttributeError: - from . import Prepared # -> delay to prevent circular imports. + from .. import Prepared # -> delay to prevent circular imports. return Prepared.normalize(getattr(dist, "name", None) or dist.metadata['Name']) @@ -30,7 +30,7 @@ def ep_matches(ep: EntryPoint, **params) -> bool: try: return ep.matches(**params) except AttributeError: - from . import EntryPoint # -> delay to prevent circular imports. + from .. import EntryPoint # -> delay to prevent circular imports. # Reconstruct the EntryPoint object to make sure it is compatible. return EntryPoint(ep.name, ep.value, ep.group).matches(**params) From ffa719bbd8876b43964b704af34b6bce50ac7271 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 05:14:31 -0400 Subject: [PATCH 07/15] Moved compatibility module to compat package. --- tests/compat/__init__.py | 0 tests/{py39compat.py => compat/py39.py} | 0 tests/fixtures.py | 2 +- tests/test_main.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests/compat/__init__.py rename tests/{py39compat.py => compat/py39.py} (100%) diff --git a/tests/compat/__init__.py b/tests/compat/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/py39compat.py b/tests/compat/py39.py similarity index 100% rename from tests/py39compat.py rename to tests/compat/py39.py diff --git a/tests/fixtures.py b/tests/fixtures.py index 88478050..07f10963 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -9,7 +9,7 @@ import functools import contextlib -from .py39compat import os_helper +from .compat.py39 import os_helper from . import _path from ._path import FilesSpec diff --git a/tests/test_main.py b/tests/test_main.py index c94ab0f4..af79e698 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,7 +5,7 @@ import importlib import importlib_metadata import contextlib -from .py39compat import os_helper +from .compat.py39 import os_helper import pyfakefs.fake_filesystem_unittest as ffs From 5950f43b8f44a1b700342ffd4633c147309b1c7c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 05:17:34 -0400 Subject: [PATCH 08/15] Remove legacy logic for Python 3.7. --- tests/test_py39compat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_py39compat.py b/tests/test_py39compat.py index 7e6235e4..34745a4b 100644 --- a/tests/test_py39compat.py +++ b/tests/test_py39compat.py @@ -14,8 +14,7 @@ class OldStdlibFinderTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): def setUp(self): - python_version = sys.version_info[:2] - if python_version < (3, 8) or python_version > (3, 9): + if sys.version_info >= (3, 10): self.skipTest("Tests specific for Python 3.8/3.9") super().setUp() From 41ca0390dbb52543104f12c0629f0bbb882e48ea Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 05:22:24 -0400 Subject: [PATCH 09/15] Moved compatibility tests to the compat package, as they're not included in CPython. --- tests/{test_py39compat.py => compat/test_py39_compat.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/{test_py39compat.py => compat/test_py39_compat.py} (99%) diff --git a/tests/test_py39compat.py b/tests/compat/test_py39_compat.py similarity index 99% rename from tests/test_py39compat.py rename to tests/compat/test_py39_compat.py index 34745a4b..549e518a 100644 --- a/tests/test_py39compat.py +++ b/tests/compat/test_py39_compat.py @@ -2,7 +2,7 @@ import pathlib import unittest -from . import fixtures +from .. import fixtures from importlib_metadata import ( distribution, distributions, From e30a16d471f62555db5605d5652bede9a1234b1a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 06:20:46 -0400 Subject: [PATCH 10/15] Consolidated test support logic in jaraco.test.cpython. --- setup.cfg | 2 +- tests/compat/py39.py | 20 ++++---------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/setup.cfg b/setup.cfg index f67df574..02d3c7e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ testing = pyfakefs flufl.flake8 pytest-perf >= 0.9.2 - jaraco.collections + jaraco.test >= 5.4 docs = # upstream diff --git a/tests/compat/py39.py b/tests/compat/py39.py index b01ecad8..0f6f9c3a 100644 --- a/tests/compat/py39.py +++ b/tests/compat/py39.py @@ -1,18 +1,6 @@ -import types +from jaraco.test.cpython import from_test_support, try_import -from jaraco.collections import Projection - -def from_test_support(*names): - """ - Return a SimpleNamespace of names from test.support. - """ - import test.support - - return types.SimpleNamespace(**Projection(names, vars(test.support))) - - -try: - from test.support import os_helper # type: ignore -except ImportError: - os_helper = from_test_support('FS_NONASCII', 'skip_unless_symlink') +os_helper = try_import('os_helper') or from_test_support( + 'FS_NONASCII', 'skip_unless_symlink' +) From 07d894d96777e77f9dac3ec671f2dce4c584a26d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 13:49:52 -0400 Subject: [PATCH 11/15] Copy backport of isolated_modules from importlib_resources. --- tests/compat/py312.py | 18 ++++++++++++++++++ tests/compat/py39.py | 1 + 2 files changed, 19 insertions(+) create mode 100644 tests/compat/py312.py diff --git a/tests/compat/py312.py b/tests/compat/py312.py new file mode 100644 index 00000000..ea9a58ba --- /dev/null +++ b/tests/compat/py312.py @@ -0,0 +1,18 @@ +import contextlib + +from .py39 import import_helper + + +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = import_helper.modules_setup() + try: + yield + finally: + import_helper.modules_cleanup(saved) + + +vars(import_helper).setdefault('isolated_modules', isolated_modules) diff --git a/tests/compat/py39.py b/tests/compat/py39.py index 0f6f9c3a..5fe61c39 100644 --- a/tests/compat/py39.py +++ b/tests/compat/py39.py @@ -4,3 +4,4 @@ os_helper = try_import('os_helper') or from_test_support( 'FS_NONASCII', 'skip_unless_symlink' ) +import_helper = try_import('import_helper') or from_test_support() From adc4b124fc57cc3864bf68c61f7fa046757ffa02 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 13:55:54 -0400 Subject: [PATCH 12/15] Ensure tests do not leak references in sys.modules. --- tests/fixtures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/fixtures.py b/tests/fixtures.py index 07f10963..f8082df0 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -9,6 +9,7 @@ import functools import contextlib +from .compat.py312 import import_helper from .compat.py39 import os_helper from . import _path @@ -84,6 +85,7 @@ def add_sys_path(dir): def setUp(self): super().setUp() self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) class SiteBuilder(SiteDir): From 47b14acde7b15472b02a14c7abdd7e5545af37f5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 15:17:03 -0400 Subject: [PATCH 13/15] Make MetadataPathFinder.find_distributions a classmethod for consistency with CPython. Closes #484. --- importlib_metadata/__init__.py | 5 +++-- newsfragments/484.bugfix.rst | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 newsfragments/484.bugfix.rst diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index c85ffa76..32ee3b4d 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -877,8 +877,9 @@ class MetadataPathFinder(NullFinder, DistributionFinder): of Python that do not have a PathFinder find_distributions(). """ + @classmethod def find_distributions( - self, context=DistributionFinder.Context() + cls, context=DistributionFinder.Context() ) -> Iterable[PathDistribution]: """ Find distributions. @@ -888,7 +889,7 @@ def find_distributions( (or all names if ``None`` indicated) along the paths in the list of directories ``context.path``. """ - found = self._search_paths(context.name, context.path) + found = cls._search_paths(context.name, context.path) return map(PathDistribution, found) @classmethod diff --git a/newsfragments/484.bugfix.rst b/newsfragments/484.bugfix.rst new file mode 100644 index 00000000..4274419b --- /dev/null +++ b/newsfragments/484.bugfix.rst @@ -0,0 +1 @@ +Make MetadataPathFinder.find_distributions a classmethod for consistency with CPython. Closes #484. \ No newline at end of file From 1711b2c1984b2f871fe12c9c14fd7a0b42d32987 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 15:26:59 -0400 Subject: [PATCH 14/15] Need to include names from test.support for py312 compat. --- tests/compat/py39.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/compat/py39.py b/tests/compat/py39.py index 5fe61c39..16c8b574 100644 --- a/tests/compat/py39.py +++ b/tests/compat/py39.py @@ -4,4 +4,6 @@ os_helper = try_import('os_helper') or from_test_support( 'FS_NONASCII', 'skip_unless_symlink' ) -import_helper = try_import('import_helper') or from_test_support() +import_helper = try_import('import_helper') or from_test_support( + 'modules_setup', 'modules_cleanup' +) From f5d6b5f3f3f6fffe01b340c5a19562433db148a9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 15:38:57 -0400 Subject: [PATCH 15/15] Finalize --- NEWS.rst | 16 ++++++++++++++++ newsfragments/+.bugfix.rst | 1 - newsfragments/+.feature.rst | 1 - newsfragments/484.bugfix.rst | 1 - 4 files changed, 16 insertions(+), 3 deletions(-) delete mode 100644 newsfragments/+.bugfix.rst delete mode 100644 newsfragments/+.feature.rst delete mode 100644 newsfragments/484.bugfix.rst diff --git a/NEWS.rst b/NEWS.rst index 085a5305..850e8f00 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,3 +1,19 @@ +v7.1.0 +====== + +Features +-------- + +- Improve import time (python/cpython#114664). + + +Bugfixes +-------- + +- Make MetadataPathFinder.find_distributions a classmethod for consistency with CPython. Closes #484. (#484) +- Allow ``MetadataPathFinder.invalidate_caches`` to be called as a classmethod. + + v7.0.2 ====== diff --git a/newsfragments/+.bugfix.rst b/newsfragments/+.bugfix.rst deleted file mode 100644 index a8dd3dde..00000000 --- a/newsfragments/+.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ``MetadataPathFinder.invalidate_caches`` to be called as a classmethod. diff --git a/newsfragments/+.feature.rst b/newsfragments/+.feature.rst deleted file mode 100644 index 865acfc1..00000000 --- a/newsfragments/+.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Improve import time (python/cpython#114664). diff --git a/newsfragments/484.bugfix.rst b/newsfragments/484.bugfix.rst deleted file mode 100644 index 4274419b..00000000 --- a/newsfragments/484.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Make MetadataPathFinder.find_distributions a classmethod for consistency with CPython. Closes #484. \ No newline at end of file