From 6eb184b925ed19adbe852c49b2cd69c339b96394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 23 Nov 2021 16:38:02 +0100 Subject: [PATCH] =?UTF-8?q?bpo-45703:=20Invalidate=20=5FNamespacePath=20ca?= =?UTF-8?q?che=20on=20importlib.invalidate=5Fca=E2=80=A6=20(GH-29384)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consider the following directory structure: . └── PATH1 └── namespace └── sub1 └── __init__.py And both PATH1 and PATH2 in sys path: $ PYTHONPATH=PATH1:PATH2 python3.11 >>> import namespace >>> import namespace.sub1 >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace']) >>> ... While this interpreter still runs, PATH2/namespace/sub2 is created: . ├── PATH1 │ └── namespace │ └── sub1 │ └── __init__.py └── PATH2 └── namespace └── sub2 └── __init__.py The newly created module cannot be imported: >>> ... >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace']) >>> import namespace.sub2 Traceback (most recent call last): File "", line 1, in ModuleNotFoundError: No module named 'namespace.sub2' Calling importlib.invalidate_caches() now newly allows to import it: >>> import importlib >>> importlib.invalidate_caches() >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace']) >>> import namespace.sub2 >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace', '.../PATH2/namespace']) This was not previously possible. (cherry picked from commit ae1965ccb4b1fad63fab40fe8805d1b8247668d3) Co-authored-by: Miro Hrončok --- Doc/library/importlib.rst | 4 +++ Lib/importlib/_bootstrap_external.py | 11 +++++- .../test_importlib/test_namespace_pkgs.py | 35 +++++++++++++++++++ .../2021-11-03-13-41-49.bpo-45703.35AagL.rst | 5 +++ 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index c7fbcb21274a16..ac02b9979ed53c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -145,6 +145,10 @@ Functions .. versionadded:: 3.3 + .. versionchanged:: 3.10 + Namespace packages created/installed in a different :data:`sys.path` + location after the same namespace was already imported are noticed. + .. function:: reload(module) Reload a previously imported *module*. The argument must be a module object, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index fe31f437dac2ab..f3828b10e1c1ba 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1209,10 +1209,15 @@ class _NamespacePath: using path_finder. For top-level modules, the parent module's path is sys.path.""" + # When invalidate_caches() is called, this epoch is incremented + # https://bugs.python.org/issue45703 + _epoch = 0 + def __init__(self, name, path, path_finder): self._name = name self._path = path self._last_parent_path = tuple(self._get_parent_path()) + self._last_epoch = self._epoch self._path_finder = path_finder def _find_parent_path_names(self): @@ -1232,7 +1237,7 @@ def _get_parent_path(self): def _recalculate(self): # If the parent's path has changed, recalculate _path parent_path = tuple(self._get_parent_path()) # Make a copy - if parent_path != self._last_parent_path: + if parent_path != self._last_parent_path or self._epoch != self._last_epoch: spec = self._path_finder(self._name, parent_path) # Note that no changes are made if a loader is returned, but we # do remember the new parent path @@ -1240,6 +1245,7 @@ def _recalculate(self): if spec.submodule_search_locations: self._path = spec.submodule_search_locations self._last_parent_path = parent_path # Save the copy + self._last_epoch = self._epoch return self._path def __iter__(self): @@ -1320,6 +1326,9 @@ def invalidate_caches(cls): del sys.path_importer_cache[name] elif hasattr(finder, 'invalidate_caches'): finder.invalidate_caches() + # Also invalidate the caches of _NamespacePaths + # https://bugs.python.org/issue45703 + _NamespacePath._epoch += 1 @classmethod def _path_hooks(cls, path): diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index a8f95a035e2450..92771e71e70106 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -2,6 +2,7 @@ import importlib import os import sys +import tempfile import unittest from test.test_importlib import util @@ -124,6 +125,40 @@ def test_imports(self): self.assertEqual(foo.two.attr, 'portion2 foo two') +class SeparatedNamespacePackagesCreatedWhileRunning(NamespacePackageTest): + paths = ['portion1'] + + def test_invalidate_caches(self): + with tempfile.TemporaryDirectory() as temp_dir: + # we manipulate sys.path before anything is imported to avoid + # accidental cache invalidation when changing it + sys.path.append(temp_dir) + + import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + # the module does not exist, so it cannot be imported + with self.assertRaises(ImportError): + import foo.just_created + + # util.create_modules() manipulates sys.path + # so we must create the modules manually instead + namespace_path = os.path.join(temp_dir, 'foo') + os.mkdir(namespace_path) + module_path = os.path.join(namespace_path, 'just_created.py') + with open(module_path, 'w', encoding='utf-8') as file: + file.write('attr = "just_created foo"') + + # the module is not known, so it cannot be imported yet + with self.assertRaises(ImportError): + import foo.just_created + + # but after explicit cache invalidation, it is importable + importlib.invalidate_caches() + import foo.just_created + self.assertEqual(foo.just_created.attr, 'just_created foo') + + class SeparatedOverlappingNamespacePackages(NamespacePackageTest): paths = ['portion1', 'both_portions'] diff --git a/Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst b/Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst new file mode 100644 index 00000000000000..9fa9be56b8327a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst @@ -0,0 +1,5 @@ +When a namespace package is imported before another module from the same +namespace is created/installed in a different :data:`sys.path` location +while the program is running, calling the +:func:`importlib.invalidate_caches` function will now also guarantee the new +module is noticed.