From d5399ef8f183f0ad518871036607189d59a1c1cb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 7 Aug 2023 09:41:00 -0400 Subject: [PATCH 01/19] Add API docs. Closes #245. --- docs/api.rst | 26 ++++++++++++++++++++++++++ docs/conf.py | 8 ++++++++ docs/index.rst | 7 ++++--- 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 docs/api.rst diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..739be9e --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,26 @@ +============= +API Reference +============= + +``importlib_resources`` module +------------------------------ + +.. automodule:: importlib_resources + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: importlib_resources.abc + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: importlib_resources.readers + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: importlib_resources.simple + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index 1500b33..b4eaace 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,6 +34,7 @@ # Be strict about any broken references nitpicky = True +nitpick_ignore = [] # Include Python intersphinx mapping to prevent failures # jaraco/skeleton#51 @@ -46,3 +47,10 @@ autodoc_preserve_defaults = True extensions += ['jaraco.tidelift'] + +nitpick_ignore.extend( + [ + ('py:class', 'module'), + ('py:class', '_io.BufferedReader'), + ] +) diff --git a/docs/index.rst b/docs/index.rst index 24befdc..eee051c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,9 +31,10 @@ The documentation here includes a general :ref:`usage ` guide and a :maxdepth: 2 :caption: Contents: - using.rst - migration.rst - history.rst + using + api + migration + history .. tidelift-referral-banner:: From 0e2032c4754c598ba75e467c64009ba4490ddea9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 31 Aug 2023 18:42:14 -0400 Subject: [PATCH 02/19] Pin against sphinx 7.2.5 as workaround for sphinx/sphinx-doc#11662. Closes jaraco/skeleton#88. --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 46f7bdf..4f184c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,8 @@ testing = docs = # upstream sphinx >= 3.5 + # workaround for sphinx/sphinx-doc#11662 + sphinx < 7.2.5 jaraco.packaging >= 9.3 rst.linker >= 1.9 furo From 92d2d8e1aff997f3877239230c9490ed9cdd1222 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 1 Sep 2023 18:46:27 -0400 Subject: [PATCH 03/19] Allow GITHUB_* settings to pass through to tests. --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b822409..67d9d3b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,6 +36,10 @@ env: # Must be "1". TOX_PARALLEL_NO_SPINNER: 1 + # Ensure tests can sense settings about the environment + TOX_OVERRIDE: >- + testenv.pass_env+=GITHUB_* + jobs: test: From f3dc1f4776c94a9a4a7c0e8c5b49c532b0a7d411 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 1 Sep 2023 18:49:13 -0400 Subject: [PATCH 04/19] Remove spinner disablement. If it's not already fixed upstream, that's where it should be fixed. --- .github/workflows/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 67d9d3b..30c9615 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,10 +32,6 @@ env: PIP_NO_PYTHON_VERSION_WARNING: 'true' PIP_NO_WARN_SCRIPT_LOCATION: 'true' - # Disable the spinner, noise in GHA; TODO(webknjaz): Fix this upstream - # Must be "1". - TOX_PARALLEL_NO_SPINNER: 1 - # Ensure tests can sense settings about the environment TOX_OVERRIDE: >- testenv.pass_env+=GITHUB_* From 0484daa8a6f72c9ad4e1784f9181c2488a191d8e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 1 Sep 2023 18:53:55 -0400 Subject: [PATCH 05/19] Clean up 'color' environment variables. The TOX_TESTENV_PASSENV hasn't been useful for some time and by its mere presence wasted a lot of time today under the assumption that it's doing something. Instead, just rely on one variable FORCE_COLOR. If it's not honored, then that should be the fix upstream. --- .github/workflows/main.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 30c9615..f302854 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,26 +6,8 @@ permissions: contents: read env: - # Environment variables to support color support (jaraco/skeleton#66): - # Request colored output from CLI tools supporting it. Different tools - # interpret the value differently. For some, just being set is sufficient. - # For others, it must be a non-zero integer. For yet others, being set - # to a non-empty value is sufficient. For tox, it must be one of - # , 0, 1, false, no, off, on, true, yes. The only enabling value - # in common is "1". + # Environment variable to support color support (jaraco/skeleton#66) FORCE_COLOR: 1 - # MyPy's color enforcement (must be a non-zero number) - MYPY_FORCE_COLOR: -42 - # Recognized by the `py` package, dependency of `pytest` (must be "1") - PY_COLORS: 1 - # Make tox-wrapped tools see color requests - TOX_TESTENV_PASSENV: >- - FORCE_COLOR - MYPY_FORCE_COLOR - NO_COLOR - PY_COLORS - PYTEST_THEME - PYTEST_THEME_MODE # Suppress noisy pip warnings PIP_DISABLE_PIP_VERSION_CHECK: 'true' From b02bf32bae729d53bdb7c9649d6ec36afdb793ee Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 10 Sep 2023 13:27:03 -0400 Subject: [PATCH 06/19] Add diff-cover check to Github Actions CI. Closes jaraco/skeleton#90. --- .github/workflows/main.yml | 18 ++++++++++++++++++ tox.ini | 8 ++++++++ 2 files changed, 26 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f302854..fa326a2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,6 +53,24 @@ jobs: - name: Run run: tox + diffcov: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Install tox + run: | + python -m pip install tox + - name: Evaluate coverage + run: tox + env: + TOXENV: diffcov + docs: runs-on: ubuntu-latest env: diff --git a/tox.ini b/tox.ini index e51d652..3b4414b 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,14 @@ usedevelop = True extras = testing +[testenv:diffcov] +deps = + diff-cover +commands = + pytest {posargs} --cov-report xml + diff-cover coverage.xml --compare-branch=origin/main --html-report diffcov.html + diff-cover coverage.xml --compare-branch=origin/main --fail-under=100 + [testenv:docs] extras = docs From a6256e2935468b72a61aa7fda1e036faef3bfb3d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 10 Sep 2023 13:59:47 -0400 Subject: [PATCH 07/19] Add descriptions to the tox environments. Closes jaraco/skeleton#91. --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 3b4414b..1950b4e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,5 @@ [testenv] +description = perform primary checks (tests, style, types, coverage) deps = setenv = PYTHONWARNDEFAULTENCODING = 1 @@ -9,6 +10,7 @@ extras = testing [testenv:diffcov] +description = run tests and check that diff from main is covered deps = diff-cover commands = @@ -17,6 +19,7 @@ commands = diff-cover coverage.xml --compare-branch=origin/main --fail-under=100 [testenv:docs] +description = build the documentation extras = docs testing @@ -26,6 +29,7 @@ commands = python -m sphinxlint [testenv:finalize] +description = assemble changelog and tag a release skip_install = True deps = towncrier @@ -36,6 +40,7 @@ commands = [testenv:release] +description = publish the package to PyPI and GitHub skip_install = True deps = build From 928e9a86d61d3a660948bcba7689f90216cc8243 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 10 Sep 2023 14:10:31 -0400 Subject: [PATCH 08/19] Add FORCE_COLOR to the TOX_OVERRIDE for GHA. Requires tox 4.11.1. Closes jaraco/skeleton#89. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fa326a2..28e3678 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ env: # Ensure tests can sense settings about the environment TOX_OVERRIDE: >- - testenv.pass_env+=GITHUB_* + testenv.pass_env+=GITHUB_*,FORCE_COLOR jobs: From 496acc1a0d8018c830b30a3a28826c9b101975fa Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 12 Sep 2023 15:05:00 -0400 Subject: [PATCH 09/19] Replace static zip fixtures with dynamically generated zip fixtures built from the same modules as found on the file system. --- importlib_resources/tests/test_read.py | 6 +- importlib_resources/tests/test_resource.py | 77 ++++-------------- importlib_resources/tests/update-zips.py | 53 ------------ importlib_resources/tests/util.py | 48 ++++------- importlib_resources/tests/zip.py | 29 +++++++ .../tests/zipdata01/__init__.py | 0 .../tests/zipdata01/ziptestdata.zip | Bin 876 -> 0 bytes .../tests/zipdata02/__init__.py | 0 .../tests/zipdata02/ziptestdata.zip | Bin 698 -> 0 bytes 9 files changed, 66 insertions(+), 147 deletions(-) delete mode 100755 importlib_resources/tests/update-zips.py create mode 100755 importlib_resources/tests/zip.py delete mode 100644 importlib_resources/tests/zipdata01/__init__.py delete mode 100644 importlib_resources/tests/zipdata01/ziptestdata.zip delete mode 100644 importlib_resources/tests/zipdata02/__init__.py delete mode 100644 importlib_resources/tests/zipdata02/ziptestdata.zip diff --git a/importlib_resources/tests/test_read.py b/importlib_resources/tests/test_read.py index b5aaec4..5b83221 100644 --- a/importlib_resources/tests/test_read.py +++ b/importlib_resources/tests/test_read.py @@ -58,15 +58,13 @@ class ReadDiskTests(ReadTests, unittest.TestCase): class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() self.assertEqual(result, b'\0\1\2\3') def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) self.assertEqual(result, b'\0\1\2\3') diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 677110c..4069b6c 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -1,15 +1,11 @@ -import contextlib import sys import unittest import importlib_resources as resources -import uuid import pathlib from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import import_module -from ._compat import import_helper, os_helper, unlink class ResourceTests: @@ -89,34 +85,32 @@ def test_package_has_no_reader_fallback(self): class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore + ZIP_MODULE = 'data01' def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) @@ -124,7 +118,7 @@ def test_as_file_directory(self): class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore + ZIP_MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,80 +126,45 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetupBase, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') class ResourceFromNamespaceTest01(unittest.TestCase): diff --git a/importlib_resources/tests/update-zips.py b/importlib_resources/tests/update-zips.py deleted file mode 100755 index 231334a..0000000 --- a/importlib_resources/tests/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/importlib_resources/tests/util.py b/importlib_resources/tests/util.py index ce0e6fa..066f411 100644 --- a/importlib_resources/tests/util.py +++ b/importlib_resources/tests/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib from . import data01 -from . import zipdata01 from ..abc import ResourceReader -from ._compat import import_helper +from ._compat import import_helper, os_helper +from . import zip as zip_ from importlib.machinery import ModuleSpec @@ -141,39 +142,24 @@ def test_useless_loader(self): class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass + ZIP_MODULE = 'data01' def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + modules = import_helper.modules_setup() self.addCleanup(import_helper.modules_cleanup, *modules) + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + src_path = pathlib.Path(__file__).parent.joinpath(self.ZIP_MODULE) + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(src_path, modules))) + ) + + self.data = importlib.import_module(self.ZIP_MODULE) + class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore + pass diff --git a/importlib_resources/tests/zip.py b/importlib_resources/tests/zip.py new file mode 100755 index 0000000..6ab914b --- /dev/null +++ b/importlib_resources/tests/zip.py @@ -0,0 +1,29 @@ +""" +Generate zip test data files. +""" + +import contextlib +import os +import pathlib +import zipfile + + +def make_zip_file(src, dst): + """ + Zip the files in src into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for src_path, rel in walk(src): + dst_name = src.name / pathlib.PurePosixPath(rel.as_posix()) + zf.write(src_path, dst_name) + return dst + + +def walk(datapath): + for dirpath, dirnames, filenames in os.walk(datapath): + with contextlib.suppress(ValueError): + dirnames.remove('__pycache__') + for filename in filenames: + res = pathlib.Path(dirpath) / filename + rel = res.relative_to(datapath) + yield res, rel diff --git a/importlib_resources/tests/zipdata01/__init__.py b/importlib_resources/tests/zipdata01/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/importlib_resources/tests/zipdata01/ziptestdata.zip b/importlib_resources/tests/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739f87e97c1084b94d7d153680f6727738..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 876 zcmWIWW@Zs#00HOCX@Q%&m27l?Y!DU);;PJolGNgol*E!m{nC;&T|+ayw9K5;|NlG~ zQWMD z9;rDw`8o=rA#S=B3g!7lIVp-}COK17UPc zNtt;*xhM-3R!jMEPhCreO-3*u>5Df}T7+BJ{639e$2uhfsIs`pJ5Qf}C xGXyDE@VNvOv@o!wQJfLgCAgysx3f@9jKpUmiW^zkK<;1z!tFpk^MROw0RS~O%0&PG diff --git a/importlib_resources/tests/zipdata02/__init__.py b/importlib_resources/tests/zipdata02/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/importlib_resources/tests/zipdata02/ziptestdata.zip b/importlib_resources/tests/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d2807ef2fd259455283b81b02e0e45fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 698 zcmWIWW@Zs#00HOCX@Ot{ln@8fRhb1Psl_EJi6x2p@$s2?nI-Y@dIgmMI5kP5Y0A$_ z#jWw|&p#`9ff_(q7K_HB)Z+ZoqU2OVy^@L&ph*fa0WRVlP*R?c+X1opI-R&20MZDv z&j{oIpa8N17@0(vaR(gGH(;=&5k%n(M%;#g0ulz6G@1gL$cA79E2=^00gEsw4~s!C zUxI@ZWaIMqz|BszK;s4KsL2<9jRy!Q2E6`2cTLHjr{wAk1ZCU@!+_ G1_l6Bc%f?m From 023d2c13e5c3598e187470369d94bcf39ed411b8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 12 Sep 2023 15:37:11 -0400 Subject: [PATCH 10/19] Separate 'disk' concern of namespace tests. --- importlib_resources/tests/test_resource.py | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 4069b6c..4036805 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -167,17 +167,7 @@ def test_read_text_does_not_keep_open(self): resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -207,5 +197,17 @@ def test_submodule_contents_by_name(self): self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) +class ResourceFromNamespaceDiskTests(ResourceFromNamespaceTests, unittest.TestCase): + site_dir = str(pathlib.Path(__file__).parent) + + @classmethod + def setUpClass(cls): + sys.path.append(cls.site_dir) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.site_dir) + + if __name__ == '__main__': unittest.main() From ca1831c2148fe5ddbffd001de76ff5f6005f812c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Sep 2023 11:05:36 -0400 Subject: [PATCH 11/19] Prefer ``pass_env`` in tox config. Preferred failure mode for tox-dev/tox#3127 and closes jaraco/skeleton#92. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 1950b4e..33da3de 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,7 @@ skip_install = True deps = towncrier jaraco.develop >= 7.23 -passenv = * +pass_env = * commands = python -m jaraco.develop.finalize @@ -46,7 +46,7 @@ deps = build twine>=3 jaraco.develop>=7.1 -passenv = +pass_env = TWINE_PASSWORD GITHUB_TOKEN setenv = From d3a7f69e44043719330fb68e6183ce0da31b47d3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Sep 2023 11:38:27 -0400 Subject: [PATCH 12/19] In zip namespace fixtures, explicitly generate the directory entries implied by children. Workaround for python/cpython#59110. --- importlib_resources/tests/zip.py | 3 +++ setup.cfg | 1 + 2 files changed, 4 insertions(+) diff --git a/importlib_resources/tests/zip.py b/importlib_resources/tests/zip.py index 6ab914b..962195a 100755 --- a/importlib_resources/tests/zip.py +++ b/importlib_resources/tests/zip.py @@ -7,6 +7,8 @@ import pathlib import zipfile +import zipp + def make_zip_file(src, dst): """ @@ -16,6 +18,7 @@ def make_zip_file(src, dst): for src_path, rel in walk(src): dst_name = src.name / pathlib.PurePosixPath(rel.as_posix()) zf.write(src_path, dst_name) + zipp.CompleteDirs.inject(zf) return dst diff --git a/setup.cfg b/setup.cfg index 52ec4c2..571356d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,6 +44,7 @@ testing = pytest-ruff # local + zipp >= 3.17 docs = # upstream From 1d91410d0d709718d9f77fc729f9d3738c6fda1f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 12 Sep 2023 15:42:35 -0400 Subject: [PATCH 13/19] Add xfail tests for namespace packages in a zip, capturing missed expectation reported in python/importlib_resources#287. --- importlib_resources/tests/test_resource.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 4036805..850d029 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -3,6 +3,8 @@ import importlib_resources as resources import pathlib +import pytest + from . import data01 from . import util from importlib import import_module @@ -209,5 +211,14 @@ def tearDownClass(cls): sys.path.remove(cls.site_dir) +@pytest.mark.xfail +class ResourceFromNamespaceZipTests( + util.ZipSetupBase, + ResourceFromNamespaceTests, + unittest.TestCase, +): + ZIP_MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() From c02bc7ed1e1b1468cf0ba781ab01ab08916f1d2f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 19 Sep 2023 09:43:24 -0400 Subject: [PATCH 14/19] Update MultiplexedPath to expect Traversable and add a compatibility shim with deprecation warning. --- importlib_resources/_compat.py | 17 +++++++++++++++++ importlib_resources/readers.py | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/importlib_resources/_compat.py b/importlib_resources/_compat.py index a93a882..d7e9f0d 100644 --- a/importlib_resources/_compat.py +++ b/importlib_resources/_compat.py @@ -4,6 +4,7 @@ import os import sys import pathlib +import warnings from contextlib import suppress from typing import Union @@ -107,3 +108,19 @@ def wrap_spec(package): else: # PathLike is only subscriptable at runtime in 3.9+ StrPath = Union[str, "os.PathLike[str]"] + + +def ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/importlib_resources/readers.py b/importlib_resources/readers.py index 51d030a..dc1ec94 100644 --- a/importlib_resources/readers.py +++ b/importlib_resources/readers.py @@ -6,7 +6,7 @@ from . import abc from ._itertools import only -from ._compat import ZipPath +from ._compat import ZipPath, ensure_traversable def remove_duplicates(items): @@ -62,7 +62,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) From f004f042e9a418e1f4bc94b9cd20d426b7c6d74e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 19 Sep 2023 09:45:16 -0400 Subject: [PATCH 15/19] Update tests for MultiplexedPath to pass traversables, addressing some deprecation warnings. --- importlib_resources/tests/test_reader.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/importlib_resources/tests/test_reader.py b/importlib_resources/tests/test_reader.py index e2bdf19..a1eadb2 100644 --- a/importlib_resources/tests/test_reader.py +++ b/importlib_resources/tests/test_reader.py @@ -10,8 +10,7 @@ class MultiplexedPathTest(unittest.TestCase): @classmethod def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + cls.folder = pathlib.Path(__file__).parent / 'namespacedata01' def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +18,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -30,7 +29,7 @@ def test_iterdir(self): self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) + data01 = pathlib.Path(__file__).parent.joinpath('data01') contents = { path.name for path in MultiplexedPath(self.folder, data01).iterdir() } @@ -60,8 +59,8 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') + data01 = pathlib.Path(__file__).parent.joinpath('data01') + prefix = str(data01.parent) path = MultiplexedPath(self.folder, data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], @@ -82,9 +81,9 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') + data01 = pathlib.Path(__file__).parent.joinpath('data01') + data02 = pathlib.Path(__file__).parent.joinpath('data02') + prefix = str(data01.parent) path = MultiplexedPath(data01, data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( From f23b743f45fe75b6b10d004fd3ce69d181849ae4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 19 Sep 2023 10:02:24 -0400 Subject: [PATCH 16/19] Update changelog --- newsfragments/+13deb64a.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/+13deb64a.feature.rst diff --git a/newsfragments/+13deb64a.feature.rst b/newsfragments/+13deb64a.feature.rst new file mode 100644 index 0000000..a765911 --- /dev/null +++ b/newsfragments/+13deb64a.feature.rst @@ -0,0 +1 @@ +MultiplexedPath now expects Traversable paths. String arguments to MultiplexedPath are now deprecated. \ No newline at end of file From 463331b37e9811d4f231d2b70d17df7cea7a6551 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 19 Sep 2023 09:16:01 -0400 Subject: [PATCH 17/19] When constructing a MultiplexedPath, resolve submodule_search_locations to Traversable objects. Closes python/importlib_resources#287. --- importlib_resources/readers.py | 28 +++++++++++++++++++++- importlib_resources/tests/test_resource.py | 3 --- newsfragments/287.bugfix.rst | 1 + 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 newsfragments/287.bugfix.rst diff --git a/importlib_resources/readers.py b/importlib_resources/readers.py index dc1ec94..6a45a23 100644 --- a/importlib_resources/readers.py +++ b/importlib_resources/readers.py @@ -1,7 +1,9 @@ import collections +import contextlib import itertools import pathlib import operator +import re from . import abc @@ -130,7 +132,31 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*map(self._resolve, namespace_path)) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable: + """ + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir``. + """ + (dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return dir + + @classmethod + def _candidate_paths(cls, path_str): + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str): + for match in reversed(list(re.finditer('/', path_str))): + with contextlib.suppress(FileNotFoundError, IsADirectoryError): + inner = path_str[match.end() :] + yield ZipPath(path_str[: match.start()], inner + '/' * len(inner)) def resource_path(self, resource): """ diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 850d029..de7d734 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -3,8 +3,6 @@ import importlib_resources as resources import pathlib -import pytest - from . import data01 from . import util from importlib import import_module @@ -211,7 +209,6 @@ def tearDownClass(cls): sys.path.remove(cls.site_dir) -@pytest.mark.xfail class ResourceFromNamespaceZipTests( util.ZipSetupBase, ResourceFromNamespaceTests, diff --git a/newsfragments/287.bugfix.rst b/newsfragments/287.bugfix.rst new file mode 100644 index 0000000..9ef3c0b --- /dev/null +++ b/newsfragments/287.bugfix.rst @@ -0,0 +1 @@ +Enabled support for resources in namespace packages in zip files. From a9b0c92b303ba34a1631ce2f60c9ee16ea062d71 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Sep 2023 11:33:00 -0400 Subject: [PATCH 18/19] Honor backslashes in inner paths as found in submodule_search_locations. --- importlib_resources/readers.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/importlib_resources/readers.py b/importlib_resources/readers.py index 6a45a23..1e2d1ba 100644 --- a/importlib_resources/readers.py +++ b/importlib_resources/readers.py @@ -136,12 +136,12 @@ def __init__(self, namespace_path): @classmethod def _resolve(cls, path_str) -> abc.Traversable: - """ + r""" Given an item from a namespace path, resolve it to a Traversable. path_str might be a directory on the filesystem or a path to a zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or - ``/foo/baz.zip/inner_dir``. + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. """ (dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) return dir @@ -153,10 +153,12 @@ def _candidate_paths(cls, path_str): @staticmethod def _resolve_zip_path(path_str): - for match in reversed(list(re.finditer('/', path_str))): - with contextlib.suppress(FileNotFoundError, IsADirectoryError): - inner = path_str[match.end() :] - yield ZipPath(path_str[: match.start()], inner + '/' * len(inner)) + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, IsADirectoryError, PermissionError + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield ZipPath(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ From babc28763cd8e4f6fc4d6d67136ff51fb9182069 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Sep 2023 14:00:30 -0400 Subject: [PATCH 19/19] Finalize --- NEWS.rst | 15 +++++++++++++++ newsfragments/+13deb64a.feature.rst | 1 - newsfragments/287.bugfix.rst | 1 - 3 files changed, 15 insertions(+), 2 deletions(-) delete mode 100644 newsfragments/+13deb64a.feature.rst delete mode 100644 newsfragments/287.bugfix.rst diff --git a/NEWS.rst b/NEWS.rst index 5c97c82..83126c2 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,3 +1,18 @@ +v6.1.0 +====== + +Features +-------- + +- MultiplexedPath now expects Traversable paths. String arguments to MultiplexedPath are now deprecated. + + +Bugfixes +-------- + +- Enabled support for resources in namespace packages in zip files. (#287) + + v6.0.1 ====== diff --git a/newsfragments/+13deb64a.feature.rst b/newsfragments/+13deb64a.feature.rst deleted file mode 100644 index a765911..0000000 --- a/newsfragments/+13deb64a.feature.rst +++ /dev/null @@ -1 +0,0 @@ -MultiplexedPath now expects Traversable paths. String arguments to MultiplexedPath are now deprecated. \ No newline at end of file diff --git a/newsfragments/287.bugfix.rst b/newsfragments/287.bugfix.rst deleted file mode 100644 index 9ef3c0b..0000000 --- a/newsfragments/287.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Enabled support for resources in namespace packages in zip files.