diff --git a/.coveragerc b/.coveragerc index 45823064..054b5e59 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,19 @@ [run] -omit = .tox/* +omit = + # leading `*/` for pytest-dev/pytest-cov#456 + */.tox/* + */_itertools.py + */_legacy.py + */simple.py + */_path.py +disable_warnings = + couldnt-parse [report] show_missing = True +exclude_also = + # Exclude common false positives per + # https://coverage.readthedocs.io/en/latest/excluding.html#advanced-exclusion + # Ref jaraco/skeleton#97 and jaraco/skeleton#135 + class .*\bProtocol\): + if TYPE_CHECKING: diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..304196f8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.py] +indent_style = space +max_line_length = 88 + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 + +[*.rst] +indent_style = space diff --git a/.flake8 b/.flake8 deleted file mode 100644 index bc470da1..00000000 --- a/.flake8 +++ /dev/null @@ -1,13 +0,0 @@ -[flake8] -max-line-length = 88 - -# jaraco/skeleton#34 -max-complexity = 10 - -extend-ignore = - # Black creates whitespace before colon - E203 -exclude = - # Exclude the entire top-level __init__.py file since its only purpose is - # to expose the version string and to handle Python 2/3 compatibility. - importlib_resources/__init__.py diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..8ebda610 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +tidelift: pypi/importlib-resources diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index 4f70acfb..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: automerge -on: - pull_request: - types: - - labeled - - unlabeled - - synchronize - - opened - - edited - - ready_for_review - - reopened - - unlocked - pull_request_review: - types: - - submitted - check_suite: - types: - - completed - status: {} -jobs: - automerge: - runs-on: ubuntu-latest - steps: - - name: automerge - uses: "pascalgn/automerge-action@v0.12.0" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2d00d652..928acf2c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,59 +1,130 @@ name: tests -on: [push, pull_request] +on: + merge_group: + push: + branches-ignore: + # temporary GH branches relating to merge queues (jaraco/skeleton#93) + - gh-readonly-queue/** + tags: + # required if branches-ignore is supplied (jaraco/skeleton#103) + - '**' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +env: + # Environment variable to support color support (jaraco/skeleton#66) + FORCE_COLOR: 1 + + # Suppress noisy pip warnings + PIP_DISABLE_PIP_VERSION_CHECK: 'true' + PIP_NO_WARN_SCRIPT_LOCATION: 'true' + + # Ensure tests can sense settings about the environment + TOX_OVERRIDE: >- + testenv.pass_env+=GITHUB_*,FORCE_COLOR + jobs: test: strategy: + # https://blog.jaraco.com/efficient-use-of-ci-resources/ matrix: - python: [3.6, 3.8, 3.9] - platform: [ubuntu-latest, macos-latest, windows-latest] + python: + - "3.9" + - "3.13" + platform: + - ubuntu-latest + - macos-latest + - windows-latest + include: + - python: "3.10" + platform: ubuntu-latest + - python: "3.11" + platform: ubuntu-latest + - python: "3.12" + platform: ubuntu-latest + - python: "3.14" + platform: ubuntu-latest + - python: pypy3.10 + platform: ubuntu-latest runs-on: ${{ matrix.platform }} + continue-on-error: ${{ matrix.python == '3.14' }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + - name: Install build dependencies + # Install dependencies for building packages on pre-release Pythons + # jaraco/skeleton#161 + if: matrix.python == '3.14' && matrix.platform == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y libxml2-dev libxslt-dev - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Install tox - run: | - python -m pip install tox - - name: Run tests + run: python -m pip install tox + - name: Run run: tox - diffcov: + collateral: + strategy: + fail-fast: false + matrix: + job: + - diffcov + - docs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.x - name: Install tox - run: | - python -m pip install tox - - name: Evaluate coverage - run: tox - env: - TOXENV: diffcov + run: python -m pip install tox + - name: Eval ${{ matrix.job }} + run: tox -e ${{ matrix.job }} + + check: # This job does nothing and is only used for the branch protection + if: always() + + needs: + - test + - collateral + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} release: - needs: test + permissions: + contents: write + needs: + - check if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.x - name: Install tox - run: | - python -m pip install tox - - name: Release + run: python -m pip install tox + - name: Run run: tox -e release env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c15ab0c9..633e3648 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,7 @@ repos: -- repo: https://github.com/psf/black - rev: 20.8b1 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.9 hooks: - - id: black - -- repo: https://github.com/asottile/blacken-docs - rev: v1.9.1 - hooks: - - id: blacken-docs + - id: ruff + args: [--fix, --unsafe-fixes] + - id: ruff-format diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..72437063 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,19 @@ +version: 2 +python: + install: + - path: . + extra_requirements: + - doc + +sphinx: + configuration: docs/conf.py + +# required boilerplate readthedocs/readthedocs.org#10401 +build: + os: ubuntu-lts-latest + tools: + python: latest + # post-checkout job to ensure the clone isn't shallow jaraco/skeleton#114 + jobs: + post_checkout: + - git fetch --unshallow || true diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index cc698548..00000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -python: - install: - - path: . - extra_requirements: - - docs diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index ced83de8..00000000 --- a/CHANGES.rst +++ /dev/null @@ -1,271 +0,0 @@ -v5.0.7 -====== - -* bpo-45419: Correct ``DegenerateFiles.Path`` ``.name`` - and ``.open()`` interfaces to match ``Traversable``. - -v5.0.6 -====== - -* bpo-38693: Prefer f-strings to ``.format`` calls. - -v5.0.5 -====== - -* #216: Make MultiplexedPath.name a property per the - spec. - -v5.0.4 -====== - -* Fixed non-hermetic test in test_reader, revealed by - GH-24670. - -v5.0.3 -====== - -* Simplified DegenerateFiles.Path. - -v5.0.2 -====== - -* #214: Added ``_adapters`` module to ensure that degenerate - ``files`` behavior can be made available for legacy loaders - whose resource readers don't implement it. Fixes issue where - backport compatibility module was masking this fallback - behavior only to discover the defect when applying changes to - CPython. - -v5.0.1 -====== - -* Remove pyinstaller hook for hidden 'trees' module. - -v5.0.0 -====== - -* Removed ``importlib_resources.trees``, deprecated since 1.3.0. - -v4.1.1 -====== - -* Fixed badges in README. - -v4.1.0 -====== - -* #209: Adopt - `jaraco/skeleton `_. - -* Cleaned up some straggling Python 2 compatibility code. - -* Refreshed test zip files without .pyc and .pyo files. - -v4.0.0 -====== - -* #108: Drop support for Python 2.7. Now requires Python 3.6+. - -v3.3.1 -====== - -* Minor cleanup. - -v3.3.0 -====== - -* #107: Drop support for Python 3.5. Now requires Python 2.7 or 3.6+. - -v3.2.1 -====== - -* #200: Minor fixes and improved tests for namespace package support. - -v3.2.0 -====== - -* #68: Resources in PEP 420 Namespace packages are now supported. - -v3.1.1 -====== - -* bpo-41490: ``contents`` is now also more aggressive about - consuming any iterator from the ``Reader``. - -v3.1.0 -====== - -* #110 and bpo-41490: ``path`` method is more aggressive about - releasing handles to zipfile objects early, enabling use-cases - like ``certifi`` to leave the context open but delete the underlying - zip file. - -v3.0.0 -====== - -* Package no longer exposes ``importlib_resources.__version__``. - Users that wish to inspect the version of ``importlib_resources`` - should instead invoke ``.version('importlib_resources')`` from - ``importlib-metadata`` ( - `stdlib `_ - or `backport `_) - directly. This change eliminates the dependency on - ``importlib_metadata``. Closes #100. -* Package now always includes its data. Closes #93. -* Declare hidden imports for PyInstaller. Closes #101. - -v2.0.1 -====== - -* Select pathlib and contextlib imports based on Python version - and avoid pulling in deprecated - [pathlib](https://pypi.org/project/pathlib). Closes #97. - -v2.0.0 -====== - -* Loaders are no longer expected to implement the - ``abc.TraversableResources`` interface, but are instead - expected to return ``TraversableResources`` from their - ``get_resource_reader`` method. - -v1.5.0 -====== - -* Traversable is now a Protocol instead of an Abstract Base - Class (Python 2.7 and Python 3.8+). - -* Traversable objects now require a ``.name`` property. - -v1.4.0 -====== - -* #79: Temporary files created will now reflect the filename of - their origin. - -v1.3.1 -====== - -* For improved compatibility, ``importlib_resources.trees`` is - now imported implicitly. Closes #88. - -v1.3.0 -====== - -* Add extensibility support for non-standard loaders to supply - ``Traversable`` resources. Introduces a new abstract base - class ``abc.TraversableResources`` that supersedes (but - implements for compatibility) ``abc.ResourceReader``. Any - loader that implements (implicitly or explicitly) the - ``TraversableResources.files`` method will be capable of - supplying resources with subdirectory support. Closes #77. -* Preferred way to access ``as_file`` is now from top-level module. - ``importlib_resources.trees.as_file`` is deprecated and discouraged. - Closes #86. -* Moved ``Traversable`` abc to ``abc`` module. Closes #87. - -v1.2.0 -====== - -* Traversable now requires an ``open`` method. Closes #81. -* Fixed error on ``Python 3.5.{0,3}``. Closes #83. -* Updated packaging to resolve version from package metadata. - Closes #82. - -v1.1.0 -====== - -* Add support for retrieving resources from subdirectories of packages - through the new ``files()`` function, which returns a ``Traversable`` - object with ``joinpath`` and ``read_*`` interfaces matching those - of ``pathlib.Path`` objects. This new function supersedes all of the - previous functionality as it provides a more general-purpose access - to a package's resources. - - With this function, subdirectories are supported (Closes #58). - - The - documentation has been updated to reflect that this function is now - the preferred interface for loading package resources. It does not, - however, support resources from arbitrary loaders. It currently only - supports resources from file system path and zipfile packages (a - consequence of the ResourceReader interface only operating on - Python packages). - -1.0.2 -===== - -* Fix ``setup_requires`` and ``install_requires`` metadata in ``setup.cfg``. - Given by Anthony Sottile. - -1.0.1 -===== - -* Update Trove classifiers. Closes #63 - -1.0 -=== - -* Backport fix for test isolation from Python 3.8/3.7. Closes #61 - -0.8 -=== - -* Strip ``importlib_resources.__version__``. Closes #56 -* Fix a metadata problem with older setuptools. Closes #57 -* Add an ``__all__`` to ``importlib_resources``. Closes #59 - -0.7 -=== - -* Fix ``setup.cfg`` metadata bug. Closes #55 - -0.6 -=== - -* Move everything from ``pyproject.toml`` to ``setup.cfg``, with the added - benefit of fixing the PyPI metadata. Closes #54 -* Turn off mypy's ``strict_optional`` setting for now. - -0.5 -=== - -* Resynchronize with Python 3.7; changes the return type of ``contents()`` to - be an ``Iterable``. Closes #52 - -0.4 -=== - -* Correctly find resources in subpackages inside a zip file. Closes #51 - -0.3 -=== - -* The API, implementation, and documentation is synchronized with the Python - 3.7 standard library. Closes #47 -* When run under Python 3.7 this API shadows the stdlib versions. Closes #50 - -0.2 -=== - -* **Backward incompatible change**. Split the ``open()`` and ``read()`` calls - into separate binary and text versions, i.e. ``open_binary()``, - ``open_text()``, ``read_binary()``, and ``read_text()``. Closes #41 -* Fix a bug where unrelated resources could be returned from ``contents()``. - Closes #44 -* Correctly prevent namespace packages from containing resources. Closes #20 - -0.1 -=== - -* Initial release. - - -.. - Local Variables: - mode: change-log-mode - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 78 - coding: utf-8 - End: diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 378b991a..00000000 --- a/LICENSE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2017-2019 Brett Cannon, Barry Warsaw - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/NEWS.rst b/NEWS.rst new file mode 100644 index 00000000..a7c95e80 --- /dev/null +++ b/NEWS.rst @@ -0,0 +1,636 @@ +v6.5.2 +====== + +Bugfixes +-------- + +- Replaced reference to typing_extensions with stdlib Literal. (#323) + + +v6.5.1 +====== + +Bugfixes +-------- + +- Updated ``Traversable.read_text()`` to reflect the ``errors`` parameter (python/cpython#127012). (#321) + + +v6.5.0 +====== + +Features +-------- + +- Add type annotations for Traversable.open. (#317) +- Require Python 3.9 or later. + + +v6.4.5 +====== + +Bugfixes +-------- + +- Omit sentinel values from a namespace path. (#311) + + +v6.4.4 +====== + +No significant changes. + + +v6.4.3 +====== + +Bugfixes +-------- + +- When inferring the caller in ``files()`` correctly detect one's own module even when the resources package source is not present. (python/cpython#123085) + + +v6.4.2 +====== + +Bugfixes +-------- + +- Merged fix for UTF-16 BOM handling in functional tests. (#312) + + +v6.4.1 +====== + +Bugfixes +-------- + +- When constructing ZipReader, only append the name if the indicated module is a package. (python/cpython#121735) + + +v6.4.0 +====== + +Features +-------- + +- The functions + ``is_resource()``, + ``open_binary()``, + ``open_text()``, + ``path()``, + ``read_binary()``, and + ``read_text()`` are un-deprecated, and support + subdirectories via multiple positional arguments. + The ``contents()`` function also allows subdirectories, + but remains deprecated. (#303) +- Deferred select imports in for a speedup (python/cpython#109829). + + +v6.3.2 +====== + +Bugfixes +-------- + +- Restored expectation that local standard readers are preferred over degenerate readers. (#298) + + +v6.3.1 +====== + +Bugfixes +-------- + +- Restored expectation that stdlib readers are suppressed on Python 3.10. (#257) + + +v6.3.0 +====== + +Features +-------- + +- Add ``Anchor`` to ``importlib.resources`` (in order for the code to comply with the documentation) + + +v6.2.0 +====== + +Features +-------- + +- Future compatibility adapters now ensure that standard library readers are replaced without overriding non-standard readers. (#295) + + +v6.1.3 +====== + +No significant changes. + + +v6.1.2 +====== + +Bugfixes +-------- + +- Fixed NotADirectoryError when calling files on a subdirectory of a namespace package. (#293) + + +v6.1.1 +====== + +Bugfixes +-------- + +- Added missed stream argument in simple.ResourceHandle. Ref python/cpython#111775. + + +v6.1.0 +====== + +Features +-------- + +- MultiplexedPath now expects Traversable paths. String arguments to MultiplexedPath are now deprecated. + + +Bugfixes +-------- + +- Enabled support for resources in namespace packages in zip files. (#287) + + +v6.0.1 +====== + +Bugfixes +-------- + +- Restored Apache license. (#285) + + +v6.0.0 +====== + +Deprecations and Removals +------------------------- + +- Removed legacy functions deprecated in 5.3. (#80) + + +v5.13.0 +======= + +Features +-------- + +- Require Python 3.8 or later. + + +v5.12.0 +======= + +* #257: ``importlib_resources`` (backport) now gives + precedence to built-in readers (file system, zip, + namespace packages), providing forward-compatibility + of behaviors like ``MultiplexedPath``. + +v5.11.1 +======= + +v5.10.4 +======= + +* #280: Fixed one more ``EncodingWarning`` in test suite. + +v5.11.0 +======= + +* #265: ``MultiplexedPath`` now honors multiple subdirectories + in ``iterdir`` and ``joinpath``. + +v5.10.3 +======= + +* Packaging refresh, including fixing EncodingWarnings + and some tests cleanup. + +v5.10.2 +======= + +* #274: Prefer ``write_bytes`` to context manager as + proposed in gh-100586. + +v5.10.1 +======= + +* #274: Fixed ``ResourceWarning`` in ``_common``. + +v5.10.0 +======= + +* #203: Lifted restriction on modules passed to ``files``. + Now modules need not be a package and if a non-package + module is passed, resources will be resolved adjacent to + those modules, even for modules not found in any package. + For example, ``files(import_module('mod.py'))`` will + resolve resources found at the root. The parameter to + files was renamed from 'package' to 'anchor', with a + compatibility shim for those passing by keyword. + +* #259: ``files`` no longer requires the anchor to be + specified and can infer the anchor from the caller's scope + (defaults to the caller's module). + +v5.9.0 +====== + +* #228: ``as_file`` now also supports a ``Traversable`` + representing a directory and (when needed) renders the + full tree to a temporary directory. + +v5.8.1 +====== + +* #253: In ``MultiplexedPath``, restore expectation that + a compound path with a non-existent directory does not + raise an exception. + +v5.8.0 +====== + +* #250: Now ``Traversable.joinpath`` provides a concrete + implementation, replacing the implementation in ``.simple`` + and converging with the behavior in ``MultiplexedPath``. + +v5.7.1 +====== + +* #249: In ``simple.ResourceContainer.joinpath``, honor + names split by ``posixpath.sep``. + +v5.7.0 +====== + +* #248: ``abc.Traversable.joinpath`` now allows for multiple + arguments and specifies that ``posixpath.sep`` is allowed + in any argument to accept multiple arguments, matching the + behavior found in ``zipfile.Path`` and ``pathlib.Path``. + + ``simple.ResourceContainer`` now honors this behavior. + +v5.6.0 +====== + +* #244: Add type declarations in ABCs. + +v5.5.0 +====== + +* Require Python 3.7 or later. +* #243: Fix error when no ``__pycache__`` directories exist + when testing ``update-zips``. + +v5.4.0 +====== + +* #80: Test suite now relies entirely on the traversable + API. + +v5.3.0 +====== + +* #80: Now raise a ``DeprecationWarning`` for all legacy + functions. Instead, users should rely on the ``files()`` + API introduced in importlib_resources 1.3. See + `Migrating from Legacy `_ + for guidance on avoiding the deprecated functions. + +v5.2.3 +====== + +* Updated readme to reflect current behavior and show + which versions correspond to which behavior in CPython. + +v5.0.7 +====== + +* bpo-45419: Correct ``DegenerateFiles.Path`` ``.name`` + and ``.open()`` interfaces to match ``Traversable``. + +v5.2.2 +====== + +* #234: Fix refleak in ``as_file`` caught by CPython tests. + +v5.2.1 +====== + +* bpo-38291: Avoid DeprecationWarning on ``typing.io``. + +v5.2.0 +====== + +* #80 via #221: Legacy API (``path``, ``contents``, ...) + is now supported entirely by the ``.files()`` API with + a compatibility shim supplied for resource loaders without + that functionality. + +v5.0.6 +====== + +* bpo-38693: Prefer f-strings to ``.format`` calls. + +v5.1.4 +====== + +* #225: Require + `zipp 3.1.0 `_ + or later on Python prior to 3.10 to incorporate those fixes. + +v5.0.5 +====== + +* #216: Make MultiplexedPath.name a property per the + spec. + +v5.1.3 +====== + +* Refresh packaging and improve tests. +* #216: Make MultiplexedPath.name a property per the + spec. + +v5.1.2 +====== + +* Re-release with changes from 5.0.4. + +v5.0.4 +====== + +* Fixed non-hermetic test in test_reader, revealed by + GH-24670. + +v5.1.1 +====== + +* Re-release with changes from 5.0.3. + +v5.0.3 +====== + +* Simplified DegenerateFiles.Path. + +v5.0.2 +====== + +* #214: Added ``_adapters`` module to ensure that degenerate + ``files`` behavior can be made available for legacy loaders + whose resource readers don't implement it. Fixes issue where + backport compatibility module was masking this fallback + behavior only to discover the defect when applying changes to + CPython. + +v5.1.0 +====== + +* Added ``simple`` module implementing adapters from + a low-level resource reader interface to a + ``TraversableResources`` interface. Closes #90. + +v5.0.1 +====== + +* Remove pyinstaller hook for hidden 'trees' module. + +v5.0.0 +====== + +* Removed ``importlib_resources.trees``, deprecated since 1.3.0. + +v4.1.1 +====== + +* Fixed badges in README. + +v4.1.0 +====== + +* #209: Adopt + `jaraco/skeleton `_. + +* Cleaned up some straggling Python 2 compatibility code. + +* Refreshed test zip files without .pyc and .pyo files. + +v4.0.0 +====== + +* #108: Drop support for Python 2.7. Now requires Python 3.6+. + +v3.3.1 +====== + +* Minor cleanup. + +v3.3.0 +====== + +* #107: Drop support for Python 3.5. Now requires Python 2.7 or 3.6+. + +v3.2.1 +====== + +* #200: Minor fixes and improved tests for namespace package support. + +v3.2.0 +====== + +* #68: Resources in PEP 420 Namespace packages are now supported. + +v3.1.1 +====== + +* bpo-41490: ``contents`` is now also more aggressive about + consuming any iterator from the ``Reader``. + +v3.1.0 +====== + +* #110 and bpo-41490: ``path`` method is more aggressive about + releasing handles to zipfile objects early, enabling use-cases + like ``certifi`` to leave the context open but delete the underlying + zip file. + +v3.0.0 +====== + +* Package no longer exposes ``importlib_resources.__version__``. + Users that wish to inspect the version of ``importlib_resources`` + should instead invoke ``.version('importlib_resources')`` from + ``importlib-metadata`` ( + `stdlib `_ + or `backport `_) + directly. This change eliminates the dependency on + ``importlib_metadata``. Closes #100. +* Package now always includes its data. Closes #93. +* Declare hidden imports for PyInstaller. Closes #101. + +v2.0.1 +====== + +* Select pathlib and contextlib imports based on Python version + and avoid pulling in deprecated + [pathlib](https://pypi.org/project/pathlib). Closes #97. + +v2.0.0 +====== + +* Loaders are no longer expected to implement the + ``abc.TraversableResources`` interface, but are instead + expected to return ``TraversableResources`` from their + ``get_resource_reader`` method. + +v1.5.0 +====== + +* Traversable is now a Protocol instead of an Abstract Base + Class (Python 2.7 and Python 3.8+). + +* Traversable objects now require a ``.name`` property. + +v1.4.0 +====== + +* #79: Temporary files created will now reflect the filename of + their origin. + +v1.3.1 +====== + +* For improved compatibility, ``importlib_resources.trees`` is + now imported implicitly. Closes #88. + +v1.3.0 +====== + +* Add extensibility support for non-standard loaders to supply + ``Traversable`` resources. Introduces a new abstract base + class ``abc.TraversableResources`` that supersedes (but + implements for compatibility) ``abc.ResourceReader``. Any + loader that implements (implicitly or explicitly) the + ``TraversableResources.files`` method will be capable of + supplying resources with subdirectory support. Closes #77. +* Preferred way to access ``as_file`` is now from top-level module. + ``importlib_resources.trees.as_file`` is deprecated and discouraged. + Closes #86. +* Moved ``Traversable`` abc to ``abc`` module. Closes #87. + +v1.2.0 +====== + +* Traversable now requires an ``open`` method. Closes #81. +* Fixed error on ``Python 3.5.{0,3}``. Closes #83. +* Updated packaging to resolve version from package metadata. + Closes #82. + +v1.1.0 +====== + +* Add support for retrieving resources from subdirectories of packages + through the new ``files()`` function, which returns a ``Traversable`` + object with ``joinpath`` and ``read_*`` interfaces matching those + of ``pathlib.Path`` objects. This new function supersedes all of the + previous functionality as it provides a more general-purpose access + to a package's resources. + + With this function, subdirectories are supported (Closes #58). + + The + documentation has been updated to reflect that this function is now + the preferred interface for loading package resources. It does not, + however, support resources from arbitrary loaders. It currently only + supports resources from file system path and zipfile packages (a + consequence of the ResourceReader interface only operating on + Python packages). + +1.0.2 +===== + +* Fix ``setup_requires`` and ``install_requires`` metadata in ``setup.cfg``. + Given by Anthony Sottile. + +1.0.1 +===== + +* Update Trove classifiers. Closes #63 + +1.0 +=== + +* Backport fix for test isolation from Python 3.8/3.7. Closes #61 + +0.8 +=== + +* Strip ``importlib_resources.__version__``. Closes #56 +* Fix a metadata problem with older setuptools. Closes #57 +* Add an ``__all__`` to ``importlib_resources``. Closes #59 + +0.7 +=== + +* Fix ``setup.cfg`` metadata bug. Closes #55 + +0.6 +=== + +* Move everything from ``pyproject.toml`` to ``setup.cfg``, with the added + benefit of fixing the PyPI metadata. Closes #54 +* Turn off mypy's ``strict_optional`` setting for now. + +0.5 +=== + +* Resynchronize with Python 3.7; changes the return type of ``contents()`` to + be an ``Iterable``. Closes #52 + +0.4 +=== + +* Correctly find resources in subpackages inside a zip file. Closes #51 + +0.3 +=== + +* The API, implementation, and documentation is synchronized with the Python + 3.7 standard library. Closes #47 +* When run under Python 3.7 this API shadows the stdlib versions. Closes #50 + +0.2 +=== + +* **Backward incompatible change**. Split the ``open()`` and ``read()`` calls + into separate binary and text versions, i.e. ``open_binary()``, + ``open_text()``, ``read_binary()``, and ``read_text()``. Closes #41 +* Fix a bug where unrelated resources could be returned from ``contents()``. + Closes #44 +* Correctly prevent namespace packages from containing resources. Closes #20 + +0.1 +=== + +* Initial release. + + +.. + Local Variables: + mode: change-log-mode + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 78 + coding: utf-8 + End: diff --git a/README.rst b/README.rst index 65b799e3..30fa5b1e 100644 --- a/README.rst +++ b/README.rst @@ -1,31 +1,66 @@ .. image:: https://img.shields.io/pypi/v/importlib_resources.svg - :target: `PyPI link`_ + :target: https://pypi.org/project/importlib_resources .. image:: https://img.shields.io/pypi/pyversions/importlib_resources.svg - :target: `PyPI link`_ -.. _PyPI link: https://pypi.org/project/importlib_resources - -.. image:: https://github.com/python/importlib_resources/workflows/tests/badge.svg +.. image:: https://github.com/python/importlib_resources/actions/workflows/main.yml/badge.svg :target: https://github.com/python/importlib_resources/actions?query=workflow%3A%22tests%22 :alt: tests -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - :alt: Code style: Black +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff .. image:: https://readthedocs.org/projects/importlib-resources/badge/?version=latest :target: https://importlib-resources.readthedocs.io/en/latest/?badge=latest +.. image:: https://img.shields.io/badge/skeleton-2025-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/importlib-resources + :target: https://tidelift.com/subscription/pkg/pypi-importlib-resources?utm_source=pypi-importlib-resources&utm_medium=readme + ``importlib_resources`` is a backport of Python standard library `importlib.resources `_ -module for older Pythons. Users of Python 3.9 and beyond -should use the standard library module, since for these versions, -``importlib_resources`` just delegates to that module. +module for older Pythons. The key goal of this module is to replace parts of `pkg_resources `_ with a solution in Python's stdlib that relies on well-defined APIs. This makes reading resources included in packages easier, with more stable and consistent semantics. + +Compatibility +============= + +New features are introduced in this third-party library and later merged +into CPython. The following table indicates which versions of this library +were contributed to different versions in the standard library: + +.. list-table:: + :header-rows: 1 + + * - importlib_resources + - stdlib + * - 6.0 + - 3.13 + * - 5.12 + - 3.12 + * - 5.7 + - 3.11 + * - 5.0 + - 3.10 + * - 1.3 + - 3.9 + * - 0.5 (?) + - 3.7 + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more `_. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..54f99acb --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Contact + +To report a security vulnerability, please use the [Tidelift security contact](https://tidelift.com/security). Tidelift will coordinate the fix and disclosure. diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..739be9ed --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,26 @@ +============= +API Reference +============= + +``importlib_resources`` module +------------------------------ + +.. automodule:: importlib_resources + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: importlib_resources.abc + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: importlib_resources.readers + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: importlib_resources.simple + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index d24ee04b..570346b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,12 +1,17 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- +from __future__ import annotations -extensions = ['sphinx.ext.autodoc', 'jaraco.packaging.sphinx', 'rst.linker'] +extensions = [ + 'sphinx.ext.autodoc', + 'jaraco.packaging.sphinx', +] master_doc = "index" +html_theme = "furo" +# Link dates and other references in the changelog +extensions += ['rst.linker'] link_files = { - '../CHANGES.rst': dict( + '../NEWS.rst': dict( using=dict(GH='https://github.com'), replace=[ dict( @@ -19,12 +24,48 @@ ), dict( pattern=r'PEP[- ](?P\d+)', - url='https://www.python.org/dev/peps/pep-{pep_number:0>4}/', + url='https://peps.python.org/pep-{pep_number:0>4}/', ), dict( - pattern=r'(Python #|bpo-)(?P\d+)', - url='http://bugs.python.org/issue{python}', + pattern=r'(python/cpython#|Python #)(?P\d+)', + url='https://github.com/python/cpython/issues/{python}', + ), + dict( + pattern=r'bpo-(?P\d+)', + url='http://bugs.python.org/issue{bpo}', ), ], ), } + +# Be strict about any broken references +nitpicky = True +nitpick_ignore: list[tuple[str, str]] = [] + +# Include Python intersphinx mapping to prevent failures +# jaraco/skeleton#51 +extensions += ['sphinx.ext.intersphinx'] +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), +} + +# Preserve authored syntax for defaults +autodoc_preserve_defaults = True + +# Add support for linking usernames, PyPI projects, Wikipedia pages +github_url = 'https://github.com/' +extlinks = { + 'user': (f'{github_url}%s', '@%s'), + 'pypi': ('https://pypi.org/project/%s', '%s'), + 'wiki': ('https://wikipedia.org/wiki/%s', '%s'), +} +extensions += ['sphinx.ext.extlinks'] + +# local + +extensions += ['jaraco.tidelift'] + +nitpick_ignore.extend([ + ('py:class', 'module'), + ('py:class', '_io.BufferedReader'), +]) diff --git a/docs/history.rst b/docs/history.rst index 8e217503..5bdc2320 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -5,4 +5,4 @@ History ******* -.. include:: ../CHANGES (links).rst +.. include:: ../NEWS (links).rst diff --git a/docs/index.rst b/docs/index.rst index 82cb007a..eee051cc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,21 +1,26 @@ Welcome to |project| documentation! =================================== +.. sidebar-links:: + :home: + :pypi: + ``importlib_resources`` is a library which provides for access to *resources* in Python packages. It provides functionality similar to ``pkg_resources`` `Basic Resource Access`_ API, but without all of the overhead and performance problems of ``pkg_resources``. -In our terminology, a *resource* is a file tree that is located within an -importable `Python package`_. Resources can live on the file system or in a +In our terminology, a *resource* is a file tree that is located alongside an +importable `Python module`_. Resources can live on the file system or in a zip file, with support for other loader_ classes that implement the appropriate API for reading resources. -``importlib_resources`` is a backport of Python 3.9's standard library -`importlib.resources`_ module for Python 2.7, and 3.5 through 3.8. Users of -Python 3.9 and beyond are encouraged to use the standard library module. -Developers looking for detailed API descriptions should refer to the Python -3.9 standard library documentation. +``importlib_resources`` supplies a backport of :mod:`importlib.resources`, +enabling early access to features of future Python versions and making +functionality available for older Python versions. Users are encouraged to +use the Python standard library where suitable and fall back to +this library for future compatibility. Developers looking for detailed API +descriptions should refer to the standard library documentation. The documentation here includes a general :ref:`usage ` guide and a :ref:`migration ` guide for projects that want to adopt @@ -26,9 +31,12 @@ The documentation here includes a general :ref:`usage ` guide and a :maxdepth: 2 :caption: Contents: - using.rst - migration.rst - history.rst + using + api + migration + history + +.. tidelift-referral-banner:: Indices and tables @@ -39,7 +47,6 @@ Indices and tables * :ref:`search` -.. _`importlib.resources`: https://docs.python.org/3.7/library/importlib.html#module-importlib.resources .. _`Basic Resource Access`: http://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access -.. _`Python package`: https://docs.python.org/3/reference/import.html#packages +.. _`Python module`: https://docs.python.org/3/glossary.html#term-module .. _loader: https://docs.python.org/3/reference/import.html#finders-and-loaders diff --git a/docs/migration.rst b/docs/migration.rst index 04d059cc..5121d068 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -91,7 +91,7 @@ bytes. E.g.:: The equivalent code in ``importlib_resources`` is pretty straightforward:: ref = importlib_resources.files('my.package').joinpath('resource.dat') - with ref.open() as fp: + with ref.open('rb') as fp: my_bytes = fp.read() @@ -109,7 +109,7 @@ following example is often written for clarity as:: This can be easily rewritten like so:: ref = importlib_resources.files('my.package').joinpath('resource.dat') - contents = f.read_bytes() + contents = ref.read_bytes() pkg_resources.resource_listdir() @@ -148,7 +148,7 @@ a package is a directory or not:: The ``importlib_resources`` equivalent is straightforward:: - if importlib_resources.files('my.package').joinpath('resource').isdir(): + if importlib_resources.files('my.package').joinpath('resource').is_dir(): print('A directory') diff --git a/docs/using.rst b/docs/using.rst index c0a60ad1..0c66810f 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -5,11 +5,11 @@ =========================== ``importlib_resources`` is a library that leverages Python's import system to -provide access to *resources* within *packages*. Given that this library is -built on top of the import system, it is highly efficient and easy to use. -This library's philosophy is that, if you can import a package, you can access -resources within that package. Resources can be opened or read, in either -binary or text mode. +provide access to *resources* within *packages* and alongside *modules*. Given +that this library is built on top of the import system, it is highly efficient +and easy to use. This library's philosophy is that, if one can import a +module, one can access resources associated with that module. Resources can be +opened or read, in either binary or text mode. What exactly do we mean by "a resource"? It's easiest to think about the metaphor of files and directories on the file system, though it's important to @@ -23,15 +23,18 @@ If you have a file system layout such as:: one/ __init__.py resource1.txt + module1.py resources1/ resource1.1.txt two/ __init__.py resource2.txt + standalone.py + resource3.txt then the directories are ``data``, ``data/one``, and ``data/two``. Each of these are also Python packages by virtue of the fact that they all contain -``__init__.py`` files [#fn1]_. That means that in Python, all of these import +``__init__.py`` files. That means that in Python, all of these import statements work:: import data @@ -41,18 +44,21 @@ statements work:: Each import statement gives you a Python *module* corresponding to the ``__init__.py`` file in each of the respective directories. These modules are packages since packages are just special module instances that have an -additional attribute, namely a ``__path__`` [#fn2]_. +additional attribute, namely a ``__path__`` [#fn1]_. In this analogy then, resources are just files or directories contained in a package directory, so ``data/one/resource1.txt`` and ``data/two/resource2.txt`` are both resources, as are the ``__init__.py`` files in all the directories. -Resources are always accessed relative to the package that they live in. -``resource1.txt`` and ``resources1/resource1.1.txt`` are resources within -the ``data.one`` package, and -``two/resource2.txt`` is a resource within the -``data`` package. +Resources in packages are always accessed relative to the package that they +live in. ``resource1.txt`` and ``resources1/resource1.1.txt`` are resources +within the ``data.one`` package, and ``two/resource2.txt`` is a resource +within the ``data`` package. + +Resources may also be referenced relative to another *anchor*, a module in a +package (``data.one.module1``) or a standalone module (``standalone``). In +this case, resources are loaded from the same loader that loaded that module. Example @@ -103,34 +109,55 @@ using ``importlib_resources`` would look like:: eml = files('email.tests.data').joinpath('message.eml').read_text() -Packages or package names -========================= +Anchors +======= -All of the ``importlib_resources`` APIs take a *package* as their first -parameter, but this can either be a package name (as a ``str``) or an actual -module object, though the module *must* be a package [#fn3]_. If a string is -passed in, it must name an importable Python package, and this is first -imported. Thus the above example could also be written as:: +The ``importlib_resources`` ``files`` API takes an *anchor* as its first +parameter, which can either be a package name (as a ``str``) or an actual +module object. If a string is passed in, it must name an importable Python +module, which is imported prior to loading any resources. Thus the above +example could also be written as:: import email.tests.data eml = files(email.tests.data).joinpath('message.eml').read_text() +Namespace Packages +================== + +``importlib_resources`` supports namespace packages as anchors just like +any other package. Similar to modules in a namespace package, +resources in a namespace package are not allowed to collide by name. +For example, if two packages both expose ``nspkg/data/foo.txt``, those +resources are unsupported by this library. The package will also likely +experience problems due to the collision with installers. + +It's perfectly valid, however, for two packages to present different resources +in the same namespace package, regular package, or subdirectory. +For example, one package could expose ``nspkg/data/foo.txt`` and another +expose ``nspkg/data/bar.txt`` and those two packages could be installed +into separate paths, and the resources should be queryable:: + + data = importlib_resources.files('nspkg').joinpath('data') + data.joinpath('foo.txt').read_text() + data.joinpath('bar.txt').read_text() + + File system or zip file ======================= -In general you never have to worry whether your package is on the file system -or in a zip file, as the ``importlib_resources`` APIs hide those details from -you. Sometimes though, you need a path to an actual file on the file system. +A consumer need not worry whether any given package is on the file system +or in a zip file, as the ``importlib_resources`` APIs abstracts those details. +Sometimes though, the user needs a path to an actual file on the file system. For example, some SSL APIs require a certificate file to be specified by a real file system path, and C's ``dlopen()`` function also requires a real file system path. -To support this, ``importlib_resources`` provides an API that will extract the -resource from a zip file to a temporary file, and return the file system path -to this temporary file as a :py:class:`pathlib.Path` object. In order to -properly clean up this temporary file, what's actually returned is a context -manager that you can use in a ``with``-statement:: +To support this need, ``importlib_resources`` provides an API to extract the +resource from a zip file to a temporary file or folder and return the file +system path to this materialized resource as a :py:class:`pathlib.Path` +object. In order to properly clean up this temporary file, what's actually +returned is a context manager for use in a ``with``-statement:: from importlib_resources import files, as_file @@ -138,29 +165,24 @@ manager that you can use in a ``with``-statement:: with as_file(source) as eml: third_party_api_requiring_file_system_path(eml) -You can use all the standard :py:mod:`contextlib` APIs to manage this context -manager. +Use all the standard :py:mod:`contextlib` APIs to manage this context manager. -.. attention:: - There is an odd interaction with Python 3.4, 3.5, and 3.6 regarding adding - zip or wheel file paths to ``sys.path``. Due to limitations in `zipimport - `_, which can't be - changed without breaking backward compatibility, you **must** use an - absolute path to the zip/wheel file. If you use a relative path, you will - not be able to find resources inside these zip files. E.g.: +Migrating from Legacy +===================== - **No**:: +Starting with Python 3.9 and ``importlib_resources`` 1.4, this package +introduced the ``files()`` API, to be preferred over the legacy API, +i.e. the functions ``open_binary``, ``open_text``, ``path``, +``contents``, ``read_text``, ``read_binary``, and ``is_resource``. - sys.path.append('relative/path/to/foo.whl') - files('foo') # This will fail! - - **Yes**:: - - sys.path.append(os.path.abspath('relative/path/to/foo.whl')) - files('foo') - -Both relative and absolute paths work for Python 3.7 and newer. +To port to the ``files()`` API, refer to the +`_legacy module `_ +to see simple wrappers that enable drop-in replacement based on the +preferred API, and either copy those or adapt the usage to utilize the +``files`` and +`Traversable `_ +interfaces directly. Extending @@ -175,23 +197,11 @@ should return a ``TraversableResources`` instance. .. rubric:: Footnotes -.. [#fn1] We're ignoring `PEP 420 - `_ style namespace - packages, since ``importlib_resources`` does not support resources - within namespace packages. Also, the example assumes that the - parent directory containing ``data/`` is on ``sys.path``. - -.. [#fn2] As of `PEP 451 `_ this +.. [#fn1] As of `PEP 451 `_ this information is also available on the module's ``__spec__.submodule_search_locations`` attribute, which will not be ``None`` for packages. -.. [#fn3] Specifically, this means that in Python 2, the module object must - have an ``__path__`` attribute, while in Python 3, the module's - ``__spec__.submodule_search_locations`` must not be ``None``. - Otherwise a ``TypeError`` is raised. - - .. _`pkg_resources API`: http://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access .. _`loader`: https://docs.python.org/3/reference/import.html#finders-and-loaders .. _`ResourceReader`: https://docs.python.org/3.7/library/importlib.html#importlib.abc.ResourceReader diff --git a/importlib_resources/__init__.py b/importlib_resources/__init__.py index ea4c7efe..27d6c7f8 100644 --- a/importlib_resources/__init__.py +++ b/importlib_resources/__init__.py @@ -1,13 +1,19 @@ -"""Read resources contained within a package.""" +""" +Read resources contained within a package. + +This codebase is shared between importlib.resources in the stdlib +and importlib_resources in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" from ._common import ( + Anchor, + Package, as_file, files, ) - -from importlib_resources._py3 import ( - Package, - Resource, +from ._functional import ( contents, is_resource, open_binary, @@ -16,16 +22,15 @@ read_binary, read_text, ) -from importlib_resources.abc import ResourceReader - +from .abc import ResourceReader __all__ = [ 'Package', - 'Resource', + 'Anchor', 'ResourceReader', 'as_file', - 'contents', 'files', + 'contents', 'is_resource', 'open_binary', 'open_text', diff --git a/importlib_resources/_adapters.py b/importlib_resources/_adapters.py index e72edd10..50688fbb 100644 --- a/importlib_resources/_adapters.py +++ b/importlib_resources/_adapters.py @@ -1,4 +1,5 @@ from contextlib import suppress +from io import TextIOWrapper from . import abc @@ -25,33 +26,117 @@ def __init__(self, spec): self.spec = spec def get_resource_reader(self, name): - return DegenerateFiles(self.spec)._native() + return CompatibilityFiles(self.spec)._native() -class DegenerateFiles: +def _io_wrapper(file, mode='r', *args, **kwargs): + if mode == 'r': + return TextIOWrapper(file, *args, **kwargs) + elif mode == 'rb': + return file + raise ValueError(f"Invalid mode value '{mode}', only 'r' and 'rb' are supported") + + +class CompatibilityFiles: """ - Adapter for an existing or non-existant resource reader - to provide a degenerate .files(). + Adapter for an existing or non-existent resource reader + to provide a compatibility .files(). """ - class Path(abc.Traversable): + class SpecPath(abc.Traversable): + """ + Path tied to a module spec. + Can be read and exposes the resource reader children. + """ + + def __init__(self, spec, reader): + self._spec = spec + self._reader = reader + + def iterdir(self): + if not self._reader: + return iter(()) + return iter( + CompatibilityFiles.ChildPath(self._reader, path) + for path in self._reader.contents() + ) + + def is_file(self): + return False + + is_dir = is_file + + def joinpath(self, other): + if not self._reader: + return CompatibilityFiles.OrphanPath(other) + return CompatibilityFiles.ChildPath(self._reader, other) + + @property + def name(self): + return self._spec.name + + def open(self, mode='r', *args, **kwargs): + return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs) + + class ChildPath(abc.Traversable): + """ + Path tied to a resource reader child. + Can be read but doesn't expose any meaningful children. + """ + + def __init__(self, reader, name): + self._reader = reader + self._name = name + def iterdir(self): return iter(()) + def is_file(self): + return self._reader.is_resource(self.name) + def is_dir(self): + return not self.is_file() + + def joinpath(self, other): + return CompatibilityFiles.OrphanPath(self.name, other) + + @property + def name(self): + return self._name + + def open(self, mode='r', *args, **kwargs): + return _io_wrapper( + self._reader.open_resource(self.name), mode, *args, **kwargs + ) + + class OrphanPath(abc.Traversable): + """ + Orphan path, not tied to a module spec or resource reader. + Can't be read and doesn't expose any meaningful children. + """ + + def __init__(self, *path_parts): + if len(path_parts) < 1: + raise ValueError('Need at least one path part to construct a path') + self._path = path_parts + + def iterdir(self): + return iter(()) + + def is_file(self): return False - is_file = exists = is_dir # type: ignore + is_dir = is_file def joinpath(self, other): - return DegenerateFiles.Path() + return CompatibilityFiles.OrphanPath(*self._path, other) @property def name(self): - return '' + return self._path[-1] - def open(self, mode='rb', *args, **kwargs): - raise ValueError() + def open(self, mode='r', *args, **kwargs): + raise FileNotFoundError("Can't open orphan path") def __init__(self, spec): self.spec = spec @@ -72,7 +157,7 @@ def __getattr__(self, attr): return getattr(self._reader, attr) def files(self): - return DegenerateFiles.Path() + return CompatibilityFiles.SpecPath(self.spec, self._reader) def wrap_spec(package): diff --git a/importlib_resources/_common.py b/importlib_resources/_common.py index d3223bdd..5f41c265 100644 --- a/importlib_resources/_common.py +++ b/importlib_resources/_common.py @@ -1,42 +1,62 @@ +import contextlib +import functools +import importlib +import inspect +import itertools import os import pathlib import tempfile -import functools -import contextlib import types -import importlib +import warnings +from typing import Optional, Union, cast -from typing import Union, Any, Optional from .abc import ResourceReader, Traversable -from ._compat import wrap_spec - Package = Union[types.ModuleType, str] +Anchor = Package -def files(package): - # type: (Package) -> Traversable - """ - Get a Traversable resource from a package +def package_to_anchor(func): """ - return from_package(get_package(package)) + Replace 'package' parameter as 'anchor' and warn about the change. + Other errors should fall through. -def normalize_path(path): - # type: (Any) -> str - """Normalize a path by ensuring it is a string. + >>> files('a', 'b') + Traceback (most recent call last): + TypeError: files() takes from 0 to 1 positional arguments but 2 were given - If the resulting string contains path separators, an exception is raised. + Remove this compatibility in Python 3.14. + """ + undefined = object() + + @functools.wraps(func) + def wrapper(anchor=undefined, package=undefined): + if package is not undefined: + if anchor is not undefined: + return func(anchor, package) + warnings.warn( + "First parameter to files is renamed to 'anchor'", + DeprecationWarning, + stacklevel=2, + ) + return func(package) + elif anchor is undefined: + return func() + return func(anchor) + + return wrapper + + +@package_to_anchor +def files(anchor: Optional[Anchor] = None) -> Traversable: + """ + Get a Traversable resource for an anchor. """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name + return from_package(resolve(anchor)) -def get_resource_reader(package): - # type: (types.ModuleType) -> Optional[ResourceReader] +def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: """ Return the package's loader if it's a ResourceReader. """ @@ -46,64 +66,109 @@ def get_resource_reader(package): # zipimport.zipimporter does not support weak references, resulting in a # TypeError. That seems terrible. spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore[union-attr] if reader is None: return None - return reader(spec.name) # type: ignore + return reader(spec.name) # type: ignore[union-attr] -def resolve(cand): - # type: (Package) -> types.ModuleType - return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand) +@functools.singledispatch +def resolve(cand: Optional[Anchor]) -> types.ModuleType: + return cast(types.ModuleType, cand) + + +@resolve.register +def _(cand: str) -> types.ModuleType: + return importlib.import_module(cand) -def get_package(package): - # type: (Package) -> types.ModuleType - """Take a package name or module object and return the module. +@resolve.register +def _(cand: None) -> types.ModuleType: + return resolve(_infer_caller().f_globals['__name__']) - Raise an exception if the resolved module is not a package. + +def _infer_caller(): + """ + Walk the stack and find the frame of the first caller not in this module. """ - resolved = resolve(package) - if wrap_spec(resolved).submodule_search_locations is None: - raise TypeError(f'{package!r} is not a package') - return resolved + def is_this_file(frame_info): + return frame_info.filename == stack[0].filename -def from_package(package): + def is_wrapper(frame_info): + return frame_info.function == 'wrapper' + + stack = inspect.stack() + not_this_file = itertools.filterfalse(is_this_file, stack) + # also exclude 'wrapper' due to singledispatch in the call stack + callers = itertools.filterfalse(is_wrapper, not_this_file) + return next(callers).frame + + +def from_package(package: types.ModuleType): """ Return a Traversable object for the given package. """ + # deferred for performance (python/cpython#109829) + from .future.adapters import wrap_spec + spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() @contextlib.contextmanager -def _tempfile(reader, suffix=''): +def _tempfile( + reader, + suffix='', + # gh-93353: Keep a reference to call os.remove() in late Python + # finalization. + *, + _os_remove=os.remove, +): # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' # blocks due to the need to close the temporary file to work on Windows # properly. fd, raw_path = tempfile.mkstemp(suffix=suffix) try: - os.write(fd, reader()) - os.close(fd) + try: + os.write(fd, reader()) + finally: + os.close(fd) del reader yield pathlib.Path(raw_path) finally: try: - os.remove(raw_path) + _os_remove(raw_path) except FileNotFoundError: pass +def _temp_file(path): + return _tempfile(path.read_bytes, suffix=path.name) + + +def _is_present_dir(path: Traversable) -> bool: + """ + Some Traversables implement ``is_dir()`` to raise an + exception (i.e. ``FileNotFoundError``) when the + directory doesn't exist. This function wraps that call + to always return a boolean and only return True + if there's a dir and it exists. + """ + with contextlib.suppress(FileNotFoundError): + return path.is_dir() + return False + + @functools.singledispatch def as_file(path): """ Given a Traversable object, return that object as a path on the local file system in a context manager. """ - return _tempfile(path.read_bytes, suffix=path.name) + return _temp_dir(path) if _is_present_dir(path) else _temp_file(path) @as_file.register(pathlib.Path) @@ -113,3 +178,34 @@ def _(path): Degenerate behavior for pathlib.Path objects. """ yield path + + +@contextlib.contextmanager +def _temp_path(dir: tempfile.TemporaryDirectory): + """ + Wrap tempfile.TemporaryDirectory to return a pathlib object. + """ + with dir as result: + yield pathlib.Path(result) + + +@contextlib.contextmanager +def _temp_dir(path): + """ + Given a traversable dir, recursively replicate the whole tree + to the file system in a context manager. + """ + assert path.is_dir() + with _temp_path(tempfile.TemporaryDirectory()) as temp_dir: + yield _write_contents(temp_dir, path) + + +def _write_contents(target, source): + child = target.joinpath(source.name) + if source.is_dir(): + child.mkdir() + for item in source.iterdir(): + _write_contents(child, item) + else: + child.write_bytes(source.read_bytes()) + return child diff --git a/importlib_resources/_compat.py b/importlib_resources/_compat.py deleted file mode 100644 index deb071b9..00000000 --- a/importlib_resources/_compat.py +++ /dev/null @@ -1,87 +0,0 @@ -# flake8: noqa - -import abc -import sys -import pathlib -from contextlib import suppress - -try: - from zipfile import Path as ZipPath # type: ignore -except ImportError: - from zipp import Path as ZipPath # type: ignore - - -try: - from typing import runtime_checkable # type: ignore -except ImportError: - - def runtime_checkable(cls): # type: ignore - return cls - - -try: - from typing import Protocol # type: ignore -except ImportError: - Protocol = abc.ABC # type: ignore - - -class TraversableResourcesLoader: - """ - Adapt loaders to provide TraversableResources and other - compatibility. - """ - - def __init__(self, spec): - self.spec = spec - - @property - def path(self): - return self.spec.origin - - def get_resource_reader(self, name): - from . import readers, _adapters - - def _zip_reader(spec): - with suppress(AttributeError): - return readers.ZipReader(spec.loader, spec.name) - - def _namespace_reader(spec): - with suppress(AttributeError, ValueError): - return readers.NamespaceReader(spec.submodule_search_locations) - - def _available_reader(spec): - with suppress(AttributeError): - return spec.loader.get_resource_reader(spec.name) - - def _native_reader(spec): - reader = _available_reader(spec) - return reader if hasattr(reader, 'files') else None - - def _file_reader(spec): - if pathlib.Path(self.path).exists(): - return readers.FileReader(self) - - return ( - # native reader if it supplies 'files' - _native_reader(self.spec) - or - # local ZipReader if a zip module - _zip_reader(self.spec) - or - # local NamespaceReader if a namespace module - _namespace_reader(self.spec) - or - # local FileReader - _file_reader(self.spec) - or _adapters.DegenerateFiles(self.spec) - ) - - -def wrap_spec(package): - """ - Construct a package spec with traversable compatibility - on the spec/loader/reader. - """ - from . import _adapters - - return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/importlib_resources/_functional.py b/importlib_resources/_functional.py new file mode 100644 index 00000000..b08a5c6e --- /dev/null +++ b/importlib_resources/_functional.py @@ -0,0 +1,84 @@ +"""Simplified function-based API for importlib.resources""" + +import warnings + +from ._common import as_file, files +from .abc import TraversalError + +_MISSING = object() + + +def open_binary(anchor, *path_names): + """Open for binary reading the *resource* within *package*.""" + return _get_resource(anchor, path_names).open('rb') + + +def open_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Open for text reading the *resource* within *package*.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.open('r', encoding=encoding, errors=errors) + + +def read_binary(anchor, *path_names): + """Read and return contents of *resource* within *package* as bytes.""" + return _get_resource(anchor, path_names).read_bytes() + + +def read_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Read and return contents of *resource* within *package* as str.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.read_text(encoding=encoding, errors=errors) + + +def path(anchor, *path_names): + """Return the path to the *resource* as an actual file system path.""" + return as_file(_get_resource(anchor, path_names)) + + +def is_resource(anchor, *path_names): + """Return ``True`` if there is a resource named *name* in the package, + + Otherwise returns ``False``. + """ + try: + return _get_resource(anchor, path_names).is_file() + except TraversalError: + return False + + +def contents(anchor, *path_names): + """Return an iterable over the named resources within the package. + + The iterable returns :class:`str` resources (e.g. files). + The iterable does not recurse into subdirectories. + """ + warnings.warn( + "importlib.resources.contents is deprecated. " + "Use files(anchor).iterdir() instead.", + DeprecationWarning, + stacklevel=1, + ) + return (resource.name for resource in _get_resource(anchor, path_names).iterdir()) + + +def _get_encoding_arg(path_names, encoding): + # For compatibility with versions where *encoding* was a positional + # argument, it needs to be given explicitly when there are multiple + # *path_names*. + # This limitation can be removed in Python 3.15. + if encoding is _MISSING: + if len(path_names) > 1: + raise TypeError( + "'encoding' argument required with multiple path names", + ) + else: + return 'utf-8' + return encoding + + +def _get_resource(anchor, path_names): + if anchor is None: + raise TypeError("anchor must be module or string, got None") + return files(anchor).joinpath(*path_names) diff --git a/importlib_resources/_itertools.py b/importlib_resources/_itertools.py new file mode 100644 index 00000000..7b775ef5 --- /dev/null +++ b/importlib_resources/_itertools.py @@ -0,0 +1,38 @@ +# from more_itertools 9.0 +def only(iterable, default=None, too_long=None): + """If *iterable* has only one item, return it. + If it has zero items, return *default*. + If it has more than one item, raise the exception given by *too_long*, + which is ``ValueError`` by default. + >>> only([], default='missing') + 'missing' + >>> only([1]) + 1 + >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Expected exactly one item in iterable, but got 1, 2, + and perhaps more.' + >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError + Note that :func:`only` attempts to advance *iterable* twice to ensure there + is only one item. See :func:`spy` or :func:`peekable` to check + iterable contents less destructively. + """ + it = iter(iterable) + first_value = next(it, default) + + try: + second_value = next(it) + except StopIteration: + pass + else: + msg = ( + 'Expected exactly one item in iterable, but got {!r}, {!r}, ' + 'and perhaps more.'.format(first_value, second_value) + ) + raise too_long or ValueError(msg) + + return first_value diff --git a/importlib_resources/_py3.py b/importlib_resources/_py3.py deleted file mode 100644 index 17cc8a47..00000000 --- a/importlib_resources/_py3.py +++ /dev/null @@ -1,166 +0,0 @@ -import os -import io - -from . import _common -from contextlib import suppress -from importlib.abc import ResourceLoader -from importlib.machinery import ModuleSpec -from io import BytesIO, TextIOWrapper -from pathlib import Path -from types import ModuleType -from typing import ContextManager, Iterable, Union -from typing import cast -from typing.io import BinaryIO, TextIO -from collections.abc import Sequence -from functools import singledispatch - -Package = Union[str, ModuleType] -Resource = Union[str, os.PathLike] - - -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - resource = _common.normalize_path(resource) - package = _common.get_package(package) - reader = _common.get_resource_reader(package) - if reader is not None: - return reader.open_resource(resource) - spec = cast(ModuleSpec, package.__spec__) - # Using pathlib doesn't work well here due to the lack of 'strict' - # argument for pathlib.Path.resolve() prior to Python 3.6. - if spec.submodule_search_locations is not None: - paths = spec.submodule_search_locations - elif spec.origin is not None: - paths = [os.path.dirname(os.path.abspath(spec.origin))] - - for package_path in paths: - full_path = os.path.join(package_path, resource) - try: - return open(full_path, mode='rb') - except OSError: - # Just assume the loader is a resource loader; all the relevant - # importlib.machinery loaders are and an AttributeError for - # get_data() will make it clear what is needed from the loader. - loader = cast(ResourceLoader, spec.loader) - data = None - if hasattr(spec.loader, 'get_data'): - with suppress(OSError): - data = loader.get_data(full_path) - if data is not None: - return BytesIO(data) - - raise FileNotFoundError(f'{resource!r} resource not found in {spec.name!r}') - - -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return TextIOWrapper( - open_binary(package, resource), encoding=encoding, errors=errors - ) - - -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - with open_binary(package, resource) as fp: - return fp.read() - - -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -def path( - package: Package, - resource: Resource, -) -> 'ContextManager[Path]': - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - reader = _common.get_resource_reader(_common.get_package(package)) - return ( - _path_from_reader(reader, _common.normalize_path(resource)) - if reader - else _common.as_file( - _common.files(package).joinpath(_common.normalize_path(resource)) - ) - ) - - -def _path_from_reader(reader, resource): - return _path_from_resource_path(reader, resource) or _path_from_open_resource( - reader, resource - ) - - -def _path_from_resource_path(reader, resource): - with suppress(FileNotFoundError): - return Path(reader.resource_path(resource)) - - -def _path_from_open_resource(reader, resource): - saved = io.BytesIO(reader.open_resource(resource).read()) - return _common._tempfile(saved.read, suffix=resource) - - -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - package = _common.get_package(package) - _common.normalize_path(name) - reader = _common.get_resource_reader(package) - if reader is not None: - return reader.is_resource(name) - package_contents = set(contents(package)) - if name not in package_contents: - return False - return (_common.from_package(package) / name).is_file() - - -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - package = _common.get_package(package) - reader = _common.get_resource_reader(package) - if reader is not None: - return _ensure_sequence(reader.contents()) - transversable = _common.from_package(package) - if transversable.is_dir(): - return list(item.name for item in transversable.iterdir()) - return [] - - -@singledispatch -def _ensure_sequence(iterable): - return list(iterable) - - -@_ensure_sequence.register(Sequence) -def _(iterable): - return iterable diff --git a/importlib_resources/abc.py b/importlib_resources/abc.py index 56dc8127..883d3328 100644 --- a/importlib_resources/abc.py +++ b/importlib_resources/abc.py @@ -1,7 +1,26 @@ import abc -from typing import BinaryIO, Iterable, Text - -from ._compat import runtime_checkable, Protocol +import itertools +import os +import pathlib +from typing import ( + Any, + BinaryIO, + Iterable, + Iterator, + NoReturn, + Literal, + Optional, + Protocol, + Text, + TextIO, + Union, + overload, + runtime_checkable, +) + +StrPath = Union[str, os.PathLike[str]] + +__all__ = ["ResourceReader", "Traversable", "TraversableResources"] class ResourceReader(metaclass=abc.ABCMeta): @@ -46,37 +65,46 @@ def contents(self) -> Iterable[str]: raise FileNotFoundError +class TraversalError(Exception): + pass + + @runtime_checkable class Traversable(Protocol): """ An object with a subset of pathlib.Path methods suitable for traversing directories and opening files. + + Any exceptions that occur when accessing the backing resource + may propagate unaltered. """ @abc.abstractmethod - def iterdir(self): + def iterdir(self) -> Iterator["Traversable"]: """ Yield Traversable objects in self """ - def read_bytes(self): + def read_bytes(self) -> bytes: """ Read contents of self as bytes """ with self.open('rb') as strm: return strm.read() - def read_text(self, encoding=None): + def read_text( + self, encoding: Optional[str] = None, errors: Optional[str] = None + ) -> str: """ Read contents of self as text """ - with self.open(encoding=encoding) as strm: + with self.open(encoding=encoding, errors=errors) as strm: return strm.read() @abc.abstractmethod def is_dir(self) -> bool: """ - Return True if self is a dir + Return True if self is a directory """ @abc.abstractmethod @@ -85,20 +113,47 @@ def is_file(self) -> bool: Return True if self is a file """ - @abc.abstractmethod - def joinpath(self, child): + def joinpath(self, *descendants: StrPath) -> "Traversable": """ - Return Traversable child in self + Return Traversable resolved with any descendants applied. + + Each descendant should be a path segment relative to self + and each may contain multiple levels separated by + ``posixpath.sep`` (``/``). """ + if not descendants: + return self + names = itertools.chain.from_iterable( + path.parts for path in map(pathlib.PurePosixPath, descendants) + ) + target = next(names) + matches = ( + traversable for traversable in self.iterdir() if traversable.name == target + ) + try: + match = next(matches) + except StopIteration: + raise TraversalError( + "Target not found during traversal.", target, list(names) + ) + return match.joinpath(*names) - def __truediv__(self, child): + def __truediv__(self, child: StrPath) -> "Traversable": """ Return Traversable child in self """ return self.joinpath(child) + @overload + def open(self, mode: Literal['r'] = 'r', *args: Any, **kwargs: Any) -> TextIO: ... + + @overload + def open(self, mode: Literal['rb'], *args: Any, **kwargs: Any) -> BinaryIO: ... + @abc.abstractmethod - def open(self, mode='r', *args, **kwargs): + def open( + self, mode: str = 'r', *args: Any, **kwargs: Any + ) -> Union[TextIO, BinaryIO]: """ mode may be 'r' or 'rb' to open as text or binary. Return a handle suitable for reading (same as pathlib.Path.open). @@ -107,7 +162,8 @@ def open(self, mode='r', *args, **kwargs): accepted by io.TextIOWrapper. """ - @abc.abstractproperty + @property + @abc.abstractmethod def name(self) -> str: """ The base name of this object without any parent references. @@ -121,17 +177,17 @@ class TraversableResources(ResourceReader): """ @abc.abstractmethod - def files(self): + def files(self) -> "Traversable": """Return a Traversable object for the loaded package.""" - def open_resource(self, resource): + def open_resource(self, resource: StrPath) -> BinaryIO: return self.files().joinpath(resource).open('rb') - def resource_path(self, resource): + def resource_path(self, resource: Any) -> NoReturn: raise FileNotFoundError(resource) - def is_resource(self, path): + def is_resource(self, path: StrPath) -> bool: return self.files().joinpath(path).is_file() - def contents(self): + def contents(self) -> Iterator[str]: return (item.name for item in self.files().iterdir()) diff --git a/importlib_resources/tests/data01/__init__.py b/importlib_resources/compat/__init__.py similarity index 100% rename from importlib_resources/tests/data01/__init__.py rename to importlib_resources/compat/__init__.py diff --git a/importlib_resources/compat/py39.py b/importlib_resources/compat/py39.py new file mode 100644 index 00000000..684d3c63 --- /dev/null +++ b/importlib_resources/compat/py39.py @@ -0,0 +1,9 @@ +import sys + +__all__ = ['ZipPath'] + + +if sys.version_info >= (3, 10): + from zipfile import Path as ZipPath +else: + from zipp import Path as ZipPath diff --git a/importlib_resources/tests/data01/subdirectory/__init__.py b/importlib_resources/future/__init__.py similarity index 100% rename from importlib_resources/tests/data01/subdirectory/__init__.py rename to importlib_resources/future/__init__.py diff --git a/importlib_resources/future/adapters.py b/importlib_resources/future/adapters.py new file mode 100644 index 00000000..239e52b7 --- /dev/null +++ b/importlib_resources/future/adapters.py @@ -0,0 +1,102 @@ +import functools +import pathlib +from contextlib import suppress +from types import SimpleNamespace + +from .. import _adapters, readers + + +def _block_standard(reader_getter): + """ + Wrap _adapters.TraversableResourcesLoader.get_resource_reader + and intercept any standard library readers. + """ + + @functools.wraps(reader_getter) + def wrapper(*args, **kwargs): + """ + If the reader is from the standard library, return None to allow + allow likely newer implementations in this library to take precedence. + """ + try: + reader = reader_getter(*args, **kwargs) + except NotADirectoryError: + # MultiplexedPath may fail on zip subdirectory + return + except ValueError as exc: + # NamespaceReader in stdlib may fail for editable installs + # (python/importlib_resources#311, python/importlib_resources#318) + # Remove after bugfix applied to Python 3.13. + if "not enough values to unpack" not in str(exc): + raise + return + # Python 3.10+ + mod_name = reader.__class__.__module__ + if mod_name.startswith('importlib.') and mod_name.endswith('readers'): + return + # Python 3.8, 3.9 + if isinstance(reader, _adapters.CompatibilityFiles) and ( + reader.spec.loader.__class__.__module__.startswith('zipimport') + or reader.spec.loader.__class__.__module__.startswith( + '_frozen_importlib_external' + ) + ): + return + return reader + + return wrapper + + +def _skip_degenerate(reader): + """ + Mask any degenerate reader. Ref #298. + """ + is_degenerate = ( + isinstance(reader, _adapters.CompatibilityFiles) and not reader._reader + ) + return reader if not is_degenerate else None + + +class TraversableResourcesLoader(_adapters.TraversableResourcesLoader): + """ + Adapt loaders to provide TraversableResources and other + compatibility. + + Ensures the readers from importlib_resources are preferred + over stdlib readers. + """ + + def get_resource_reader(self, name): + return ( + _skip_degenerate(_block_standard(super().get_resource_reader)(name)) + or self._standard_reader() + or super().get_resource_reader(name) + ) + + def _standard_reader(self): + return self._zip_reader() or self._namespace_reader() or self._file_reader() + + def _zip_reader(self): + with suppress(AttributeError): + return readers.ZipReader(self.spec.loader, self.spec.name) + + def _namespace_reader(self): + with suppress(AttributeError, ValueError): + return readers.NamespaceReader(self.spec.submodule_search_locations) + + def _file_reader(self): + try: + path = pathlib.Path(self.spec.origin) + except TypeError: + return None + if path.exists(): + return readers.FileReader(SimpleNamespace(path=path)) + + +def wrap_spec(package): + """ + Override _adapters.wrap_spec to use TraversableResourcesLoader + from above. Ensures that future behavior is always available on older + Pythons. + """ + return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader) diff --git a/importlib_resources/readers.py b/importlib_resources/readers.py index c918ef82..99884b6a 100644 --- a/importlib_resources/readers.py +++ b/importlib_resources/readers.py @@ -1,9 +1,17 @@ +from __future__ import annotations + import collections +import contextlib +import itertools +import operator import pathlib +import re +import warnings +from collections.abc import Iterator from . import abc - -from ._compat import ZipPath +from ._itertools import only +from .compat.py39 import ZipPath def remove_duplicates(items): @@ -28,8 +36,10 @@ def files(self): class ZipReader(abc.TraversableResources): def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.prefix = loader.prefix.replace('\\', '/') + if loader.is_package(module): + _, _, name = module.rpartition('.') + self.prefix += name + '/' self.archive = loader.archive def open_resource(self, resource): @@ -39,8 +49,10 @@ def open_resource(self, resource): raise FileNotFoundError(exc.args[0]) def is_resource(self, path): - # workaround for `zipfile.Path.is_file` returning true - # for non-existent paths. + """ + Workaround for `zipfile.Path.is_file` returning true + for non-existent paths. + """ target = self.files().joinpath(path) return target.is_file() and target.exists() @@ -57,7 +69,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(_ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) @@ -65,13 +77,10 @@ def __init__(self, *paths): raise NotADirectoryError('MultiplexedPath only supports directories') def iterdir(self): - visited = [] - for path in self._paths: - for file in path.iterdir(): - if file.name in visited: - continue - visited.append(file.name) - yield file + children = (child for path in self._paths for child in path.iterdir()) + by_name = operator.attrgetter('name') + groups = itertools.groupby(sorted(children, key=by_name), key=by_name) + return map(self._follow, (locs for name, locs in groups)) def read_bytes(self): raise FileNotFoundError(f'{self} is not a file') @@ -85,15 +94,32 @@ def is_dir(self): def is_file(self): return False - def joinpath(self, child): - # first try to find child in current paths - for file in self.iterdir(): - if file.name == child: - return file - # if it does not exist, construct it with the first path - return self._paths[0] / child + def joinpath(self, *descendants): + try: + return super().joinpath(*descendants) + except abc.TraversalError: + # One of the paths did not resolve (a directory does not exist). + # Just return something that will not exist. + return self._paths[0].joinpath(*descendants) + + @classmethod + def _follow(cls, children): + """ + Construct a MultiplexedPath if needed. + + If children contains a sole element, return it. + Otherwise, return a MultiplexedPath of the items. + Unless one of the items is not a Directory, then return the first. + """ + subdirs, one_dir, one_file = itertools.tee(children, 3) - __truediv__ = joinpath + try: + return only(one_dir) + except ValueError: + try: + return cls(*subdirs) + except NotADirectoryError: + return next(one_file) def open(self, *args, **kwargs): raise FileNotFoundError(f'{self} is not a file') @@ -111,7 +137,40 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path))) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable | None: + r""" + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + + path_str might also be a sentinel used by editable packages to + trigger other behaviors (see python/importlib_resources#311). + In that case, return None. + """ + dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return next(dirs, None) + + @classmethod + def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]: + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str: str): + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield ZipPath(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ @@ -123,3 +182,21 @@ def resource_path(self, resource): def files(self): return self.path + + +def _ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + + Remove with Python 3.15. + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/importlib_resources/simple.py b/importlib_resources/simple.py new file mode 100644 index 00000000..2e75299b --- /dev/null +++ b/importlib_resources/simple.py @@ -0,0 +1,106 @@ +""" +Interface adapters for low-level readers. +""" + +import abc +import io +import itertools +from typing import BinaryIO, List + +from .abc import Traversable, TraversableResources + + +class SimpleReader(abc.ABC): + """ + The minimum, low-level interface required from a resource + provider. + """ + + @property + @abc.abstractmethod + def package(self) -> str: + """ + The name of the package for which this reader loads resources. + """ + + @abc.abstractmethod + def children(self) -> List['SimpleReader']: + """ + Obtain an iterable of SimpleReader for available + child containers (e.g. directories). + """ + + @abc.abstractmethod + def resources(self) -> List[str]: + """ + Obtain available named resources for this virtual package. + """ + + @abc.abstractmethod + def open_binary(self, resource: str) -> BinaryIO: + """ + Obtain a File-like for a named resource. + """ + + @property + def name(self): + return self.package.split('.')[-1] + + +class ResourceContainer(Traversable): + """ + Traversable container for a package's resources via its reader. + """ + + def __init__(self, reader: SimpleReader): + self.reader = reader + + def is_dir(self): + return True + + def is_file(self): + return False + + def iterdir(self): + files = (ResourceHandle(self, name) for name in self.reader.resources) + dirs = map(ResourceContainer, self.reader.children()) + return itertools.chain(files, dirs) + + def open(self, *args, **kwargs): + raise IsADirectoryError() + + +class ResourceHandle(Traversable): + """ + Handle to a named resource in a ResourceReader. + """ + + def __init__(self, parent: ResourceContainer, name: str): + self.parent = parent + self.name = name # type: ignore[misc] + + def is_file(self): + return True + + def is_dir(self): + return False + + def open(self, mode='r', *args, **kwargs): + stream = self.parent.reader.open_binary(self.name) + if 'b' not in mode: + stream = io.TextIOWrapper(stream, *args, **kwargs) + return stream + + def joinpath(self, name): + raise RuntimeError("Cannot traverse into a resource") + + +class TraversableReader(TraversableResources, SimpleReader): + """ + A TraversableResources based on SimpleReader. Resource providers + may derive from this class to provide the TraversableResources + interface by supplying the SimpleReader interface. + """ + + def files(self): + return ResourceContainer(self) diff --git a/importlib_resources/tests/_compat.py b/importlib_resources/tests/_compat.py deleted file mode 100644 index 4c99cffd..00000000 --- a/importlib_resources/tests/_compat.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - - -try: - from test.support import import_helper # type: ignore -except ImportError: - # Python 3.9 and earlier - class import_helper: # type: ignore - from test.support import modules_setup, modules_cleanup - - -try: - # Python 3.10 - from test.support.os_helper import unlink -except ImportError: - from test.support import unlink as _unlink - - def unlink(target): - return _unlink(os.fspath(target)) diff --git a/importlib_resources/tests/_path.py b/importlib_resources/tests/_path.py new file mode 100644 index 00000000..0033983d --- /dev/null +++ b/importlib_resources/tests/_path.py @@ -0,0 +1,90 @@ +import functools +import pathlib +from typing import Dict, Protocol, Union, runtime_checkable + +#### +# from jaraco.path 3.7.1 + + +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover + + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] +): + """ + Build a set of files/directories, as described by the spec. + + Each key represents a pathname, and the value represents + the content. Content may be a nested directory. + + >>> spec = { + ... 'README.txt': "A README file", + ... "foo": { + ... "__init__.py": "", + ... "bar": { + ... "__init__.py": "", + ... }, + ... "baz.py": "# Some code", + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), + ... } + >>> target = getfixture('tmp_path') + >>> build(spec, target) + >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') + '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' + """ + for name, contents in spec.items(): + create(contents, _ensure_tree_maker(prefix) / name) + + +@functools.singledispatch +def create(content: Union[str, bytes, FilesSpec], path): + path.mkdir(exist_ok=True) + build(content, prefix=path) # type: ignore[arg-type] + + +@create.register +def _(content: bytes, path): + path.write_bytes(content) + + +@create.register +def _(content: str, path): + path.write_text(content, encoding='utf-8') + + +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + +# end from jaraco.path +#### diff --git a/importlib_resources/tests/data02/__init__.py b/importlib_resources/tests/compat/__init__.py similarity index 100% rename from importlib_resources/tests/data02/__init__.py rename to importlib_resources/tests/compat/__init__.py diff --git a/importlib_resources/tests/compat/py312.py b/importlib_resources/tests/compat/py312.py new file mode 100644 index 00000000..ea9a58ba --- /dev/null +++ b/importlib_resources/tests/compat/py312.py @@ -0,0 +1,18 @@ +import contextlib + +from .py39 import import_helper + + +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = import_helper.modules_setup() + try: + yield + finally: + import_helper.modules_cleanup(saved) + + +vars(import_helper).setdefault('isolated_modules', isolated_modules) diff --git a/importlib_resources/tests/compat/py39.py b/importlib_resources/tests/compat/py39.py new file mode 100644 index 00000000..e01d276b --- /dev/null +++ b/importlib_resources/tests/compat/py39.py @@ -0,0 +1,13 @@ +""" +Backward-compatability shims to support Python 3.9 and earlier. +""" + +from jaraco.test.cpython import from_test_support, try_import + +import_helper = try_import('import_helper') or from_test_support( + 'modules_setup', 'modules_cleanup', 'DirsOnSysPath' +) +os_helper = try_import('os_helper') or from_test_support('temp_dir') +warnings_helper = try_import('warnings_helper') or from_test_support( + 'ignore_warnings', 'check_warnings' +) diff --git a/importlib_resources/tests/data01/binary.file b/importlib_resources/tests/data01/binary.file deleted file mode 100644 index eaf36c1d..00000000 Binary files a/importlib_resources/tests/data01/binary.file and /dev/null differ diff --git a/importlib_resources/tests/data01/subdirectory/binary.file b/importlib_resources/tests/data01/subdirectory/binary.file deleted file mode 100644 index eaf36c1d..00000000 Binary files a/importlib_resources/tests/data01/subdirectory/binary.file and /dev/null differ diff --git a/importlib_resources/tests/data01/utf-16.file b/importlib_resources/tests/data01/utf-16.file deleted file mode 100644 index 2cb77229..00000000 Binary files a/importlib_resources/tests/data01/utf-16.file and /dev/null differ diff --git a/importlib_resources/tests/data01/utf-8.file b/importlib_resources/tests/data01/utf-8.file deleted file mode 100644 index 1c0132ad..00000000 --- a/importlib_resources/tests/data01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/importlib_resources/tests/data02/one/__init__.py b/importlib_resources/tests/data02/one/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/data02/one/resource1.txt b/importlib_resources/tests/data02/one/resource1.txt deleted file mode 100644 index 61a813e4..00000000 --- a/importlib_resources/tests/data02/one/resource1.txt +++ /dev/null @@ -1 +0,0 @@ -one resource diff --git a/importlib_resources/tests/data02/two/__init__.py b/importlib_resources/tests/data02/two/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/data02/two/resource2.txt b/importlib_resources/tests/data02/two/resource2.txt deleted file mode 100644 index a80ce46e..00000000 --- a/importlib_resources/tests/data02/two/resource2.txt +++ /dev/null @@ -1 +0,0 @@ -two resource diff --git a/importlib_resources/tests/namespacedata01/binary.file b/importlib_resources/tests/namespacedata01/binary.file deleted file mode 100644 index eaf36c1d..00000000 Binary files a/importlib_resources/tests/namespacedata01/binary.file and /dev/null differ diff --git a/importlib_resources/tests/namespacedata01/utf-16.file b/importlib_resources/tests/namespacedata01/utf-16.file deleted file mode 100644 index 2cb77229..00000000 Binary files a/importlib_resources/tests/namespacedata01/utf-16.file and /dev/null differ diff --git a/importlib_resources/tests/namespacedata01/utf-8.file b/importlib_resources/tests/namespacedata01/utf-8.file deleted file mode 100644 index 1c0132ad..00000000 --- a/importlib_resources/tests/namespacedata01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/importlib_resources/tests/test_compatibilty_files.py b/importlib_resources/tests/test_compatibilty_files.py new file mode 100644 index 00000000..e8aac284 --- /dev/null +++ b/importlib_resources/tests/test_compatibilty_files.py @@ -0,0 +1,103 @@ +import io +import unittest + +import importlib_resources as resources +from importlib_resources._adapters import ( + CompatibilityFiles, + wrap_spec, +) + +from . import util + + +class CompatibilityFilesTests(unittest.TestCase): + @property + def package(self): + bytes_data = io.BytesIO(b'Hello, world!') + return util.create_package( + file=bytes_data, + path='some_path', + contents=('a', 'b', 'c'), + ) + + @property + def files(self): + return resources.files(self.package) + + def test_spec_path_iter(self): + self.assertEqual( + sorted(path.name for path in self.files.iterdir()), + ['a', 'b', 'c'], + ) + + def test_child_path_iter(self): + self.assertEqual(list((self.files / 'a').iterdir()), []) + + def test_orphan_path_iter(self): + self.assertEqual(list((self.files / 'a' / 'a').iterdir()), []) + self.assertEqual(list((self.files / 'a' / 'a' / 'a').iterdir()), []) + + def test_spec_path_is(self): + self.assertFalse(self.files.is_file()) + self.assertFalse(self.files.is_dir()) + + def test_child_path_is(self): + self.assertTrue((self.files / 'a').is_file()) + self.assertFalse((self.files / 'a').is_dir()) + + def test_orphan_path_is(self): + self.assertFalse((self.files / 'a' / 'a').is_file()) + self.assertFalse((self.files / 'a' / 'a').is_dir()) + self.assertFalse((self.files / 'a' / 'a' / 'a').is_file()) + self.assertFalse((self.files / 'a' / 'a' / 'a').is_dir()) + + def test_spec_path_name(self): + self.assertEqual(self.files.name, 'testingpackage') + + def test_child_path_name(self): + self.assertEqual((self.files / 'a').name, 'a') + + def test_orphan_path_name(self): + self.assertEqual((self.files / 'a' / 'b').name, 'b') + self.assertEqual((self.files / 'a' / 'b' / 'c').name, 'c') + + def test_spec_path_open(self): + self.assertEqual(self.files.read_bytes(), b'Hello, world!') + self.assertEqual(self.files.read_text(encoding='utf-8'), 'Hello, world!') + + def test_child_path_open(self): + self.assertEqual((self.files / 'a').read_bytes(), b'Hello, world!') + self.assertEqual( + (self.files / 'a').read_text(encoding='utf-8'), 'Hello, world!' + ) + + def test_orphan_path_open(self): + with self.assertRaises(FileNotFoundError): + (self.files / 'a' / 'b').read_bytes() + with self.assertRaises(FileNotFoundError): + (self.files / 'a' / 'b' / 'c').read_bytes() + + def test_open_invalid_mode(self): + with self.assertRaises(ValueError): + self.files.open('0') + + def test_orphan_path_invalid(self): + with self.assertRaises(ValueError): + CompatibilityFiles.OrphanPath() + + def test_wrap_spec(self): + spec = wrap_spec(self.package) + self.assertIsInstance(spec.loader.get_resource_reader(None), CompatibilityFiles) + + +class CompatibilityFilesNoReaderTests(unittest.TestCase): + @property + def package(self): + return util.create_package_from_loader(None) + + @property + def files(self): + return resources.files(self.package) + + def test_spec_path_joinpath(self): + self.assertIsInstance(self.files / 'a', CompatibilityFiles.OrphanPath) diff --git a/importlib_resources/tests/test_contents.py b/importlib_resources/tests/test_contents.py new file mode 100644 index 00000000..dcb872ec --- /dev/null +++ b/importlib_resources/tests/test_contents.py @@ -0,0 +1,39 @@ +import unittest + +import importlib_resources as resources + +from . import util + + +class ContentsTests: + expected = { + '__init__.py', + 'binary.file', + 'subdirectory', + 'utf-16.file', + 'utf-8.file', + } + + def test_contents(self): + contents = {path.name for path in resources.files(self.data).iterdir()} + assert self.expected <= contents + + +class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase): + pass + + +class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): + pass + + +class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + expected = { + # no __init__ because of namespace design + 'binary.file', + 'subdirectory', + 'utf-16.file', + 'utf-8.file', + } diff --git a/importlib_resources/tests/test_custom.py b/importlib_resources/tests/test_custom.py new file mode 100644 index 00000000..25ae0e75 --- /dev/null +++ b/importlib_resources/tests/test_custom.py @@ -0,0 +1,48 @@ +import contextlib +import pathlib +import unittest + +import importlib_resources as resources + +from .. import abc +from ..abc import ResourceReader, TraversableResources +from . import util +from .compat.py39 import os_helper + + +class SimpleLoader: + """ + A simple loader that only implements a resource reader. + """ + + def __init__(self, reader: ResourceReader): + self.reader = reader + + def get_resource_reader(self, package): + return self.reader + + +class MagicResources(TraversableResources): + """ + Magically returns the resources at path. + """ + + def __init__(self, path: pathlib.Path): + self.path = path + + def files(self): + return self.path + + +class CustomTraversableResourcesTests(unittest.TestCase): + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + def test_custom_loader(self): + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) + loader = SimpleLoader(MagicResources(temp_dir)) + pkg = util.create_package_from_loader(loader) + files = resources.files(pkg) + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/importlib_resources/tests/test_files.py b/importlib_resources/tests/test_files.py index d701d81f..be206603 100644 --- a/importlib_resources/tests/test_files.py +++ b/importlib_resources/tests/test_files.py @@ -1,10 +1,23 @@ -import typing +import contextlib +import importlib +import pathlib +import py_compile +import textwrap import unittest +import warnings import importlib_resources as resources -from importlib_resources.abc import Traversable -from . import data01 + +from ..abc import Traversable from . import util +from .compat.py39 import import_helper, os_helper + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx class FilesTests: @@ -18,22 +31,164 @@ def test_read_text(self): actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + + def test_old_parameter(self): + """ + Files used to take a 'package' parameter. Make sure anyone + passing by name is still supported. + """ + with suppress_known_deprecation(): + resources.files(package=self.data) -class OpenDiskTests(FilesTests, unittest.TestCase): - def setUp(self): - self.data = data01 + +class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): + pass class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_non_paths_in_dunder_path(self): + """ + Non-path items in a namespace package's ``__path__`` are ignored. + + As reported in python/importlib_resources#311, some tools + like Setuptools, when creating editable packages, will inject + non-paths into a namespace package's ``__path__``, a + sentinel like + ``__editable__.sample_namespace-1.0.finder.__path_hook__`` + to cause the ``PathEntryFinder`` to be called when searching + for packages. In that case, resources should still be loadable. + """ + import namespacedata01 # type: ignore[import-not-found] + + namespacedata01.__path__.append( + '__editable__.sample_namespace-1.0.finder.__path_hook__' + ) + + resources.files(namespacedata01) + + +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' + + +class DirectSpec: + """ + Override behavior of ModuleSetup to write a full spec directly. + """ + + MODULE = 'unused' + + def load_fixture(self, name): + self.tree_on_path(self.spec) + + +class ModulesFiles: + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } + + def test_module_resources(self): + """ + A module can have resources found adjacent to the module. + """ + import mod # type: ignore[import-not-found] + + actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') + assert actual == self.spec['res.txt'] + + +class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase): + pass + + +class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase): + pass + + +class ImplicitContextFiles: + set_val = textwrap.dedent( + f""" + import {resources.__name__} as res + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') + """ + ) + spec = { + 'somepkg': { + '__init__.py': set_val, + 'submod.py': set_val, + 'res.txt': 'resources are the best', + }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, + } + + def test_implicit_files_package(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg').val == 'resources are the best' + + def test_implicit_files_submodule(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg.submod').val == 'resources are the best' + + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix( + '.pyc' + ) + py_compile.compile(source_path, c_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + + +class ImplicitContextFilesDiskTests( + DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + +class ImplicitContextFilesZipTests( + DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + if __name__ == '__main__': unittest.main() diff --git a/importlib_resources/tests/test_functional.py b/importlib_resources/tests/test_functional.py new file mode 100644 index 00000000..9eb2d815 --- /dev/null +++ b/importlib_resources/tests/test_functional.py @@ -0,0 +1,267 @@ +import importlib +import os +import unittest + +import importlib_resources as resources + +from . import util +from .compat.py39 import warnings_helper + + +class StringAnchorMixin: + anchor01 = 'data01' + anchor02 = 'data02' + + +class ModuleAnchorMixin: + @property + def anchor01(self): + return importlib.import_module('data01') + + @property + def anchor02(self): + return importlib.import_module('data02') + + +class FunctionalAPIBase: + def setUp(self): + super().setUp() + self.load_fixture('data02') + + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest""" + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def assertEndsWith(self, string, suffix): + """Assert that `string` ends with `suffix`. + + Used to ignore an architecture-specific UTF-16 byte-order mark.""" + self.assertEqual(string[-len(suffix) :], suffix) + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises((OSError, resources.abc.TraversalError)): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEndsWith( # ignore the BOM + resources.read_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises((OSError, resources.abc.TraversalError)): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEndsWith( # ignore the BOM + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, + *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with ( + self.assertRaises(OSError), + warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )), + ): + list(resources.contents(self.anchor01, 'utf-8.file')) + + for path_parts in self._gen_resourcetxt_path_parts(): + with ( + self.assertRaises((OSError, resources.abc.TraversalError)), + warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )), + ): + list(resources.contents(self.anchor01, *path_parts)) + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor_Disk( + StringAnchorMixin, + FunctionalAPIBase, + util.DiskSetup, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_ModuleAnchor_Disk( + ModuleAnchorMixin, + FunctionalAPIBase, + util.DiskSetup, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_StringAnchor_Memory( + StringAnchorMixin, + FunctionalAPIBase, + util.MemorySetup, + unittest.TestCase, +): + pass diff --git a/importlib_resources/tests/test_open.py b/importlib_resources/tests/test_open.py index 919fc0db..8a4b68e3 100644 --- a/importlib_resources/tests/test_open.py +++ b/importlib_resources/tests/test_open.py @@ -1,43 +1,51 @@ import unittest import importlib_resources as resources -from . import data01 + from . import util class CommonBinaryTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): - with resources.open_binary(package, path): + target = resources.files(package).joinpath(path) + with target.open('rb'): pass class CommonTextTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): - with resources.open_text(package, path): + target = resources.files(package).joinpath(path) + with target.open(encoding='utf-8'): pass class OpenTests: def test_open_binary(self): - with resources.open_binary(self.data, 'utf-8.file') as fp: + target = resources.files(self.data) / 'binary.file' + with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'Hello, UTF-8 world!\n') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): - with resources.open_text(self.data, 'utf-8.file') as fp: + target = resources.files(self.data) / 'utf-8.file' + with target.open(encoding='utf-8') as fp: result = fp.read() self.assertEqual(result, 'Hello, UTF-8 world!\n') def test_open_text_given_encoding(self): - with resources.open_text(self.data, 'utf-16.file', 'utf-16', 'strict') as fp: + target = resources.files(self.data) / 'utf-16.file' + with target.open(encoding='utf-16', errors='strict') as fp: result = fp.read() self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_open_text_with_errors(self): - # Raises UnicodeError without the 'errors' argument. - with resources.open_text(self.data, 'utf-16.file', 'utf-8', 'strict') as fp: + """ + Raises UnicodeError without the 'errors' argument. + """ + target = resources.files(self.data) / 'utf-16.file' + with target.open(encoding='utf-8', errors='strict') as fp: self.assertRaises(UnicodeError, fp.read) - with resources.open_text(self.data, 'utf-16.file', 'utf-8', 'ignore') as fp: + with target.open(encoding='utf-8', errors='ignore') as fp: result = fp.read() self.assertEqual( result, @@ -47,31 +55,31 @@ def test_open_text_with_errors(self): ) def test_open_binary_FileNotFoundError(self): - self.assertRaises( - FileNotFoundError, resources.open_binary, self.data, 'does-not-exist' - ) + target = resources.files(self.data) / 'does-not-exist' + with self.assertRaises(FileNotFoundError): + target.open('rb') def test_open_text_FileNotFoundError(self): - self.assertRaises( - FileNotFoundError, resources.open_text, self.data, 'does-not-exist' - ) - + target = resources.files(self.data) / 'does-not-exist' + with self.assertRaises(FileNotFoundError): + target.open(encoding='utf-8') -class OpenDiskTests(OpenTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase): + pass -class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/importlib_resources/tests/test_path.py b/importlib_resources/tests/test_path.py index 9180626f..0be673d2 100644 --- a/importlib_resources/tests/test_path.py +++ b/importlib_resources/tests/test_path.py @@ -1,40 +1,39 @@ import io +import pathlib import unittest import importlib_resources as resources -from . import data01 + from . import util class CommonTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): - with resources.path(package, path): + with resources.as_file(resources.files(package).joinpath(path)): pass class PathTests: def test_reading(self): - # Path should be readable. - # Test also implicitly verifies the returned object is a pathlib.Path - # instance. - with resources.path(self.data, 'utf-8.file') as path: + """ + Path should be readable and a pathlib.Path instance. + """ + target = resources.files(self.data) / 'utf-8.file' + with resources.as_file(target) as path: + self.assertIsInstance(path, pathlib.Path) self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) -class PathDiskTests(PathTests, unittest.TestCase): - data = data01 - +class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): def test_natural_path(self): """ Guarantee the internal implementation detail that file-system-backed resources do not get the tempdir treatment. """ - with resources.path(self.data, 'utf-8.file') as path: + target = resources.files(self.data) / 'utf-8.file' + with resources.as_file(target) as path: assert 'data' in str(path) @@ -51,9 +50,12 @@ def setUp(self): class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): def test_remove_in_context_manager(self): - # It is not an error if the file that was temporarily stashed on the - # file system is removed inside the `with` stanza. - with resources.path(self.data, 'utf-8.file') as path: + """ + It is not an error if the file that was temporarily stashed on the + file system is removed inside the `with` stanza. + """ + target = resources.files(self.data) / 'utf-8.file' + with resources.as_file(target) as path: path.unlink() diff --git a/importlib_resources/tests/test_read.py b/importlib_resources/tests/test_read.py index 5422eea3..216c8feb 100644 --- a/importlib_resources/tests/test_read.py +++ b/importlib_resources/tests/test_read.py @@ -1,38 +1,49 @@ import unittest +from importlib import import_module + import importlib_resources as resources -from . import data01 from . import util -from importlib import import_module class CommonBinaryTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): - resources.read_binary(package, path) + resources.files(package).joinpath(path).read_bytes() class CommonTextTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): - resources.read_text(package, path) + resources.files(package).joinpath(path).read_text(encoding='utf-8') class ReadTests: - def test_read_binary(self): - result = resources.read_binary(self.data, 'binary.file') - self.assertEqual(result, b'\0\1\2\3') + def test_read_bytes(self): + result = resources.files(self.data).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): - result = resources.read_text(self.data, 'utf-8.file') + result = ( + resources.files(self.data) + .joinpath('utf-8.file') + .read_text(encoding='utf-8') + ) self.assertEqual(result, 'Hello, UTF-8 world!\n') def test_read_text_given_encoding(self): - result = resources.read_text(self.data, 'utf-16.file', encoding='utf-16') + result = ( + resources.files(self.data) + .joinpath('utf-16.file') + .read_text(encoding='utf-16') + ) self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_read_text_with_errors(self): - # Raises UnicodeError without the 'errors' argument. - self.assertRaises(UnicodeError, resources.read_text, self.data, 'utf-16.file') - result = resources.read_text(self.data, 'utf-16.file', errors='ignore') + """ + Raises UnicodeError without the 'errors' argument. + """ + target = resources.files(self.data) / 'utf-16.file' + self.assertRaises(UnicodeError, target.read_text, encoding='utf-8') + result = target.read_text(encoding='utf-8', errors='ignore') self.assertEqual( result, 'H\x00e\x00l\x00l\x00o\x00,\x00 ' @@ -41,19 +52,42 @@ def test_read_text_with_errors(self): ) -class ReadDiskTests(ReadTests, unittest.TestCase): - data = data01 +class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase): + pass class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') - result = resources.read_binary(submodule, 'binary.file') - self.assertEqual(result, b'\0\1\2\3') + submodule = import_module('data01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): - result = resources.read_binary('ziptestdata.subdirectory', 'binary.file') - self.assertEqual(result, b'\0\1\2\3') + result = ( + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() + ) + self.assertEqual(result, bytes(range(4, 8))) + + +class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') + .joinpath('binary.file') + .read_bytes() + ) + self.assertEqual(result, bytes(range(12, 16))) if __name__ == '__main__': diff --git a/importlib_resources/tests/test_reader.py b/importlib_resources/tests/test_reader.py index 16841a50..f8cfd8de 100644 --- a/importlib_resources/tests/test_reader.py +++ b/importlib_resources/tests/test_reader.py @@ -1,17 +1,21 @@ import os.path -import sys import pathlib import unittest - from importlib import import_module + from importlib_resources.readers import MultiplexedPath, NamespaceReader +from . import util + -class MultiplexedPathTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) +class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def setUp(self): + super().setUp() + self.folder = pathlib.Path(self.data.__path__[0]) + self.data01 = pathlib.Path(self.load_fixture('data01').__file__).parent + self.data02 = pathlib.Path(self.load_fixture('data02').__file__).parent def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +23,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,12 +31,13 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) contents = { - path.name for path in MultiplexedPath(self.folder, data01).iterdir() + path.name for path in MultiplexedPath(self.folder, self.data01).iterdir() } for remove in ('__pycache__', '__init__.pyc'): try: @@ -60,21 +65,34 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - path = MultiplexedPath(self.folder, data01) + prefix = str(self.folder.parent) + path = MultiplexedPath(self.folder, self.data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), ) + self.assertEqual(path.joinpath(), path) + + def test_join_path_compound(self): + path = MultiplexedPath(self.folder) + assert not path.joinpath('imaginary/foo.py').exists() + + def test_join_path_common_subdir(self): + prefix = str(self.data02.parent) + path = MultiplexedPath(self.data01, self.data02) + self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) + self.assertEqual( + str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], + os.path.join('data02', 'subdirectory', 'subsubdir'), + ) def test_repr(self): self.assertEqual( @@ -89,16 +107,8 @@ def test_name(self): ) -class NamespaceReaderTest(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' def test_init_error(self): with self.assertRaises(ValueError): @@ -108,7 +118,7 @@ def test_resource_path(self): namespacedata01 = import_module('namespacedata01') reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + root = self.data.__path__[0] self.assertEqual( reader.resource_path('binary.file'), os.path.join(root, 'binary.file') ) @@ -117,9 +127,8 @@ def test_resource_path(self): ) def test_files(self): - namespacedata01 = import_module('namespacedata01') - reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + reader = NamespaceReader(self.data.__spec__.submodule_search_locations) + root = self.data.__path__[0] self.assertIsInstance(reader.files(), MultiplexedPath) self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 75748e65..c80afdc7 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -1,92 +1,82 @@ -import sys import unittest +from importlib import import_module + import importlib_resources as resources -import uuid -import pathlib -from . import data01 -from . import zipdata01, zipdata02 from . import util -from importlib import import_module -from ._compat import import_helper, unlink class ResourceTests: # Subclasses are expected to set the `data` attribute. - def test_is_resource_good_path(self): - self.assertTrue(resources.is_resource(self.data, 'binary.file')) + def test_is_file_exists(self): + target = resources.files(self.data) / 'binary.file' + self.assertTrue(target.is_file()) - def test_is_resource_missing(self): - self.assertFalse(resources.is_resource(self.data, 'not-a-file')) + def test_is_file_missing(self): + target = resources.files(self.data) / 'not-a-file' + self.assertFalse(target.is_file()) - def test_is_resource_subresource_directory(self): - # Directories are not resources. - self.assertFalse(resources.is_resource(self.data, 'subdirectory')) - - def test_contents(self): - contents = set(resources.contents(self.data)) - # There may be cruft in the directory listing of the data directory. - # It could have a __pycache__ directory, - # an artifact of the - # test suite importing these modules, which - # are not germane to this test, so just filter them out. - contents.discard('__pycache__') - self.assertEqual( - contents, - { - '__init__.py', - 'subdirectory', - 'utf-8.file', - 'binary.file', - 'utf-16.file', - }, - ) + def test_is_dir(self): + target = resources.files(self.data) / 'subdirectory' + self.assertFalse(target.is_file()) + self.assertTrue(target.is_dir()) -class ResourceDiskTests(ResourceTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): + pass class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): pass -class ResourceLoaderTests(unittest.TestCase): +def names(traversable): + return {item.name for item in traversable.iterdir()} + + +class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) - self.assertEqual(set(resources.contents(package)), {'A', 'B', 'C'}) + self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) - def test_resource_is_resource(self): + def test_is_file(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertTrue(resources.is_resource(package, 'B')) + self.assertTrue(resources.files(package).joinpath('B').is_file()) - def test_resource_directory_is_not_resource(self): + def test_is_dir(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertFalse(resources.is_resource(package, 'D')) + self.assertTrue(resources.files(package).joinpath('D').is_dir()) - def test_resource_missing_is_not_resource(self): + def test_resource_missing(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertFalse(resources.is_resource(package, 'Z')) + self.assertFalse(resources.files(package).joinpath('Z').is_file()) -class ResourceCornerCaseTests(unittest.TestCase): +class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): def test_package_has_no_reader_fallback(self): - # Test odd ball packages which: + """ + Test odd ball packages which: # 1. Do not have a ResourceReader as a loader # 2. Are not on the file system # 3. Are not in a zip file + """ module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) # Give the module a dummy loader. module.__loader__ = object() @@ -94,36 +84,42 @@ def test_package_has_no_reader_fallback(self): module.__file__ = '/path/which/shall/not/be/named' module.__spec__.loader = module.__loader__ module.__spec__.origin = module.__file__ - self.assertFalse(resources.is_resource(module, 'A')) - + self.assertFalse(resources.files(module).joinpath('A').is_file()) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore +class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') - self.assertTrue(resources.is_resource(submodule, 'binary.file')) + submodule = import_module('data01.subdirectory') + self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.is_resource('ziptestdata.subdirectory', 'binary.file') + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( - set(resources.contents(submodule)), {'__init__.py', 'binary.file'} + names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - set(resources.contents('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) + def test_as_file_directory(self): + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' + assert data.is_dir() + assert data.joinpath('subdirectory').is_dir() + assert len(list(data.iterdir())) + assert not data.parent.exists() + -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore +class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): + MODULE = 'data02' def test_unrelated_contents(self): """ @@ -131,123 +127,111 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - set(resources.contents('ziptestdata.one')), {'__init__.py', 'resource1.txt'} + names(resources.files('data02.one')), + {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - set(resources.contents('ziptestdata.two')), {'__init__.py', 'resource2.txt'} + names(resources.files('data02.two')), + {'__init__.py', 'resource2.txt'}, ) -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetup, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - ZIP_MODULE = zipdata01 - - def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - data_path = pathlib.Path(self.ZIP_MODULE.__file__) - data_dir = data_path.parent - self.source_zip_path = data_dir / 'ziptestdata.zip' - self.zip_path = pathlib.Path(f'{uuid.uuid4()}.zip').absolute() - self.zip_path.write_bytes(self.source_zip_path.read_bytes()) - sys.path.append(str(self.zip_path)) - self.data = import_module('ziptestdata') - - def tearDown(self): - try: - sys.path.remove(str(self.zip_path)) - except ValueError: - pass - - try: - del sys.path_importer_cache[str(self.zip_path)] - del sys.modules[self.data.__name__] - except KeyError: - pass - - try: - unlink(self.zip_path) - except OSError: - # If the test fails, this will probably fail too - pass - - def test_contents_does_not_keep_open(self): - c = resources.contents('ziptestdata') - self.zip_path.unlink() - del c + def test_iterdir_does_not_keep_open(self): + [item.name for item in resources.files('data01').iterdir()] - def test_is_resource_does_not_keep_open(self): - c = resources.is_resource('ziptestdata', 'binary.file') - self.zip_path.unlink() - del c + def test_is_file_does_not_keep_open(self): + resources.files('data01').joinpath('binary.file').is_file() - def test_is_resource_failure_does_not_keep_open(self): - c = resources.is_resource('ziptestdata', 'not-present') - self.zip_path.unlink() - del c + def test_is_file_failure_does_not_keep_open(self): + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") - def test_path_does_not_keep_open(self): - c = resources.path('ziptestdata', 'binary.file') - self.zip_path.unlink() - del c + def test_as_file_does_not_keep_open(self): # pragma: no cover + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): - # This is what certifi does on import to make its bundle - # available for the process duration. - c = resources.path('ziptestdata', 'binary.file').__enter__() - self.zip_path.unlink() - del c + """ + Mimic what certifi does on import to make its bundle + available for the process duration. + """ + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - c = resources.read_binary('ziptestdata', 'binary.file') - self.zip_path.unlink() - del c + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - c = resources.read_text('ziptestdata', 'utf-8.file', encoding='utf-8') - self.zip_path.unlink() - del c - - -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( - resources.is_resource(import_module('namespacedata01'), 'binary.file') + resources.files(import_module('namespacedata01')) + .joinpath('binary.file') + .is_file() ) def test_read_submodule_resource_by_name(self): - self.assertTrue(resources.is_resource('namespacedata01', 'binary.file')) + self.assertTrue( + resources.files('namespacedata01').joinpath('binary.file').is_file() + ) def test_submodule_contents(self): - contents = set(resources.contents(import_module('namespacedata01'))) + contents = names(resources.files(import_module('namespacedata01'))) try: contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): - contents = set(resources.contents('namespacedata01')) + contents = names(resources.files('namespacedata01')) try: contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests( + util.DiskSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' + + +class ResourceFromNamespaceZipTests( + util.ZipSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/importlib_resources/tests/test_util.py b/importlib_resources/tests/test_util.py new file mode 100644 index 00000000..de304b6f --- /dev/null +++ b/importlib_resources/tests/test_util.py @@ -0,0 +1,29 @@ +import unittest + +from .util import MemorySetup, Traversable + + +class TestMemoryTraversableImplementation(unittest.TestCase): + def test_concrete_methods_are_not_overridden(self): + """`MemoryTraversable` must not override `Traversable` concrete methods. + + This test is not an attempt to enforce a particular `Traversable` protocol; + it merely catches changes in the `Traversable` abstract/concrete methods + that have not been mirrored in the `MemoryTraversable` subclass. + """ + + traversable_concrete_methods = { + method + for method, value in Traversable.__dict__.items() + if callable(value) and method not in Traversable.__abstractmethods__ + } + memory_traversable_concrete_methods = { + method + for method, value in MemorySetup.MemoryTraversable.__dict__.items() + if callable(value) and not method.startswith("__") + } + overridden_methods = ( + memory_traversable_concrete_methods & traversable_concrete_methods + ) + + assert not overridden_methods diff --git a/importlib_resources/tests/update-zips.py b/importlib_resources/tests/update-zips.py deleted file mode 100755 index 9ef0224c..00000000 --- a/importlib_resources/tests/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(KeyError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/importlib_resources/tests/util.py b/importlib_resources/tests/util.py index 3247bcf1..0340c150 100644 --- a/importlib_resources/tests/util.py +++ b/importlib_resources/tests/util.py @@ -1,18 +1,18 @@ import abc +import contextlib +import functools import importlib import io +import pathlib import sys import types -from pathlib import Path, PurePath - -from . import data01 -from . import zipdata01 -from ..abc import ResourceReader -from ._compat import import_helper - - from importlib.machinery import ModuleSpec +from ..abc import ResourceReader, Traversable, TraversableResources +from . import _path +from . import zip as zip_ +from .compat.py39 import import_helper, os_helper + class Reader(ResourceReader): def __init__(self, **kwargs): @@ -25,25 +25,25 @@ def open_resource(self, path): self._path = path if isinstance(self.file, Exception): raise self.file - else: - return self.file + return self.file def resource_path(self, path_): self._path = path_ if isinstance(self.path, Exception): raise self.path - else: - return self.path + return self.path def is_resource(self, path_): self._path = path_ if isinstance(self.path, Exception): raise self.path - for entry in self._contents: - parts = entry.split('/') - if len(parts) == 1 and parts[0] == path_: - return True - return False + + def part(entry): + return entry.split('/') + + return any( + len(parts) == 1 and parts[0] == path_ for parts in map(part, self._contents) + ) def contents(self): if isinstance(self.path, Exception): @@ -51,75 +51,84 @@ def contents(self): yield from self._contents -def create_package(file, path, is_package=True, contents=()): +def create_package_from_loader(loader, is_package=True): name = 'testingpackage' module = types.ModuleType(name) - loader = Reader(file=file, path=path, _contents=contents) spec = ModuleSpec(name, loader, origin='does-not-exist', is_package=is_package) module.__spec__ = spec module.__loader__ = loader return module -class CommonTests(metaclass=abc.ABCMeta): +def create_package(file=None, path=None, is_package=True, contents=()): + return create_package_from_loader( + Reader(file=file, path=path, _contents=contents), + is_package, + ) + + +class CommonTestsBase(metaclass=abc.ABCMeta): + """ + Tests shared by test_open, test_path, and test_read. + """ + @abc.abstractmethod def execute(self, package, path): - raise NotImplementedError + """ + Call the pertinent legacy API function (e.g. open_text, path) + on package and path. + """ def test_package_name(self): - # Passing in the package name should succeed. - self.execute(data01.__name__, 'utf-8.file') + """ + Passing in the package name should succeed. + """ + self.execute(self.data.__name__, 'utf-8.file') def test_package_object(self): - # Passing in the package itself should succeed. - self.execute(data01, 'utf-8.file') + """ + Passing in the package itself should succeed. + """ + self.execute(self.data, 'utf-8.file') def test_string_path(self): - # Passing in a string for the path should succeed. + """ + Passing in a string for the path should succeed. + """ path = 'utf-8.file' - self.execute(data01, path) + self.execute(self.data, path) def test_pathlib_path(self): - # Passing in a pathlib.PurePath object for the path should succeed. - path = PurePath('utf-8.file') - self.execute(data01, path) - - def test_absolute_path(self): - # An absolute path is a ValueError. - path = Path(__file__) - full_path = path.parent / 'utf-8.file' - with self.assertRaises(ValueError): - self.execute(data01, full_path) - - def test_relative_path(self): - # A reative path is a ValueError. - with self.assertRaises(ValueError): - self.execute(data01, '../data01/utf-8.file') + """ + Passing in a pathlib.PurePath object for the path should succeed. + """ + path = pathlib.PurePath('utf-8.file') + self.execute(self.data, path) def test_importing_module_as_side_effect(self): - # The anchor package can already be imported. - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') - - def test_non_package_by_name(self): - # The anchor package cannot be a module. - with self.assertRaises(TypeError): - self.execute(__name__, 'utf-8.file') - - def test_non_package_by_package(self): - # The anchor package cannot be a module. - with self.assertRaises(TypeError): - module = sys.modules['importlib_resources.tests.util'] - self.execute(module, 'utf-8.file') - - def test_resource_opener(self): + """ + The anchor package can already be imported. + """ + del sys.modules[self.data.__name__] + self.execute(self.data.__name__, 'utf-8.file') + + def test_missing_path(self): + """ + Attempting to open or read or request the path for a + non-existent path should succeed if open_resource + can return a viable data stream. + """ bytes_data = io.BytesIO(b'Hello, world!') package = create_package(file=bytes_data, path=FileNotFoundError()) self.execute(package, 'utf-8.file') self.assertEqual(package.__loader__._path, 'utf-8.file') - def test_resource_path(self): + def test_extant_path(self): + # Attempting to open or read or request the path when the + # path does exist should still succeed. Does not assert + # anything about the result. bytes_data = io.BytesIO(b'Hello, world!') + # any path that exists path = __file__ package = create_package(file=bytes_data, path=path) self.execute(package, 'utf-8.file') @@ -131,40 +140,169 @@ def test_useless_loader(self): self.execute(package, 'utf-8.file') -class ZipSetupBase: - ZIP_MODULE = None +fixtures = dict( + data01={ + '__init__.py': '', + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + '__init__.py': '', + 'binary.file': bytes(range(4, 8)), + }, + }, + data02={ + '__init__.py': '', + 'one': {'__init__.py': '', 'resource1.txt': 'one resource'}, + 'two': {'__init__.py': '', 'resource2.txt': 'two resource'}, + 'subdirectory': {'subsubdir': {'resource.txt': 'a resource'}}, + }, + namespacedata01={ + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + 'binary.file': bytes(range(12, 16)), + }, + }, +) + + +class ModuleSetup: + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) - @classmethod - def setUpClass(cls): - data_path = Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') + self.fixtures.enter_context(import_helper.isolated_modules()) + self.data = self.load_fixture(self.MODULE) - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass + def load_fixture(self, module): + self.tree_on_path({module: fixtures[module]}) + return importlib.import_module(module) - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - try: - del cls.data - del cls._zip_path - except AttributeError: - pass +class ZipSetup(ModuleSetup): + MODULE = 'data01' - def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(spec, modules))) + ) -class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore +class DiskSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + _path.build(spec, pathlib.Path(temp_dir)) + self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) + + +class MemorySetup(ModuleSetup): + """Support loading a module in memory.""" + + MODULE = 'data01' + + def load_fixture(self, module): + self.fixtures.enter_context(self.augment_sys_metapath(module)) + return importlib.import_module(module) + + @contextlib.contextmanager + def augment_sys_metapath(self, module): + finder_instance = self.MemoryFinder(module) + sys.meta_path.append(finder_instance) + yield + sys.meta_path.remove(finder_instance) + + class MemoryFinder(importlib.abc.MetaPathFinder): + def __init__(self, module): + self._module = module + + def find_spec(self, fullname, path, target=None): + if fullname != self._module: + return None + + return importlib.machinery.ModuleSpec( + name=fullname, + loader=MemorySetup.MemoryLoader(self._module), + is_package=True, + ) + + class MemoryLoader(importlib.abc.Loader): + def __init__(self, module): + self._module = module + + def exec_module(self, module): + pass + + def get_resource_reader(self, fullname): + return MemorySetup.MemoryTraversableResources(self._module, fullname) + + class MemoryTraversableResources(TraversableResources): + def __init__(self, module, fullname): + self._module = module + self._fullname = fullname + + def files(self): + return MemorySetup.MemoryTraversable(self._module, self._fullname) + + class MemoryTraversable(Traversable): + """Implement only the abstract methods of `Traversable`. + + Besides `.__init__()`, no other methods may be implemented or overridden. + This is critical for validating the concrete `Traversable` implementations. + """ + + def __init__(self, module, fullname): + self._module = module + self._fullname = fullname + + def _resolve(self): + """ + Fully traverse the `fixtures` dictionary. + + This should be wrapped in a `try/except KeyError` + but it is not currently needed and lowers the code coverage numbers. + """ + path = pathlib.PurePosixPath(self._fullname) + return functools.reduce(lambda d, p: d[p], path.parts, fixtures) + + def iterdir(self): + directory = self._resolve() + if not isinstance(directory, dict): + # Filesystem openers raise OSError, and that exception is mirrored here. + raise OSError(f"{self._fullname} is not a directory") + for path in directory: + yield MemorySetup.MemoryTraversable( + self._module, f"{self._fullname}/{path}" + ) + + def is_dir(self) -> bool: + return isinstance(self._resolve(), dict) + + def is_file(self) -> bool: + return not self.is_dir() + + def open(self, mode='r', encoding=None, errors=None, *_, **__): + contents = self._resolve() + if isinstance(contents, dict): + # Filesystem openers raise OSError when attempting to open a directory, + # and that exception is mirrored here. + raise OSError(f"{self._fullname} is a directory") + if isinstance(contents, str): + contents = contents.encode("utf-8") + result = io.BytesIO(contents) + if "b" in mode: + return result + return io.TextIOWrapper(result, encoding=encoding, errors=errors) + + @property + def name(self): + return pathlib.PurePosixPath(self._fullname).name + + +class CommonTests(DiskSetup, CommonTestsBase): + pass diff --git a/importlib_resources/tests/zip.py b/importlib_resources/tests/zip.py new file mode 100755 index 00000000..51ee5648 --- /dev/null +++ b/importlib_resources/tests/zip.py @@ -0,0 +1,26 @@ +""" +Generate zip test data files. +""" + +import zipfile + +import zipp + + +def make_zip_file(tree, dst): + """ + Zip the files in tree into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for name, contents in walk(tree): + zf.writestr(name, contents) + zipp.CompleteDirs.inject(zf) + return dst + + +def walk(tree, prefix=''): + for name, contents in tree.items(): + if isinstance(contents, dict): + yield from walk(contents, prefix=f'{prefix}{name}/') + else: + yield f'{prefix}{name}', contents diff --git a/importlib_resources/tests/zipdata01/__init__.py b/importlib_resources/tests/zipdata01/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/zipdata01/ziptestdata.zip b/importlib_resources/tests/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb073..00000000 Binary files a/importlib_resources/tests/zipdata01/ziptestdata.zip and /dev/null differ diff --git a/importlib_resources/tests/zipdata02/__init__.py b/importlib_resources/tests/zipdata02/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/zipdata02/ziptestdata.zip b/importlib_resources/tests/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512..00000000 Binary files a/importlib_resources/tests/zipdata02/ziptestdata.zip and /dev/null differ diff --git a/mypy.ini b/mypy.ini index 976ba029..d8f1411c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,23 @@ [mypy] +# Is the project well-typed? +strict = False + +# Early opt-in even when strict = False +warn_unused_ignores = True +warn_redundant_casts = True +enable_error_code = ignore-without-code + +# Support namespace packages per https://github.com/python/mypy/issues/14057 +explicit_package_bases = True + +disable_error_code = + # Disable due to many false positives + overload-overlap, + +# jaraco/zipp#123 +[mypy-zipp] +ignore_missing_imports = True + +# jaraco/jaraco.test#7 +[mypy-jaraco.test.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index b6ebc0be..1892638c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,76 @@ [build-system] -requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4.1"] +requires = ["setuptools>=61.2", "setuptools_scm[toml]>=3.4.1"] build-backend = "setuptools.build_meta" -[tool.black] -skip-string-normalization = true +[project] +name = "importlib_resources" +authors = [ + { name = "Barry Warsaw", email = "barry@python.org" }, +] +maintainers = [ + { name = "Jason R. Coombs", email = "jaraco@jaraco.com" }, +] +description = "Read resources from Python packages" +readme = "README.rst" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", +] +requires-python = ">=3.9" +license = "Apache-2.0" +dependencies = [ + "zipp >= 3.1.0; python_version < '3.10'", +] +dynamic = ["version"] -[tool.setuptools_scm] +[project.urls] +Source = "https://github.com/python/importlib_resources" + +[project.optional-dependencies] +test = [ + # upstream + "pytest >= 6, != 8.1.*", + + # local + "zipp >= 3.17", + "jaraco.test >= 5.4", +] + +doc = [ + # upstream + "sphinx >= 3.5", + "jaraco.packaging >= 9.3", + "rst.linker >= 1.9", + "furo", + "sphinx-lint", + + # tidelift + "jaraco.tidelift >= 1.4", -[pytest.enabler.black] -addopts = "--black" + # local +] -[pytest.enabler.mypy] -addopts = "--mypy" +check = [ + "pytest-checkdocs >= 2.4", + "pytest-ruff >= 0.2.1; sys_platform != 'cygwin'", +] -[pytest.enabler.flake8] -addopts = "--flake8" +cover = [ + "pytest-cov", +] -[pytest.enabler.cov] -addopts = "--cov" +enabler = [ + "pytest-enabler >= 2.2", +] + +type = [ + # upstream + "pytest-mypy", + + # local +] + + +[tool.setuptools_scm] diff --git a/pytest.ini b/pytest.ini index d7f0b115..9a0f3bce 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,9 +1,25 @@ [pytest] norecursedirs=dist build .tox .eggs -addopts=--doctest-modules -doctest_optionflags=ALLOW_UNICODE ELLIPSIS -# workaround for warning pytest-dev/pytest#6178 -junit_family=xunit2 +addopts= + --doctest-modules + --import-mode importlib +consider_namespace_packages=true filterwarnings= - # https://github.com/pytest-dev/pytest/issues/6928 - ignore:direct construction of .*Item has been deprecated:DeprecationWarning + ## upstream + + # Ensure ResourceWarnings are emitted + default::ResourceWarning + + # realpython/pytest-mypy#152 + ignore:'encoding' argument not specified::pytest_mypy + + # python/cpython#100750 + ignore:'encoding' argument not specified::platform + + # pypa/build#615 + ignore:'encoding' argument not specified::build.env + + # dateutil/dateutil#1284 + ignore:datetime.datetime.utcfromtimestamp:DeprecationWarning:dateutil.tz.tz + + ## end upstream diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..63c0825f --- /dev/null +++ b/ruff.toml @@ -0,0 +1,51 @@ +[lint] +extend-select = [ + # upstream + + "C901", # complex-structure + "I", # isort + "PERF401", # manual-list-comprehension + + # Ensure modern type annotation syntax and best practices + # Not including those covered by type-checkers or exclusive to Python 3.11+ + "FA", # flake8-future-annotations + "F404", # late-future-import + "PYI", # flake8-pyi + "UP006", # non-pep585-annotation + "UP007", # non-pep604-annotation + "UP010", # unnecessary-future-import + "UP035", # deprecated-import + "UP037", # quoted-annotation + "UP043", # unnecessary-default-type-args + + # local +] +ignore = [ + # upstream + + # Typeshed rejects complex or non-literal defaults for maintenance and testing reasons, + # irrelevant to this project. + "PYI011", # typed-argument-default-in-stub + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", + + # local +] + +[format] +# Enable preview to get hugged parenthesis unwrapping and other nice surprises +# See https://github.com/jaraco/skeleton/pull/133#issuecomment-2239538373 +preview = true +# https://docs.astral.sh/ruff/settings/#format_quote-style +quote-style = "preserve" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 7d9424d0..00000000 --- a/setup.cfg +++ /dev/null @@ -1,53 +0,0 @@ -[metadata] -license_files = - LICENSE -name = importlib_resources -author = Barry Warsaw -author_email = barry@python.org -description = Read resources from Python packages -long_description = file: README.rst -url = https://github.com/python/importlib_resources -license = Apache2 -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - License :: OSI Approved :: Apache Software License - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only -project_urls = - Documentation = https://importlib-resources.readthedocs.io/ - -[options] -packages = find_namespace: -include_package_data = true -python_requires = >=3.6 -install_requires = - zipp >= 0.4; python_version < '3.8' -setup_requires = setuptools_scm[toml] >= 3.4.1 - -[options.packages.find] -exclude = - build* - docs* - tests* - -[options.extras_require] -testing = - # upstream - pytest >= 3.5, !=3.7.3 - pytest-checkdocs >= 1.2.3 - pytest-flake8 - pytest-black >= 0.3.7; python_implementation != "PyPy" - pytest-cov - pytest-mypy; python_implementation != "PyPy" - pytest-enabler - - # local - -docs = - # upstream - sphinx - jaraco.packaging >= 8.2 - rst.linker >= 1.9 - - # local diff --git a/setup.py b/setup.py deleted file mode 100644 index bac24a43..00000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -import setuptools - -if __name__ == "__main__": - setuptools.setup() diff --git a/skeleton.md b/skeleton.md deleted file mode 100644 index 0938f892..00000000 --- a/skeleton.md +++ /dev/null @@ -1,166 +0,0 @@ -# Overview - -This project is merged with [skeleton](https://github.com/jaraco/skeleton). What is skeleton? It's the scaffolding of a Python project jaraco [introduced in his blog](https://blog.jaraco.com/a-project-skeleton-for-python-projects/). It seeks to provide a means to re-use techniques and inherit advances when managing projects for distribution. - -## An SCM-Managed Approach - -While maintaining dozens of projects in PyPI, jaraco derives best practices for project distribution and publishes them in the [skeleton repo](https://github.com/jaraco/skeleton), a Git repo capturing the evolution and culmination of these best practices. - -It's intended to be used by a new or existing project to adopt these practices and honed and proven techniques. Adopters are encouraged to use the project directly and maintain a small deviation from the technique, make their own fork for more substantial changes unique to their environment or preferences, or simply adopt the skeleton once and abandon it thereafter. - -The primary advantage to using an SCM for maintaining these techniques is that those tools help facilitate the merge between the template and its adopting projects. - -Another advantage to using an SCM-managed approach is that tools like GitHub recognize that a change in the skeleton is the _same change_ across all projects that merge with that skeleton. Without the ancestry, with a traditional copy/paste approach, a [commit like this](https://github.com/jaraco/skeleton/commit/12eed1326e1bc26ce256e7b3f8cd8d3a5beab2d5) would produce notifications in the upstream project issue for each and every application, but because it's centralized, GitHub provides just the one notification when the change is added to the skeleton. - -# Usage - -## new projects - -To use skeleton for a new project, simply pull the skeleton into a new project: - -``` -$ git init my-new-project -$ cd my-new-project -$ git pull gh://jaraco/skeleton -``` - -Now customize the project to suit your individual project needs. - -## existing projects - -If you have an existing project, you can still incorporate the skeleton by merging it into the codebase. - -``` -$ git merge skeleton --allow-unrelated-histories -``` - -The `--allow-unrelated-histories` is necessary because the history from the skeleton was previously unrelated to the existing codebase. Resolve any merge conflicts and commit to the master, and now the project is based on the shared skeleton. - -## Updating - -Whenever a change is needed or desired for the general technique for packaging, it can be made in the skeleton project and then merged into each of the derived projects as needed, recommended before each release. As a result, features and best practices for packaging are centrally maintained and readily trickle into a whole suite of packages. This technique lowers the amount of tedious work necessary to create or maintain a project, and coupled with other techniques like continuous integration and deployment, lowers the cost of creating and maintaining refined Python projects to just a few, familiar Git operations. - -For example, here's a session of the [path project](https://pypi.org/project/path) pulling non-conflicting changes from the skeleton: - - - -Thereafter, the target project can make whatever customizations it deems relevant to the scaffolding. The project may even at some point decide that the divergence is too great to merit renewed merging with the original skeleton. This approach applies maximal guidance while creating minimal constraints. - -## Periodic Collapse - -In late 2020, this project [introduced](https://github.com/jaraco/skeleton/issues/27) the idea of a periodic but infrequent (O(years)) collapse of commits to limit the number of commits a new consumer will need to accept to adopt the skeleton. - -The full history of commits is collapsed into a single commit and that commit becomes the new mainline head. - -When one of these collapse operations happens, any project that previously pulled from the skeleton will no longer have a related history with that new main branch. For those projects, the skeleton provides a "handoff" branch that reconciles the two branches. Any project that has previously merged with the skeleton but now gets an error "fatal: refusing to merge unrelated histories" should instead use the handoff branch once to incorporate the new main branch. - -``` -$ git pull https://github.com/jaraco/skeleton 2020-handoff -``` - -This handoff needs to be pulled just once and thereafter the project can pull from the main head. - -The archive and handoff branches from prior collapses are indicate here: - -| refresh | archive | handoff | -|---------|-----------------|--------------| -| 2020-12 | archive/2020-12 | 2020-handoff | - -# Features - -The features/techniques employed by the skeleton include: - -- PEP 517/518-based build relying on Setuptools as the build tool -- Setuptools declarative configuration using setup.cfg -- tox for running tests -- A README.rst as reStructuredText with some popular badges, but with Read the Docs and AppVeyor badges commented out -- A CHANGES.rst file intended for publishing release notes about the project -- Use of [Black](https://black.readthedocs.io/en/stable/) for code formatting (disabled on unsupported Python 3.5 and earlier) -- Integrated type checking through [mypy](https://github.com/python/mypy/). - -## Packaging Conventions - -A pyproject.toml is included to enable PEP 517 and PEP 518 compatibility and declares the requirements necessary to build the project on Setuptools (a minimum version compatible with setup.cfg declarative config). - -The setup.cfg file implements the following features: - -- Assumes universal wheel for release -- Advertises the project's LICENSE file (MIT by default) -- Reads the README.rst file into the long description -- Some common Trove classifiers -- Includes all packages discovered in the repo -- Data files in the package are also included (not just Python files) -- Declares the required Python versions -- Declares install requirements (empty by default) -- Declares setup requirements for legacy environments -- Supplies two 'extras': - - testing: requirements for running tests - - docs: requirements for building docs - - these extras split the declaration into "upstream" (requirements as declared by the skeleton) and "local" (those specific to the local project); these markers help avoid merge conflicts -- Placeholder for defining entry points - -Additionally, the setup.py file declares `use_scm_version` which relies on [setuptools_scm](https://pypi.org/project/setuptools_scm) to do two things: - -- derive the project version from SCM tags -- ensure that all files committed to the repo are automatically included in releases - -## Running Tests - -The skeleton assumes the developer has [tox](https://pypi.org/project/tox) installed. The developer is expected to run `tox` to run tests on the current Python version using [pytest](https://pypi.org/project/pytest). - -Other environments (invoked with `tox -e {name}`) supplied include: - - - a `docs` environment to build the documentation - - a `release` environment to publish the package to PyPI - -A pytest.ini is included to define common options around running tests. In particular: - -- rely on default test discovery in the current directory -- avoid recursing into common directories not containing tests -- run doctests on modules and invoke Flake8 tests -- in doctests, allow Unicode literals and regular literals to match, allowing for doctests to run on Python 2 and 3. Also enable ELLIPSES, a default that would be undone by supplying the prior option. -- filters out known warnings caused by libraries/functionality included by the skeleton - -Relies on a .flake8 file to correct some default behaviors: - -- disable mutually incompatible rules W503 and W504 -- support for Black format - -## Continuous Integration - -The project is pre-configured to run Continuous Integration tests. - -### Github Actions - -[Github Actions](https://docs.github.com/en/free-pro-team@latest/actions) are the preferred provider as they provide free, fast, multi-platform services with straightforward configuration. Configured in `.github/workflows`. - -Features include: -- test against multiple Python versions -- run on late (and updated) platform versions -- automated releases of tagged commits -- [automatic merging of PRs](https://github.com/marketplace/actions/merge-pull-requests) (requires [protecting branches with required status checks](https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/enabling-required-status-checks), [not possible through API](https://github.community/t/set-all-status-checks-to-be-required-as-branch-protection-using-the-github-api/119493)) - - -### Continuous Deployments - -In addition to running tests, an additional publish stage is configured to automatically release tagged commits to PyPI using [API tokens](https://pypi.org/help/#apitoken). The release process expects an authorized token to be configured with each Github project (or org) `PYPI_TOKEN` [secret](https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets). Example: - -``` -pip-run -q jaraco.develop -- -m jaraco.develop.add-github-secrets -``` - -## Building Documentation - -Documentation is automatically built by [Read the Docs](https://readthedocs.org) when the project is registered with it, by way of the .readthedocs.yml file. To test the docs build manually, a tox env may be invoked as `tox -e docs`. Both techniques rely on the dependencies declared in `setup.cfg/options.extras_require.docs`. - -In addition to building the Sphinx docs scaffolded in `docs/`, the docs build a `history.html` file that first injects release dates and hyperlinks into the CHANGES.rst before incorporating it as history in the docs. - -## Cutting releases - -By default, tagged commits are released through the continuous integration deploy stage. - -Releases may also be cut manually by invoking the tox environment `release` with the PyPI token set as the TWINE_PASSWORD: - -``` -TWINE_PASSWORD={token} tox -e release -``` diff --git a/towncrier.toml b/towncrier.toml new file mode 100644 index 00000000..6fa480e4 --- /dev/null +++ b/towncrier.toml @@ -0,0 +1,2 @@ +[tool.towncrier] +title_format = "{version}" diff --git a/tox.ini b/tox.ini index 01828205..14243051 100644 --- a/tox.ini +++ b/tox.ini @@ -1,48 +1,63 @@ -[tox] -envlist = python -minversion = 3.2 -# https://github.com/jaraco/skeleton/issues/6 -tox_pip_extensions_ext_venv_update = true -toxworkdir={env:TOX_WORK_DIR:.tox} - - [testenv] +description = perform primary checks (tests, style, types, coverage) deps = +setenv = + PYTHONWARNDEFAULTENCODING = 1 commands = pytest {posargs} usedevelop = True -extras = testing - -[testenv:docs] extras = - docs - testing -changedir = docs -commands = - python -m sphinx . {toxinidir}/build/html + test + check + cover + enabler + type [testenv:diffcov] +description = run tests and check that diff from main is covered deps = + {[testenv]deps} diff-cover commands = pytest {posargs} --cov-report xml diff-cover coverage.xml --compare-branch=origin/main --html-report diffcov.html diff-cover coverage.xml --compare-branch=origin/main --fail-under=100 +[testenv:docs] +description = build the documentation +extras = + doc + test +changedir = docs +commands = + python -m sphinx -W --keep-going . {toxinidir}/build/html + python -m sphinxlint + +[testenv:finalize] +description = assemble changelog and tag a release +skip_install = True +deps = + towncrier + jaraco.develop >= 7.23 +pass_env = * +commands = + python -m jaraco.develop.finalize + + [testenv:release] +description = publish the package to PyPI and GitHub skip_install = True deps = build - twine[keyring]>=1.13 - path + twine>=3 jaraco.develop>=7.1 -passenv = +pass_env = TWINE_PASSWORD GITHUB_TOKEN setenv = TWINE_USERNAME = {env:TWINE_USERNAME:__token__} commands = - python -c "import path; path.Path('dist').rmtree_p()" + python -c "import shutil; shutil.rmtree('dist', ignore_errors=True)" python -m build python -m twine upload dist/* python -m jaraco.develop.create-github-release