diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..45eeeac2 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,18 @@ +# .github/release.yml + +changelog: + exclude: + labels: + - ignore-for-release-notes + authors: + - pre-commit-ci[bot] + categories: + - title: Fixes + labels: + - bug + - title: Exciting New Features 🎉 + labels: + - enhancement + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index 65bda202..00000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Run tests - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - # Test the oldest and newest configuration on Mac and Windows - - os: macos-latest - python-version: 3.6 - toxenv: py36-test-mpl20 - - os: macos-latest - python-version: "3.10" - toxenv: py310-test-mpl35 - - os: windows-latest - python-version: 3.6 - toxenv: py36-test-mpl20 - - os: windows-latest - python-version: "3.10" - toxenv: py310-test-mpl35 - # Test all configurations on Linux - - os: ubuntu-latest - python-version: 3.6 - toxenv: py36-test-mpl20 - - os: ubuntu-latest - python-version: 3.6 - toxenv: py36-test-mpl21 - - os: ubuntu-latest - python-version: 3.6 - toxenv: py36-test-mpl22 - - os: ubuntu-latest - python-version: 3.7 - toxenv: py37-test-mpl30 - - os: ubuntu-latest - python-version: 3.7 - toxenv: py37-test-mpl31 - - os: ubuntu-latest - python-version: 3.7 - toxenv: py37-test-mpl32 - - os: ubuntu-latest - python-version: 3.8 - toxenv: py38-test-mpl33 - - os: ubuntu-latest - python-version: 3.9 - toxenv: py39-test-mpl34 - - os: ubuntu-latest - python-version: "3.10" - toxenv: py310-test-mpl35 - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: python -m pip install tox - - name: Run tox - run: python -m tox -e ${{ matrix.toxenv }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 4053bc3e..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Build and Upload Python Package - -on: - push: - tags: - - "v*" - branches: - - master - pull_request: - -jobs: - build_sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U setuptools wheel build - - name: Build - run: | - python -m build --sdist --outdir dist/ . - - name: Install - run: | - pip install dist/* - - name: Run tests - run: | - pytest tests - pytest tests --mpl - - name: Store sdist - uses: actions/upload-artifact@v2 - with: - name: sdist - path: | - dist/* - - build_universal_wheel: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U setuptools wheel build - - name: Build - run: | - python -m build --wheel --outdir dist/ . - - name: Install - run: | - pip install dist/* - - name: Run tests - run: | - pytest tests - pytest tests --mpl - - name: Store sdist - uses: actions/upload-artifact@v2 - with: - name: wheel - path: | - dist/* - - upload: - runs-on: ubuntu-latest - # Only upload on a tag - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - needs: - - build_universal_wheel - - build_sdist - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U setuptools wheel twine - - - name: Download all workflow run artifacts - uses: actions/download-artifact@v2 - with: - path: dist/ - - name: List all downloaded Artifacts - run: | - ls -lha dist/wheel dist/sdist - - name: Publish to PyPI - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - twine upload dist/wheel/* dist/sdist/* diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml new file mode 100644 index 00000000..4888ba44 --- /dev/null +++ b/.github/workflows/test_and_publish.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: + - master + tags: + - 'v*' + - '!*dev*' + - '!*pre*' + - '!*post*' + pull_request: + # Allow manual runs through the web UI + workflow_dispatch: + +jobs: + test: + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + with: + envs: | + # Test the oldest and newest configuration on Mac and Windows + - macos: py36-test-mpl20 + - macos: py310-test-mpl35 + - windows: py36-test-mpl20 + - windows: py310-test-mpl35 + # Test all configurations on Linux + - linux: py36-test-mpl20 + - linux: py36-test-mpl21 + - linux: py36-test-mpl22 + - linux: py37-test-mpl30 + - linux: py37-test-mpl31 + - linux: py37-test-mpl32 + - linux: py38-test-mpl33 + - linux: py39-test-mpl34 + - linux: py310-test-mpl35 + coverage: 'codecov' + + publish: + if: github.event_name != 'pull_request' + needs: [test] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 + with: + test_command: pytest $GITHUB_WORKSPACE/tests; pytest --mpl $GITHUB_WORKSPACE/tests + secrets: + pypi_token: ${{ secrets.pypi_password }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97cd8689..e7c0ffc4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,11 +80,14 @@ repos: - id: check-ast - id: check-case-conflict - id: trailing-whitespace + exclude: CHANGES.md - id: check-yaml - id: debug-statements - id: check-added-large-files - id: end-of-file-fixer - id: mixed-line-ending + - id: check-toml + ci: autofix_prs: false diff --git a/CHANGES.md b/CHANGES.md index 470ace47..5c6ab0fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,158 +1,161 @@ -0.15 (unreleased) ------------------ +## v0.15.0 - 2022-04-21 -- An updated hash library will be saved to the results directory when - generating a HTML summary page or when the `--mpl-results-always` flag is - set. A button to download this file is included in the HTML summary. - Various bugfixes, test improvements and documentation updates. [#138] +### Features -0.14 (2022-02-09) ------------------ +- Remove Python 2 from package classifiers by @dopplershift in https://github.com/matplotlib/pytest-mpl/pull/137 +- Downloadable hash library in HTML summary by @ConorMacBride in https://github.com/matplotlib/pytest-mpl/pull/138 -- Add `--mpl-results-always` flag which disables removing of test images for - tests which pass. Test images are also stored and generated for hash tests - when a baseline dir is also provided. [#108] +### Fixes -- When generating a HTML summary page, the `--mpl-results-always` flag is - automatically applied. [#131] +- No need to warn when falling back to other URL by @pllim in https://github.com/matplotlib/pytest-mpl/pull/139 +- Automatically update changelog in the repo after release by @Cadair in https://github.com/matplotlib/pytest-mpl/pull/143 -- Add a `--mpl-generate-summary=json` option which saves a JSON summary of the - image comparison results. [#127] +**Full Changelog**: https://github.com/matplotlib/pytest-mpl/compare/v0.14.0...v0.15.0 -- Add a significantly improved HTML summary page, and rename the old simple - summary page to `basic-html`. [#128] +## 0.14 (2022-02-09) +- Add `--mpl-results-always` flag which disables removing of test images for +- tests which pass. Test images are also stored and generated for hash tests +- when a baseline dir is also provided. [#108] +- +- When generating a HTML summary page, the `--mpl-results-always` flag is +- automatically applied. [#131] +- +- Add a `--mpl-generate-summary=json` option which saves a JSON summary of the +- image comparison results. [#127] +- +- Add a significantly improved HTML summary page, and rename the old simple +- summary page to `basic-html`. [#128] +- - Various bugfixes, test improvements and documentation updates [#134] +- -0.13 (2021-07-02) ------------------ +## 0.13 (2021-07-02) - Ensure all test files are included in the sdist. [#109] - +- - Print hash for new figure tests. [#111] - +- - Do not error if a baseline image can not be retrieved when using figure hashes. [#118] - +- - Allow generation of hash library and testing against hash library simultaneously. [#121] +- -0.12.1 (2021-07-02) -------------------- +## 0.12.1 (2021-07-02) - Fix specification of required Python version in setup.cfg. [#122] -0.12 (2020-11-05) ------------------ +## 0.12 (2020-11-05) - Fix passing a https url for baseline images from the CLI. [#89] - +- - Added `--mpl-baseline-relative` option to specify baseline images relative to the test path. [#96] - +- - Add option to do comparisons against a json library of sha256 hashes. [#98] - +- - Drop support for matplotlib 1.5 and Python < 3.6. [#100] - +- - Add support for generating a HTML summary of test faliures. [#101] - +- - Add support for falling back to baseline image comparison if hash comparison fails. [#101] +- -0.11 (2019-11-15) ------------------ +## 0.11 (2019-11-15) - Improve error message if image shapes don't match. [#79] - +- - Properly register mpl_image_compare marker with pytest. [#83] - +- - Drop support for Python 3.5 and earlier, and Matplotlib 1.5. [#87] +- -0.10 (2018-09-25) ------------------ +## 0.10 (2018-09-25) - Improve error message when baseline image is not found. [#76] - +- - Update compatibility with pytest 3.6. [#72] - +- - Only close figures if they are a valid Matplotlib figure. [#66] - +- - Improve tests to not assume pytest executable is called py.test. [#65] - +- - Make sure local matplotlib files are completely ignored. [#64] +- -0.9 (2017-10-12) ----------------- +## 0.9 (2017-10-12) - Fix compatibility with Matplotlib 2.1. [#54] - +- - Allow baseline_dir to be comma-separated URL list to allow mirrors to - be specified. [#59] - +- be specified. [#59] +- - Make sure figures get closed even if not running with the --mpl - option, and only close actual Matplotlib Figure objects. [#60] +- option, and only close actual Matplotlib Figure objects. [#60] +- -0.8 (2017-07-19) ----------------- +## 0.8 (2017-07-19) - Fixed use of mpl_image_compare on methods of test classes that also - use setup_method. [#51] - +- use setup_method. [#51] +- - Make it possible to specify the directory in which to put the results - from the tests using --mpl-results-path. [#39] - +- from the tests using --mpl-results-path. [#39] +- - Only import Matplotlib if the plugin is actually used. [#47] - +- - Make sure figures are closed after saving. [#46] - +- - Allow the backend to be set on a per-test basis. [#38] - +- - If test name contains slashes (normally from parameters in - parametrized tests), replace with _. [#50] +- parametrized tests), replace with _. [#50] +- -0.7 (2016-11-26) ----------------- +## 0.7 (2016-11-26) - Properly define dependencies in setup.py. [#32] -0.6 (2016-11-22) ----------------- - -- Added ``style`` and ``remove_text`` options. [#20] +## 0.6 (2016-11-22) +- Added `style` and `remove_text` options. [#20] +- - Properly support parametrized tests. [#24] +- -0.5 (2016-05-06) ----------------- +## 0.5 (2016-05-06) - Minor fixes to detection of remote baseline directories. - +- - Minor improvements to documentation. +- -0.4 (2016-05-04) ----------------- +## 0.4 (2016-05-04) - Add support for remote baseline images. [#18] - +- - When providing two conflicting options, warn instead of raising an - exception. [#19] +- exception. [#19] +- -0.3 (2015-06-26) ----------------- +## 0.3 (2015-06-26) - Changed default tolerance from 10 to 2. [#9] - -- Added ``tox.ini``. - +- +- Added `tox.ini`. +- - Improvements to documentation +- -0.2 (2015-06-25) ----------------- +## 0.2 (2015-06-25) - Added globally-configurable baseline directory with the - ``--mpl-baseline-dir`` option. [#8] - -- Added ``baseline_dir`` and ``filename`` options in decorator. - +- `--mpl-baseline-dir` option. [#8] +- +- Added `baseline_dir` and `filename` options in decorator. +- - Improvements to documentation +- -0.1 (2015-06-25) ----------------- +## 0.1 (2015-06-25) - Initial version diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 78fbcc19..0f30566d 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -130,6 +130,8 @@ def _rms_sort(self): if self.image_status == 'match': return "000000" elif self.image_status == 'diff': + if self.rms is None: # Shape mismatch + return "999999" # RMS will be in [0, 255] return f"{(self.rms + 2) * 1000:06.0f}" else: # Missing baseline image diff --git a/pytest_mpl/summary/templates/result.html b/pytest_mpl/summary/templates/result.html index 06d34973..6e7986bd 100644 --- a/pytest_mpl/summary/templates/result.html +++ b/pytest_mpl/summary/templates/result.html @@ -8,14 +8,18 @@
- {% if r.diff_image -%} + {% if r.image_status and r.image_status == "diff" -%}
+ {% if r.diff_image -%}
diff image
result image
+ {%- else -%} + result image + {%- endif %}
{%- elif r.result_image -%} result image diff --git a/tests/subtests/baseline/test_hdiff_idiffshape.png b/tests/subtests/baseline/test_hdiff_idiffshape.png new file mode 100644 index 00000000..2bc3cbce Binary files /dev/null and b/tests/subtests/baseline/test_hdiff_idiffshape.png differ diff --git a/tests/subtests/baseline/test_hmatch_idiffshape.png b/tests/subtests/baseline/test_hmatch_idiffshape.png new file mode 100644 index 00000000..82b5e0ae Binary files /dev/null and b/tests/subtests/baseline/test_hmatch_idiffshape.png differ diff --git a/tests/subtests/baseline/test_hmissing_idiffshape.png b/tests/subtests/baseline/test_hmissing_idiffshape.png new file mode 100644 index 00000000..0ca56a91 Binary files /dev/null and b/tests/subtests/baseline/test_hmissing_idiffshape.png differ diff --git a/tests/subtests/hashes/mpl33_ft261.json b/tests/subtests/hashes/mpl33_ft261.json index e285cd6f..630e40b6 100644 --- a/tests/subtests/hashes/mpl33_ft261.json +++ b/tests/subtests/hashes/mpl33_ft261.json @@ -1,9 +1,11 @@ { "subtests.subtest.test_hmatch_imatch": "42c391b37022e2c4edb53f5fd988e94f421905b40cea1544e62ffb3c049292a8", "subtests.subtest.test_hmatch_idiff": "c14ba098dbda0988e35be5724ffb15b8e666253a4b37dec6a21203607c17473d", + "subtests.subtest.test_hmatch_idiffshape": "d23fa57068c6888307575623e5bdbe5e577d935910fee8d41deab426677acecb", "subtests.subtest.test_hmatch_imissing": "6c07931bac1a926c88bea5d07c40c8c1ce30648712e3fc963028193863e3ae65", "subtests.subtest.test_hdiff_imatch": "d1ff383721a0c395c856302be7de8a8138a2693651425dc181ede262860aef7b", "subtests.subtest.test_hdiff_idiff": "d1fff55ace5ef7e45dcd9913b54e0d9970028cae59666e937ccb3586d0f76e9a", + "subtests.subtest.test_hdiff_idiffshape": "d1ff76e20951e78fd3dedfff6a6f8f2eab4c569860d1a0da7867114cdcdf7c2c", "subtests.subtest.test_hdiff_imissing": "d1ff35845c5887c034230e02aa4b60e053c779c693867e4803e1d72dde9240f7", "subtests.subtest.test_hdiff_imatch_tolerance": "d1ff6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b", "subtests.subtest.test_hdiff_idiff_tolerance": "d1ff6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b", diff --git a/tests/subtests/hashes/mpl34_ft261.json b/tests/subtests/hashes/mpl34_ft261.json index a8012e51..c58c78d3 100644 --- a/tests/subtests/hashes/mpl34_ft261.json +++ b/tests/subtests/hashes/mpl34_ft261.json @@ -1,9 +1,11 @@ { "subtests.subtest.test_hmatch_imatch": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672", "subtests.subtest.test_hmatch_idiff": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857", + "subtests.subtest.test_hmatch_idiffshape": "2ee75301c4de2dcb9f839b278c6371be2e751de40b131213e375d4dcc5542382", "subtests.subtest.test_hmatch_imissing": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f", "subtests.subtest.test_hdiff_imatch": "d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1", "subtests.subtest.test_hdiff_idiff": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43", + "subtests.subtest.test_hdiff_idiffshape": "d1ffae8ab2b65de3fa297be17ce973ff871e703c9550679e9566179dd785f6eb", "subtests.subtest.test_hdiff_imissing": "d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e", "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96", "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96", diff --git a/tests/subtests/hashes/mpl35_ft261.json b/tests/subtests/hashes/mpl35_ft261.json index 2ae41a7e..0c9354fe 100644 --- a/tests/subtests/hashes/mpl35_ft261.json +++ b/tests/subtests/hashes/mpl35_ft261.json @@ -1,9 +1,11 @@ { "subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", "subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "subtests.subtest.test_hmatch_idiffshape": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", "subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", "subtests.subtest.test_hdiff_imatch": "d1ff8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", "subtests.subtest.test_hdiff_idiff": "d1ff21206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "subtests.subtest.test_hdiff_idiffshape": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", "subtests.subtest.test_hdiff_imissing": "d1ff11cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", "subtests.subtest.test_hdiff_imatch_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", "subtests.subtest.test_hdiff_idiff_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", diff --git a/tests/subtests/result_hashes/mpl33_ft261.json b/tests/subtests/result_hashes/mpl33_ft261.json index b5bf8b7a..1d32aed3 100644 --- a/tests/subtests/result_hashes/mpl33_ft261.json +++ b/tests/subtests/result_hashes/mpl33_ft261.json @@ -1,12 +1,15 @@ { "subtests.subtest.test_hmatch_imatch": "42c391b37022e2c4edb53f5fd988e94f421905b40cea1544e62ffb3c049292a8", "subtests.subtest.test_hmatch_idiff": "c14ba098dbda0988e35be5724ffb15b8e666253a4b37dec6a21203607c17473d", + "subtests.subtest.test_hmatch_idiffshape": "d23fa57068c6888307575623e5bdbe5e577d935910fee8d41deab426677acecb", "subtests.subtest.test_hmatch_imissing": "6c07931bac1a926c88bea5d07c40c8c1ce30648712e3fc963028193863e3ae65", "subtests.subtest.test_hdiff_imatch": "b48a383721a0c395c856302be7de8a8138a2693651425dc181ede262860aef7b", "subtests.subtest.test_hdiff_idiff": "44edf55ace5ef7e45dcd9913b54e0d9970028cae59666e937ccb3586d0f76e9a", + "subtests.subtest.test_hdiff_idiffshape": "1c6176e20951e78fd3dedfff6a6f8f2eab4c569860d1a0da7867114cdcdf7c2c", "subtests.subtest.test_hdiff_imissing": "042235845c5887c034230e02aa4b60e053c779c693867e4803e1d72dde9240f7", "subtests.subtest.test_hmissing_imatch": "d87e9aa72d2cd0ccd9959ec7af2dc00ab86e2fe8f06a6c54a017cde62f7e5c1f", "subtests.subtest.test_hmissing_idiff": "00b074ef61b5061b8ec50dbe32b3214ffd21c9489ae212671558f338ad792e5f", + "subtests.subtest.test_hmissing_idiffshape": "3abf54dbc88ce374b40d677597685fc83b4dc05c9d9a50fa1580f348d71fb99d", "subtests.subtest.test_hmissing_imissing": "3e5c4a4386e32133083c36ca0c95f5e38be904ed238e49ab1e06edd35603abfc", "subtests.subtest.test_hdiff_imatch_tolerance": "04eb6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b", "subtests.subtest.test_hdiff_idiff_tolerance": "04eb6912989a4b47ea910b04edfa58cf5d756d60825ea52ad59dcde8e03d4d8b", diff --git a/tests/subtests/result_hashes/mpl34_ft261.json b/tests/subtests/result_hashes/mpl34_ft261.json index b30b198a..3511556b 100644 --- a/tests/subtests/result_hashes/mpl34_ft261.json +++ b/tests/subtests/result_hashes/mpl34_ft261.json @@ -1,12 +1,15 @@ { "subtests.subtest.test_hmatch_imatch": "573f4c1482192b7b15bbe4f2bd370ae3b5ab40c9afa441543b87e15a71e9f672", "subtests.subtest.test_hmatch_idiff": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857", + "subtests.subtest.test_hmatch_idiffshape": "2ee75301c4de2dcb9f839b278c6371be2e751de40b131213e375d4dcc5542382", "subtests.subtest.test_hmatch_imissing": "fd069e642e3b154c24077a4996b545e1c4dbffdbed34ea5ad34c7b36873af68f", "subtests.subtest.test_hdiff_imatch": "6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1", "subtests.subtest.test_hdiff_idiff": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43", + "subtests.subtest.test_hdiff_idiffshape": "3379ae8ab2b65de3fa297be17ce973ff871e703c9550679e9566179dd785f6eb", "subtests.subtest.test_hdiff_imissing": "301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e", "subtests.subtest.test_hmissing_imatch": "eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824", "subtests.subtest.test_hmissing_idiff": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab", + "subtests.subtest.test_hmissing_idiffshape": "fa566a1620537f5c10aa8bcbebed55065f13fa9f8e8e82d7f7c67f7b39edd552", "subtests.subtest.test_hmissing_imissing": "5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b", "subtests.subtest.test_hdiff_imatch_tolerance": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96", "subtests.subtest.test_hdiff_idiff_tolerance": "aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96", diff --git a/tests/subtests/result_hashes/mpl35_ft261.json b/tests/subtests/result_hashes/mpl35_ft261.json index edc8605a..4bbb2cb1 100644 --- a/tests/subtests/result_hashes/mpl35_ft261.json +++ b/tests/subtests/result_hashes/mpl35_ft261.json @@ -1,12 +1,15 @@ { "subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", "subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "subtests.subtest.test_hmatch_idiffshape": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", "subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", "subtests.subtest.test_hdiff_imatch": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", "subtests.subtest.test_hdiff_idiff": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "subtests.subtest.test_hdiff_idiffshape": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", "subtests.subtest.test_hdiff_imissing": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", "subtests.subtest.test_hmissing_imatch": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", "subtests.subtest.test_hmissing_idiff": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "subtests.subtest.test_hmissing_idiffshape": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d", "subtests.subtest.test_hmissing_imissing": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", "subtests.subtest.test_hdiff_imatch_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", "subtests.subtest.test_hdiff_idiff_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", diff --git a/tests/subtests/subtest.py b/tests/subtests/subtest.py index ba927e57..9791ebd2 100644 --- a/tests/subtests/subtest.py +++ b/tests/subtests/subtest.py @@ -25,6 +25,11 @@ def test_hmatch_idiff(): return plot([1, 3, 2, 4]) +@pytest.mark.mpl_image_compare() +def test_hmatch_idiffshape(): + return plot([4, 2, 3, 1, 2]) + + @pytest.mark.mpl_image_compare() def test_hmatch_imissing(): return plot([4, 3, 2, 1]) @@ -42,6 +47,11 @@ def test_hdiff_idiff(): return plot([1, 2, 4, 3]) +@pytest.mark.mpl_image_compare() +def test_hdiff_idiffshape(): + return plot([4, 2, 3, 1, 3]) + + @pytest.mark.mpl_image_compare() def test_hdiff_imissing(): return plot([3, 2, 4, 1]) @@ -59,6 +69,11 @@ def test_hmissing_idiff(): return plot([1, 4, 3, 2]) +@pytest.mark.mpl_image_compare() +def test_hmissing_idiffshape(): + return plot([4, 2, 3, 1, 4]) + + @pytest.mark.mpl_image_compare() def test_hmissing_imissing(): return plot([2, 4, 3, 1]) diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index b4b51444..f5e295fa 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -25,6 +25,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", "image_status": "missing", @@ -64,6 +77,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "image_status": "missing", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "image_status": "missing", diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json index 6f9f78e2..401081a8 100644 --- a/tests/subtests/summaries/test_generate.json +++ b/tests/subtests/summaries/test_generate.json @@ -25,6 +25,19 @@ "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", @@ -64,6 +77,19 @@ "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", @@ -103,6 +129,19 @@ "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d", + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", "image_status": "generated", diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json index 8dee25fa..a53ef2ca 100644 --- a/tests/subtests/summaries/test_generate_hashes_only.json +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -25,6 +25,19 @@ "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", "image_status": "missing", @@ -64,6 +77,19 @@ "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "image_status": "missing", @@ -103,6 +129,19 @@ "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d", + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "image_status": "missing", diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json index 7720807d..553b5589 100644 --- a/tests/subtests/summaries/test_generate_images_only.json +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -25,6 +25,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", @@ -64,6 +77,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", "image_status": "generated", diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index 4304bf03..ac51e659 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -25,6 +25,19 @@ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857", "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857" }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e" + }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", "image_status": null, @@ -64,6 +77,19 @@ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43", "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43" }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580" + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab" }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": null, + "hash_status": "missing", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": null, + "result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d" + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json index 84a8e8b9..d4a2835e 100644 --- a/tests/subtests/summaries/test_html_generate.json +++ b/tests/subtests/summaries/test_html_generate.json @@ -25,6 +25,19 @@ "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", @@ -64,6 +77,19 @@ "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", @@ -103,6 +129,19 @@ "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d", + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", "image_status": "generated", diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json index 0a813734..d679f033 100644 --- a/tests/subtests/summaries/test_html_generate_hashes_only.json +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -25,6 +25,19 @@ "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", "image_status": "missing", @@ -64,6 +77,19 @@ "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "image_status": "missing", @@ -103,6 +129,19 @@ "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d", + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "image_status": "missing", diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json index bed75fae..9d1947eb 100644 --- a/tests/subtests/summaries/test_html_generate_images_only.json +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -25,6 +25,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", @@ -64,6 +77,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", "image_status": "generated", diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index a55666ac..42442399 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -25,6 +25,19 @@ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857", "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857" }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e" + }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", "image_status": null, @@ -64,6 +77,19 @@ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43", "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43" }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580" + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab" }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": null, + "hash_status": "missing", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": null, + "result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d" + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index 0b33035e..bd23837d 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -25,6 +25,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", "image_status": "missing", @@ -64,6 +77,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "image_status": "missing", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": null }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": null, + "result_hash": null + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "image_status": "missing", diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 851e67f1..d65ae9cb 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -25,6 +25,19 @@ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857", "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857" }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e" + }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", "image_status": null, @@ -64,6 +77,19 @@ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43", "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43" }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580" + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab" }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "missing", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": null, + "result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d" + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index ea7af1d2..333c1e2c 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -25,6 +25,19 @@ "baseline_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857", "result_hash": "8b5c00365e6f784d5c8947091f09a92fd7d222e790431f297b4848f173e64857" }, + "subtests.subtest.test_hmatch_idiffshape": { + "status": "passed", + "image_status": "diff", + "hash_status": "match", + "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "baseline_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e", + "result_hash": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e" + }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", "image_status": "missing", @@ -64,6 +77,19 @@ "baseline_hash": "d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43", "result_hash": "443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43" }, + "subtests.subtest.test_hdiff_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash 5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 doesn't match hash d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "baseline_hash": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580", + "result_hash": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580" + }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", @@ -103,6 +129,19 @@ "baseline_hash": null, "result_hash": "e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab" }, + "subtests.subtest.test_hmissing_idiffshape": { + "status": "failed", + "image_status": "diff", + "hash_status": "missing", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is 204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(400, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "baseline_hash": null, + "result_hash": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d" + }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",