From 88d1bf27871f1362b4c1d140a99183a1364f0099 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Wed, 20 Nov 2024 09:29:24 +0100 Subject: [PATCH 1/9] Release 15.0 (#281) --- CHANGES.rst | 8 +++++++- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3abdade..6cee36f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,13 @@ Changelog ========= -15.0 (unreleased) +15.1 (unreleased) +----------------- + +- Nothing changed yet. + + +15.0 (2024-11-20) ----------------- Breaking changes diff --git a/pyproject.toml b/pyproject.toml index dee3a48..a71e0c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ ignore = [".pre-commit-config.yaml"] [project] name = "pytest-rerunfailures" -version = "15.0.dev0" +version = "15.1.dev0" description = "pytest plugin to re-run tests to eliminate flaky failures" dynamic = [ "readme", From 4bbc0bea3ba206639313d165d7252bd7501c96a5 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Thu, 21 Nov 2024 08:25:02 +0100 Subject: [PATCH 2/9] Fix duplicate keyword. (#282) --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a71e0c6..c5bdb21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ keywords = [ "failures", "flaky", "pytest", - "pytest", "rerun", ] license = {text = "Mozilla Public License 2.0 (MPL 2.0)"} From 5a1a3b2f48ad64e6944354b50eb188378e96dab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=81ajszczak?= Date: Fri, 22 Nov 2024 08:40:45 +0100 Subject: [PATCH 3/9] Initial draft docs (#283) --- .gitignore | 1 + README.rst | 16 +++++++++ docs/changelog.rst | 1 + docs/cli.rst | 82 ++++++++++++++++++++++++++++++++++++++++++ docs/conf.py | 21 +++++++++++ docs/configuration.rst | 63 ++++++++++++++++++++++++++++++++ docs/contributing.rst | 5 +++ docs/index.rst | 18 ++++++++++ docs/installation.rst | 10 ++++++ docs/mark.rst | 82 ++++++++++++++++++++++++++++++++++++++++++ docs/quickstart.rst | 35 ++++++++++++++++++ 11 files changed, 334 insertions(+) create mode 100644 docs/changelog.rst create mode 100644 docs/cli.rst create mode 100644 docs/conf.py create mode 100644 docs/configuration.rst create mode 100644 docs/contributing.rst create mode 100644 docs/index.rst create mode 100644 docs/installation.rst create mode 100644 docs/mark.rst create mode 100644 docs/quickstart.rst diff --git a/.gitignore b/.gitignore index e004867..fab6be7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ bin/ build/ dist/ +docs/_build/ include/ lib/ pip-selfcheck.json diff --git a/README.rst b/README.rst index 1ca22c5..d7804ef 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,13 @@ pytest-rerunfailures ==================== +.. START-SHORT-DESCRIPTION + pytest-rerunfailures is a plugin for `pytest `_ that re-runs tests to eliminate intermittent failures. +.. END-SHORT-DESCRIPTION + .. image:: https://img.shields.io/badge/license-MPL%202.0-blue.svg :target: https://github.com/pytest-dev/pytest-rerunfailures/blob/master/LICENSE :alt: License @@ -14,6 +18,8 @@ re-runs tests to eliminate intermittent failures. :target: https://github.com/pytest-dev/pytest-rerunfailures/actions :alt: GitHub Actions +.. START-INSTALLATION + Requirements ------------ @@ -40,6 +46,8 @@ To install pytest-rerunfailures: $ pip install pytest-rerunfailures +.. END-INSTALLATION + Recover from hard crashes ------------------------- @@ -212,6 +220,8 @@ Here's an example of the output provided by the plugin when run with Note that output will show all re-runs. Tests that fail on all the re-runs will be marked as failed. +.. START-COMPATIBILITY + Compatibility ------------- @@ -222,6 +232,10 @@ Compatibility `flaky `_, you can only have ``pytest-rerunfailures`` or ``flaky`` but not both. +.. END-COMPATIBILITY + +.. START-CONTRIBUTING + Resources --------- @@ -239,3 +253,5 @@ Development @hookimpl(tryfirst=True) def pytest_runtest_makereport(item, call): print(item.execution_count) + +.. END-CONTRIBUTING diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..d9e113e --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1 @@ +.. include:: ../CHANGES.rst diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000..764c76f --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,82 @@ +Command Line Interface Options +============================== + +.. option:: --only-rerun + + **Description**: + Specify regex patterns for errors to rerun. Use this option multiple times to accumulate a list of regexes. + + **Type**: + String (repeatable) + + **Default**: + None + + **Example**: + .. code-block:: bash + + pytest --only-rerun AssertionError --only-rerun ValueError + +.. option:: --reruns + + **Description**: + The number of times to rerun failing tests. + + **Type**: + Integer + + **Default**: + Not set (must be provided) + + **Example**: + .. code-block:: bash + + pytest --reruns 5 + +.. option:: --reruns-delay + + **Description**: + Delay in seconds between reruns. + + **Type**: + Float + + **Default**: + Not set (must be provided) + + **Example**: + .. code-block:: bash + + pytest --reruns 5 --reruns-delay 1 + +.. option:: --rerun-except + + **Description**: + Specify regex patterns for errors to exclude from reruns. Use this option multiple times to accumulate a list of regexes. + + **Type**: + String (repeatable) + + **Default**: + None + + **Example**: + .. code-block:: bash + + pytest --reruns 5 --rerun-except AssertionError --rerun-except OSError + +.. option:: --fail-on-flaky + + **Description**: + If set, the test run will fail with exit code 7 if a flaky test passes on rerun. + + **Type**: + Boolean flag + + **Default**: + False + + **Example**: + .. code-block:: bash + + pytest --fail-on-flaky diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..9f0ce8c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,21 @@ +import os +import sys +from datetime import datetime, timezone + +sys.path.insert(0, os.path.abspath("../src")) + +project = "pytest-rerunfailures" +copyright = ( + f"2012-{datetime.now(tz=timezone.utc).year}, Leah Klearman and pytest-dev team" +) +author = "Leah Klearman" + +extensions = [ + "sphinx.ext.autodoc", +] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 0000000..de56fbe --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,63 @@ +Configuration +============= + +The ``pytest.ini`` configuration file allows you to set default values for the plugin's options, +enabling a consistent test execution environment without the need to specify :doc:`command-line options ` every time. + +Available ``pytest.ini`` Options +-------------------------------- + +Below are the ``pytest.ini`` options supported by the plugin: + +``reruns`` +^^^^^^^^^^ + +- **Description**: Sets the default number of times to rerun failed tests. If not set, you must provide the :option:`--reruns` option on the command line. +- **Type**: String +- **Default**: Not set (must be provided as a CLI argument if not configured). +- **Example**: + + .. code-block:: ini + + [pytest] + reruns = 3 + +``reruns_delay`` +^^^^^^^^^^^^^^^^ + +- **Description**: Sets the default delay (in seconds) between reruns of failed tests. +- **Type**: String +- **Default**: Not set (optional). +- **Example**: + + .. code-block:: ini + + [pytest] + reruns_delay = 2.5 + +Example +------- + +To configure your test environment for consistent retries and delays, add the following options to your ``pytest.ini`` file: + +.. code-block:: ini + + [pytest] + reruns = 3 + reruns_delay = 2.0 + +This setup ensures that: + +- Failed tests will be retried up to 3 times. +- There will be a 2-second delay between each retry. + +Overriding ``pytest.ini`` Options +--------------------------------- + +Command-line arguments always override ``pytest.ini`` settings. For example: + +.. code-block:: bash + + pytest --reruns 5 --reruns-delay 1.5 + +This will retry tests 5 times with a 1.5-second delay, regardless of the values set in ``pytest.ini``. diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..8322209 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,5 @@ +.. include:: ../CONTRIBUTING.rst + +.. include:: ../README.rst + :start-after: .. START-CONTRIBUTING + :end-before: .. END-CONTRIBUTING diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..2c474e0 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,18 @@ +pytest-rerunfailures +==================== + +.. include:: ../README.rst + :start-after: .. START-SHORT-DESCRIPTION + :end-before: .. END-SHORT-DESCRIPTION + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + quickstart + cli + configuration + mark + contributing + changelog diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..4e8a53f --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,10 @@ +Requirements and Installation +============================= + +.. include:: ../README.rst + :start-after: .. START-INSTALLATION + :end-before: .. END-INSTALLATION + +.. include:: ../README.rst + :start-after: .. START-COMPATIBILITY + :end-before: .. END-COMPATIBILITY diff --git a/docs/mark.rst b/docs/mark.rst new file mode 100644 index 0000000..7755178 --- /dev/null +++ b/docs/mark.rst @@ -0,0 +1,82 @@ +Mark Specific Tests as Flaky +============================ + +The ``@pytest.mark.flaky`` decorator allows you to mark individual tests as flaky and configure them to +automatically re-run a specified number of times upon failure. This is particularly useful for specific tests +that are intermittently failing due to non-deterministic conditions (e.g., network latency, race conditions). +That mark also allows to override global settings specified via :doc:`command-line options `. + +Basic Usage +----------- + +To use the ``@pytest.mark.flaky`` decorator, include it in your test function and specify the number of retries using the ``reruns`` argument: + +.. code-block:: python + + @pytest.mark.flaky(reruns=3) + def test_example(): + import random + assert random.choice([True, False]) + +In this example, ``test_example`` will automatically re-run up to 3 times if it fails. + +Additional Options +------------------ + +The ``@pytest.mark.flaky`` decorator supports the following optional arguments: + +``reruns_delay`` +^^^^^^^^^^^^^^^^ + +Specify a delay (in seconds) between re-runs. + +.. code-block:: python + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + def test_example(): + import random + assert random.choice([True, False]) + +This will retry the test 5 times with a 2-second pause between attempts. + +``condition`` +^^^^^^^^^^^^^ + +Re-run the test only if a specified condition is met. +The condition can be any expression that evaluates to ``True`` or ``False``. + +.. code-block:: python + + import sys + + @pytest.mark.flaky(reruns=3, condition=sys.platform.startswith("win32")) + def test_example(): + import random + assert random.choice([True, False]) + +In this example, the test will only be re-run if the operating system is Windows. + + +``only_rerun`` +^^^^^^^^^^^^^^ + +Re-run the test only for specific exception types or patterns. +That overrides the :option:`--only-rerun` command-line option. + +.. code-block:: python + + @pytest.mark.flaky(reruns=5, only_rerun=["AssertionError", "ValueError"]) + def test_example(): + raise AssertionError() + +``rerun_except`` +^^^^^^^^^^^^^^^^ + +Exclude specific exception types or patterns from triggering a re-run. +That overrides the :option:`--rerun-except` command-line option. + +.. code-block:: python + + @pytest.mark.flaky(reruns=5, rerun_except="AssertionError") + def test_example(): + raise ValueError() diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 0000000..aa0e407 --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,35 @@ +Quickstart +========== + +Basic Usage +----------- + +To re-run all test failures, use the :option:`--reruns` option with the maximum number of times you'd like the tests to re-run: + +.. code-block:: bash + + pytest --reruns 3 + +This command will execute the test suite, and any failed tests will be retried up to 3 times. + +Delay Between Re-runs +--------------------- + +To add a delay between re-runs, use the :option:`--reruns-delay` option: + +.. code-block:: bash + + pytest --reruns 3 --reruns-delay 2 + +This will retry failed tests up to 3 times with a 2-second delay between each retry. + +Re-run Specific Failures +------------------------ + +To re-run only specific types of failures, use the :option:`--only-rerun` option with a regular expression. For example: + +.. code-block:: bash + + pytest --reruns 3 --only-rerun AssertionError + +This will re-run failed tests only if they match the error type ``AssertionError``. From c4992c38f17090debe74519eb3945e54e0c01d66 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Tue, 3 Dec 2024 19:53:36 +0100 Subject: [PATCH 4/9] Configure for readthedocs (#284) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/docs.yml | 46 ++++++++++++++++++++++++++++++++++++++ .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 2 +- .readthedocs.yaml | 23 +++++++++++++++++++ README.rst | 3 +++ docs/requirements.txt | 2 ++ tox.ini | 8 +++++++ 7 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..c1000db --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,46 @@ +name: Docs + +on: + push: + pull_request: + schedule: + - cron: '0 12 * * 0' # run once a week on Sunday + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + +env: + FORCE_COLOR: 1 + +jobs: + tests: + name: Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: 3.13 + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: + docs-v1-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + docs-v1- + + - name: Install dependencies + run: | + python -m pip install -U pip tox + + - name: Docs + run: | + tox -e docs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3d1f84a..f8863a8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: ~/.cache/pip ~/.cache/pre-commit key: - lint-v1-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/.pre-commit-config.yaml') }} + lint-v1-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/.pre-commit-config.yaml') }} restore-keys: | lint-v1- diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 44306cc..2396266 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: with: path: ${{ steps.pip-cache.outputs.dir }} key: - ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.pytest-version }}-v1-${{ hashFiles('**/setup.py') }} + ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.pytest-version }}-v1-${{ hashFiles('**/pyproject.toml') }} restore-keys: | ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.pytest-version }}-v1- diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..31511c6 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,23 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-lts-latest + tools: + python: "3" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt + - method: pip + path: . diff --git a/README.rst b/README.rst index d7804ef..5419922 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,9 @@ re-runs tests to eliminate intermittent failures. .. image:: https://github.com/pytest-dev/pytest-rerunfailures/workflows/Test/badge.svg :target: https://github.com/pytest-dev/pytest-rerunfailures/actions :alt: GitHub Actions +.. image:: https://readthedocs.org/projects/pytest-rerunfailures/badge/?version=latest + :target: https://pytest-rerunfailures.readthedocs.io/latest/?badge=latest + :alt: Documentation Status .. START-INSTALLATION diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..ab3f3dd --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +Sphinx +sphinx_rtd_theme diff --git a/tox.ini b/tox.ini index 924a5f7..db4e3ee 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ max-line-length = 88 envlist = linting py{39,310,311,312,313,py3}-pytest{74,80,81,82,83,main} + docs minversion = 4.0 [testenv] @@ -30,3 +31,10 @@ basepython = python3 commands = pre-commit run --all-files --show-diff-on-failure {posargs:} deps = pre-commit>=1.11.0 skip_install = True + +[testenv:docs] +basepython = python3 +skip_install = True +deps = -r docs/requirements.txt +commands = + sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html From d0ff1d4337993cb0f74a46e59f1e6bed0857a0a6 Mon Sep 17 00:00:00 2001 From: Ricardo Busquet Date: Fri, 14 Feb 2025 05:54:41 -0500 Subject: [PATCH 5/9] Adjust `--fail-on-flaky` behavior to improve exit status handling (#288) --- CHANGES.rst | 7 +++- src/pytest_rerunfailures.py | 9 +++-- tests/test_pytest_rerunfailures.py | 57 ++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6cee36f..c8fe1ee 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,12 @@ Changelog 15.1 (unreleased) ----------------- -- Nothing changed yet. +Bug fixes ++++++++++ + +- Fix ``--fail-on-flaky`` option to fail the test run with custom exit code + only when reruns are detected. + (`#287 `_) 15.0 (2024-11-20) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 6197444..48b4761 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -629,5 +629,10 @@ def pytest_sessionfinish(session, exitstatus): return if session.config.option.fail_on_flaky: - if session.config.getvalue("reruns") > 0: - session.exitstatus = 7 + for item in session.items: + if not hasattr(item, "execution_count"): + # no rerun requested + continue + if item.execution_count > 1: + session.exitstatus = 7 + break diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index ccd7141..e6cfe89 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -199,6 +199,63 @@ def test_run_fails_with_code_1_after_consistent_test_failure_even_with_fail_on_f assert result.ret == 1 +def test_run_mark_and_fail_on_flaky_fails_with_custom_error_code_after_pass_on_rerun( + testdir, +): + testdir.makepyfile(f""" + import pytest + + @pytest.mark.flaky(reruns=1) + def test_fail(): + {temporary_failure()} + """) + result = testdir.runpytest("--fail-on-flaky") + assert_outcomes(result, passed=1, rerun=1) + assert result.ret == 7 + + +def test_run_fails_with_code_1_after_test_failure_with_fail_on_flaky_and_mark( + testdir, +): + testdir.makepyfile(""" + import pytest + + @pytest.mark.flaky(reruns=2) + def test_fail(): + assert False + """) + result = testdir.runpytest("--fail-on-flaky") + assert_outcomes(result, passed=0, failed=1, rerun=2) + assert result.ret == 1 + + +def test_run_with_mark_and_fail_on_flaky_succeeds_if_all_tests_pass_without_reruns( + testdir, +): + testdir.makepyfile(""" + import pytest + + @pytest.mark.flaky(reruns=2) + def test_marked_pass(): + assert True + + def test_unmarked_pass(): + assert True + """) + result = testdir.runpytest("--fail-on-flaky") + assert_outcomes(result, passed=2, rerun=0) + assert result.ret == pytest.ExitCode.OK + + +def test_run_with_fail_on_flaky_succeeds_if_all_tests_pass_without_reruns( + testdir, +): + testdir.makepyfile("def test_pass(): assert True") + result = testdir.runpytest("--reruns", "1", "--fail-on-flaky") + assert_outcomes(result, passed=1, rerun=0) + assert result.ret == pytest.ExitCode.OK + + @pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") def test_rerun_passes_after_temporary_test_crash(testdir): # note: we need two tests because there is a bug where xdist From 3064a7782efbee1ede8f76ae59d87b670b10b829 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 26 Mar 2025 08:55:00 +0100 Subject: [PATCH 6/9] Use SPDX license identifier (#289) --- .pre-commit-config.yaml | 2 +- CHANGES.rst | 2 ++ pyproject.toml | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 725fe58..b6e8611 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,6 +28,6 @@ repos: - id: pyproject-fmt additional_dependencies: [ tox ] - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.16 + rev: v0.24.1 hooks: - id: validate-pyproject diff --git a/CHANGES.rst b/CHANGES.rst index c8fe1ee..3d13224 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,8 @@ Bug fixes only when reruns are detected. (`#287 `_) +- Switched to using the SPDX license identifier in the project metadata. + 15.0 (2024-11-20) ----------------- diff --git a/pyproject.toml b/pyproject.toml index c5bdb21..0d07c59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,14 +20,13 @@ keywords = [ "pytest", "rerun", ] -license = {text = "Mozilla Public License 2.0 (MPL 2.0)"} +license.text = "MPL-2.0" authors = [{name = "Leah Klearman", email = "lklrmn@gmail.com"}] requires-python = ">=3.9" classifiers = [ "Development Status :: 5 - Production/Stable", "Framework :: Pytest", "Intended Audience :: Developers", - "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", From c5a282cf4e1b20452440da6c16baa9ef9dda83a3 Mon Sep 17 00:00:00 2001 From: Yugo Kato Date: Wed, 7 May 2025 23:32:36 -0700 Subject: [PATCH 7/9] Update exception filtering to match against actual exception (#293) --- CHANGES.rst | 6 +++++ README.rst | 7 ------ src/pytest_rerunfailures.py | 37 +++++++++++++++--------------- tests/test_pytest_rerunfailures.py | 16 +++++++++---- 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3d13224..2298c57 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,12 @@ Bug fixes - Switched to using the SPDX license identifier in the project metadata. +Other changes ++++++++++++++ + +- Change exception filtering logic to match ``AssertionError`` raised via + ``assert`` statements when filtering by "AssertionError". + (`#292 `_) 15.0 (2024-11-20) ----------------- diff --git a/README.rst b/README.rst index 5419922..ecafcbd 100644 --- a/README.rst +++ b/README.rst @@ -118,13 +118,6 @@ would only rerun those errors that does not match with ``AssertionError`` or ``O $ pytest --reruns 5 --rerun-except AssertionError --rerun-except OSError -.. note:: - - When the ```AssertionError``` comes from the use of the ``assert`` keyword, - use ``--rerun-except assert`` instead:: - - $ pytest --reruns 5 --rerun-except assert - Re-run individual failures -------------------------- diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 48b4761..dba6718 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -258,26 +258,24 @@ def _get_rerun_filter_regex(item, regex_name): return regex -def _matches_any_rerun_error(rerun_errors, report): - return _try_match_reprcrash(rerun_errors, report) +def _matches_any_rerun_error(rerun_errors, excinfo): + return _try_match_error(rerun_errors, excinfo) -def _matches_any_rerun_except_error(rerun_except_errors, report): - return _try_match_reprcrash(rerun_except_errors, report) +def _matches_any_rerun_except_error(rerun_except_errors, excinfo): + return _try_match_error(rerun_except_errors, excinfo) -def _try_match_reprcrash(rerun_errors, report): - for rerun_regex in rerun_errors: - try: - if re.search(rerun_regex, report.longrepr.reprcrash.message): - return True - except AttributeError: - if re.search(rerun_regex, report.longreprtext): +def _try_match_error(rerun_errors, excinfo): + if excinfo: + err = f"{excinfo.type.__name__}: {excinfo.value}" + for rerun_regex in rerun_errors: + if re.search(rerun_regex, err): return True return False -def _should_hard_fail_on_error(item, report): +def _should_hard_fail_on_error(item, report, excinfo): if report.outcome != "failed": return False @@ -290,24 +288,24 @@ def _should_hard_fail_on_error(item, report): elif rerun_errors and (not rerun_except_errors): # Using --only-rerun but not --rerun-except - return not _matches_any_rerun_error(rerun_errors, report) + return not _matches_any_rerun_error(rerun_errors, excinfo) elif (not rerun_errors) and rerun_except_errors: # Using --rerun-except but not --only-rerun - return _matches_any_rerun_except_error(rerun_except_errors, report) + return _matches_any_rerun_except_error(rerun_except_errors, excinfo) else: # Using both --only-rerun and --rerun-except - matches_rerun_only = _matches_any_rerun_error(rerun_errors, report) + matches_rerun_only = _matches_any_rerun_error(rerun_errors, excinfo) matches_rerun_except = _matches_any_rerun_except_error( - rerun_except_errors, report + rerun_except_errors, excinfo ) return (not matches_rerun_only) or matches_rerun_except def _should_not_rerun(item, report, reruns): xfail = hasattr(report, "wasxfail") - is_terminal_error = _should_hard_fail_on_error(item, report) + is_terminal_error = item._terminal_errors[report.when] condition = get_reruns_condition(item) return ( item.execution_count > reruns @@ -530,8 +528,9 @@ def pytest_runtest_makereport(item, call): _test_failed_statuses = getattr(item, "_test_failed_statuses", {}) _test_failed_statuses[result.when] = result.failed item._test_failed_statuses = _test_failed_statuses - - item._terminal_errors[result.when] = _should_hard_fail_on_error(item, result) + item._terminal_errors[result.when] = _should_hard_fail_on_error( + item, result, call.excinfo + ) def pytest_runtest_protocol(item, nextitem): diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index e6cfe89..afb706f 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -598,12 +598,18 @@ def pytest_runtest_logfinish(nodeid, location): ], ) def test_only_rerun_flag(testdir, only_rerun_texts, should_rerun): - testdir.makepyfile('def test_only_rerun(): raise AssertionError("ERR")') + testdir.makepyfile(""" + def test_only_rerun1(): + raise AssertionError("ERR") - num_failed = 1 + def test_only_rerun2(): + assert False, "ERR" + """) + + num_failed = 2 num_passed = 0 - num_reruns = 1 - num_reruns_actual = num_reruns if should_rerun else 0 + num_reruns = 2 + num_reruns_actual = num_reruns * 2 if should_rerun else 0 pytest_args = ["--reruns", str(num_reruns)] for only_rerun_text in only_rerun_texts: @@ -1205,7 +1211,7 @@ def test_1(self): raise AssertionError("fail") def test_2(self): - assert False + raise ValueError("fail") """ ) From 1ddc2455289ef9a25c8b8736dfb84b401ca9be89 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Thu, 8 May 2025 08:35:32 +0200 Subject: [PATCH 8/9] Pimp MANIFEST. --- MANIFEST.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index f88eebe..6083763 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,10 @@ include *.rst include *.toml include LICENSE include tox.ini + +include *.yaml +exclude *.yaml + +recursive-include docs *.py +recursive-include docs *.rst +recursive-include docs *.txt From beaa1a46d3d57e30c8e6fa2ad26d42721eeb7d73 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Thu, 8 May 2025 08:36:15 +0200 Subject: [PATCH 9/9] Preparing release 15.1 --- CHANGES.rst | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2298c57..84dd598 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,7 @@ Changelog ========= -15.1 (unreleased) +15.1 (2025-05-08) ----------------- Bug fixes diff --git a/pyproject.toml b/pyproject.toml index 0d07c59..acddd46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ ignore = [".pre-commit-config.yaml"] [project] name = "pytest-rerunfailures" -version = "15.1.dev0" +version = "15.1" description = "pytest plugin to re-run tests to eliminate flaky failures" dynamic = [ "readme",