From 29ad7dc8062ce3a39baca3e0624640104bd3d08e Mon Sep 17 00:00:00 2001 From: codejedi365 Date: Sun, 18 May 2025 23:34:18 -0600 Subject: [PATCH 1/2] test(fixtures): always run e2e tests in very verbose mode for test failure debugging --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 16298e98b..2d081f62b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -207,10 +207,11 @@ def _run_cli( cli_runner = CliRunner(mix_stderr=False) env_vars = {**clean_os_environment, **(env or {})} + args = ["-vv", *(argv or [])] with mock.patch.dict(os.environ, env_vars, clear=True): # run the CLI with the provided arguments - return cli_runner.invoke(main, args=(argv or []), **(invoke_kwargs or {})) + return cli_runner.invoke(main, args=args, **(invoke_kwargs or {})) return _run_cli From 8fb3e7b6a4d0e20c229e627eadf198bdb37b96ea Mon Sep 17 00:00:00 2001 From: codejedi365 Date: Sun, 25 May 2025 13:34:06 -0600 Subject: [PATCH 2/2] test(e2e): update tests to ignore logging messages when validing stderr output --- tests/e2e/cmd_version/test_version.py | 7 +++-- tests/e2e/cmd_version/test_version_print.py | 31 +++++++++++++------- tests/e2e/cmd_version/test_version_strict.py | 7 +++-- tests/e2e/conftest.py | 21 ++++++++++++- tests/e2e/test_main.py | 17 ++++++++--- 5 files changed, 64 insertions(+), 19 deletions(-) diff --git a/tests/e2e/cmd_version/test_version.py b/tests/e2e/cmd_version/test_version.py index 5cb9700cf..0af19f8a3 100644 --- a/tests/e2e/cmd_version/test_version.py +++ b/tests/e2e/cmd_version/test_version.py @@ -26,6 +26,7 @@ from requests_mock import Mocker from tests.conftest import RunCliFn + from tests.e2e.conftest import StripLoggingMessagesFn from tests.fixtures.example_project import GetWheelFileFn, UpdatePyprojectTomlFn from tests.fixtures.git_repo import BuiltRepoResult, GetVersionsFromRepoBuildDefFn @@ -151,6 +152,7 @@ def test_version_on_nonrelease_branch( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): """ Given repo is on a non-release branch, @@ -175,7 +177,7 @@ def test_version_on_nonrelease_branch( # Evaluate (expected -> actual) assert_successful_exit_code(result, cli_cmd) assert not result.stdout - assert expected_error_msg == result.stderr + assert expected_error_msg == strip_logging_messages(result.stderr) # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) tags_after = sorted([tag.name for tag in repo.tags]) @@ -196,6 +198,7 @@ def test_version_on_last_release( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): """ Given repo is on the last release version, @@ -229,7 +232,7 @@ def test_version_on_last_release( # Evaluate (expected -> actual) assert_successful_exit_code(result, cli_cmd) assert f"{latest_release_version}\n" == result.stdout - assert f"{expected_error_msg}\n" == result.stderr + assert f"{expected_error_msg}\n" == strip_logging_messages(result.stderr) # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) assert repo_status_before == repo_status_after diff --git a/tests/e2e/cmd_version/test_version_print.py b/tests/e2e/cmd_version/test_version_print.py index 4efd1e02f..18c0fc5f7 100644 --- a/tests/e2e/cmd_version/test_version_print.py +++ b/tests/e2e/cmd_version/test_version_print.py @@ -35,6 +35,7 @@ from requests_mock import Mocker from tests.conftest import RunCliFn + from tests.e2e.conftest import StripLoggingMessagesFn from tests.fixtures.git_repo import ( BuiltRepoResult, GetCfgValueFromDefFn, @@ -442,6 +443,7 @@ def test_version_print_last_released_prints_version( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] latest_release_version = get_versions_from_repo_build_def( @@ -465,7 +467,7 @@ def test_version_print_last_released_prints_version( # Evaluate assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{latest_release_version}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -493,6 +495,7 @@ def test_version_print_last_released_prints_released_if_commits( mocked_git_push: MagicMock, post_mocker: Mocker, file_in_repo: str, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] latest_release_version = get_versions_from_repo_build_def( @@ -520,7 +523,7 @@ def test_version_print_last_released_prints_released_if_commits( # Evaluate assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{latest_release_version}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -585,6 +588,7 @@ def test_version_print_last_released_on_detached_head( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] latest_release_version = get_versions_from_repo_build_def( @@ -611,7 +615,7 @@ def test_version_print_last_released_on_detached_head( # Evaluate (expected -> actual) assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{latest_release_version}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -632,6 +636,7 @@ def test_version_print_last_released_on_nonrelease_branch( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] latest_release_version = get_versions_from_repo_build_def( @@ -658,7 +663,7 @@ def test_version_print_last_released_on_nonrelease_branch( # Evaluate (expected -> actual) assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{latest_release_version}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -688,6 +693,7 @@ def test_version_print_last_released_tag_prints_correct_tag( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] repo_def = repo_result["definition"] @@ -712,7 +718,7 @@ def test_version_print_last_released_tag_prints_correct_tag( # Evaluate assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{latest_release_tag}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -748,6 +754,7 @@ def test_version_print_last_released_tag_prints_released_if_commits( mocked_git_push: MagicMock, post_mocker: Mocker, file_in_repo: str, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] repo_def = repo_result["definition"] @@ -776,7 +783,7 @@ def test_version_print_last_released_tag_prints_released_if_commits( # Evaluate assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{latest_release_tag}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -850,6 +857,7 @@ def test_version_print_last_released_tag_on_detached_head( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] repo_def = repo_result["definition"] @@ -877,7 +885,7 @@ def test_version_print_last_released_tag_on_detached_head( # Evaluate (expected -> actual) assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{latest_release_tag}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -907,6 +915,7 @@ def test_version_print_last_released_tag_on_nonrelease_branch( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] repo_def = repo_result["definition"] @@ -934,7 +943,7 @@ def test_version_print_last_released_tag_on_nonrelease_branch( # Evaluate (expected -> actual) assert_successful_exit_code(result, cli_cmd) - assert not result.stderr + assert not strip_logging_messages(result.stderr) assert f"{last_release_tag}\n" == result.stdout # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) @@ -961,6 +970,7 @@ def test_version_print_next_version_fails_on_detached_head( get_commit_def_fn: GetCommitDefFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] expected_error_msg = ( @@ -994,7 +1004,7 @@ def test_version_print_next_version_fails_on_detached_head( # Evaluate (expected -> actual) assert_exit_code(1, result, cli_cmd) assert not result.stdout - assert f"{expected_error_msg}\n" == result.stderr + assert f"{expected_error_msg}\n" == strip_logging_messages(result.stderr) # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) assert repo_status_before == repo_status_after @@ -1020,6 +1030,7 @@ def test_version_print_next_tag_fails_on_detached_head( get_commit_def_fn: GetCommitDefFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): repo = repo_result["repo"] expected_error_msg = ( @@ -1053,7 +1064,7 @@ def test_version_print_next_tag_fails_on_detached_head( # Evaluate (expected -> actual) assert_exit_code(1, result, cli_cmd) assert not result.stdout - assert f"{expected_error_msg}\n" == result.stderr + assert f"{expected_error_msg}\n" == strip_logging_messages(result.stderr) # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) assert repo_status_before == repo_status_after diff --git a/tests/e2e/cmd_version/test_version_strict.py b/tests/e2e/cmd_version/test_version_strict.py index 951a9966f..a41998ded 100644 --- a/tests/e2e/cmd_version/test_version_strict.py +++ b/tests/e2e/cmd_version/test_version_strict.py @@ -17,6 +17,7 @@ from requests_mock import Mocker from tests.conftest import RunCliFn + from tests.e2e.conftest import StripLoggingMessagesFn from tests.fixtures.git_repo import BuiltRepoResult, GetVersionsFromRepoBuildDefFn @@ -30,6 +31,7 @@ def test_version_already_released_when_strict( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): """ Given repo has no new changes since the last release, @@ -60,7 +62,7 @@ def test_version_already_released_when_strict( # Evaluate assert_exit_code(2, result, cli_cmd) assert f"{latest_release_version}\n" == result.stdout - assert f"{expected_error_msg}\n" == result.stderr + assert f"{expected_error_msg}\n" == strip_logging_messages(result.stderr) # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) assert repo_status_before == repo_status_after @@ -78,6 +80,7 @@ def test_version_on_nonrelease_branch_when_strict( run_cli: RunCliFn, mocked_git_push: MagicMock, post_mocker: Mocker, + strip_logging_messages: StripLoggingMessagesFn, ): """ Given repo is on a non-release branch, @@ -103,7 +106,7 @@ def test_version_on_nonrelease_branch_when_strict( # Evaluate assert_exit_code(2, result, cli_cmd) assert not result.stdout - assert expected_error_msg == result.stderr + assert expected_error_msg == strip_logging_messages(result.stderr) # assert nothing else happened (no code changes, no commit, no tag, no push, no vcs release) tags_after = sorted([tag.name for tag in repo.tags]) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b64d5aecf..209f3654e 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -2,7 +2,7 @@ import os from pathlib import Path -from re import IGNORECASE, compile as regexp +from re import IGNORECASE, MULTILINE, compile as regexp from typing import TYPE_CHECKING from unittest.mock import MagicMock @@ -47,6 +47,9 @@ class RetrieveRuntimeContextFn(Protocol): def __call__(self, repo: Repo) -> RuntimeContext: ... + class StripLoggingMessagesFn(Protocol): + def __call__(self, log: str) -> str: ... + @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: @@ -116,6 +119,22 @@ def _retrieve_runtime_context(repo: Repo) -> RuntimeContext: return _retrieve_runtime_context +@pytest.fixture(scope="session") +def strip_logging_messages() -> StripLoggingMessagesFn: + """Fixture to strip logging messages from the output.""" + # Log levels match SemanticReleaseLogLevel enum values + logger_msg_pattern = regexp( + r"^\s*(?:\[\d\d:\d\d:\d\d\])?\s*(FATAL|CRITICAL|ERROR|WARNING|INFO|DEBUG|SILLY).*?\n(?:\s+\S.*?\n)*(?!\n[ ]{11})", + MULTILINE, + ) + + def _strip_logging_messages(log: str) -> str: + # Make sure it ends with a newline + return logger_msg_pattern.sub("", log.rstrip("\n") + "\n") + + return _strip_logging_messages + + @pytest.fixture(scope="session") def long_hash_pattern() -> Pattern: return regexp(r"\b([0-9a-f]{40})\b", IGNORECASE) diff --git a/tests/e2e/test_main.py b/tests/e2e/test_main.py index ff290146e..8ce3c58a5 100644 --- a/tests/e2e/test_main.py +++ b/tests/e2e/test_main.py @@ -8,6 +8,7 @@ import git import pytest +from click.testing import CliRunner from pytest_lazy_fixtures.lazy_fixture import lf as lazy_fixture from semantic_release import __version__ @@ -20,6 +21,7 @@ from pathlib import Path from tests.conftest import RunCliFn + from tests.e2e.conftest import StripLoggingMessagesFn from tests.fixtures.example_project import ExProjectDir, UpdatePyprojectTomlFn from tests.fixtures.git_repo import BuiltRepoResult @@ -59,8 +61,13 @@ def test_main_prints_version_and_exits(run_cli: RunCliFn): assert result.output == f"semantic-release, version {__version__}\n" -def test_main_no_args_prints_help_text(run_cli: RunCliFn): - assert_successful_exit_code(run_cli(), [MAIN_PROG_NAME]) +def test_main_no_args_passes_w_help_text(): + from semantic_release.cli.commands.main import main + + cli_cmd = [MAIN_PROG_NAME] + result = CliRunner().invoke(main, prog_name=cli_cmd[0]) + assert_successful_exit_code(result, cli_cmd) + assert "Usage: " in result.output @pytest.mark.parametrize( @@ -210,7 +217,9 @@ def test_errors_when_config_file_does_not_exist_and_passed_explicitly( @pytest.mark.usefixtures(repo_w_no_tags_conventional_commits.__name__) def test_errors_when_config_file_invalid_configuration( - run_cli: RunCliFn, update_pyproject_toml: UpdatePyprojectTomlFn + run_cli: RunCliFn, + update_pyproject_toml: UpdatePyprojectTomlFn, + strip_logging_messages: StripLoggingMessagesFn, ): # Setup update_pyproject_toml("tool.semantic_release.remote.type", "invalidType") @@ -220,7 +229,7 @@ def test_errors_when_config_file_invalid_configuration( result = run_cli(cli_cmd[1:]) # preprocess results - stderr_lines = result.stderr.splitlines() + stderr_lines = strip_logging_messages(result.stderr).splitlines() # Evaluate assert_exit_code(1, result, cli_cmd)