diff --git a/AUTHORS b/AUTHORS index 9539e8dc4f4..4c0227de2af 100644 --- a/AUTHORS +++ b/AUTHORS @@ -463,6 +463,7 @@ Tomer Keren Tony Narlock Tor Colvin Trevor Bekolay +Trey Shaffer Tushar Sadhwani Tyler Goodlet Tyler Smart diff --git a/changelog/13201.bugfix.rst b/changelog/13201.bugfix.rst new file mode 100644 index 00000000000..6808eed8b7e --- /dev/null +++ b/changelog/13201.bugfix.rst @@ -0,0 +1 @@ +Fixed color inconsistency in verbose mode where test status showed green instead of yellow for passed tests with warnings. diff --git a/codecov.yml b/codecov.yml index c37e5ec4a09..51d5c0fb8eb 100644 --- a/codecov.yml +++ b/codecov.yml @@ -9,5 +9,7 @@ coverage: patch: default: target: 100% # require patches to be 100% + paths: + - "src/" # only check source files, not test files project: false comment: false diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index ed62c9e345e..cb7ae87fbd4 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -635,10 +635,19 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: return if markup is None: was_xfail = hasattr(report, "wasxfail") - if rep.passed and not was_xfail: - markup = {"green": True} - elif rep.passed and was_xfail: - markup = {"yellow": True} + # Check if report has warnings via user_properties + from _pytest.warnings import HAS_WARNINGS_KEY + + has_warnings = any( + name == HAS_WARNINGS_KEY and value is True + for name, value in getattr(report, "user_properties", []) + ) + + if rep.passed: + if was_xfail or has_warnings: + markup = {"yellow": True} + else: + markup = {"green": True} elif rep.failed: markup = {"red": True} elif rep.skipped: diff --git a/src/_pytest/warnings.py b/src/_pytest/warnings.py index 1dbf0025a31..69bc2d41b89 100644 --- a/src/_pytest/warnings.py +++ b/src/_pytest/warnings.py @@ -6,6 +6,7 @@ from contextlib import ExitStack import sys from typing import Literal +from typing import TYPE_CHECKING import warnings from _pytest.config import apply_warning_filters @@ -13,11 +14,23 @@ from _pytest.config import parse_warning_filter from _pytest.main import Session from _pytest.nodes import Item +from _pytest.stash import StashKey from _pytest.terminal import TerminalReporter from _pytest.tracemalloc import tracemalloc_message import pytest +if TYPE_CHECKING: + from _pytest.reports import TestReport + from _pytest.runner import CallInfo + +# StashKey for storing warning log on items +warning_captured_log_key = StashKey[list[warnings.WarningMessage]]() + +# Key name for storing warning flag in report.user_properties +HAS_WARNINGS_KEY = "has_warnings" + + @contextmanager def catch_warnings_for_item( config: Config, @@ -51,6 +64,9 @@ def catch_warnings_for_item( for mark in item.iter_markers(name="filterwarnings"): for arg in mark.args: warnings.filterwarnings(*parse_warning_filter(arg, escape=False)) + # Store the warning log on the item so it can be accessed during reporting + if record and log is not None: + item.stash[warning_captured_log_key] = log try: yield @@ -89,6 +105,21 @@ def pytest_runtest_protocol(item: Item) -> Generator[None, object, object]: return (yield) +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport( + item: Item, call: CallInfo[None] +) -> Generator[None, TestReport, None]: + """Attach warning information to test reports for terminal coloring.""" + outcome = yield + report: TestReport = outcome.get_result() + + # Only mark warnings during the call phase, not setup/teardown + if report.passed and report.when == "call": + warning_log = item.stash.get(warning_captured_log_key, None) + if warning_log is not None and len(warning_log) > 0: + report.user_properties.append((HAS_WARNINGS_KEY, True)) + + @pytest.hookimpl(wrapper=True, tryfirst=True) def pytest_collection(session: Session) -> Generator[None, object, object]: config = session.config diff --git a/testing/test_terminal.py b/testing/test_terminal.py index e6b77ae5546..bac8ff31720 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2191,7 +2191,7 @@ def test_foobar(i): raise ValueError() [ r"test_axfail.py {yellow}x{reset}{green} \s+ \[ 4%\]{reset}", r"test_bar.py ({green}\.{reset}){{10}}{green} \s+ \[ 52%\]{reset}", - r"test_foo.py ({green}\.{reset}){{5}}{yellow} \s+ \[ 76%\]{reset}", + r"test_foo.py ({yellow}\.{reset}){{5}}{yellow} \s+ \[ 76%\]{reset}", r"test_foobar.py ({red}F{reset}){{5}}{red} \s+ \[100%\]{reset}", ] ) @@ -2208,6 +2208,179 @@ def test_foobar(i): raise ValueError() ) ) + def test_verbose_colored_warnings( + self, pytester: Pytester, monkeypatch, color_mapping + ) -> None: + """Test that verbose mode shows yellow PASSED for tests with warnings.""" + monkeypatch.setenv("PY_COLORS", "1") + pytester.makepyfile( + test_warning=""" + import warnings + def test_with_warning(): + warnings.warn("test warning", DeprecationWarning) + + def test_without_warning(): + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.re_match_lines( + color_mapping.format_for_rematch( + [ + r"test_warning.py::test_with_warning {yellow}PASSED{reset}{green} \s+ \[ 50%\]{reset}", + r"test_warning.py::test_without_warning {green}PASSED{reset}{yellow} \s+ \[100%\]{reset}", + ] + ) + ) + + def test_verbose_colored_warnings_xdist( + self, pytester: Pytester, monkeypatch, color_mapping + ) -> None: + """Test that warning coloring works correctly with pytest-xdist parallel execution.""" + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + monkeypatch.setenv("PY_COLORS", "1") + pytester.makepyfile( + test_warning_xdist=""" + import warnings + def test_with_warning_1(): + warnings.warn("warning in test 1", DeprecationWarning) + pass + + def test_with_warning_2(): + warnings.warn("warning in test 2", DeprecationWarning) + pass + + def test_without_warning(): + pass + """ + ) + + output = pytester.runpytest("-v", "-n2") + # xdist outputs in random order, and uses format: + # [gw#][cyan] [%] [reset][color]STATUS[reset] test_name + # Note: \x1b[36m is cyan, which isn't in color_mapping + output.stdout.re_match_lines_random( + color_mapping.format_for_rematch( + [ + r"\[gw\d\]\x1b\[36m \[\s*\d+%\] {reset}{yellow}PASSED{reset} " + r"test_warning_xdist.py::test_with_warning_1", + r"\[gw\d\]\x1b\[36m \[\s*\d+%\] {reset}{yellow}PASSED{reset} " + r"test_warning_xdist.py::test_with_warning_2", + r"\[gw\d\]\x1b\[36m \[\s*\d+%\] {reset}{green}PASSED{reset} " + r"test_warning_xdist.py::test_without_warning", + ] + ) + ) + + def test_failed_test_with_warnings_shows_red( + self, pytester: Pytester, monkeypatch, color_mapping + ) -> None: + """Test that failed tests with warnings show RED, not yellow.""" + monkeypatch.setenv("PY_COLORS", "1") + pytester.makepyfile( + test_failed_warning=""" + import warnings + def test_fails_with_warning(): + warnings.warn("This will fail", DeprecationWarning) + assert False, "Expected failure" + + def test_passes_with_warning(): + warnings.warn("This passes", DeprecationWarning) + assert True + """ + ) + result = pytester.runpytest("-v") + # Failed test should be RED even though it has warnings + result.stdout.re_match_lines( + color_mapping.format_for_rematch( + [ + r"test_failed_warning.py::test_fails_with_warning {red}FAILED{reset}", + r"test_failed_warning.py::test_passes_with_warning {yellow}PASSED{reset}", + ] + ) + ) + + def test_non_verbose_mode_with_warnings( + self, pytester: Pytester, monkeypatch, color_mapping + ) -> None: + """Test that non-verbose mode (dot output) works correctly with warnings.""" + monkeypatch.setenv("PY_COLORS", "1") + pytester.makepyfile( + test_dots=""" + import warnings + def test_with_warning(): + warnings.warn("warning", DeprecationWarning) + pass + + def test_without_warning(): + pass + """ + ) + result = pytester.runpytest() # No -v flag + # Should show dots, yellow for warning, green for clean pass + result.stdout.re_match_lines( + color_mapping.format_for_rematch( + [ + r"test_dots.py {yellow}\.{reset}{green}\.{reset}", + ] + ) + ) + + def test_multiple_warnings_single_test( + self, pytester: Pytester, monkeypatch, color_mapping + ) -> None: + """Test that tests with multiple warnings still show yellow.""" + monkeypatch.setenv("PY_COLORS", "1") + pytester.makepyfile( + test_multi=""" + import warnings + def test_multiple_warnings(): + warnings.warn("warning 1", DeprecationWarning) + warnings.warn("warning 2", DeprecationWarning) + warnings.warn("warning 3", DeprecationWarning) + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.re_match_lines( + color_mapping.format_for_rematch( + [ + r"test_multi.py::test_multiple_warnings {yellow}PASSED{reset}", + ] + ) + ) + + def test_warning_with_filterwarnings_mark( + self, pytester: Pytester, monkeypatch, color_mapping + ) -> None: + """Test that warnings with filterwarnings mark still show yellow.""" + monkeypatch.setenv("PY_COLORS", "1") + pytester.makepyfile( + test_marked=""" + import warnings + import pytest + + @pytest.mark.filterwarnings("ignore::DeprecationWarning") + def test_with_ignored_warning(): + warnings.warn("ignored warning", DeprecationWarning) + pass + + def test_with_visible_warning(): + warnings.warn("visible warning", DeprecationWarning) + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.re_match_lines( + color_mapping.format_for_rematch( + [ + r"test_marked.py::test_with_ignored_warning {green}PASSED{reset}", + r"test_marked.py::test_with_visible_warning {yellow}PASSED{reset}", + ] + ) + ) + def test_count(self, many_tests_files, pytester: Pytester) -> None: pytester.makeini( """ diff --git a/testing/test_warnings.py b/testing/test_warnings.py index 4800a916eac..a6c8eb970dd 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -888,3 +888,85 @@ def test_resource_warning(tmp_path): else [] ) result.stdout.fnmatch_lines([*expected_extra, "*1 passed*"]) + + +def test_warning_captured_in_user_properties(pytester: Pytester) -> None: + """Test that warnings are captured in report.user_properties for terminal coloring.""" + pytester.makepyfile( + """ + import warnings + def test_with_warning(): + warnings.warn("test warning", DeprecationWarning) + assert True + + def test_without_warning(): + assert True + """ + ) + # Use inline_run to get access to reports + reprec = pytester.inline_run() + reports = reprec.getreports("pytest_runtest_logreport") + + # Find the call phase reports + call_reports = [r for r in reports if r.when == "call"] + assert len(call_reports) == 2 + + # First test should have warnings in user_properties + test_with_warning_report = call_reports[0] + assert test_with_warning_report.nodeid.endswith("test_with_warning") + assert test_with_warning_report.passed + + # Check that HAS_WARNINGS_KEY is in user_properties + from _pytest.warnings import HAS_WARNINGS_KEY + + has_warnings = any( + name == HAS_WARNINGS_KEY and value is True + for name, value in test_with_warning_report.user_properties + ) + assert has_warnings, ( + "Expected HAS_WARNINGS_KEY in user_properties for test with warning" + ) + + # Second test should NOT have warnings in user_properties + test_without_warning_report = call_reports[1] + assert test_without_warning_report.nodeid.endswith("test_without_warning") + assert test_without_warning_report.passed + + has_warnings = any( + name == HAS_WARNINGS_KEY and value is True + for name, value in test_without_warning_report.user_properties + ) + assert not has_warnings, "Did not expect HAS_WARNINGS_KEY for test without warning" + + +def test_warning_stash_storage(pytester: Pytester) -> None: + """Test that warning log is stored in item.stash during test execution.""" + pytester.makepyfile( + """ + import warnings + + def test_with_warning(): + warnings.warn("test warning", DeprecationWarning) + pass + """ + ) + + # Use a plugin to capture the item and check the stash + captured_item = [] + + class StashChecker: + def pytest_runtest_call(self, item): + captured_item.append(item) + + pytester.inline_run(plugins=[StashChecker()]) + + assert len(captured_item) == 1 + item = captured_item[0] + + # Check that the warning log was stored in the stash + from _pytest.warnings import warning_captured_log_key + + warning_log = item.stash.get(warning_captured_log_key, None) + assert warning_log is not None, "Expected warning log to be stored in item.stash" + assert len(warning_log) > 0, "Expected at least one warning in the log" + assert "test warning" in str(warning_log[0].message)