diff --git a/AUTHORS b/AUTHORS index 2f8e26b2cb1..7ade456b1ff 100644 --- a/AUTHORS +++ b/AUTHORS @@ -325,6 +325,7 @@ Michał Zięba Mickey Pashov Mihai Capotă Mihail Milushev +Mike Fiedler (miketheman) Mike Hoyle (hoylemd) Mike Lundy Milan Lesnek diff --git a/changelog/14371.feature.rst b/changelog/14371.feature.rst new file mode 100644 index 00000000000..dbfd8636d9f --- /dev/null +++ b/changelog/14371.feature.rst @@ -0,0 +1 @@ +Added :option:`--max-warnings` command-line option and :confval:`max_warnings` configuration option to fail the test run when the number of warnings exceeds a given threshold. diff --git a/doc/en/how-to/capture-warnings.rst b/doc/en/how-to/capture-warnings.rst index e5bc4d27874..909c22ae342 100644 --- a/doc/en/how-to/capture-warnings.rst +++ b/doc/en/how-to/capture-warnings.rst @@ -204,6 +204,42 @@ decorator or to all tests in a module by setting the :globalvar:`pytestmark` var .. _`pytest-warnings`: https://github.com/fschulze/pytest-warnings +Setting a maximum number of warnings +------------------------------------- + +You can use the :option:`--max-warnings` command-line option to fail the test run +if the total number of warnings exceeds a given threshold: + +.. code-block:: bash + + pytest --max-warnings 10 + +If the number of warnings exceeds the threshold, pytest will exit with code ``6`` +(:class:`~pytest.ExitCode` ``WARNINGS_ERROR``). This is useful for gradually +ratcheting down warnings in a codebase. + +The threshold can also be set in the configuration file using :confval:`max_warnings`: + +.. tab:: toml + + .. code-block:: toml + + [pytest] + max_warnings = 10 + +.. tab:: ini + + .. code-block:: ini + + [pytest] + max_warnings = 10 + +.. note:: + + If tests fail, the exit code will be ``1`` (:class:`~pytest.ExitCode` ``TESTS_FAILED``) + regardless of the warning count. ``WARNINGS_ERROR`` is only reported when all tests pass + but the warning threshold is exceeded. + Disabling warnings summary -------------------------- diff --git a/doc/en/reference/exit-codes.rst b/doc/en/reference/exit-codes.rst index 49aaca19121..485bd4fe20a 100644 --- a/doc/en/reference/exit-codes.rst +++ b/doc/en/reference/exit-codes.rst @@ -3,7 +3,7 @@ Exit codes ======================================================== -Running ``pytest`` can result in six different exit codes: +Running ``pytest`` can result in seven different exit codes: :Exit code 0: All tests were collected and passed successfully :Exit code 1: Tests were collected and run but some of the tests failed @@ -11,6 +11,7 @@ Running ``pytest`` can result in six different exit codes: :Exit code 3: Internal error happened while executing tests :Exit code 4: pytest command line usage error :Exit code 5: No tests were collected +:Exit code 6: Maximum number of warnings exceeded (see :option:`--max-warnings`) They are represented by the :class:`pytest.ExitCode` enum. The exit codes being a part of the public API can be imported and accessed directly using: diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index c1df110e822..0b347cc08ef 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1630,6 +1630,31 @@ passed multiple times. The expected format is ``name=value``. For example:: into errors. For more information please refer to :ref:`warnings`. +.. confval:: max_warnings + :type: ``int`` + + Maximum number of warnings allowed before the test run is considered a failure. + When the total number of warnings exceeds this value, pytest exits with + :class:`pytest.ExitCode` ``WARNINGS_ERROR`` (code ``6``). + + .. tab:: toml + + .. code-block:: toml + + [pytest] + max_warnings = 10 + + .. tab:: ini + + .. code-block:: ini + + [pytest] + max_warnings = 10 + + Can also be set via the :option:`--max-warnings` command-line option. + For more information please refer to :ref:`warnings`. + + .. confval:: junit_duration_report :type: ``str`` :default: ``"total"`` @@ -3121,6 +3146,12 @@ Warnings Set which warnings to report, see ``-W`` option of Python itself. Can be specified multiple times. +.. option:: --max-warnings=NUM + + Exit with :class:`pytest.ExitCode` ``WARNINGS_ERROR`` (code ``6``) if the number + of warnings exceeds the given threshold. By default there is no limit. + Can also be set via the :confval:`max_warnings` configuration option. + Doctest ~~~~~~~ @@ -3409,6 +3440,8 @@ All the command-line flags can also be obtained by running ``pytest --help``:: -W, --pythonwarnings PYTHONWARNINGS Set which warnings to report, see -W option of Python itself + --max-warnings=num Exit with error if the number of warnings exceeds + this threshold collection: --collect-only, --co Only collect tests, don't execute them @@ -3531,6 +3564,9 @@ All the command-line flags can also be obtained by running ``pytest --help``:: Each line specifies a pattern for warnings.filterwarnings. Processed after -W/--pythonwarnings. + max_warnings (string): + Maximum number of warnings allowed before failing + the test run norecursedirs (args): Directory patterns to avoid for recursion testpaths (args): Directories to search for tests when no files or directories are given on the command line diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index f7c4de5d7e9..0936201b1fb 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -117,6 +117,8 @@ class ExitCode(enum.IntEnum): USAGE_ERROR = 4 #: pytest couldn't find tests. NO_TESTS_COLLECTED = 5 + #: Maximum number of warnings exceeded. + WARNINGS_ERROR = 6 __module__ = "pytest" diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 02c7fb373fd..50d3bb80974 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -125,6 +125,15 @@ def pytest_addoption(parser: Parser) -> None: action="append", help="Set which warnings to report, see -W option of Python itself", ) + group.addoption( + "--max-warnings", + action="store", + type=int, + default=None, + metavar="num", + dest="max_warnings", + help="Exit with error if the number of warnings exceeds this threshold", + ) parser.addini( "filterwarnings", type="linelist", @@ -132,6 +141,10 @@ def pytest_addoption(parser: Parser) -> None: "warnings.filterwarnings. " "Processed after -W/--pythonwarnings.", ) + parser.addini( + "max_warnings", + help="Maximum number of warnings allowed before failing the test run", + ) group = parser.getgroup("collect", "collection") group.addoption( diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index e9049bb82ec..a0d46a566c9 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -966,11 +966,23 @@ def pytest_sessionfinish( ExitCode.INTERRUPTED, ExitCode.USAGE_ERROR, ExitCode.NO_TESTS_COLLECTED, + ExitCode.WARNINGS_ERROR, ) if exitstatus in summary_exit_codes and not self.no_summary: self.config.hook.pytest_terminal_summary( terminalreporter=self, exitstatus=exitstatus, config=self.config ) + # Check --max-warnings threshold after all warnings have been collected. + max_warnings = self._get_max_warnings() + if max_warnings is not None and session.exitstatus == ExitCode.OK: + warning_count = len(self.stats.get("warnings", [])) + if warning_count > max_warnings: + session.exitstatus = ExitCode.WARNINGS_ERROR + self.write_line( + f"Maximum allowed warnings exceeded: " + f"{warning_count} > {max_warnings}", + red=True, + ) if session.shouldfail: self.write_sep("!", str(session.shouldfail), red=True) if exitstatus == ExitCode.INTERRUPTED: @@ -1057,6 +1069,16 @@ def _getcrashline(self, rep): except AttributeError: return "" + def _get_max_warnings(self) -> int | None: + """Return the max_warnings threshold, from CLI or INI, or None if unset.""" + value = self.config.option.max_warnings + if value is not None: + return int(value) + ini_value = self.config.getini("max_warnings") + if ini_value: + return int(ini_value) + return None + # # Summaries for sessionfinish. # diff --git a/testing/test_warnings.py b/testing/test_warnings.py index 2625f0959e6..0b7a5104fe0 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -5,6 +5,7 @@ import sys import warnings +from _pytest.config import ExitCode from _pytest.fixtures import FixtureRequest from _pytest.pytester import Pytester import pytest @@ -885,3 +886,87 @@ def test_resource_warning(tmp_path): else [] ) result.stdout.fnmatch_lines([*expected_extra, "*1 passed*"]) + + +class TestMaxWarnings: + """Tests for the --max-warnings feature.""" + + PYFILE = """ + import warnings + def test_one(): + warnings.warn(UserWarning("warning one")) + def test_two(): + warnings.warn(UserWarning("warning two")) + """ + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_not_set(self, pytester: Pytester) -> None: + """Without --max-warnings, warnings don't affect exit code.""" + pytester.makepyfile(self.PYFILE) + result = pytester.runpytest() + result.assert_outcomes(passed=2, warnings=2) + assert result.ret == ExitCode.OK + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_not_exceeded(self, pytester: Pytester) -> None: + """When warning count is below the threshold, exit code is OK.""" + pytester.makepyfile(self.PYFILE) + result = pytester.runpytest("--max-warnings", "10") + result.assert_outcomes(passed=2, warnings=2) + assert result.ret == ExitCode.OK + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_exceeded(self, pytester: Pytester) -> None: + """When warning count exceeds threshold, exit code is WARNINGS_ERROR.""" + pytester.makepyfile(self.PYFILE) + result = pytester.runpytest("--max-warnings", "1") + assert result.ret == ExitCode.WARNINGS_ERROR + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_equal_to_count(self, pytester: Pytester) -> None: + """When warning count equals threshold exactly, exit code is OK.""" + pytester.makepyfile(self.PYFILE) + result = pytester.runpytest("--max-warnings", "2") + result.assert_outcomes(passed=2, warnings=2) + assert result.ret == ExitCode.OK + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_zero(self, pytester: Pytester) -> None: + """--max-warnings 0 means no warnings are allowed.""" + pytester.makepyfile(self.PYFILE) + result = pytester.runpytest("--max-warnings", "0") + assert result.ret == ExitCode.WARNINGS_ERROR + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_exceeded_message(self, pytester: Pytester) -> None: + """Verify the output message when max warnings is exceeded.""" + pytester.makepyfile(self.PYFILE) + result = pytester.runpytest("--max-warnings", "1") + result.stdout.fnmatch_lines(["*Maximum allowed warnings exceeded: 2 > 1*"]) + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_ini_option(self, pytester: Pytester) -> None: + """max_warnings can be set via INI configuration.""" + pytester.makeini( + """ + [pytest] + max_warnings = 1 + """ + ) + pytester.makepyfile(self.PYFILE) + result = pytester.runpytest() + assert result.ret == ExitCode.WARNINGS_ERROR + + @pytest.mark.filterwarnings("default::UserWarning") + def test_max_warnings_with_test_failure(self, pytester: Pytester) -> None: + """When tests fail AND warnings exceed max, TESTS_FAILED takes priority.""" + pytester.makepyfile( + """ + import warnings + def test_fail(): + warnings.warn(UserWarning("a warning")) + assert False + """ + ) + result = pytester.runpytest("--max-warnings", "0") + assert result.ret == ExitCode.TESTS_FAILED