Implement --fail-all-on-failed-ordering (#246)

Author: victor-nexthop

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### New features
+* added option `--fail-all-on-failed-ordering` to abort the whole test run
+  without executing any tests if some tests could not be ordered
+
 ### Fixes
 - transitive relative chains are resolved as a single globally consistent order
 

docs/source/configuration.rst

@@ -653,6 +653,36 @@ If you use the option `--error-on-failed-ordering`, "test_two" will now error:
     ERROR test_failed_ordering.py::test_two - Failed: The test could not be ordered
     ========================= 1 passed, 1 error in 0.75s ==========================
 
+``--fail-all-on-failed-ordering``
+---------------------------------
+This option is analogous to ``--error-on-failed-ordering``, but instead of failing only
+the tests that could not be ordered, it aborts the whole test run immediately, without
+executing any tests. This is useful if running the tests in a wrong order has
+unwanted effects.
+
+Using the example shown above::
+
+    $ pytest tests -vv --fail-all-on-failed-ordering
+    ============================= test session starts ==============================
+    ...
+    collected 2 items
+
+    ============================ no tests ran in 0.02s =============================
+    ERROR: pytest-order: cannot execute 'test_two' relative to others: 'test_three'
+
+The test run exits with the usage error exit code (4).
+
+The option also works with ``--collect-only``, as the ordering happens during test
+collection. This allows validating the ordering, e.g. in CI, without executing any
+tests::
+
+    $ pytest tests --collect-only --fail-all-on-failed-ordering
+    ERROR: pytest-order: cannot execute 'test_two' relative to others: 'test_three'
+    ============================= test session starts ==============================
+    ...
+    ========================== 2 tests collected in 0.01s ==========================
+
+The exit code is 4 in this case as well, and 0 if all tests could be ordered.
 
 
 .. _`pytest-dependency`: https://pypi.org/project/pytest-dependency/

src/pytest_order/item.py

@@ -2,7 +2,7 @@
 from typing import Optional, Generic, TypeVar
 from collections import defaultdict
 
-from _pytest.python import Function
+from pytest import Function, UsageError
 
 from .settings import Scope, Settings
 
@@ -176,6 +176,10 @@ def print_unhandled_items(self) -> None:
         msg = " ".join([item.node_id for item in failed_items])
         sys.stdout.write("\nWARNING: cannot execute test relative to others: ")
         sys.stdout.write(msg)
+        if self.settings.fail_all_on_failed_ordering:
+            raise UsageError(
+                f"pytest-order: cannot execute test relative to others: {msg}"
+            )
         if self.settings.error_on_failed_ordering:
             sys.stdout.write(" - ignoring the marker.\n")
         else:

src/pytest_order/plugin.py

@@ -1,11 +1,11 @@
 from collections.abc import Generator, Callable
 
 import pytest
+from pytest import Function
 from _pytest.config import Config
 from _pytest.config.argparsing import Parser
 from _pytest.main import Session
 from _pytest.mark import Mark
-from _pytest.python import Function
 
 from .sorter import Sorter
 
@@ -126,6 +126,15 @@ def pytest_addoption(parser: Parser) -> None:
             "will error instead of generating only a warning."
         ),
     )
+    group.addoption(
+        "--fail-all-on-failed-ordering",
+        action="store_true",
+        dest="fail_all_on_failed_ordering",
+        help=(
+            "If set, the whole test run fails immediately without running "
+            "any tests if some tests with relative markers could not be ordered."
+        ),
+    )
     group.addoption(
         "--order-after-ff",
         action="store_true",

src/pytest_order/settings.py

@@ -26,6 +26,9 @@ def __init__(self, config: Config) -> None:
         self.error_on_failed_ordering: str = config.getoption(
             "error_on_failed_ordering"
         )
+        self.fail_all_on_failed_ordering: bool = config.getoption(
+            "fail_all_on_failed_ordering"
+        )
         scope: str = config.getoption("order_scope")
         if scope in self.valid_scopes:
             self.scope: Scope = self.valid_scopes[scope]

src/pytest_order/sorter.py

@@ -7,7 +7,7 @@
 
 from _pytest.config import Config
 from _pytest.mark import Mark
-from _pytest.python import Function
+from pytest import Function, UsageError
 
 from .item import Item, ItemList, ItemGroup, filter_marks, move_item, RelativeMark
 from .settings import Settings, Scope
@@ -270,15 +270,15 @@ def handle_relative_marks(self, item: Item, mark: Mark) -> bool:
         return has_relative_marks
 
     def warn_about_unknown_test(self, item: Item, rel_mark: str) -> None:
+        msg = f"cannot execute '{item.item.name}' relative to others: '{rel_mark}'"
+        if self.settings.fail_all_on_failed_ordering:
+            raise UsageError(f"pytest-order: {msg}")
         if self.settings.error_on_failed_ordering:
             item.item.fixturenames.insert(0, "fail_after_cannot_order")
             ignore_msg = ""
         else:
             ignore_msg = " - ignoring the marker"
-        sys.stdout.write(
-            f"\nWARNING: cannot execute '{item.item.name}' relative to others: "
-            f"'{rel_mark}'{ignore_msg}."
-        )
+        sys.stdout.write(f"\nWARNING: {msg}{ignore_msg}.")
 
     def collect_markers(self) -> None:
         aliases: dict[str, list[Item]] = {}

tests/test_relative_ordering.py

@@ -474,6 +474,30 @@ def test_3():
     )
 
 
+def test_failing_dependency_fails_run(test_path):
+    test_path.makepyfile(
+        test_failed_ordering="""
+        import pytest
+
+        def test_1():
+            pass
+
+        @pytest.mark.order(before="test_4")
+        def test_2():
+            pass
+
+        def test_3():
+            pass
+        """
+    )
+    result = test_path.runpytest("-v", "--fail-all-on-failed-ordering")
+    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    result.assert_outcomes(passed=0, failed=0)
+    result.stderr.fnmatch_lines(
+        ["ERROR: pytest-order: cannot execute 'test_2' relative to others: 'test_4'"]
+    )
+
+
 def test_dependency_in_class_before_unknown_test(item_names_for, capsys):
     test_content = """
         import pytest
@@ -565,6 +589,35 @@ def test_4():
     )
 
 
+def test_failed_run_after_dependency_loop(test_path):
+    test_path.makepyfile(
+        test_failed_ordering="""
+        import pytest
+
+        @pytest.mark.order(after="test_3")
+        def test_1():
+            pass
+
+        @pytest.mark.order(1)
+        def test_2():
+            pass
+
+        @pytest.mark.order(after="test_1")
+        def test_3():
+            pass
+        """
+    )
+    result = test_path.runpytest("-v", "--fail-all-on-failed-ordering")
+    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    result.assert_outcomes(passed=0, failed=0)
+    result.stderr.fnmatch_lines(
+        [
+            "ERROR: pytest-order: cannot execute test relative to others: "
+            "test_failed_ordering.py::test_* test_failed_ordering.py::test_*"
+        ]
+    )
+
+
 def test_dependency_on_parametrized_test(item_names_for):
     test_content = """
         import pytest