Added option --error-on-failed-ordering

- makes tests error that cannot be ordered
- closes #140

Author: mrbean-bremen

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### New features
+- added option `--error-on-failed-ordering` to make tests that cannot be ordered fail
+  (see [#140](https://github.com/pytest-dev/pytest-order/issues/140))
+
 ## Infrastructure
 - added missing documentation examples, structured examples according to documentation structure
 

docs/source/configuration.rst

@@ -603,6 +603,49 @@ tests first from the start, then from the end if there are negative numbers,
 and the rest will be in between (e.g. between positive and negative numbers),
 as it is without this option.
 
+``--error-on-failed-ordering``
+------------------------------
+Relative ordering of tests my fail under some circumstances. Mostly this happens if the related marker
+is not found, or if the tests have a cyclic dependency.
+The default behavior in this case is not to order the test in question, issue a warning during test
+collection and execute the test as usual. If you want to make sure that your relative markers work
+without checking all warning messages, you can also make the tests that cannot be ordered fail, so that
+they show up as errored in the report:
+
+.. code:: python
+
+ import pytest
+
+
+ def test_one():
+     assert True
+
+
+ @pytest.mark.order(before="test_three")
+ def test_two():
+     assert True
+
+
+In this example, the test "test_three" on which "test_two" depends, does not exist.
+If you use the option `--error-on-failed-ordering`, "test_two" will now error:
+
+    $ pytest tests -vv --error-on-failed-ordering
+    ============================= test session starts ==============================
+    ...
+    WARNING: cannot execute 'test_two' relative to others: 'test_three'
+
+    test_failed_ordering.py::test_one PASSED
+    test_failed_ordering.py::test_two ERROR
+
+    =================================== ERRORS ====================================
+    __________________________ ERROR at setup of test_two__________________________
+    ...
+    =========================== short test summary info ===========================
+    ERROR test_failed_ordering.py::test_two - Failed: The test could not be ordered
+    ========================= 1 passed, 1 error in 0.75s ==========================
+
+
+
 .. _`pytest-dependency`: https://pypi.org/project/pytest-dependency/
 .. _`dynamic compilation of marked parameters`: https://pytest-dependency.readthedocs.io/en/stable/advanced.html#dynamic-compilation-of-marked-parameters
 .. _`add dependencies at runtime`: https://pytest-dependency.readthedocs.io/en/stable/usage.html#marking-dependencies-at-runtime

pytest_order/item.py

@@ -102,15 +102,20 @@ def sort_numbered_items(self) -> List[Item]:
         return sorted_list
 
     def print_unhandled_items(self) -> None:
-        msg = " ".join(
-            [mark.item.node_id for mark in self.rel_marks]
-            + [mark.item.node_id for mark in self.dep_marks]
-        )
-        if msg:
-            sys.stdout.write("\nWARNING: cannot execute test relative to others: ")
-            sys.stdout.write(msg)
+        failed_items = [mark.item for mark in self.rel_marks] + [
+            mark.item for mark in self.dep_marks
+        ]
+        msg = " ".join([item.node_id for item in failed_items])
+        sys.stdout.write("\nWARNING: cannot execute test relative to others: ")
+        sys.stdout.write(msg)
+        if self.settings.error_on_failed_ordering:
             sys.stdout.write(" - ignoring the marker.\n")
-            sys.stdout.flush()
+        else:
+            sys.stdout.write(".\n")
+        sys.stdout.flush()
+        if self.settings.error_on_failed_ordering:
+            for item in failed_items:
+                item.item.fixturenames.insert(0, "fail_after_cannot_order")
 
     def number_of_rel_groups(self) -> int:
         return len(self.rel_marks) + len(self.dep_marks)

pytest_order/plugin.py

@@ -110,6 +110,15 @@ def pytest_addoption(parser: Parser) -> None:
             "are handled like order markers with an index."
         ),
     )
+    group.addoption(
+        "--error-on-failed-ordering",
+        action="store_true",
+        dest="error_on_failed_ordering",
+        help=(
+            "If set, tests with relative markers that could not be ordered "
+            "will error instead of generating only a warning."
+        ),
+    )
 
 
 def _get_mark_description(mark: Mark):
@@ -146,6 +155,11 @@ def pytest_generate_tests(metafunc):
                 metafunc.parametrize("order", args)
 
 
+@pytest.fixture
+def fail_after_cannot_order():
+    pytest.fail("The test could not be ordered")
+
+
 class OrderingPlugin:
     """
     Plugin implementation.

pytest_order/settings.py

@@ -23,6 +23,9 @@ def __init__(self, config: Config) -> None:
         self.sparse_ordering: bool = config.getoption("sparse_ordering")
         self.order_dependencies: bool = config.getoption("order_dependencies")
         self.marker_prefix: str = config.getoption("order_marker_prefix")
+        self.error_on_failed_ordering: str = config.getoption(
+            "error_on_failed_ordering"
+        )
         scope: str = config.getoption("order_scope")
         if scope in self.valid_scopes:
             self.scope: Scope = self.valid_scopes[scope]

tests/test_relative_ordering.py

@@ -419,6 +419,36 @@ def test_3():
     assert warning in out
 
 
+def test_failing_dependency_fails_test(test_path):
+    test_path.makepyfile(
+        test_failed_ordering="""
+        import pytest
+
+        def test_1():
+            pass
+
+        @pytest.mark.order(before="test_4")
+        def test_2():
+            pass
+
+        def test_3():
+            pass
+        """
+    )
+    result = test_path.runpytest("-v", "--error-on-failed-ordering")
+    if int(pytest.__version__.split(".")[0]) < 6:
+        result.assert_outcomes(passed=2, error=1)
+    else:
+        result.assert_outcomes(passed=2, errors=1)
+    result.stdout.fnmatch_lines(
+        [
+            "test_failed_ordering.py::test_1 PASSED",
+            "test_failed_ordering.py::test_2 ERROR",
+            "test_failed_ordering.py::test_3 PASSED",
+        ]
+    )
+
+
 def test_dependency_in_class_before_unknown_test(item_names_for, capsys):
     test_content = """
         import pytest
@@ -470,6 +500,38 @@ def test_3():
     assert warning in out
 
 
+def test_failed_tests_after_dependency_loop(test_path):
+    test_path.makepyfile(
+        test_failed_ordering="""
+        import pytest
+
+        @pytest.mark.order(after="test_3")
+        def test_1():
+            pass
+
+        @pytest.mark.order(1)
+        def test_2():
+            pass
+
+        @pytest.mark.order(before="test_1")
+        def test_3():
+            pass
+        """
+    )
+    result = test_path.runpytest("-v", "--error-on-failed-ordering")
+    if int(pytest.__version__.split(".")[0]) < 6:
+        result.assert_outcomes(passed=1, error=2)
+    else:
+        result.assert_outcomes(passed=1, errors=2)
+    result.stdout.fnmatch_lines(
+        [
+            "test_failed_ordering.py::test_2 PASSED",
+            "test_failed_ordering.py::test_1 ERROR",
+            "test_failed_ordering.py::test_3 ERROR",
+        ]
+    )
+
+
 def test_dependency_on_parametrized_test(item_names_for):
     test_content = """
         import pytest