Add support for --indulgent-ordering option

- Author: Andrew Gilbert <[email protected]>

This requests that the plugin be run first, allowing other plugins to
override its ordering. This is useful if the order requirements
provided by this plugin are intended to be priorities, rather than
absolute requirements. For example, if some test cases run slower than
others in your test suite, you can mark them as desirable to run first
when running tests in parallel, allowing a more efficient use of a
multi-core processor. By adding the `--indulgent-ordering` option,
this ordering can still be overridden by, for example, the
`--failed-first` option, which prioritizes tests which failed during
the last run. Without `--indulgent-ordering`, the `--failed-first`
option will be overridden by pytest-ordering, reducing its usefulness
by increasing the total time until the results are available.

Author: mrbean-bremen

AUTHORS

@@ -4,3 +4,4 @@ Ben Greene <[email protected]>
 Adam Talsma <[email protected]>
 Brian Maissy <[email protected]>
 Jonas Zinn <[email protected]>
+Andrew Gilbert <[email protected]>

CHANGELOG

@@ -1,3 +1,9 @@
+Unreleased
+---
+### Added
+- #50 Add ``--indulgent-ordering`` to request that the sort from
+  pytest-ordering be run before other plugins.  This allows the built-in
+  ``--failed-first`` implementation to override the ordering.
 
 0.6
 ---

docs/source/index.rst

@@ -120,6 +120,21 @@ You can also use markers such as "first", "second", "last", and "second_to_last"
 
     =========================== 4 passed in 0.02 seconds ===========================
 
+``--indulgent-ordering`` and overriding ordering
+-------------
+
+You may sometimes find that you want to suggest an ordering of tests, while
+allowing it to be overridden for good reson. For example, if you run your test
+suite in parallel and have a number of tests which are particularly slow, it
+might be desirable to start those tests running first, in order to optimize
+your completion time. You can use the pytest-ordering plugin to inform pytest
+of this. Now suppose you also want to prioritize tests which failed during the
+previous run, by using the ``--failed-first`` option. By default,
+pytest-ordering will override the ``--failed-first`` order, but by adding the
+``--indulgent-ordering`` option, you can ask pytest to run the sort from
+pytest-ordering *before* the sort from ``--failed-first``, allowing the failed
+tests to be sorted to the front.
+
 
 Aspirational
 ============

pytest_ordering/__init__.py

@@ -3,6 +3,8 @@
 import re
 import sys
 
+import pytest
+
 from ._version import __version__
 
 orders_map = {
@@ -26,7 +28,8 @@
 
 
 def pytest_configure(config):
-    """Register the "run" marker."""
+    """Register the "run" marker and configure the plugin depending on the CLI
+    options"""
 
     provided_by_pytest_ordering = (
         'Provided by pytest-ordering. '
@@ -45,6 +48,41 @@ def pytest_configure(config):
                                                    provided_by_pytest_ordering)
         config.addinivalue_line('markers', config_line)
 
+    if config.getoption('indulgent-ordering'):
+        # We need to dynamically add this `tryfirst` decorator to the plugin:
+        # only when the CLI option is present should the decorator be added.
+        # Thus, we manually run the decorator on the class function and
+        # manually replace it.
+        # Python 2.7 didn't allow arbitrary attributes on methods, so we have
+        # to keep the function as a function and then add it to the class as a
+        # pseudomethod.  Since the class is purely for structuring and `self`
+        # is never referenced, this seems reasonable.
+        OrderingPlugin.pytest_collection_modifyitems = pytest.hookimpl(
+            function=modify_items, tryfirst=True)
+    else:
+        OrderingPlugin.pytest_collection_modifyitems = pytest.hookimpl(
+            function=modify_items, trylast=True)
+    config.pluginmanager.register(OrderingPlugin(), 'orderingplugin')
+
+
+def pytest_addoption(parser):
+    """Set up CLI option for pytest"""
+    group = parser.getgroup('ordering')
+    group.addoption('--indulgent-ordering', action='store_true',
+                    dest='indulgent-ordering', help=
+                    '''Request that the sort \
+order provided by pytest-ordering be applied before other sorting, \
+allowing the other sorting to have priority''')
+
+
+class OrderingPlugin(object):
+    """
+    Plugin implementation
+
+    By putting this in a class, we are able to dynamically register it after
+    the CLI is parsed.
+    """
+
 
 def get_filename(item):
     name = item.location[0]
@@ -105,7 +143,6 @@ def mark_binning(item, keys, start, end, before, after, unordered):
 
 
 def insert(items, sort):
-    list_items = []
     if isinstance(items, tuple):
         list_items = items[1]
     else:
@@ -139,7 +176,7 @@ def insert_after(name, items, sort):
     return False
 
 
-def pytest_collection_modifyitems(session, config, items):
+def modify_items(session, config, items):
     before_item = {}
     after_item = {}
     start_item = {}

tests/test_indulgent_ordering.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+import re
+
+import pytest
+import pytest_ordering
+
+pytest_plugins = ["pytester"]
+
+
+def test_run_marker_registered(capsys, tmpdir):
+    testname = str(tmpdir.join("failing.py"))
+    with open(testname, "w") as fi:
+        fi.write(
+            """
+import pytest
+
+@pytest.mark.second
+def test_me_second():
+    assert True
+
+def test_that_fails():
+    assert False
+
+@pytest.mark.first
+def test_me_first():
+    assert True
+"""
+        )
+    args = ["--quiet", "--color=no", testname]
+    pytest.main(args, [pytest_ordering])
+    out, err = capsys.readouterr()
+    assert "..F" in out
+    args.insert(0, "--ff")
+    pytest.main(args, [pytest_ordering])
+    out, err = capsys.readouterr()
+    assert "..F" in out
+    args.insert(0, "--indulgent-ordering")
+    pytest.main(args, [pytest_ordering])
+    out, err = capsys.readouterr()
+    assert "F.." in out

tests/test_ordering.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-import re
 
 import pytest