Merge pull request #2 from jbasko/dev

Docs & plugin failure handling

Author: jbasko

README.rst

@@ -1,17 +1,45 @@
 pytest-random-order
 ===================================
 
-**pytest** plugin to randomise the order of tests within module, package, or globally.
+This is a **pytest** plugin to randomise the order in which tests are run with a little bit of control
+over how much randomness one allows.
 
-This plugin allows you to control the level at which the order of tests is randomised
-through ``--random-order-mode`` command line option.
+It is a good idea to randomise the order in which your tests run
+because a test running early as part of a larger suite of tests may have left
+the system under test in a particularly fortunate state for a subsequent test to pass.
+
+How Random
+__________
+
+**pytest-random-order** groups tests in buckets, shuffles them within buckets and then shuffles the buckets.
+
+User can choose among four types of buckets to use:
+
+    * ``class``
+    * ``module`` -- **this is the default setting**
+    * ``package``
+    * ``global`` -- all tests fall in the same bucket, full randomness, tests probably take longer to run
+
+If you have three buckets of tests ``A``, ``B``, and ``C`` with three tests ``1`` and ``2``, and ``3`` in each of them,
+then here are just two of many potential orderings that non-global randomisation can produce:
+
+::
+
+    C2  C1  C3  A3  A1  A2  B3  B2  B1
+
+    A2  A1  A3  C1  C2  C3  B2  B1  B3
+
+As you can see, all C tests are executed "next" to each other and so are tests in buckets A and B.
+Tests from any bucket X are guaranteed to not be interspersed with tests from another bucket Y.
+For example, if you choose bucket type ``module`` then bucket X contains all tests that are in this module.
+
+Note that modules (and hence tests inside those modules) that belong to package ``x.y`` do not belong
+to package ``x.y.z``, so they will fall in different buckets when randomising with ``package`` bucket type.
 
 By default, your tests will be randomised at ``module`` level which means that
 tests within a single module X will be executed in no particular order, but tests from
 other modules will not be mixed in between tests of module X.
 
-Similarly, you can randomise the order of tests at ``package`` and ``global`` levels.
-
 ----
 
 Installation
@@ -25,21 +53,38 @@ Installation
 Usage
 -----
 
+The plugin is enabled by default. To randomise the order of tests within modules, just run pytest as always:
+
 ::
 
     $ pytest -v
 
-    $ pytest -v --random-order-mode=global
+It is best to start with smallest bucket type (``class`` or ``module`` depending on whether you have class-based tests),
+and switch to a larger bucket type when you are sure your tests handle that.
+
+If your tests rely on fixtures that are module or session-scoped, more randomised order of tests will mean slower tests.
+You probably don't want to randomise at ``global`` or ``package`` level while you are developing and need a quick confirmation
+that nothing big is broken.
+
+::
+
+    $ pytest -v --random-order-bucket=class
 
-    $ pytest -v --random-order-mode=package
+    $ pytest -v --random-order-bucket=module
 
-    $ pytest -v --random-order-mode=module
+    $ pytest -v --random-order-bucket=package
 
-    $ pytest -v --random-order-mode=class
+    $ pytest -v --random-order-bucket=global
+
+If the plugin misbehaves or you just want to assure yourself that it is not the plugin making your tests fail or
+pass undeservedly, you can disable it:
+
+::
+
+    $ pytest -p no:random-order -v
 
 
 License
 -------
 
 Distributed under the terms of the MIT license, "pytest-random-order" is free and open source software
-

pytest_random_order.py

@@ -1,14 +1,16 @@
 # -*- coding: utf-8 -*-
 
 import random
+import sys
+import traceback
 
 
 def pytest_addoption(parser):
     group = parser.getgroup('random-order')
     group.addoption(
-        '--random-order-mode',
+        '--random-order-bucket',
         action='store',
-        dest='random_order_mode',
+        dest='random_order_bucket',
         default='module',
         choices=('global', 'package', 'module', 'class'),
         help='Limit reordering of test items across units of code',
@@ -18,9 +20,9 @@ def pytest_addoption(parser):
 def pytest_report_header(config):
     out = None
 
-    if config.getoption('random_order_mode'):
-        mode = config.getoption('random_order_mode')
-        out = "Using --random-order-mode={0}".format(mode)
+    if config.getoption('random_order_bucket'):
+        bucket = config.getoption('random_order_bucket')
+        out = "Using --random-order-bucket={0}".format(bucket)
 
     return out
 
@@ -75,6 +77,34 @@ def _shuffle_items(items, key=None, preserve_bucket_order=False):
     return
 
 
+def _get_set_of_item_ids(items):
+    s = {}
+    try:
+        s = set(item.nodeid for item in items)
+    finally:
+        return s
+
+
 def pytest_collection_modifyitems(session, config, items):
-    shuffle_mode = config.getoption('random_order_mode')
-    _shuffle_items(items, key=_random_order_item_keys[shuffle_mode])
+    failure = None
+    item_ids = _get_set_of_item_ids(items)
+
+    try:
+        shuffle_mode = config.getoption('random_order_bucket')
+        _shuffle_items(items, key=_random_order_item_keys[shuffle_mode])
+
+    except Exception as e:
+        # If the number of items is still the same, we assume that we haven't messed up too hard
+        # and we can just return the list of items as it is.
+        _, _, exc_tb = sys.exc_info()
+        failure = 'pytest-random-order plugin has failed with {!r}:\n{}'.format(
+            e, ''.join(traceback.format_tb(exc_tb, 10))
+        )
+        config.warn(0, failure, None)
+
+    finally:
+        # Fail only if we have lost user's tests
+        if item_ids != _get_set_of_item_ids(items):
+            if not failure:
+                failure = 'pytest-random-order plugin has failed miserably'
+            raise RuntimeError(failure)

requirements.txt

@@ -0,0 +1,4 @@
+pytest
+tox
+coverage
+py

setup.py

@@ -13,7 +13,7 @@ def read(fname):
 
 setup(
     name='pytest-random-order',
-    version='0.2.5',
+    version='0.3.0',
     author='Jazeps Basko',
     author_email='[email protected]',
     maintainer='Jazeps Basko',

tests/test_actual_test_runs.py

@@ -104,7 +104,7 @@ def test_ex2(self):
     return testdir
 
 
-def check_call_sequence(seq, shuffle_mode='module'):
+def check_call_sequence(seq, bucket='module'):
     all_values = collections.defaultdict(list)
     num_switches = collections.defaultdict(int)
 
@@ -131,39 +131,39 @@ def inspect_attr(this_call, prev_call, attr_name):
     # These are just sanity tests, the actual shuffling is tested in test_shuffle,
     # assertions here are very relaxed.
 
-    if shuffle_mode == 'global':
+    if bucket == 'global':
         if num_module_switches <= num_modules:
             pytest.fail('Too few module switches for global shuffling')
         if num_package_switches <= num_packages:
             pytest.fail('Too few package switches for global shuffling')
 
-    elif shuffle_mode == 'package':
+    elif bucket == 'package':
         assert num_package_switches == num_packages
         if num_module_switches <= num_modules:
             pytest.fail('Too few module switches for package-limited shuffling')
 
-    elif shuffle_mode == 'module':
+    elif bucket == 'module':
         assert num_module_switches == num_modules
 
-    elif shuffle_mode == 'class':
+    elif bucket == 'class':
         # Each class can contribute to 1 or 2 switches.
         assert num_class_switches <= num_classes * 2
 
-        # Class shuffle is a subset of module shuffle.
+        # Class bucket is a special case of module bucket.
         # We have two classes in one module and these could be reshuffled so
         # the module could appear in sequence of buckets two times.
         assert num_modules <= num_module_switches <= num_modules + 1
 
 
-@pytest.mark.parametrize('mode', ['class', 'module', 'package', 'global'])
-def test_it_works_with_actual_tests(tmp_tree_of_tests, mode):
+@pytest.mark.parametrize('bucket', ['class', 'module', 'package', 'global'])
+def test_it_works_with_actual_tests(tmp_tree_of_tests, bucket):
     sequences = set()
 
     for x in range(5):
-        result = tmp_tree_of_tests.runpytest('--random-order-mode={}'.format(mode), '--verbose')
+        result = tmp_tree_of_tests.runpytest('--random-order-bucket={}'.format(bucket), '--verbose')
         result.assert_outcomes(passed=14, failed=3)
         seq = get_runtest_call_sequence(result)
-        check_call_sequence(seq, shuffle_mode=mode)
+        check_call_sequence(seq, bucket=bucket)
         assert len(seq) == 17
         sequences.add(seq)
 

tests/test_cli.py

@@ -4,5 +4,5 @@ def test_help_message(testdir):
     )
     result.stdout.fnmatch_lines([
         'random-order:',
-        '*--random-order-mode={global,package,module,class}*',
+        '*--random-order-bucket={global,package,module,class}*',
     ])

tests/test_plugin_failure.py

@@ -0,0 +1,57 @@
+import pytest
+
+
+def acceptably_failing_shuffle_items(items, **kwargs):
+    # Does not mess up items collection
+    raise ValueError('shuffling failed')
+
+
+def critically_failing_shuffle_items(items, **kwargs):
+    # Messes up items collection, an item is effectively lost
+    items[1] = items[0]
+    raise ValueError('shuffling failed')
+
+
+def critically_not_failing_shuffle_items(items, **kwargs):
+    # This shuffler doesn't raise an exception but it does lose test cases
+    items[1] = items[0]
+
+
+@pytest.fixture
+def simple_testdir(testdir):
+    testdir.makepyfile("""
+        def test_a1():
+            assert True
+
+        def test_a2():
+            assert True
+    """)
+    return testdir
+
+
+def test_faulty_shuffle_that_preserves_items_does_not_fail_test_run(monkeypatch, simple_testdir):
+    monkeypatch.setattr('pytest_random_order._shuffle_items', acceptably_failing_shuffle_items)
+
+    result = simple_testdir.runpytest()
+    result.assert_outcomes(passed=2)
+    result.stdout.fnmatch_lines("""
+        *W0 None pytest-random-order plugin has failed with ValueError*
+    """)
+
+
+def test_faulty_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_testdir):
+    monkeypatch.setattr('pytest_random_order._shuffle_items', critically_failing_shuffle_items)
+    result = simple_testdir.runpytest()
+    result.assert_outcomes(passed=0, failed=0, skipped=0)
+    result.stdout.fnmatch_lines("""
+        *INTERNALERROR> RuntimeError: pytest-random-order plugin has failed with ValueError*
+    """)
+
+
+def test_seemingly_ok_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_testdir):
+    monkeypatch.setattr('pytest_random_order._shuffle_items', critically_not_failing_shuffle_items)
+    result = simple_testdir.runpytest()
+    result.assert_outcomes(passed=0, failed=0, skipped=0)
+    result.stdout.fnmatch_lines("""
+        *INTERNALERROR> RuntimeError: pytest-random-order plugin has failed miserably*
+    """)