* Terminology: rename sections to buckets

* Implements bucket shuffling
* Adds random-order-mode option "class" to use classes as buckets.
* Flake8 cleanup
* Adds TestCase class based tests to the tests

Author: jbasko

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+ignore = E501

README.rst

@@ -26,6 +26,8 @@ Usage
 
     $ pytest -v --random-order-mode=module
 
+    $ pytest -v --random-order-mode=class
+
 
 License
 -------

pytest_random_order.py

@@ -2,25 +2,15 @@
 
 import random
 
-import pytest
-
 
 def pytest_addoption(parser):
     group = parser.getgroup('random-order')
-    # group.addoption(
-    #     '--random-order-seed',
-    #     action='store',
-    #     type=int,
-    #     dest='random_order_seed',
-    #     default=None,
-    #     help='Seed value to reproduce a particular order',
-    # )
     group.addoption(
         '--random-order-mode',
         action='store',
         dest='random_order_mode',
         default='module',
-        choices=('global', 'package', 'module'),
+        choices=('global', 'package', 'module', 'class'),
         help='Limit reordering of test items across units of code',
     )
 
@@ -39,32 +29,52 @@ def pytest_report_header(config):
     'global': lambda x: None,
     'package': lambda x: x.module.__package__,
     'module': lambda x: x.module.__name__,
+    'class': lambda x: (x.module.__name__, x.cls.__name__) if x.cls else None,
 }
 
 
-def pytest_collection_modifyitems(session, config, items):
-    sections = []
+def _shuffle_items(items, key=None, preserve_bucket_order=False):
+    """
+    Shuffles `items`, a list, in place.
 
-    # One of: global, package, module
-    shuffle_mode = config.getoption('random_order_mode')
+    If `key` is None, items are shuffled across the entire list.
+
+    Otherwise `key` is a function called for each item in `items` to
+    calculate key of bucket in which the item falls.
+
+    Bucket defines the boundaries across which tests will not
+    be reordered.
+
+    `preserve_bucket_order` is only customisable for testing purposes.
+    There is no use case for predefined bucket order, is there?
+    """
+
+    # If `key` is falsey, shuffle is global.
+    if not key:
+        random.shuffle(items)
+        return
 
-    item_key = _random_order_item_keys[shuffle_mode]
-
-    # Mark the beginning and ending of each key's test items in the items collection
-    # so we know the boundaries of reshuffle.
-    for i, item in enumerate(items):
-        key = item_key(item)
-        if not sections:
-            assert i == 0
-            sections.append([key, i, None])
-        elif sections[-1][0] != key:
-            sections[-1][2] = i
-            sections.append([key, i, None])
-
-    if sections:
-        sections[-1][2] = len(items)
-
-    for key, i, j in sections:
-        key_items = items[i:j]
-        random.shuffle(key_items)
-        items[i:j] = key_items
+    buckets = []
+    this_key = '__not_initialised__'
+    for item in items:
+        prev_key = this_key
+        this_key = key(item)
+        if this_key != prev_key:
+            buckets.append([])
+        buckets[-1].append(item)
+
+    # Shuffle within bucket
+    for bucket in buckets:
+        random.shuffle(bucket)
+
+    # Shuffle buckets
+    if not preserve_bucket_order:
+        random.shuffle(buckets)
+
+    items[:] = [item for bucket in buckets for item in bucket]
+    return
+
+
+def pytest_collection_modifyitems(session, config, items):
+    shuffle_mode = config.getoption('random_order_mode')
+    _shuffle_items(items, key=_random_order_item_keys[shuffle_mode])

setup.py

@@ -13,7 +13,7 @@ def read(fname):
 
 setup(
     name='pytest-random-order',
-    version='0.1.0',
+    version='0.2.0',
     author='Jazeps Basko',
     author_email='[email protected]',
     maintainer='Jazeps Basko',

tests/test_actual_test_runs.py

@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+import collections
+
+import py
+import pytest
+
+
+Call = collections.namedtuple('Call', field_names=('package', 'module', 'cls', 'name'))
+
+
+def get_runtest_call_sequence(result, key=None):
+    """
+    Returns a tuple of names of test methods that were run
+    in the order they were run.
+    """
+    calls = []
+
+    for c in result.reprec.getcalls('pytest_runtest_call'):
+        calls.append(Call(
+            package=c.item.module.__package__,
+            module=c.item.module.__name__,
+            cls=(c.item.module.__name__, c.item.cls.__name__) if c.item.cls else None,
+            name=c.item.name,
+        ))
+    return tuple(calls)
+
+
+@pytest.fixture
+def tmp_tree_of_tests(testdir):
+    """
+    Creates a directory structure:
+        tmpdir/
+            shallow_tests/
+                test_a.py 2 passing, 1 failing
+                deep_tests/
+                    test_b.py 2 passing, 1 failing
+                    test_c.py 1 passing
+                    test_d.py 2 passing
+                    test_e.py a class, 2 passing, 1 failing
+
+    If module name doesn't start with "test_", it isn't picked up by runpytest.
+    """
+
+    sup = testdir.mkpydir('shallow_tests')
+
+    sup.join('test_a.py').write(py.code.Source("""
+        def test_a1():
+            assert False
+        def test_a2():
+            assert True
+        def test_a3():
+            assert True
+    """))
+
+    sup.join('test_ax.py').write(py.code.Source("""
+        def test_ax1():
+            assert True
+        def test_ax2():
+            assert True
+        def test_ax3():
+            assert True
+    """))
+
+    sub = testdir.mkpydir('shallow_tests/deep_tests')
+
+    sub.join('test_b.py').write(py.code.Source("""
+        def test_b1():
+            assert True
+        def test_b2():
+            assert False
+        def test_b3():
+            assert True
+    """))
+
+    sub.join('test_c.py').write(py.code.Source("""
+        def test_c1():
+            assert True
+    """))
+
+    sub.join('test_d.py').write(py.code.Source("""
+        def test_d1():
+            assert True
+        def test_d2():
+            assert True
+    """))
+
+    sub.join('test_e.py').write(py.code.Source("""
+        from unittest import TestCase
+        class EeTest(TestCase):
+            def test_ee1(self):
+                self.assertTrue(True)
+            def test_ee2(self):
+                self.assertFalse(True)
+            def test_ee3(self):
+                self.assertTrue(True)
+
+        class ExTest(TestCase):
+            def test_ex1(self):
+                self.assertTrue(True)
+            def test_ex2(self):
+                self.assertTrue(True)
+    """))
+
+    return testdir
+
+
+def check_call_sequence(seq, shuffle_mode='module'):
+    all_values = collections.defaultdict(list)
+    num_switches = collections.defaultdict(int)
+
+    def inspect_attr(this_call, prev_call, attr_name):
+        attr_value = getattr(this_call, attr_name)
+        prev_value = getattr(prev_call, attr_name) if prev_call else -1
+        all_values[attr_name].append(attr_value)
+        if attr_value != prev_value:
+            num_switches[attr_name] += 1
+
+    for i, this_call in enumerate(seq):
+        prev_call = seq[i - 1] if i > 0 else None
+        inspect_attr(this_call, prev_call, 'package')
+        inspect_attr(this_call, prev_call, 'module')
+        inspect_attr(this_call, prev_call, 'cls')
+
+    num_packages = len(set(all_values['package']))
+    num_package_switches = num_switches['package']
+    num_modules = len(set(all_values['module']))
+    num_module_switches = num_switches['module']
+    num_classes = len(set(all_values['class']))
+    num_class_switches = num_switches['class']
+
+    # These are just sanity tests, the actual shuffling is tested in test_shuffle,
+    # assertions here are very relaxed.
+
+    if shuffle_mode == 'global':
+        if num_module_switches <= num_modules:
+            pytest.fail('Too few module switches for global shuffling')
+        if num_package_switches <= num_packages:
+            pytest.fail('Too few package switches for global shuffling')
+
+    elif shuffle_mode == 'package':
+        assert num_package_switches == num_packages
+        if num_module_switches <= num_modules:
+            pytest.fail('Too few module switches for package-limited shuffling')
+
+    elif shuffle_mode == 'module':
+        assert num_module_switches == num_modules
+
+    elif shuffle_mode == 'cls':
+        # Each class can contribute to 1 or 2 switches.
+        assert num_class_switches <= num_classes * 2
+
+        # Class shuffle is a subset of module shuffle
+        assert num_module_switches == num_modules
+
+
+@pytest.mark.parametrize('mode', ['class', 'module', 'package', 'global'])
+def test_it_works_with_actual_tests(tmp_tree_of_tests, mode):
+    sequences = set()
+
+    for x in range(5):
+        result = tmp_tree_of_tests.runpytest('--random-order-mode={}'.format(mode), '--verbose')
+        result.assert_outcomes(passed=14, failed=3)
+        seq = get_runtest_call_sequence(result)
+        check_call_sequence(seq, shuffle_mode=mode)
+        assert len(seq) == 17
+        sequences.add(seq)
+
+    assert 1 < len(sequences) <= 5

tests/test_cli.py

@@ -0,0 +1,8 @@
+def test_help_message(testdir):
+    result = testdir.runpytest(
+        '--help',
+    )
+    result.stdout.fnmatch_lines([
+        'random-order:',
+        '*--random-order-mode={global,package,module,class}*',
+    ])