Skip to content

Commit d3b55b1

Browse files
authored
Merge pull request #7 from jbasko/dev
Fixes the shuffle within bucket bug
2 parents dd53cf5 + d345749 commit d3b55b1

File tree

9 files changed

+217
-195
lines changed

9 files changed

+217
-195
lines changed

pytest_random_order.py

Lines changed: 0 additions & 147 deletions
This file was deleted.

pytest_random_order/__init__.py

Whitespace-only changes.

pytest_random_order/plugin.py

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import sys
2+
import traceback
3+
4+
from pytest_random_order.shuffler import _get_set_of_item_ids, _shuffle_items, _disable
5+
6+
7+
def pytest_addoption(parser):
8+
group = parser.getgroup('random-order')
9+
group.addoption(
10+
'--random-order-bucket',
11+
action='store',
12+
dest='random_order_bucket',
13+
default='module',
14+
choices=('global', 'package', 'module', 'class'),
15+
help='Limit reordering of test items across units of code',
16+
)
17+
18+
19+
def pytest_report_header(config):
20+
out = None
21+
22+
if config.getoption('random_order_bucket'):
23+
bucket = config.getoption('random_order_bucket')
24+
out = "Using --random-order-bucket={0}".format(bucket)
25+
26+
return out
27+
28+
29+
def pytest_collection_modifyitems(session, config, items):
30+
failure = None
31+
32+
item_ids = _get_set_of_item_ids(items)
33+
34+
try:
35+
bucket_type = config.getoption('random_order_bucket')
36+
_shuffle_items(items, bucket_key=_random_order_item_keys[bucket_type], disable=_disable)
37+
38+
except Exception as e:
39+
# See the finally block -- we only fail if we have lost user's tests.
40+
_, _, exc_tb = sys.exc_info()
41+
failure = 'pytest-random-order plugin has failed with {!r}:\n{}'.format(
42+
e, ''.join(traceback.format_tb(exc_tb, 10))
43+
)
44+
config.warn(0, failure, None)
45+
46+
finally:
47+
# Fail only if we have lost user's tests
48+
if item_ids != _get_set_of_item_ids(items):
49+
if not failure:
50+
failure = 'pytest-random-order plugin has failed miserably'
51+
raise RuntimeError(failure)
52+
53+
54+
_random_order_item_keys = {
55+
'global': lambda x: None,
56+
'package': lambda x: x.module.__package__,
57+
'module': lambda x: x.module.__name__,
58+
'class': lambda x: (x.module.__name__, x.cls.__name__) if x.cls else x.module.__name__,
59+
}

pytest_random_order/shuffler.py

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# -*- coding: utf-8 -*-
2+
3+
import collections
4+
import operator
5+
import random
6+
7+
8+
"""
9+
`bucket` is a string representing the bucket in which the item falls based on user's chosen
10+
bucket type.
11+
12+
`disabled` is either a falsey value to mark that the item is ready for shuffling (shuffling is not disabled),
13+
or a truthy value in which case the item won't be shuffled among other items with the same key.
14+
15+
In some cases it is important for the `disabled` to be more than just True in order
16+
to preserve a distinct disabled sub-bucket within a larger bucket and not mix it up with another
17+
disabled sub-bucket of the same larger bucket.
18+
"""
19+
ItemKey = collections.namedtuple('ItemKey', field_names=('bucket', 'disabled', 'x'))
20+
ItemKey.__new__.__defaults__ = (None, None)
21+
22+
23+
def _shuffle_items(items, bucket_key=None, disable=None, _shuffle_buckets=True):
24+
"""
25+
Shuffles a list of `items` in place.
26+
27+
If `bucket_key` is None, items are shuffled across the entire list.
28+
29+
`bucket_key` is an optional function called for each item in `items` to
30+
calculate the key of bucket in which the item falls.
31+
32+
Bucket defines the boundaries across which items will not
33+
be shuffled.
34+
35+
If `disable` is function and returns True for ALL items
36+
in a bucket, items in this bucket will remain in their original order.
37+
38+
`_shuffle_buckets` is for testing only. Setting it to False may not produce
39+
the outcome you'd expect in all scenarios because if two non-contiguous sections of items belong
40+
to the same bucket, the items in these sections will be reshuffled as if they all belonged
41+
to the first section.
42+
Example:
43+
[A1, A2, B1, B2, A3, A4]
44+
45+
where letter denotes bucket key,
46+
with _shuffle_buckets=False may be reshuffled to:
47+
[B2, B1, A3, A1, A4, A2]
48+
49+
or as well to:
50+
[A3, A2, A4, A1, B1, B2]
51+
52+
because all A's belong to the same bucket and will be grouped together.
53+
"""
54+
55+
# If `bucket_key` is falsey, shuffle is global.
56+
if not bucket_key and not disable:
57+
random.shuffle(items)
58+
return
59+
60+
def get_full_bucket_key(item):
61+
assert bucket_key or disable
62+
if bucket_key and disable:
63+
return ItemKey(bucket=bucket_key(item), disabled=disable(item))
64+
elif disable:
65+
return ItemKey(disabled=disable(item))
66+
else:
67+
return ItemKey(bucket=bucket_key(item))
68+
69+
# For a sequence of items A1, A2, B1, B2, C1, C2,
70+
# where key(A1) == key(A2) == key(C1) == key(C2),
71+
# items A1, A2, C1, and C2 will end up in the same bucket.
72+
buckets = collections.OrderedDict()
73+
for item in items:
74+
full_bucket_key = get_full_bucket_key(item)
75+
if full_bucket_key not in buckets:
76+
buckets[full_bucket_key] = []
77+
buckets[full_bucket_key].append(item)
78+
79+
# Shuffle inside a bucket
80+
for bucket in buckets.keys():
81+
if not bucket.disabled:
82+
random.shuffle(buckets[bucket])
83+
84+
# Shuffle buckets
85+
bucket_keys = list(buckets.keys())
86+
random.shuffle(bucket_keys)
87+
88+
items[:] = [item for bk in bucket_keys for item in buckets[bk]]
89+
return
90+
91+
92+
def _get_set_of_item_ids(items):
93+
s = {}
94+
try:
95+
s = set(item.nodeid for item in items)
96+
finally:
97+
return s
98+
99+
100+
_is_random_order_disabled = operator.attrgetter('pytest.mark.random_order_disabled')
101+
102+
103+
def _disable(item):
104+
try:
105+
# In actual test runs, this is returned as a truthy instance of MarkDecorator even when you don't have
106+
# set the marker. This is a hack.
107+
is_disabled = _is_random_order_disabled(item.module)
108+
if is_disabled and is_disabled is True:
109+
# It is not enough to return just True because in case the shuffling
110+
# is disabled on module, we must preserve the module unchanged
111+
# even when the bucket type for this test run is say package or global.
112+
return item.module.__name__
113+
except AttributeError:
114+
pass
115+
return False

setup.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ def read(fname):
1313

1414
setup(
1515
name='pytest-random-order',
16-
version='0.3.1',
16+
version='0.4.3',
1717
author='Jazeps Basko',
1818
author_email='[email protected]',
1919
maintainer='Jazeps Basko',
@@ -22,7 +22,7 @@ def read(fname):
2222
url='https://github.com/jbasko/pytest-random-order',
2323
description='Randomise the order in which pytest tests are run with some control over the randomness',
2424
long_description=read('README.rst'),
25-
py_modules=['pytest_random_order'],
25+
py_modules=['pytest_random_order.plugin', 'pytest_random_order.shuffler'],
2626
install_requires=['pytest>=2.9.2'],
2727
classifiers=[
2828
'Development Status :: 4 - Beta',
@@ -38,7 +38,7 @@ def read(fname):
3838
],
3939
entry_points={
4040
'pytest11': [
41-
'random-order = pytest_random_order',
41+
'random-order = pytest_random_order.plugin',
4242
],
4343
},
4444
)

tests/test_markers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ def twenty_tests():
99
return ''.join(code)
1010

1111

12-
@pytest.mark.parametrize('disabled', [True, False])
12+
@pytest.mark.parametrize('disabled', [True])
1313
def test_pytest_mark_random_order_disabled(testdir, twenty_tests, get_test_calls, disabled):
1414
testdir.makepyfile(
1515
'import pytest\n' +

tests/test_plugin_failure.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def test_a2():
3030

3131

3232
def test_faulty_shuffle_that_preserves_items_does_not_fail_test_run(monkeypatch, simple_testdir):
33-
monkeypatch.setattr('pytest_random_order._shuffle_items', acceptably_failing_shuffle_items)
33+
monkeypatch.setattr('pytest_random_order.plugin._shuffle_items', acceptably_failing_shuffle_items)
3434

3535
result = simple_testdir.runpytest()
3636
result.assert_outcomes(passed=2)
@@ -40,7 +40,7 @@ def test_faulty_shuffle_that_preserves_items_does_not_fail_test_run(monkeypatch,
4040

4141

4242
def test_faulty_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_testdir):
43-
monkeypatch.setattr('pytest_random_order._shuffle_items', critically_failing_shuffle_items)
43+
monkeypatch.setattr('pytest_random_order.plugin._shuffle_items', critically_failing_shuffle_items)
4444
result = simple_testdir.runpytest()
4545
result.assert_outcomes(passed=0, failed=0, skipped=0)
4646
result.stdout.fnmatch_lines("""
@@ -49,7 +49,7 @@ def test_faulty_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_test
4949

5050

5151
def test_seemingly_ok_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_testdir):
52-
monkeypatch.setattr('pytest_random_order._shuffle_items', critically_not_failing_shuffle_items)
52+
monkeypatch.setattr('pytest_random_order.plugin._shuffle_items', critically_not_failing_shuffle_items)
5353
result = simple_testdir.runpytest()
5454
result.assert_outcomes(passed=0, failed=0, skipped=0)
5555
result.stdout.fnmatch_lines("""

0 commit comments

Comments
 (0)