Initial commit

Author: jbasko

.gitignore

@@ -0,0 +1,70 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask instance folder
+instance/
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+

.travis.yml

@@ -0,0 +1,17 @@
+# Config file for automatic testing at travis-ci.org
+
+sudo: false
+language: python
+python:
+  - "2.7"
+
+install:
+  - pip install tox
+  - "TOX_ENV=${TRAVIS_PYTHON_VERSION/[0-9].[0-9]/py${TRAVIS_PYTHON_VERSION/.}}"
+script: tox -e $TOX_ENV
+
+before_cache:
+  - rm -rf $HOME/.cache/pip/log
+cache:
+  directories:
+    - $HOME/.cache/pip

LICENSE

@@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+
+Copyright (c) 2016 Jazeps Basko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

MANIFEST.in

@@ -0,0 +1,5 @@
+include LICENSE
+include README.rst
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

pytest_random_order.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+
+import random
+
+import pytest
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup('random-order')
+    # group.addoption(
+    #     '--random-order-seed',
+    #     action='store',
+    #     type=int,
+    #     dest='random_order_seed',
+    #     default=None,
+    #     help='Seed value to reproduce a particular order',
+    # )
+    group.addoption(
+        '--random-order-mode',
+        action='store',
+        dest='random_order_mode',
+        default='module',
+        choices=('global', 'package', 'module'),
+        help='Limit reordering of test items across units of code',
+    )
+
+
+def pytest_report_header(config):
+    out = None
+
+    if config.getoption('random_order_mode'):
+        mode = config.getoption('random_order_mode')
+        out = "Using --random-order-mode={0}".format(mode)
+
+    return out
+
+
+_random_order_item_keys = {
+    'global': lambda x: None,
+    'package': lambda x: x.module.__package__,
+    'module': lambda x: x.module.__name__,
+}
+
+
+def pytest_collection_modifyitems(session, config, items):
+    sections = []
+
+    # One of: global, package, module
+    shuffle_mode = config.getoption('random_order_mode')
+
+    item_key = _random_order_item_keys[shuffle_mode]
+
+    # Mark the beginning and ending of each key's test items in the items collection
+    # so we know the boundaries of reshuffle.
+    for i, item in enumerate(items):
+        key = item_key(item)
+        if not sections:
+            assert i == 0
+            sections.append([key, i, None])
+        elif sections[-1][0] != key:
+            sections[-1][2] = i
+            sections.append([key, i, None])
+
+    if sections:
+        sections[-1][2] = len(items)
+
+    for key, i, j in sections:
+        key_items = items[i:j]
+        random.shuffle(key_items)
+        items[i:j] = key_items

setup.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import codecs
+from setuptools import setup
+
+
+def read(fname):
+    file_path = os.path.join(os.path.dirname(__file__), fname)
+    return codecs.open(file_path, encoding='utf-8').read()
+
+
+setup(
+    name='pytest-random-order',
+    version='0.1.0',
+    author='Jazeps Basko',
+    author_email='[email protected]',
+    maintainer='Jazeps Basko',
+    maintainer_email='[email protected]',
+    license='MIT',
+    url='https://github.com/jbasko/pytest-random-order',
+    description='Randomise the order in which tests are run',
+    long_description=read('README.rst'),
+    py_modules=['pytest_random_order'],
+    install_requires=['pytest>=2.9.2'],
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Framework :: Pytest',
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Testing',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'License :: OSI Approved :: MIT License',
+    ],
+    entry_points={
+        'pytest11': [
+            'random-order = pytest_random_order',
+        ],
+    },
+)

tests/conftest.py

@@ -0,0 +1 @@
+pytest_plugins = 'pytester'

tests/test_random_order.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+import py
+import pytest
+
+
+def test_help_message(testdir):
+    result = testdir.runpytest(
+        '--help',
+    )
+    result.stdout.fnmatch_lines([
+        'random-order:',
+        # '*--random-order-seed=RANDOM_ORDER_SEED*',
+        '*--random-order-mode={global,package,module}*',
+    ])
+
+
+def get_runtest_call_sequence(result, key=None):
+    """
+    Returns a tuple of names of test methods that were run
+    in the order they were run.
+    """
+    calls = []
+
+    for c in result.reprec.getcalls('pytest_runtest_call'):
+        calls.append((c.item.module.__package__, c.item.module.__name__, c.item.name))
+    return tuple(calls)
+
+
+@pytest.fixture
+def tmp_tree_of_tests(testdir):
+    """
+    Creates a directory structure:
+        tmpdir/
+            shallow_tests/
+                test_a.py
+                deep_tests/
+                    test_b.py
+                    test_c.py
+
+    If module name doesn't start with "test_", it isn't picked up by runpytest.
+    """
+
+    sup = testdir.mkpydir('shallow_tests')
+
+    sup.join('test_a.py').write(py.code.Source("""
+        def test_a1():
+            assert False
+        def test_a2():
+            assert True
+        def test_a3():
+            assert True
+    """))
+
+    sup.join('test_ax.py').write(py.code.Source("""
+        def test_ax1():
+            assert True
+        def test_ax2():
+            assert True
+        def test_ax3():
+            assert True
+    """))
+
+    sub = testdir.mkpydir('shallow_tests/deep_tests')
+
+    sub.join('test_b.py').write(py.code.Source("""
+        def test_b1():
+            assert True
+        def test_b2():
+            assert False
+        def test_b3():
+            assert True
+    """))
+
+    sub.join('test_c.py').write(py.code.Source("""
+        def test_c1():
+            assert True
+    """))
+
+    sub.join('test_d.py').write(py.code.Source("""
+        def test_d1():
+            assert True
+        def test_d2():
+            assert True
+    """))
+
+    return testdir
+
+
+def check_call_sequence(seq, shuffle_mode='module'):
+    packages_seen = []
+    modules_seen = []
+
+    num_package_switches = 0
+    num_module_switches = 0
+
+    for package, module, item in seq:
+        if not packages_seen:
+            num_package_switches += 1
+            packages_seen.append(package)
+        elif packages_seen[-1] == package:
+            # Same package as the previous item
+            pass
+        else:
+            num_package_switches += 1
+            if package not in packages_seen:
+                packages_seen.append(package)
+
+        if not modules_seen:
+            num_module_switches += 1
+            modules_seen.append(module)
+        elif modules_seen[-1] == module:
+            # Same module as the previous item
+            pass
+        else:
+            num_module_switches += 1
+            if module not in modules_seen:
+                modules_seen.append(module)
+
+    if shuffle_mode == 'global':
+        if len(packages_seen) >= num_package_switches:
+            pytest.fail(
+                'Too few package switches ({}) for '
+                'random-shuffle-mode=global. Packages seen: {}'.format(
+                    num_package_switches,
+                    packages_seen,
+                )
+            )
+
+    if shuffle_mode == 'package':
+        if len(packages_seen) != num_package_switches:
+            pytest.fail('There were more package switches than number of packages')
+
+        if len(modules_seen) >= num_module_switches:
+            pytest.fail(
+                'Suspiciously few module switches ({}) for '
+                'random-shuffle-mode=package. Modules seen: {}'.format(
+                    num_module_switches,
+                    modules_seen,
+                )
+            )
+
+    elif shuffle_mode == 'module' and len(modules_seen) != num_module_switches:
+        pytest.fail('There were more module switches than number of modules')
+
+
+@pytest.mark.parametrize('mode', ['module', 'package', 'global'])
+def test_it_works(tmp_tree_of_tests, mode):
+    sequences = set()
+
+    for x in xrange(5):
+        result = tmp_tree_of_tests.runpytest('--random-order-mode={}'.format(mode), '--verbose')
+        result.assert_outcomes(passed=10, failed=2)
+        seq = get_runtest_call_sequence(result)
+        check_call_sequence(seq, shuffle_mode=mode)
+        assert len(seq) == 12
+        sequences.add(seq)
+
+    assert 1 < len(sequences) <= 5