diff --git a/.gitignore b/.gitignore index 9fb4879..5929919 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ __pycache__/ /doc/latex/ /doc/linkcheck/ /pytest_dependency.egg-info/ +/src/pytest_dependency.egg-info/ /python2_6.patch diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..30ae3da --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Run Tests", + "type": "python", + "request": "launch", + "module": "pytest", + "args": ["-vvx", "tests/test_05_ordering.py::test_order_scopes"] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..cb8f801 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "restructuredtext.confPath": "", + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/src/pytest_dependency.py b/src/pytest_dependency.py index 43224ee..12a4028 100644 --- a/src/pytest_dependency.py +++ b/src/pytest_dependency.py @@ -1,89 +1,145 @@ """$DOC""" -__version__ = "$VERSION" - import logging +from collections import deque + import pytest + +__version__ = "$VERSION" + logger = logging.getLogger(__name__) _automark = False _ignore_unknown = False +def _get_name(item, scope): + # Old versions of pytest used to add an extra "::()" to + # the node ids of class methods to denote the class + # instance. This has been removed in pytest 4.0.0. + nodeid = item.nodeid.replace("::()::", "::") + if scope == "session" or scope == "package": + name = nodeid + elif scope == "module": + name = nodeid.split("::", 1)[1] + elif scope == "class": + name = nodeid.split("::", 2)[2] + else: + raise RuntimeError( + "Internal error: invalid scope '%s'" % scope + ) + return name + + +def _remove_parametrization(item, name): + original = item.originalname if item.originalname is not None else item.name + # remove the parametrization part at the end + if not name.endswith(original): + index = name.rindex(original) + len(original) + name = name[:index] + return name + + class DependencyItemStatus(object): - """Status of a test item in a dependency manager. + """ + Status of a test item in a dependency manager. """ - Phases = ('setup', 'call', 'teardown') + phases = ("setup", "call", "teardown") def __init__(self): - self.results = { w:None for w in self.Phases } + self.results = {w: None for w in self.phases} def __str__(self): - l = ["%s: %s" % (w, self.results[w]) for w in self.Phases] - return "Status(%s)" % ", ".join(l) + return "Status(%s)" % ", ".join( + "%s: %s" % (w, self.results[w]) for w in self.phases + ) - def addResult(self, rep): + def add_result(self, rep): self.results[rep.when] = rep.outcome - def isSuccess(self): - return list(self.results.values()) == ['passed', 'passed', 'passed'] + def is_success(self): + return all(v == "passed" for v in self.results.values()) + + def is_done(self): + return None not in self.results.values() class DependencyManager(object): - """Dependency manager, stores the results of tests. + """ + Dependency manager, stores the results of tests. """ - ScopeCls = { - 'session': pytest.Session, - 'package': pytest.Package, - 'module': pytest.Module, - 'class': pytest.Class, + scope_cls = { + "session": pytest.Session, + "package": pytest.Package, + "module": pytest.Module, + "class": pytest.Class, } @classmethod - def getManager(cls, item, scope): - """Get the DependencyManager object from the node at scope level. + def get_manager(cls, item, scope): + """ + Get the DependencyManager object from the node at scope level. Create it, if not yet present. """ - node = item.getparent(cls.ScopeCls[scope]) + node = item.getparent(cls.scope_cls[scope]) if not node: return None - if not hasattr(node, 'dependencyManager'): - node.dependencyManager = cls(scope) - return node.dependencyManager + if not hasattr(node, "dependency_manager"): + node.dependency_manager = cls(scope) + return node.dependency_manager def __init__(self, scope): - self.results = {} self.scope = scope + self.results = {} + self.names = set() + self.dependencies = set() - def addResult(self, item, name, rep): + def register_dependency(self, name): + self.dependencies.add(name) + + def register_dependency_name(self, name): + self.names.add(name) + + def add_result(self, item, name, rep): if not name: - # Old versions of pytest used to add an extra "::()" to - # the node ids of class methods to denote the class - # instance. This has been removed in pytest 4.0.0. - nodeid = item.nodeid.replace("::()::", "::") - if self.scope == 'session' or self.scope == 'package': - name = nodeid - elif self.scope == 'module': - name = nodeid.split("::", 1)[1] - elif self.scope == 'class': - name = nodeid.split("::", 2)[2] - else: - raise RuntimeError("Internal error: invalid scope '%s'" - % self.scope) - status = self.results.setdefault(name, DependencyItemStatus()) - logger.debug("register %s %s %s in %s scope", - rep.when, name, rep.outcome, self.scope) - status.addResult(rep) - - def checkDepend(self, depends, item): - logger.debug("check dependencies of %s in %s scope ...", - item.name, self.scope) + name = _get_name(item, self.scope) + parameterless_name = _remove_parametrization(item, name) + if parameterless_name not in self.results: + self.results[parameterless_name] = DependencyItemStatus() + status = self.results[parameterless_name] + # only add the result if the status is incomplete or it's (still) a success + # this prevents overwriting a failed status of one parametrized test, + # with a success status of the following tests + if not status.is_done() or status.is_success(): + status.add_result(rep) + + if name not in self.results: + self.results[name] = DependencyItemStatus() + # add the result + logger.debug( + "register %s %s %s in %s scope", + rep.when, name, rep.outcome, self.scope + ) + self.results[name].add_result(rep) + + @classmethod + def add_all_scopes(cls, item, name, rep): + for scope in cls.scope_cls: + manager = cls.get_manager(item, scope=scope) + if manager is not None: + manager.add_result(item, name, rep) + + def check_depends(self, depends, item): + logger.debug( + "check dependencies of %s in %s scope ...", + item.name, self.scope + ) for i in depends: if i in self.results: - if self.results[i].isSuccess(): + if self.results[i].is_success(): logger.debug("... %s succeeded", i) continue else: @@ -95,9 +151,25 @@ def checkDepend(self, depends, item): logger.info("skip %s because it depends on %s", item.name, i) pytest.skip("%s depends on %s" % (item.name, i)) + def check_order(self, depends, item, name): + for d in depends: + if d not in self.names: + item.warn(pytest.PytestWarning( + "Dependency '%s' of '%s' doesn't exist, " + "or has incorrect scope!" % (d, name) + )) + if _ignore_unknown: + continue + else: + return False + elif d not in self.dependencies: + return False + return True + -def depends(request, other, scope='module'): - """Add dependency on other test. +def depends(request, other, scope="module"): + """ + Add dependency on other test. Call pytest.skip() unless a successful outcome of all of the tests in other has been registered previously. This has the same effect as @@ -121,52 +193,108 @@ def depends(request, other, scope='module'): the scope parameter has been added. """ item = request.node - manager = DependencyManager.getManager(item, scope=scope) - manager.checkDepend(other, item) + manager = DependencyManager.get_manager(item, scope=scope) + manager.check_depends(other, item) def pytest_addoption(parser): - parser.addini("automark_dependency", - "Add the dependency marker to all tests automatically", - type="bool", default=False) - parser.addoption("--ignore-unknown-dependency", - action="store_true", default=False, - help="ignore dependencies whose outcome is not known") + parser.addini( + "automark_dependency", + "Add the dependency marker to all tests automatically", + type="bool", + default=False, + ) + parser.addoption( + "--ignore-unknown-dependency", + action="store_true", + default=False, + help="ignore dependencies whose outcome is not known", + ) def pytest_configure(config): global _automark, _ignore_unknown _automark = config.getini("automark_dependency") _ignore_unknown = config.getoption("--ignore-unknown-dependency") - config.addinivalue_line("markers", - "dependency(name=None, depends=[]): " - "mark a test to be used as a dependency for " - "other tests or to depend on other tests.") + config.addinivalue_line( + "markers", + "dependency(name=None, depends=[]): " + "mark a test to be used as a dependency for " + "other tests or to depend on other tests." + ) @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_runtest_makereport(item, call): - """Store the test outcome if this item is marked "dependency". + """ + Store the test outcome if this item is marked "dependency". """ outcome = yield marker = item.get_closest_marker("dependency") if marker is not None or _automark: rep = outcome.get_result() - name = marker.kwargs.get('name') if marker is not None else None - for scope in DependencyManager.ScopeCls: - manager = DependencyManager.getManager(item, scope=scope) - if (manager): - manager.addResult(item, name, rep) + name = marker.kwargs.get("name") if marker is not None else None + DependencyManager.add_all_scopes(item, name, rep) def pytest_runtest_setup(item): - """Check dependencies if this item is marked "dependency". + """ + Check dependencies if this item is marked "dependency". Skip if any of the dependencies has not been run successfully. """ marker = item.get_closest_marker("dependency") if marker is not None: - depends = marker.kwargs.get('depends') + depends = marker.kwargs.get("depends") if depends: - scope = marker.kwargs.get('scope', 'module') - manager = DependencyManager.getManager(item, scope=scope) - manager.checkDepend(depends, item) + scope = marker.kwargs.get("scope", "module") + manager = DependencyManager.get_manager(item, scope=scope) + manager.check_depends(depends, item) + + +# special hook to make pytest-dependency support reordering based on deps +def pytest_collection_modifyitems(items): + # store the markers between passes - improves speed + markers_cache = {} + # register items and their names, according to scopes + for item in items: + markers_cache[item] = markers = [] + for marker in item.iter_markers("dependency"): + depends = marker.kwargs.get("depends", []) + scope = marker.kwargs.get("scope", "module") + name = marker.kwargs.get("name") + if not name: + name = _remove_parametrization(item, _get_name(item, scope)) + manager = DependencyManager.get_manager(item, scope) + if manager is None: + continue + markers.append((depends, name, manager)) + manager.register_dependency_name(name) + miss_list = [] + final_items = [] + deque_items = deque(items) + # loop until all items are sorted out + while deque_items: + item = deque_items.popleft() + # store managers and only register when adding to the final list + to_register = [] + for depends, name, manager in markers_cache[item]: + if manager.check_order(depends, item, name): + to_register.append((manager, name)) + else: + miss_list.append(item) + break + else: + # runs only when the for loop wasn't broken out of + for manager, name in to_register: + manager.register_dependency(name) + final_items.append(item) + # add the missing items back in the correct order + if miss_list: + deque_items.extendleft(reversed(miss_list)) + miss_list = [] # can't use '.clear()' in 2.7 + if miss_list: + # this list being non-empty here means there are + # cyclic or missing dependencies + final_items.extend(miss_list) + assert len(items) == len(final_items) + items[:] = final_items diff --git a/tests/test_01_marker.py b/tests/test_01_marker.py index 7be11f1..e9f9b87 100644 --- a/tests/test_01_marker.py +++ b/tests/test_01_marker.py @@ -1,7 +1,6 @@ -"""The most basic test: check that the marker works. """ - -import pytest +The most basic test: check that the marker works. +""" def test_marker_registered(ctestdir): @@ -19,9 +18,9 @@ def test_marker(ctestdir): @pytest.mark.dependency() def test_marker(request): node = request.node.getparent(pytest.Module) - assert hasattr(node, 'dependencyManager') - assert isinstance(node.dependencyManager, DependencyManager) - assert 'test_marker' in node.dependencyManager.results + assert hasattr(node, 'dependency_manager') + assert isinstance(node.dependency_manager, DependencyManager) + assert 'test_marker' in node.dependency_manager.results """) result = ctestdir.runpytest("--verbose") result.assert_outcomes(passed=1) diff --git a/tests/test_02_simple_dependency.py b/tests/test_02_simple_dependency.py index 3c45f2c..83a73c8 100644 --- a/tests/test_02_simple_dependency.py +++ b/tests/test_02_simple_dependency.py @@ -1,11 +1,11 @@ -"""Simple dependencies between tests. """ - -import pytest +Simple dependencies between tests. +""" def test_no_skip(ctestdir): - """One test is skipped, but no other test depends on it, + """ + One test is skipped, but no other test depends on it, so all other tests pass. """ ctestdir.makepyfile(""" @@ -38,7 +38,8 @@ def test_d(): def test_skip_depend(ctestdir): - """One test is skipped, other dependent tests are skipped as well. + """ + One test is skipped, other dependent tests are skipped as well. This also includes indirect dependencies. """ ctestdir.makepyfile(""" @@ -71,7 +72,8 @@ def test_d(): def test_fail_depend(ctestdir): - """One test fails, other dependent tests are skipped. + """ + One test fails, other dependent tests are skipped. This also includes indirect dependencies. """ ctestdir.makepyfile(""" @@ -104,7 +106,8 @@ def test_d(): def test_named_fail_depend(ctestdir): - """Same as test_fail_depend, but using custom test names. + """ + Same as test_fail_depend, but using custom test names. """ ctestdir.makepyfile(""" import pytest @@ -136,7 +139,8 @@ def test_d(): def test_explicit_select(ctestdir): - """Explicitly select only a single test that depends on another one. + """ + Explicitly select only a single test that depends on another one. Since the other test has not been run at all, the selected test will be skipped. @@ -168,7 +172,8 @@ def test_d(): def test_depend_unknown(ctestdir): - """Depend on an unknown test that is not even defined in the test set. + """ + Depend on an unknown test that is not even defined in the test set. Note that is not an error to depend on an undefined test, but the dependent test will be skipped since the non-existent dependency diff --git a/tests/test_03_class.py b/tests/test_03_class.py index 64bdb6d..b7b8348 100644 --- a/tests/test_03_class.py +++ b/tests/test_03_class.py @@ -1,11 +1,11 @@ -"""Usage with test classes. """ - -import pytest +Usage with test classes. +""" def test_class_simple(ctestdir): - """Simple dependencies of test methods in a class. + """ + Simple dependencies of test methods in a class. test_a() deliberately fails, some other methods depend on it, some don't. """ ctestdir.makepyfile(""" @@ -29,7 +29,7 @@ def test_c(self): def test_d(self): pass - @pytest.mark.dependency(depends=["TestClass::test_b", + @pytest.mark.dependency(depends=["TestClass::test_b", "TestClass::test_c"]) def test_e(self): pass @@ -46,8 +46,9 @@ def test_e(self): def test_class_simple_named(ctestdir): - """Mostly the same as test_class_simple(), but name the test methods - now explicitly. + """ + Mostly the same as test_class_simple(), but name the test methods + now explicitely. """ ctestdir.makepyfile(""" import pytest diff --git a/tests/test_03_multiple_dependency.py b/tests/test_03_multiple_dependency.py index a5e585a..df55eaf 100644 --- a/tests/test_03_multiple_dependency.py +++ b/tests/test_03_multiple_dependency.py @@ -1,7 +1,6 @@ -"""A complicated scenario with tests having multiple dependencies. """ - -import pytest +A complicated scenario with tests having multiple dependencies. +""" def test_multiple(ctestdir): diff --git a/tests/test_03_param.py b/tests/test_03_param.py index 62cb5ba..9ca3c0a 100644 --- a/tests/test_03_param.py +++ b/tests/test_03_param.py @@ -1,7 +1,58 @@ -"""A scenario featuring parametrized tests. """ +A scenario featuring parametrized tests. +""" + + +def test_removed_params(ctestdir): + """ + Test for a dependency on a parametrized test, but with parametrization removed. + """ + ctestdir.makepyfile(""" + import pytest + + @pytest.mark.parametrize("x", [0, 1]) + @pytest.mark.dependency() + def test_a(x): + # passes, then fails + assert x == 0 + + @pytest.mark.parametrize("x", [0, 1]) + @pytest.mark.dependency() + def test_b(x): + # fails, then passes + assert x == 1 -import pytest + @pytest.mark.parametrize("x", [0, 1]) + @pytest.mark.dependency() + def test_c(x): + # always passes + pass + + @pytest.mark.dependency(depends=["test_a"]) + def test_d(): + pass + + @pytest.mark.dependency(depends=["test_b"]) + def test_e(): + pass + + @pytest.mark.dependency(depends=["test_c"]) + def test_f(): + pass + """) + result = ctestdir.runpytest("--verbose") + result.assert_outcomes(passed=5, skipped=2, failed=2) + result.stdout.re_match_lines(r""" + .*::test_a\[0\] PASSED + .*::test_a\[1\] FAILED + .*::test_b\[0\] FAILED + .*::test_b\[1\] PASSED + .*::test_c\[0\] PASSED + .*::test_c\[1\] PASSED + .*::test_d SKIPPED(?:\s+\(.*\))? + .*::test_e SKIPPED(?:\s+\(.*\))? + .*::test_f PASSED + """) def test_simple_params(ctestdir): @@ -14,7 +65,7 @@ def test_simple_params(ctestdir): _md = pytest.mark.dependency - @pytest.mark.parametrize("x", [ 0, 1 ]) + @pytest.mark.parametrize("x", [0, 1]) @pytest.mark.dependency() def test_a(x): assert x == 0 diff --git a/tests/test_03_runtime.py b/tests/test_03_runtime.py index fb23e2c..253e4f5 100644 --- a/tests/test_03_runtime.py +++ b/tests/test_03_runtime.py @@ -1,11 +1,11 @@ -"""Using depends() to mark dependencies at runtime. """ - -import pytest +Using depends() to mark dependencies at runtime. +""" def test_skip_depend_runtime(ctestdir): - """One test is skipped, other dependent tests are skipped as well. + """ + One test is skipped, other dependent tests are skipped as well. This also includes indirect dependencies. """ ctestdir.makepyfile(""" diff --git a/tests/test_03_scope.py b/tests/test_03_scope.py index 17efe82..492e468 100644 --- a/tests/test_03_scope.py +++ b/tests/test_03_scope.py @@ -1,11 +1,11 @@ -"""Specifying the scope of dependencies. """ - -import pytest +Specifying the scope of dependencies. +""" def test_scope_module(ctestdir): - """One single module, module scope is explicitly set in the + """ + One single module, module scope is explicitely set in the pytest.mark.dependency() marker. """ ctestdir.makepyfile(""" @@ -41,8 +41,10 @@ def test_e(): test_scope_module.py::test_e SKIPPED(?:\s+\(.*\))? """) + def test_scope_session(ctestdir): - """Two modules, some cross module dependencies in session scope. + """ + Two modules, some cross module dependencies in session scope. """ ctestdir.makepyfile(test_scope_session_01=""" import pytest @@ -114,8 +116,10 @@ def test_h(): test_scope_session_02.py::test_h PASSED """) + def test_scope_package(ctestdir): - """Two packages, some cross module dependencies within the package and + """ + Two packages, some cross module dependencies within the package and across package boundaries. """ ctestdir.mkpydir("test_scope_package_a") @@ -184,8 +188,10 @@ def test_h(): test_scope_package_b/test_03.py::test_h SKIPPED(?:\s+\(.*\))? """) + def test_scope_class(ctestdir): - """Dependencies in class scope. + """ + Dependencies in class scope. """ ctestdir.makepyfile(""" import pytest @@ -249,8 +255,10 @@ def test_h(self): test_scope_class.py::TestClass2::test_h SKIPPED(?:\s+\(.*\))? """) + def test_scope_nodeid(ctestdir): - """The default name of a test is the node id. + """ + The default name of a test is the node id. The references to the default names must be adapted according to the scope. """ @@ -363,23 +371,25 @@ def test_o(self): result.stdout.re_match_lines(r""" test_scope_nodeid.py::test_a PASSED test_scope_nodeid.py::test_b PASSED + test_scope_nodeid.py::TestClass::test_f PASSED + test_scope_nodeid.py::TestClass::test_k PASSED test_scope_nodeid.py::test_c SKIPPED(?:\s+\(.*\))? test_scope_nodeid.py::test_d SKIPPED(?:\s+\(.*\))? test_scope_nodeid.py::test_e PASSED - test_scope_nodeid.py::TestClass::test_f PASSED test_scope_nodeid.py::TestClass::test_g PASSED test_scope_nodeid.py::TestClass::test_h SKIPPED(?:\s+\(.*\))? test_scope_nodeid.py::TestClass::test_i SKIPPED(?:\s+\(.*\))? test_scope_nodeid.py::TestClass::test_j SKIPPED(?:\s+\(.*\))? - test_scope_nodeid.py::TestClass::test_k PASSED test_scope_nodeid.py::TestClass::test_l SKIPPED(?:\s+\(.*\))? test_scope_nodeid.py::TestClass::test_m SKIPPED(?:\s+\(.*\))? test_scope_nodeid.py::TestClass::test_n SKIPPED(?:\s+\(.*\))? test_scope_nodeid.py::TestClass::test_o PASSED """) + def test_scope_named(ctestdir): - """Explicitly named tests are always referenced by that name, + """ + Explicitely named tests are always referenced by that name, regardless of the scope. """ ctestdir.makepyfile(""" @@ -470,20 +480,22 @@ def test_l(self): result.stdout.re_match_lines(r""" test_scope_named.py::test_a PASSED test_scope_named.py::test_b PASSED + test_scope_named.py::TestClass::test_f PASSED + test_scope_named.py::TestClass::test_i PASSED test_scope_named.py::test_c SKIPPED(?:\s+\(.*\))? test_scope_named.py::test_d PASSED test_scope_named.py::test_e SKIPPED(?:\s+\(.*\))? - test_scope_named.py::TestClass::test_f PASSED test_scope_named.py::TestClass::test_g PASSED test_scope_named.py::TestClass::test_h SKIPPED(?:\s+\(.*\))? - test_scope_named.py::TestClass::test_i PASSED test_scope_named.py::TestClass::test_j SKIPPED(?:\s+\(.*\))? test_scope_named.py::TestClass::test_k PASSED test_scope_named.py::TestClass::test_l SKIPPED(?:\s+\(.*\))? """) + def test_scope_dependsfunc(ctestdir): - """Test the scope argument to the depends() function. + """ + Test the scope argument to the depends() function. """ ctestdir.makepyfile(test_scope_dependsfunc_01=""" import pytest diff --git a/tests/test_03_skipmsgs.py b/tests/test_03_skipmsgs.py index bfdc833..6e4b4d1 100644 --- a/tests/test_03_skipmsgs.py +++ b/tests/test_03_skipmsgs.py @@ -1,11 +1,11 @@ -"""Verify the messages issued when a dependent test is skipped. """ - -import pytest +Verify the messages issued when a dependent test is skipped. +""" def test_simple(ctestdir): - """One test fails, other dependent tests are skipped. + """ + One test fails, other dependent tests are skipped. This also includes indirect dependencies. """ ctestdir.makepyfile(""" diff --git a/tests/test_04_automark.py b/tests/test_04_automark.py index 09fbc17..0d5973a 100644 --- a/tests/test_04_automark.py +++ b/tests/test_04_automark.py @@ -1,11 +1,13 @@ -"""Test the automark_dependency option. +""" +Test the automark_dependency option. """ import pytest def test_not_set(ctestdir): - """No pytest.ini file, e.g. automark_dependency is not set. + """ + No pytest.ini file, e.g. automark_dependency is not set. Since automark_dependency defaults to false and test_a is not marked, the outcome of test_a will not be recorded. As a result, diff --git a/tests/test_04_ignore_unknown.py b/tests/test_04_ignore_unknown.py index bc145e5..c3e6362 100644 --- a/tests/test_04_ignore_unknown.py +++ b/tests/test_04_ignore_unknown.py @@ -1,11 +1,11 @@ -"""Test the ignore-unknown-dependency command line option. """ - -import pytest +Test the ignore-unknown-dependency command line option. +""" def test_no_ignore(ctestdir): - """No command line option, e.g. ignore-unknown-dependency is not set. + """ + No command line option, e.g. ignore-unknown-dependency is not set. Explicitly select only a single test that depends on another one. Since the other test has not been run at all, the selected test @@ -38,7 +38,8 @@ def test_d(): def test_ignore(ctestdir): - """Set the ignore-unknown-dependency command line option. + """ + Set the ignore-unknown-dependency command line option. Explicitly select only a single test that depends on another one. The other test has not been run at all, but since unknown @@ -64,7 +65,7 @@ def test_c(): def test_d(): pass """) - result = ctestdir.runpytest("--verbose", "--ignore-unknown-dependency", + result = ctestdir.runpytest("--verbose", "--ignore-unknown-dependency", "test_ignore.py::test_d") result.assert_outcomes(passed=1, skipped=0, failed=0) result.stdout.re_match_lines(r""" diff --git a/tests/test_05_ordering.py b/tests/test_05_ordering.py new file mode 100644 index 0000000..6bc188b --- /dev/null +++ b/tests/test_05_ordering.py @@ -0,0 +1,253 @@ +""" +Tests ordering. +""" + + +def test_order_standard(ctestdir): + """ + One module and 7 tests designed to cover most cases the ordering can fail in. + """ + ctestdir.makepyfile(""" + import pytest + + # this empty one should stay first + @pytest.mark.dependency() + def test_a(): + pass + + # misordered dependencies, this should end up near the bottom + @pytest.mark.dependency(depends=["test_f", "test_d", "test_e"]) + def test_b(): + pass + + # this empty one should occur after 'test_a' but before 'test_d' + @pytest.mark.dependency(depends=["test_a"]) + def test_c(): + pass + + # right after 'test_c' + @pytest.mark.dependency() + def test_d(): + pass + + # correct order already + @pytest.mark.dependency(depends=["test_d"]) + def test_e(): + pass + + # same here + @pytest.mark.dependency(depends=["test_c", "test_d"]) + def test_f(): + pass + + # and here - 'test_b' should land just before this test + @pytest.mark.dependency(depends=["test_c"]) + def test_g(): + pass + """) + result = ctestdir.runpytest("--verbose") + result.assert_outcomes(passed=7, skipped=0, failed=0) + result.stdout.re_match_lines(r""" + .*::test_a PASSED + .*::test_c PASSED + .*::test_d PASSED + .*::test_e PASSED + .*::test_f PASSED + .*::test_b PASSED + .*::test_g PASSED + """) + + +def test_order_missing(ctestdir): + """ + 5 tests, with 2 of them having mislabeled (missing) dependencies. + """ + ctestdir.makepyfile(""" + import pytest + + @pytest.mark.dependency() + def test_a(): + pass + + # mislabeled 'test_c' + @pytest.mark.dependency(depends=["tets_c"]) + def test_b(): + pass + + @pytest.mark.dependency() + def test_c(): + pass + + # mislabeled 'test_a' + @pytest.mark.dependency(depends=["tets_a"]) + def test_d(): + pass + + @pytest.mark.dependency() + def test_e(): + pass + """) + result = ctestdir.runpytest("--verbose") + result.assert_outcomes(passed=3, skipped=2, failed=0) + result.stdout.re_match_lines(r""" + .*::test_a PASSED + .*::test_c PASSED + .*::test_e PASSED + .*::test_b SKIPPED(?:\s+\(.*\))? + .*::test_d SKIPPED(?:\s+\(.*\))? + """) + + +def test_order_cycles(ctestdir): + """ + 5 tests, with 2 of them creating an "accidental" cycle. + """ + ctestdir.makepyfile(""" + import pytest + + @pytest.mark.dependency() + def test_a(): + pass + + # depends on 'test_d' - cycle + @pytest.mark.dependency(depends=["test_d"]) + def test_b(): + pass + + @pytest.mark.dependency() + def test_c(): + pass + + # depends on 'test_b' - cycle + @pytest.mark.dependency(depends=["test_b", "test_c"]) + def test_d(): + pass + + @pytest.mark.dependency() + def test_e(): + pass + """) + result = ctestdir.runpytest("--verbose") + result.assert_outcomes(passed=3, skipped=2, failed=0) + result.stdout.re_match_lines(r""" + .*::test_a PASSED + .*::test_c PASSED + .*::test_e PASSED + .*::test_b SKIPPED(?:\s+\(.*\))? + .*::test_d SKIPPED(?:\s+\(.*\))? + """) + + +def test_order_nesting(ctestdir): + """ + 8 tests, with tests depending on tests that depend on + other tests that might be reordered later too. + """ + ctestdir.makepyfile(""" + import pytest + + @pytest.mark.dependency() + def test_a(): + pass + + @pytest.mark.dependency(depends=["test_d"]) + def test_b(): + pass + + @pytest.mark.dependency() + def test_c(): + pass + + @pytest.mark.dependency(depends=["test_f"]) + def test_d(): + pass + + @pytest.mark.dependency() + def test_e(): + pass + + @pytest.mark.dependency(depends=["test_g"]) + def test_f(): + pass + + @pytest.mark.dependency() + def test_g(): + pass + + @pytest.mark.dependency() + def test_h(): + pass + """) + result = ctestdir.runpytest("--verbose") + result.assert_outcomes(passed=8, skipped=0, failed=0) + result.stdout.re_match_lines(r""" + .*::test_a PASSED + .*::test_c PASSED + .*::test_e PASSED + .*::test_g PASSED + .*::test_f PASSED + .*::test_d PASSED + .*::test_b PASSED + .*::test_h PASSED + """) + + +def test_order_scopes(ctestdir): + """ + 9 tests, with dependencies spread between 'module' and 'session' scopes. + """ + ctestdir.makepyfile(""" + import pytest + + @pytest.mark.dependency() + def test_a(): + pass + + @pytest.mark.dependency(depends=["test_c"]) + def test_b(): + pass + + @pytest.mark.dependency() + def test_c(): + pass + + @pytest.mark.dependency(depends=["test_order_scopes.py::test_e"], scope="session") + def test_d(): + pass + + @pytest.mark.dependency() + @pytest.mark.dependency(scope="session") + def test_e(): + pass + + @pytest.mark.dependency(depends=["test_g"]) + @pytest.mark.dependency(depends=["test_order_scopes.py::test_e"], scope="session") + def test_f(): + pass + + @pytest.mark.dependency() + def test_g(): + pass + + @pytest.mark.dependency(depends=["test_order_scopes.py::test_i"], scope="session") + @pytest.mark.dependency(depends=["test_e"]) + def test_h(): + pass + + @pytest.mark.dependency(scope="session") + def test_i(): + pass + """) + result = ctestdir.runpytest("--verbose") + result.assert_outcomes(passed=9, skipped=0, failed=0) + result.stdout.re_match_lines(r""" + .*::test_a PASSED + .*::test_c PASSED + .*::test_b PASSED + .*::test_e PASSED + .*::test_d PASSED + .*::test_g PASSED + .*::test_f PASSED + .*::test_i PASSED + .*::test_h PASSED + """) diff --git a/tests/test_09_examples_advanced.py b/tests/test_09_examples_advanced.py index f2a9f95..d54c5cc 100644 --- a/tests/test_09_examples_advanced.py +++ b/tests/test_09_examples_advanced.py @@ -1,7 +1,7 @@ -"""Test the included examples. +""" +Test the included examples. """ -import pytest from conftest import get_example @@ -133,13 +133,13 @@ def test_all_params(ctestdir): .*::test_a\[14\] PASSED .*::test_a\[15\] PASSED .*::test_a\[16\] PASSED - .*::test_b SKIPPED(?:\s+\(.*\))? .*::test_c\[0-2\] PASSED .*::test_c\[2-3\] PASSED .*::test_c\[4-4\] PASSED .*::test_c\[6-5\] (?:XFAIL(?:\s+\(.*\))?|xfail) - .*::test_d SKIPPED(?:\s+\(.*\))? .*::test_e\[abc\] PASSED .*::test_e\[def\] (?:XFAIL(?:\s+\(.*\))?|xfail) + .*::test_b SKIPPED(?:\s+\(.*\))? + .*::test_d SKIPPED(?:\s+\(.*\))? .*::test_f SKIPPED(?:\s+\(.*\))? """) diff --git a/tests/test_09_examples_names.py b/tests/test_09_examples_names.py index 7d45ba7..b66218d 100644 --- a/tests/test_09_examples_names.py +++ b/tests/test_09_examples_names.py @@ -1,7 +1,7 @@ -"""Test the included examples. +""" +Test the included examples. """ -import pytest from conftest import get_example diff --git a/tests/test_09_examples_scope.py b/tests/test_09_examples_scope.py index 7975e5a..1ef96f8 100644 --- a/tests/test_09_examples_scope.py +++ b/tests/test_09_examples_scope.py @@ -1,8 +1,8 @@ -"""Test the included examples. +""" +Test the included examples. """ from pathlib import Path -import pytest from conftest import get_example diff --git a/tests/test_09_examples_usage.py b/tests/test_09_examples_usage.py index 85db269..0e8bc8d 100644 --- a/tests/test_09_examples_usage.py +++ b/tests/test_09_examples_usage.py @@ -1,7 +1,7 @@ -"""Test the included examples. +""" +Test the included examples. """ -import pytest from conftest import get_example