Change performance tests to only check our code

- should make it more reliable
- tests are Python 3 only

Author: mrbean-bremen

perf_tests/__init__.py

@@ -0,0 +1,35 @@
+import os
+import shutil
+from unittest import mock
+
+import pytest
+import pytest_order
+from perf_tests.util import TimedSorter
+from tests.utils import write_test
+
+
+@pytest.fixture
+def fixture_path_ordinal(tmpdir_factory):
+    fixture_path = str(tmpdir_factory.mktemp("ordinal_perf"))
+    for module_index in range(10):
+        testname = os.path.join(
+            fixture_path, "test_performance{}.py".format(module_index))
+        test_contents = """
+import pytest
+"""
+        for i in range(100):
+            test_contents += """
+@pytest.mark.order({})
+def test_{}():
+    assert True
+""".format(50 - i, i)
+        write_test(testname, test_contents)
+    yield fixture_path
+    shutil.rmtree(fixture_path, ignore_errors=True)
+
+
+@mock.patch("pytest_order.Sorter", TimedSorter)
+def test_performance_ordinal(fixture_path_ordinal):
+    args = [fixture_path_ordinal]
+    pytest.main(args, [pytest_order])
+    assert TimedSorter.elapsed < 0.02

perf_tests/test_order.py

@@ -0,0 +1,41 @@
+import os
+import shutil
+from unittest import mock
+
+import pytest
+
+import pytest_order
+from perf_tests.util import TimedSorter
+from tests.utils import write_test
+
+
+@pytest.fixture
+def fixture_path_relative(tmpdir_factory):
+    fixture_path = str(tmpdir_factory.mktemp("relative_perf"))
+    for module_index in range(10):
+        testname = os.path.join(
+            fixture_path, "test_relative_perf{}.py".format(module_index))
+        test_contents = """
+import pytest
+"""
+        for i in range(90):
+            test_contents += """
+@pytest.mark.order(after="test_{}")
+def test_{}():
+    assert True
+""".format(i + 10, i)
+        for i in range(10):
+            test_contents += """
+def test_{}():
+    assert True
+""".format(i + 90)
+        write_test(testname, test_contents)
+    yield fixture_path
+    shutil.rmtree(fixture_path, ignore_errors=True)
+
+
+@mock.patch("pytest_order.Sorter", TimedSorter)
+def test_performance_relative(fixture_path_relative):
+    args = [fixture_path_relative]
+    pytest.main(args, [pytest_order])
+    assert TimedSorter.elapsed < 1.2

perf_tests/test_ordinal.py

@@ -0,0 +1,15 @@
+import time
+
+from pytest_order import Sorter
+
+
+class TimedSorter(Sorter):
+    elapsed = 0
+
+    def sort_items(self):
+        self.__class__.elapsed = 0
+        start_time = time.time()
+        items = super().sort_items()
+        self.__class__.elapsed += time.time() - start_time
+        print("Time per test: {:.3f} ms".format(self.__class__.elapsed))
+        return items