Add performance test for the ordering overhead

Author: mrbean-bremen

.github/workflows/performance.yml

@@ -0,0 +1,42 @@
+name: PerformanceTests
+
+on:
+  push
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [2.7, 3.6, 3.9]
+    env:
+      OS: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        python -m pip install -U pip
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: pip cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: py${{ matrix.python-version }}-${{ matrix.os }}-pip
+    - name: Test environment setup
+      run: |
+        python -m pip install wheel
+        python -m pip install pytest
+    - name: Run tests
+      run: |
+        python -m pytest -s perf_tests

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+### Infrastructure
+- add performance tests to prevent performance degradation
+
 ## [Version 0.9.5](https://pypi.org/project/pytest-order/0.9.5/) (2021-02-16)
 Introduces hierarchical ordering option and fixes ordering of session-scoped
 dependency markers. 

perf_tests/test_order.py

@@ -0,0 +1,94 @@
+import os
+import shutil
+import time
+
+import pytest
+
+import pytest_order
+from tests.utils import write_test
+
+
+@pytest.fixture
+def fixture_path_baseline(tmpdir_factory):
+    fixture_path = str(tmpdir_factory.mktemp("baseline"))
+    for module_index in range(10):
+        testname = os.path.join(
+            fixture_path, "test_baseline{}.py".format(module_index))
+        test_contents = """
+import pytest
+"""
+        for i in range(100):
+            test_contents += """
+def test_{}():
+    assert True
+""".format(i)
+        write_test(testname, test_contents)
+    yield fixture_path
+    shutil.rmtree(fixture_path, ignore_errors=True)
+
+
+@pytest.fixture
+def fixture_path_relative(tmpdir_factory):
+    fixture_path = str(tmpdir_factory.mktemp("relative_perf"))
+    for module_index in range(10):
+        testname = os.path.join(
+            fixture_path, "test_relative_perf{}.py".format(module_index))
+        test_contents = """
+import pytest
+"""
+        for i in range(100):
+            test_contents += """
+@pytest.mark.order(after="test_{}")
+def test_{}():
+    assert True
+""".format(i + 10 % 100, i)
+        write_test(testname, test_contents)
+    yield fixture_path
+    shutil.rmtree(fixture_path, ignore_errors=True)
+
+
+@pytest.fixture
+def fixture_path_ordinal(tmpdir_factory):
+    fixture_path = str(tmpdir_factory.mktemp("ordinal_perf"))
+    for module_index in range(10):
+        testname = os.path.join(
+            fixture_path, "test_performance{}.py".format(module_index))
+        test_contents = """
+import pytest
+"""
+        for i in range(100):
+            test_contents += """
+@pytest.mark.order({})
+def test_{}():
+    assert True
+""".format(50 - i, i)
+        write_test(testname, test_contents)
+    yield fixture_path
+    shutil.rmtree(fixture_path, ignore_errors=True)
+
+
+class TestPerformance:
+    base_time = None
+
+    @pytest.mark.order(0)
+    def test_baseline_times(self, fixture_path_baseline):
+        start_time = time.time()
+        args = [fixture_path_baseline]
+        pytest.main(args, [pytest_order])
+        self.__class__.base_time = time.time() - start_time
+
+    def test_performance_ordinal(self, fixture_path_ordinal):
+        start_time = time.time()
+        args = [fixture_path_ordinal]
+        pytest.main(args, [pytest_order])
+        overhead_time = time.time() - start_time - self.__class__.base_time
+        print("Overhead per test: {:.3f} ms".format(overhead_time))
+        assert overhead_time < 0.4
+
+    def test_performance_relative(self, fixture_path_relative):
+        start_time = time.time()
+        args = ["--quiet", fixture_path_relative]
+        pytest.main(args, [pytest_order])
+        overhead_time = time.time() - start_time - self.__class__.base_time
+        print("Overhead per test: {:.3f} ms".format(overhead_time))
+        assert overhead_time < 0.8

tests/__init__.py

@@ -1,15 +1,12 @@
 import os
 import shutil
 import uuid
+from random import randint
 
 import pytest
 
 from pytest_order.sorter import SESSION
-
-try:
-    from tests.utils import write_test
-except ImportError:
-    from utils import write_test
+from tests.utils import write_test
 
 pytest_plugins = ["pytester"]
 
@@ -73,3 +70,9 @@ def test_node(nodeid):
 
     yield fixture_path
     shutil.rmtree(fixture_path, ignore_errors=True)
+
+
+def pytest_collection_modifyitems(config, items):
+    for item in items:
+        if item.name.startswith("test_performance"):
+            item.add_marker(pytest.mark.order(randint(-100, 100)))

tests/conftest.py

@@ -6,12 +6,7 @@
 import pytest
 
 import pytest_order
-
-try:
-    from tests.utils import write_test, assert_test_order
-except ImportError:
-    from utils import write_test, assert_test_order
-
+from tests.utils import write_test, assert_test_order
 
 NODEID_ROOT = ""
 

tests/test_dependency.py

@@ -5,11 +5,7 @@
 import pytest
 
 import pytest_order
-
-try:
-    from tests.utils import write_test, assert_test_order
-except ImportError:
-    from utils import write_test, assert_test_order
+from tests.utils import write_test, assert_test_order
 
 
 @pytest.fixture(scope="module")

tests/test_order_group_scope.py

@@ -6,12 +6,7 @@
 import pytest
 
 import pytest_order
-
-try:
-    from tests.utils import write_test, assert_test_order
-except ImportError:
-    from utils import write_test, assert_test_order
-
+from tests.utils import write_test, assert_test_order
 
 NODEID_ROOT = ""
 

tests/test_order_group_scope_dep.py

@@ -6,10 +6,7 @@
 
 import pytest_order
 
-try:
-    from tests.utils import write_test, assert_test_order
-except ImportError:
-    from utils import write_test, assert_test_order
+from tests.utils import write_test, assert_test_order
 
 
 @pytest.fixture(scope="module")

tests/test_order_group_scope_named_dep.py

@@ -6,10 +6,7 @@
 
 import pytest_order
 
-try:
-    from tests.utils import write_test, assert_test_order
-except ImportError:
-    from utils import write_test, assert_test_order
+from tests.utils import write_test, assert_test_order
 
 
 @pytest.fixture(scope="module")