v0.3.0 (#3)

Author: provinzkraut

.github/workflows/test.yaml

@@ -51,9 +51,9 @@ jobs:
       - name: Install dependencies
         run: |
           uv sync
-          uv pip install pytest=="${{matrix.pytest-version}}"
+          uv pip install pytest~="${{matrix.pytest-version}}.0"
       - name: Test
-        run: uv run pytest
+        run: uv run --no-sync pytest
 
   mypy:
     runs-on: ubuntu-latest

README.md

@@ -1,6 +1,7 @@
 # pytest-cdist
 
-Like pytest-xdist, but for distributed environments.
+Like pytest-xdist, but for distributed environments - pytest-cdist allows to split a 
+test suite into multiple parts and run them independently.
 
 **This is a work in progress**
 
@@ -13,8 +14,6 @@ large. pytest-cdist can help with this by allowing to execute individual chunks
 test suite in a deterministic order, so you can use multiple concurrent jobs to run each
 individual chunk.
 
-The individual invocation can *still* make use of pytest-xdist.
-
 
 ## How?
 
@@ -105,5 +104,56 @@ When running under pytest-xdist, pytest-cdist will honour tests marked with
 
 ### With pytest-randomly
 
-At the moment, pytest-cdist does not work out of the box with pytest randomly's test
-reordering, unless an explicit seed is passed
+To use pytest-cdist with 
+[pytest-randomly](https://github.com/pytest-dev/pytest-randomly)'s test reordering, a 
+randomness seed that's consistent across the different pytest 
+invocations needs to be specified, otherwise, test cases would be dropped, since 
+pytest-cdist has to rely on a consistent collection order to split test cases among its
+groups.
+
+To achieve this, a random source such as the current timestamp can be used:
+
+```yaml
+jobs:
+  setup_randomly_seed:
+    runs-on: ubuntu-latest
+    outputs:
+      randomly_seed: ${{ steps.set-seed.outputs.randomly_seed }}
+    steps:
+      - name: Set seed
+        id: set-seed
+        run: echo "randomly_seed=$(date +%s)" >> $GITHUB_OUTPUT
+        
+  test:
+    runs-on: ubuntu-latest
+    needs: setup_randomly_seed
+    matrix:
+      strategy:
+        cdist-groups: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run pytest
+        run: > 
+          pytest 
+          --cdist-group=${{ matrix.cdist-group }}/4 
+          --randomly-seed=${{ needs.setup_randomly_seeds.outputs.randomly_seed }}
+```
+
+or a bit simpler (but less random), using the current git hash:
+
+```yaml
+jobs:        
+  test:
+    runs-on: ubuntu-latest
+    needs: setup_randomly_seed
+    matrix:
+      strategy:
+        cdist-groups: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run pytest
+        run: > 
+          pytest 
+          --cdist-group=${{ matrix.cdist-group }}/4 
+          --randomly-seed=$((16#$(git rev-parse HEAD)))
+```

pyproject.toml

@@ -1,7 +1,10 @@
 [project]
 name = "pytest-cdist"
-version = "0.2.0"
-description = "Add your description here"
+version = "0.3.0"
+description = "A pytest plugin to split your test suite into multiple parts"
+authors = [
+  { name = "Janek Nouvertné", email = "provinzkraut@posteo.de" },
+]
 readme = "README.md"
 requires-python = ">=3.9"
 classifiers = [
@@ -32,6 +35,7 @@ dev = [
     "mypy>=1.14.0",
     "pre-commit>=4.0.1",
     "typing-extensions>=4.12.2",
+    "pytest-randomly>=3.16.0",
 ]
 
 [project.entry-points.pytest11]
@@ -52,3 +56,7 @@ packages = ["pytest_cdist", "tests"]
 
 [tool.ruff]
 target-version = "py39"
+
+
+[tool.pytest.ini_options]
+addopts = "-p no:randomly"

pytest_cdist/plugin.py

@@ -167,7 +167,9 @@ def from_pytest_config(cls, config: pytest.Config) -> CdistConfig | None:
         ) or config.getini("cdist-justify-items")
 
         group_steal = _get_group_steal_opt(
-            config.getoption("cdist_group_steal") or config.getini("cdist-group-steal")
+            config.getoption("cdist_group_steal")
+            or config.getini("cdist-group-steal")
+            or None  # pytest<8 returns an empty string here
         )
 
         current_group, total_groups = map(int, cdist_option.split("/"))
@@ -213,6 +215,17 @@ def pytest_addoption(parser: pytest.Parser) -> None:
 def pytest_configure(config: pytest.Config) -> None:
     cdist_config = CdistConfig.from_pytest_config(config)
     config.stash[_CDIST_CONFIG_KEY] = cdist_config
+    if (
+        config.pluginmanager.hasplugin("randomly")
+        and config.getoption("randomly_seed") == "default"
+        and config.getoption("randomly_reorganize")
+    ):
+        raise pytest.UsageError(
+            "pytest-cdist is incompatible with the current pytest-randomly "
+            "configuration and will produce incorrect results. To use both of them "
+            "together, either specify a randomness seed using '--randomly-seed=...' or "
+            "disable test reorganization by passing '--randomly-dont-reorganize'."
+        )
 
 
 def pytest_collection_modifyitems(

test.py

@@ -1 +1,13 @@
+import pytest
+
 pytest_plugins = ["pytester"]
+
+
+@pytest.fixture()
+def ini_tpl() -> str:
+    return '[pytest]\naddopts="-p no:randomly"'
+
+
+@pytest.fixture(autouse=True)
+def make_ini(pytester: pytest.Pytester, ini_tpl: str) -> None:
+    pytester.makeini(ini_tpl)

tests/conftest.py

@@ -87,7 +87,10 @@ def test_three(self):
     ],
 )
 def test_justify_cli_ini_cfg(
-    pytester: pytest.Pytester, cli_opt: str, ini_opt: str | None
+    pytester: pytest.Pytester,
+    cli_opt: str,
+    ini_opt: str | None,
+    ini_tpl: str,
 ) -> None:
     pytester.makepyfile("""
     def test_one():
@@ -100,7 +103,7 @@ def test_three():
         assert True
     """)
     if ini_opt is not None:
-        pytester.makeini(f"[pytest]\n{ini_opt}")
+        pytester.makeini(ini_tpl + f"\n{ini_opt}")
 
     result = pytester.runpytest("--cdist-group=1/2", cli_opt)
     result.assert_outcomes(passed=3)
@@ -179,7 +182,9 @@ def test_two():
         ("--cdist-group-steal=2:50", "cdist-group-steal=2:50"),
     ],
 )
-def test_steal(pytester: pytest.Pytester, cli_opt: str, ini_opt: str | None) -> None:
+def test_steal(
+    pytester: pytest.Pytester, cli_opt: str, ini_opt: str | None, ini_tpl: str
+) -> None:
     pytester.makepyfile("""
     def test_one():
         assert True
@@ -195,10 +200,68 @@ def test_four():
     """)
 
     if ini_opt is not None:
-        pytester.makeini(f"[pytest]\n{ini_opt}")
+        pytester.makeini(ini_tpl + f"\n{ini_opt}")
 
     result = pytester.runpytest("--cdist-group=1/2", cli_opt)
     result.assert_outcomes(passed=1, deselected=3)
 
     result = pytester.runpytest("--cdist-group=2/2", cli_opt)
     result.assert_outcomes(passed=3, deselected=1)
+
+
+def test_raises_for_invalid_randomly_cfg(pytester: pytest.Pytester) -> None:
+    pytester.makeini("")
+
+    result = pytester.runpytest("--cdist-group=1/2")
+    assert (
+        "pytest-cdist is incompatible with the current pytest-randomly configuration"
+        in result.stderr.str()
+    )
+
+
+@pytest.mark.parametrize("i", range(10))
+def test_randomly_with_seed(pytester: pytest.Pytester, i: int) -> None:
+    pytester.makeini('[pytest]\naddopts="-p randomly"')
+
+    pytester.makepyfile("""
+    def test_one():
+        assert True
+
+    def test_two():
+        assert True
+
+    def test_three():
+        assert False
+    """)
+
+    result = pytester.runpytest_subprocess("--cdist-group=1/2", "--randomly-seed=123")
+    result.assert_outcomes(passed=1, failed=1, deselected=1)
+
+    result = pytester.runpytest_subprocess("--cdist-group=2/2", "--randomly-seed=123")
+    result.assert_outcomes(passed=1, failed=0, deselected=2)
+
+
+@pytest.mark.parametrize("i", range(10))
+def test_randomly_with_dont_reorganize(pytester: pytest.Pytester, i: int) -> None:
+    pytester.makeini('[pytest]\naddopts="-p randomly"')
+
+    pytester.makepyfile("""
+    def test_one():
+        assert True
+
+    def test_two():
+        assert True
+
+    def test_three():
+        assert False
+    """)
+
+    result = pytester.runpytest_subprocess(
+        "--cdist-group=1/2", "--randomly-dont-reorganize"
+    )
+    result.assert_outcomes(passed=2, failed=0, deselected=1)
+
+    result = pytester.runpytest_subprocess(
+        "--cdist-group=2/2", "--randomly-dont-reorganize"
+    )
+    result.assert_outcomes(passed=0, failed=1, deselected=2)