use pytest-split in GHA test workflow

Author: janosh

.github/workflows/combine_durations.py

@@ -0,0 +1,27 @@
+import json
+from pathlib import Path
+
+split_prefix = "split-"
+durations_path = Path(".test_durations")
+
+split_paths = Path(".").glob(f"{split_prefix}*/{durations_path.name}")
+
+try:
+    previous_durations = json.loads(durations_path.read_text())
+except FileNotFoundError:
+    previous_durations = {}
+
+new_durations = previous_durations.copy()
+
+for path in split_paths:
+    durations = json.loads(path.read_text())
+    new_durations.update(
+        {
+            key: duration
+            for key, duration in durations.items()
+            if previous_durations[key] != duration
+        }
+    )
+
+durations_path.parent.mkdir(parents=True, exist_ok=True)
+durations_path.write_text(json.dumps(new_durations))

.github/workflows/test.yml

@@ -10,6 +10,10 @@ jobs:
   test:
     if: github.repository_owner == 'hackingmaterials' # don't run on forks
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # pytest-split groups for automatically equally split concurrent test runners
+        group: [1, 2, 3, 4, 5]
     services:
       mongodb:
         image: mongo
@@ -45,6 +49,49 @@ jobs:
           pip install -r requirements-ci.txt
           pip install .[complete]
 
-      - name: pytest
+      - name: Get durations from cache
+        uses: actions/cache@v2
+        with:
+          path: test_durations
+          # the key must never match, even when restarting workflows, as that
+          # will cause durations to get out of sync between groups, the
+          # combined durations will be loaded if available
+          key: test-durations-split-${{ github.run_id }}-${{ github.run_number}}-${{ matrix.group }}
+          restore-keys: |
+            test-durations-combined-${{ github.sha }}
+            test-durations-combined
+
+      - name: Run tests
         run: |
-          pytest --ignore=atomate/qchem/test_files --cov=atomate --cov-report html:coverage_reports atomate
+          pytest --splits 5 --group ${{ matrix.group }} --store-durations \
+            --cov=atomate --cov-report html:coverage_reports atomate
+
+      - name: Upload partial durations
+        uses: actions/upload-artifact@v2
+        with:
+          name: split-${{ matrix.group }}
+          path: .test_durations
+
+  update_durations:
+    name: Combine and update test durations
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Get durations from cache
+        uses: actions/cache@v2
+        with:
+          path: .test_durations
+          # key won't match during the first run for the given commit, but
+          # restore-key will if there's a previous stored durations file,
+          # so cache will both be loaded and stored
+          key: test-durations-combined-${{ github.sha }}
+          restore-keys: test-durations-combined
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+
+      - name: Combine test durations
+        run: python3 .github/workflows/combine_durations.py

requirements-ci.txt

@@ -3,3 +3,4 @@ coverage==5.5
 moto==2.0.10
 pytest-cov==2.12.1
 pytest==6.2.4
+pytest-split