Explore test coverage (#1281)

* wip use latest wk docker version for update cassettes by default and add coverage.

* use uv cache also for test.sh

* restore uv cache for python version.

* coverage only for py311

* typo.

* coverage report.

* Update workflow.

* Update workflow.

* Update workflow.

* Update workflow.

* Update workflow.

* Update workflow.

* Update ci workflow.

* move logic for fetching latest webknossos version from nightly to test

* Update ci.

* Update ci.

* update ci.

* Update ci.

* Update ci.

* Update ci.

* Update ci.

* Update ci.

* Update ci.

* implement requested changes.

* use gh actions coverage.

* Change working dir.

* Update ci.

* debuf download artifacts.

* Update ci.

* Update ci.

* Update ci.

* remove debug ci step.

* Add thresholds.

Author: markbader

.github/workflows/ci.yml

@@ -180,7 +180,7 @@ jobs:
 
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
-      
+
       - name: Check formatting
         if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }}
         run: ./format.sh check
@@ -198,17 +198,70 @@ jobs:
           sed -i '/def _launch/a\ \ \ \ \ \ \ \ raise Exception("fork is not allowed.")' /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py
           cat /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py
 
+      - name: Python tests (with coverage)
+        if: ${{ matrix.python-version == '3.11' }}
+        timeout-minutes: 30
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          COVERAGE_FILE: "~/.local/share/coverage/${{ matrix.group }}.coverage"
+        run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration --cov=webknossos --cov-report=
+
       - name: Python tests
+        if: ${{ matrix.python-version != '3.11' }}
         timeout-minutes: 30
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
         run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration
 
+      - name: Persist coverage reports
+        if: ${{ matrix.python-version == '3.11' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report-${{ matrix.group }}
+          path: "~/.local/share/coverage/${{ matrix.group }}.coverage"
+          retention-days: 1  # Automatically deletes after 1 day
+
       - name: Check if git is dirty
         run: |
           git diff --no-ext-diff --exit-code
           [[ -z $(git status -s) ]]
 
+  coverage_report:
+    needs: [webknossos_linux]
+    if: success()
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: webknossos
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install coverage.py
+        run: pip install coverage
+
+      - name: Download all coverage reports
+        uses: actions/download-artifact@v4
+        with:
+          path: ~/coverage-files
+
+      - name: Generate coverage report
+        env:
+          COVERAGE_FILE: "~/coverage-files/result.coverage"
+        run: |
+          coverage combine ~/coverage-files/**/*.coverage
+          coverage xml -i -o ~/coverage-files/result.xml
+
+      - name: Report coverage
+        uses: orgoro/[email protected]
+        with:
+          coverageFile: /home/runner/coverage-files/result.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          thresholdAll: 0.8
+          thresholdNew: 0.8
+      
+      - name: Cleanup temporary files
+        run: rm -rf ~/coverage-files
+
   webknossos_cli_docker:
     needs: [cluster_tools, webknossos_linux]
     if: |

.github/workflows/nightly.yml

@@ -37,10 +37,6 @@ jobs:
         git diff --no-ext-diff --quiet --exit-code
         [[ -z $(git status -s) ]]
 
-    - name: Use currently deployed Webknossos version
-      run: |
-        sed -i "s/export DOCKER_TAG=.*$/export DOCKER_TAG=master__$(curl https://webknossos.org/api/buildinfo | jq -r .webknossos.version)/" local_wk_setup.sh
-
     - name: Python tests, refreshing the network snapshots
       env:
         WK_TOKEN: ${{ secrets.WK_TOKEN }}

webknossos/local_wk_setup.sh

@@ -1,7 +1,11 @@
 function export_vars {
+    # Fetch current version of webknossos.org this can be replaced with a fixed version for testing
+    WK_VERSION=$(curl -s https://webknossos.org/api/buildinfo | jq -r '.webknossos.version')
+
+    # Export the necessary environment variables
     export WK_TOKEN=1b88db86331a38c21a0b235794b9e459856490d70408bcffb767f64ade0f83d2bdb4c4e181b9a9a30cdece7cb7c65208cc43b6c1bb5987f5ece00d348b1a905502a266f8fc64f0371cd6559393d72e031d0c2d0cabad58cccf957bb258bc86f05b5dc3d4fff3d5e3d9c0389a6027d861a21e78e3222fb6c5b7944520ef21761e
     export WK_URL=http://localhost:9000
-    export DOCKER_TAG=master__32090
+    export DOCKER_TAG="master__${WK_VERSION}"
 }
 
 function ensure_local_test_wk {
@@ -54,13 +58,6 @@ function ensure_local_test_wk {
     fi
 
     curl -s -X POST -H "X-Auth-Token: $WK_TOKEN" localhost:9000/data/triggers/checkInboxBlocking
-
-    WK_ORG_VERSION="$(curl -s https://webknossos.org/api/buildinfo | tr ',"' "\n" | sed -n '/version/{n;n;p;q;}')"
-    LOCAL_VERSION="$(curl -s http://localhost:9000/api/buildinfo | tr ',"' "\n" | sed -n '/version/{n;n;p;q;}')"
-
-    if [ "$WK_ORG_VERSION" != "$LOCAL_VERSION" ]; then
-        echo "The local webknossos version is $LOCAL_VERSION, differing from the webknossos.org version $WK_ORG_VERSION"
-    fi
 }
 
 

webknossos/pyproject.toml

@@ -88,13 +88,15 @@ all = [
 [tool.uv]
 extra-index-url = ["https://pypi.scm.io/simple"]
 dev-dependencies = [
+    "coverage>=7.8.0",
     "fsspec ~=2024.6.0",
     "hypothesis ~=6.35.0",
     "icecream ~=2.1.1",
     "inducoapi ~=2.0.2",
     "jsonschema ~=4.6.0",
     "mypy ~=1.15.0",
     "pytest ~=8.3.2",
+    "pytest-cov>=6.0.0",
     "pytest-custom-exit-code ~=0.3.0",
     "pytest-split ~=0.10.0",
     "pytest-sugar ~=1.0.0",