Migrate to open source test data

.github/actions/install_eitprocessing/action.yml

@@ -8,36 +8,20 @@ inputs:
   dependencies:
     description: "The optional dependencies of eitprocessing to install"
     required: false
-  extract-data:
-    description: "Whether to extract testing data"
-    required: false
-    default: "false"
   token:
     description: "GitHub TOKEN"
     required: true
-  data-directory:
-    description: "Directory where to store eitprocessing data"
+  zenodo-record-id:
+    description: "Zenodo record ID for test dataset"
     required: false
-    default: ${{ github.workspace }}/../eitprocessing_data/
+    default: "17423608"
 
 runs:
   using: "composite"
   steps:
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
-    - uses: docker/login-action@v3
-      if: ${{ inputs.extract-data == 'true' }}
-      with:
-        registry: ghcr.io
-        username: psomhorst
-        password: ${{ inputs.token }}
-    - uses: shrink/actions-docker-extract@v3
-      if: ${{ inputs.extract-data == 'true' }}
-      with:
-        image: ghcr.io/eit-alive/eittestdata:latest
-        destination: ${{ inputs.data-directory }}
-        path: /eitprocessing/.
     - uses: actions/cache@v4
       id: cache-python-env
       with:

.github/workflows/coverage.yml

@@ -16,21 +16,45 @@ jobs:
         os: ["ubuntu-latest"]
         python-version: ["3.10"]
     env:
-      EIT_PROCESSING_TEST_DATA: ${{ github.workspace }}/../eitprocessing_data/
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
       COVERALLS_FLAG_NAME: python-${{ matrix.python-version }}
+      ZENODO_RECORD_ID: 17423608
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_eitprocessing
         with:
           dependencies: testing
-          extract-data: true
           python-version: ${{ matrix.python-version }}
           token: ${{ secrets.GITHUB_TOKEN }}
-          data-directory: ${{ env.EIT_PROCESSING_TEST_DATA }}
+      - name: Set data dir
+        shell: bash
+        id: set-path
+        run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT"
+      - name: Restore dataset cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.set-path.outputs.data_dir }}
+          key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }}
+      - name: Install zenodo-get
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: python3 -m pip install zenodo-get
+        shell: bash
+      - name: Download test dataset
+        shell: bash
+        if: steps.cache.outputs['cache-hit'] != 'true'
+        run: |
+          mkdir -p ${{ steps.set-path.outputs.data_dir }}
+          cd ${{ steps.set-path.outputs.data_dir }}
+          zenodo_get $ZENODO_RECORD_ID
+          cd -
+      - name: Uninstall zenodo-get
+        if: steps.cache.outputs['cache-hit'] != 'true'
+        run: python3 -m pip uninstall --yes zenodo-get
+        shell: bash
       - name: Run coveralls
         run: |
-          pytest --cov --cov-report xml --cov-report term --cov-report html 
-          git config --global --add safe.directory /ci 
+          pytest --cov --cov-report xml --cov-report term --cov-report html
+          git config --global --add safe.directory /ci
           coveralls --service=github

.github/workflows/release_github.yml

@@ -1,5 +1,4 @@
 name: Draft GitHub Release
-
 on:
   workflow_dispatch:
     inputs:
@@ -25,6 +24,7 @@ permissions:
 env:
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   GH_PAGER: cat
+  ZENODO_RECORD_ID: 17423608
 
 jobs:
   checks:
@@ -77,8 +77,6 @@ jobs:
         os: ["ubuntu-latest"]
         python-version: ["3.10", "3.11", "3.12", "3.13"]
     name: Build for ${{ matrix.python-version }}, ${{ matrix.os }}
-    env:
-      EIT_PROCESSING_TEST_DATA: ${{ github.workspace }}/../eitprocessing_data/
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -87,11 +85,34 @@ jobs:
         uses: ./.github/actions/install_eitprocessing
         with:
           dependencies: testing
-          extract-data: true
           python-version: ${{ matrix.python-version }}
           token: ${{ secrets.GITHUB_TOKEN }}
-          data-directory: ${{ env.EIT_PROCESSING_TEST_DATA }}
-
+      - name: Set data dir
+        shell: bash
+        id: set-path
+        run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT"
+      - name: Restore dataset cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.set-path.outputs.data_dir }}
+          key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }}
+      - name: Install zenodo-get
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: python3 -m pip install zenodo-get
+        shell: bash
+      - name: Download test dataset
+        shell: bash
+        if: steps.cache.outputs['cache-hit'] != 'true'
+        run: |
+          mkdir -p ${{ steps.set-path.outputs.data_dir }}
+          cd ${{ steps.set-path.outputs.data_dir }}
+          zenodo_get $ZENODO_RECORD_ID
+          cd -
+      - name: Uninstall zenodo-get
+        if: steps.cache.outputs['cache-hit'] != 'true'
+        run: python3 -m pip uninstall --yes zenodo-get
+        shell: bash
       - name: Run pytest
         run: pytest -v --runslow
 

.github/workflows/release_pypi.yml

@@ -16,7 +16,6 @@ jobs:
       - uses: ./.github/actions/install_eitprocessing
         with:
           dependencies: publishing
-          extract-data: false
           python-version: "3.10"
       - name: Build wheel and source distribution
         run: python -m build

.github/workflows/test_build_documentation.yml

@@ -23,7 +23,6 @@ jobs:
       - uses: ./.github/actions/install_eitprocessing
         with:
           dependencies: docs
-          extract-data: false
           python-version: "3.10"
       - name: Link notebooks
         run: ln -s ../../notebooks docs/examples

.github/workflows/testing.yml

@@ -1,5 +1,6 @@
 name: Build and Test
-
+env:
+  ZENODO_RECORD_ID: 17423608
 on:
   push:
     branches:
@@ -14,7 +15,6 @@ on:
     branches:
       - main
       - develop
-
 jobs:
   test_and_build:
     if: github.event.pull_request.draft == false
@@ -25,16 +25,38 @@ jobs:
         os: ["ubuntu-latest"]
         python-version: ["3.10"]
     name: Test and build for ${{ matrix.python-version }}, ${{ matrix.os }}
-    env:
-      EIT_PROCESSING_TEST_DATA: ${{ github.workspace }}/../eitprocessing_data/
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_eitprocessing
         with:
           dependencies: testing
-          extract-data: true
           python-version: ${{ matrix.python-version }}
           token: ${{ secrets.GITHUB_TOKEN }}
-          data-directory: ${{ env.EIT_PROCESSING_TEST_DATA }}
+      - name: Set data dir
+        shell: bash
+        id: set-path
+        run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT"
+      - name: Restore dataset cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.set-path.outputs.data_dir }}
+          key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }}
+      - name: Install zenodo-get
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: python3 -m pip install zenodo-get
+        shell: bash
+      - name: Download test dataset
+        shell: bash
+        if: steps.cache.outputs['cache-hit'] != 'true'
+        run: |
+          mkdir -p ${{ steps.set-path.outputs.data_dir }}
+          cd ${{ steps.set-path.outputs.data_dir }}
+          zenodo_get $ZENODO_RECORD_ID
+          cd -
+      - name: Uninstall zenodo-get
+        if: steps.cache.outputs['cache-hit'] != 'true'
+        run: python3 -m pip uninstall --yes zenodo-get
+        shell: bash
       - name: Run pytest
         run: pytest -v

README.dev.md

@@ -12,7 +12,7 @@ Please follow these steps:
 
 1. (**important**) announce your plan to the rest of the community _before you start working_. This announcement should be in the form of a (new) issue;
 1. (**important**) wait until some kind of consensus is reached about your idea being a good idea;
-1. if needed, fork the repository to your own Github profile and create your own feature branch off of the latest master commit. While working on your feature branch, make sure to stay up to date with the master branch by pulling in changes, possibly from the 'upstream' repository (follow the instructions [here](https://help.github.com/articles/configuring-a-remote-for-a-fork/) and [here](https://help.github.com/articles/syncing-a-fork/));
+1. if needed, fork the repository to your own GitHub profile and create your own feature branch off of the latest master commit. While working on your feature branch, make sure to stay up to date with the master branch by pulling in changes, possibly from the 'upstream' repository (follow the instructions [here](https://help.github.com/articles/configuring-a-remote-for-a-fork/) and [here](https://help.github.com/articles/syncing-a-fork/));
 1. make sure the existing tests still work by running `pytest` (see also [here](#testing-locally));
 1. add your own tests (if necessary);
 1. update or expand the documentation;
@@ -31,7 +31,7 @@ readibility or simplicity is more important than absolute correctness.
 It is hard to define the precise balance we are looking for, so instead we will refer
 to the [Zen of python](https://peps.python.org/pep-0020/).
 
-Note that all contrubtions to this project will be published under our [Apache 2.0 licence]
+Note that all contributions to this project will be published under our [Apache 2.0 licence]
 (<https://github.com/EIT-ALIVE/eitprocessing/blob/main/LICENSE>).
 
 #### Docstrings
@@ -49,7 +49,7 @@ place to start. This extension is currently in preview, but seems to work more r
 #### Branch naming convention
 
 Please try to adhere to the following branch naming convention:
-<github-issue-number>_<brief-description>_<username>.
+`<github-issue-number>_<brief-description>_<username>`.
 E.g., `042_life_universe_everything_douglasadams`.
 
 This allows, at a single glance, to see in the issue that you're
@@ -98,6 +98,38 @@ the CI.
 Make sure you have developer options installed as described in the [README](README.md)
 (otherwise run: `pip install -e .[dev]` on the repository folder in your environment)
 
+##### Downloading test data
+Some tests require access to test data. You can download the test data from Zenodo via the button below. Note that for
+some reason downloading all files at ones results in a corrupted zip file. Please download the files one by one.
+
+[![](https://zenodo.org/badge/DOI/10.5281/zenodo.17423608.svg)](https://doi.org/10.5281/zenodo.17423608)
+
+Test data should reside in the `test_data/` folder in the root of the repository.
+
+Alternatively, use zenodo-get to download the data directly into the `test_data/` folder:
+
+Using `uv`:
+
+```shell
+mkdir -p test_data
+cd test_data
+uv tool run zenodo_get 17423608
+cd -
+```
+
+Using `pip`:
+
+```shell
+pip install zenodo-get
+mkdir -p test_data
+cd test_data
+zenodo_get 17423608
+cd -
+```
+
+
+##### Running tests
+
 For testing all you need to do is run:
 
 ```shell
@@ -120,7 +152,9 @@ coverage report
 
 We use [ruff](https://docs.astral.sh/ruff/) for linting, sorting imports and formatting of python (notebook) files. The configurations of `ruff` are set in [pyproject.toml](pyproject.toml) file.
 
-If you are using VS code, please install and activate the [Ruff extension](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff) to automatically format and check linting.
+If you are using VS code, please install and activate the [Ruff
+extension](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff) to automatically format and check
+linting. Make sure to use the ruff version installed in your environment.
 
 Otherwise, please ensure check both linting (`ruff fix .`) and formatting (`ruff format .`) before requesting a review.
 

eitprocessing/datahandling/eitdata.py

@@ -60,6 +60,15 @@ def __post_init__(self, suppress_simulated_warning: bool) -> None:
             self.path = self.path[0]
 
         self.name = self.name or self.label
+        old_sample_frequency = self.sample_frequency
+        self.sample_frequency = float(self.sample_frequency)
+        if self.sample_frequency != old_sample_frequency:
+            msg = (
+                "Sample frequency could not be correctly converted from "
+                f"{old_sample_frequency} ({type(old_sample_frequency)}) to "
+                f"{self.sample_frequency:.1f} (float)."
+            )
+            raise TypeError(msg)
 
         if (lv := len(self.pixel_impedance)) != (lt := len(self.time)):
             msg = f"The number of time points ({lt}) does not match the number of pixel impedance values ({lv})."