Skip to content

Commit ff99a77

Browse files
psomhorstpsomhorst
authored
Merge pull request #455 from EIT-ALIVE/tests/330-open-source-test-data
Migrate to open source test data and fix revealed issues
2 parents 20ed9b7 + 50a94c5 commit ff99a77

37 files changed

+1328
-711
lines changed

‎.github/actions/install_eitprocessing/action.yml‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,36 +8,20 @@ inputs:
88
dependencies:
99
description: "The optional dependencies of eitprocessing to install"
1010
required: false
11-
extract-data:
12-
description: "Whether to extract testing data"
13-
required: false
14-
default: "false"
1511
token:
1612
description: "GitHub TOKEN"
1713
required: true
18-
data-directory:
19-
description: "Directory where to store eitprocessing data"
14+
zenodo-record-id:
15+
description: "Zenodo record ID for test dataset"
2016
required: false
21-
default: ${{ github.workspace }}/../eitprocessing_data/
17+
default: "17423608"
2218

2319
runs:
2420
using: "composite"
2521
steps:
2622
- uses: actions/setup-python@v5
2723
with:
2824
python-version: ${{ inputs.python-version }}
29-
- uses: docker/login-action@v3
30-
if: ${{ inputs.extract-data == 'true' }}
31-
with:
32-
registry: ghcr.io
33-
username: psomhorst
34-
password: ${{ inputs.token }}
35-
- uses: shrink/actions-docker-extract@v3
36-
if: ${{ inputs.extract-data == 'true' }}
37-
with:
38-
image: ghcr.io/eit-alive/eittestdata:latest
39-
destination: ${{ inputs.data-directory }}
40-
path: /eitprocessing/.
4125
- uses: actions/cache@v4
4226
id: cache-python-env
4327
with:

‎.github/workflows/coverage.yml‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,45 @@ jobs:
1616
os: ["ubuntu-latest"]
1717
python-version: ["3.10"]
1818
env:
19-
EIT_PROCESSING_TEST_DATA: ${{ github.workspace }}/../eitprocessing_data/
2019
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2120
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
2221
COVERALLS_FLAG_NAME: python-${{ matrix.python-version }}
22+
ZENODO_RECORD_ID: 17423608
2323
steps:
2424
- uses: actions/checkout@v4
2525
- uses: ./.github/actions/install_eitprocessing
2626
with:
2727
dependencies: testing
28-
extract-data: true
2928
python-version: ${{ matrix.python-version }}
3029
token: ${{ secrets.GITHUB_TOKEN }}
31-
data-directory: ${{ env.EIT_PROCESSING_TEST_DATA }}
30+
- name: Set data dir
31+
shell: bash
32+
id: set-path
33+
run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT"
34+
- name: Restore dataset cache
35+
id: cache
36+
uses: actions/cache@v4
37+
with:
38+
path: ${{ steps.set-path.outputs.data_dir }}
39+
key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }}
40+
- name: Install zenodo-get
41+
if: steps.cache.outputs.cache-hit != 'true'
42+
run: python3 -m pip install zenodo-get
43+
shell: bash
44+
- name: Download test dataset
45+
shell: bash
46+
if: steps.cache.outputs['cache-hit'] != 'true'
47+
run: |
48+
mkdir -p ${{ steps.set-path.outputs.data_dir }}
49+
cd ${{ steps.set-path.outputs.data_dir }}
50+
zenodo_get $ZENODO_RECORD_ID
51+
cd -
52+
- name: Uninstall zenodo-get
53+
if: steps.cache.outputs['cache-hit'] != 'true'
54+
run: python3 -m pip uninstall --yes zenodo-get
55+
shell: bash
3256
- name: Run coveralls
3357
run: |
34-
pytest --cov --cov-report xml --cov-report term --cov-report html
35-
git config --global --add safe.directory /ci
58+
pytest --cov --cov-report xml --cov-report term --cov-report html
59+
git config --global --add safe.directory /ci
3660
coveralls --service=github

‎.github/workflows/release_github.yml‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
name: Draft GitHub Release
2-
32
on:
43
workflow_dispatch:
54
inputs:
@@ -25,6 +24,7 @@ permissions:
2524
env:
2625
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2726
GH_PAGER: cat
27+
ZENODO_RECORD_ID: 17423608
2828

2929
jobs:
3030
checks:
@@ -77,8 +77,6 @@ jobs:
7777
os: ["ubuntu-latest"]
7878
python-version: ["3.10", "3.11", "3.12", "3.13"]
7979
name: Build for ${{ matrix.python-version }}, ${{ matrix.os }}
80-
env:
81-
EIT_PROCESSING_TEST_DATA: ${{ github.workspace }}/../eitprocessing_data/
8280
steps:
8381
- name: Checkout repository
8482
uses: actions/checkout@v4
@@ -87,11 +85,34 @@ jobs:
8785
uses: ./.github/actions/install_eitprocessing
8886
with:
8987
dependencies: testing
90-
extract-data: true
9188
python-version: ${{ matrix.python-version }}
9289
token: ${{ secrets.GITHUB_TOKEN }}
93-
data-directory: ${{ env.EIT_PROCESSING_TEST_DATA }}
94-
90+
- name: Set data dir
91+
shell: bash
92+
id: set-path
93+
run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT"
94+
- name: Restore dataset cache
95+
id: cache
96+
uses: actions/cache@v4
97+
with:
98+
path: ${{ steps.set-path.outputs.data_dir }}
99+
key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }}
100+
- name: Install zenodo-get
101+
if: steps.cache.outputs.cache-hit != 'true'
102+
run: python3 -m pip install zenodo-get
103+
shell: bash
104+
- name: Download test dataset
105+
shell: bash
106+
if: steps.cache.outputs['cache-hit'] != 'true'
107+
run: |
108+
mkdir -p ${{ steps.set-path.outputs.data_dir }}
109+
cd ${{ steps.set-path.outputs.data_dir }}
110+
zenodo_get $ZENODO_RECORD_ID
111+
cd -
112+
- name: Uninstall zenodo-get
113+
if: steps.cache.outputs['cache-hit'] != 'true'
114+
run: python3 -m pip uninstall --yes zenodo-get
115+
shell: bash
95116
- name: Run pytest
96117
run: pytest -v --runslow
97118

‎.github/workflows/release_pypi.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ jobs:
1616
- uses: ./.github/actions/install_eitprocessing
1717
with:
1818
dependencies: publishing
19-
extract-data: false
2019
python-version: "3.10"
2120
- name: Build wheel and source distribution
2221
run: python -m build

‎.github/workflows/test_build_documentation.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ jobs:
2323
- uses: ./.github/actions/install_eitprocessing
2424
with:
2525
dependencies: docs
26-
extract-data: false
2726
python-version: "3.10"
2827
- name: Link notebooks
2928
run: ln -s ../../notebooks docs/examples

‎.github/workflows/testing.yml‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
name: Build and Test
2-
2+
env:
3+
ZENODO_RECORD_ID: 17423608
34
on:
45
push:
56
branches:
@@ -14,7 +15,6 @@ on:
1415
branches:
1516
- main
1617
- develop
17-
1818
jobs:
1919
test_and_build:
2020
if: github.event.pull_request.draft == false
@@ -25,16 +25,38 @@ jobs:
2525
os: ["ubuntu-latest"]
2626
python-version: ["3.10"]
2727
name: Test and build for ${{ matrix.python-version }}, ${{ matrix.os }}
28-
env:
29-
EIT_PROCESSING_TEST_DATA: ${{ github.workspace }}/../eitprocessing_data/
3028
steps:
3129
- uses: actions/checkout@v4
3230
- uses: ./.github/actions/install_eitprocessing
3331
with:
3432
dependencies: testing
35-
extract-data: true
3633
python-version: ${{ matrix.python-version }}
3734
token: ${{ secrets.GITHUB_TOKEN }}
38-
data-directory: ${{ env.EIT_PROCESSING_TEST_DATA }}
35+
- name: Set data dir
36+
shell: bash
37+
id: set-path
38+
run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT"
39+
- name: Restore dataset cache
40+
id: cache
41+
uses: actions/cache@v4
42+
with:
43+
path: ${{ steps.set-path.outputs.data_dir }}
44+
key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }}
45+
- name: Install zenodo-get
46+
if: steps.cache.outputs.cache-hit != 'true'
47+
run: python3 -m pip install zenodo-get
48+
shell: bash
49+
- name: Download test dataset
50+
shell: bash
51+
if: steps.cache.outputs['cache-hit'] != 'true'
52+
run: |
53+
mkdir -p ${{ steps.set-path.outputs.data_dir }}
54+
cd ${{ steps.set-path.outputs.data_dir }}
55+
zenodo_get $ZENODO_RECORD_ID
56+
cd -
57+
- name: Uninstall zenodo-get
58+
if: steps.cache.outputs['cache-hit'] != 'true'
59+
run: python3 -m pip uninstall --yes zenodo-get
60+
shell: bash
3961
- name: Run pytest
4062
run: pytest -v

‎README.dev.md‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ Please follow these steps:
1212

1313
1. (**important**) announce your plan to the rest of the community _before you start working_. This announcement should be in the form of a (new) issue;
1414
1. (**important**) wait until some kind of consensus is reached about your idea being a good idea;
15-
1. if needed, fork the repository to your own Github profile and create your own feature branch off of the latest master commit. While working on your feature branch, make sure to stay up to date with the master branch by pulling in changes, possibly from the 'upstream' repository (follow the instructions [here](https://help.github.com/articles/configuring-a-remote-for-a-fork/) and [here](https://help.github.com/articles/syncing-a-fork/));
15+
1. if needed, fork the repository to your own GitHub profile and create your own feature branch off of the latest master commit. While working on your feature branch, make sure to stay up to date with the master branch by pulling in changes, possibly from the 'upstream' repository (follow the instructions [here](https://help.github.com/articles/configuring-a-remote-for-a-fork/) and [here](https://help.github.com/articles/syncing-a-fork/));
1616
1. make sure the existing tests still work by running `pytest` (see also [here](#testing-locally));
1717
1. add your own tests (if necessary);
1818
1. update or expand the documentation;
@@ -31,7 +31,7 @@ readibility or simplicity is more important than absolute correctness.
3131
It is hard to define the precise balance we are looking for, so instead we will refer
3232
to the [Zen of python](https://peps.python.org/pep-0020/).
3333

34-
Note that all contrubtions to this project will be published under our [Apache 2.0 licence]
34+
Note that all contributions to this project will be published under our [Apache 2.0 licence]
3535
(<https://github.com/EIT-ALIVE/eitprocessing/blob/main/LICENSE>).
3636

3737
#### Docstrings
@@ -49,7 +49,7 @@ place to start. This extension is currently in preview, but seems to work more r
4949
#### Branch naming convention
5050

5151
Please try to adhere to the following branch naming convention:
52-
<github-issue-number>_<brief-description>_<username>.
52+
`<github-issue-number>_<brief-description>_<username>`.
5353
E.g., `042_life_universe_everything_douglasadams`.
5454

5555
This allows, at a single glance, to see in the issue that you're
@@ -98,6 +98,38 @@ the CI.
9898
Make sure you have developer options installed as described in the [README](README.md)
9999
(otherwise run: `pip install -e .[dev]` on the repository folder in your environment)
100100

101+
##### Downloading test data
102+
Some tests require access to test data. You can download the test data from Zenodo via the button below. Note that for
103+
some reason downloading all files at ones results in a corrupted zip file. Please download the files one by one.
104+
105+
[![](https://zenodo.org/badge/DOI/10.5281/zenodo.17423608.svg)](https://doi.org/10.5281/zenodo.17423608)
106+
107+
Test data should reside in the `test_data/` folder in the root of the repository.
108+
109+
Alternatively, use zenodo-get to download the data directly into the `test_data/` folder:
110+
111+
Using `uv`:
112+
113+
```shell
114+
mkdir -p test_data
115+
cd test_data
116+
uv tool run zenodo_get 17423608
117+
cd -
118+
```
119+
120+
Using `pip`:
121+
122+
```shell
123+
pip install zenodo-get
124+
mkdir -p test_data
125+
cd test_data
126+
zenodo_get 17423608
127+
cd -
128+
```
129+
130+
131+
##### Running tests
132+
101133
For testing all you need to do is run:
102134

103135
```shell
@@ -120,7 +152,9 @@ coverage report
120152

121153
We use [ruff](https://docs.astral.sh/ruff/) for linting, sorting imports and formatting of python (notebook) files. The configurations of `ruff` are set in [pyproject.toml](pyproject.toml) file.
122154

123-
If you are using VS code, please install and activate the [Ruff extension](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff) to automatically format and check linting.
155+
If you are using VS code, please install and activate the [Ruff
156+
extension](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff) to automatically format and check
157+
linting. Make sure to use the ruff version installed in your environment.
124158

125159
Otherwise, please ensure check both linting (`ruff fix .`) and formatting (`ruff format .`) before requesting a review.
126160

‎eitprocessing/datahandling/eitdata.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,15 @@ def __post_init__(self, suppress_simulated_warning: bool) -> None:
6060
self.path = self.path[0]
6161

6262
self.name = self.name or self.label
63+
old_sample_frequency = self.sample_frequency
64+
self.sample_frequency = float(self.sample_frequency)
65+
if self.sample_frequency != old_sample_frequency:
66+
msg = (
67+
"Sample frequency could not be correctly converted from "
68+
f"{old_sample_frequency} ({type(old_sample_frequency)}) to "
69+
f"{self.sample_frequency:.1f} (float)."
70+
)
71+
raise TypeError(msg)
6372

6473
if (lv := len(self.pixel_impedance)) != (lt := len(self.time)):
6574
msg = f"The number of time points ({lt}) does not match the number of pixel impedance values ({lv})."

0 commit comments

Comments
 (0)