Skip to content

Commit 5b8bd97

Browse files
FynnBeFynnBe
authored
Merge pull request #440 from bioimage-io/fix_torch_load
Update adapters and converters
2 parents 7962d3e + 7e9888a commit 5b8bd97

File tree

80 files changed

+3761
-2311
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+3761
-2311
lines changed

‎.github/workflows/build.yaml‎

Lines changed: 175 additions & 172 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88

99
defaults:
1010
run:
11-
shell: micromamba-shell {0}
11+
shell: bash -el {0}
1212

1313
jobs:
1414
black:
@@ -22,215 +22,218 @@ jobs:
2222
jupyter: true
2323
version: "24.3"
2424

25-
test-spec-conda:
25+
populate-cache:
2626
runs-on: ubuntu-latest
27-
strategy:
28-
matrix:
29-
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
27+
outputs:
28+
cache-key: ${{steps.cache-key.outputs.cache-key}}
3029
steps:
31-
- uses: actions/checkout@v4
32-
- name: Install Conda environment with Micromamba
33-
if: matrix.python-version != '3.8'
34-
uses: mamba-org/setup-micromamba@v1
35-
with:
36-
cache-downloads: true
37-
cache-environment: true
38-
environment-file: dev/env-wo-python.yaml
39-
create-args: >-
40-
python=${{ matrix.python-version }}
41-
post-cleanup: 'all'
42-
- name: Install py3.8 environment
43-
if: matrix.python-version == '3.8'
44-
uses: mamba-org/setup-micromamba@v1
45-
with:
46-
cache-downloads: true
47-
cache-environment: true
48-
environment-file: dev/env-py38.yaml
49-
post-cleanup: 'all'
50-
- name: additional setup
51-
run: pip install --no-deps -e .
52-
- name: Get Date
53-
id: get-date
54-
run: |
55-
echo "date=$(date +'%Y-%b')"
56-
echo "date=$(date +'%Y-%b')" >> $GITHUB_OUTPUT
57-
shell: bash
58-
- uses: actions/cache@v4
59-
with:
60-
path: bioimageio_cache
61-
key: "test-spec-conda-${{ steps.get-date.outputs.date }}"
62-
- name: pytest-spec-conda
63-
run: pytest --disable-pytest-warnings
64-
env:
65-
BIOIMAGEIO_CACHE_PATH: bioimageio_cache
66-
67-
test-spec-main:
30+
- name: Get Date
31+
id: get-date
32+
run: |
33+
echo "date=$(date +'%Y-%b')"
34+
echo "date=$(date +'%Y-%b')" >> $GITHUB_OUTPUT
35+
- id: cache-key
36+
run: echo "cache-key=test-${{steps.get-date.outputs.date}}" >> $GITHUB_OUTPUT
37+
- uses: actions/cache/restore@v4
38+
id: look-up
39+
with:
40+
path: bioimageio_cache
41+
key: ${{steps.cache-key.outputs.cache-key}}
42+
lookup-only: true
43+
- uses: actions/checkout@v4
44+
if: steps.look-up.outputs.cache-hit != 'true'
45+
- uses: actions/cache@v4
46+
if: steps.look-up.outputs.cache-hit != 'true'
47+
with:
48+
path: bioimageio_cache
49+
key: ${{steps.cache-key.outputs.cache-key}}
50+
- uses: actions/setup-python@v5
51+
if: steps.look-up.outputs.cache-hit != 'true'
52+
with:
53+
python-version: '3.12'
54+
cache: 'pip'
55+
- name: Install dependencies
56+
if: steps.look-up.outputs.cache-hit != 'true'
57+
run: |
58+
pip install --upgrade pip
59+
pip install -e .[dev]
60+
- run: pytest --disable-pytest-warnings tests/test_bioimageio_collection.py::test_rdf_format_to_populate_cache
61+
if: steps.look-up.outputs.cache-hit != 'true'
62+
env:
63+
BIOIMAGEIO_POPULATE_CACHE: '1'
64+
BIOIMAGEIO_CACHE_PATH: bioimageio_cache
65+
test:
66+
needs: populate-cache
6867
runs-on: ubuntu-latest
6968
strategy:
7069
matrix:
71-
python-version: ['3.8', '3.12']
7270
include:
71+
- python-version: '3.8'
72+
conda-env: py38
73+
spec: conda
74+
- python-version: '3.8'
75+
conda-env: py38
76+
spec: main
77+
- python-version: '3.9'
78+
conda-env: dev
79+
spec: conda
80+
- python-version: '3.10'
81+
conda-env: dev
82+
spec: conda
83+
- python-version: '3.11'
84+
conda-env: full
85+
spec: main
86+
run-expensive-tests: true
87+
report-coverage: true
88+
save-cache: true
7389
- python-version: '3.12'
74-
is-dev-version: true
90+
conda-env: dev
91+
spec: conda
92+
- python-version: '3.13'
93+
conda-env: dev
94+
spec: main
95+
save-cache: true
96+
7597
steps:
7698
- uses: actions/checkout@v4
77-
- name: Install Conda environment with Micromamba
78-
if: matrix.python-version != '3.8'
79-
uses: mamba-org/setup-micromamba@v1
80-
with:
81-
cache-downloads: true
82-
cache-environment: true
83-
environment-file: dev/env-wo-python.yaml
84-
create-args: >-
85-
python=${{ matrix.python-version }}
86-
post-cleanup: 'all'
87-
- name: Install py3.8 environment
88-
if: matrix.python-version == '3.8'
89-
uses: mamba-org/setup-micromamba@v1
90-
with:
91-
cache-downloads: true
92-
cache-environment: true
93-
environment-file: dev/env-py38.yaml
94-
post-cleanup: 'all'
95-
- name: additional setup spec
99+
- id: setup
100+
run: |
101+
echo "env-name=${{ matrix.spec }}-${{ matrix.conda-env }}-${{ matrix.python-version }}"
102+
echo "env-name=${{ matrix.spec }}-${{ matrix.conda-env }}-${{ matrix.python-version }}" >> $GITHUB_OUTPUT
103+
echo "env-file=dev/env-${{ matrix.conda-env }}.yaml"
104+
echo "env-file=dev/env-${{ matrix.conda-env }}.yaml" >> $GITHUB_OUTPUT
105+
- name: check on env-file
106+
shell: python
96107
run: |
97-
conda remove --yes --force bioimageio.spec || true # allow failure for cached env
98-
pip install --no-deps git+https://github.com/bioimage-io/spec-bioimage-io
99-
- name: additional setup core
100-
run: pip install --no-deps -e .
108+
from pathlib import Path
109+
from pprint import pprint
110+
if not (env_path:=Path("${{steps.setup.outputs.env-file}}")).exists():
111+
if env_path.parent.exists():
112+
pprint(env_path.parent.glob("*"))
113+
else:
114+
pprint(Path().glob("*"))
115+
raise FileNotFoundError(f"{env_path} does not exist")
116+
117+
- uses: conda-incubator/setup-miniconda@v3
118+
with:
119+
auto-update-conda: true
120+
auto-activate-base: true
121+
activate-environment: ${{steps.setup.outputs.env-name}}
122+
channel-priority: strict
123+
miniforge-version: latest
101124
- name: Get Date
102125
id: get-date
103126
run: |
104-
echo "date=$(date +'%Y-%b')"
105-
echo "date=$(date +'%Y-%b')" >> $GITHUB_OUTPUT
106-
shell: bash
107-
- uses: actions/cache@v4
127+
echo "today=$(date -u '+%Y%m%d')"
128+
echo "today=$(date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
129+
- name: Restore cached env
130+
uses: actions/cache/restore@v4
131+
with:
132+
path: ${{env.CONDA}}/envs/${{steps.setup.outputs.env-name}}
133+
key: >-
134+
conda-${{runner.os}}-${{runner.arch}}
135+
-${{steps.get-date.outputs.today}}
136+
-${{hashFiles(steps.setup.outputs.env-file)}}
137+
-${{env.CACHE_NUMBER}}
138+
env:
139+
CACHE_NUMBER: 0
140+
id: cache-env
141+
- name: Install env
142+
run: conda env update --name=${{steps.setup.outputs.env-name}} --file=${{steps.setup.outputs.env-file}} python=${{matrix.python-version}}
143+
if: steps.cache-env.outputs.cache-hit != 'true'
144+
- name: Install uncached pip dependencies
145+
run: |
146+
pip install --upgrade pip
147+
pip install --no-deps -e .
148+
- name: Install uncached pip dependencies for 'full' environment
149+
if: matrix.conda-env == 'full'
150+
run: |
151+
pip install git+https://github.com/ChaoningZhang/MobileSAM.git
152+
- name: Cache env
153+
if: steps.cache-env.outputs.cache-hit != 'true'
154+
uses: actions/cache/save@v4
155+
with:
156+
path: ${{env.CONDA}}/envs/${{steps.setup.outputs.env-name}}
157+
key: >-
158+
conda-${{runner.os}}-${{runner.arch}}
159+
-${{steps.get-date.outputs.today}}
160+
-${{hashFiles(steps.setup.outputs.env-file)}}
161+
-${{env.CACHE_NUMBER}}
162+
env:
163+
CACHE_NUMBER: 0
164+
- run: conda list
165+
- name: Pyright
166+
run: |
167+
pyright --version
168+
pyright -p pyproject.toml --pythonversion ${{ matrix.python-version }}
169+
if: matrix.run-expensive-tests
170+
- name: Restore bioimageio cache
171+
uses: actions/cache/restore@v4
172+
id: bioimageio-cache
108173
with:
109174
path: bioimageio_cache
110-
key: "test-spec-main-${{ steps.get-date.outputs.date }}"
111-
- name: pytest-spec-main
175+
key: ${{needs.populate-cache.outputs.cache-key}}${{matrix.run-expensive-tests && '' || '-light'}}
176+
- name: pytest
112177
run: pytest --disable-pytest-warnings
113178
env:
114179
BIOIMAGEIO_CACHE_PATH: bioimageio_cache
115-
- if: matrix.is-dev-version && github.event_name == 'pull_request'
180+
RUN_EXPENSIVE_TESTS: ${{ matrix.run-expensive-tests && 'true' || 'false' }}
181+
- name: Save updated bioimageio cache
182+
if: matrix.save-cache
183+
uses: actions/cache/save@v4
184+
with:
185+
path: bioimageio_cache
186+
key: ${{needs.populate-cache.outputs.cache-key}}${{matrix.run-expensive-tests && '' || '-light'}}
187+
- if: matrix.report-coverage && github.event_name == 'pull_request'
116188
uses: orgoro/[email protected]
117189
with:
118190
coverageFile: coverage.xml
119-
token: ${{ secrets.GITHUB_TOKEN }}
120-
- if: matrix.is-dev-version && github.ref == 'refs/heads/main'
191+
token: ${{secrets.GITHUB_TOKEN}}
192+
- if: matrix.report-coverage && github.ref == 'refs/heads/main'
121193
run: |
122194
pip install genbadge[coverage]
123195
genbadge coverage --input-file coverage.xml --output-file ./dist/coverage/coverage-badge.svg
124196
coverage html -d dist/coverage
125-
- if: matrix.is-dev-version && github.ref == 'refs/heads/main'
197+
- if: matrix.report-coverage && github.ref == 'refs/heads/main'
126198
uses: actions/upload-artifact@v4
127199
with:
128200
name: coverage
129201
retention-days: 1
130202
path: dist
131203

132-
133-
test-spec-main-tf:
134-
runs-on: ubuntu-latest
135-
strategy:
136-
matrix:
137-
python-version: ['3.9', '3.12']
138-
steps:
139-
- uses: actions/checkout@v4
140-
- uses: mamba-org/setup-micromamba@v1
141-
with:
142-
cache-downloads: true
143-
cache-environment: true
144-
environment-file: dev/env-tf.yaml
145-
condarc: |
146-
channel-priority: flexible
147-
create-args: >-
148-
python=${{ matrix.python-version }}
149-
post-cleanup: 'all'
150-
- name: additional setup spec
151-
run: |
152-
conda remove --yes --force bioimageio.spec || true # allow failure for cached env
153-
pip install --no-deps git+https://github.com/bioimage-io/spec-bioimage-io
154-
- name: additional setup core
155-
run: pip install --no-deps -e .
156-
- name: Get Date
157-
id: get-date
158-
run: |
159-
echo "date=$(date +'%Y-%b')"
160-
echo "date=$(date +'%Y-%b')" >> $GITHUB_OUTPUT
161-
shell: bash
162-
- uses: actions/cache@v4
163-
with:
164-
path: bioimageio_cache
165-
key: "test-spec-main-tf-${{ steps.get-date.outputs.date }}"
166-
- run: pytest --disable-pytest-warnings
167-
env:
168-
BIOIMAGEIO_CACHE_PATH: bioimageio_cache
169-
170-
test-spec-conda-tf:
171-
runs-on: ubuntu-latest
172-
strategy:
173-
matrix:
174-
python-version: ['3.9', '3.12']
175-
steps:
176-
- uses: actions/checkout@v4
177-
- uses: mamba-org/setup-micromamba@v1
178-
with:
179-
cache-downloads: true
180-
cache-environment: true
181-
environment-file: dev/env-tf.yaml
182-
condarc: |
183-
channel-priority: flexible
184-
create-args: >-
185-
python=${{ matrix.python-version }}
186-
post-cleanup: 'all'
187-
- name: additional setup
188-
run: pip install --no-deps -e .
189-
- name: Get Date
190-
id: get-date
191-
run: |
192-
echo "date=$(date +'%Y-%b')"
193-
echo "date=$(date +'%Y-%b')" >> $GITHUB_OUTPUT
194-
shell: bash
195-
- uses: actions/cache@v4
196-
with:
197-
path: bioimageio_cache
198-
key: "test-spec-conda-tf-${{ steps.get-date.outputs.date }}"
199-
- name: pytest-spec-tf
200-
run: pytest --disable-pytest-warnings
201-
env:
202-
BIOIMAGEIO_CACHE_PATH: bioimageio_cache
203-
204204
conda-build:
205+
needs: test
205206
runs-on: ubuntu-latest
206-
needs: test-spec-conda
207207
steps:
208-
- name: checkout
209-
uses: actions/checkout@v4
210-
with:
211-
fetch-depth: 0
212-
- name: Install Conda environment with Micromamba
213-
uses: mamba-org/setup-micromamba@v1
214-
with:
215-
cache-downloads: true
216-
cache-environment: true
217-
environment-name: build-env
218-
condarc: |
219-
channels:
220-
- conda-forge
221-
create-args: |
222-
boa
223-
- name: linux conda build
224-
run: |
225-
conda mambabuild -c conda-forge conda-recipe
208+
- uses: actions/checkout@v4
209+
with:
210+
fetch-depth: 0
211+
- uses: conda-incubator/setup-miniconda@v3
212+
with:
213+
auto-update-conda: true
214+
auto-activate-base: true
215+
activate-environment: ""
216+
channel-priority: strict
217+
miniforge-version: latest
218+
conda-solver: libmamba
219+
- name: install common conda dependencies
220+
run: conda install -n base -c conda-forge conda-build -y
221+
- uses: actions/cache@v4
222+
with:
223+
path: |
224+
pkgs/noarch
225+
pkgs/channeldata.json
226+
key: ${{ github.sha }}-packages
227+
- name: linux conda build test
228+
shell: bash -l {0}
229+
run: |
230+
mkdir -p ./pkgs/noarch
231+
conda-build -c conda-forge conda-recipe --no-test --output-folder ./pkgs
226232
227233
docs:
228-
needs: [test-spec-main]
234+
needs: test
229235
if: github.ref == 'refs/heads/main'
230236
runs-on: ubuntu-latest
231-
defaults:
232-
run:
233-
shell: bash -l {0}
234237
steps:
235238
- uses: actions/checkout@v4
236239
- uses: actions/download-artifact@v4

0 commit comments

Comments
 (0)