Merge pull request #474 from bioimage-io/dev

Improve testing in dedicated conda env

Author: FynnBe

.github/workflows/build.yaml

@@ -44,7 +44,6 @@ jobs:
       - name: Install dependencies
         if: steps.look-up.outputs.cache-hit != 'true'
         run: |
-          pip install --upgrade pip
           pip install -e .[dev]
       - run: pytest --disable-pytest-warnings tests/test_bioimageio_collection.py::test_rdf_format_to_populate_cache
         if: steps.look-up.outputs.cache-hit != 'true'
@@ -94,7 +93,7 @@ jobs:
           path: bioimageio_cache
           key: ${{needs.populate-cache.outputs.cache-key}}
       - name: pytest
-        run: pytest --cov bioimageio --cov-report xml --cov-append --capture no --disable-pytest-warnings
+        run: pytest --cov bioimageio.core --cov-append --capture no --disable-pytest-warnings
         env:
           BIOIMAGEIO_CACHE_PATH: bioimageio_cache
           RUN_EXPENSIVE_TESTS: ${{ matrix.run-expensive-tests && 'true' || 'false' }}
@@ -105,25 +104,50 @@ jobs:
           path: bioimageio_cache
           key: ${{needs.populate-cache.outputs.cache-key}}
 
-      - if: matrix.report-coverage && github.event_name == 'pull_request'
-        uses: orgoro/[email protected]
+      - run: cp .coverage .coverage.${{matrix.python-version}}-${{matrix.numpy-version}}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: .coverage.${{matrix.python-version}}-${{matrix.numpy-version}}
+          retention-days: 1
+          path: .coverage.${{matrix.python-version}}-${{matrix.numpy-version}}
+          include-hidden-files: true
+
+  coverage:
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v6
+      - run: |
+          pip install coverage
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: .coverage.*
+          merge-multiple: true
+      - run: |
+          ls -la .coverage*
+          coverage combine
+          coverage xml -o coverage.xml
+      - uses: orgoro/[email protected]
         with:
           coverageFile: coverage.xml
-          token: ${{secrets.GITHUB_TOKEN}}
-      - if: matrix.report-coverage && github.ref == 'refs/heads/main'
+          token: ${{ secrets.GITHUB_TOKEN }}
+          thresholdAll: 0.7
+          thresholdNew: 0.9
+          thresholdModified: 0.6
+      - name: generate coverage badge and html report
         run: |
           pip install genbadge[coverage]
           genbadge coverage --input-file coverage.xml --output-file ./dist/coverage/coverage-badge.svg
           coverage html -d dist/coverage
-      - if: matrix.report-coverage && github.ref == 'refs/heads/main'
-        uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v4
         with:
-          name: coverage
+          name: coverage-summary
           retention-days: 1
           path: dist
 
   conda-build:
-    needs: test
+    needs: test # only so we run tests even if the pinned bioimageio.spec version is not yet published on conda-forge yet
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -159,7 +183,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
         with:
-          name: coverage
+          name: coverage-summary
           path: dist
       - uses: actions/setup-python@v6
         with:
@@ -179,65 +203,66 @@ jobs:
           branch: gh-pages
           folder: dist
 
-  pip-build:
-    name: Build with pip and publish to PyPI
-    needs: [test, conda-build]
+  build:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out the repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-
-      - name: Set up Python
-        uses: actions/setup-python@v6
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v6
         with:
           python-version: '3.10'
+          cache: 'pip'
       - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip --version
-          pip install --upgrade build
-          pip install .
+        run: pip install --upgrade build
       - name: Build package
-        run: |
-          python -m build
+        run: python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          path: dist/
+          name: dist
 
+  publish:
+    needs: [test, build, conda-build, docs]
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/project/bioimageio.core/
+    permissions:
+      contents: write # required for tag creation
+      id-token: write # required for pypi publish action
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
       - name: Check if there is a parent commit
         id: check-parent-commit
         run: |
           echo "sha=$(git rev-parse --verify --quiet HEAD^)" >> $GITHUB_OUTPUT
 
       - name: Detect new version
-        id: check-version
+        id: tag-version
         if: github.ref == 'refs/heads/main' && steps.check-parent-commit.outputs.sha
         uses: salsify/[email protected]
         with:
-          create-tag: false
+          create-tag: true
           version-command: |
             python -c "from pathlib import Path;import json;print(p_src.read_text().split('__version__ = \"')[1].split('\"')[0] if (p_src:=Path('src/bioimageio/core/__init__.py')).exists() else json.loads(Path('bioimageio/core/VERSION').read_text())['version'])"
-
-      - name: Push tag
-        id: tag-version
-        if: github.ref == 'refs/heads/main' && steps.check-version.outputs.previous-version != steps.check-version.outputs.current-version
-        uses: mathieudutour/[email protected]
+      - uses: actions/download-artifact@v4
+        if: github.ref == 'refs/heads/main' && steps.tag-version.outputs.tag
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          custom_tag: ${{ steps.check-version.outputs.current-version }}
-
+          name: dist
+          path: dist
       - name: Publish package on PyPI
-        if: github.ref == 'refs/heads/main' && steps.tag-version.outputs.new_tag
-        uses: pypa/gh-action-pypi-publish@release/v1.12
+        if: github.ref == 'refs/heads/main' && steps.tag-version.outputs.tag
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          user: __token__
-          password: '${{ secrets.PYPI_TOKEN }}'
           packages-dir: dist/
-          verbose: true
+
       - name: Publish the release notes
         if: github.ref == 'refs/heads/main'
         uses: release-drafter/[email protected]
         with:
-          publish: "${{ steps.tag-version.outputs.new_tag != '' }}"
-          tag: '${{ steps.tag-version.outputs.new_tag }}'
+          publish: "${{ steps.tag-version.outputs.tag != '' }}"
+          tag: '${{ steps.tag-version.outputs.tag }}'
         env:
           GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'

README.md

@@ -347,15 +347,24 @@ stats: inputs/dataset_statistics.json
 
 ## Set up Development Environment
 
-To set up a development conda environment run the following commands:
+To set up a development environment run the following commands:
 
 ```console
-conda env create -f dev/env.yaml
+conda create -n core python=$(grep -E '^requires-python' pyproject.toml | grep -oE '[0-9]+\.[0-9]+')
 conda activate core
-pip install -e . --no-deps
+pip install -e .[dev,partners]
 ```
 
-There are different environment files available that only install tensorflow or pytorch as dependencies, see [dev folder](https://github.com/bioimage-io/core-bioimage-io-python/tree/main/dev).
+### Joint development of bioimageio.spec and bioimageio.core
+
+Assuming [spec-bioimage-io](https://github.com/bioimage-io/spec-bioimage-io) is cloned to the parent folder
+a joint development environment can be created with the following commands:
+
+```console
+conda create -n core python=$(grep -E '^requires-python' pyproject.toml | grep -oE '[0-9]+\.[0-9]+')
+conda activate core
+pip install -e .[dev,partners] -e ../spec-bioimage-io[dev]
+```
 
 ## Logging level
 

changelog.md

@@ -1,3 +1,9 @@
+### 0.9.5
+
+- bump bioimageio.spec library version to 0.5.6.0
+- improved ONNX export from pytorch state dict or torchscript using torch dynamo
+- default `opset_version` for `pytorch_to_onnx`/`torchscript_for_onnx` conversions bumped to 18.
+
 ### 0.9.4
 
 - bump bioimageio.spec library version to 0.5.5.6

conda-recipe/meta.yaml

@@ -54,9 +54,7 @@ test:
     - tests
   requires:
     {% for dep in pyproject['project']['optional-dependencies']['dev'] %}
-      {% if dep.startswith('torch>=') %}  # pip: torch  -> conda: pytorch
-    - py{{ dep.lower() }}
-      {% else %}
+      {% if 'torch' not in dep %}  # can't install pytorch>=2.8 from conda-forge smh
     - {{ dep.lower().replace('_', '-') }}
       {% endif %}
     {% endfor %}

pyproject.toml

@@ -6,7 +6,7 @@ requires-python = ">=3.9"
 readme = "README.md"
 dynamic = ["version"]
 dependencies = [
-    "bioimageio.spec ==0.5.5.6",
+    "bioimageio.spec ==0.5.6.0",
     "h5py",
     "imagecodecs",
     "imageio>=2.10",
@@ -40,7 +40,7 @@ Documentation = "https://bioimage-io.github.io/core-bioimage-io-python/bioimagei
 Source = "https://github.com/bioimage-io/core-bioimage-io-python"
 
 [project.optional-dependencies]
-onnx = ["onnxruntime"]
+onnx = ["onnxruntime", "onnxscript"]
 pytorch = ["torch>=1.6,<3", "torchvision>=0.21", "keras>=3.0,<4"]
 tensorflow = ["tensorflow", "keras>=2.15,<4"]
 partners = [
@@ -59,10 +59,11 @@ dev = [
     "numpy",
     "onnx",
     "onnxruntime",
+    "onnxscript",
     "packaging>=17.0",
     "pdoc",
     "pre-commit",
-    "pyright==1.1.404",
+    "pyright==1.1.407",
     "pytest-cov",
     "pytest",
     "python-dotenv",
@@ -124,7 +125,6 @@ testpaths = ["src", "tests"]
 
 [tool.ruff]
 line-length = 88
-target-version = "py39"
 include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"]
 exclude = [
     "presentations",

src/bioimageio/core/__init__.py

@@ -3,7 +3,7 @@
 """
 # ruff: noqa: E402
 
-__version__ = "0.9.4"
+__version__ = "0.9.5"
 from loguru import logger
 
 logger.disable("bioimageio.core")