Merge branch 'main' into auto-update-10.5281/zenodo.7614645

Author: FynnBe

.github/workflows/auto_update_main.yaml

@@ -45,11 +45,11 @@ jobs:
       found_new_resources: ${{ steps.update_external.outputs.found_new_resources || 'no' }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0  # get all branches to check for pending resources by checking if auto-update-... branch exists
     - name: checkout gh-pages for update_partner_resources.py
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         ref: gh-pages
         path: gh-pages
@@ -59,7 +59,7 @@ jobs:
       id: update_external
       run: python scripts/update_external_resources.py --max-resource-count 15 ${{ github.event_name == 'pull_request' && '--no-ignore-status-5xx' || '--ignore-status-5xx' }}
     - name: "Upload collection update"
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: collection-update
         path: dist
@@ -74,7 +74,7 @@ jobs:
       run: python scripts/update_partner_resources.py
     - name: Upload preview of updated partner resources
       if: github.event_name == 'pull_request'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: preview-partner-resources
         path: dist
@@ -108,8 +108,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+    - uses: actions/checkout@v3
       with:
         ref: gh-pages
         path: gh-pages
@@ -119,7 +119,7 @@ jobs:
       run: python scripts/generate_collection_rdf.py
     - name: Upload preview of collection.json
       if: github.event_name == 'pull_request'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: preview-collection-json
         path: dist/collection.json
@@ -151,16 +151,16 @@ jobs:
       matrix: ${{ fromJson(needs.update-resources.outputs.updated_resources_matrix) }}
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+    - uses: actions/checkout@v3
       with:
         ref: gh-pages
         path: gh-pages
     - run: mkdir dist
     - run: cp gh-pages/pr_urls.json dist
       continue-on-error: true
     - name: download new collection
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v3
       with:
         name: collection-update
         path: collection-update
@@ -177,7 +177,7 @@ jobs:
     - name: Create Pull Request
       if: github.event_name != 'pull_request'
       id: cpr
-      uses: peter-evans/create-pull-request@v3
+      uses: peter-evans/create-pull-request@v4
       with:
         token: ${{ secrets.PAT }}  # using a PAT will make sure we can trigger the CI in PR
         commit-message: 'Update ${{ matrix.update.resource_id }}'

.github/workflows/post_pr.yaml

@@ -10,7 +10,7 @@ jobs:
     if: github.event.pull_request.merged == false  # if pr was merged the branch is deleted via gh settings
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: check if remote still exists
       id: ls-remote
       run: |

.github/workflows/reset_pending_auto_update.yaml

@@ -6,7 +6,7 @@ jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: install script deps
       run: pip install typer
     - id: detect_auto_updates

.github/workflows/trigger_partner_wf.yaml

@@ -25,7 +25,7 @@ jobs:
 #        - ...
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         ref: gh-pages
     - name: trigger ${{ matrix.partner_id }}

.github/workflows/validate_resources.yaml

@@ -29,12 +29,12 @@ jobs:
       has_dynamic_test_cases: ${{ steps.static_validation.outputs.has_dynamic_test_cases }}
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+    - uses: actions/checkout@v3
       with:
         ref: last_ci_run
         path: last_ci_run
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         ref: gh-pages
         path: gh-pages
@@ -53,7 +53,7 @@ jobs:
       run: python scripts/static_validation.py '${{ steps.update_rdfs.outputs.pending_matrix_bioimageio }}'
     - name: Upload static validation summaries and conda envs
       if: steps.update_rdfs.outputs.has_pending_matrix_bioimageio == 'yes'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: static_validation_artifact
         path: dist/static_validation_artifact
@@ -71,35 +71,39 @@ jobs:
       matrix: ${{ fromJson(needs.static-validation.outputs.dynamic_test_cases) }}  # [{env_name: ..., resource_id: ..., version_id: ..., weight_format: ...}, ...]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: download environment.yaml
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v3
       with:
         name: static_validation_artifact
         path: artifacts/static_validation_artifact
     - name: install validation dependencies
       id: create_env
-      continue-on-error: true  # we inspect this step's outcome in dynamic_validation.py
-      timeout-minutes: 30
-      uses: conda-incubator/setup-miniconda@v2
+      uses: mamba-org/provision-with-micromamba@main
       with:
-        auto-update-conda: true
-        mamba-version: "*"
-        channel-priority: strict
-        activate-environment: ${{ matrix.env_name }}
+        cache-downloads: true
+        environment-name: ${{ matrix.env_name }}
         environment-file: artifacts/static_validation_artifact/${{ matrix.resource_id }}/${{ matrix.version_id }}/conda_env_${{ matrix.weight_format }}.yaml
-    - name: install minimal script deps
-      if: steps.create_env.outcome != 'success'
-      run: pip install typer bioimageio.spec
+        extra-specs: |  # script dependencies
+          typer
+          bioimageio.spec
+      continue-on-error: true  # we inspect this step's outcome in dynamic_validation.py
+      timeout-minutes: 60
+# atm bioimageio.core imports directly from tqdm, PR to import from spec is on the way...
+# todo: remove tqdm to avoid spamming logs
+#    - name: remove tqdm to avoid spamming logs
+#      shell: bash -l {0}
+#      run: conda remove --yes --force tqdm
+#      continue-on-error: true
     - name: get artifact name wo forward slashes
       id: artifact_name
       run: echo ::set-output name=name::dynamic_validation_artifact_$(echo ${{ matrix.resource_id }}_${{ matrix.version_id }}_${{ matrix.weight_format }} | sed 's#/##g')
     - name: dynamic validation
       shell: bash -l {0}
-      timeout-minutes: 30
+      timeout-minutes: 60
       run: python scripts/dynamic_validation.py dist/dynamic_validation_artifact ${{ matrix.resource_id }} ${{ matrix.version_id }} ${{ matrix.weight_format }} --create-env-outcome ${{ steps.create_env.outcome }} --${{ contains(inputs.deploy_to, 'gh-pages') && 'no-ignore' || 'ignore' }}-rdf-source-field-in-validation
     - name: Upload validation summary
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: ${{ steps.artifact_name.outputs.name }}
         path: dist/dynamic_validation_artifact/${{ matrix.resource_id }}/${{ matrix.version_id }}/${{ matrix.weight_format }}
@@ -117,13 +121,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+    - uses: actions/checkout@v3
       with:
         ref: gh-pages
         path: gh-pages
     - name: download artifacts
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v3
       with:
         path: artifacts
     - name: install script deps
@@ -144,7 +148,7 @@ jobs:
       run: python scripts/download_documentation.py --folder dist/gh_pages_update
     - name: Upload preview
       if: contains(inputs.deploy_to, 'preview')
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: preview
         path: dist/gh_pages_update

collection_rdf_template.yaml

@@ -43,9 +43,9 @@ config:
       branch: master
       collection_file_name: manifest.bioimage.io.yaml
       test_summaries:
-        repository: deepimagej/models
+        repository: deepimagej/CI-deepimagej-bioimage-io
         deploy_branch: gh-pages
-        deploy_folder: test_summaries
+        deploy_folder: .
         workflow: test_bioimageio_resources.yaml
         workflow_ref: refs/heads/master
     - id: fiji

scripts/prepare_to_deploy.py

@@ -100,8 +100,9 @@ def main(
             for sp in dyn_sums:
                 for sub_summary in get_sub_summaries(sp):
                     test_summary["tests"]["bioimageio"].append(sub_summary)
-                    core_versions.add(Version(sub_summary["bioimageio_core_version"]))
                     success &= sub_summary.get("status") == "passed"
+                    if "bioimageio_core_version" in sub_summary:
+                        core_versions.add(Version(sub_summary["bioimageio_core_version"]))
 
             if spec_versions:
                 test_summary["bioimageio_spec_version"] = str(max(spec_versions))

scripts/static_validation.py

@@ -12,7 +12,7 @@
 from bare_utils import set_gh_actions_outputs
 from bioimageio.spec import load_raw_resource_description, validate
 from bioimageio.spec.model.raw_nodes import Model, WeightsFormat
-from bioimageio.spec.rdf.raw_nodes import RDF
+from bioimageio.spec.rdf.raw_nodes import RDF_Base
 from bioimageio.spec.shared import yaml
 from bioimageio.spec.shared.raw_nodes import Dependencies, URI
 from utils import ADJECTIVES, ANIMALS, iterate_over_gh_matrix, split_animal_nickname
@@ -134,9 +134,7 @@ def ensure_valid_conda_env_name(name: str) -> str:
     return name or "empty"
 
 
-def prepare_dynamic_test_cases(
-    rd: Union[Model, RDF], resource_id: str, version_id: str, dist: Path
-) -> List[Dict[str, str]]:
+def prepare_dynamic_test_cases(rd: RDF_Base, resource_id: str, version_id: str, dist: Path) -> List[Dict[str, str]]:
     validation_cases = []
     # construct test cases based on resource type
     if isinstance(rd, Model):
@@ -158,7 +156,7 @@ def prepare_dynamic_test_cases(
             validation_cases.append(
                 {"env_name": env_name, "resource_id": resource_id, "version_id": version_id, "weight_format": wf}
             )
-    elif isinstance(rd, RDF):
+    elif isinstance(rd, RDF_Base):
         pass
     else:
         raise TypeError(rd)
@@ -211,7 +209,7 @@ def main(
             latest_static_summary = validate(rdf_path, update_format=True)
             if not latest_static_summary["error"]:
                 rd = load_raw_resource_description(rdf_path, update_to_format="latest")
-                assert isinstance(rd, RDF)
+                assert isinstance(rd, RDF_Base)
                 dynamic_test_cases += prepare_dynamic_test_cases(rd, resource_id, version_id, dist)
 
             if "name" not in latest_static_summary: