Merge pull request #530 from bioimage-io/faster_ci

FynnBe · web-flow · commit 942656dee58c · 2023-02-15T11:40:15.000+01:00
Improve CI speed with lighter/cached (mamba) environments
diff --git a/.github/workflows/auto_update_main.yaml b/.github/workflows/auto_update_main.yaml
@@ -54,9 +54,21 @@ jobs:
         ref: gh-pages
         path: gh-pages
     - name: install script deps
-      run: pip install requests typer bioimageio.spec lxml
+      uses: mamba-org/provision-with-micromamba@main
+      with:
+        cache-downloads: true
+        cache-env: true
+        environment-file: false
+        environment-name: scriptenv
+        channels: conda-forge
+        extra-specs: |  # script dependencies
+          bioimageio.spec
+          lxml
+          requests
+          typer
     - name: update external resources
       id: update_external
+      shell: bash -l {0}
       run: python scripts/update_external_resources.py --max-resource-count 15 ${{ github.event_name == 'pull_request' && '--no-ignore-status-5xx' || '--ignore-status-5xx' }}
     - name: "Upload collection update"
       uses: actions/upload-artifact@v3
@@ -71,6 +83,7 @@ jobs:
         mkdir dist
         mv tmp_download_counts.json dist/download_counts.json
     - name: update partner resources
+      shell: bash -l {0}
       run: python scripts/update_partner_resources.py
     - name: Upload preview of updated partner resources
       if: github.event_name == 'pull_request'
@@ -114,8 +127,21 @@ jobs:
         ref: gh-pages
         path: gh-pages
     - name: install script deps
-      run: pip install typer bioimageio.spec boltons lxml requests
+      uses: mamba-org/provision-with-micromamba@main
+      with:
+        cache-downloads: true
+        cache-env: true
+        environment-file: false
+        environment-name: scriptenv
+        channels: conda-forge
+        extra-specs: |  # script dependencies
+          bioimageio.spec
+          boltons
+          lxml
+          requests
+          typer
     - name: generate collection rdf
+      shell: bash -l {0}
       run: python scripts/generate_collection_rdf.py
     - name: Upload preview of collection.json
       if: github.event_name == 'pull_request'
@@ -169,8 +195,6 @@ jobs:
         mkdir -p "collection/${{ matrix.update.resource_id }}"
         cp -r "collection-update/${{ matrix.update.resource_id }}"/* "collection/${{ matrix.update.resource_id }}"
         rm -rf collection-update
-    - name: install script deps
-      run: pip install requests
     - name: get urls of previous PRs
       id: pr_urls
       run: python scripts/get_previous_pr_urls.py ${{ matrix.update.resource_id }}
diff --git a/.github/workflows/reset_pending_auto_update.yaml b/.github/workflows/reset_pending_auto_update.yaml
@@ -7,8 +7,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: install script deps
-      run: pip install typer
     - id: detect_auto_updates
       run: python scripts/detect_auto_updates.py auto-update-
     - name: Delete auto-update-* branches
diff --git a/.github/workflows/validate_resources.yaml b/.github/workflows/validate_resources.yaml
@@ -39,9 +39,22 @@ jobs:
         ref: gh-pages
         path: gh-pages
     - name: install script deps
-      run: pip install typer bioimageio.spec lxml bioimageio.core packaging
+      uses: mamba-org/provision-with-micromamba@main
+      with:
+        cache-downloads: true
+        cache-env: true
+        environment-file: false
+        environment-name: scriptenv
+        channels: conda-forge
+        extra-specs: |  # script dependencies
+          bioimageio.core
+          bioimageio.spec
+          lxml
+          packaging
+          typer
     - name: update RDFs
       id: update_rdfs
+      shell: bash -l {0}
       run: python scripts/update_rdfs.py --branch ${{ github.head_ref || github.ref }}
     - name: check pending versions limit
       if: inputs.check_validation == 'yes' && steps.pending.outputs.retrigger == 'yes'
@@ -50,6 +63,7 @@ jobs:
     - name: static validation
       if: steps.update_rdfs.outputs.has_pending_matrix_bioimageio == 'yes'
       id: static_validation
+      shell: bash -l {0}
       run: python scripts/static_validation.py '${{ steps.update_rdfs.outputs.pending_matrix_bioimageio }}'
     - name: Upload static validation summaries and conda envs
       if: steps.update_rdfs.outputs.has_pending_matrix_bioimageio == 'yes'
@@ -60,6 +74,7 @@ jobs:
         retention-days: 1
     - name: check if validation passed
       if: steps.update_rdfs.outputs.has_pending_matrix_bioimageio == 'yes' && inputs.check_validation == 'yes'
+      shell: bash -l {0}
       run: python scripts/check_validation_passed.py dist/static_validation_artifact
 
   dynamic-validation:
@@ -86,33 +101,29 @@ jobs:
         environment-file: artifacts/static_validation_artifact/${{ matrix.resource_id }}/${{ matrix.version_id }}/conda_env_${{ matrix.weight_format }}.yaml
         extra-specs: |  # script dependencies
           typer
-          bioimageio.spec
+          conda-forge::bioimageio.spec
       continue-on-error: true  # we inspect this step's outcome in dynamic_validation.py
       timeout-minutes: 60
-# atm bioimageio.core imports directly from tqdm, PR to import from spec is on the way...
-# todo: remove tqdm to avoid spamming logs
-#    - name: remove tqdm to avoid spamming logs
-#      shell: bash -l {0}
-#      run: conda remove --yes --force tqdm
-#      continue-on-error: true
+    - name: remove tqdm to avoid spamming logs
+      shell: bash -l {0}
+      run: conda remove --yes --force tqdm
+      continue-on-error: true
     - name: get artifact name wo forward slashes
       id: artifact_name
       run: echo ::set-output name=name::dynamic_validation_artifact_$(echo ${{ matrix.resource_id }}_${{ matrix.version_id }}_${{ matrix.weight_format }} | sed 's#/##g')
     - name: dynamic validation
       shell: bash -l {0}
-      timeout-minutes: 60
       run: python scripts/dynamic_validation.py dist/dynamic_validation_artifact ${{ matrix.resource_id }} ${{ matrix.version_id }} ${{ matrix.weight_format }} --create-env-outcome ${{ steps.create_env.outcome }} --${{ contains(inputs.deploy_to, 'gh-pages') && 'no-ignore' || 'ignore' }}-rdf-source-field-in-validation
+      timeout-minutes: 60
     - name: Upload validation summary
       uses: actions/upload-artifact@v3
       with:
         name: ${{ steps.artifact_name.outputs.name }}
         path: dist/dynamic_validation_artifact/${{ matrix.resource_id }}/${{ matrix.version_id }}/${{ matrix.weight_format }}
         retention-days: 1
-    - name: install script deps
-      if: steps.create_env.outcome == 'success' && inputs.check_validation == 'yes'
-      run: pip install typer bioimageio.spec
     - name: check if validation passed
       if: inputs.check_validation == 'yes'
+      shell: bash -l {0}
       run: python scripts/check_validation_passed.py dist/dynamic_validation_artifact/${{ matrix.resource_id }}/${{ matrix.version_id }}/${{ matrix.weight_format }}
 
   deploy:
@@ -131,10 +142,24 @@ jobs:
       with:
         path: artifacts
     - name: install script deps
-      run: pip install typer packaging numpy bioimageio.spec lxml
+      uses: mamba-org/provision-with-micromamba@main
+      with:
+        cache-downloads: true
+        cache-env: true
+        environment-file: false
+        environment-name: scriptenv
+        channels: conda-forge
+        extra-specs: |  # script dependencies
+          bioimageio.spec
+          lxml
+          numpy
+          packaging
+          typer
     - name: download partner test summaries
+      shell: bash -l {0}
       run: python scripts/download_partner_test_summaries.py
     - name: prepare to deploy
+      shell: bash -l {0}
       run: python scripts/prepare_to_deploy.py --branch ${{ github.head_ref || github.ref }}
     - name: Deploy to gh-pages 🚀
       if: contains(inputs.deploy_to, 'gh-pages')
@@ -145,6 +170,7 @@ jobs:
         folder: dist/gh_pages_update
     - name: add documentation files to preview
       if: contains(inputs.deploy_to, 'preview')  # only download documentation for preview to ease review
+      shell: bash -l {0}
       run: python scripts/download_documentation.py --folder dist/gh_pages_update
     - name: Upload preview
       if: contains(inputs.deploy_to, 'preview')
diff --git a/scripts/detect_auto_updates.py b/scripts/detect_auto_updates.py
@@ -1,8 +1,7 @@
+import argparse
 import subprocess
 from pprint import pprint
 
-import typer
-
 from bare_utils import set_gh_actions_output
 
 
@@ -19,4 +18,10 @@ def main(prefix: str):
 
 
 if __name__ == "__main__":
-    typer.run(main)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "prefix",
+        help="prefix to identify branches, e.g. 'auto-update-'",
+    )
+    args = parser.parse_args()
+    main(args.prefix)
diff --git a/scripts/static_validation.py b/scripts/static_validation.py
@@ -36,16 +36,13 @@ def get_env_from_deps(deps: Dependencies):
             dep_file_content = r.text
             if deps.manager == "conda":
                 conda_env = yaml.load(dep_file_content)
-                # add bioimageio.core if not present
-                channels = conda_env.get("channels", [])
-                if "conda-forge" not in channels:
-                    conda_env["channels"] = channels + ["conda-forge"]
 
+                # add bioimageio.core to dependencies
                 deps = conda_env.get("dependencies", [])
                 if not isinstance(deps, list):
                     raise TypeError(f"expected dependencies in conda environment.yaml to be a list, but got: {deps}")
                 if not any(isinstance(d, str) and d.startswith("bioimageio.core") for d in deps):
-                    conda_env["dependencies"] = deps + ["bioimageio.core"]
+                    conda_env["dependencies"] = deps + ["conda-forge::bioimageio.core"]
             elif deps.manager == "pip":
                 pip_req = [d for d in dep_file_content.split("\n") if not d.strip().startswith("#")]
                 conda_env["dependencies"].append("pip")
