Merge pull request #699 from bioimage-io/dev

Fix uncached downloads

Author: FynnBe

.github/workflows/build.yml

@@ -12,7 +12,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11","3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11","3.12", "3.13"]
+        pydantic-version: ["2.10.3", "2.11.0"]
         include:
           - python-version: "3.12"
             is-dev-version: true
@@ -26,7 +27,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install --upgrade pip
-          pip install -e .[dev]
+          pip install pydantic~=${{matrix.pydantic-version }} -e .[dev]
       - name: Get Date
         id: get-date
         run: |
@@ -41,20 +42,18 @@ jobs:
         run: python scripts/generate_version_submodule_imports.py check
       - run: black --check .
         if: matrix.is-dev-version
-      - name: install additinal deps for a full pyright check
-        run: pip install json_schema_for_humans
-        if: matrix.is-dev-version
       - run: ruff check
         if: matrix.is-dev-version
-      - run: pyright --version
+      - name: Run Pyright
         if: matrix.is-dev-version
-      - run: pyright -p pyproject.toml --pythonversion ${{ matrix.python-version }}
-        if: matrix.is-dev-version
-      - run: pytest
+        run: |
+          pyright --version
+          pyright -p pyproject.toml --pythonversion ${{ matrix.python-version }}
+      - run: pytest --cov bioimageio --cov-report xml --cov-append --capture no
         env:
           BIOIMAGEIO_CACHE_PATH: bioimageio_cache
           RUN_EXPENSIVE_TESTS: ${{ matrix.run-expensive-tests && 'true' || 'false' }}
-      - run: pytest --cov-append scripts  # also test docstrings in scripts for dev-version
+      - run: pytest --cov bioimageio --cov-report xml --cov-append --capture no scripts  # also test docstrings in scripts for dev-version
         if: ${{matrix.is-dev-version}}
         env:
           BIOIMAGEIO_CACHE_PATH: bioimageio_cache
@@ -110,7 +109,6 @@ jobs:
         run: python -m interactive_docs
       - name: Generate JSON schemas
         run: python scripts/generate_json_schemas.py
-      - run: pip install json_schema_for_humans
       - name: Generate JSON schema documentation
         run: python scripts/generate_json_schema_documentation.py
       - name: Generate developer docs

bioimageio/spec/_description.py

@@ -8,7 +8,7 @@
 
 from ._description_impl import DISCOVER, build_description_impl, get_rd_class_impl
 from ._internal.common_nodes import InvalidDescr
-from ._internal.io import BioimageioYamlContent
+from ._internal.io import BioimageioYamlContent, BioimageioYamlContentView
 from ._internal.types import FormatVersionPlaceholder
 from ._internal.validation_context import get_validation_context
 from .application import (
@@ -151,7 +151,7 @@ def _get_rd_class(typ: Any, format_version: Any):
 
 @overload
 def build_description(
-    content: BioimageioYamlContent,
+    content: BioimageioYamlContentView,
     /,
     *,
     context: Optional[ValidationContext] = None,
@@ -161,7 +161,7 @@ def build_description(
 
 @overload
 def build_description(
-    content: BioimageioYamlContent,
+    content: BioimageioYamlContentView,
     /,
     *,
     context: Optional[ValidationContext] = None,
@@ -170,7 +170,7 @@ def build_description(
 
 
 def build_description(
-    content: BioimageioYamlContent,
+    content: BioimageioYamlContentView,
     /,
     *,
     context: Optional[ValidationContext] = None,

bioimageio/spec/_description_impl.py

@@ -1,9 +1,10 @@
 """implementation details for building a bioimage.io resource description"""
 
+import collections.abc
 from typing import Any, Callable, List, Literal, Mapping, Optional, Type, TypeVar, Union
 
 from ._internal.common_nodes import InvalidDescr, ResourceDescrBase
-from ._internal.io import BioimageioYamlContent
+from ._internal.io import BioimageioYamlContentView
 from ._internal.types import FormatVersionPlaceholder
 from ._internal.validation_context import ValidationContext, get_validation_context
 from .summary import (
@@ -48,7 +49,7 @@ def get_rd_class_impl(
 
 
 def build_description_impl(
-    content: BioimageioYamlContent,
+    content: BioimageioYamlContentView,
     /,
     *,
     context: Optional[ValidationContext] = None,
@@ -57,7 +58,7 @@ def build_description_impl(
 ) -> Union[ResourceDescrT, InvalidDescr]:
     context = context or get_validation_context()
     errors: List[ErrorEntry] = []
-    if isinstance(content, dict):
+    if isinstance(content, collections.abc.Mapping):
         for minimum in ("type", "format_version"):
             if minimum not in content:
                 errors.append(

bioimageio/spec/_get_conda_env.py

@@ -3,12 +3,11 @@
 from typing_extensions import assert_never
 
 from ._internal.gh_utils import set_github_warning
-from ._internal.io_utils import ZipPath, read_yaml
-from .common import RelativeFilePath
+from ._internal.io import FileDescr, get_reader
+from ._internal.io_utils import read_yaml
 from .conda_env import BioimageioCondaEnv, PipDeps
 from .model import v0_4, v0_5
 from .model.v0_5 import Version
-from .utils import download
 
 SupportedWeightsEntry = Union[
     v0_4.KerasHdf5WeightsDescr,
@@ -202,35 +201,26 @@ def _get_default_tf_env(tensorflow_version: Optional[Version]) -> BioimageioCond
 
 
 def _get_env_from_deps(
-    deps: Union[v0_4.Dependencies, v0_5.EnvironmentFileDescr],
+    deps: Union[v0_4.Dependencies, FileDescr],
 ) -> BioimageioCondaEnv:
     if isinstance(deps, v0_4.Dependencies):
+        deps_reader = get_reader(deps.file)
         if deps.manager == "pip":
-            pip_deps_str = download(deps.file).path.read_text(encoding="utf-8")
-            assert isinstance(pip_deps_str, str)
+            pip_deps_str = deps_reader.read_text()
             pip_deps = [d.strip() for d in pip_deps_str.split("\n")]
             if "bioimageio.core" not in pip_deps:
                 pip_deps.append("bioimageio.core")
 
             return BioimageioCondaEnv(
                 dependencies=[PipDeps(pip=pip_deps)],
             )
-        elif deps.manager not in ("conda", "mamba"):
-            raise ValueError(f"Dependency manager {deps.manager} not supported")
+        elif deps.manager in ("conda", "mamba"):
+            return BioimageioCondaEnv.model_validate(read_yaml(deps_reader))
         else:
-            deps_source = (
-                deps.file.absolute()
-                if isinstance(deps.file, RelativeFilePath)
-                else deps.file
-            )
-            if isinstance(deps_source, ZipPath):
-                local = deps_source
-            else:
-                local = download(deps_source).path
-
-            return BioimageioCondaEnv.model_validate(read_yaml(local))
-    elif isinstance(deps, v0_5.EnvironmentFileDescr):
-        local = download(deps.source).path
-        return BioimageioCondaEnv.model_validate(read_yaml(local))
+            raise ValueError(f"Dependency manager {deps.manager} not supported")
+
+    elif isinstance(deps, FileDescr):
+        deps_reader = deps.get_reader()
+        return BioimageioCondaEnv.model_validate(read_yaml(deps_reader))
     else:
         assert_never(deps)

bioimageio/spec/_internal/_settings.py

@@ -1,11 +1,17 @@
+import os
+from functools import cached_property
 from pathlib import Path
 from typing import Optional, Union
 
 import pooch  # pyright: ignore [reportMissingTypeStubs]
-from pydantic import Field
+from genericache import DiskCache
+from genericache.digest import UrlDigest
+from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing_extensions import Annotated
 
+from .root_url import RootHttpUrl
+
 
 class Settings(BaseSettings, extra="ignore"):
     """environment variables for bioimageio.spec"""
@@ -15,11 +21,16 @@ class Settings(BaseSettings, extra="ignore"):
     )
 
     allow_pickle: bool = False
-    """Sets the `allow_pickle` argument for `numpy.load()`/`numpy.save()`"""
+    """Sets the `allow_pickle` argument for `numpy.load()`"""
 
     cache_path: Path = pooch.os_cache("bioimageio")
     """bioimageio cache location"""
 
+    @field_validator("cache_path", mode="after")
+    @classmethod
+    def _expand_user(cls, value: Path):
+        return Path(os.path.expanduser(str(value)))
+
     collection_http_pattern: str = (
         "https://hypha.aicell.io/bioimage-io/artifacts/{bioimageio_id}/files/rdf.yaml"
     )
@@ -41,20 +52,21 @@ class Settings(BaseSettings, extra="ignore"):
     )
     """URL to bioimageio id_map_draft.json to resolve draft IDs ending with '/draft'."""
 
+    perform_io_checks: bool = True
+    """Wether or not to perform validation that requires file io,
+    e.g. downloading a remote files.
+
+    Existence of any local absolute file paths is still being checked."""
+
     resolve_draft: bool = True
     """Flag to resolve draft resource versions following the pattern
     <resource id>/draft.
+
     Note that anyone may stage a new draft and that such a draft version
     may not have been reviewed yet.
     Set this flag to False to avoid this potential security risk
     and disallow loading draft versions."""
 
-    perform_io_checks: bool = True
-    """Wether or not to perform validation that requires file io,
-    e.g. downloading a remote files.
-
-    Existence of any local absolute file paths is still being checked."""
-
     log_warnings: bool = True
     """Log validation warnings to console."""
 
@@ -77,6 +89,15 @@ def github_auth(self):
         else:
             return (self.github_username, self.github_token)
 
+    @cached_property
+    def disk_cache(self):
+        cache = DiskCache[RootHttpUrl].create(
+            url_type=RootHttpUrl,
+            cache_dir=self.cache_path,
+            url_hasher=UrlDigest.from_str,
+        )
+        return cache
+
 
 settings = Settings()
 """parsed environment variables for bioimageio.spec"""