Merge pull request #698 from bioimage-io/dev

bump version and update changelog

Author: FynnBe

.github/workflows/build.yml

@@ -220,7 +220,7 @@ jobs:
 
       - name: Publish package on PyPI
         if: steps.tag-version.outputs.new_tag
-        uses: pypa/gh-action-pypi-publish@v1.8.14
+        uses: pypa/gh-action-pypi-publish@release/v1.12
         with:
           user: __token__
           password: "${{ secrets.PYPI_TOKEN }}"

bioimageio/spec/VERSION

@@ -1,3 +1,3 @@
 {
-    "version": "0.5.4.1"
+    "version": "0.5.4.2"
 }

bioimageio/spec/_internal/io.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import collections.abc
 import hashlib
 import io
 import sys
@@ -48,6 +47,7 @@
     model_validator,
 )
 from pydantic_core import core_schema
+from tqdm import tqdm
 from typing_extensions import (
     Annotated,
     LiteralString,
@@ -726,10 +726,16 @@ def resolve(
         if settings.user_agent is not None:
             headers["User-Agent"] = settings.user_agent
 
-        if settings.cache_path and not get_validation_context().disable_cache:
+        chunk_size = 1024
+        if (
+            settings.cache_path
+            and not get_validation_context().disable_cache
+            and any(v is not None for v in kwargs.values())
+        ):
             downloader = pooch.HTTPDownloader(
                 headers=headers,
                 progressbar=progressbar,  # pyright: ignore[reportArgumentType]
+                chunk_size=chunk_size,
             )
             fname = _get_unique_file_name(source)
             _ls: Any = pooch.retrieve(
@@ -747,23 +753,47 @@ def resolve(
             )
         else:
             # cacheless download to memory using an in memory zip file
-            r = requests.get(str(source))
+            r = requests.get(str(source), stream=True)
             r.raise_for_status()
-            if not isinstance(r.content, bytes):
-                raise TypeError(
-                    f"Request content of {source} has unexpected type {type(r.content)}."
-                )
 
             zf = zipfile.ZipFile(io.BytesIO(), "w")
             fn = extract_file_name(source)
-            # alternative implementation; TODO: remove commented code
-            # zf.writestr(fn, r.content)
-            # zp = ZipPath(zf) / fn
-            # assert zp.exists(), f"failed to write {fn} to in-memory zip file"
+            total = int(r.headers.get("content-length", 0))
+
+            if isinstance(progressbar, bool):
+                if progressbar:
+                    use_ascii = bool(sys.platform == "win32")
+                    pbar = tqdm(
+                        total=total,
+                        ncols=79,
+                        ascii=use_ascii,
+                        unit="B",
+                        unit_scale=True,
+                        leave=True,
+                    )
+                    pbar = tqdm(desc=f"Downloading {fn}")
+                else:
+                    pbar = None
+            else:
+                pbar = progressbar
+
             zp = ZipPath(zf, fn)
             with zp.open("wb") as z:
                 assert not isinstance(z, io.TextIOWrapper)
-                _ = z.write(r.content)
+                for chunk in r.iter_content(chunk_size=chunk_size):
+                    n = z.write(chunk)
+                    if pbar is not None:
+                        _ = pbar.update(n)
+
+            # Make sure the progress bar gets filled even if the actual number
+            # is chunks is smaller than expected. This happens when streaming
+            # text files that are compressed by the server when sending (gzip).
+            # Binary files don't experience this.
+            # (adapted from pooch.HttpDownloader)
+            if pbar is not None:
+                pbar.reset()
+                _ = pbar.update(total)
+                pbar.close()
 
             return FileInZip(
                 path=zp,
@@ -833,20 +863,15 @@ def _validate_sha256(self) -> Self:
     def validate_sha256(self):
         context = get_validation_context()
         if (src_str := str(self.source)) in context.known_files:
-            if isinstance(context.known_files, collections.abc.Mapping):
-                actual_sha = context.known_files[src_str]
-            else:
-                return
+            actual_sha = context.known_files[src_str]
         else:
             local_source = download(self.source, sha256=self.sha256).path
-            if isinstance(context.known_files, collections.abc.Mapping):
-                actual_sha = get_sha256(local_source)
-                context.known_files[src_str] = actual_sha
-            else:
-                context.known_files.add(src_str)
-                return
+            actual_sha = get_sha256(local_source)
+            context.known_files[src_str] = actual_sha
 
-        if self.sha256 == actual_sha:
+        if actual_sha is None:
+            return
+        elif self.sha256 == actual_sha:
             pass
         elif self.sha256 is None or context.update_hashes:
             self.sha256 = actual_sha

bioimageio/spec/_internal/type_guards.py

@@ -1,5 +1,5 @@
 import collections.abc
-from typing import Any, Dict, List, Mapping, Sequence, Tuple
+from typing import Any, Dict, List, Mapping, Sequence, Set, Tuple
 
 import numpy as np
 from numpy.typing import NDArray
@@ -11,6 +11,11 @@ def is_dict(v: Any) -> TypeGuard[Dict[Any, Any]]:
     return isinstance(v, dict)
 
 
+def is_set(v: Any) -> TypeGuard[Set[Any]]:
+    """to avoid Set[Unknown]"""
+    return isinstance(v, set)
+
+
 def is_kwargs(v: Any) -> TypeGuard[Dict[str, Any]]:
     return isinstance(v, dict) and all(
         isinstance(k, str) for k in v  # pyright: ignore[reportUnknownVariableType]

bioimageio/spec/_internal/url.py

@@ -23,6 +23,14 @@ def _validate_url_impl(
 ) -> pydantic.HttpUrl:
 
     url = str(url)
+    context = get_validation_context()
+    if url in context.known_files:
+        with context.replace(perform_io_checks=False):
+            return (  # pyright: ignore[reportUnknownVariableType]
+                # TODO: remove pyright ignore for pydantic > 2.9
+                pydantic.HttpUrl(url)  # pyright: ignore[reportCallIssue]
+            )
+
     val_url = url
 
     if url.startswith("http://example.com") or url.startswith("https://example.com"):
@@ -113,6 +121,7 @@ def _validate_url_impl(
         else:
             assert_never(request_mode)
 
+    context.known_files[url] = None
     return (  # pyright: ignore[reportUnknownVariableType]
         # TODO: remove pyright ignore for pydantic > 2.9
         pydantic.HttpUrl(url)  # pyright: ignore[reportCallIssue]
@@ -128,10 +137,7 @@ class HttpUrl(RootHttpUrl):
     def _after_validator(self):
         self = super()._after_validator()
         context = get_validation_context()
-        if (
-            context.perform_io_checks
-            and str(self._validated) not in context.known_files
-        ):
+        if context.perform_io_checks:
             self._validated = _validate_url(self._validated)
             self._exists = True
 

bioimageio/spec/_internal/validation_context.py

@@ -4,7 +4,7 @@
 from copy import copy
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, List, Literal, Optional, Set, Union
+from typing import Dict, List, Literal, Optional, Union
 from urllib.parse import urlsplit, urlunsplit
 from zipfile import ZipFile
 
@@ -28,7 +28,7 @@ class ValidationContextBase:
 
     Existence of local absolute file paths is still being checked."""
 
-    known_files: Union[Set[str], Dict[str, Sha256]] = field(default_factory=dict)
+    known_files: Dict[str, Optional[Sha256]] = field(default_factory=dict)
     """Allows to bypass download and hashing of referenced files."""
 
     update_hashes: bool = False
@@ -112,7 +112,7 @@ def replace(  # TODO: probably use __replace__ when py>=3.13
         log_warnings: Optional[bool] = None,
         file_name: Optional[str] = None,
         perform_io_checks: Optional[bool] = None,
-        known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+        known_files: Optional[Dict[str, Optional[Sha256]]] = None,
         raise_errors: Optional[bool] = None,
         update_hashes: Optional[bool] = None,
     ) -> Self:

bioimageio/spec/_io.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Dict, Literal, Optional, Set, TextIO, Union, cast, overload
+from typing import Dict, Literal, Optional, TextIO, Union, cast, overload
 from zipfile import ZipFile
 
 from loguru import logger
@@ -35,7 +35,7 @@ def load_description(
     *,
     format_version: Literal["latest"],
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> Union[LatestResourceDescr, InvalidDescr]: ...
 
@@ -47,7 +47,7 @@ def load_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> Union[ResourceDescr, InvalidDescr]: ...
 
@@ -58,7 +58,7 @@ def load_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> Union[ResourceDescr, InvalidDescr]:
     """load a bioimage.io resource description
@@ -73,6 +73,8 @@ def load_description(
                            absolute file paths is still being checked.
         known_files: Allows to bypass download and hashing of referenced files
                      (even if perform_io_checks is True).
+                     Checked files will be added to this dictionary
+                     with their SHA-256 value.
         sha256: Optional SHA-256 value of **source**
 
     Returns:
@@ -107,7 +109,7 @@ def load_model_description(
     *,
     format_version: Literal["latest"],
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> ModelDescr: ...
 
@@ -119,7 +121,7 @@ def load_model_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyModelDescr: ...
 
@@ -130,7 +132,7 @@ def load_model_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyModelDescr:
     """same as `load_description`, but addtionally ensures that the loaded
@@ -156,7 +158,7 @@ def load_dataset_description(
     *,
     format_version: Literal["latest"],
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> DatasetDescr: ...
 
@@ -168,7 +170,7 @@ def load_dataset_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyDatasetDescr: ...
 
@@ -179,7 +181,7 @@ def load_dataset_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyDatasetDescr:
     """same as `load_description`, but addtionally ensures that the loaded
@@ -231,7 +233,7 @@ def load_description_and_validate_format_only(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> ValidationSummary:
     """same as `load_description`, but only return the validation summary.

bioimageio/spec/summary.py

@@ -658,11 +658,6 @@ def append_detail(
             if code is not None:
                 details.append(["", "", code.text])
 
-            text, *additional_lines = format_text(text)
-
-            details.append([status, format_loc(loc, target), text])
-            details.extend([["", "", line] for line in additional_lines])
-
         for d in summary.details:
             details.append([d.status_icon, format_loc(d.loc, target), d.name])
 

changelog.md

@@ -10,6 +10,14 @@ In this file we log both:
 
 This changelog includes implementation details and my reference the [changes to the Resource Description Format](#changes-to-the-resource-description-format), e.g. in entry [bioimageio.spec 0.5.2](#bioimageiospec-052).
 
+#### bioimageio.spec 0.5.4.2
+
+- improved validation summary formatting
+- new validation context `disable_cache` (equivalent to an empty `cache_path` in `settings`)
+  circumvents caching to disk and keeps downloads in memory only
+- new setting `allow_pickle` to control `numpy.load`/`numpy.save` behavior
+- allow the `ValidationContext`'s `known_files` to include `None` values (isntead of known SHA values) to only check for file existence without comparing file hashes.
+
 #### bioimageio.spec 0.5.4.1
 
 (patch to 0.5.4.0)