known_files with None value

Author: FynnBe

bioimageio/spec/_internal/io.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import collections.abc
 import hashlib
 import io
 import sys
@@ -833,20 +832,15 @@ def _validate_sha256(self) -> Self:
     def validate_sha256(self):
         context = get_validation_context()
         if (src_str := str(self.source)) in context.known_files:
-            if isinstance(context.known_files, collections.abc.Mapping):
-                actual_sha = context.known_files[src_str]
-            else:
-                return
+            actual_sha = context.known_files[src_str]
         else:
             local_source = download(self.source, sha256=self.sha256).path
-            if isinstance(context.known_files, collections.abc.Mapping):
-                actual_sha = get_sha256(local_source)
-                context.known_files[src_str] = actual_sha
-            else:
-                context.known_files.add(src_str)
-                return
+            actual_sha = get_sha256(local_source)
+            context.known_files[src_str] = actual_sha
 
-        if self.sha256 == actual_sha:
+        if actual_sha is None:
+            return
+        elif self.sha256 == actual_sha:
             pass
         elif self.sha256 is None or context.update_hashes:
             self.sha256 = actual_sha

bioimageio/spec/_internal/type_guards.py

@@ -1,5 +1,5 @@
 import collections.abc
-from typing import Any, Dict, List, Mapping, Sequence, Tuple
+from typing import Any, Dict, List, Mapping, Sequence, Set, Tuple
 
 import numpy as np
 from numpy.typing import NDArray
@@ -11,6 +11,11 @@ def is_dict(v: Any) -> TypeGuard[Dict[Any, Any]]:
     return isinstance(v, dict)
 
 
+def is_set(v: Any) -> TypeGuard[Set[Any]]:
+    """to avoid Set[Unknown]"""
+    return isinstance(v, set)
+
+
 def is_kwargs(v: Any) -> TypeGuard[Dict[str, Any]]:
     return isinstance(v, dict) and all(
         isinstance(k, str) for k in v  # pyright: ignore[reportUnknownVariableType]

bioimageio/spec/_internal/url.py

@@ -23,6 +23,14 @@ def _validate_url_impl(
 ) -> pydantic.HttpUrl:
 
     url = str(url)
+    context = get_validation_context()
+    if url in context.known_files:
+        with context.replace(perform_io_checks=False):
+            return (  # pyright: ignore[reportUnknownVariableType]
+                # TODO: remove pyright ignore for pydantic > 2.9
+                pydantic.HttpUrl(url)  # pyright: ignore[reportCallIssue]
+            )
+
     val_url = url
 
     if url.startswith("http://example.com") or url.startswith("https://example.com"):
@@ -113,6 +121,7 @@ def _validate_url_impl(
         else:
             assert_never(request_mode)
 
+    context.known_files[url] = None
     return (  # pyright: ignore[reportUnknownVariableType]
         # TODO: remove pyright ignore for pydantic > 2.9
         pydantic.HttpUrl(url)  # pyright: ignore[reportCallIssue]
@@ -128,10 +137,7 @@ class HttpUrl(RootHttpUrl):
     def _after_validator(self):
         self = super()._after_validator()
         context = get_validation_context()
-        if (
-            context.perform_io_checks
-            and str(self._validated) not in context.known_files
-        ):
+        if context.perform_io_checks:
             self._validated = _validate_url(self._validated)
             self._exists = True
 

bioimageio/spec/_internal/validation_context.py

@@ -4,7 +4,7 @@
 from copy import copy
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, List, Literal, Optional, Set, Union
+from typing import Dict, List, Literal, Optional, Union
 from urllib.parse import urlsplit, urlunsplit
 from zipfile import ZipFile
 
@@ -28,7 +28,7 @@ class ValidationContextBase:
 
     Existence of local absolute file paths is still being checked."""
 
-    known_files: Union[Set[str], Dict[str, Sha256]] = field(default_factory=dict)
+    known_files: Dict[str, Optional[Sha256]] = field(default_factory=dict)
     """Allows to bypass download and hashing of referenced files."""
 
     update_hashes: bool = False
@@ -112,7 +112,7 @@ def replace(  # TODO: probably use __replace__ when py>=3.13
         log_warnings: Optional[bool] = None,
         file_name: Optional[str] = None,
         perform_io_checks: Optional[bool] = None,
-        known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+        known_files: Optional[Dict[str, Optional[Sha256]]] = None,
         raise_errors: Optional[bool] = None,
         update_hashes: Optional[bool] = None,
     ) -> Self:

bioimageio/spec/_io.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Dict, Literal, Optional, Set, TextIO, Union, cast, overload
+from typing import Dict, Literal, Optional, TextIO, Union, cast, overload
 from zipfile import ZipFile
 
 from loguru import logger
@@ -35,7 +35,7 @@ def load_description(
     *,
     format_version: Literal["latest"],
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> Union[LatestResourceDescr, InvalidDescr]: ...
 
@@ -47,7 +47,7 @@ def load_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> Union[ResourceDescr, InvalidDescr]: ...
 
@@ -58,7 +58,7 @@ def load_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> Union[ResourceDescr, InvalidDescr]:
     """load a bioimage.io resource description
@@ -73,6 +73,8 @@ def load_description(
                            absolute file paths is still being checked.
         known_files: Allows to bypass download and hashing of referenced files
                      (even if perform_io_checks is True).
+                     Checked files will be added to this dictionary
+                     with their SHA-256 value.
         sha256: Optional SHA-256 value of **source**
 
     Returns:
@@ -107,7 +109,7 @@ def load_model_description(
     *,
     format_version: Literal["latest"],
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> ModelDescr: ...
 
@@ -119,7 +121,7 @@ def load_model_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyModelDescr: ...
 
@@ -130,7 +132,7 @@ def load_model_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyModelDescr:
     """same as `load_description`, but addtionally ensures that the loaded
@@ -156,7 +158,7 @@ def load_dataset_description(
     *,
     format_version: Literal["latest"],
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> DatasetDescr: ...
 
@@ -168,7 +170,7 @@ def load_dataset_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyDatasetDescr: ...
 
@@ -179,7 +181,7 @@ def load_dataset_description(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> AnyDatasetDescr:
     """same as `load_description`, but addtionally ensures that the loaded
@@ -231,7 +233,7 @@ def load_description_and_validate_format_only(
     *,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
     perform_io_checks: Optional[bool] = None,
-    known_files: Optional[Union[Set[str], Dict[str, Sha256]]] = None,
+    known_files: Optional[Dict[str, Optional[Sha256]]] = None,
     sha256: Optional[Sha256] = None,
 ) -> ValidationSummary:
     """same as `load_description`, but only return the validation summary.

changelog.md

@@ -16,7 +16,7 @@ This changelog includes implementation details and my reference the [changes to
 - new validation context `disable_cache` (equivalent to an empty `cache_path` in `settings`)
   circumvents caching to disk and keeps downloads in memory only
 - new setting `allow_pickle` to control `numpy.load`/`numpy.save` behavior
-- allow the `ValidationContext`'s `known_files` to be a set of file names to only check for file existence without comparing file hashes.
+- allow the `ValidationContext`'s `known_files` to include `None` values (isntead of known SHA values) to only check for file existence without comparing file hashes.
 
 #### bioimageio.spec 0.5.4.1