Merge pull request #709 from bioimage-io/dev

Housekeeping

Author: FynnBe

bioimageio/spec/VERSION

@@ -1,3 +1,3 @@
 {
-    "version": "0.5.4.3"
+    "version": "0.5.4.4"
 }

bioimageio/spec/_internal/common_nodes.py

@@ -10,6 +10,7 @@
     Any,
     ClassVar,
     Dict,
+    Iterable,
     List,
     Literal,
     Mapping,
@@ -149,13 +150,15 @@ def _ignore_future_patch(cls, data: Any, /) -> Any:
     @model_validator(mode="after")
     def _set_init_validation_summary(self) -> Self:
         context = get_validation_context()
+
         self._validation_summary = ValidationSummary(
             name="bioimageio format validation",
             source_name=context.source_name,
             id=getattr(self, "id", None),
             type=self.type,
             format_version=self.format_version,
             status="failed" if isinstance(self, InvalidDescr) else "valid-format",
+            metadata_completeness=self._get_metadata_completeness(),
             details=(
                 []
                 if isinstance(self, InvalidDescr)
@@ -245,6 +248,25 @@ def load(
 
         return rd
 
+    def _get_metadata_completeness(self) -> float:
+        if isinstance(self, InvalidDescr):
+            return 0.0
+
+        given = self.model_dump(mode="json", exclude_unset=True, exclude_defaults=False)
+        full = self.model_dump(mode="json", exclude_unset=False, exclude_defaults=False)
+
+        def extract_flat_keys(d: Dict[Any, Any], key: str = "") -> Iterable[str]:
+            for k, v in d.items():
+                if is_dict(v):
+                    yield from extract_flat_keys(v, key=f"{key}.{k}" if key else k)
+
+                yield f"{key}.{k}" if key else k
+
+        given_keys = set(extract_flat_keys(given))
+        full_keys = set(extract_flat_keys(full))
+        assert len(full_keys) >= len(given_keys)
+        return len(given_keys) / len(full_keys) if full_keys else 0.0
+
     @classmethod
     def _load_impl(
         cls, data: BioimageioYamlContent

bioimageio/spec/_internal/constants.py

@@ -56,29 +56,31 @@ class _DtypeLimit(NamedTuple):
     }
 )
 
-# TODO: cache/store known gh users in file
-KNOWN_GH_USERS: Set[str] = {
-    "clementcaporal",
-    "donglaiw",
-    "jansanrom",
-    "pedgomgal1",
+# TODO: cache/store known GitHub users in file
+KNOWN_GITHUB_USERS: Set[str] = {
     "aaitorg",
+    "anwai98",
     "bioimageiobot",
     "carlosuc3m",
     "cfusterbarcelo",
+    "clementcaporal",
     "constantinpape",
     "ctr26",
     "danifranco",
+    "donglaiw",
     "esgomezm",
     "fynnbe",
     "githubuser2",
     "iarganda",
+    "ilastik",
     "ivanhcenalmor",
     "jansanrom",
     "k-dominik",
     "lenkaback",
     "oeway",
+    "pedgomgal1",
 }
-N_KNOWN_GH_USERS = len(KNOWN_GH_USERS)
-KNOWN_INVALID_GH_USERS: Set[str] = {"arratemunoz", "lmescu"}
-N_KNOWN_INVALID_GH_USERS = len(KNOWN_INVALID_GH_USERS)
+
+N_KNOWN_GITHUB_USERS = len(KNOWN_GITHUB_USERS)
+KNOWN_INVALID_GITHUB_USERS: Set[str] = {"arratemunoz", "lmescu"}
+N_KNOWN_INVALID_GITHUB_USERS = len(KNOWN_INVALID_GITHUB_USERS)

bioimageio/spec/_internal/field_validation.py

@@ -12,7 +12,7 @@
 import httpx
 
 from ._settings import settings
-from .constants import KNOWN_GH_USERS, KNOWN_INVALID_GH_USERS
+from .constants import KNOWN_GITHUB_USERS, KNOWN_INVALID_GITHUB_USERS
 from .field_warning import issue_warning
 from .type_guards import is_mapping, is_sequence, is_tuple
 from .validation_context import get_validation_context
@@ -57,18 +57,20 @@ def validate_unique_entries(seq: Sequence[Hashable]):
     return seq
 
 
-def validate_gh_user(username: str, hotfix_known_errorenous_names: bool = True) -> str:
+def validate_github_user(
+    username: str, hotfix_known_errorenous_names: bool = True
+) -> str:
     if hotfix_known_errorenous_names:
         if username == "Constantin Pape":
             return "constantinpape"
 
     if (
-        username.lower() in KNOWN_GH_USERS
+        username.lower() in KNOWN_GITHUB_USERS
         or not get_validation_context().perform_io_checks
     ):
         return username
 
-    if username.lower() in KNOWN_INVALID_GH_USERS:
+    if username.lower() in KNOWN_INVALID_GITHUB_USERS:
         raise ValueError(f"Known invalid GitHub user '{username}'")
 
     try:
@@ -89,9 +91,9 @@ def validate_gh_user(username: str, hotfix_known_errorenous_names: bool = True)
                 value=username,
             )
         elif r.status_code != 200:
-            KNOWN_INVALID_GH_USERS.add(username.lower())
+            KNOWN_INVALID_GITHUB_USERS.add(username.lower())
             raise ValueError(f"Could not find GitHub user '{username}'")
 
-        KNOWN_GH_USERS.add(username.lower())
+        KNOWN_GITHUB_USERS.add(username.lower())
 
     return username

bioimageio/spec/_internal/io.py

@@ -705,18 +705,23 @@ def _open_url(
         or hashlib.sha256(str(source).encode(encoding="utf-8")).hexdigest()
     )
 
-    return BytesReader(
-        cache.fetch(
+    try:
+        reader = cache.fetch(
             source,
             fetcher=partial(_fetch_url, progressbar=progressbar),
             force_refetch=digest,
-        ),
-        suffix=source_path.suffix,
-        sha256=sha,
-        original_file_name=source_path.name,
-        original_root=source.parent,
-        is_zipfile=None,
-    )
+        )
+    except Exception as e:
+        raise ValueError(f"Failed to fetch {source}.") from e
+    else:
+        return BytesReader(
+            reader,
+            suffix=source_path.suffix,
+            sha256=sha,
+            original_file_name=source_path.name,
+            original_root=source.parent,
+            is_zipfile=None,
+        )
 
 
 def _fetch_url(

bioimageio/spec/conda_env.py

@@ -50,8 +50,8 @@ def _ensure_valid_conda_env_name(cls, value: Optional[str]) -> Optional[str]:
     def wo_name(self):
         return self.model_construct(**{k: v for k, v in self if k != "name"})
 
-    def _get_version(self, package: str):
-        """Helper to return any verison pin for **package**
+    def _get_version_pin(self, package: str):
+        """Helper to return any version pin for **package**
 
         TODO: improve: interprete version pin and return structured information.
         """
@@ -62,6 +62,10 @@ def _get_version(self, package: str):
                         return p[len(package) :]
             elif d.startswith(package):
                 return d[len(package) :]
+            elif "::" in d and (d_wo_channel := d.split("::", 1)[-1]).startswith(
+                package
+            ):
+                return d_wo_channel[len(package) :]
 
     def get_pip_deps(self) -> List[str]:
         """Get the pip dependencies of this conda env."""

bioimageio/spec/generic/v0_3.py

@@ -27,7 +27,7 @@
 
 from .._internal.common_nodes import Node, ResourceDescrBase
 from .._internal.constants import TAG_CATEGORIES
-from .._internal.field_validation import validate_gh_user
+from .._internal.field_validation import validate_github_user
 from .._internal.field_warning import as_warning, warn
 from .._internal.io import (
     BioimageioYamlContent,
@@ -112,11 +112,11 @@ class Author(_Author_v0_2):
     github_user: Optional[str] = None
 
     @field_validator("github_user", mode="after")
-    def _validate_gh_user(cls, value: Optional[str]):
+    def _validate_github_user(cls, value: Optional[str]):
         if value is None:
             return None
         else:
-            return validate_gh_user(value)
+            return validate_github_user(value)
 
 
 class _AuthorConv(Converter[_Author_v0_2, Author]):
@@ -140,8 +140,8 @@ class Maintainer(_Maintainer_v0_2):
     github_user: str
 
     @field_validator("github_user", mode="after")
-    def validate_gh_user(cls, value: str):
-        return validate_gh_user(value)
+    def validate_github_user(cls, value: str):
+        return validate_github_user(value)
 
 
 class _MaintainerConv(Converter[_Maintainer_v0_2, Maintainer]):

bioimageio/spec/model/v0_5.py

@@ -43,6 +43,7 @@
     RootModel,
     SerializationInfo,
     SerializerFunctionWrapHandler,
+    StrictInt,
     Tag,
     ValidationInfo,
     WrapSerializer,
@@ -382,7 +383,7 @@ class SizeReference(Node):
     axis_id: AxisId
     """axis id of the reference axis"""
 
-    offset: int = 0
+    offset: StrictInt = 0
 
     def get_size(
         self,
@@ -2488,8 +2489,8 @@ class LinkedModel(LinkedResourceBase):
 
 
 class _DataDepSize(NamedTuple):
-    min: int
-    max: Optional[int]
+    min: StrictInt
+    max: Optional[StrictInt]
 
 
 class _AxisSizes(NamedTuple):

bioimageio/spec/summary.py

@@ -23,6 +23,7 @@
     no_type_check,
 )
 
+import annotated_types
 import markdown
 import rich.console
 import rich.markdown
@@ -36,7 +37,7 @@
     model_validator,
 )
 from pydantic_core.core_schema import ErrorType
-from typing_extensions import Self, assert_never
+from typing_extensions import Annotated, Self, assert_never
 
 from bioimageio.spec._internal.type_guards import is_dict
 
@@ -241,25 +242,32 @@ class ValidationSummary(BaseModel, extra="allow"):
     for one specific `ResourceDescr` instance."""
 
     name: str
-    """name of the validation"""
+    """Name of the validation"""
     source_name: str
-    """source of the validated bioimageio description"""
+    """Source of the validated bioimageio description"""
     id: Optional[str] = None
     """ID of the resource being validated"""
     type: str
-    """type of the resource being validated"""
+    """Type of the resource being validated"""
     format_version: str
-    """format version of the resource being validated"""
+    """Format version of the resource being validated"""
     status: Literal["passed", "valid-format", "failed"]
     """overall status of the bioimageio validation"""
+    metadata_completeness: Annotated[float, annotated_types.Interval(ge=0, le=1)] = 0.0
+    """Estimate of completeness of the metadata in the resource description.
+
+    Note: This completeness estimate may change with subsequent releases
+        and should be considered bioimageio.spec version specific.
+    """
+
     details: List[ValidationDetail]
-    """list of validation details"""
+    """List of validation details"""
     env: Set[InstalledPackage] = Field(
         default_factory=lambda: {
             InstalledPackage(name="bioimageio.spec", version=VERSION)
         }
     )
-    """list of selected, relevant package versions"""
+    """List of selected, relevant package versions"""
 
     saved_conda_list: Optional[str] = None
 

changelog.md

@@ -10,9 +10,10 @@ In this file we log both:
 
 This changelog includes implementation details and my reference the [changes to the Resource Description Format](#changes-to-the-resource-description-format), e.g. in entry [bioimageio.spec 0.5.2](#bioimageiospec-052).
 
-#### bioimageio.spec (next release)
+#### bioimageio.spec 0.5.4.4
 
 - infinity and not-a-number values are no longer allowed (when used in a tensor description under data.range they are replaced with `None`)
+- stricter validation of integers; float values no longer allowed for input/output fields `size.offset`, `size.min`, `size.max`
 
 #### bioimageio.spec 0.5.4.3