add estimate of metadata completeness

Author: FynnBe

bioimageio/spec/_internal/common_nodes.py

@@ -10,6 +10,7 @@
     Any,
     ClassVar,
     Dict,
+    Iterable,
     List,
     Literal,
     Mapping,
@@ -149,13 +150,15 @@ def _ignore_future_patch(cls, data: Any, /) -> Any:
     @model_validator(mode="after")
     def _set_init_validation_summary(self) -> Self:
         context = get_validation_context()
+
         self._validation_summary = ValidationSummary(
             name="bioimageio format validation",
             source_name=context.source_name,
             id=getattr(self, "id", None),
             type=self.type,
             format_version=self.format_version,
             status="failed" if isinstance(self, InvalidDescr) else "valid-format",
+            metadata_completeness=self._get_metadata_completeness(),
             details=(
                 []
                 if isinstance(self, InvalidDescr)
@@ -245,6 +248,25 @@ def load(
 
         return rd
 
+    def _get_metadata_completeness(self) -> float:
+        if isinstance(self, InvalidDescr):
+            return 0.0
+
+        given = self.model_dump(mode="json", exclude_unset=True, exclude_defaults=False)
+        full = self.model_dump(mode="json", exclude_unset=False, exclude_defaults=False)
+
+        def extract_flat_keys(d: Dict[Any, Any], key: str = "") -> Iterable[str]:
+            for k, v in d.items():
+                if is_dict(v):
+                    yield from extract_flat_keys(v, key=f"{key}.{k}" if key else k)
+
+                yield k
+
+        given_keys = set(extract_flat_keys(given))
+        full_keys = set(extract_flat_keys(full))
+        assert len(full_keys) >= len(given_keys)
+        return len(given_keys) / len(full_keys) if full_keys else 0.0
+
     @classmethod
     def _load_impl(
         cls, data: BioimageioYamlContent

bioimageio/spec/summary.py

@@ -23,6 +23,7 @@
     no_type_check,
 )
 
+import annotated_types
 import markdown
 import rich.console
 import rich.markdown
@@ -36,7 +37,7 @@
     model_validator,
 )
 from pydantic_core.core_schema import ErrorType
-from typing_extensions import Self, assert_never
+from typing_extensions import Annotated, Self, assert_never
 
 from bioimageio.spec._internal.type_guards import is_dict
 
@@ -241,25 +242,32 @@ class ValidationSummary(BaseModel, extra="allow"):
     for one specific `ResourceDescr` instance."""
 
     name: str
-    """name of the validation"""
+    """Name of the validation"""
     source_name: str
-    """source of the validated bioimageio description"""
+    """Source of the validated bioimageio description"""
     id: Optional[str] = None
     """ID of the resource being validated"""
     type: str
-    """type of the resource being validated"""
+    """Type of the resource being validated"""
     format_version: str
-    """format version of the resource being validated"""
+    """Format version of the resource being validated"""
     status: Literal["passed", "valid-format", "failed"]
     """overall status of the bioimageio validation"""
+    metadata_completeness: Annotated[float, annotated_types.Interval(ge=0, le=1)] = 0.0
+    """Estimate of completeness of the metadata in the resource description.
+
+    Note: This completeness estimate may change with subsequent releases
+        and should be considered bioimageio.spec version specific.
+    """
+
     details: List[ValidationDetail]
-    """list of validation details"""
+    """List of validation details"""
     env: Set[InstalledPackage] = Field(
         default_factory=lambda: {
             InstalledPackage(name="bioimageio.spec", version=VERSION)
         }
     )
-    """list of selected, relevant package versions"""
+    """List of selected, relevant package versions"""
 
     saved_conda_list: Optional[str] = None