Merge pull request #402 from bioimage-io/always_test_all_weight_formats

always test all weight formats

Author: FynnBe

README.md

@@ -124,6 +124,11 @@ The model specification and its validation tools can be found at <https://github
 
 ## Changelog
 
+### 0.6.8 (to be released)
+
+* testing model inference will now check all weight formats
+  (previously only the first one for which model adapter creation succeeded had been checked)
+
 ### 0.6.7
 
 * `predict()` argument `inputs` may be sample

bioimageio/core/_resource_tests.py

@@ -103,9 +103,16 @@ def load_description_and_test(
         _test_expected_resource_type(rd, expected_type)
 
     if isinstance(rd, (v0_4.ModelDescr, v0_5.ModelDescr)):
-        _test_model_inference(rd, weight_format, devices, decimal)
-        if not isinstance(rd, v0_4.ModelDescr):
-            _test_model_inference_parametrized(rd, weight_format, devices)
+        if weight_format is None:
+            weight_formats: List[WeightsFormat] = [
+                w for w, we in rd.weights if we is not None
+            ]  # pyright: ignore[reportAssignmentType]
+        else:
+            weight_formats = [weight_format]
+        for w in weight_formats:
+            _test_model_inference(rd, w, devices, decimal)
+            if not isinstance(rd, v0_4.ModelDescr):
+                _test_model_inference_parametrized(rd, w, devices)
 
     # TODO: add execution of jupyter notebooks
     # TODO: add more tests
@@ -115,7 +122,7 @@ def load_description_and_test(
 
 def _test_model_inference(
     model: Union[v0_4.ModelDescr, v0_5.ModelDescr],
-    weight_format: Optional[WeightsFormat],
+    weight_format: WeightsFormat,
     devices: Optional[List[str]],
     decimal: int,
 ) -> None:
@@ -161,11 +168,7 @@ def _test_model_inference(
                 if error is None
                 else [
                     ErrorEntry(
-                        loc=(
-                            ("weights",)
-                            if weight_format is None
-                            else ("weights", weight_format)
-                        ),
+                        loc=("weights", weight_format),
                         msg=error,
                         type="bioimageio.core",
                         traceback=tb,
@@ -178,7 +181,7 @@ def _test_model_inference(
 
 def _test_model_inference_parametrized(
     model: v0_5.ModelDescr,
-    weight_format: Optional[WeightsFormat],
+    weight_format: WeightsFormat,
     devices: Optional[List[str]],
 ) -> None:
     if not any(
@@ -300,19 +303,15 @@ def get_ns(n: int):
 
                 model.validation_summary.add_detail(
                     ValidationDetail(
-                        name="Run inference for inputs with batch_size:"
-                        + f" {batch_size} and size parameter n: {n}",
+                        name=f"Run {weight_format} inference for inputs with"
+                        + f" batch_size: {batch_size} and size parameter n: {n}",
                         status="passed" if error is None else "failed",
                         errors=(
                             []
                             if error is None
                             else [
                                 ErrorEntry(
-                                    loc=(
-                                        ("weights",)
-                                        if weight_format is None
-                                        else ("weights", weight_format)
-                                    ),
+                                    loc=("weights", weight_format),
                                     msg=error,
                                     type="bioimageio.core",
                                 )
@@ -325,15 +324,11 @@ def get_ns(n: int):
         tb = traceback.format_tb(e.__traceback__)
         model.validation_summary.add_detail(
             ValidationDetail(
-                name="Run inference for parametrized inputs",
+                name=f"Run {weight_format} inference for parametrized inputs",
                 status="failed",
                 errors=[
                     ErrorEntry(
-                        loc=(
-                            ("weights",)
-                            if weight_format is None
-                            else ("weights", weight_format)
-                        ),
+                        loc=("weights", weight_format),
                         msg=error,
                         type="bioimageio.core",
                         traceback=tb,

tests/conftest.py

@@ -115,23 +115,12 @@
         "unet2d_nuclei_broad_model",
     ]
 )
-ONNX_MODELS = (
-    []
-    if onnxruntime is None
-    else [
-        "hpa_densenet",
-        "unet2d_multi_tensor",
-        "unet2d_nuclei_broad_model",
-    ]
-)
+ONNX_MODELS = [] if onnxruntime is None else ["hpa_densenet"]
 TENSORFLOW_MODELS = (
     []
     if tensorflow is None
     else (
-        [
-            "hpa_densenet",
-            "stardist",
-        ]
+        ["stardist"]
         if tf_major_version == 1
         else [
             "unet2d_keras_tf2",