Merge pull request #232 from bioimage-io/test_model_robust

catch more errors in test_model

Author: FynnBe

bioimageio/core/resource_tests.py

@@ -30,15 +30,34 @@ def test_model(
 ) -> dict:
     """Test whether the test output(s) of a model can be reproduced.
 
-    Returns summary dict with "error" and "traceback" key; summary["error"] is None if no errors were encountered.
+    Returns: summary dict with keys: name, status, error, traceback, bioimageio_spec_version, bioimageio_core_version
     """
-    model = load_resource_description(
-        model_rdf, weights_priority_order=None if weight_format is None else [weight_format]
-    )
+    # todo: reuse more of 'test_resource'
+    tb = None
+    try:
+        model = load_resource_description(
+            model_rdf, weights_priority_order=None if weight_format is None else [weight_format]
+        )
+    except Exception as e:
+        model = None
+        error = str(e)
+        tb = traceback.format_tb(e.__traceback__)
+    else:
+        error = None
+
     if isinstance(model, Model):
         return test_resource(model, weight_format=weight_format, devices=devices, decimal=decimal)
     else:
-        return {"error": f"Expected RDF type Model, got {type(model)} instead.", "traceback": None}
+        error = error or f"Expected RDF type Model, got {type(model)} instead."
+
+    return dict(
+        name="reproduced test outputs from test inputs",
+        status="failed",
+        error=error,
+        traceback=tb,
+        bioimageio_spec_version=bioimageio_spec_version,
+        bioimageio_core_version=bioimageio_core_version,
+    )
 
 
 def _validate_input_shape(shape: Tuple[int, ...], shape_spec) -> bool:
@@ -80,7 +99,7 @@ def _validate_output_shape(shape: Tuple[int, ...], shape_spec, input_shapes) ->
 
 
 def test_resource(
-    model_rdf: Union[RawResourceDescription, ResourceDescription, URI, Path, str],
+    rdf: Union[RawResourceDescription, ResourceDescription, URI, Path, str],
     *,
     weight_format: Optional[WeightsFormat] = None,
     devices: Optional[List[str]] = None,
@@ -95,9 +114,7 @@ def test_resource(
     test_name: str = "load resource description"
 
     try:
-        rd = load_resource_description(
-            model_rdf, weights_priority_order=None if weight_format is None else [weight_format]
-        )
+        rd = load_resource_description(rdf, weights_priority_order=None if weight_format is None else [weight_format])
     except Exception as e:
         error = str(e)
         tb = traceback.format_tb(e.__traceback__)