WIP fix tests and cleanup

Author: FynnBe

bioimageio/core/_prediction_pipeline.py

@@ -164,20 +164,20 @@ def predict_sample_without_blocking(
                 if out is not None
             },
             stat=sample.stat,
-            id=self.get_output_sample_id(sample.id),
+            id=sample.id,
         )
         if not skip_postprocessing:
             self.apply_postprocessing(output)
 
         return output
 
     def get_output_sample_id(self, input_sample_id: SampleId):
-        if input_sample_id is None:
-            return None
-        else:
-            return f"{input_sample_id}_" + (
-                self.model_description.id or self.model_description.name
-            )
+        warnings.warn(
+            "`PredictionPipeline.get_output_sample_id()` is deprecated and will be"
+            + " removed soon. Output sample id is equal to input sample id, hence this"
+            + " function is not needed."
+        )
+        return input_sample_id
 
     def predict_sample_with_fixed_blocking(
         self,

bioimageio/core/cli.py

@@ -116,7 +116,7 @@ class ValidateFormatCmd(CmdBase, WithSource):
     """validate the meta data format of a bioimageio resource."""
 
     def run(self):
-        validate_format(self.descr)
+        sys.exit(validate_format(self.descr))
 
 
 class TestCmd(CmdBase, WithSource):
@@ -134,11 +134,13 @@ class TestCmd(CmdBase, WithSource):
     """Precision for numerical comparisons"""
 
     def run(self):
-        test(
-            self.descr,
-            weight_format=self.weight_format,
-            devices=self.devices,
-            decimal=self.decimal,
+        sys.exit(
+            test(
+                self.descr,
+                weight_format=self.weight_format,
+                devices=self.devices,
+                decimal=self.decimal,
+            )
         )
 
 
@@ -158,10 +160,12 @@ def run(self):
             self.descr.validation_summary.display()
             raise ValueError("resource description is invalid")
 
-        package(
-            self.descr,
-            self.path,
-            weight_format=self.weight_format,
+        sys.exit(
+            package(
+                self.descr,
+                self.path,
+                weight_format=self.weight_format,
+            )
         )
 
 

bioimageio/core/commands.py

@@ -1,8 +1,6 @@
-"""deprecated,
-use the CLI object `bioimageio.core.cli.Bioimageio` programmatically instead.
-"""
+"""These functions implement the logic of the bioimageio command line interface
+defined in `bioimageio.core.cli`."""
 
-import sys
 from pathlib import Path
 from typing import Optional, Sequence, Union
 
@@ -28,7 +26,7 @@ def test(
     weight_format: WeightFormatArgAll = "all",
     devices: Optional[Union[str, Sequence[str]]] = None,
     decimal: int = 4,
-):
+) -> int:
     """test a bioimageio resource
 
     Args:
@@ -40,7 +38,7 @@ def test(
     """
     if isinstance(descr, InvalidDescr):
         descr.validation_summary.display()
-        sys.exit(1)
+        return 1
 
     summary = test_description(
         descr,
@@ -49,7 +47,7 @@ def test(
         decimal=decimal,
     )
     summary.display()
-    sys.exit(0 if summary.status == "passed" else 1)
+    return 0 if summary.status == "passed" else 1
 
 
 def validate_format(
@@ -61,7 +59,7 @@ def validate_format(
         descr: a bioimageio resource description
     """
     descr.validation_summary.display()
-    sys.exit(0 if descr.validation_summary.status == "passed" else 1)
+    return 0 if descr.validation_summary.status == "passed" else 1
 
 
 def package(
@@ -98,3 +96,4 @@ def package(
             output_path=path,
             weights_priority_order=weights_priority_order,
         )
+    return 0

bioimageio/core/digest_spec.py

@@ -159,7 +159,7 @@ def get_test_inputs(model: AnyModelDescr) -> Sample:
             for m, arr, ax in zip(member_ids, arrays, axes)
         },
         stat={},
-        id="test-input",
+        id="test-sample",
     )
 
 
@@ -180,7 +180,7 @@ def get_test_outputs(model: AnyModelDescr) -> Sample:
             for m, arr, ax in zip(member_ids, arrays, axes)
         },
         stat={},
-        id="test-output",
+        id="test-sample",
     )
 
 

tests/conftest.py

@@ -165,39 +165,39 @@ def conda_cmd():
 #
 
 
-@fixture(params=TORCH_MODELS)
+@fixture(scope="session", params=TORCH_MODELS)
 def any_torch_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
-@fixture(params=TORCHSCRIPT_MODELS)
+@fixture(scope="session", params=TORCHSCRIPT_MODELS)
 def any_torchscript_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
-@fixture(params=ONNX_MODELS)
+@fixture(scope="session", params=ONNX_MODELS)
 def any_onnx_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
-@fixture(params=TENSORFLOW_MODELS)
+@fixture(scope="session", params=TENSORFLOW_MODELS)
 def any_tensorflow_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
-@fixture(params=KERAS_MODELS)
+@fixture(scope="session", params=KERAS_MODELS)
 def any_keras_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
-@fixture(params=TENSORFLOW_JS_MODELS)
+@fixture(scope="session", params=TENSORFLOW_JS_MODELS)
 def any_tensorflow_js_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # fixture to test with all models that should run in the current environment
 # we exclude any 'wrong' model here
-@fixture(params=sorted({m for m in ALL_MODELS if "wrong" not in m}))
+@fixture(scope="session", params=sorted({m for m in ALL_MODELS if "wrong" not in m}))
 def any_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
@@ -239,48 +239,52 @@ def unet2d_keras(request: FixtureRequest):
 
 
 # written as model group to automatically skip on missing torch
-@fixture(params=[] if skip_torch else ["unet2d_nuclei_broad_model"])
+@fixture(scope="session", params=[] if skip_torch else ["unet2d_nuclei_broad_model"])
 def unet2d_nuclei_broad_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # written as model group to automatically skip on missing torch
-@fixture(params=[] if skip_torch else ["unet2d_diff_output_shape"])
+@fixture(scope="session", params=[] if skip_torch else ["unet2d_diff_output_shape"])
 def unet2d_diff_output_shape(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # written as model group to automatically skip on missing torch
-@fixture(params=[] if skip_torch else ["unet2d_expand_output_shape"])
+@fixture(scope="session", params=[] if skip_torch else ["unet2d_expand_output_shape"])
 def unet2d_expand_output_shape(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # written as model group to automatically skip on missing torch
-@fixture(params=[] if skip_torch else ["unet2d_fixed_shape"])
+@fixture(scope="session", params=[] if skip_torch else ["unet2d_fixed_shape"])
 def unet2d_fixed_shape(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # written as model group to automatically skip on missing torch
-@fixture(params=[] if skip_torch else ["shape_change"])
+@fixture(scope="session", params=[] if skip_torch else ["shape_change"])
 def shape_change_model(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # written as model group to automatically skip on missing tensorflow 1
-@fixture(params=["stardist_wrong_shape"] if tf_major_version == 1 else [])
+@fixture(
+    scope="session", params=["stardist_wrong_shape"] if tf_major_version == 1 else []
+)
 def stardist_wrong_shape(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # written as model group to automatically skip on missing tensorflow 1
-@fixture(params=["stardist_wrong_shape2"] if tf_major_version == 1 else [])
+@fixture(
+    scope="session", params=["stardist_wrong_shape2"] if tf_major_version == 1 else []
+)
 def stardist_wrong_shape2(request: FixtureRequest):
     return MODEL_SOURCES[request.param]
 
 
 # written as model group to automatically skip on missing tensorflow 1
-@fixture(params=["stardist"] if tf_major_version == 1 else [])
+@fixture(scope="session", params=["stardist"] if tf_major_version == 1 else [])
 def stardist(request: FixtureRequest):
     return MODEL_SOURCES[request.param]

tests/test_commands.py

@@ -6,10 +6,10 @@
 from bioimageio.core import load_model
 from bioimageio.core.commands import package, validate_format
 from bioimageio.core.commands import test as command_tst
-from bioimageio.spec.model import ModelDescr
+from bioimageio.spec import AnyModelDescr
 
 
-@pytest.mark.fixture(scope="module")
+@pytest.fixture(scope="module")
 def model(unet2d_nuclei_broad_model: str):
     return load_model(unet2d_nuclei_broad_model, perform_io_checks=False)
 
@@ -23,14 +23,14 @@ def model(unet2d_nuclei_broad_model: str):
 )
 def test_package(
     weight_format: Literal["all", "pytorch_state_dict"],
-    model: ModelDescr,
+    model: AnyModelDescr,
     tmp_path: Path,
 ):
-    _ = package(model, weight_format=weight_format, path=tmp_path / "out.zip")
+    assert package(model, weight_format=weight_format, path=tmp_path / "out.zip") == 0
 
 
-def test_validate_format(model: ModelDescr):
-    _ = validate_format(model)
+def test_validate_format(model: AnyModelDescr):
+    assert validate_format(model) == 0
 
 
 @pytest.mark.parametrize(
@@ -39,6 +39,6 @@ def test_validate_format(model: ModelDescr):
 def test_test(
     weight_format: Literal["all", "pytorch_state_dict"],
     devices: Optional[str],
-    model: ModelDescr,
+    model: AnyModelDescr,
 ):
-    _ = command_tst(model, weight_format=weight_format, devices=devices)
+    assert command_tst(model, weight_format=weight_format, devices=devices) == 0