replace skip_<weith format> flags by model groups

FynnBe · FynnBe · commit 31e54cb60d73 · 2021-09-15T16:56:34.000+02:00
diff --git a/tests/build_spec/test_build_spec.py b/tests/build_spec/test_build_spec.py
@@ -1,10 +1,8 @@
 import os
 
-import pytest
-
-import bioimageio.spec as spec
 from marshmallow import missing
 
+import bioimageio.spec as spec
 from bioimageio.core.resource_io.io_ import load_raw_resource_description
 
 
@@ -56,33 +54,27 @@ def _test_build_spec(path, weight_type, tensorflow_version=None):
     spec.model.schema.Model().dump(raw_model)
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_build_spec_pytorch(any_torch_model):
     _test_build_spec(any_torch_model, "pytorch_state_dict")
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_build_spec_torchscript(any_torchscript_model):
     _test_build_spec(any_torchscript_model, "pytorch_script")
 
 
-@pytest.mark.skipif(pytest.skip_onnx, reason="requires onnx")
 def test_build_spec_onnx(any_onnx_model):
     _test_build_spec(any_onnx_model, "onnx")
 
 
-@pytest.mark.skipif(pytest.skip_tensorflow or pytest.tf_major_version != 1, reason="requires tensorflow 1")
 def test_build_spec_keras(any_keras_model):
     _test_build_spec(any_keras_model, "keras_hdf5", tensorflow_version="1.12")  # todo: keras for tf 2??
 
 
-@pytest.mark.skipif(pytest.skip_tensorflow, reason="requires tensorflow")
 def test_build_spec_tf(any_tensorflow_model):
     _test_build_spec(
         any_tensorflow_model, "tensorflow_saved_model_bundle", tensorflow_version="1.12"
     )  # check tf version
 
 
-@pytest.mark.skipif(pytest.skip_tensorflow, reason="requires tensorflow")
 def test_build_spec_tfjs(any_tensorflow_js_model):
     _test_build_spec(any_tensorflow_js_model, "tensorflow_js", tensorflow_version="1.12")
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -49,6 +49,7 @@
 
 skip_tensorflow = tensorflow is None
 skip_tensorflow = True  # todo: update FruNet and remove this
+skip_tensorflow_js = True  # todo: update FruNet and figure out how to test tensorflow_js weights in python
 
 try:
     import keras
@@ -74,80 +75,71 @@
         load_model_packages |= set(tensorflow2_models)
 
 
-# set 'skip_<FRAMEWORK>' flags as global pytest variables,
-# to deselect tests that require frameworks not available in current env
 def pytest_configure():
-    pytest.skip_torch = skip_torch
-    pytest.skip_onnx = skip_onnx
-    pytest.skip_tensorflow = skip_tensorflow
-    pytest.tf_major_version = tf_major_version
-    pytest.skip_keras = skip_keras
-
-    pytest.model_packages = {
-        name: export_resource_package(model_sources[name])
-        for name in (load_model_packages | {"unet2d_nuclei_broad_model"})  # always load unet2d_nuclei_broad_model
-    }
-
 
-@pytest.fixture
-def unet2d_nuclei_broad_model():
-    return pytest.model_packages["unet2d_nuclei_broad_model"]
+    # explicit skip flag needed for pytorch to onnx converter test
+    pytest.skip_onnx = skip_onnx
 
+    # load all model packages used in tests
+    pytest.model_packages = {name: export_resource_package(model_sources[name]) for name in load_model_packages}
 
-@pytest.fixture
-def unet2d_multi_tensor():
-    return pytest.model_packages["unet2d_multi_tensor"]
 
+#
+# model groups of the form any_<weight format>_model that include all models providing a specific weight format
+#
 
-@pytest.fixture
-def FruNet_model():
-    return pytest.model_packages["FruNet_model"]
+# written as model group to automatically skip on missing torch
+@pytest.fixture(params=[] if skip_torch else ["unet2d_nuclei_broad_model"])
+def unet2d_nuclei_broad_model(request):
+    return pytest.model_packages[request.param]
 
 
-#
-# model groups
-#
+# written as model group to automatically skip on missing tensorflow 1
+@pytest.fixture(params=[] if skip_tensorflow or tf_major_version != 1 else ["FruNet_model"])
+def FruNet_model(request):
+    return pytest.model_packages[request.param]
 
 
-@pytest.fixture(params=torch_models)
+@pytest.fixture(params=[] if skip_torch else torch_models)
 def any_torch_model(request):
     return pytest.model_packages[request.param]
 
 
-@pytest.fixture(params=torchscript_models)
+@pytest.fixture(params=[] if skip_torch else torchscript_models)
 def any_torchscript_model(request):
     return pytest.model_packages[request.param]
 
 
-@pytest.fixture(params=onnx_models)
+@pytest.fixture(params=[] if skip_onnx else onnx_models)
 def any_onnx_model(request):
     return pytest.model_packages[request.param]
 
 
-@pytest.fixture(params=set(tensorflow1_models) | set(tensorflow2_models))
+@pytest.fixture(params=[] if skip_tensorflow else (set(tensorflow1_models) | set(tensorflow2_models)))
 def any_tensorflow_model(request):
     name = request.param
-    if (pytest.tf_major_version == 1 and name in tensorflow1_models) or (
-        pytest.tf_major_version == 2 and name in tensorflow2_models
-    ):
+    if (tf_major_version == 1 and name in tensorflow1_models) or (tf_major_version == 2 and name in tensorflow2_models):
         return pytest.model_packages[name]
 
 
-@pytest.fixture(params=keras_models)
+@pytest.fixture(params=[] if skip_keras else keras_models)
 def any_keras_model(request):
     return pytest.model_packages[request.param]
 
 
-@pytest.fixture(params=tensorflow_js_models)
+@pytest.fixture(params=[] if skip_tensorflow_js else tensorflow_js_models)
 def any_tensorflow_js_model(request):
     return pytest.model_packages[request.param]
 
 
+# fixture to test with all models that should run in the current environment
 @pytest.fixture(params=load_model_packages)
 def any_model(request):
     return pytest.model_packages[request.param]
 
 
-@pytest.fixture(params=["unet2d_nuclei_broad_model", "unet2d_fixed_shape"])
+# temporary fixture to test not with all, but only a manual selection of models
+# (models/functionality should be improved to get rid of this specific model group)
+@pytest.fixture(params=[] if skip_torch else ["unet2d_nuclei_broad_model", "unet2d_fixed_shape"])
 def unet2d_fixed_shape_or_not(request):
     return pytest.model_packages[request.param]
diff --git a/tests/prediction_pipeline/test_prediction_pipeline.py b/tests/prediction_pipeline/test_prediction_pipeline.py
@@ -1,5 +1,4 @@
 import numpy as np
-import pytest
 import xarray as xr
 from numpy.testing import assert_array_almost_equal
 
@@ -31,31 +30,21 @@ def _test_prediction_pipeline(model_package, weight_format):
         assert_array_almost_equal(out, exp, decimal=4)
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_prediction_pipeline_torch(any_torch_model):
     _test_prediction_pipeline(any_torch_model, "pytorch_state_dict")
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_prediction_pipeline_torchscript(any_torchscript_model):
     _test_prediction_pipeline(any_torchscript_model, "pytorch_script")
 
 
-@pytest.mark.skipif(pytest.skip_onnx, reason="requires onnx")
 def test_prediction_pipeline_onnx(any_onnx_model):
     _test_prediction_pipeline(any_onnx_model, "onnx")
 
 
-@pytest.mark.skipif(pytest.skip_tensorflow or pytest.tf_major_version != 1, reason="requires tensorflow 1")
-def test_prediction_pipeline_tensorflow(any_tensorflow1_model):
-    _test_prediction_pipeline(any_tensorflow1_model, "tensorflow_saved_model_bundle")
+def test_prediction_pipeline_tensorflow(any_tensorflow_model):
+    _test_prediction_pipeline(any_tensorflow_model, "tensorflow_saved_model_bundle")
 
 
-@pytest.mark.skipif(pytest.skip_tensorflow or pytest.tf_major_version != 2, reason="requires tensorflow 2")
-def test_prediction_pipeline_tensorflow(any_tensorflow2_model):
-    _test_prediction_pipeline(any_tensorflow2_model, "tensorflow_saved_model_bundle")
-
-
-@pytest.mark.skipif(pytest.skip_keras, reason="requires keras")
 def test_prediction_pipeline_keras(any_keras_model):
     _test_prediction_pipeline(any_keras_model, "keras_hdf5")
diff --git a/tests/resource_io/test_load_rdf.py b/tests/resource_io/test_load_rdf.py
@@ -52,7 +52,6 @@ def test_load_model(any_model):
     assert model
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires pytorch")
 def test_load_model_with_abs_path_source(unet2d_nuclei_broad_model):
     from bioimageio.core.resource_io import load_raw_resource_description, load_resource_description
 
@@ -63,7 +62,6 @@ def test_load_model_with_abs_path_source(unet2d_nuclei_broad_model):
     assert model
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires pytorch")
 def test_load_model_with_rel_path_source(unet2d_nuclei_broad_model):
     from bioimageio.core.resource_io import load_raw_resource_description, load_resource_description
 
@@ -74,7 +72,6 @@ def test_load_model_with_rel_path_source(unet2d_nuclei_broad_model):
     assert model
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires pytorch")
 def test_load_model_with_abs_str_source(unet2d_nuclei_broad_model):
     from bioimageio.core.resource_io import load_raw_resource_description, load_resource_description
 
@@ -85,7 +82,6 @@ def test_load_model_with_abs_str_source(unet2d_nuclei_broad_model):
     assert model
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires pytorch")
 def test_load_model_with_rel_str_source(unet2d_nuclei_broad_model):
     from bioimageio.core.resource_io import load_raw_resource_description, load_resource_description
 
@@ -96,7 +92,6 @@ def test_load_model_with_rel_str_source(unet2d_nuclei_broad_model):
     assert model
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires pytorch")
 def test_load_remote_model_with_folders():
     from bioimageio.core import load_resource_description, load_raw_resource_description
     from bioimageio.core.resource_io import nodes
diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -2,16 +2,15 @@
 
 import numpy as np
 import pytest
+
 from bioimageio.core import load_resource_description
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_cli_test_model(unet2d_nuclei_broad_model):
     ret = subprocess.run(["bioimageio", "test-model", unet2d_nuclei_broad_model])
     assert ret.returncode == 0
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_cli_predict_image(unet2d_nuclei_broad_model, tmp_path):
     spec = load_resource_description(unet2d_nuclei_broad_model)
     in_path = spec.test_inputs[0]
@@ -23,7 +22,6 @@ def test_cli_predict_image(unet2d_nuclei_broad_model, tmp_path):
     assert out_path.exists()
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_cli_predict_images(unet2d_nuclei_broad_model, tmp_path):
     n_images = 3
     shape = (1, 1, 128, 128)
@@ -50,7 +48,6 @@ def test_cli_predict_images(unet2d_nuclei_broad_model, tmp_path):
         assert np.load(out_path).shape == expected_shape
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_torch_to_torchscript(unet2d_nuclei_broad_model, tmp_path):
     out_path = tmp_path.with_suffix(".pt")
     ret = subprocess.run(
@@ -60,7 +57,7 @@ def test_torch_to_torchscript(unet2d_nuclei_broad_model, tmp_path):
     assert out_path.exists()
 
 
-@pytest.mark.skipif(pytest.skip_torch or pytest.skip_onnx, reason="requires torch and onnx")
+@pytest.mark.skipif(pytest.skip_onnx, reason="requires torch and onnx")
 def test_torch_to_onnx(unet2d_nuclei_broad_model, tmp_path):
     out_path = tmp_path.with_suffix(".onnx")
     ret = subprocess.run(["bioimageio", "convert-torch-weights-to-onnx", str(unet2d_nuclei_broad_model), str(out_path)])
diff --git a/tests/test_prediction.py b/tests/test_prediction.py
@@ -2,22 +2,18 @@
 
 import imageio
 import numpy as np
-import pytest
-
-from bioimageio.core import load_resource_description
 from numpy.testing import assert_array_almost_equal
 
+from bioimageio.core import load_resource_description
 from bioimageio.core.resource_io.nodes import Model
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_test_model(unet2d_nuclei_broad_model):
     from bioimageio.core.prediction import test_model
 
     assert test_model(unet2d_nuclei_broad_model)
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_predict_image(unet2d_fixed_shape_or_not, tmpdir):
     any_model = unet2d_fixed_shape_or_not  # todo: replace 'unet2d_fixed_shape_or_not' with 'any_model'
     from bioimageio.core.prediction import predict_image
@@ -37,7 +33,6 @@ def test_predict_image(unet2d_fixed_shape_or_not, tmpdir):
         assert_array_almost_equal(res, exp, decimal=4)
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_predict_image_with_padding(unet2d_fixed_shape_or_not, tmp_path):
     any_model = unet2d_fixed_shape_or_not  # todo: replace 'unet2d_fixed_shape_or_not' with 'any_model'
     from bioimageio.core.prediction import predict_image
@@ -74,7 +69,6 @@ def check_result():
     check_result()
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_predict_image_with_tiling(unet2d_nuclei_broad_model, tmp_path):
     from bioimageio.core.prediction import predict_image
 
@@ -104,7 +98,6 @@ def check_result():
     check_result()
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires torch")
 def test_predict_images(unet2d_nuclei_broad_model, tmp_path):
     from bioimageio.core.prediction import predict_images
 
diff --git a/tests/weight_converter/torch/test_onnx.py b/tests/weight_converter/torch/test_onnx.py
@@ -2,7 +2,6 @@
 import pytest
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires pytorch")
 def test_onnx_converter_from_torch(any_torch_model, tmp_path):
     from bioimageio.core.weight_converter.torch.onnx import convert_weights_to_onnx
 
diff --git a/tests/weight_converter/torch/test_torchscript.py b/tests/weight_converter/torch/test_torchscript.py
@@ -1,8 +1,6 @@
 import os
-import pytest
 
 
-@pytest.mark.skipif(pytest.skip_torch, reason="requires pytorch")
 def test_torchscript_converter(any_torch_model, tmp_path):
     from bioimageio.core.weight_converter.torch import convert_weights_to_pytorch_script
 
