use determinism and refactor decimal

Author: FynnBe

bioimageio/core/_resource_tests.py

@@ -100,16 +100,18 @@ def test_model(
     absolute_tolerance: float = 1.5e-4,
     relative_tolerance: float = 1e-4,
     decimal: Optional[int] = None,
+    *,
+    determinism: Literal["seed_only", "full"] = "seed_only",
 ) -> ValidationSummary:
     """Test model inference"""
-    # NOTE: `decimal` is a legacy argument and is handled in `_test_model_inference`
     return test_description(
         source,
         weight_format=weight_format,
         devices=devices,
         absolute_tolerance=absolute_tolerance,
         relative_tolerance=relative_tolerance,
         decimal=decimal,
+        determinism=determinism,
         expected_type="model",
     )
 
@@ -123,10 +125,10 @@ def test_description(
     absolute_tolerance: float = 1.5e-4,
     relative_tolerance: float = 1e-4,
     decimal: Optional[int] = None,
+    determinism: Literal["seed_only", "full"] = "seed_only",
     expected_type: Optional[str] = None,
 ) -> ValidationSummary:
     """Test a bioimage.io resource dynamically, e.g. prediction of test tensors for models"""
-    # NOTE: `decimal` is a legacy argument and is handled in `_test_model_inference`
     rd = load_description_and_test(
         source,
         format_version=format_version,
@@ -135,6 +137,7 @@ def test_description(
         absolute_tolerance=absolute_tolerance,
         relative_tolerance=relative_tolerance,
         decimal=decimal,
+        determinism=determinism,
         expected_type=expected_type,
     )
     return rd.validation_summary
@@ -149,10 +152,10 @@ def load_description_and_test(
     absolute_tolerance: float = 1.5e-4,
     relative_tolerance: float = 1e-4,
     decimal: Optional[int] = None,
+    determinism: Literal["seed_only", "full"] = "seed_only",
     expected_type: Optional[str] = None,
 ) -> Union[ResourceDescr, InvalidDescr]:
     """Test RDF dynamically, e.g. model inference of test inputs"""
-    # NOTE: `decimal` is a legacy argument and is handled in `_test_model_inference`
     if (
         isinstance(source, ResourceDescrBase)
         and format_version != "discover"
@@ -184,10 +187,25 @@ def load_description_and_test(
             ]  # pyright: ignore[reportAssignmentType]
         else:
             weight_formats = [weight_format]
-        for w in weight_formats:
-            _test_model_inference(
-                rd, w, devices, absolute_tolerance, relative_tolerance, decimal
+
+        if decimal is None:
+            atol = absolute_tolerance
+            rtol = relative_tolerance
+        else:
+            warnings.warn(
+                "The argument `decimal` has been deprecated in favour of"
+                + " `relative_tolerance` and `absolute_tolerance`, with different"
+                + " validation logic, using `numpy.testing.assert_allclose, see"
+                + " 'https://numpy.org/doc/stable/reference/generated/"
+                + " numpy.testing.assert_allclose.html'. Passing a value for `decimal`"
+                + " will cause validation to revert to the old behaviour."
             )
+            atol = 1.5 * 10 ** (-decimal)
+            rtol = 0
+
+        enable_determinism(determinism)
+        for w in weight_formats:
+            _test_model_inference(rd, w, devices, atol, rtol)
             if not isinstance(rd, v0_4.ModelDescr):
                 _test_model_inference_parametrized(rd, w, devices)
 
@@ -201,21 +219,14 @@ def _test_model_inference(
     model: Union[v0_4.ModelDescr, v0_5.ModelDescr],
     weight_format: WeightsFormat,
     devices: Optional[Sequence[str]],
-    absolute_tolerance: float,
-    relative_tolerance: float,
-    decimal: Optional[int],
+    atol: float,
+    rtol: float,
 ) -> None:
     test_name = f"Reproduce test outputs from test inputs ({weight_format})"
     logger.info("starting '{}'", test_name)
     error: Optional[str] = None
     tb: List[str] = []
 
-    precision_args = _handle_legacy_precision_args(
-        absolute_tolerance=absolute_tolerance,
-        relative_tolerance=relative_tolerance,
-        decimal=decimal,
-    )
-
     try:
         inputs = get_test_inputs(model)
         expected = get_test_outputs(model)
@@ -238,8 +249,8 @@ def _test_model_inference(
                     np.testing.assert_allclose(
                         res.data,
                         exp.data,
-                        rtol=precision_args["relative_tolerance"],
-                        atol=precision_args["absolute_tolerance"],
+                        rtol=rtol,
+                        atol=atol,
                     )
                 except AssertionError as e:
                     error = f"Output and expected output disagree:\n {e}"
@@ -455,39 +466,6 @@ def _test_expected_resource_type(
     )
 
 
-def _handle_legacy_precision_args(
-    absolute_tolerance: float, relative_tolerance: float, decimal: Optional[int]
-) -> Dict[str, float]:
-    """
-    Transform the precision arguments to conform with the current implementation.
-
-    If the deprecated `decimal` argument is used it overrides the new behaviour with
-    the old behaviour.
-    """
-    # Already conforms with current implementation
-    if decimal is None:
-        return {
-            "absolute_tolerance": absolute_tolerance,
-            "relative_tolerance": relative_tolerance,
-        }
-    else:
-        warnings.warn(
-            "The argument `decimal` has been depricated in favour of "
-            + "`relative_tolerance` and `absolute_tolerance`, with different validation "
-            + "logic, using `numpy.testing.assert_allclose, see "
-            + "'https://numpy.org/doc/stable/reference/generated/"
-            + "numpy.testing.assert_allclose.html'. Passing a value for `decimal` will "
-            + "cause validation to revert to the old behaviour."
-        )
-
-    # decimal overrides new behaviour,
-    #   have to convert the params to emulate old behaviour
-    return {
-        "absolute_tolerance": 1.5 * 10 ** (-decimal),
-        "relative_tolerance": 0,
-    }
-
-
 # TODO: Implement `debug_model()`
 # def debug_model(
 #     model_rdf: Union[RawResourceDescr, ResourceDescr, URI, Path, str],