Merge pull request #485 from bioimage-io/forward_compat

Report on forward compatibility attempt in validation errors

Author: FynnBe

bioimageio/spec/io_.py

@@ -12,7 +12,7 @@
 from types import ModuleType
 from typing import Dict, IO, Optional, Sequence, Tuple, Union
 
-from marshmallow import missing
+from marshmallow import ValidationError, missing
 from packaging.version import Version
 
 from bioimageio.spec.shared import RDF_NAMES, raw_nodes, resolve_rdf_source, resolve_rdf_source_and_type, resolve_source
@@ -199,7 +199,8 @@ def load_raw_resource_description(
             # => 'format_version' may be invalid or the issue lies with 'type_'...
             raise e
 
-    if odv and Version(sub_spec.format_version) < odv:
+    downgrade_format_version = odv and Version(sub_spec.format_version) < odv
+    if downgrade_format_version:
         warnings.warn(
             f"Loading future {type_} format version {original_data_version} as (latest known) "
             f"{sub_spec.format_version}."
@@ -218,7 +219,16 @@ def load_raw_resource_description(
     schema: SharedBioImageIOSchema = getattr(sub_spec.schema, class_name)()
 
     data = sub_spec.converters.maybe_convert(data)
-    raw_rd = schema.load(data)
+    try:
+        raw_rd = schema.load(data)
+    except ValidationError as e:
+        if downgrade_format_version:
+            e.messages["format_version"] = (
+                f"Other errors may be caused by a possibly incompatible future format version {original_data_version} "
+                f"treated as {sub_spec.format_version}."
+            )
+
+        raise e
 
     if isinstance(root, pathlib.Path):
         root = root.resolve()

tests/test_validation_errors.py

@@ -11,6 +11,20 @@ def test_list_instead_of_nested_schema(unet2d_nuclei_broad_latest):
     data["run_mode"] = [{"name": "something"}]
 
     error = validate(data)["error"]
-    print(error)
     assert len(error) == 1
     assert error["run_mode"] == ["Expected dictionary, but got list."]
+
+
+def test_forward_compatibility_error(unet2d_fixed_shape):
+    from bioimageio.spec.commands import validate
+
+    data = yaml.load(unet2d_fixed_shape)
+
+    data["authors"] = 42  # make sure rdf is invalid
+    data["format_version"] = "9999.0.0"  # assume it is valid in a future format version
+
+    error = validate(data)["error"]
+
+    # even though the format version is correctly formatted, it should be mentioned here as we treat the future format
+    # version as the current latest. If this attempted forward compatibility fails we have to report that we did it.
+    assert "format_version" in error