Merge branch 'fix-build-spec' into code-examples

Author: constantinpape

bioimageio/core/build_spec/build_model.py

@@ -100,10 +100,7 @@ def _get_weights(weight_uri, weight_type, source, root, **kwargs):
 
     elif weight_type == "onnx":
         weights = model_spec.raw_nodes.OnnxWeightsEntry(
-            source=weight_source,
-            sha256=weight_hash,
-            opset_version=kwargs.get("opset_version", 12),
-            **attachments
+            source=weight_source, sha256=weight_hash, opset_version=kwargs.get("opset_version", 12), **attachments
         )
         language = None
         framework = None
@@ -197,9 +194,7 @@ def _get_input_tensor(path, name, step, min_shape, data_range, axes, preprocessi
 
     kwargs = {}
     if preprocessing is not None:
-        kwargs["preprocessing"] = [
-            {"name": k, "kwargs": v} for k, v in preprocessing.items()
-        ]
+        kwargs["preprocessing"] = [{"name": k, "kwargs": v} for k, v in preprocessing.items()]
 
     inputs = model_spec.raw_nodes.InputTensor(
         name="input" if name is None else name,
@@ -229,9 +224,7 @@ def _get_output_tensor(path, name, reference_tensor, scale, offset, axes, data_r
 
     kwargs = {}
     if postprocessing is not None:
-        kwargs["postprocessing"] = [
-            {"name": k, "kwargs": v} for k, v in postprocessing.items()
-        ]
+        kwargs["postprocessing"] = [{"name": k, "kwargs": v} for k, v in postprocessing.items()]
     if halo is not None:
         kwargs["halo"] = halo
 
@@ -389,8 +382,9 @@ def build_model(
 
     inputs = [
         _get_input_tensor(test_in, name, step, min_shape, axes, data_range, preproc)
-        for test_in, name, step, min_shape, axes, data_range, preproc in
-        zip(test_inputs, input_name, input_step, input_min_shape, input_axes, input_data_range, preprocessing)
+        for test_in, name, step, min_shape, axes, data_range, preproc in zip(
+            test_inputs, input_name, input_step, input_min_shape, input_axes, input_data_range, preprocessing
+        )
     ]
 
     n_outputs = len(test_outputs)
@@ -405,8 +399,7 @@ def build_model(
 
     outputs = [
         _get_output_tensor(test_out, name, reference, scale, offset, axes, data_range, postproc, hal)
-        for test_out, name, reference, scale, offset, axes, data_range, postproc, hal in
-        zip(
+        for test_out, name, reference, scale, offset, axes, data_range, postproc, hal in zip(
             test_outputs,
             output_name,
             output_reference,
@@ -415,7 +408,7 @@ def build_model(
             output_axes,
             output_data_range,
             postprocessing,
-            halo
+            halo,
         )
     ]
 
@@ -449,7 +442,7 @@ def build_model(
         "source": source,
         "sha256": source_hash,
         "kwargs": model_kwargs,
-        "links": links
+        "links": links,
     }
     kwargs = {k: v for k, v in optional_kwargs.items() if v is not None}
     if dependencies is not None:
@@ -495,13 +488,13 @@ def add_weights(
     weight_uri: Union[str, Path],
     weight_type: Optional[str] = None,
     output_path: Optional[Union[str, Path]] = None,
-    **weight_kwargs
+    **weight_kwargs,
 ):
     """Add weight entry to bioimage.io model."""
     # we need to patss the weight path as abs path to avoid confusion with different root directories
-    new_weights = _get_weights(
-        Path(weight_uri).absolute(), weight_type, source=None, root=Path("."), **weight_kwargs
-    )[0]
+    new_weights = _get_weights(Path(weight_uri).absolute(), weight_type, source=None, root=Path("."), **weight_kwargs)[
+        0
+    ]
     model.weights.update(new_weights)
     if output_path is not None:
         model_package = export_resource_package(model, output_path=output_path)

bioimageio/core/resource_io/common.py

@@ -1,48 +1,97 @@
 import os
 import pathlib
 import warnings
+import zipfile
 from copy import deepcopy
-from typing import Any, Dict, Optional, Sequence, Tuple, Union
+from typing import Dict, IO, Optional, Sequence, Tuple, Union
 from zipfile import ZIP_DEFLATED, ZipFile
 
-from marshmallow import ValidationError, missing
+from marshmallow import missing
 
 from bioimageio import spec
 from bioimageio.core.resource_io.nodes import ResourceDescription
+from bioimageio.spec.io_ import resolve_rdf_source
 from bioimageio.spec.shared import raw_nodes
-from bioimageio.spec.shared.common import get_class_name_from_type
+from bioimageio.spec.shared.common import BIOIMAGEIO_CACHE_PATH, get_class_name_from_type
 from bioimageio.spec.shared.raw_nodes import ResourceDescription as RawResourceDescription
 from bioimageio.spec.shared.utils import PathToRemoteUriTransformer
 from . import nodes
-from .common import BIOIMAGEIO_CACHE_PATH, yaml
-from .utils import _download_uri_to_local_path, resolve_local_uri, resolve_raw_resource_description, resolve_uri
+from .utils import resolve_raw_resource_description, resolve_uri
 
-
-ROOT_PATH = "root_path"
 serialize_raw_resource_description = spec.io_.serialize_raw_resource_description
 save_raw_resource_description = spec.io_.save_raw_resource_description
 
 
+def extract_resource_package(
+    source: Union[os.PathLike, IO, str, bytes, raw_nodes.URI]
+) -> Tuple[dict, str, pathlib.Path]:
+    """extract a zip source to BIOIMAGEIO_CACHE_PATH"""
+    source, source_name, root = resolve_rdf_source(source)
+    if isinstance(root, bytes):
+        raise NotImplementedError("package source was bytes")
+
+    cache_folder = BIOIMAGEIO_CACHE_PATH / "extracted_packages"
+    cache_folder.mkdir(exist_ok=True, parents=True)
+
+    if isinstance(root, raw_nodes.URI):
+        from urllib.request import urlretrieve
+
+        package_path = cache_folder / root.scheme / root.authority / root.path.strip("/") / root.query
+        if (package_path / "rdf.yaml").exists():
+            download = None
+        else:
+            download, header = urlretrieve(str(root))
+
+        local_source = download
+    else:
+        download = None
+        local_source = root
+        package_path = cache_folder / root.relative_to(list(root.parents)[-1])
+
+    if local_source is not None:
+        with zipfile.ZipFile(local_source) as zf:
+            zf.extractall(package_path)
+
+    if not (package_path / "rdf.yaml").exists():
+        raise FileNotFoundError(f"missing 'rdf.yaml' in {root} extracted from {download}")
+
+    if download is not None:
+        try:
+            os.remove(download)
+        except Exception as e:
+            warnings.warn(f"Could not remove download {download} due to {e}")
+
+    assert isinstance(package_path, pathlib.Path)
+    return source, source_name, package_path
+
+
 def _replace_relative_paths_for_remote_source(
-    raw_rd: RawResourceDescription, source: Union[Any, str, raw_nodes.URI]
+    raw_rd: RawResourceDescription, root: Union[pathlib.Path, raw_nodes.URI, bytes]
 ) -> RawResourceDescription:
-    if isinstance(source, raw_nodes.URI) or isinstance(source, str) and source.startswith("http"):
+    if isinstance(root, raw_nodes.URI):
         # for a remote source relative paths are invalid; replace all relative file paths in source with URLs
-        if isinstance(source, str):
-            source = raw_nodes.URI(source)
-
         warnings.warn(
-            f"changing file paths in RDF to URIs due to a remote {source.scheme} source "
+            f"changing file paths in RDF to URIs due to a remote {root.scheme} source "
             "(may result in an invalid node)"
         )
-        raw_rd = PathToRemoteUriTransformer(remote_source=source).transform(raw_rd)
-        raw_rd.root_path = pathlib.Path()  # root_path cannot be URI
+        raw_rd = PathToRemoteUriTransformer(remote_source=root).transform(raw_rd)
+        root_path = pathlib.Path()  # root_path cannot be URI
+    elif isinstance(root, pathlib.Path):
+        if zipfile.is_zipfile(root):
+            _, _, root_path = extract_resource_package(root)
+        else:
+            root_path = root
+    elif isinstance(root, bytes):
+        raise NotImplementedError("root as bytes (io)")
+    else:
+        raise TypeError(root)
 
+    raw_rd.root_path = root_path
     return raw_rd
 
 
 def load_raw_resource_description(
-    source: Union[os.PathLike, str, dict, raw_nodes.URI, RawResourceDescription]
+    source: Union[dict, os.PathLike, IO, str, bytes, raw_nodes.URI]
 ) -> RawResourceDescription:
     """load a raw python representation from a BioImage.IO resource description file (RDF).
     Use `load_resource_description` for a more convenient representation.
@@ -53,12 +102,8 @@ def load_raw_resource_description(
     Returns:
         raw BioImage.IO resource
     """
-    if isinstance(source, RawResourceDescription):
-        return source
-
-    data, type_ = resolve_rdf_source_and_type(source)
-    raw_rd = spec.load_raw_resource_description(data, update_to_current_format=True)
-    raw_rd = _replace_relative_paths_for_remote_source(raw_rd, source)
+    raw_rd = spec.load_raw_resource_description(source, update_to_current_format=True)
+    raw_rd = _replace_relative_paths_for_remote_source(raw_rd, raw_rd.root_path)
     return raw_rd
 
 
@@ -186,26 +231,6 @@ def _get_tmp_package_path(raw_rd: RawResourceDescription, weights_priority_order
     return package_path
 
 
-def extract_resource_package(source: Union[os.PathLike, str, raw_nodes.URI]) -> pathlib.Path:
-    """extract a zip source to BIOIMAGEIO_CACHE_PATH"""
-    local_source = resolve_uri(source)
-    assert isinstance(local_source, pathlib.Path)
-    cache_folder = BIOIMAGEIO_CACHE_PATH / "extracted_packages"
-    cache_folder.mkdir(exist_ok=True, parents=True)
-    package_path = cache_folder / f"{local_source.stem}"
-    with ZipFile(local_source) as zf:
-        zf.extractall(package_path)
-
-    for rdf_name in ["rdf.yaml", "model.yaml", "rdf.yml", "model.yml"]:
-        rdf_path = package_path / rdf_name
-        if rdf_path.exists():
-            break
-    else:
-        raise FileNotFoundError(local_source / "rdf.yaml")
-
-    return rdf_path
-
-
 def make_zip(
     path: os.PathLike, content: Dict[str, Union[str, pathlib.Path]], *, compression: int, compression_level: int
 ) -> None:
@@ -225,53 +250,3 @@ def make_zip(
                 myzip.writestr(arc_name, file_or_str_content)
             else:
                 myzip.write(file_or_str_content, arcname=arc_name)
-
-
-def resolve_rdf_source_and_type(source: Union[os.PathLike, str, dict, raw_nodes.URI]) -> Tuple[dict, str]:
-    if isinstance(source, dict):
-        data = source
-        if ROOT_PATH not in data:
-            data[ROOT_PATH] = pathlib.Path()
-    else:
-        data = get_dict_from_yaml_source(source)
-
-    type_ = data.get("type", "model")  # todo: remove default 'model' type
-
-    return data, type_
-
-
-def get_dict_from_yaml_source(source: Union[os.PathLike, str, raw_nodes.URI, dict]) -> dict:
-    if isinstance(source, dict):
-        if ROOT_PATH not in source:
-            source[ROOT_PATH] = pathlib.Path()
-
-        return source
-    elif isinstance(source, (str, os.PathLike, raw_nodes.URI)):
-        source = resolve_local_uri(source, pathlib.Path())
-    else:
-        raise TypeError(source)
-
-    if isinstance(source, raw_nodes.URI):  # remote uri
-        local_source = _download_uri_to_local_path(source)
-        root_path = pathlib.Path()
-    else:
-        local_source = source
-        root_path = source.parent
-
-    assert isinstance(local_source, pathlib.Path)
-    if local_source.suffix == ".zip":
-        local_source = extract_resource_package(local_source)
-        root_path = local_source.parent
-
-    if local_source.suffix == ".yml":
-        warnings.warn(
-            "suffix '.yml' is not recommended and will raise a ValidationError in the future. Use '.yaml' instead "
-            "(https://yaml.org/faq.html)"
-        )
-    elif local_source.suffix != ".yaml":
-        raise ValidationError(f"invalid suffix {local_source.suffix} for source {source}")
-
-    data = yaml.load(local_source)
-    assert isinstance(data, dict)
-    data[ROOT_PATH] = root_path
-    return data

bioimageio/core/resource_io/io_.py

@@ -14,9 +14,9 @@
 from marshmallow import ValidationError
 
 from bioimageio.spec.shared import fields, raw_nodes
+from bioimageio.spec.shared.common import BIOIMAGEIO_CACHE_PATH
 from bioimageio.spec.shared.utils import GenericRawNode, GenericRawRD, NodeTransformer, NodeVisitor
 from . import nodes
-from .common import BIOIMAGEIO_CACHE_PATH
 
 GenericResolvedNode = typing.TypeVar("GenericResolvedNode", bound=nodes.Node)
 GenericNode = typing.Union[GenericRawNode, GenericResolvedNode]

bioimageio/core/resource_io/utils.py

@@ -18,10 +18,10 @@ build:
 
 requirements:
   host:
-    - python >=3.7
+    - python >=3.7,<3.10
     - pip
   run:
-    - python >=3.7
+    - python >=3.7,<3.10
     - tqdm
     - typer
     {% for dep in setup_py_data['install_requires'] %}

conda-recipe/meta.yaml

@@ -44,7 +44,7 @@ def _test_build_spec(spec_path, out_path, weight_type, tensorflow_version=None,
         cite=cite,
         root=model_spec.root_path,
         weight_type=weight_type_,
-        output_path=out_path
+        output_path=out_path,
     )
     if tensorflow_version is not None:
         kwargs["tensorflow_version"] = tensorflow_version

tests/build_spec/test_build_spec.py

@@ -18,26 +18,6 @@ def test_load_non_existing_rdf():
         load_resource_description(spec_path)
 
 
-def test_load_non_valid_rdf_name_no_suffix():
-    from bioimageio.core import load_resource_description
-
-    with NamedTemporaryFile() as f:
-        spec_path = pathlib.Path(f.name)
-
-        with pytest.raises(ValidationError):
-            load_resource_description(spec_path)
-
-
-def test_load_non_valid_rdf_name_invalid_suffix():
-    from bioimageio.core import load_resource_description
-
-    with NamedTemporaryFile(suffix=".invalid_suffix") as f:
-        spec_path = pathlib.Path(f.name)
-
-        with pytest.raises(ValidationError):
-            load_resource_description(spec_path)
-
-
 def test_load_raw_model(any_model):
     from bioimageio.core import load_raw_resource_description
 

tests/resource_io/test_load_rdf.py

@@ -6,6 +6,11 @@
 from bioimageio.core import load_resource_description
 
 
+def test_validate_model(unet2d_nuclei_broad_model):
+    ret = subprocess.run(["bioimageio", "validate", unet2d_nuclei_broad_model])
+    assert ret.returncode == 0
+
+
 def test_cli_test_model(unet2d_nuclei_broad_model):
     ret = subprocess.run(["bioimageio", "test-model", unet2d_nuclei_broad_model])
     assert ret.returncode == 0