|
| 1 | +import traceback |
| 2 | +import warnings |
| 3 | +from pathlib import Path |
| 4 | +from typing import List, Optional, Union |
| 5 | + |
| 6 | +import numpy as np |
| 7 | +import xarray as xr |
| 8 | + |
| 9 | +from bioimageio.core import load_resource_description |
| 10 | +from bioimageio.core.prediction import predict |
| 11 | +from bioimageio.core.prediction_pipeline import create_prediction_pipeline |
| 12 | +from bioimageio.core.resource_io.nodes import Model, ResourceDescription, URI |
| 13 | +from bioimageio.spec.model.raw_nodes import WeightsFormat |
| 14 | +from bioimageio.spec.shared.raw_nodes import ResourceDescription as RawResourceDescription |
| 15 | + |
| 16 | + |
| 17 | +def test_model( |
| 18 | + model_rdf: Union[URI, Path, str], |
| 19 | + weight_format: Optional[WeightsFormat] = None, |
| 20 | + devices: Optional[List[str]] = None, |
| 21 | + decimal: int = 4, |
| 22 | +) -> dict: |
| 23 | + """Test whether the test output(s) of a model can be reproduced. |
| 24 | +
|
| 25 | + Returns summary dict with "error" and "traceback" key; summary["error"] is None if no errors were encountered. |
| 26 | + """ |
| 27 | + model = load_resource_description(model_rdf) |
| 28 | + if isinstance(model, Model): |
| 29 | + return test_resource(model, weight_format=weight_format, devices=devices, decimal=decimal) |
| 30 | + else: |
| 31 | + return {"error": f"Expected RDF type Model, got {type(model)} instead.", "traceback": None} |
| 32 | + |
| 33 | + |
| 34 | +def test_resource( |
| 35 | + model_rdf: Union[RawResourceDescription, ResourceDescription, URI, Path, str], |
| 36 | + *, |
| 37 | + weight_format: Optional[WeightsFormat] = None, |
| 38 | + devices: Optional[List[str]] = None, |
| 39 | + decimal: int = 4, |
| 40 | +): |
| 41 | + """Test RDF dynamically |
| 42 | +
|
| 43 | + Returns summary dict with "error" and "traceback" key; summary["error"] is None if no errors were encountered. |
| 44 | + """ |
| 45 | + error: Optional[str] = None |
| 46 | + tb: Optional = None |
| 47 | + |
| 48 | + try: |
| 49 | + model = load_resource_description(model_rdf) |
| 50 | + except Exception as e: |
| 51 | + error = str(e) |
| 52 | + tb = traceback.format_tb(e.__traceback__) |
| 53 | + else: |
| 54 | + if isinstance(model, Model): |
| 55 | + try: |
| 56 | + prediction_pipeline = create_prediction_pipeline( |
| 57 | + bioimageio_model=model, devices=devices, weight_format=weight_format |
| 58 | + ) |
| 59 | + inputs = [np.load(str(in_path)) for in_path in model.test_inputs] |
| 60 | + results = predict(prediction_pipeline, inputs) |
| 61 | + if isinstance(results, (np.ndarray, xr.DataArray)): |
| 62 | + results = [results] |
| 63 | + |
| 64 | + expected = [np.load(str(out_path)) for out_path in model.test_outputs] |
| 65 | + if len(results) != len(expected): |
| 66 | + error = ( |
| 67 | + f"Number of outputs and number of expected outputs disagree: {len(results)} != {len(expected)}" |
| 68 | + ) |
| 69 | + else: |
| 70 | + for res, exp in zip(results, expected): |
| 71 | + try: |
| 72 | + np.testing.assert_array_almost_equal(res, exp, decimal=decimal) |
| 73 | + except AssertionError as e: |
| 74 | + error = f"Output and expected output disagree:\n {e}" |
| 75 | + except Exception as e: |
| 76 | + error = str(e) |
| 77 | + tb = traceback.format_tb(e.__traceback__) |
| 78 | + |
| 79 | + # todo: add tests for non-model resources |
| 80 | + |
| 81 | + return {"error": error, "traceback": tb} |
0 commit comments