Strict Flag for Config Manager

Author: Lars-Kraemer

README.md

@@ -1,8 +1,8 @@
 # ViData: Vision Data Management, Processing and Analysis
 
-> A unified Python toolkit for managing, processing, and analyzing vision datasets — from raw data to task specific analytics.
+> A unified Python toolkit for managing, processing, and analyzing vision datasets, from raw data to task specific analytics.
 >
-> streamlines the entire data pipeline for computer vision and medical imaging:
+> Streamlines the entire data pipeline for computer vision and medical imaging:
 >
 > - Load and save 2D/3D data across common formats with a consistent API.
 > - Organize datasets using flexible file managers and YAML-based configs.
@@ -52,6 +52,7 @@ pip install napari-data-inspection
 - Use `load_xxx` / `save_xxx` for all supported formats.
 - **Images/arrays:** PNG/JPG/TIFF (`imageio`, `tifffile`), NIfTI/NRRD/MHA (`sitk`, `nibabel`), Blosc2, NumPy (`.npy`, `.npz`).
 - **Configs/metadata:** JSON, YAML, Pickle, TXT.
+- **Extendable** by custom _load_ and _save_ functions.
 - Functions always follow the same pattern:
 
 ```python
@@ -153,6 +154,52 @@ obj = load_txt("config.txt")
 save_txt(obj, "out.txt")
 ```
 
+### Custom Load and Save Functions
+
+- Register a reader / writer with a decorator.
+- Reader must return (numpy_array, metadata_dict).
+- Writer must return list\[str\] of the file(s) it wrote.
+- Registration happens at import time—make sure this module is imported (e.g., from your package’s __init__.py).
+- See [here](https://github.com/MIC-DKFZ/ViData/blob/main/src/vidata/io/image_io.py) for an example.
+
+```py
+# custom_io_template.py  — fill in the TODOs and import this module somewhere at startup.
+import numpy as np
+from typing import Tuple, Dict, List
+
+# TODO: import your backend library (e.g., imageio, tifffile, nibabel, SimpleITK, ...)
+# import imageio.v3 as iio
+
+from vidata.registry import register_loader, register_writer
+
+# --------------------------- READER ------------------------------------------
+# Replace file extension and backend name to your custom function
+@register_loader("image", ".png", ".jpg", ".jpeg", ".bmp", backend="imageio")  # To Register Image Loading
+@register_loader("mask", ".png", ".bmp", backend="imageio") # To Register Label Loading
+def load_custom(file: str) -> tuple[np.ndarray, dict]:
+    """
+    Load a file and return (data, metadata).
+    metadata can be empty or include keys like: spacing, origin, direction, shear, dtype, etc.
+    """
+    # data = iio.imread(file)  # example for imageio
+    data = ...  # TODO: replace
+    meta = {}   # TODO: replace
+    return data, meta
+    # return meta # for metadata files like json or yaml
+
+# --------------------------- WRITER ------------------------------------------
+# Replace file extension and backend name to your custom function
+@register_writer("image", ".png", ".jpg", ".jpeg", ".bmp", backend="imageio") # To Register Image Writer
+@register_writer("mask", ".png", ".bmp", backend="imageio") # To Register Label Writer
+def save_custom(data: np.ndarray, file: str) -> list[str]:
+    """
+    Save array to `file`. Return all created paths (include sidecars if any).
+    """
+    # TODO: write using your backend
+    # iio.imwrite(file, data)
+    return [file]
+```
+
 </details>
 
 # Loaders/Writers
@@ -579,8 +626,7 @@ This repository is developed and maintained by the Applied Computer Vision Lab (
 of [Helmholtz Imaging](https://www.helmholtz-imaging.de/) and the
 [Division of Medical Image Computing](https://www.dkfz.de/en/medical-image-computing) at DKFZ.
 
-This [napari] plugin was generated with [copier] using the [napari-plugin-template].
+This repository was generated with [copier] using the [napari-plugin-template].
 
 [copier]: https://copier.readthedocs.io/en/stable/
-[napari]: https://github.com/napari/napari
 [napari-plugin-template]: https://github.com/napari/napari-plugin-template

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "vidata"
 dynamic = ["version"]
-description = "Data IO"
+description = "A unified Python toolkit for managing, processing, and analyzing vision datasets, from raw data to task specific analytics."
 readme = "README.md"
 license = {file = "LICENSE"}
 authors = [

src/vidata/config_manager.py

@@ -50,7 +50,7 @@
 
 
 class LayerConfigManager:
-    def __init__(self, layer_config, split_config=None, splits_file=None):
+    def __init__(self, layer_config, split_config=None, splits_file=None, strict: bool = True):
         self.layer_config = layer_config
         self.split_config = split_config if split_config is not None else {}
 
@@ -111,13 +111,13 @@ def __init__(self, layer_config, split_config=None, splits_file=None):
                 f"Unknown layer type '{self.layer_config['type']}' for layer '{self.layer_config.get('name')}'"
             )
 
-        if errs != []:
+        if errs != [] and strict:
             for err in errs:
                 print(err)
             raise ValueError(f"Config for layer '{self.layer_config.get('name')}' is invalid")
 
         # Check if splits_file is valid
-        if splits_file is not None and not Path(splits_file).exists():
+        if splits_file is not None and not Path(splits_file).exists() and strict:
             raise FileNotFoundError(f"splits_file not found: {splits_file}")
         self.splits_file = splits_file
 
@@ -322,7 +322,7 @@ def task_manager(self) -> TaskManager:
 
 
 class ConfigManager:
-    def __init__(self, config: dict | DictConfig | str | Path):
+    def __init__(self, config: dict | DictConfig | str | Path, strict: bool = True):
         if isinstance(config, (str | Path)):
             self.config = OmegaConf.load(config)
         else:
@@ -337,7 +337,9 @@ def __init__(self, config: dict | DictConfig | str | Path):
                 if split_cfg.get(k, {}) is not None and layer_cfg["name"] in split_cfg.get(k, {}):
                     ovrds = split_cfg[k][layer_cfg["name"]]
                     layer_split[k] = ovrds if ovrds is not None else {}
-            lcm = LayerConfigManager(layer_cfg, layer_split, split_cfg.get("splits_file"))
+            lcm = LayerConfigManager(
+                layer_cfg, layer_split, split_cfg.get("splits_file"), strict=strict
+            )
 
             self.layers.append(lcm)
 

src/vidata/io/image_io.py

@@ -16,56 +16,3 @@ def load_image(file: str):
 def save_image(data: np.ndarray, file: str) -> list[str]:
     iio.imwrite(file, data)
     return [file]
-
-
-# @register_loader("image", ".png", ".jpg", ".jpeg", ".bmp")
-# def load_rgb(file: str):
-#     """Loads an image from file using OpenCV.
-#
-#     Args:
-#         file (Union[str, bytes]): Path to the image file.
-#
-#     Returns:
-#         np.ndarray: Loaded image as a NumPy array.
-#     """
-#     data = cv2.imread(file)
-#     data = cv2.cvtColor(data, cv2.COLOR_BGR2RGB)
-#
-#     return data, None
-#
-# # TODO How to handle RGB vs Grayscale data
-# @register_writer("image", ".png", ".jpg", ".jpeg", ".bmp")
-# def save_rgb(data: np.ndarray, file: str):
-#     """Saves an image to file using OpenCV.
-#
-#     Args:
-#         data (np.ndarray): Image array to save.
-#         file (Union[str, bytes]): Destination file path.
-#     """
-#     data = cv2.cvtColor(data, cv2.COLOR_RGB2BGR)
-#     cv2.imwrite(file, data)
-#
-#
-# @register_loader("mask", ".png", ".bmp")
-# def load_mask(file: str):
-#     """Loads an image from file using OpenCV.
-#
-#     Args:
-#         file (Union[str, bytes]): Path to the image file.
-#
-#     Returns:
-#         np.ndarray: Loaded image as a NumPy array.
-#     """
-#     data = cv2.imread(file, cv2.IMREAD_UNCHANGED)
-#     return data, None
-#
-#
-# @register_writer("image", ".png", ".bmp")
-# def save_mask(data: np.ndarray, file: str):
-#     """Saves an image to file using OpenCV.
-#
-#     Args:
-#         data (np.ndarray): Image array to save.
-#         file (Union[str, bytes]): Destination file path.
-#     """
-#     cv2.imwrite(file, data)