Changed multiprocessor; Add NibabelRO IO; Changed File Manager

Author: Lars-Kraemer

.github/workflows/test_and_deploy.yml

@@ -7,13 +7,11 @@ on:
   push:
     branches:
       - main
-      - npe2
     tags:
       - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10
   pull_request:
     branches:
       - main
-      - npe2
   workflow_dispatch:
 
 jobs:

.gitignore

@@ -1,5 +1,6 @@
 # Local dev/testing scripts and folders
 playground.py
+metric.py
 stats/
 dataset_cfg/
 tests/temp/

examples/template.yaml

@@ -2,7 +2,7 @@ name: DatasetName
 
 # Define an arbitrary number of layers
 layers:
-  - name: SomeImages # unique layer name
+  SomeImages: # unique layer name
     type: image # 'image' for image data and for label data one of: 'semseg' | 'multilabel'
     path: some/path/to/data # directory
     file_type: .png # required extension incl. dot (e.g., .png | .nii.gz | .b2nd)
@@ -11,7 +11,7 @@ layers:
     channels: 3 # number of channels, required for image
     file_stack: False # True if each channel is a separate file: *_0000, *_0001, ...
 
-  - name: SomeLabels
+  SomeLabels:
     type: semseg # semseg --> Semantic Segmentation | multilabel --> Multilabel Segmentation
     path: some/path/to/data
     file_type: .png|.nii.gz|.b2nd|...

src/vidata/config_manager.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from omegaconf import DictConfig, OmegaConf
+from omegaconf import DictConfig, ListConfig, OmegaConf
 
 from vidata.file_manager import FileManager, FileManagerStacked
 from vidata.io import load_json
@@ -330,7 +330,20 @@ def __init__(self, config: dict | DictConfig | str | Path, strict: bool = True):
         self.layers = []
 
         split_cfg = self.config.get("splits", {})
-        for layer_cfg in self.config.get("layers", []):
+
+        layers_cfg = self.config.get("layers", [])
+        if isinstance(layers_cfg, (dict | DictConfig)):
+            layer_list = []
+            for layer_name, layer_cfg in layers_cfg.items():
+                layer_cfg = dict(layer_cfg)
+                layer_cfg["name"] = layer_name
+                layer_list.append(layer_cfg)
+        elif isinstance(layers_cfg, (list | ListConfig)):
+            layer_list = layers_cfg
+        else:
+            raise ValueError(f"Invalid type for layers: {type(layers_cfg)}. Must be dict or list.")
+
+        for layer_cfg in layer_list:
             layer_split = {}
 
             for k in _VALID_SPLITS:
@@ -358,3 +371,6 @@ def layer_names(self):
 
     def __len__(self):
         return len(self.layers)
+
+    def __getitem__(self, layer_name: str) -> LayerConfigManager:
+        return self.layer(layer_name)

src/vidata/file_manager/file_manager.py

@@ -23,6 +23,8 @@ class FileManager:
         Keep files whose RELATIVE path contains ANY of these substrings.
     exclude_names: list[str] | None
         Drop files whose RELATIVE path contains ANY of these substrings. (Exclude wins.)
+    recursive: bool
+        Whether to recursively search subdirectories.
     """
 
     def __init__(
@@ -32,12 +34,14 @@ def __init__(
         pattern: str | None = None,
         include_names: list[str] | None = None,
         exclude_names: list[str] | None = None,
+        recursive: bool = False,
     ):
         self.path = path
         self.file_type = file_type
         self.pattern = pattern
         self.include_names = include_names
         self.exclude_names = exclude_names
+        self.recursive = recursive
         self.collect_files()
         self.filter_files()
 
@@ -69,17 +73,31 @@ def collect_files(self):
             pattern = "*" + self.pattern
         else:
             pattern = self.pattern
-        files = list(Path(self.path).glob(pattern + self.file_type))
+
+        if self.recursive:
+            files = list(Path(self.path).rglob(pattern + self.file_type))
+        else:
+            files = list(Path(self.path).glob(pattern + self.file_type))
         self.files = natsorted(files, key=lambda p: p.name)
 
     def get_name(self, file: str | int, with_file_type=True) -> str:
+        """Just keep this for backwards compatibility"""
+        return self.name_from_path(file, with_file_type)
+
+    def name_from_path(self, file: str | int, include_ext: bool = True) -> str:
         if isinstance(file, int):
             file = str(self.files[file])
         name = str(Path(file).relative_to(self.path))
-        if not with_file_type:
+        if not include_ext:
             name = name.replace(self.file_type, "")
         return name
 
+    def path_from_name(self, name: str | Path, include_ext=True):
+        rel = Path(name)
+        if include_ext and rel.suffix != self.file_type:
+            rel = rel.with_suffix(self.file_type)
+        return (self.path / rel).resolve()
+
     def __getitem__(self, item: int):
         return self.files[item]
 

src/vidata/io/__init__.py

@@ -1,8 +1,8 @@
-# isort: skip_file
+# isort: skip_file # order matters, first ones in list are the defaults
 # ruff: noqa: I001, I002  # disable Ruff's import-sorting checks for this file
 from .image_io import load_image, save_image
 from .sitk_io import load_sitk, save_sitk
-from .nib_io import load_nib, save_nib
+from .nib_io import load_nib, save_nib, load_nibRO, save_nibRO
 from .tif_io import load_tif, save_tif
 from .blosc2_io import load_blosc2, load_blosc2pkl, save_blosc2, save_blosc2pkl
 from .numpy_io import load_npy, load_npz, save_npy, save_npz
@@ -16,6 +16,8 @@
     "save_sitk",
     "load_nib",
     "save_nib",
+    "load_nibRO",
+    "save_nibRO",
     "load_blosc2",
     "save_blosc2",
     "load_blosc2pkl",

src/vidata/io/blosc2_io.py

@@ -6,6 +6,7 @@
 
 import math
 from copy import deepcopy
+from pathlib import Path
 from typing import Union
 
 import blosc2
@@ -19,7 +20,7 @@
 @register_writer("mask", ".b2nd", backend="blosc2")
 def save_blosc2(
     data: np.ndarray,
-    file: str,
+    file: str | Path,
     patch_size: Union[tuple[int, int], tuple[int, int, int]] | None = None,
     clevel: int = 8,
     nthreads: int = 8,
@@ -42,11 +43,7 @@ def save_blosc2(
         _is_float = np.issubdtype(data.dtype.type, np.floating)
         _is_2d = data.ndim == 2
 
-        # if _is_2d:
         base_patch_size = (512 if _is_float else 1024) if _is_2d else (64 if _is_float else 96)
-        # else:
-        #    base_patch_size = 64 if _is_float else 96
-
         patch_size = tuple([min(s, base_patch_size) for s in data.shape])
 
     blocks, chunks = comp_blosc2_params(data.shape, patch_size, data.itemsize)
@@ -60,12 +57,12 @@ def save_blosc2(
         mmap_mode="w+",
         meta=metadata,
     )
-    return [file]
+    return [str(file)]
 
 
 @register_loader("image", ".b2nd", backend="blosc2")
 @register_loader("mask", ".b2nd", backend="blosc2")
-def load_blosc2(file: str, nthreads: int = 1) -> tuple[blosc2.NDArray, dict]:
+def load_blosc2(file: str | Path, nthreads: int = 1) -> tuple[blosc2.NDArray, dict]:
     """Reads a Blosc2 file and returns the data and metadata.
 
     Args:
@@ -86,7 +83,7 @@ def load_blosc2(file: str, nthreads: int = 1) -> tuple[blosc2.NDArray, dict]:
 @register_writer("mask", ".b2nd", backend="blosc2pkl")
 def save_blosc2pkl(
     data: np.ndarray,
-    file: str,
+    file: str | Path,
     patch_size: Union[tuple[int, int], tuple[int, int, int]] | None = None,
     clevel: int = 8,
     nthreads: int = 8,
@@ -112,7 +109,7 @@ def save_blosc2pkl(
 
 @register_loader("image", ".b2nd", backend="blosc2pkl")
 @register_loader("mask", ".b2nd", backend="blosc2pkl")
-def load_blosc2pkl(file: str, nthreads: int = 1) -> tuple[blosc2.NDArray, dict]:
+def load_blosc2pkl(file: str | Path, nthreads: int = 1) -> tuple[blosc2.NDArray, dict]:
     """Reads a Blosc2 file and returns the data and metadata.
 
     Args:

src/vidata/io/image_io.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import imageio.v3 as iio
 import numpy as np
 
@@ -6,13 +8,13 @@
 
 @register_loader("image", ".png", ".jpg", ".jpeg", ".bmp", backend="imageio")
 @register_loader("mask", ".png", ".bmp", backend="imageio")
-def load_image(file: str):
+def load_image(file: str | Path):
     data = iio.imread(file)  # automatically handles RGB, grayscale, masks
     return data, {}
 
 
 @register_writer("image", ".png", ".jpg", ".jpeg", ".bmp", backend="imageio")
 @register_writer("mask", ".png", ".bmp", backend="imageio")
-def save_image(data: np.ndarray, file: str) -> list[str]:
+def save_image(data: np.ndarray, file: str | Path) -> list[str]:
     iio.imwrite(file, data)
-    return [file]
+    return [str(file)]

src/vidata/io/json_io.py

@@ -1,8 +1,9 @@
 import json
+from pathlib import Path
 from typing import Any
 
 
-def load_json(json_file: str) -> Any:
+def load_json(json_file: str | Path) -> Any:
     """Load data from a JSON file.
 
     Args:
@@ -16,7 +17,7 @@ def load_json(json_file: str) -> Any:
     return data
 
 
-def save_json(data: Any, json_file: str, indent: int = 4) -> None:
+def save_json(data: Any, json_file: str | Path, indent: int = 4) -> None:
     """Write data to a JSON file.
 
     Args: