search for datasets also in the path with the long mag name (#243)

Author: rschwanhold

tests/test_dataset.py

@@ -1,7 +1,7 @@
 import filecmp
 import json
 import os
-from os.path import dirname
+from os.path import dirname, join
 import pytest
 
 import numpy as np
@@ -147,7 +147,7 @@ def test_create_wk_dataset_with_explicit_header_fields():
     delete_dir("./testoutput/wk_dataset_advanced")
 
     ds = WKDataset.create("./testoutput/wk_dataset_advanced", scale=(1, 1, 1))
-    ds.add_layer("color", "color", dtype_per_layer="uint48", num_channels=3)
+    ds.add_layer("color", Layer.COLOR_TYPE, dtype_per_layer="uint48", num_channels=3)
 
     ds.get_layer("color").add_mag("1", block_len=64, file_len=64)
     ds.get_layer("color").add_mag("2-2-1")
@@ -696,7 +696,9 @@ def test_tiled_tiff_read_and_write_multichannel():
         pattern="{xxx}_{yyy}_{zzz}.tif",
     )
 
-    mag = tiled_tiff_ds.add_layer("color", "color", num_channels=3).add_mag("1")
+    mag = tiled_tiff_ds.add_layer("color", Layer.COLOR_TYPE, num_channels=3).add_mag(
+        "1"
+    )
 
     data = get_multichanneled_data(np.uint8)
 
@@ -715,7 +717,7 @@ def test_tiled_tiff_read_and_write():
         pattern="{xxx}_{yyy}_{zzz}.tif",
     )
 
-    mag = tiled_tiff_ds.add_layer("color", "color").add_mag("1")
+    mag = tiled_tiff_ds.add_layer("color", Layer.COLOR_TYPE).add_mag("1")
 
     data = np.zeros((250, 200, 10), dtype=np.uint8)
     for h in range(10):
@@ -812,11 +814,11 @@ def test_largest_segment_id_requirement():
     ds = WKDataset.create(path, scale=(10, 10, 10))
 
     with pytest.raises(AssertionError):
-        ds.add_layer("segmentation", "segmentation")
+        ds.add_layer("segmentation", Layer.SEGMENTATION_TYPE)
 
     largest_segment_id = 10
     ds.add_layer(
-        "segmentation", "segmentation", largest_segment_id=largest_segment_id
+        "segmentation", Layer.SEGMENTATION_TYPE, largest_segment_id=largest_segment_id
     ).add_mag(Mag(1))
 
     ds = WKDataset(path)
@@ -1018,7 +1020,7 @@ def test_view_write_without_open():
     delete_dir("./testoutput/wk_dataset_write_without_open")
 
     ds = WKDataset.create("./testoutput/wk_dataset_write_without_open", scale=(1, 1, 1))
-    ds.add_layer("color", "color")
+    ds.add_layer("color", Layer.COLOR_TYPE)
 
     ds.get_layer("color").add_mag("1")
 
@@ -1214,7 +1216,7 @@ def test_view_offsets():
     delete_dir("./testoutput/wk_offset_tests")
 
     ds = WKDataset.create("./testoutput/wk_offset_tests", scale=(1, 1, 1))
-    mag = ds.add_layer("color", "color").add_mag("1")
+    mag = ds.add_layer("color", Layer.COLOR_TYPE).add_mag("1")
 
     # The dataset is new -> no data has been written.
     # Therefore, the size of the bounding box in the properties.json is (0, 0, 0)
@@ -1288,12 +1290,14 @@ def test_adding_layer_with_invalid_dtype_per_layer():
     ds = WKDataset.create("./testoutput/invalid_dtype", scale=(1, 1, 1))
     with pytest.raises(TypeError):
         # this would lead to a dtype_per_channel of "uint10", but that is not a valid dtype
-        ds.add_layer("color", "color", dtype_per_layer="uint30", num_channels=3)
+        ds.add_layer(
+            "color", Layer.COLOR_TYPE, dtype_per_layer="uint30", num_channels=3
+        )
     with pytest.raises(TypeError):
         # "int" is interpreted as "int64", but 64 bit cannot be split into 3 channels
-        ds.add_layer("color", "color", dtype_per_layer="int", num_channels=3)
+        ds.add_layer("color", Layer.COLOR_TYPE, dtype_per_layer="int", num_channels=3)
     ds.add_layer(
-        "color", "color", dtype_per_layer="int", num_channels=4
+        "color", Layer.COLOR_TYPE, dtype_per_layer="int", num_channels=4
     )  # "int"/"int64" works with 4 channels
 
 
@@ -1485,3 +1489,39 @@ def test_add_symlink_layer():
 
     assert np.array_equal(mag.read(size=(10, 10, 10)), write_data)
     assert np.array_equal(original_mag.read(size=(10, 10, 10)), write_data)
+
+
+def test_search_dataset_also_for_long_layer_name():
+    delete_dir("./testoutput/long_layer_name")
+
+    ds = WKDataset.create("./testoutput/long_layer_name", scale=(1, 1, 1))
+    mag = ds.add_layer("color", Layer.COLOR_TYPE).add_mag("2")
+
+    assert mag.name == "2"
+    short_mag_file_path = join(ds.path, "color", Mag(mag.name).to_layer_name())
+    long_mag_file_path = join(ds.path, "color", Mag(mag.name).to_long_layer_name())
+
+    assert os.path.exists(short_mag_file_path)
+    assert not os.path.exists(long_mag_file_path)
+
+    write_data = (np.random.rand(10, 10, 10) * 255).astype(np.uint8)
+    mag.write(write_data, offset=(10, 10, 10))
+
+    assert np.array_equal(
+        mag.read(offset=(10, 10, 10), size=(10, 10, 10)), np.expand_dims(write_data, 0)
+    )
+
+    # rename the path from "long_layer_name/color/2" to "long_layer_name/color/2-2-2"
+    os.rename(short_mag_file_path, long_mag_file_path)
+
+    with pytest.raises(WKWException):
+        # the dataset has to be reopened to notice the changed directory
+        mag.read(offset=(10, 10, 10), size=(10, 10, 10))
+
+    # when opening the dataset, it searches both for the long and the short path
+    layer = WKDataset("./testoutput/long_layer_name").get_layer("color")
+    mag = layer.get_mag("2")
+    assert np.array_equal(
+        mag.read(offset=(10, 10, 10), size=(10, 10, 10)), np.expand_dims(write_data, 0)
+    )
+    layer.delete_mag("2")

wkcuber/api/Layer.py

@@ -12,6 +12,7 @@
     WKMagDataset,
     TiffMagDataset,
     TiledTiffMagDataset,
+    find_mag_path_on_disk,
 )
 from wkcuber.mag import Mag
 from wkcuber.utils import DEFAULT_WKW_FILE_LEN
@@ -48,7 +49,7 @@ def delete_mag(self, mag):
         del self.mags[mag]
         self.dataset.properties._delete_mag(self.name, mag)
         # delete files on disk
-        full_path = join(self.dataset.path, self.name, mag)
+        full_path = find_mag_path_on_disk(self.dataset.path, self.name, mag)
         rmtree(full_path)
 
     def _create_dir_for_mag(self, mag):
@@ -138,7 +139,9 @@ def setup_mag(self, mag):
 
         self._assert_mag_does_not_exist_yet(mag)
 
-        with wkw.Dataset.open(join(self.dataset.path, self.name, mag)) as wkw_dataset:
+        with wkw.Dataset.open(
+            find_mag_path_on_disk(self.dataset.path, self.name, mag)
+        ) as wkw_dataset:
             wk_header = wkw_dataset.header
 
         self.mags[mag] = WKMagDataset(

wkcuber/api/MagDataset.py

@@ -1,3 +1,4 @@
+import os
 from os.path import join
 
 from wkw import wkw
@@ -9,6 +10,16 @@
 from wkcuber.mag import Mag
 
 
+def find_mag_path_on_disk(dataset_path: str, layer_name: str, mag_name: str):
+    mag = Mag(mag_name)
+    short_mag_file_path = join(dataset_path, layer_name, mag.to_layer_name())
+    long_mag_file_path = join(dataset_path, layer_name, mag.to_long_layer_name())
+    if os.path.exists(short_mag_file_path):
+        return short_mag_file_path
+    else:
+        return long_mag_file_path
+
+
 class MagDataset:
     def __init__(self, layer, name):
         self.layer = layer
@@ -99,7 +110,9 @@ def get_view(self, size=None, offset=None, is_bounded=True):
                         f"Use is_bounded=False if you intend to write outside out the existing bounding box."
                     )
 
-        mag_file_path = join(self.layer.dataset.path, self.layer.name, self.name)
+        mag_file_path = find_mag_path_on_disk(
+            self.layer.dataset.path, self.layer.name, self.name
+        )
         return self._get_view_type()(
             mag_file_path, self.header, size, offset, is_bounded
         )

wkcuber/api/Properties/DatasetProperties.py

@@ -5,7 +5,7 @@
     SegmentationLayerProperties,
     LayerProperties,
 )
-from wkcuber.api.Properties.ResolutionProperties import WkResolution, TiffResolution
+from wkcuber.api.Properties.ResolutionProperties import WkResolution, Resolution
 from wkcuber.mag import Mag
 
 
@@ -155,11 +155,11 @@ def _from_json(cls, path) -> Properties:
             for layer in data["dataLayers"]:
                 if layer["category"] == Layer.SEGMENTATION_TYPE:
                     data_layers[layer["name"]] = SegmentationLayerProperties._from_json(
-                        layer, TiffResolution, path
+                        layer, Resolution, path
                     )
                 else:
                     data_layers[layer["name"]] = LayerProperties._from_json(
-                        layer, TiffResolution, path
+                        layer, Resolution, path
                     )
 
             return cls(
@@ -196,5 +196,5 @@ def _add_mag(self, layer_name, mag):
                 for res in self.data_layers[layer_name].wkw_magnifications
             ]
         ):
-            self.data_layers[layer_name]._add_resolution(TiffResolution(mag))
+            self.data_layers[layer_name]._add_resolution(Resolution(mag))
             self._export_as_json()

wkcuber/api/Properties/LayerProperties.py

@@ -106,7 +106,11 @@ def _add_resolution(self, resolution):
         self._wkw_magnifications.append(resolution)
 
     def _delete_resolution(self, resolution):
-        self._wkw_magnifications.delete(resolution)
+        resolutions_to_delete = [
+            res for res in self._wkw_magnifications if res.mag == Mag(resolution)
+        ]
+        for res in resolutions_to_delete:
+            self._wkw_magnifications.remove(res)
 
     def get_bounding_box(self) -> BoundingBox:
 

wkcuber/api/Properties/ResolutionProperties.py

@@ -2,15 +2,6 @@
 
 
 class Resolution:
-    def _to_json(self) -> dict:
-        pass
-
-    @classmethod
-    def _from_json(cls, json_data):
-        pass
-
-
-class TiffResolution(Resolution):
     def __init__(self, mag):
         self._mag = Mag(mag)
 
@@ -28,7 +19,7 @@ def mag(self) -> Mag:
 
 class WkResolution(Resolution):
     def __init__(self, mag, cube_length):
-        self._mag = Mag(mag)
+        super().__init__(mag)
         self._cube_length = cube_length
 
     def _to_json(self) -> dict: