Support adding a layer via symlink (#240)

* support adding layers with symlinks

* reformat code

Author: rschwanhold

tests/test_dataset.py

@@ -1,5 +1,6 @@
 import filecmp
 import json
+import os
 from os.path import dirname
 import pytest
 
@@ -1276,3 +1277,31 @@ def test_view_offsets():
         raise Exception(expected_error_msg)
     except AssertionError:
         pass
+
+
+def test_add_symlink_layer():
+    delete_dir("./testoutput/wk_dataset_with_symlink")
+    delete_dir("./testoutput/simple_wk_dataset_copy")
+    copytree("./testdata/simple_wk_dataset/", "./testoutput/simple_wk_dataset_copy/")
+
+    original_mag = (
+        WKDataset("./testoutput/simple_wk_dataset_copy/")
+        .get_layer("color")
+        .get_mag("1")
+    )
+
+    ds = WKDataset.create("./testoutput/wk_dataset_with_symlink", scale=(1, 1, 1))
+    symlink_layer = ds.add_symlink_layer("./testoutput/simple_wk_dataset_copy/color/")
+    mag = symlink_layer.get_mag("1")
+
+    assert path.exists("./testoutput/wk_dataset_with_symlink/color/1")
+
+    assert len(ds.properties.data_layers) == 1
+    assert len(ds.properties.data_layers["color"].wkw_magnifications) == 1
+
+    # write data in symlink layer
+    write_data = (np.random.rand(3, 10, 10, 10) * 255).astype(np.uint8)
+    mag.write(write_data)
+
+    assert np.array_equal(mag.read((10, 10, 10)), write_data)
+    assert np.array_equal(original_mag.read((10, 10, 10)), write_data)

wkcuber/api/Dataset.py

@@ -129,6 +129,32 @@ def delete_layer(self, layer_name):
         # delete files on disk
         rmtree(join(self.path, layer_name))
 
+    def add_symlink_layer(self, foreign_layer_path) -> Layer:
+        foreign_layer_path = os.path.abspath(foreign_layer_path)
+        layer_name = os.path.basename(os.path.normpath(foreign_layer_path))
+        if layer_name in self.layers.keys():
+            raise IndexError(
+                f"Cannot create symlink to {foreign_layer_path}. This dataset already has a layer called {layer_name}."
+            )
+
+        os.symlink(foreign_layer_path, join(self.path, layer_name))
+
+        # copy the properties of the layer into the properties of this dataset
+        layer_properties = self._get_type()(
+            Path(foreign_layer_path).parent
+        ).properties.data_layers[layer_name]
+        self.properties.data_layers[layer_name] = layer_properties
+        self.properties._export_as_json()
+
+        self.layers[layer_name] = self._create_layer(
+            layer_name,
+            np.dtype(layer_properties.element_class),
+            layer_properties.num_channels,
+        )
+        for resolution in layer_properties.wkw_magnifications:
+            self.layers[layer_name].setup_mag(resolution.mag.to_layer_name())
+        return self.layers[layer_name]
+
     def get_view(self, layer_name, mag, size, offset=None, is_bounded=True) -> View:
         layer = self.get_layer(layer_name)
         mag_ds = layer.get_mag(mag)
@@ -142,6 +168,10 @@ def _create_layer(self, layer_name, dtype, num_channels) -> Layer:
     def _get_properties_type(self):
         pass
 
+    @abstractmethod
+    def _get_type(self):
+        pass
+
 
 class WKDataset(AbstractDataset):
     @classmethod
@@ -177,6 +207,9 @@ def _create_layer(self, layer_name, dtype, num_channels) -> Layer:
     def _get_properties_type(self):
         return WKProperties
 
+    def _get_type(self):
+        return WKDataset
+
 
 class TiffDataset(AbstractDataset):
     @classmethod
@@ -226,6 +259,9 @@ def _create_layer(self, layer_name, dtype, num_channels) -> Layer:
     def _get_properties_type(self):
         return TiffProperties
 
+    def _get_type(self):
+        return TiffDataset
+
 
 class TiledTiffDataset(AbstractDataset):
     @classmethod
@@ -280,6 +316,9 @@ def _create_layer(self, layer_name, dtype, num_channels) -> Layer:
     def _get_properties_type(self):
         return TiffProperties
 
+    def _get_type(self):
+        return TiledTiffDataset
+
 
 def validate_pattern(pattern):
     assert pattern.count("{") > 0 and pattern.count("}") > 0, (