Support tiled tiffs in high level dataset API (#185)

* only log downsampling for each MB ob precessed data

* log every gigabyte instead of every megabyte

* implement PR feedback

* WIP: implement high level dataset api

* WIP: implement TiffMag

* implement test for dataset api

* refactor high level dataset api

* add test data for WKDataset

* improve high level dataset api and add more tests

* use seed in tests to achieve deterministic behaviour

* implement a test for writing out of bounds with a wk_slice

* add possibility to define pattern for TiffDatasets

* implement tiled tiff

* improve quality of dataset tests

* make naming schema of test files consistent

* implement PR feedback

* rename tiffs in testdata

* Improve error message and reformat code

* add type annotation

* rename Slice to View

* remove comments and reformat code

* add scikit-image (which is needed for skimage) as dependency

* update version of scikit-image

* fix relative paths in tests

* add possibility to get data of specific tile

* reformat code

* add test for png images

* reformat code

* make TiledTiffDataset a seperate class

* add support for opening datasets which do not have num_channels in properties.json

* add test for advanced pattern

* use more efficient method to detect tile_ranges

* rename file path of test case

* support largestSegmentationId in properties

* restructure properties

* add dataType attribute to properties

* implement PR feedback

* fix writing data to an unopened dataset and support different orders of the dimensions with patterns

* Adjust docstring of method

Co-Authored-By: Philipp Otto <[email protected]>

* Support tiled tiffs in high level dataset API (#178)

* add possibility to define pattern for TiffDatasets

* implement tiled tiff

* add possibility to get data of specific tile

* add test for png images

* reformat code

* make TiledTiffDataset a seperate class

* add support for opening datasets which do not have num_channels in properties.json

* add test for advanced pattern

* use more efficient method to detect tile_ranges

* rename file path of test case

* support largestSegmentationId in properties

* restructure properties

* add dataType attribute to properties

* implement PR feedback

* fix writing data to an unopened dataset and support different orders of the dimensions with patterns

* Adjust docstring of method

Co-Authored-By: Philipp Otto <[email protected]>

Co-authored-by: Philipp Otto <[email protected]>

Co-authored-by: Philipp Otto <[email protected]>

Author: rschwanhold

testdata/complex_property_ds/color/1/header.wkw

@@ -0,0 +1,70 @@
+{
+  "id" : {
+    "name" : "ROI2017_wkw",
+    "team" : "Connectomics department"
+  },
+  "dataLayers" : [ {
+    "name" : "color",
+    "category" : "color",
+    "boundingBox" : {
+      "topLeft" : [ 0, 0, 0 ],
+      "width" : 1024,
+      "height" : 1024,
+      "depth" : 1024
+    },
+    "wkwResolutions" : [ {
+      "resolution" : [1, 1, 1],
+      "cubeLength" : 1024
+    }, {
+      "resolution" : [16, 16, 16],
+      "cubeLength" : 64
+    }, {
+      "resolution" : [2, 2, 2],
+      "cubeLength" : 512
+    }, {
+      "resolution" : [32, 32, 32],
+      "cubeLength" : 32
+    }, {
+      "resolution" : [4, 4, 4],
+      "cubeLength" : 256
+    }, {
+      "resolution" : [8, 8, 8],
+      "cubeLength" : 128
+    } ],
+    "elementClass" : "uint8",
+    "dataFormat" : "wkw"
+  }, {
+    "name" : "segmentation",
+    "boundingBox" : {
+      "topLeft" : [ 0, 0, 0 ],
+      "width" : 1024,
+      "height" : 1024,
+      "depth" : 1024
+    },
+    "wkwResolutions" : [ {
+      "resolution" : [1, 1, 1],
+      "cubeLength" : 1024
+    }, {
+      "resolution" : [16, 16, 16],
+      "cubeLength" : 64
+    }, {
+      "resolution" : [2, 2, 2],
+      "cubeLength" : 512
+    }, {
+      "resolution" : [32, 32, 32],
+      "cubeLength" : 32
+    }, {
+      "resolution" : [4, 4, 4],
+      "cubeLength" : 256
+    }, {
+      "resolution" : [8, 8, 8],
+      "cubeLength" : 128
+    } ],
+    "elementClass" : "uint32",
+    "mappings" : [ "larger5um1", "axons", "astrocyte-ge-7", "astrocyte", "mitochondria", "astrocyte-full" ],
+    "largestSegmentId" : 1000000000,
+    "category" : "segmentation",
+    "dataFormat" : "wkw"
+  } ],
+  "scale" : [ 11.239999771118164, 11.239999771118164, 28 ]
+}

testdata/complex_property_ds/datasource-properties.json

@@ -0,0 +1,26 @@
+{
+  "id" : {
+    "name" : "ROI2017_wkw",
+    "team" : "Connectomics department"
+  },
+  "dataLayers" : [ {
+    "name" : "color",
+    "category" : "color",
+    "boundingBox" : {
+      "topLeft" : [ 0, 0, 0 ],
+      "width" : 1024,
+      "height" : 1024,
+      "depth" : 1024
+    },
+    "wkwResolutions" : [ {
+      "resolution" : 1,
+      "cubeLength" : 1024
+    }, {
+      "resolution" : 2,
+      "cubeLength" : 512
+    }],
+    "elementClass" : "uint8",
+    "dataFormat" : "wkw"
+  }],
+  "scale" : [ 11.239999771118164, 11.239999771118164, 28 ]
+}

testdata/complex_property_ds/segmentation/1/header.wkw

@@ -6,10 +6,12 @@
     "scale": [
         1
     ],
+    "pattern": "{zzzzz}.tif",
     "dataLayers": [
         {
             "name": "color",
             "category": "color",
+            "dataFormat": "tiff",
             "elementClass": "uint8",
             "num_channels": 1,
             "boundingBox": {
@@ -32,9 +34,5 @@
                 }
             ]
         }
-    ],
-    "grid_shape": [
-        0,
-        0
     ]
 }

testdata/old_wk_dataset/color/1/header.wkw

@@ -30,9 +30,9 @@
                         1,
                         1
                     ],
-                    "cube_length": 32
+                    "cubeLength": 32
                 }
             ]
         }
     ]
-}
+}