Add zarrita support (#912)

* adds zarrita

* fixes for zarrita

* fixes

* cassettes

* fix

* fix

* fix

* fix dask example

* fixes

* fixes for tests

* format

* fixes

* refresh

* tests

* tests

* types

* durations

* new zarr3 data

* no faulthandler in ci

* fixes

* changelog

* bump zarrita

* pr feedback

* fix tests

* update .test_durations

---------

Co-authored-by: Norman Rzepka <[email protected]>

Author: normanrz

.github/workflows/ci.yml

@@ -16,6 +16,8 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 
 ### Added
 - Added `duration_in_seconds` and `modified` to `AnnotationInfo`. [#914](https://github.com/scalableminds/webknossos-libs/pull/914)
+- Added [`zarrita`](https://github.com/scalableminds/zarrita) storage backend for arrays. zarrita supports Zarr v2 and v3 including sharding. To activate zarrita, the environment variable `WK_USE_ZARRITA` must be set. [#912](https://github.com/scalableminds/webknossos-libs/pull/912)
+- Added a `Zarr3` data format which supports sharding. [#912](https://github.com/scalableminds/webknossos-libs/pull/912)
 
 ### Changed
 - Integrated the `wkcuber` CLI tool into `webknossos` package. [#903](https://github.com/scalableminds/webknossos-libs/pull/903)

webknossos/.test_durations

@@ -63,14 +63,15 @@ JPype1 = { version = "^1.3.0", optional = true }
 pims = { version = "^0.6.0", optional = true }
 tifffile = { version = ">=2021.11.2", optional = true }
 pylibCZIrw = { version = "3.4.0", source = "scm", optional = true }
+zarrita = "0.1.0a12"
 
 [tool.poetry.extras]
 pims = ["pims"]
 tifffile = ["pims", "tifffile"]
 imagecodecs = ["pims", "imagecodecs"]
 bioformats = ["pims","JPype1"]
 czi = ["pims","pylibCZIrw"]
-all = ["pims","tifffile","imagecodecs","JPype1","pylibCZIrw",]
+all = ["pims","tifffile","imagecodecs","JPype1","pylibCZIrw","zarrita"]
 
 [tool.poetry.dev-dependencies]
 # autoflake

webknossos/Changelog.md

@@ -11,6 +11,7 @@ export_vars
 # (which is standard python behavior). This is necessary so that the imports
 # refer to the checked out (and potentially modified) code.
 PYTEST="poetry run python -m pytest --suppress-no-test-exit-code"
+export WK_USE_ZARRITA=True
 
 
 if [ $# -gt 0 ] && [ "$1" = "--refresh-snapshots" ]; then

webknossos/poetry.lock

@@ -0,0 +1 @@
+{"shape": [3, 24, 24, 24], "data_type": "uint8", "chunk_grid": {"configuration": {"chunk_shape": [3, 32, 32, 32]}, "name": "regular"}, "chunk_key_encoding": {"configuration": {"separator": "/"}, "name": "default"}, "fill_value": 0, "attributes": {}, "codecs": [{"configuration": {"chunk_shape": [3, 16, 16, 16], "codecs": []}, "name": "sharding_indexed"}], "dimension_names": null, "zarr_format": 3, "node_type": "array"}

webknossos/pyproject.toml

@@ -0,0 +1,45 @@
+{
+  "node_type": "group",
+  "zarr_format": 3,
+  "attributes": {
+    "ome": {
+      "multiscales": [
+        {
+          "version": "0.4",
+          "axes": [
+            {
+              "name": "c",
+              "type": "channel"
+            },
+            {
+              "name": "x",
+              "type": "space",
+              "unit": "nanometer"
+            },
+            {
+              "name": "y",
+              "type": "space",
+              "unit": "nanometer"
+            },
+            {
+              "name": "z",
+              "type": "space",
+              "unit": "nanometer"
+            }
+          ],
+          "datasets": [
+            {
+              "path": "1",
+              "coordinateTransformations": [
+                {
+                  "type": "scale",
+                  "scale": [1.0, 1.0, 1.0, 1.0]
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  }
+}

webknossos/test.sh

@@ -0,0 +1,33 @@
+{
+  "id": {
+    "name": "simple_zarr_dataset",
+    "team": ""
+  },
+  "scale": [1.0, 1.0, 1.0],
+  "dataLayers": [
+    {
+      "name": "color",
+      "category": "color",
+      "boundingBox": {
+        "topLeft": [0, 0, 0],
+        "width": 24,
+        "height": 24,
+        "depth": 24
+      },
+      "elementClass": "uint24",
+      "dataFormat": "zarr3",
+      "mags": [
+        {
+          "mag": [1, 1, 1]
+        }
+      ],
+      "numChannels": 3,
+      "defaultViewConfiguration": {
+        "color": [255, 0, 0]
+      }
+    }
+  ],
+  "defaultViewConfiguration": {
+    "position": [12, 12, 12]
+  }
+}