Refactor wkcuber for Zarr (#679)

Author: normanrz

.github/workflows/ci.yml

@@ -299,7 +299,7 @@ jobs:
             --jobs 2 \
             --batch_size 8 \
             --layer_name color \
-            --wkw_file_len 32 \
+            --chunks_per_shard 32 \
             --scale 1 \
             testdata/tiff testoutput/tiff
 

.gitignore

@@ -124,3 +124,6 @@ webknossos/testoutput/*
 */*/output.nml
 */cell_*/*
 cell_*/*
+
+# minio
+minio_data

_tooling/local_minio_setup.sh

@@ -0,0 +1,19 @@
+export MINIO_ROOT_USER="TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur"
+export MINIO_ROOT_PASSWORD="ANTN35UAENTS5UIAEATD"
+
+# Minio is an S3 clone and is used as local test server
+docker run \
+  -p 8000:9000 \
+  -e MINIO_ROOT_USER=$MINIO_ROOT_USER \
+  -e MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD \
+  --name minio \
+  --rm \
+  -d \
+  minio/minio server /data
+
+stop_minio () {
+    ARG=$?
+    docker stop minio
+    exit $ARG
+}
+trap stop_minio EXIT

webknossos/.gitignore

@@ -105,3 +105,5 @@ ENV/
 
 /webknossos/version.py
 testoutput
+# Unpacked version of WT1_wkw.tar.gz
+testdata/WT1_wkw

webknossos/poetry.lock

@@ -31,7 +31,7 @@ zarr = "^2.11.0"
 numcodecs = "^0.9.1"
 zipp = "^3.5.0"
 fsspec = "^2022.2.0"
-universal-pathlib = "^0.0.14"
+universal-pathlib = "0.0.16"
 
 
 [tool.poetry.dev-dependencies]

webknossos/pyproject.toml

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Tuple
 
-from wkcuber.utils import (  # pylint: disable=import-error
+from wkcuber._internal.utils import (  # pylint: disable=import-error
     add_distribution_flags,
     add_scale_flag,
     get_executor_for_args,

webknossos/script_collection/move_dataset_slices_by_one.py

@@ -2,28 +2,10 @@
 set -eEuo pipefail
 
 source local_wk_setup.sh
+source ../_tooling/local_minio_setup.sh
 
 export_vars
 
-export MINIO_ROOT_USER="TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur"
-export MINIO_ROOT_PASSWORD="ANTN35UAENTS5UIAEATD"
-
-# Minio is an S3 clone and is used as local test server
-docker run \
-  -p 8000:9000 \
-  -e MINIO_ROOT_USER=$MINIO_ROOT_USER \
-  -e MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD \
-  --name minio \
-  --rm \
-  -d \
-  minio/minio server /data
-
-stop_minio () {
-    ARG=$?
-    docker stop minio
-    exit $ARG
-}
-trap stop_minio EXIT
 
 if [ $# -eq 1 ] && [ "$1" = "--refresh-snapshots" ]; then
     ensure_local_test_wk

webknossos/test.sh

@@ -1,52 +1,34 @@
 {
-    "id": {
-        "name": "simple_wk_dataset",
-        "team": ""
-    },
-    "scale": [
-        1, 1, 1
-    ],
-    "defaultViewConfiguration": {
-        "position": [
-            12,
-            12,
-            12
-        ]
-    },
-    "dataLayers": [
+  "id": {
+    "name": "simple_wkw_dataset",
+    "team": ""
+  },
+  "scale": [1, 1, 1],
+  "defaultViewConfiguration": {
+    "position": [12, 12, 12]
+  },
+  "dataLayers": [
+    {
+      "dataFormat": "wkw",
+      "name": "color",
+      "category": "color",
+      "elementClass": "uint24",
+      "numChannels": 3,
+      "boundingBox": {
+        "topLeft": [0, 0, 0],
+        "width": 24,
+        "height": 24,
+        "depth": 24
+      },
+      "wkwResolutions": [
         {
-            "dataFormat": "wkw",
-            "name": "color",
-            "category": "color",
-            "elementClass": "uint24",
-            "numChannels": 3,
-            "boundingBox": {
-                "topLeft": [
-                    0,
-                    0,
-                    0
-                ],
-                "width": 24,
-                "height": 24,
-                "depth": 24
-            },
-            "wkwResolutions": [
-                {
-                    "resolution": [
-                        1,
-                        1,
-                        1
-                    ],
-                    "cubeLength": 32
-                }
-            ],
-            "defaultViewConfiguration": {
-                "color": [
-                    255,
-                    0,
-                    0
-                ]
-            }
+          "resolution": [1, 1, 1],
+          "cubeLength": 32
         }
-    ]
+      ],
+      "defaultViewConfiguration": {
+        "color": [255, 0, 0]
+      }
+    }
+  ]
 }