[wk-libs] Dataset: fix bug with upload linking, add remove_layer (#584)

* cancel previous CI runs on same branch

* add cluster_tools to Readme

* dataset.upload: fix bug with linking, added dataset.remove_layer

* update changelog

* undo remove_layer

* update changelog

* minor fix

Author: jstriebel

.github/workflows/ci.yml

@@ -2,6 +2,10 @@ name: CI
 
 on: push
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   changes:
     runs-on: ubuntu-latest

README.md

@@ -25,3 +25,6 @@ Use this for:
 - converting Tiff-stacks and other data formats for volume image data to webKnossos-compatible *.wkw files from the CLI
 - up/downsampling of *.wkw files to different magnification levels (image pyramid) from the CLI
 - compressing your *.wkw files to save disk space from the CLI
+
+## [Cluster Tools](cluster_tools)
+The `cluster_tools` package provides python `Executor` classes for distributing tasks on a slurm cluster or via multi processing.

webknossos/Changelog.md

@@ -16,6 +16,7 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 ### Changed
 
 ### Fixed
+- `dataset.upload(layers_to_link=…)`: Fixed a bug where the upload did not complete if layers_to_link contained layers present in uploading dataset. [#584](https://github.com/scalableminds/webknossos-libs/pull/584)
 
 
 ## [0.9.2](https://github.com/scalableminds/webknossos-libs/releases/tag/v0.9.2) - 2022-02-03

webknossos/examples/learned_segmenter.py

@@ -52,10 +52,8 @@ def main() -> None:
     )  # wk data has dimensions (Channels, X, Y, Z)
     # move channels to last dimension, remove z dimension to match skimage's shape
     X_train = np.moveaxis(np.squeeze(img_data_train), 0, -1)
-    print(volume_annotation.mags[mag].bounding_box)
     Y_train = np.squeeze(volume_annotation.mags[mag].read())
 
-    print(training_data_bbox, X_train.shape, Y_train.shape)
     segmenter.fit(X_train, Y_train)
 
     # Step 4: Use our trained model and predict a class for each pixel in the dataset

webknossos/webknossos/client/_upload_dataset.py

@@ -1,6 +1,7 @@
 import os
 from functools import lru_cache
 from pathlib import Path
+from tempfile import TemporaryDirectory
 from time import gmtime, strftime
 from typing import Dict, Iterator, List, NamedTuple, Optional, Tuple
 from uuid import uuid4
@@ -58,7 +59,8 @@ def _cached_get_upload_datastore(context: _WebknossosContext) -> str:
 
 
 def _walk(
-    path: Path, base_path: Optional[Path] = None
+    path: Path,
+    base_path: Optional[Path] = None,
 ) -> Iterator[Tuple[Path, Path, int]]:
     if base_path is None:
         base_path = path
@@ -80,6 +82,19 @@ def upload_dataset(
     if layers_to_link is None:
         layers_to_link = []
     context = _get_context()
+    layer_names_to_link = set(i.new_layer_name or i.layer_name for i in layers_to_link)
+    if len(layer_names_to_link.intersection(dataset.layers.keys())) > 0:
+        with TemporaryDirectory() as tmpdir:
+            tmp_ds = dataset.shallow_copy_dataset(
+                tmpdir, name=dataset.name, layers_to_ignore=layer_names_to_link
+            )
+            return upload_dataset(
+                tmp_ds,
+                new_dataset_name=new_dataset_name,
+                layers_to_link=layers_to_link,
+                jobs=jobs,
+            )
+
     file_infos = list(_walk(dataset.path))
     total_file_size = sum(size for _, _, size in file_infos)
     # replicates https://github.com/scalableminds/webknossos/blob/master/frontend/javascripts/admin/dataset/dataset_upload_view.js

webknossos/webknossos/dataset/dataset.py

@@ -8,12 +8,12 @@
 from os import PathLike, makedirs
 from os.path import basename, join, normpath
 from pathlib import Path
-from shutil import rmtree
 from typing import (
     TYPE_CHECKING,
     Any,
     ContextManager,
     Dict,
+    Iterable,
     List,
     Optional,
     Tuple,
@@ -559,12 +559,17 @@ def delete_layer(self, layer_name: str) -> None:
             raise IndexError(
                 f"Removing layer {layer_name} failed. There is no layer with this name"
             )
+        layer_path = self._layers[layer_name].path
         del self._layers[layer_name]
         self._properties.data_layers = [
             layer for layer in self._properties.data_layers if layer.name != layer_name
         ]
         # delete files on disk
-        rmtree(join(self.path, layer_name))
+        if layer_path.is_symlink():
+            layer_path.unlink()
+        else:
+            # rmtree does not recurse into linked dirs, but removes the link
+            shutil.rmtree(layer_path)
         self._export_as_json()
 
     def add_symlink_layer(
@@ -714,10 +719,10 @@ def copy_dataset(
 
     def shallow_copy_dataset(
         self,
-        new_dataset_path: Path,
+        new_dataset_path: Union[str, PathLike],
         name: Optional[str] = None,
         make_relative: bool = False,
-        layers_to_ignore: Optional[List[str]] = None,
+        layers_to_ignore: Optional[Iterable[str]] = None,
     ) -> "Dataset":
         """
         Create a new dataset at the given path. Link all mags of all existing layers.