add webknossos-client with examples (#452)

* add v1 of user facing rest client with examples
* only allow binary mode with zipfile
* Undo global_offset change when setting bounding_box on layer, fix in example
* 🙈
* fix roundtrip for int zoomLevels
* Merge remote-tracking branch 'origin/master' into user-facing-rest-api
* Update webknossos/tests/conftest.py

Co-authored-by: Philipp Otto <[email protected]>
* undo typecheck change from merge
* Merge branch 'master' into user-facing-rest-api
* apply feedback part 1
* Merge remote-tracking branch 'origin/master' into user-facing-rest-api
* rm unused imports
* Merge remote-tracking branch 'origin/master' into user-facing-rest-api
* apply PR feedback part 2 & speed up learned_segmenter test
* fix chunk for single list entry
* remove unset handling in generated client responses
* PR feedback part 3
* formatting
* formatting
* formatting
* add PR feedback & lint everything
* add changelog entries
* apply PR feedback feedback :-)
* Merge remote-tracking branch 'origin/master' into user-facing-rest-api
* remove unused import
* remove more unused imports

Author: jstriebel

.github/workflows/ci.yml

@@ -45,19 +45,19 @@ jobs:
         poetry install
 
     - name: Check formatting
-      run: poetry run ./format.sh check
+      run: ./format.sh check
 
     - name: Lint code
-      run: poetry run ./lint.sh
+      run: ./lint.sh
 
     - name: Check typing
       run: |
-        poetry run ./typecheck.sh
+        ./typecheck.sh
 
     - name: Python tests
       env:
         WK_TOKEN: ${{ secrets.WK_TOKEN }}
-      run: poetry run pytest tests
+      run: ./test.sh
 
     - name: Check if git is dirty
       run: |

webknossos/.gitignore

@@ -1,3 +1,5 @@
+tests/.webknossos-server
+
 .idea
 *.iml
 

webknossos/Changelog.md

@@ -11,7 +11,12 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 [Commits](https://github.com/scalableminds/webknossos-cuber/compare/v0.8.18...HEAD)
 
 ### Breaking Changes
+- `BoundingBox.chunk()`'s 2nd parameter `chunk_border_alignments` now does not accept a list with a single `int` anymore. [#437](https://github.com/scalableminds/webknossos-libs/pull/452)
+
 ### Added
+- Added a new Annotation class which includes skeletons as well as volume-annotations. [#437](https://github.com/scalableminds/webknossos-libs/pull/452)
+- Added dataset down- and upload as well as annotation download, see the examples `learned_segmenter.py` and `upload_image_data.py`. [#437](https://github.com/scalableminds/webknossos-libs/pull/452)
+
 ### Changed
 ### Fixed
 

webknossos/__generate_client.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Dict, Iterable, Tuple
+from typing import Any, Dict, Iterable, Tuple
 
 import httpx
 from inducoapi import build_openapi
@@ -13,9 +13,8 @@
     Project,
     _get_project_for_url_or_path,
 )
-from openapi_python_client.cli import handle_errors
 
-from webknossos.client import _get_generated_client
+from webknossos.client.context import get_generated_client
 from webknossos.utils import snake_to_camel_case
 
 SCHEMA_URL = "https://converter.swagger.io/api/convert?url=https%3A%2F%2Fwebknossos.org%2Fswagger.json"
@@ -39,9 +38,10 @@ def generate_client(openapi_schema: Dict) -> None:
             meta=MetaType.POETRY,
             config=generator_config,
         )
-        assert isinstance(generator_project, Project)
-        errors = generator_project.update()
-        # handle_errors(errors)  # prints warnings
+        assert isinstance(generator_project, Project), generator_project.detail
+        _errors = generator_project.update()  # pylint: disable=no-member
+        # from openapi_python_client.cli import handle_errors
+        # handle_errors(_errors)  # prints warnings
 
 
 def add_api_prefix_for_non_data_paths(openapi_schema: Dict) -> None:
@@ -68,7 +68,7 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
 
     d = datetime.utcnow()
     unixtime = calendar.timegm(d.utctimetuple())
-    client = _get_generated_client(enforce_token=True)
+    client = get_generated_client(enforce_auth=True)
 
     annotation_info_response = annotation_info.sync_detailed(
         typ="Explorational",
@@ -91,9 +91,30 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
         api_endpoint_name = api_endpoint.__name__.split(".")[-1]
         api_endpoint_name = snake_to_camel_case(api_endpoint_name)
 
-        response = api_endpoint.sync_detailed(client=client)
-        assert response.status_code == 200
-        yield api_endpoint_name, response.content
+        api_endpoint_response = api_endpoint.sync_detailed(client=client)
+        assert api_endpoint_response.status_code == 200
+        yield api_endpoint_name, api_endpoint_response.content
+
+
+FIELDS_WITH_VARYING_CONTENT = ["adminViewConfiguration"]
+
+
+def make_properties_required(x: Any) -> None:
+    if isinstance(x, dict):
+        for key, value in x.items():
+            # do not recurse into objects where the contents might be varying
+            if key in FIELDS_WITH_VARYING_CONTENT:
+                continue
+            make_properties_required(value)
+    elif isinstance(x, list):
+        for i in x:
+            make_properties_required(i)
+
+    if isinstance(x, dict) and "properties" in x:
+        properties = x["properties"]
+        if isinstance(properties, dict) and len(properties) > 0:
+            assert "required" not in x
+            x["required"] = list(properties.keys())
 
 
 def set_response_schema_by_example(
@@ -120,16 +141,17 @@ def set_response_schema_by_example(
         for path_method in path.values()
         if path_method["operationId"] == operation_id
     ][0]
+    make_properties_required(recorded_response_schema)
     request_schema["responses"]["200"]["content"] = recorded_response_schema
 
 
 def bootstrap_response_schemas(openapi_schema: Dict) -> None:
     """Inserts the response schemas into openapi_schema (in-place),
     as recorded by example requests."""
     assert_valid_schema(openapi_schema)
-    for operation_id, response in iterate_request_ids_with_responses():
+    for operation_id, example_response in iterate_request_ids_with_responses():
         set_response_schema_by_example(
-            openapi_schema, example_response=response, operation_id=operation_id
+            openapi_schema, example_response=example_response, operation_id=operation_id
         )
 
 

webknossos/_helpers/pylint_plugins/attrs_ng_api.py

@@ -7,7 +7,7 @@
 
 def register(linter):
     # Needed for registering the plugin.
-    pass
+    del linter
 
 
 ATTRIB_NAMES = frozenset(("attr.ib", "attrib", "attr.attrib", "attr.field", "field"))

webknossos/examples/WIP/learned_segmenter.py

@@ -1,4 +1,5 @@
 # type: ignore
+# pylint: skip-file
 from itertools import combinations
 
 import networkx as nx

webknossos/examples/WIP/merge_trees_at_closest_nodes.py

@@ -1,4 +1,5 @@
 # type: ignore
+# pylint: skip-file
 import numpy as np
 
 import webknossos as wk

webknossos/examples/WIP/offline_merger_mode.py

@@ -0,0 +1,67 @@
+from functools import partial
+from time import gmtime, strftime
+
+import numpy as np
+from skimage import feature
+from skimage.future import TrainableSegmenter
+
+import webknossos as wk
+from webknossos.client.context import webknossos_context
+from webknossos.geometry import BoundingBox, Mag
+
+annotation = wk.open_annotation(
+    "https://webknossos.org/annotations/Explorational/616457c2010000870032ced4"
+)
+training_data_bbox = BoundingBox.from_tuple6(
+    annotation.skeleton.user_bounding_boxes[0]  # pylint: disable=unsubscriptable-object
+)
+time_str = strftime("%Y-%m-%d_%H-%M-%S", gmtime())
+new_dataset_name = annotation.dataset_name + f"_segmented_{time_str}"
+dataset = wk.download_dataset(
+    annotation.dataset_name,
+    "scalable_minds",
+    path=new_dataset_name,
+)
+dataset.name = new_dataset_name
+
+volume_annotation = annotation.save_volume_annotation(dataset)
+volume_annotation.bounding_box = training_data_bbox
+
+mag = Mag(1)
+mag_view = dataset.layers["color"].mags[mag]
+
+features_func = partial(
+    feature.multiscale_basic_features, multichannel=True, edges=False
+)
+segmenter = TrainableSegmenter(features_func=features_func)
+
+print("Starting training…")
+img_data_train = mag_view.read(
+    training_data_bbox.in_mag(mag).topleft, training_data_bbox.in_mag(mag).size
+)
+# move channels to last dimension, remove z dimension
+X_train = np.moveaxis(np.squeeze(img_data_train), 0, -1)
+Y_train = np.squeeze(volume_annotation.mags[mag].get_view().read())
+segmenter.fit(X_train, Y_train)
+
+
+print("Starting prediction…")
+X_predict = np.moveaxis(np.squeeze(mag_view.read()), 0, -1)
+Y_predicted = segmenter.predict(X_predict)
+segmentation = Y_predicted[:, :, None]  # adds z dimension
+assert segmentation.max() < 256
+segmentation = segmentation.astype("uint8")
+
+segmentation_layer = dataset.add_layer(
+    "segmentation",
+    "segmentation",
+    segmentation.dtype,
+    compressed=True,
+    largest_segment_id=int(segmentation.max()),
+)
+segmentation_layer.bounding_box = dataset.layers["color"].bounding_box
+segmentation_layer.add_mag(mag, compress=True).write(segmentation)
+
+with webknossos_context(url="http://localhost:9000", token="secretScmBoyToken"):
+    url = dataset.upload()
+print(f"Successfully uploaded {url}")

webknossos/examples/learned_segmenter.py

@@ -0,0 +1,22 @@
+from time import gmtime, strftime
+
+from skimage import data
+
+from webknossos.client.context import webknossos_context
+from webknossos.dataset.dataset import Dataset
+
+with webknossos_context(url="http://localhost:9000", token="secretScmBoyToken"):
+    img = data.cell()
+    time_str = strftime("%Y-%m-%d_%H-%M-%S", gmtime())
+    name = f"cell_{time_str}"
+    ds = Dataset.create(name, scale=(107, 107, 107))
+    layer = ds.add_layer(
+        "color",
+        "color",
+        dtype_per_layer=img.dtype,
+    )
+    # add channel and z dimensions and put X before Y,
+    # resulting dimensions are C, X, Y, Z.
+    layer.add_mag(1, compress=True).write(img.T[None, :, :, None])
+    url = ds.upload()
+    print(f"Successfully uploaded {url}")