Announce dataset creation with API (#1170)

* Add trigger_reload method.

* Add dataset reserve manual upload method.

* Add example and add method to dataset class for announce manual upload.

* Add trigger reload as method to dataset class and update example.

* Remove returns for post requests.

* Adapt naming and docstring.

* Update example.

Author: markbader

docs/mkdocs.yml

@@ -99,6 +99,7 @@ nav:
               - webknossos-py/examples/remote_datasets.md
               - webknossos-py/examples/zarr_and_dask.md
               - webknossos-py/examples/convert_4d_tiff.md
+              - webknossos-py/examples/announce_dataset_upload.md
           - Annotation Examples:
               - webknossos-py/examples/apply_merger_mode.md
               - webknossos-py/examples/learned_segmenter.md

docs/src/webknossos-py/examples/announce_dataset_upload.md

@@ -0,0 +1,9 @@
+# Announce manual Dataset upload
+
+This example demonstrates the manual process of uploading a Dataset. If users have file-system access to the WEBKNOSSOS datastore, they can upload a dataset by manually copying it to the binary_data folder. To ensure proper database entry creation and access rights, the Dataset can be announced using the WEBKNOSSOS API.
+
+```python
+--8<--
+webknossos/examples/announce_dataset_upload.py
+--8<--
+```

webknossos/examples/announce_dataset_upload.py

@@ -0,0 +1,30 @@
+import webknossos as wk
+from webknossos.dataset.remote_folder import RemoteFolder
+
+
+def main() -> None:
+    # Get the folder id of the upload destination:
+    folder_id = RemoteFolder.get_by_path("Datasets/").id
+
+    # Announce the manual upload of a new dataset:
+    wk.Dataset.announce_manual_upload(
+        dataset_name="my_new_dataset_name",
+        organization="sample_organization",
+        initial_team_ids=[],
+        folder_id=folder_id,
+    )
+
+    # Move/Copy the dataset manually to the datastore
+
+    # After a few minutes the dataset list is updated
+    # automatically. The dataset is then visible for all
+    # users that are part of one team defined in initial_team_ids.
+    # To trigger a reload of the dataset properties manually, use:
+
+    wk.Dataset.trigger_reload_in_datastore(
+        dataset_name="my_new_dataset_name", organization="sample_organization"
+    )
+
+
+if __name__ == "__main__":
+    main()

webknossos/webknossos/client/api_client/datastore_api_client.py

@@ -1,6 +1,7 @@
 from typing import Dict, Optional, Tuple
 
 from webknossos.client.api_client.models import (
+    ApiDatasetAnnounceUpload,
     ApiDatasetUploadInformation,
     ApiReserveDatasetUploadInformation,
 )
@@ -34,7 +35,7 @@ def dataset_finish_upload(
         retry_count: int,
     ) -> None:
         route = "/datasets/finishUpload"
-        return self._post_json(
+        self._post_json(
             route,
             upload_information,
             query={"token": token},
@@ -49,13 +50,32 @@ def dataset_reserve_upload(
         retry_count: int,
     ) -> None:
         route = "/datasets/reserveUpload"
-        return self._post_json(
+        self._post_json(
             route,
             reserve_upload_information,
             query={"token": token},
             retry_count=retry_count,
         )
 
+    def dataset_trigger_reload(
+        self,
+        organization_name: str,
+        dataset_name: str,
+        token: Optional[str] = None,
+    ) -> None:
+        route = f"/triggers/reload/{organization_name}/{dataset_name}"
+        query: Query = {"token": token}
+        self._post(route, query=query)
+
+    def dataset_reserve_manual_upload(
+        self,
+        dataset_announce: ApiDatasetAnnounceUpload,
+        token: Optional[str],
+    ) -> None:
+        route = "/datasets/reserveManualUpload"
+        query: Query = {"token": token}
+        self._post_json(route, dataset_announce, query)
+
     def dataset_get_raw_data(
         self,
         organization_name: str,

webknossos/webknossos/client/api_client/models.py

@@ -86,6 +86,14 @@ class ApiDataset:
     description: Optional[str] = None
 
 
+@attr.s(auto_attribs=True)
+class ApiDatasetAnnounceUpload:
+    dataset_name: str
+    organization: str
+    initial_team_ids: List[str]
+    folder_id: str
+
+
 @attr.s(auto_attribs=True)
 class ApiDatasetIsValidNewNameResponse:
     is_valid: bool

webknossos/webknossos/dataset/dataset.py

@@ -396,6 +396,59 @@ def open(cls, dataset_path: Union[str, PathLike]) -> "Dataset":
             exist_ok=True,
         )
 
+    @classmethod
+    def announce_manual_upload(
+        cls,
+        dataset_name: str,
+        organization: str,
+        initial_team_ids: List[str],
+        folder_id: str,
+        token: Optional[str] = None,
+    ) -> None:
+        """
+        Announces a manual dataset upload to webknossos. This is useful when users with access
+        to the file system of the datastore want to upload a dataset manually. It creates an entry
+        in the database and sets the access rights accordingly.
+        """
+        from ..client._upload_dataset import _cached_get_upload_datastore
+        from ..client.api_client.models import ApiDatasetAnnounceUpload
+        from ..client.context import _get_context
+
+        context = _get_context()
+        dataset_announce = ApiDatasetAnnounceUpload(
+            dataset_name=dataset_name,
+            organization=organization,
+            initial_team_ids=initial_team_ids,
+            folder_id=folder_id,
+        )
+        token = token or context.token
+        upload_url = _cached_get_upload_datastore(context)
+        datastore_api = context.get_datastore_api_client(upload_url)
+        datastore_api.dataset_reserve_manual_upload(dataset_announce, token=token)
+
+    @classmethod
+    def trigger_reload_in_datastore(
+        cls,
+        dataset_name: str,
+        organization: str,
+        token: Optional[str] = None,
+    ) -> None:
+        """
+        This method is used for datasets that were uploaded manually
+        to a webknossos datastore. It can not be used for local datasets.
+        After a manual upload of a dataset, the datasets properties are
+        updated automatically after a few minutes. To trigger a reload
+        of the datasets properties manually, use this method.
+        """
+        from ..client._upload_dataset import _cached_get_upload_datastore
+        from ..client.context import _get_context
+
+        context = _get_context()
+        token = token or context.token
+        upload_url = _cached_get_upload_datastore(context)
+        datastore_api = context.get_datastore_api_client(upload_url)
+        datastore_api.dataset_trigger_reload(organization, dataset_name, token=token)
+
     @classmethod
     def _parse_remote(
         cls,