Create handles for Tasks + Projects (#574)

* [WIP] create handles for Tasks + Projects

* add new routes from swagger, task example

* add user info by id, use in project

* rename project_info_by_id, add by_name

* fix imports

* fix type errors

* move to “administration” pacakge

* get tasks for project

* fetch annotation infos by task id

* [WIP] pagination

* fetch all using pagination if requested

* create tasks from files

* pass BytesIO directly

* task state

* create new task from downloaded annotation

* experiences are varying, binary is property

* fix optional field

* format

* remove name field from annotation layer response

* undo wrong renaming. IDE tried to be clever...

* one more usage...

* document new example

* changelog

* docs

* implement pr feedback (part 1)

* lru cache instead of cachedproperty

* new generated client

* format

* fix generated client

* format again after poetry install

* add missing call parentheses

* lru cache python37 compatible (hopefully)

Author: fm3

docs/mkdocs.yml

@@ -72,6 +72,7 @@ nav:
       - webknossos-py/examples/learned_segmenter.md
       - webknossos-py/examples/skeleton_synapse_candidates.md
       - webknossos-py/examples/user_times.md
+      - webknossos-py/examples/annotation_project_administration.md
   - API Reference:
     - Overview: api/webknossos.md
     - Geometry:
@@ -90,8 +91,10 @@ nav:
       - Node: api/webknossos/skeleton/node.md
     - Annotation: api/webknossos/annotation/annotation.md
     - Authentication & Server Context: api/webknossos/client/context.md
-    - Admin Resources:
-      - User: api/webknossos/client/user.md
+    - Administration:
+      - User: api/webknossos/administration/user.md
+      - Project: api/webknossos/administration/project.md
+      - Task: api/webknossos/administration/task.md
   - Getting Help:
     - webknossos-py/development.md
     - Community Support: https://forum.image.sc/tag/webknossos" target="_blank

docs/src/webknossos-py/examples/annotation_project_administration.md

@@ -0,0 +1,9 @@
+# Annotation Project Administration
+
+This example uses the [`Project` class](../../api/webknossos/administration/project.md#Project) and [`Task` class](../../api/webknossos/administration/task.md#task) to check annotation task status and submit new tasks.
+
+```python
+--8<--
+webknossos/examples/annotation_project_administration.py
+--8<--
+```

docs/src/webknossos-py/examples/user_times.md

@@ -1,6 +1,6 @@
 # Logged User Times
 
-This example uses the [`User` class](../../api/webknossos/client/user.md#User) to load the logged times
+This example uses the [`User` class](../../api/webknossos/administration/user.md#User) to load the logged times
 for all users of whom the current user is admin or team-manager.
 
 *This example additionally needs the pandas and tabulate packages.*

webknossos/Changelog.md

@@ -12,6 +12,7 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 ### Breaking Changes
 
 ### Added
+ - Added AnnotationInfo, Project and Task classes for handling annotation information and annotation project administration. [#574](https://github.com/scalableminds/webknossos-libs/pull/574):
 
 ### Changed
 

webknossos/__generate_client.py

@@ -16,11 +16,10 @@
     _get_project_for_url_or_path,
 )
 
-from webknossos.client.context import _get_generated_client
 from webknossos.utils import snake_to_camel_case
 
-SCHEMA_URL = "https://master.webknossos.xyz/swagger.json"
-# SCHEMA_URL = "http://localhost:9000/swagger.json"
+# SCHEMA_URL = "https://master.webknossos.xyz/swagger.json"
+SCHEMA_URL = "http://localhost:9000/swagger.json"
 CONVERTER_URL = "https://converter.swagger.io/api/convert"
 
 
@@ -64,37 +63,114 @@ def add_api_prefix_for_non_data_paths(openapi_schema: Dict) -> None:
 
 
 def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
+    """Send requests to webKnossos and record the schema of their replies"""
     from webknossos.client._generated.api.default import (
         annotation_info,
+        annotation_infos_by_task_id,
         build_info,
         current_user_info,
         dataset_info,
         datastore_list,
         generate_token_for_data_store,
+        project_info_by_id,
+        project_info_by_name,
+        task_info,
+        task_infos_by_project_id,
+        user_info_by_id,
         user_list,
         user_logged_time,
     )
+    from webknossos.client.context import _get_generated_client
+
+    # webKnossos.org setup:
+    # explorative_annotation_id = "6114d9410100009f0096c640"
+    # organization_name = "scalable_minds",
+    # dataset_name = "l4dense_motta_et_al_demo"
+    # task_id = "61f151c10100000a01249afe"
+    # user_id = "5b5dd2fb1c00008230ec8174"
+    # project_id = "61f1515e0100002f01249afa"
+    # project_name = "sampleProject"
+    # local setup, probably long gone by the time you read this:
+    explorative_annotation_id = "62011da6fa0100b202ec50db"
+    organization_name = "sample_organization"
+    dataset_name = "l4_sample"
+    task_id = "62011dddfa0100ad02ec50de"
+    user_id = "6200df39f70100f70157d983"
+    project_id = "6200df45f70100440257d987"
+    project_name = "sampleProject"
 
     d = datetime.utcnow()
     unixtime = calendar.timegm(d.utctimetuple())
     client = _get_generated_client(enforce_auth=True)
 
-    annotation_info_response = annotation_info.sync_detailed(
-        typ="Explorational",
-        id="6114d9410100009f0096c640",
-        client=client,
-        timestamp=unixtime,
+    yield extract_200_response(
+        "annotationInfo",
+        annotation_info.sync_detailed(
+            typ="Explorational",
+            id=explorative_annotation_id,
+            client=client,
+            timestamp=unixtime,
+        ),
     )
-    assert annotation_info_response.status_code == 200
-    yield "annotationInfo", annotation_info_response.content
 
-    dataset_info_response = dataset_info.sync_detailed(
-        organization_name="scalable_minds",
-        data_set_name="l4dense_motta_et_al_demo",
-        client=client,
+    yield extract_200_response(
+        "datasetInfo",
+        dataset_info.sync_detailed(
+            organization_name=organization_name,
+            data_set_name=dataset_name,
+            client=client,
+        ),
+    )
+
+    yield extract_200_response(
+        "taskInfo",
+        task_info.sync_detailed(
+            id=task_id,
+            client=client,
+        ),
+    )
+
+    yield extract_200_response(
+        "userInfoById",
+        user_info_by_id.sync_detailed(
+            id=user_id,
+            client=client,
+        ),
+    )
+
+    yield extract_200_response(
+        "projectInfoById",
+        project_info_by_id.sync_detailed(
+            id=project_id,
+            client=client,
+        ),
+    )
+
+    yield extract_200_response(
+        "projectInfoByName",
+        project_info_by_name.sync_detailed(name=project_name, client=client),
+    )
+
+    yield extract_200_response(
+        "taskInfosByProjectId",
+        task_infos_by_project_id.sync_detailed(
+            id=project_id,
+            client=client,
+        ),
+    )
+
+    yield extract_200_response(
+        "annotationInfosByTaskId",
+        annotation_infos_by_task_id.sync_detailed(id=task_id, client=client),
+    )
+
+    yield extract_200_response(
+        "userLoggedTime",
+        user_logged_time.sync_detailed(
+            id=user_id,
+            client=client,
+        ),
     )
-    assert dataset_info_response.status_code == 200
-    yield "datasetInfo", dataset_info_response.content
 
     for api_endpoint in [
         datastore_list,
@@ -106,28 +182,30 @@ def iterate_request_ids_with_responses() -> Iterable[Tuple[str, bytes]]:
         api_endpoint_name = api_endpoint.__name__.split(".")[-1]
         api_endpoint_name = snake_to_camel_case(api_endpoint_name)
 
-        api_endpoint_response = api_endpoint.sync_detailed(client=client)
-        assert api_endpoint_response.status_code == 200
-        yield api_endpoint_name, api_endpoint_response.content
-
-        if api_endpoint == current_user_info:
-            user_id = json.loads(api_endpoint_response.content)["id"]
-
-    user_logged_time_response = user_logged_time.sync_detailed(
-        id=user_id,
-        client=client,
-    )
-    assert user_logged_time_response.status_code == 200
-    yield "userLoggedTime", user_logged_time_response.content
+        yield extract_200_response(
+            api_endpoint_name, api_endpoint.sync_detailed(client=client)
+        )
 
 
 FIELDS_WITH_VARYING_CONTENT = [
+    "experiences",
+    "adminViewConfiguration",
+    "novelUserExperienceInfos",
+    "viewConfiguration",
+]
+
+OPTIONAL_FIELDS = [
     "adminViewConfiguration",
     "novelUserExperienceInfos",
     "viewConfiguration",
 ]
 
 
+def extract_200_response(name: str, response: Any) -> Tuple[str, bytes]:
+    assert response.status_code == 200, response.content
+    return name, response.content
+
+
 def make_properties_required(x: Any) -> None:
     if isinstance(x, dict):
         for key, value in x.items():
@@ -146,7 +224,7 @@ def make_properties_required(x: Any) -> None:
             x["required"] = list(
                 property
                 for property in properties.keys()
-                if property not in FIELDS_WITH_VARYING_CONTENT
+                if property not in OPTIONAL_FIELDS
             )
 
 

webknossos/examples/annotation_project_administration.py

@@ -0,0 +1,43 @@
+from webknossos import AnnotationState, Project, Task
+
+
+def main() -> None:
+    # Assume running annotation project with previously created tasks.
+    # Compare webKnossos documentation at https://docs.webknossos.org/webknossos/tasks.html
+
+    # Fetch statistics about how many annotation tasks have been completed yet.
+    sample_project = Project.get_by_name("sampleProject")
+    tasks = sample_project.get_tasks()
+    total_active_instances = sum([task.status.active_instance_count for task in tasks])
+    total_open_instances = sum([task.status.open_instance_count for task in tasks])
+    print(
+        f"There are {total_active_instances} active and {total_open_instances} open task instances."
+    )
+
+    # Find and download all of the project’s annotations that are already finished by annotators
+    finished_annotations = []
+    for task in tasks:
+        for annotation_info in task.get_annotation_infos():
+            if annotation_info.state == AnnotationState.FINISHED:
+                finished_annotation = annotation_info.download_annotation()
+                finished_annotations.append(finished_annotation)
+
+    assert len(finished_annotations) > 0, "No annotations are finished yet!"
+
+    # Assume a second task type is in place that instructs annotators to take the previously created
+    # annotations and perform a secondary annotation step on them
+    # (e.g. first task is to seed positions, second is to fully label around those)
+
+    task_type_id = "61f90e4efe0100b102553009"  # from webKnossos web interface
+    tasks = Task.create_from_annotations(
+        task_type_id=task_type_id,  # New task type instructs annotators what to do in the task
+        project_name="sampleProject2",
+        base_annotations=finished_annotations,
+        needed_experience_domain="sampleExp",  # Only annotators with the experience "sampleExp" of at least value 2 will get this task
+        needed_experience_value=2,
+    )
+    print(f"New tasks: {tasks}")
+
+
+if __name__ == "__main__":
+    main()

webknossos/tests/client/test_user.py

@@ -4,7 +4,7 @@
 
 
 def assert_valid_user(user: User) -> None:
-    assert user.id
+    assert user.user_id
     assert user.first_name
     assert user.last_name
     assert user.email

webknossos/webknossos/__init__.py

@@ -22,6 +22,7 @@
 
 # The table above contains zero-width spaces in the code examples after each dot to enforce correct line-breaks.
 
+from webknossos.administration import *
 from webknossos.annotation import *
 from webknossos.client import *
 from webknossos.dataset import *

webknossos/webknossos/administration/__init__.py

@@ -0,0 +1,3 @@
+from webknossos.administration.project import Project
+from webknossos.administration.task import Task
+from webknossos.administration.user import User