✨ Webserver reports copy progress (ITISFoundation#3287)

Author: sanderegg

.github/workflows/ci-testing-deploy.yml

@@ -1076,7 +1076,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-storage:
-    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    timeout-minutes: 25 # if this timeout gets too small, then split the tests
     name: "[unit] storage"
     runs-on: ${{ matrix.os }}
     strategy:

api/specs/webserver/components/schemas/task.yaml renamed to api/specs/common/schemas/task.yaml

@@ -23,11 +23,19 @@ servers:
         enum:
           - v0
         default: v0
+tags:
+  - name: maintenance
+  - name: location
+  - name: dataset
+  - name: file
+  - name: tasks
 paths:
   /:
     get:
       summary: Service health-check endpoint
       description: Some general information on the API and state of the service behind
+      tags:
+        - maintenance
       operationId: health_check
       responses:
         "200":
@@ -43,6 +51,8 @@ paths:
     get:
       summary: checks status of self and connected services
       operationId: get_status
+      tags:
+        - maintenance
       responses:
         "200":
           description: returns app status check
@@ -51,6 +61,8 @@ paths:
     get:
       summary: Lists available storage locations
       operationId: get_storage_locations
+      tags:
+        - location
       parameters:
         - name: user_id
           in: query
@@ -71,6 +83,8 @@ paths:
     post:
       summary: Manually triggers the synchronisation of the file meta data table in the database
       operationId: synchronise_meta_data_table
+      tags:
+        - location
       parameters:
         - name: location_id
           in: path
@@ -103,6 +117,8 @@ paths:
     get:
       summary: Lists all dataset's metadata
       operationId: get_datasets_metadata
+      tags:
+        - dataset
       parameters:
         - name: location_id
           in: path
@@ -128,6 +144,8 @@ paths:
     get:
       summary: Lists all file's metadata
       operationId: get_files_metadata
+      tags:
+        - file
       parameters:
         - name: location_id
           in: path
@@ -158,6 +176,8 @@ paths:
     get:
       summary: Get dataset metadata
       operationId: get_files_metadata_dataset
+      tags:
+        - dataset
       parameters:
         - name: location_id
           in: path
@@ -188,6 +208,8 @@ paths:
     get:
       summary: Get file metadata
       operationId: get_file_metadata
+      tags:
+        - file
       parameters:
         - name: file_id
           in: path
@@ -218,6 +240,8 @@ paths:
     get:
       summary: Gets download link for file at location
       operationId: download_file
+      tags:
+        - file
       parameters:
         - name: file_id
           in: path
@@ -256,6 +280,8 @@ paths:
     put:
       summary: Returns upload object
       operationId: upload_file
+      tags:
+        - file
       parameters:
         - name: file_id
           in: path
@@ -316,6 +342,8 @@ paths:
     delete:
       summary: Deletes file
       operationId: delete_file
+      tags:
+        - file
       parameters:
         - name: file_id
           in: path
@@ -342,6 +370,8 @@ paths:
     post:
       summary: Asks the server to abort the upload and revert to the last valid version if any
       operationId: abort_upload_file
+      tags:
+        - file
       parameters:
         - name: file_id
           in: path
@@ -368,6 +398,8 @@ paths:
     post:
       summary: Asks the server to complete the upload
       operationId: complete_upload_file
+      tags:
+        - file
       parameters:
         - name: file_id
           in: path
@@ -427,6 +459,8 @@ paths:
     post:
       summary: Check for upload completion
       operationId: is_completed_upload_file
+      tags:
+        - file
       parameters:
         - name: future_id
           in: path
@@ -462,6 +496,8 @@ paths:
     post:
       summary: Returns the temporary access credentials for the user storage space
       operationId: get_or_create_temporary_s3_access
+      tags:
+        - file
       parameters:
         - name: user_id
           in: query
@@ -482,6 +518,8 @@ paths:
     post:
       summary: Returns metadata for all files matching a pattern
       operationId: search_files_starting_with
+      tags:
+        - file
       parameters:
         - name: user_id
           in: query
@@ -508,6 +546,8 @@ paths:
     post:
       summary: Deep copies of all data from source to destination project in s3
       operationId: copy_folders_from_project
+      tags:
+        - file
       parameters:
         - name: user_id
           in: query
@@ -543,6 +583,8 @@ paths:
     delete:
       summary: Deletes all objects within a node_id or within a project_id if node_id is omitted
       operationId: delete_folders_of_project
+      tags:
+        - file
       parameters:
         - name: folder_id
           in: path
@@ -567,6 +609,8 @@ paths:
     post:
       summary: Copy as soft link
       operationId: copy_as_soft_link
+      tags:
+        - file
       parameters:
         - name: file_id
           in: path
@@ -597,6 +641,68 @@ paths:
                 $ref: "#/components/schemas/FileMetaEnvelope"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
+
+  /tasks:
+    get:
+      operationId: list_tasks
+      tags:
+        - tasks
+      responses:
+        "200":
+          description: Returns the list of active tasks (running and/or done)
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "../common/schemas/task.yaml#/TaskEnveloped"
+  /tasks/{task_id}:
+    parameters:
+      - name: task_id
+        in: path
+        required: true
+        schema:
+          type: string
+    get:
+      operationId: get_task_status
+      tags:
+        - tasks
+      responses:
+        "200":
+          description: Returns the task status
+          content:
+            application/json:
+              schema:
+                $ref: "../common/schemas/task.yaml#/TaskStatusEnveloped"
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+    delete:
+      operationId: cancel_and_delete_task
+      description: Aborts and remove the task
+      tags:
+        - tasks
+      responses:
+        "204":
+          description: Task was successfully aborted
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
+  /tasks/{task_id}/result:
+    parameters:
+      - name: task_id
+        in: path
+        required: true
+        schema:
+          type: string
+    get:
+      operationId: get_task_result
+      tags:
+        - tasks
+      responses:
+        "2XX":
+          description: Retrieve the task result and returns directly its HTTP code
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
 components:
   schemas:
     HealthCheckEnveloped:

api/specs/storage/openapi.yaml

@@ -85,7 +85,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "./components/schemas/task.yaml#/TaskEnveloped"
+                $ref: "../common/schemas/task.yaml#/TaskEnveloped"
           links:
             CreationStatus:
               operationId: get_task_status

api/specs/webserver/openapi-projects.yaml

@@ -12,7 +12,7 @@ paths:
               schema:
                 type: array
                 items:
-                  $ref: "./components/schemas/task.yaml#/TaskEnveloped"
+                  $ref: "../common/schemas/task.yaml#/TaskEnveloped"
 
   /tasks/{task_id}:
     parameters:
@@ -31,7 +31,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "./components/schemas/task.yaml#/TaskStatusEnveloped"
+                $ref: "../common/schemas/task.yaml#/TaskStatusEnveloped"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
     delete:

api/specs/webserver/openapi-tasks.yaml

@@ -19,10 +19,12 @@
 NodeID = UUID
 
 
-class NodeIDStr(ConstrainedStr):
+class UUIDStr(ConstrainedStr):
     regex: Optional[Pattern[str]] = re.compile(UUID_RE)
 
 
+NodeIDStr = UUIDStr
+
 LocationID = int
 LocationName = str
 

packages/models-library/src/models_library/projects_nodes_io.py

@@ -22,7 +22,7 @@
 _DEFAULT_AIOHTTP_RETRY_POLICY = dict(
     retry=retry_if_exception_type(ClientConnectionError),
     wait=wait_random_exponential(max=20),
-    stop=stop_after_delay(30),
+    stop=stop_after_delay(60),
     reraise=True,
 )
 
@@ -45,12 +45,12 @@ async def _wait_for_completion(
     session: ClientSession,
     task_id: TaskId,
     status_url: URL,
-    wait_timeout_s: int,
+    client_timeout: int,
 ) -> AsyncGenerator[TaskProgress, None]:
     try:
 
         async for attempt in AsyncRetrying(
-            stop=stop_after_delay(wait_timeout_s),
+            stop=stop_after_delay(client_timeout),
             reraise=True,
             retry=retry_if_exception_type(TryAgain),
         ):
@@ -78,7 +78,7 @@ async def _wait_for_completion(
     except TryAgain as exc:
         # this is a timeout
         raise asyncio.TimeoutError(
-            f"Long running task {task_id}, calling to {status_url} timed-out after {wait_timeout_s} seconds"
+            f"Long running task {task_id}, calling to {status_url} timed-out after {client_timeout} seconds"
         ) from exc
 
 

packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py

@@ -5,6 +5,7 @@
 The server only has to return a `TaskId` in the handler creating the long
 running task.
 """
+from ...long_running_tasks._models import ProgressMessage, ProgressPercent
 from ...long_running_tasks._task import (
     TaskAlreadyRunningError,
     TaskCancelledError,
@@ -21,6 +22,8 @@
 __all__: tuple[str, ...] = (
     "create_task_name_from_request",
     "get_tasks_manager",
+    "ProgressMessage",
+    "ProgressPercent",
     "setup",
     "start_long_running_task",
     "TaskAlreadyRunningError",

packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py

@@ -4,7 +4,14 @@
 from datetime import datetime
 from typing import Any, Awaitable, Callable, Coroutine, Optional
 
-from pydantic import BaseModel, Field, PositiveFloat, confloat, validator
+from pydantic import (
+    BaseModel,
+    Field,
+    PositiveFloat,
+    confloat,
+    validate_arguments,
+    validator,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +37,7 @@ class TaskProgress(BaseModel):
     message: ProgressMessage = Field(default="")
     percent: ProgressPercent = Field(default=0.0)
 
+    @validate_arguments
     def update(
         self,
         *,
@@ -50,6 +58,11 @@ def update(
     def create(cls) -> "TaskProgress":
         return cls.parse_obj(dict(message="", percent=0.0))
 
+    @validator("percent")
+    @classmethod
+    def round_value_to_3_digit(cls, v):
+        return round(v, 3)
+
 
 class TrackedTask(BaseModel):
     task_id: str

packages/service-library/src/servicelib/long_running_tasks/_models.py

@@ -0,0 +1,6 @@
+from servicelib.long_running_tasks._models import TaskProgress
+
+
+def test_progress_has_no_more_than_3_digits():
+    progress = TaskProgress(percent=0.45646)
+    assert progress.percent == 0.456