✨ Storage: allow creation of multipart presigned upload links (ITISFoundation#3021)

Author: sanderegg

.env-devel

@@ -70,7 +70,10 @@ SMTP_PORT=25
 
 SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet
 
+# NOTE: STORAGE_ENDPOINT is needed by director-v0
 STORAGE_ENDPOINT=storage:8080
+STORAGE_HOST=storage
+STORAGE_PORT=8080
 
 SWARM_STACK_NAME=master-simcore
 

api/specs/storage/openapi.yaml

@@ -47,36 +47,6 @@ paths:
         "200":
           description: returns app status check
 
-  /check/{action}:
-    post:
-      summary: Test checkpoint to ask server to fail or echo back the transmitted data
-      parameters:
-        - in: path
-          required: true
-          name: action
-          schema:
-            type: string
-            default: "echo"
-            enum: ["fail", "echo"]
-        - in: query
-          name: data
-          schema:
-            type: string
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/Fake"
-      responses:
-        "200":
-          description: Echoes response based on action
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/FakeEnveloped"
-        default:
-          $ref: "#/components/responses/DefaultErrorResponse"
-
   /locations:
     get:
       summary: Lists available storage locations
@@ -244,32 +214,6 @@ paths:
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 
-  /locations/{location_id}/files/{file_id}:abort:
-    post:
-      summary: Asks the server to abort the upload and revert to the last valid version if any
-      operationId: abort_upload_file
-      parameters:
-        - name: file_id
-          in: path
-          required: true
-          schema:
-            type: string
-        - name: location_id
-          in: path
-          required: true
-          schema:
-            type: string
-        - name: user_id
-          in: query
-          required: true
-          schema:
-            type: string
-      responses:
-        "204":
-          description: Abort OK
-        default:
-          $ref: "#/components/responses/DefaultErrorResponse"
-
   /locations/{location_id}/files/{file_id}:
     get:
       summary: Gets download link for file at location
@@ -337,19 +281,36 @@ paths:
             enum:
               - presigned
               - s3
+        - name: file_size
+          in: query
+          required: false
+          description: will be required once all legacy services are gone or updated. if null returns the PresignedLinkEnveloped, else returns FileUploadEnveloped
+          schema:
+            type: integer
+            format: int64
+            minimum: 0
       responses:
         "200":
-          description: "Returns presigned link"
+          description: "Creates and returns file upload object"
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/PresignedLinkEnveloped"
+                oneOf:
+                  - $ref: "#/components/schemas/PresignedLinkEnveloped"
+                  - $ref: "#/components/schemas/FileUploadEnveloped"
           links:
+            CompleteUpload:
+              operationId: complete_upload_file
+              parameters:
+                path.location_id: "$request.path.location_id"
+                path.file_id: "$request.path.file_id"
+                query.user_id: "$request.query.user_id"
             AbortUpload:
               operationId: abort_upload_file
               parameters:
                 path.location_id: "$request.path.location_id"
                 path.file_id: "$request.path.file_id"
+                query.user_id: "$request.query.user_id"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
     delete:
@@ -377,6 +338,126 @@ paths:
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 
+  /locations/{location_id}/files/{file_id}:abort:
+    post:
+      summary: Asks the server to abort the upload and revert to the last valid version if any
+      operationId: abort_upload_file
+      parameters:
+        - name: file_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: location_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: user_id
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "204":
+          description: Abort OK
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
+  /locations/{location_id}/files/{file_id}:complete:
+    post:
+      summary: Asks the server to complete the upload
+      operationId: complete_upload_file
+      parameters:
+        - name: file_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: location_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: user_id
+          in: query
+          required: true
+          schema:
+            type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - parts
+              properties:
+                parts:
+                  type: array
+                  items:
+                    type: object
+                    required:
+                      - number
+                      - e_tag
+                    properties:
+                      number:
+                        type: integer
+                        minimum: 1
+                      e_tag:
+                        type: string
+      responses:
+        "202":
+          description: Completion of upload is accepted
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/FileUploadCompleteEnveloped"
+          links:
+            CompleteUploadStatus:
+              operationId: is_completed_upload_file
+              parameters:
+                path.location_id: "$request.path.location_id"
+                path.file_id: "$request.path.file_id"
+                path.future_id: "$response.body.data.links.state"
+                query.user_id: "$request.query.user_id"
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
+  /locations/{location_id}/files/{file_id}:complete/futures/{future_id}:
+    post:
+      summary: Check for upload completion
+      operationId: is_completed_upload_file
+      parameters:
+        - name: future_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: file_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: location_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: user_id
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: returns state of upload completion
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/FileUploadCompleteFutureEnveloped"
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
   /simcore-s3:access:
     post:
       summary: Returns the temporary access credentials for the user storage space
@@ -408,7 +489,7 @@ paths:
           schema:
             type: integer
         - name: startswith
-          description: matches starting string of the file_uuid
+          description: matches starting string of the file_id
           in: query
           schema:
             type: string
@@ -874,6 +955,95 @@ components:
       example:
         link: "example_link"
 
+    FileUploadEnveloped:
+      type: object
+      required:
+        - data
+        - error
+      properties:
+        data:
+          $ref: "#/components/schemas/FileUploadSchema"
+        error:
+          nullable: true
+          default: null
+
+    FileUploadSchema:
+      type: object
+      required:
+        - chunk_size
+        - urls
+        - links
+      properties:
+        chunk_size:
+          type: integer
+          format: int64
+          minimum: 0
+        urls:
+          type: array
+          items:
+            type: string
+        links:
+          type: object
+          required:
+            - abort_upload
+            - complete_upload
+          properties:
+            abort_upload:
+              type: string
+            complete_upload:
+              type: string
+
+    FileUploadCompleteEnveloped:
+      type: object
+      required:
+        - data
+        - error
+      properties:
+        data:
+          $ref: "#/components/schemas/FileUploadComplete"
+        error:
+          nullable: true
+          default: null
+
+    FileUploadComplete:
+      type: object
+      required:
+        - links
+      properties:
+        links:
+          type: object
+          required:
+            - state
+          properties:
+            state:
+              type: string
+
+    FileUploadCompleteFutureEnveloped:
+      type: object
+      required:
+        - data
+        - error
+      properties:
+        data:
+          $ref: "#/components/schemas/FileUploadCompleteFuture"
+        error:
+          nullable: true
+          default: null
+
+    FileUploadCompleteFuture:
+      type: object
+      required:
+        - state
+      properties:
+        state:
+          type: string
+          enum:
+            - ok
+            - nok
+        e_tag:
+          type: string
+          nullable: true
+
     S3AccessCredentialsEnveloped:
       type: object
       required: