Merge pull request #958 from ITISFoundation/freeze-to-staging-190718

Freeze to staging 190718
- New: Storage extension for deep copy of project files (#952)
- Enhancement: Auto retrieve (#946)

Author: pcrespov

Makefile

@@ -291,6 +291,11 @@ setup-check: .env .vscode/settings.json
 ## -------------------------------
 # Auxiliary targets.
 
+.PHONY: reset
+# target: reset – Restart docker daemon
+reset:
+	sudo systemctl restart docker
+
 .PHONY: clean
 # target: clean – Cleans all unversioned files in project
 clean: remove-intermediate-file

api/specs/storage/v0/openapi.yaml

@@ -273,7 +273,68 @@ paths:
         default:
           $ref: '#/components/responses/DefaultErrorResponse'
 
+  /simcore-s3/folders:
+    post:
+      tags:
+        - users
+      summary: Deep copies of all data from source to destination project in s3
+      operationId: copy_folders_from_project
+      parameters:
+      - name: user_id
+        in: query
+        required: true
+        schema:
+          type: integer
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                source:
+                  $ref: './openapi.yaml#/components/schemas/Project'
+                destination:
+                  $ref: './openapi.yaml#/components/schemas/Project'
+                nodes_map:
+                  type: object
+                  description: maps source and destination node uuids
+                  additionalProperties:
+                    type: string
+      responses:
+        '201':
+          description: Data from destination project copied and returns project
+          content:
+            application/json:
+              schema:
+                $ref: './openapi.yaml#/components/schemas/Project'
+        default:
+          $ref: './openapi.yaml#/components/responses/DefaultErrorResponse'
+
+  /simcore-s3/folders/{folder_id}:
+    delete:
+      tags:
+        - users
+      summary: Deletes all objects with folder_id prefix
+      operationId: delete_folders_of_project
+      parameters:
+      - name: folder_id
+        in: path
+        required: true
+        schema:
+          type: string
+      - name: user_id
+        in: query
+        required: true
+        schema:
+          type: string
+      responses:
+        '204':
+          description: folder has been successfully deleted
+
 components:
+  schemas:
+    Project:
+      $ref: '../../webserver/v0/openapi-projects.yaml#/components/schemas/Project'
   responses:
     OK_NoContent_204:
       description: everything is OK, but there is no content to return

api/tests/conftest.py

@@ -18,13 +18,12 @@
 OPENAPI_MAIN_FILENAME = 'openapi.yaml'
 
 
-def current_dir():
-    return Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
+current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
 
 
 @pytest.fixture(scope='session')
 def here():
-    return current_dir()
+    return current_dir
 
 
 @pytest.fixture(scope='session')
@@ -91,7 +90,7 @@ def list_openapi_tails():
         SEE api_specs_tail to get one at a time
     """
     tails = []
-    specs_dir = _api_specs_dir_impl(current_dir())
+    specs_dir = _api_specs_dir_impl(current_dir)
     for tail in _all_api_specs_tails_impl(specs_dir):
         specs = load_specs( specs_dir / tail)
         if not is_json_schema(specs):

scripts/demo/confirmations-invitations.csv

@@ -2,20 +2,20 @@ code,user_id,action,data,created_at
 AOuAejUGDv34i9QtxYK61V7GZmCE4B,1,INVITATION,"{
 ""guest"": ""[email protected]"" ,
 ""host"" : ""[email protected]""
-}",2019-07-08 13:18:16.368438
+}",2019-07-15 18:17:33.934830
 uQhnK20tuXWdleIRhZaBcmrWaIrb2p,1,INVITATION,"{
 ""guest"": ""[email protected]"" ,
 ""host"" : ""[email protected]""
-}",2019-07-08 13:18:16.368446
+}",2019-07-15 18:17:33.934839
 weedI0YvR6tMA7XEpaxgJZT2Z8SCUy,1,INVITATION,"{
 ""guest"": ""[email protected]"" ,
 ""host"" : ""[email protected]""
-}",2019-07-08 13:18:16.368449
+}",2019-07-15 18:17:33.934843
 Q9m5C98ALYZDr1BjilkaaXWSMKxU21,1,INVITATION,"{
 ""guest"": ""[email protected]"" ,
 ""host"" : ""[email protected]""
-}",2019-07-08 13:18:16.368453
+}",2019-07-15 18:17:33.934846
 jvhSQfoAAfin4htKgvvRYi3pkYdPhM,1,INVITATION,"{
 ""guest"": ""[email protected]"" ,
 ""host"" : ""[email protected]""
-}",2019-07-08 13:18:16.368459
+}",2019-07-15 18:17:33.934850

scripts/demo/create_portal_markdown.py

@@ -44,7 +44,7 @@
 log = logging.getLogger(__name__)
 
 params = {}
-params["154fb4ad-4913-478f-af04-19725db901a7"] = {'stimulation_period_secs': '1200'}
+params["194bb264-a717-11e9-9dff-02420aff2767"] = {'stimulation_mode': '1', 'stimulation_level': '0.5'}
 
 @contextmanager
 def _open(filepath):
@@ -62,10 +62,13 @@ def write_list(hostname, url, data, fh):
     print("## studies available @{}".format(hostname), file=fh)
     print("", file=fh)
     for prj in data:
+        prj['msg'] = ''
         study_url = origin.with_path("study/{uuid}".format(**prj))
         if prj['uuid'] in params:
-            study_url = study_url.with_query(**params[prj['uuid']])
-        print("- [{name}]({study_url})".format(study_url=str(study_url), **prj), file=fh)
+            prj_params = params[prj['uuid']]
+            study_url = study_url.with_query(**prj_params)
+            prj['msg'] = 'with ' + "and ".join([f"{k}={v} " for k,v in prj_params.items()])
+        print("- [{name}]({study_url}) {msg}".format(study_url=str(study_url), **prj), file=fh)
     print("", file=fh)
 
 

scripts/demo/portal_markdown.md

@@ -1,4 +1,4 @@
-<!-- Generated by create_portal_markdown.py on 2019-07-08 15:15:04.106152 -->
+<!-- Generated by create_portal_markdown.py on 2019-07-15 16:17:33.906421 -->
 # THE PORTAL Emulator
 
 This pages is for testing purposes for issue [#715](https://github.com/ITISFoundation/osparc-simcore/issues/715)
@@ -8,11 +8,11 @@ This pages is for testing purposes for issue [#715](https://github.com/ITISFound
 
 ## studies available @master
 
-- [ISAN2019: 3D Paraview](http://master.osparc.io/study/template-uuid-518d-a25d-8887bcae93f8)
-- [ISAN: MattWard use case](http://master.osparc.io/study/template-uuid-5a9e-9580-c53d92d18803)
-- [ISAN2019 - opencor-py model](http://master.osparc.io/study/154fb4ad-4913-478f-af04-19725db901a7?stimulation_period_secs=1200)
-- [ISAN: 2D Plot](http://master.osparc.io/study/template-uuid-5716-bedd-b409bb021760)
-- [ISAN: UCDavis use case: 0D](http://master.osparc.io/study/template-uuid-5d82-b08d-d39c436ca738)
+- [ISAN2019: 3D Paraview](http://master.osparc.io/study/template-uuid-518d-a25d-8887bcae93f8) 
+- [ISAN: osparc-opencor](http://master.osparc.io/study/194bb264-a717-11e9-9dff-02420aff2767?stimulation_mode=1&stimulation_level=0.5) with stimulation_mode=1 and stimulation_level=0.5 
+- [ISAN: 2D Plot](http://master.osparc.io/study/template-uuid-5716-bedd-b409bb021760) 
+- [ISAN: UCDavis use case: 0D](http://master.osparc.io/study/template-uuid-5d82-b08d-d39c436ca738) 
+- [ISAN: MattWard use case](http://master.osparc.io/study/template-uuid-5a9e-9580-c53d92d18803) 
 
 ## studies available @staging
 

scripts/demo/template-projects/converter.py

@@ -35,7 +35,7 @@ def load_projects(csv_path:Path ):
 
     # process
     for db_prj in db_projects:
-        if int(db_prj['published'])==1:
+        if int(db_prj.get('published', 0) or 0)==1:
             prj = _convert_to_schema_names(db_prj)
 
             # jsonifies