Merge branch 'master' into add-listing-of-comp-runs

Author: matusdrobuliak66

services/static-webserver/client/source/class/osparc/dashboard/NewPlusMenu.js

@@ -260,14 +260,15 @@ qx.Class.define("osparc.dashboard.NewPlusMenu", {
       }
     },
 
-    __addFromResourceButton: function(menuButton, category) {
-      let idx = null;
+    __addFromResourceButton: function(menuButton, category, idx = null) {
       if (category) {
         idx = this.__getLastIdxFromCategory(category);
       }
-      if (idx) {
+      if (category && idx) {
         menuButton["categoryId"] = category;
         this.addAt(menuButton, idx+1);
+      } else if (idx) {
+        this.addAt(menuButton, idx);
       } else {
         this.addAt(menuButton, this.__itemIdx);
         this.__itemIdx++;
@@ -369,9 +370,12 @@ qx.Class.define("osparc.dashboard.NewPlusMenu", {
         addListenerToButton(menuButton, latestMetadata);
       } else if ("myMostUsed" in buttonConfig) {
         const excludeFrontend = true;
-        const excludeDeprecated = true
+        const excludeDeprecated = true;
+        const old = this.__itemIdx;
+        this.__itemIdx += buttonConfig["myMostUsed"];
         osparc.store.Services.getServicesLatestList(excludeFrontend, excludeDeprecated)
-          .then(servicesList => {
+          .then(srvList => {
+            const servicesList = srvList.filter(srv => srv !== null);
             osparc.service.Utils.sortObjectsBasedOn(servicesList, {
               "sort": "hits",
               "order": "down"
@@ -385,7 +389,7 @@ qx.Class.define("osparc.dashboard.NewPlusMenu", {
                   allowGrowX: true,
                 });
                 this.__addIcon(menuButton, null, latestMetadata);
-                this.__addFromResourceButton(menuButton, buttonConfig["category"]);
+                this.__addFromResourceButton(menuButton, buttonConfig["category"], old+i);
                 addListenerToButton(menuButton, latestMetadata);
               }
             }

services/static-webserver/client/source/class/osparc/dashboard/ServiceBrowser.js

@@ -66,7 +66,7 @@ qx.Class.define("osparc.dashboard.ServiceBrowser", {
       const excludeFrontend = true;
       const excludeDeprecated = true
       osparc.store.Services.getServicesLatestList(excludeFrontend, excludeDeprecated)
-        .then(servicesList => this.__setServicesToList(servicesList));
+        .then(servicesList => this.__setServicesToList(servicesList.filter(service => service !== null)));
     },
 
     _updateServiceData: function(serviceData) {

services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js

@@ -888,78 +888,10 @@ qx.Class.define("osparc.dashboard.StudyBrowser", {
             // this one is different since it groups all new buttons in one new button
             this.__addTIPPlusButton();
             break;
-          default:
-            this.__addPlusButtons();
-            break;
         }
       }
     },
 
-    __addPlusButtons: function() {
-      const plusButtonConfig = osparc.store.Products.getInstance().getNewStudiesUiConfig();
-      if (plusButtonConfig) {
-        plusButtonConfig["resources"].forEach(newStudyData => {
-          if (newStudyData["resourceType"] === "study") {
-            this.__addEmptyStudyPlusButton(newStudyData);
-          } else if (newStudyData["resourceType"] === "service") {
-            this.__addNewStudyFromServiceButton(newStudyData);
-          }
-        });
-      }
-    },
-
-    __addEmptyStudyPlusButton: function(newStudyData) {
-      const mode = this._resourcesContainer.getMode();
-      const defTitle = this.tr("Empty") + " " + osparc.product.Utils.getStudyAlias({
-        firstUpperCase: true
-      });
-      const title = newStudyData["title"] || defTitle;
-      const desc = newStudyData["description"] || this.tr("Start with an empty study");
-      const newEmptyStudyBtn = (mode === "grid") ? new osparc.dashboard.GridButtonNew(title, desc) : new osparc.dashboard.ListButtonNew(title, desc);
-      newEmptyStudyBtn.setCardKey("new-study");
-      newEmptyStudyBtn.subscribeToFilterGroup("searchBarFilter");
-      osparc.utils.Utils.setIdToWidget(newEmptyStudyBtn, newStudyData["idToWidget"]);
-      newEmptyStudyBtn.addListener("tap", () => this.__newEmptyStudyBtnClicked(newStudyData["newStudyLabel"]));
-      this._resourcesContainer.addNonResourceCard(newEmptyStudyBtn);
-    },
-
-    __addNewStudyFromServiceButton: function(newStudyData) {
-      if ("expectedKey" in newStudyData) {
-        const key = newStudyData["expectedKey"];
-        const latestMetadata = osparc.store.Services.getLatest(key);
-        if (!latestMetadata) {
-          return;
-        }
-        const title = newStudyData.title + " " + osparc.service.Utils.extractVersionDisplay(latestMetadata);
-        const desc = newStudyData.description;
-        const mode = this._resourcesContainer.getMode();
-        const newStudyFromServiceButton = (mode === "grid") ? new osparc.dashboard.GridButtonNew(title, desc) : new osparc.dashboard.ListButtonNew(title, desc);
-        newStudyFromServiceButton.setCardKey("new-"+key);
-        if (newStudyData["idToWidget"]) {
-          osparc.utils.Utils.setIdToWidget(newStudyFromServiceButton, newStudyData["idToWidget"]);
-        }
-        newStudyFromServiceButton.addListener("tap", () => this.__newStudyFromServiceBtnClicked(latestMetadata["key"], latestMetadata["version"], newStudyData.newStudyLabel));
-        this._resourcesContainer.addNonResourceCard(newStudyFromServiceButton);
-      } else if ("myMostUsed" in newStudyData) {
-        const excludeFrontend = true;
-        const excludeDeprecated = true
-        osparc.store.Services.getServicesLatestList(excludeFrontend, excludeDeprecated)
-          .then(servicesList => {
-            osparc.service.Utils.sortObjectsBasedOn(servicesList, {
-              "sort": "hits",
-              "order": "down"
-            });
-            for (let i=0; i<newStudyData["myMostUsed"]; i++) {
-              const latestMetadata = servicesList[i];
-              const mode = this._resourcesContainer.getMode();
-              const newStudyFromServiceButton = (mode === "grid") ? new osparc.dashboard.GridButtonNew(latestMetadata["name"]) : new osparc.dashboard.ListButtonNew(latestMetadata["name"]);
-              newStudyFromServiceButton.addListener("tap", () => this.__newStudyFromServiceBtnClicked(latestMetadata["key"], latestMetadata["version"], latestMetadata["name"]));
-              this._resourcesContainer.addNonResourceCard(newStudyFromServiceButton);
-            }
-          });
-      }
-    },
-
     __addTIPPlusButton: function() {
       const mode = this._resourcesContainer.getMode();
       const title = this.tr("New Plan");

services/static-webserver/client/source/class/osparc/workbench/ServiceCatalog.js

@@ -201,7 +201,7 @@ qx.Class.define("osparc.workbench.ServiceCatalog", {
       const excludeDeprecated = true;
       osparc.store.Services.getServicesLatestList(excludeFrontend, excludeDeprecated)
         .then(servicesList => {
-          this.__servicesLatest = servicesList;
+          this.__servicesLatest = servicesList.filter(service => service !== null);
           this.__updateList();
         });
     },

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -1312,6 +1312,7 @@ async def create_s3_export(
 
             await create_and_upload_export(
                 get_s3_client(self.app),
+                ProjectRepository.instance(get_db_engine(self.app)),
                 self.simcore_bucket_name,
                 source_object_keys=source_object_keys,
                 destination_object_keys=destination_object_key,

services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py

@@ -8,8 +8,9 @@
 from aws_library.s3._constants import STREAM_READER_CHUNK_SIZE
 from aws_library.s3._models import S3ObjectKey
 from models_library.api_schemas_storage.storage_schemas import S3BucketName
-from models_library.projects import ProjectID
+from models_library.projects import ProjectID, ProjectIDStr
 from models_library.projects_nodes_io import (
+    NodeIDStr,
     SimcoreS3DirectoryID,
     SimcoreS3FileID,
     StorageFileID,
@@ -27,6 +28,7 @@
 from ..models import FileMetaData, FileMetaDataAtDB, GenericCursor, PathMetaData
 from ..modules.db.access_layer import AccessLayerRepository
 from ..modules.db.file_meta_data import FileMetaDataRepository, TotalChildren
+from ..modules.db.projects import ProjectRepository
 from .utils import convert_db_to_model
 
 
@@ -165,17 +167,55 @@ def _base_path_parent(base_path: UserSelectionStr, s3_object: S3ObjectKey) -> st
     return f"{result}"
 
 
+def _get_project_ids(user_selecton: set[UserSelectionStr]) -> list[ProjectID]:
+    results = []
+    for selected in user_selecton:
+        project_id = ProjectID(Path(selected).parts[0])
+        results.append(project_id)
+    return results
+
+
+def _replace_node_id_project_id_in_path(
+    ids_names_map: dict[ProjectID, dict[ProjectIDStr | NodeIDStr, str]], path: str
+) -> str:
+    path_parts = Path(path).parts
+    if len(path_parts) == 0:
+        return path
+
+    if len(path_parts) == 1:
+        return ids_names_map[ProjectID(path)][path].replace("/", "_")
+
+    project_id_str = path_parts[0]
+    project_id = ProjectID(project_id_str)
+    node_id_str = path_parts[1]
+    return "/".join(
+        (
+            ids_names_map[project_id][project_id_str].replace("/", "_"),
+            ids_names_map[project_id][node_id_str].replace("/", "_"),
+            *path_parts[2:],
+        )
+    )
+
+
 async def create_and_upload_export(
     s3_client: SimcoreS3API,
+    project_repository: ProjectRepository,
     bucket: S3BucketName,
     *,
     source_object_keys: set[tuple[UserSelectionStr, StorageFileID]],
     destination_object_keys: StorageFileID,
     progress_bar: ProgressBarData,
 ) -> None:
+    ids_names_map = await project_repository.get_project_id_and_node_id_to_names_map(
+        project_uuids=_get_project_ids(user_selecton={x[0] for x in source_object_keys})
+    )
+
     archive_entries: ArchiveEntries = [
         (
-            _base_path_parent(selection, s3_object),
+            _base_path_parent(
+                _replace_node_id_project_id_in_path(ids_names_map, selection),
+                _replace_node_id_project_id_in_path(ids_names_map, s3_object),
+            ),
             await s3_client.get_bytes_streamer_from_object(bucket, s3_object),
         )
         for (selection, s3_object) in source_object_keys

services/storage/tests/unit/test_simcore_s3_dsm_utils.py

@@ -1,10 +1,16 @@
 from pathlib import Path
+from typing import Final
+from uuid import UUID
 
 import pytest
 from aws_library.s3._models import S3ObjectKey
+from models_library.projects import ProjectID, ProjectIDStr
+from models_library.projects_nodes_io import NodeIDStr
+from simcore_service_storage.models import NodeID
 from simcore_service_storage.utils.simcore_s3_dsm_utils import (
     UserSelectionStr,
     _base_path_parent,
+    _replace_node_id_project_id_in_path,
     compute_file_id_prefix,
     ensure_user_selection_from_same_base_directory,
 )
@@ -73,3 +79,53 @@ def test_ensure_user_selection_from_same_base_directory(
         ensure_user_selection_from_same_base_directory([f"{x}" for x in user_selection])
         == expected
     )
+
+
+_PID1: Final[ProjectID] = UUID(int=1)
+_PID2: Final[ProjectID] = UUID(int=2)
+_NID1: Final[NodeID] = UUID(int=3)
+_NID2: Final[NodeID] = UUID(int=4)
+_IDS_NAMES_MAP: Final[dict[ProjectID, dict[ProjectIDStr | NodeIDStr, str]]] = {
+    _PID1: {
+        f"{_PID1}": "project one",
+        f"{_NID1}": "project one -> node one",
+        f"{_NID2}": "project one -> node two",
+    },
+    _PID2: {
+        f"{_PID2}": "/project/two/",
+        f"{_NID1}": "/project/two/->/node/one/",
+        f"{_NID2}": "/project/two/->/node/two/",
+    },
+}
+
+
+@pytest.mark.parametrize(
+    "path, expected",
+    [
+        ("", ""),
+        (f"{_PID1}", "project one"),
+        (f"{_PID1}/{_NID1}", "project one/project one -> node one"),
+        (f"{_PID1}/{_NID1}/something", "project one/project one -> node one/something"),
+        (f"{_PID1}/{_NID1}/{_NID2}", f"project one/project one -> node one/{_NID2}"),
+        (
+            f"{_PID1}/{_NID1}/{_NID2}/something",
+            f"project one/project one -> node one/{_NID2}/something",
+        ),
+        (f"{_PID2}", "_project_two_"),
+        (f"{_PID2}/{_NID1}", "_project_two_/_project_two_->_node_one_"),
+        (
+            f"{_PID2}/{_NID1}/something",
+            "_project_two_/_project_two_->_node_one_/something",
+        ),
+        (
+            f"{_PID2}/{_NID1}/{_NID2}",
+            f"_project_two_/_project_two_->_node_one_/{_NID2}",
+        ),
+        (
+            f"{_PID2}/{_NID1}/{_NID2}/something",
+            f"_project_two_/_project_two_->_node_one_/{_NID2}/something",
+        ),
+    ],
+)
+def test__replace_node_id_project_id_in_path(path: str, expected: str):
+    assert _replace_node_id_project_id_in_path(_IDS_NAMES_MAP, path) == expected