Merge remote-tracking branch 'upstream/master' into extract-celery-code

Author: giancarloromeo

.github/workflows/ci-testing-deploy.yml

@@ -342,7 +342,7 @@ jobs:
   unit-test-webserver-02:
     needs: changes
     if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' || github.event.inputs.force_all_builds == 'true' }}
-    timeout-minutes: 25 # if this timeout gets too small, then split the tests
+    timeout-minutes: 35 # if this timeout gets too small, then split the tests
     name: "[unit] webserver 02"
     runs-on: ${{ matrix.os }}
     strategy:

.vscode/settings.template.json

@@ -1,6 +1,7 @@
 // This is a template. Clone and replace extension ".template.json" by ".json"
 {
   "autoDocstring.docstringFormat": "pep257",
+
   "editor.tabSize": 2,
   "editor.insertSpaces": true,
   "editor.detectIndentation": false,

services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/core.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 from pprint import pformat
 from types import TracebackType
-from typing import cast
+from typing import Final, cast
 from uuid import uuid4
 
 from aiodocker import Docker
@@ -32,7 +32,6 @@
 from ..utils.dask import TaskPublisher
 from ..utils.files import (
     check_need_unzipping,
-    log_partial_file_content,
     pull_file_from_remote,
     push_file_to_remote,
 )
@@ -49,8 +48,8 @@
 from .task_shared_volume import TaskSharedVolumes
 
 _logger = logging.getLogger(__name__)
-_CONTAINER_WAIT_TIME_SECS = 2
-_MAX_LOGGED_FILE_CHARS = 40
+CONTAINER_WAIT_TIME_SECS = 2
+_TASK_PROCESSING_PROGRESS_WEIGHT: Final[float] = 0.99
 
 
 @dataclass(kw_only=True, frozen=True, slots=True)
@@ -148,17 +147,11 @@ async def _retrieve_output_data(
             upload_tasks = []
             for output_params in output_data.values():
                 if isinstance(output_params, FileUrl):
-                    assert (
+                    assert (  # nosec
                         output_params.file_mapping
                     ), f"{output_params.model_dump_json(indent=1)} expected resolved in TaskOutputData.from_task_output"
 
                     src_path = task_volumes.outputs_folder / output_params.file_mapping
-                    await log_partial_file_content(
-                        src_path,
-                        logger=_logger,
-                        log_level=logging.DEBUG,
-                        max_chars=_MAX_LOGGED_FILE_CHARS,
-                    )
                     upload_tasks.append(
                         push_file_to_remote(
                             src_path,
@@ -274,7 +267,7 @@ async def run(self, command: list[str]) -> TaskOutputData:
                 )
                 # wait until the container finished, either success or fail or timeout
                 while (container_data := await container.show())["State"]["Running"]:
-                    await asyncio.sleep(_CONTAINER_WAIT_TIME_SECS)
+                    await asyncio.sleep(CONTAINER_WAIT_TIME_SECS)
                 if container_data["State"]["ExitCode"] > os.EX_OK:
                     raise ServiceRuntimeError(
                         service_key=self.task_parameters.image,

services/dask-sidecar/src/simcore_service_dask_sidecar/utils/files.py

@@ -20,7 +20,7 @@
 from settings_library.s3 import S3Settings
 from yarl import URL
 
-_logger = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 HTTP_FILE_SYSTEM_SCHEMES: Final = ["http", "https"]
 S3_FILE_SYSTEM_SCHEMES: Final = ["s3", "s3a"]
@@ -208,7 +208,7 @@ async def pull_file_from_remote(
             await log_publishing_cb(
                 f"Uncompressing '{download_dst_path.name}'...", logging.INFO
             )
-            _logger.debug(
+            logger.debug(
                 "%s is a zip file and will be now uncompressed", download_dst_path
             )
             with repro_zipfile.ReproducibleZipFile(download_dst_path, "r") as zip_obj:
@@ -258,7 +258,7 @@ async def _push_file_to_remote(
     log_publishing_cb: LogPublishingCB,
     s3_settings: S3Settings | None,
 ) -> None:
-    _logger.debug("Uploading %s to %s...", file_to_upload, dst_url)
+    logger.debug("Uploading %s to %s...", file_to_upload, dst_url)
     assert dst_url.path  # nosec
 
     storage_kwargs: S3FsSettingsDict | dict[str, Any] = {}
@@ -306,7 +306,7 @@ async def push_file_to_remote(
                 await asyncio.get_event_loop().run_in_executor(
                     None, zfp.write, src_path, src_path.name
                 )
-            _logger.debug("%s created.", archive_file_path)
+            logger.debug("%s created.", archive_file_path)
             assert archive_file_path.exists()  # nosec
             file_to_upload = archive_file_path
             await log_publishing_cb(
@@ -319,7 +319,7 @@ async def push_file_to_remote(
         )
 
         if dst_url.scheme in HTTP_FILE_SYSTEM_SCHEMES:
-            _logger.debug("destination is a http presigned link")
+            logger.debug("destination is a http presigned link")
             await _push_file_to_http_link(file_to_upload, dst_url, log_publishing_cb)
         else:
             await _push_file_to_remote(
@@ -330,22 +330,3 @@ async def push_file_to_remote(
         f"Upload of '{src_path.name}' to '{dst_url.path.strip('/')}' complete",
         logging.INFO,
     )
-
-
-async def log_partial_file_content(
-    src_path: Path, *, logger: logging.Logger, log_level: int, max_chars: int
-) -> None:
-    if max_chars < 0:
-        msg = "max_chars must be non-negative"
-        raise ValueError(msg)
-    if max_chars == 0:
-        return
-    if not src_path.exists():
-        logger.log(log_level, "file does not exist: %s", src_path)
-        return
-    async with aiofiles.open(src_path, encoding="utf-8") as f:
-        content = await f.read(max_chars + 1)
-    if len(content) > max_chars:
-        logger.log(log_level, "file content (truncated): %s...", content[:max_chars])
-    else:
-        logger.log(log_level, "file content: %s", content)

services/dask-sidecar/tests/unit/test_utils_files.py

@@ -4,7 +4,6 @@
 
 import asyncio
 import hashlib
-import logging
 import mimetypes
 import zipfile
 from collections.abc import AsyncIterable
@@ -22,7 +21,6 @@
 from settings_library.s3 import S3Settings
 from simcore_service_dask_sidecar.utils.files import (
     _s3fs_settings_from_s3_settings,
-    log_partial_file_content,
     pull_file_from_remote,
     push_file_to_remote,
 )
@@ -513,48 +511,3 @@ async def test_push_file_to_remote_creates_reproducible_zip_archive(
     assert dst_path2.exists()
 
     assert _compute_hash(dst_path1) == _compute_hash(dst_path2)
-
-
-async def test_log_partial_file_content(
-    tmp_path: Path, caplog: pytest.LogCaptureFixture
-):
-    # Create a file with known content
-    file_content = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    file_path = tmp_path / "testfile.txt"
-    file_path.write_text(file_content)
-    logger = logging.getLogger("pytest.utils.files")
-
-    # Case 1: file longer than max_chars
-    with caplog.at_level(logging.DEBUG, logger=logger.name):
-        await log_partial_file_content(
-            file_path, logger=logger, log_level=logging.DEBUG, max_chars=10
-        )
-    assert any(
-        "file content (truncated): abcdefghij..." in record.getMessage()
-        for record in caplog.records
-    )
-
-    # Case 2: file shorter than max_chars
-    caplog.clear()
-    short_content = "short"
-    short_file = tmp_path / "short.txt"
-    short_file.write_text(short_content)
-    with caplog.at_level(logging.DEBUG, logger=logger.name):
-        await log_partial_file_content(
-            short_file, logger=logger, log_level=logging.DEBUG, max_chars=10
-        )
-    assert any(
-        "file content: short" in record.getMessage() for record in caplog.records
-    )
-
-    # Case 3: file does not exist
-    caplog.clear()
-    non_existent = tmp_path / "doesnotexist.txt"
-    with caplog.at_level(logging.DEBUG, logger=logger.name):
-        await log_partial_file_content(
-            non_existent, logger=logger, log_level=logging.DEBUG, max_chars=10
-        )
-    assert any(
-        f"file does not exist: {non_existent}" in record.getMessage()
-        for record in caplog.records
-    )

services/static-webserver/client/source/class/osparc/dashboard/GroupedCardContainer.js

@@ -23,7 +23,7 @@ qx.Class.define("osparc.dashboard.GroupedCardContainer", {
 
     this._setLayout(new qx.ui.layout.VBox());
 
-    const showAllButton = this.__showAllButton = new qx.ui.form.Button().set({
+    const showAllButton = this.__expandButton = new qx.ui.form.Button().set({
       margin: 10,
       marginBottom: 5
     });
@@ -70,7 +70,7 @@ qx.Class.define("osparc.dashboard.GroupedCardContainer", {
   },
 
   members: {
-    __showAllButton: null,
+    __expandButton: null,
     __contentContainer: null,
 
     _createChildControlImpl: function(id) {
@@ -116,7 +116,7 @@ qx.Class.define("osparc.dashboard.GroupedCardContainer", {
     __createContentContainer: function() {
       let contentContainer = null;
       const expanded = this.isExpanded();
-      const showAllBtn = this.__showAllButton;
+      const showAllBtn = this.__expandButton;
       if (expanded) {
         contentContainer = new osparc.dashboard.CardContainer();
         showAllBtn.show();
@@ -174,6 +174,10 @@ qx.Class.define("osparc.dashboard.GroupedCardContainer", {
       return this.__contentContainer;
     },
 
+    getExpandButton: function() {
+      return this.__expandButton;
+    },
+
     // overridden
     add: function(child, idx) {
       if (osparc.dashboard.CardContainer.isValidCard(child)) {

services/static-webserver/client/source/class/osparc/dashboard/ResourceBrowserBase.js

@@ -324,7 +324,7 @@ qx.Class.define("osparc.dashboard.ResourceBrowserBase", {
       this._addToLayout(resourcesContainer);
     },
 
-    __groupByChanged: function(groupBy) {
+    _groupByChanged: function(groupBy) {
       // if cards are grouped they need to be in grid mode
       this._resourcesContainer.setMode("grid");
       this.__viewModeLayout.setVisibility(groupBy ? "excluded" : "visible");
@@ -356,27 +356,34 @@ qx.Class.define("osparc.dashboard.ResourceBrowserBase", {
 
       const dontGroup = new qx.ui.menu.RadioButton(this.tr("None"));
       osparc.utils.Utils.setIdToWidget(dontGroup, "groupByNone");
-      dontGroup.addListener("execute", () => this.__groupByChanged(null));
+      dontGroup.addListener("execute", () => this._groupByChanged(null));
 
       groupByMenu.add(dontGroup);
       groupOptions.add(dontGroup);
 
       if (this._resourceType === "template") {
-        const tagByGroup = new qx.ui.menu.RadioButton(this.tr("Tags"));
-        tagByGroup.addListener("execute", () => this.__groupByChanged("tags"));
-        groupByMenu.add(tagByGroup);
-        groupOptions.add(tagByGroup);
+        const groupByTag = new qx.ui.menu.RadioButton(this.tr("Tags"));
+        groupByTag.addListener("execute", () => this._groupByChanged("tags"));
+        groupByMenu.add(groupByTag);
+        groupOptions.add(groupByTag);
         if (
           osparc.product.Utils.isProduct("s4l") ||
           osparc.product.Utils.isProduct("s4lacad") ||
           osparc.product.Utils.isProduct("s4llite")
         ) {
-          tagByGroup.execute();
+          groupByTag.execute();
         }
+      } else if (this._resourceType === "service" && osparc.product.Utils.groupServices()) {
+        const groupByFeatured = new qx.ui.menu.RadioButton(this.tr("Featured"));
+        groupByFeatured.addListener("execute", () => this._groupByChanged("groupedServices"));
+        groupByMenu.add(groupByFeatured);
+        groupOptions.add(groupByFeatured);
+        groupByFeatured.execute();
+        groupByButton.exclude(); // don't let users change the grouping
       }
 
       const groupByShared = new qx.ui.menu.RadioButton(this.tr("Shared with"));
-      groupByShared.addListener("execute", () => this.__groupByChanged("shared"));
+      groupByShared.addListener("execute", () => this._groupByChanged("shared"));
       groupByMenu.add(groupByShared);
       groupOptions.add(groupByShared);
 

services/static-webserver/client/source/class/osparc/dashboard/ResourceContainerManager.js

@@ -58,7 +58,7 @@ qx.Class.define("osparc.dashboard.ResourceContainerManager", {
     },
 
     groupBy: {
-      check: [null, "tags", "shared"],
+      check: [null, "tags", "shared", "groupedServices"],
       init: null,
       nullable: true
     }
@@ -309,8 +309,11 @@ qx.Class.define("osparc.dashboard.ResourceContainerManager", {
           case "shared":
             groupTitle = "Not Shared";
             break;
+          case "groupedServices":
+            groupTitle = "Misc";
+            break;
         }
-        const noGroupContainer = this.__createGroupContainer("no-group", groupTitle, "transparent");
+        const noGroupContainer = this.__createGroupContainer("no-group", groupTitle, "text");
         this.__groupedContainers.add(noGroupContainer);
         this._add(this.__groupedContainers);
       } else {
@@ -525,12 +528,55 @@ qx.Class.define("osparc.dashboard.ResourceContainerManager", {
       }
     },
 
+    __groupByGroupedServices: function(cards, resourceData) {
+      const groupedServicesConfig = osparc.store.Products.getInstance().getGroupedServicesUiConfig();
+      if (groupedServicesConfig == null) {
+        return;
+      }
+
+      // create group containers for each category
+      groupedServicesConfig["categories"].forEach(category => {
+        if (this.__getGroupContainer(category["id"]) === null) {
+          const groupContainer = this.__createGroupContainer(category["id"], category["title"], category["color"]);
+          groupContainer.setHeaderIcon("@FontAwesome5Solid/tag/24");
+          this.__groupedContainers.add(groupContainer);
+        }
+      });
+
+      // get the right container
+      let container = null;
+      const serviceKey = resourceData["key"];
+      if (serviceKey) {
+        const groupInfo = groupedServicesConfig["services"].find(serviceInfo => serviceInfo["serviceKey"] === serviceKey);
+        if (groupInfo) {
+          container = this.__getGroupContainer(groupInfo["category"]);
+        }
+      }
+      if (container === null) {
+        container = this.__getGroupContainer("no-group");
+        container.setHeaderIcon("@FontAwesome5Solid/tag/24");
+      }
+
+      // create the card and add it to the container
+      const card = this.__createCard(resourceData);
+      this.__addCardToContainer(card, container);
+      cards.push(card);
+
+      this.__moveNoGroupToLast();
+      this.__groupedContainersList.forEach(groupedContainer => {
+        groupedContainer.setExpanded(true);
+        groupedContainer.getExpandButton().exclude();
+      });
+    },
+
     __resourceToCards: function(resourceData) {
       const cardsCreated = [];
       if (this.getGroupBy() === "tags") {
         this.__groupByTags(cardsCreated, resourceData);
       } else if (this.getGroupBy() === "shared") {
         this.__groupByShareWith(cardsCreated, resourceData);
+      } else if (this.getGroupBy() === "groupedServices") {
+        this.__groupByGroupedServices(cardsCreated, resourceData);
       } else {
         const card = this.__createCard(resourceData);
         this.__addCardToContainer(card, this.__nonGroupedContainer);

services/static-webserver/client/source/class/osparc/desktop/preferences/window/ShowAPIKey.js

@@ -76,7 +76,7 @@ qx.Class.define("osparc.desktop.preferences.window.ShowAPIKey", {
       const hBox = this.__createEntry(title);
       if (label) {
         // partially hide the key and secret
-        hBox.getChildren()[1].setValue(label.substring(1, 8) + "****")
+        hBox.getChildren()[1].setValue(label.substring(0, 8) + "****")
       }
       return hBox;
     },