Merge branch 'master' into handle_function_errors_api

Author: mergify[bot]

packages/simcore-sdk/requirements/_base.in

@@ -13,9 +13,10 @@
 aiocache
 aiofiles
 aiohttp
+httpx
 packaging
 pint
-sqlalchemy[asyncio]
 pydantic[email]
+sqlalchemy[asyncio]
 tenacity
 tqdm

packages/simcore-sdk/requirements/_base.txt

@@ -44,6 +44,7 @@ anyio==4.8.0
     # via
     #   fast-depends
     #   faststream
+    #   httpx
 arrow==1.3.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
@@ -72,6 +73,8 @@ certifi==2025.1.31
     #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   httpcore
+    #   httpx
     #   requests
 charset-normalizer==3.4.1
     # via requests
@@ -109,10 +112,32 @@ greenlet==3.1.1
     # via sqlalchemy
 grpcio==1.70.0
     # via opentelemetry-exporter-otlp-proto-grpc
+h11==0.16.0
+    # via httpcore
+httpcore==1.0.9
+    # via httpx
+httpx==0.28.1
+    # via
+    #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -r requirements/_base.in
 idna==3.10
     # via
     #   anyio
     #   email-validator
+    #   httpx
     #   requests
     #   yarl
 importlib-metadata==8.5.0

packages/simcore-sdk/src/simcore_sdk/node_ports_common/file_io_utils.py

@@ -7,10 +7,10 @@
 from typing import IO, Any, Final, Protocol, runtime_checkable
 
 import aiofiles
+import httpx
 from aiohttp import (
     ClientConnectionError,
     ClientError,
-    ClientPayloadError,
     ClientResponse,
     ClientResponseError,
     ClientSession,
@@ -39,6 +39,7 @@
 from tqdm.contrib.logging import tqdm_logging_redirect
 from yarl import URL
 
+from ..config.http_clients import client_request_settings
 from . import exceptions
 from .constants import CHUNK_SIZE
 
@@ -148,13 +149,13 @@ async def __call__(self, log: str) -> None: ...
 
 async def _file_chunk_writer(
     file: Path,
-    response: ClientResponse,
+    response: httpx.Response,
     pbar: tqdm,
     io_log_redirect_cb: LogRedirectCB | None,
     progress_bar: ProgressBarData,
 ):
     async with aiofiles.open(file, "wb") as file_pointer:
-        while chunk := await response.content.read(CHUNK_SIZE):
+        async for chunk in response.aiter_bytes(CHUNK_SIZE):
             await file_pointer.write(chunk)
             if io_log_redirect_cb and pbar.update(len(chunk)):
                 with log_catch(_logger, reraise=False):
@@ -172,7 +173,6 @@ async def _file_chunk_writer(
 
 
 async def download_link_to_file(
-    session: ClientSession,
     url: URL,
     file_path: Path,
     *,
@@ -185,16 +185,25 @@ async def download_link_to_file(
         reraise=True,
         wait=wait_exponential(min=1, max=10),
         stop=stop_after_attempt(num_retries),
-        retry=retry_if_exception_type(ClientConnectionError),
+        retry=retry_if_exception_type(httpx.TransportError),
         before_sleep=before_sleep_log(_logger, logging.WARNING, exc_info=True),
         after=after_log(_logger, log_level=logging.ERROR),
     ):
         with attempt:
             async with AsyncExitStack() as stack:
-                response = await stack.enter_async_context(session.get(url))
-                if response.status == status.HTTP_404_NOT_FOUND:
+                client = await stack.enter_async_context(
+                    httpx.AsyncClient(
+                        timeout=httpx.Timeout(
+                            client_request_settings.HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT
+                        )
+                    )
+                )
+                response = await stack.enter_async_context(
+                    client.stream("GET", f"{url}")
+                )
+                if response.status_code == status.HTTP_404_NOT_FOUND:
                     raise exceptions.InvalidDownloadLinkError(url)
-                if response.status > _VALID_HTTP_STATUS_CODES:
+                if response.status_code > _VALID_HTTP_STATUS_CODES:
                     raise exceptions.TransferError(url)
                 file_path.parent.mkdir(parents=True, exist_ok=True)
                 # SEE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Length
@@ -231,7 +240,7 @@ async def download_link_to_file(
                         sub_progress,
                     )
                     _logger.debug("Download complete")
-                except ClientPayloadError as exc:
+                except httpx.HTTPError as exc:
                     raise exceptions.TransferError(url) from exc
 
 

packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py

@@ -217,7 +217,6 @@ async def download_path_from_s3(
         return await download_file_from_link(
             download_link,
             local_path,
-            client_session=session,
             io_log_redirect_cb=io_log_redirect_cb,
             progress_bar=progress_bar,
         )
@@ -229,7 +228,6 @@ async def download_file_from_link(
     *,
     io_log_redirect_cb: LogRedirectCB | None,
     file_name: str | None = None,
-    client_session: ClientSession | None = None,
     progress_bar: ProgressBarData,
 ) -> Path:
     # a download link looks something like:
@@ -242,15 +240,14 @@ async def download_file_from_link(
 
     if io_log_redirect_cb:
         await io_log_redirect_cb(f"downloading {local_file_path}, please wait...")
-    async with ClientSessionContextManager(client_session) as session:
-        await download_link_to_file(
-            session,
-            download_link,
-            local_file_path,
-            num_retries=NodePortsSettings.create_from_envs().NODE_PORTS_IO_NUM_RETRY_ATTEMPTS,
-            io_log_redirect_cb=io_log_redirect_cb,
-            progress_bar=progress_bar,
-        )
+
+    await download_link_to_file(
+        download_link,
+        local_file_path,
+        num_retries=NodePortsSettings.create_from_envs().NODE_PORTS_IO_NUM_RETRY_ATTEMPTS,
+        io_log_redirect_cb=io_log_redirect_cb,
+        progress_bar=progress_bar,
+    )
     if io_log_redirect_cb:
         await io_log_redirect_cb(f"download of {local_file_path} complete.")
     return local_file_path

services/docker-compose.yml

@@ -1385,10 +1385,12 @@ services:
       - "--metrics.prometheus.entryPoint=metrics"
       - "--entryPoints.http.address=:80"
       - "--entryPoints.http.forwardedHeaders.insecure"
-      - "--entryPoints.http.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805      - "--entryPoints.simcore_api.address=:10081"
+      - "--entryPoints.http.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file uploads
+      - "--entryPoints.http.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file downloads
       - "--entryPoints.simcore_api.address=:10081"
       - "--entryPoints.simcore_api.forwardedHeaders.insecure"
-      - "--entryPoints.simcore_api.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
+      - "--entryPoints.simcore_api.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file uploads
+      - "--entryPoints.simcore_api.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file downloads
       - "--entryPoints.traefik_monitor.address=:8080"
       - "--entryPoints.traefik_monitor.forwardedHeaders.insecure"
       - "--providers.swarm.endpoint=unix:///var/run/docker.sock"
@@ -1401,6 +1403,14 @@ services:
       - "--tracing.addinternals"
       - "--tracing.otlp=true"
       - "--tracing.otlp.http=true"
+    healthcheck:
+      # NOTE: this healthcheck to check if traefik is up and running must be run on the ping entrypoint defined in command!
+      test: traefik healthcheck --ping --ping.entryPoint=ping --entryPoints.ping.address=:9082
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+      start_interval: 1s
     volumes:
       # So that Traefik can listen to the Docker events
       - /var/run/docker.sock:/var/run/docker.sock

services/static-webserver/client/source/class/osparc/dashboard/CardBase.js

@@ -700,7 +700,7 @@ qx.Class.define("osparc.dashboard.CardBase", {
       }
 
       // Block card
-      const cantReadServices = osparc.study.Utils.getCantExecuteServices(services);
+      const cantReadServices = osparc.study.Utils.getCantReadServices(services);
       if (cantReadServices.length) {
         this.setBlocked("UNKNOWN_SERVICES");
         const image = "@FontAwesome5Solid/ban/";
@@ -985,8 +985,17 @@ qx.Class.define("osparc.dashboard.CardBase", {
       throw new Error("Abstract method called!");
     },
 
-    _applyMenu: function(value, old) {
-      throw new Error("Abstract method called!");
+    _applyMenu: function(menu) {
+      const menuButton = this.getChildControl("menu-button");
+      if (menu) {
+        menuButton.setMenu(menu).set({
+          appearance: "menu-wider",
+          position: "bottom-left",
+        });
+        osparc.utils.Utils.setIdToWidget(menu, "studyItemMenuMenu");
+        menu.addListener("appear", () => this.evaluateMenuButtons());
+      }
+      menuButton.setVisibility(menu ? "visible" : "excluded");
     },
 
     _setStudyPermissions: function(accessRights) {

services/static-webserver/client/source/class/osparc/dashboard/GridButtonItem.js

@@ -281,19 +281,5 @@ qx.Class.define("osparc.dashboard.GridButtonItem", {
         }
       }
     },
-
-    // overridden
-    _applyMenu: function(menu, old) {
-      const menuButton = this.getChildControl("menu-button");
-      if (menu) {
-        menuButton.setMenu(menu).set({
-          appearance: "menu-wider",
-          position: "bottom-left",
-        });
-        osparc.utils.Utils.setIdToWidget(menu, "studyItemMenuMenu");
-        this.evaluateMenuButtons();
-      }
-      menuButton.setVisibility(menu ? "visible" : "excluded");
-    }
   }
 });

services/static-webserver/client/source/class/osparc/dashboard/ListButtonItem.js

@@ -295,18 +295,5 @@ qx.Class.define("osparc.dashboard.ListButtonItem", {
         });
       });
     },
-
-    _applyMenu: function(menu, old) {
-      const menuButton = this.getChildControl("menu-button");
-      if (menu) {
-        menuButton.setMenu(menu).set({
-          appearance: "menu-wider",
-          position: "bottom-left",
-        });
-        osparc.utils.Utils.setIdToWidget(menu, "studyItemMenuMenu");
-        this.evaluateMenuButtons();
-      }
-      menuButton.setVisibility(menu ? "visible" : "excluded");
-    }
   }
 });

services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js

@@ -1598,10 +1598,12 @@ qx.Class.define("osparc.dashboard.StudyBrowser", {
       const duplicateStudyButton = this.__getDuplicateMenuButton(studyData);
       menu.add(duplicateStudyButton);
 
-      const convertToPipelineButton = this.__getConvertToPipelineMenuButton(studyData);
-      menu.add(convertToPipelineButton);
+      if (writeAccess && osparc.product.Utils.showConvertToPipeline()) {
+        const convertToPipelineButton = this.__getConvertToPipelineMenuButton(studyData);
+        menu.add(convertToPipelineButton);
+      }
 
-      if (osparc.product.Utils.hasExportCMisEnabled()) {
+      if (osparc.product.Utils.showExportCMis()) {
         const exportStudyButton = this.__getExportCMisMenuButton(studyData);
         menu.add(exportStudyButton);
       }

services/static-webserver/client/source/class/osparc/metadata/ServicesInStudyUpdate.js

@@ -60,7 +60,7 @@ qx.Class.define("osparc.metadata.ServicesInStudyUpdate", {
       osparc.store.Services.getStudyServices(this._studyData["uuid"])
         .then(resp => {
           const services = resp["services"];
-          if (osparc.study.Utils.getCantExecuteServices(services).length) {
+          if (osparc.study.Utils.getCantReadServices(services).length) {
             msg += this.tr("Some services are inaccessible. Please contact the service owner:");
             msg += "<br><br>";
           }