🎨Storage: HA (🚨🚨🚨 test with multiple replicas)

packages/service-library/src/servicelib/fastapi/http_error.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Awaitable, Callable
 from typing import TypeVar
 
@@ -10,6 +11,9 @@
 from fastapi.responses import JSONResponse
 from pydantic import ValidationError
 
+from ..logging_errors import create_troubleshotting_log_kwargs
+from ..status_codes_utils import is_5xx_server_error
+
 validation_error_response_definition["properties"] = {
     "errors": {
         "title": "Validation errors",
@@ -21,6 +25,8 @@
 
 TException = TypeVar("TException")
 
+_logger = logging.getLogger(__name__)
+
 
 def make_http_error_handler_for_exception(
     status_code: int,
@@ -36,12 +42,24 @@ def make_http_error_handler_for_exception(
     SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions
     """
 
-    async def _http_error_handler(_: Request, exc: Exception) -> JSONResponse:
+    async def _http_error_handler(request: Request, exc: Exception) -> JSONResponse:
         assert isinstance(exc, exception_cls)  # nosec
         error_content = {
             "errors": error_extractor(exc) if error_extractor else [f"{exc}"]
         }
 
+        if is_5xx_server_error(status_code):
+            _logger.exception(
+                create_troubleshotting_log_kwargs(
+                    "Unexpected error happened in the Resource Usage Tracker. Please contact support.",
+                    error=exc,
+                    error_context={
+                        "request": request,
+                        "request.method": f"{request.method}",
+                    },
+                )
+            )
+
         return JSONResponse(
             content=jsonable_encoder(
                 {"error": error_content} if envelope_error else error_content

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/storage/paths.py

@@ -5,6 +5,7 @@
     AsyncJobNameData,
 )
 from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE
+from models_library.products import ProductName
 from models_library.projects_nodes_io import LocationID
 from models_library.rabbitmq_basic_types import RPCMethodName
 from models_library.users import UserID
@@ -17,7 +18,7 @@ async def compute_path_size(
     client: RabbitMQRPCClient,
     *,
     user_id: UserID,
-    product_name: str,
+    product_name: ProductName,
     location_id: LocationID,
     path: Path,
 ) -> tuple[AsyncJobGet, AsyncJobNameData]:

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/storage/simcore_s3.py

@@ -0,0 +1,29 @@
+from models_library.api_schemas_rpc_async_jobs.async_jobs import (
+    AsyncJobGet,
+    AsyncJobNameData,
+)
+from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE
+from models_library.api_schemas_storage.storage_schemas import FoldersBody
+from models_library.rabbitmq_basic_types import RPCMethodName
+from models_library.users import UserID
+
+from ..._client_rpc import RabbitMQRPCClient
+from ..async_jobs.async_jobs import submit
+
+
+async def copy_folders_from_project(
+    client: RabbitMQRPCClient,
+    *,
+    user_id: UserID,
+    product_name: str,
+    body: FoldersBody,
+) -> tuple[AsyncJobGet, AsyncJobNameData]:
+    job_id_data = AsyncJobNameData(user_id=user_id, product_name=product_name)
+    async_job_rpc_get = await submit(
+        rabbitmq_rpc_client=client,
+        rpc_namespace=STORAGE_RPC_NAMESPACE,
+        method_name=RPCMethodName("copy_folders_from_project"),
+        job_id_data=job_id_data,
+        body=body,
+    )
+    return async_job_rpc_get, job_id_data

packages/simcore-sdk/tests/integration/test_node_data_data_manager.py

@@ -32,6 +32,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [

packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py

@@ -28,6 +28,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [

packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py

@@ -37,6 +37,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = ["minio", "adminer"]

packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py

@@ -29,6 +29,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [

packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py

@@ -48,6 +48,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [
@@ -280,7 +281,6 @@ async def test_port_file_accessors(
     request: pytest.FixtureRequest,
     constant_uuid4: None,
 ):
-
     if item_value == "symlink_path":
         item_value = request.getfixturevalue("symlink_path")
     if config_value == "config_value_symlink_path":

services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py

@@ -110,6 +110,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
     "redis",
 ]
 
@@ -934,15 +935,15 @@ async def test_nodeports_integration(
     `aioboto` instead of `docker` or `storage-data_manager API`.
     """
     # STEP 1
-    dynamic_services_urls: dict[
-        str, str
-    ] = await _start_and_wait_for_dynamic_services_ready(
-        director_v2_client=async_client,
-        product_name=osparc_product_name,
-        user_id=current_user["id"],
-        workbench_dynamic_services=workbench_dynamic_services,
-        current_study=current_study,
-        catalog_url=services_endpoint["catalog"],
+    dynamic_services_urls: dict[str, str] = (
+        await _start_and_wait_for_dynamic_services_ready(
+            director_v2_client=async_client,
+            product_name=osparc_product_name,
+            user_id=current_user["id"],
+            workbench_dynamic_services=workbench_dynamic_services,
+            current_study=current_study,
+            catalog_url=services_endpoint["catalog"],
+        )
     )
 
     # STEP 2

services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py

@@ -50,6 +50,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [
@@ -179,7 +180,6 @@ async def restore_legacy_state_archives(
     node_id: NodeID,
     state_paths_to_legacy_archives: dict[Path, Path],
 ) -> None:
-
     tasks = []
     for legacy_archive_zip in state_paths_to_legacy_archives.values():
         s3_path = f"{project_id}/{node_id}/{legacy_archive_zip.name}"

services/storage/VERSION

@@ -1 +1 @@
-0.6.0
+0.7.0

services/storage/openapi.json

@@ -1,4 +1,4 @@
 {
   "openapi": "3.1.0",
   "info": {
     "title": "simcore_service_storage",
@@ -1185,60 +1185,6 @@
         }
       }
     },
-    "/v0/simcore-s3/folders": {
-      "post": {
-        "tags": [
-          "simcore-s3"
-        ],
-        "summary": "Copy Folders From Project",
-        "operationId": "copy_folders_from_project_v0_simcore_s3_folders_post",
-        "parameters": [
-          {
-            "name": "user_id",
-            "in": "query",
-            "required": true,
-            "schema": {
-              "type": "integer",
-              "exclusiveMinimum": true,
-              "title": "User Id",
-              "minimum": 0
-            }
-          }
-        ],
-        "requestBody": {
-          "required": true,
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/FoldersBody"
-              }
-            }
-          }
-        },
-        "responses": {
-          "202": {
-            "description": "Successful Response",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/Envelope_TaskGet_"
-                }
-              }
-            }
-          },
-          "422": {
-            "description": "Validation Error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/HTTPValidationError"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
     "/v0/simcore-s3/folders/{folder_id}": {
       "delete": {
         "tags": [
@@ -1861,31 +1807,6 @@
         "type": "object",
         "title": "Envelope[S3Settings]"
       },
-      "Envelope_TaskGet_": {
-        "properties": {
-          "data": {
-            "anyOf": [
-              {
-                "$ref": "#/components/schemas/TaskGet"
-              },
-              {
-                "type": "null"
-              }
-            ]
-          },
-          "error": {
-            "anyOf": [
-              {},
-              {
-                "type": "null"
-              }
-            ],
-            "title": "Error"
-          }
-        },
-        "type": "object",
-        "title": "Envelope[TaskGet]"
-      },
       "Envelope_dict_": {
         "properties": {
           "data": {
@@ -2417,31 +2338,6 @@
         ],
         "title": "FileUploadSchema"
       },
-      "FoldersBody": {
-        "properties": {
-          "source": {
-            "type": "object",
-            "title": "Source"
-          },
-          "destination": {
-            "type": "object",
-            "title": "Destination"
-          },
-          "nodes_map": {
-            "additionalProperties": {
-              "type": "string",
-              "format": "uuid"
-            },
-            "propertyNames": {
-              "format": "uuid"
-            },
-            "type": "object",
-            "title": "Nodes Map"
-          }
-        },
-        "type": "object",
-        "title": "FoldersBody"
-      },
       "HTTPValidationError": {
         "properties": {
           "errors": {
@@ -2639,39 +2535,6 @@
         ],
         "title": "SoftCopyBody"
       },
-      "TaskGet": {
-        "properties": {
-          "task_id": {
-            "type": "string",
-            "title": "Task Id"
-          },
-          "task_name": {
-            "type": "string",
-            "title": "Task Name"
-          },
-          "status_href": {
-            "type": "string",
-            "title": "Status Href"
-          },
-          "result_href": {
-            "type": "string",
-            "title": "Result Href"
-          },
-          "abort_href": {
-            "type": "string",
-            "title": "Abort Href"
-          }
-        },
-        "type": "object",
-        "required": [
-          "task_id",
-          "task_name",
-          "status_href",
-          "result_href",
-          "abort_href"
-        ],
-        "title": "TaskGet"
-      },
       "UploadedPart": {
         "properties": {
           "number": {

services/storage/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.6.0
+current_version = 0.7.0
 commit = True
 message = services/storage api version: {current_version} → {new_version}
 tag = False