🎨Storage: HA (🚨🚨🚨 test with multiple replicas) (#7375)

Author: sanderegg

packages/service-library/src/servicelib/fastapi/http_error.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Awaitable, Callable
 from typing import TypeVar
 
@@ -10,6 +11,9 @@
 from fastapi.responses import JSONResponse
 from pydantic import ValidationError
 
+from ..logging_errors import create_troubleshotting_log_kwargs
+from ..status_codes_utils import is_5xx_server_error
+
 validation_error_response_definition["properties"] = {
     "errors": {
         "title": "Validation errors",
@@ -21,6 +25,8 @@
 
 TException = TypeVar("TException")
 
+_logger = logging.getLogger(__name__)
+
 
 def make_http_error_handler_for_exception(
     status_code: int,
@@ -36,12 +42,24 @@ def make_http_error_handler_for_exception(
     SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions
     """
 
-    async def _http_error_handler(_: Request, exc: Exception) -> JSONResponse:
+    async def _http_error_handler(request: Request, exc: Exception) -> JSONResponse:
         assert isinstance(exc, exception_cls)  # nosec
         error_content = {
             "errors": error_extractor(exc) if error_extractor else [f"{exc}"]
         }
 
+        if is_5xx_server_error(status_code):
+            _logger.exception(
+                create_troubleshotting_log_kwargs(
+                    "Unexpected error happened in the Resource Usage Tracker. Please contact support.",
+                    error=exc,
+                    error_context={
+                        "request": request,
+                        "request.method": f"{request.method}",
+                    },
+                )
+            )
+
         return JSONResponse(
             content=jsonable_encoder(
                 {"error": error_content} if envelope_error else error_content

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/storage/paths.py

@@ -5,6 +5,7 @@
     AsyncJobNameData,
 )
 from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE
+from models_library.products import ProductName
 from models_library.projects_nodes_io import LocationID
 from models_library.rabbitmq_basic_types import RPCMethodName
 from models_library.users import UserID
@@ -17,7 +18,7 @@ async def compute_path_size(
     client: RabbitMQRPCClient,
     *,
     user_id: UserID,
-    product_name: str,
+    product_name: ProductName,
     location_id: LocationID,
     path: Path,
 ) -> tuple[AsyncJobGet, AsyncJobNameData]:

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/storage/simcore_s3.py

@@ -0,0 +1,29 @@
+from models_library.api_schemas_rpc_async_jobs.async_jobs import (
+    AsyncJobGet,
+    AsyncJobNameData,
+)
+from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE
+from models_library.api_schemas_storage.storage_schemas import FoldersBody
+from models_library.rabbitmq_basic_types import RPCMethodName
+from models_library.users import UserID
+
+from ..._client_rpc import RabbitMQRPCClient
+from ..async_jobs.async_jobs import submit
+
+
+async def copy_folders_from_project(
+    client: RabbitMQRPCClient,
+    *,
+    user_id: UserID,
+    product_name: str,
+    body: FoldersBody,
+) -> tuple[AsyncJobGet, AsyncJobNameData]:
+    job_id_data = AsyncJobNameData(user_id=user_id, product_name=product_name)
+    async_job_rpc_get = await submit(
+        rabbitmq_rpc_client=client,
+        rpc_namespace=STORAGE_RPC_NAMESPACE,
+        method_name=RPCMethodName("copy_folders_from_project"),
+        job_id_data=job_id_data,
+        body=body,
+    )
+    return async_job_rpc_get, job_id_data

packages/simcore-sdk/tests/integration/test_node_data_data_manager.py

@@ -32,6 +32,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [

packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py

@@ -28,6 +28,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [

packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py

@@ -37,6 +37,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = ["minio", "adminer"]

packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py

@@ -29,6 +29,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [

packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py

@@ -48,6 +48,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [
@@ -280,7 +281,6 @@ async def test_port_file_accessors(
     request: pytest.FixtureRequest,
     constant_uuid4: None,
 ):
-
     if item_value == "symlink_path":
         item_value = request.getfixturevalue("symlink_path")
     if config_value == "config_value_symlink_path":

services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py

@@ -110,6 +110,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
     "redis",
 ]
 
@@ -934,15 +935,15 @@ async def test_nodeports_integration(
     `aioboto` instead of `docker` or `storage-data_manager API`.
     """
     # STEP 1
-    dynamic_services_urls: dict[
-        str, str
-    ] = await _start_and_wait_for_dynamic_services_ready(
-        director_v2_client=async_client,
-        product_name=osparc_product_name,
-        user_id=current_user["id"],
-        workbench_dynamic_services=workbench_dynamic_services,
-        current_study=current_study,
-        catalog_url=services_endpoint["catalog"],
+    dynamic_services_urls: dict[str, str] = (
+        await _start_and_wait_for_dynamic_services_ready(
+            director_v2_client=async_client,
+            product_name=osparc_product_name,
+            user_id=current_user["id"],
+            workbench_dynamic_services=workbench_dynamic_services,
+            current_study=current_study,
+            catalog_url=services_endpoint["catalog"],
+        )
     )
 
     # STEP 2

services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py

@@ -50,6 +50,7 @@
     "rabbit",
     "redis",
     "storage",
+    "sto-worker",
 ]
 
 pytest_simcore_ops_services_selection = [
@@ -179,7 +180,6 @@ async def restore_legacy_state_archives(
     node_id: NodeID,
     state_paths_to_legacy_archives: dict[Path, Path],
 ) -> None:
-
     tasks = []
     for legacy_archive_zip in state_paths_to_legacy_archives.values():
         s3_path = f"{project_id}/{node_id}/{legacy_archive_zip.name}"