🐛 sidecar no longer fails when saving data with missing permissions

services/director-v2/src/simcore_service_director_v2/cli/__init__.py

@@ -26,7 +26,11 @@
 DEFAULT_OUTPUTS_PUSH_ATTEMPTS: Final[int] = 3
 DEFAULT_TASK_UPDATE_INTERVAL_S: Final[int] = 1
 
-main = typer.Typer(name=PROJECT_NAME)
+main = typer.Typer(
+    name=PROJECT_NAME,
+    pretty_exceptions_enable=False,
+    pretty_exceptions_show_locals=False,
+)
 
 _logger = logging.getLogger(__name__)
 

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py

@@ -124,6 +124,7 @@ async def runs_docker_compose_down_task(
     settings: Annotated[ApplicationSettings, Depends(get_settings)],
     shared_store: Annotated[SharedStore, Depends(get_shared_store)],
     app: Annotated[FastAPI, Depends(get_application)],
+    mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)],
 ) -> TaskId:
     assert request  # nosec
 
@@ -135,6 +136,7 @@ async def runs_docker_compose_down_task(
             app=app,
             shared_store=shared_store,
             settings=settings,
+            mounted_volumes=mounted_volumes,
         )
     except TaskAlreadyRunningError as e:
         return cast(str, e.managed_task.task_id)  # type: ignore[attr-defined] # pylint:disable=no-member

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/cli.py

@@ -1,8 +1,8 @@
 import asyncio
 import json
 import logging
+from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
-from typing import AsyncIterator
 
 import typer
 from fastapi import FastAPI
@@ -17,7 +17,11 @@
 from .modules.outputs import OutputsManager, setup_outputs
 
 log = logging.getLogger(__name__)
-main = typer.Typer(name=PROJECT_NAME)
+main = typer.Typer(
+    name=PROJECT_NAME,
+    pretty_exceptions_enable=False,
+    pretty_exceptions_show_locals=False,
+)
 
 
 main.command()(create_settings_command(settings_cls=ApplicationSettings, logger=log))

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py

@@ -53,7 +53,10 @@
 from ..modules.mounted_fs import MountedVolumes
 from ..modules.notifications._notifications_ports import PortNotifier
 from ..modules.outputs import OutputsManager, event_propagation_disabled
-from .long_running_tasksutils import run_before_shutdown_actions
+from .long_running_tasks_utils import (
+    ensure_read_permissions_on_user_service_data,
+    run_before_shutdown_actions,
+)
 from .resource_tracking import send_service_started, send_service_stopped
 
 _logger = logging.getLogger(__name__)
@@ -237,6 +240,7 @@ async def task_runs_docker_compose_down(
     app: FastAPI,
     shared_store: SharedStore,
     settings: ApplicationSettings,
+    mounted_volumes: MountedVolumes,
 ) -> None:
     if shared_store.compose_spec is None:
         _logger.warning("No compose-spec was found")
@@ -312,6 +316,8 @@ async def _send_resource_tracking_stop(platform_status: SimcorePlatformStatus):
         await _send_resource_tracking_stop(SimcorePlatformStatus.OK)
         raise
 
+    await ensure_read_permissions_on_user_service_data(mounted_volumes)
+
     await _send_resource_tracking_stop(SimcorePlatformStatus.OK)
 
     # removing compose-file spec

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasksutils.py → services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks_utils.py

@@ -1,4 +1,7 @@
 import logging
+import os
+from datetime import timedelta
+from typing import Final
 
 from models_library.callbacks_mapping import UserServiceCommand
 from servicelib.logging_utils import log_context
@@ -9,10 +12,13 @@
     ContainerExecTimeoutError,
 )
 from ..models.shared_store import SharedStore
-from ..modules.container_utils import run_command_in_container
+from ..modules.mounted_fs import MountedVolumes
+from .container_utils import run_command_in_container
 
 _logger = logging.getLogger(__name__)
 
+_TIMEOUT_PERMISSION_CHANGES: Final[timedelta] = timedelta(minutes=5)
+
 
 async def run_before_shutdown_actions(
     shared_store: SharedStore, before_shutdown: list[UserServiceCommand]
@@ -40,3 +46,22 @@ async def run_before_shutdown_actions(
                     container_name,
                     exc_info=True,
                 )
+
+
+async def ensure_read_permissions_on_user_service_data(
+    mounted_volumes: MountedVolumes,
+) -> None:
+    # Makes sure sidecar has access to all files in the user services.
+    # The user could have removed read permissions form a file, which will cause an error.
+
+    # NOTE: command runs inside self container since the user service container might not always be running
+    self_container = os.environ["HOSTNAME"]
+    for path_to_store in (  # apply changes to otuputs and all state folders
+        *mounted_volumes.disk_state_paths_iter(),
+        mounted_volumes.disk_outputs_path,
+    ):
+        await run_command_in_container(
+            self_container,
+            command=f"chmod -R o+rX '{path_to_store}'",
+            timeout=_TIMEOUT_PERMISSION_CHANGES.total_seconds(),
+        )

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/prometheus_metrics.py

@@ -107,13 +107,13 @@ async def _update_metrics(self):
             )
             self._metrics_response = MetricsResponse.from_reply(metrics_fetch_result)
         except ContainerExecContainerNotFoundError as e:
-            _logger.info(
-                "Container %s was not found could nto recover metrics",
+            _logger.debug(
+                "Container %s was not found could not recover metrics",
                 container_name,
             )
             self._metrics_response = MetricsResponse.from_error(e)
         except Exception as e:  # pylint: disable=broad-exception-caught
-            _logger.info("Unexpected exception", exc_info=True)
+            _logger.debug("Could not recover metrics", exc_info=True)
             self._metrics_response = MetricsResponse.from_error(e)
 
     async def _task_metrics_recovery(self) -> None:

services/dynamic-sidecar/tests/unit/conftest.py

@@ -11,6 +11,7 @@
 from aiodocker.volumes import DockerVolume
 from async_asgi_testclient import TestClient
 from fastapi import FastAPI
+from pytest_mock.plugin import MockerFixture
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict
 from simcore_service_dynamic_sidecar.core.application import AppState, create_app
 from simcore_service_dynamic_sidecar.core.docker_compose_utils import (
@@ -157,3 +158,10 @@ def port_notifier(app: FastAPI) -> PortNotifier:
         settings.DY_SIDECAR_PROJECT_ID,
         settings.DY_SIDECAR_NODE_ID,
     )
+
+
+@pytest.fixture
+def mock_ensure_read_permissions_on_user_service_data(mocker: MockerFixture) -> None:
+    mocker.patch(
+        "simcore_service_dynamic_sidecar.modules.long_running_tasks.ensure_read_permissions_on_user_service_data",
+    )

services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py

@@ -520,6 +520,7 @@ def _get_awaitable() -> Awaitable:
 
 
 async def test_containers_down_after_starting(
+    mock_ensure_read_permissions_on_user_service_data: None,
     httpx_async_client: AsyncClient,
     client: Client,
     compose_spec: str,

services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py

@@ -172,7 +172,7 @@ async def _get_task_id_docker_compose_down(httpx_async_client: AsyncClient) -> T
 
 
 def _get_resource_tracking_messages(
-    mock_core_rabbitmq: dict[str, AsyncMock]
+    mock_core_rabbitmq: dict[str, AsyncMock],
 ) -> list[RabbitResourceTrackingMessages]:
     return [
         x[0][1]
@@ -200,6 +200,7 @@ async def _wait_for_containers_to_be_running(app: FastAPI) -> None:
 
 
 async def test_service_starts_and_closes_as_expected(
+    mock_ensure_read_permissions_on_user_service_data: None,
     mock_core_rabbitmq: dict[str, AsyncMock],
     app: FastAPI,
     httpx_async_client: AsyncClient,
@@ -383,6 +384,7 @@ async def _mocked_get_container_states(
 
 @pytest.mark.parametrize("expected_platform_state", SimcorePlatformStatus)
 async def test_user_services_crash_when_running(
+    mock_ensure_read_permissions_on_user_service_data: None,
     mock_core_rabbitmq: dict[str, AsyncMock],
     app: FastAPI,
     httpx_async_client: AsyncClient,

services/dynamic-sidecar/tests/unit/test_modules_attribute_monitor.py

@@ -125,6 +125,7 @@ async def logging_event_handler_observer(
     ],
 )
 async def test_chown_triggers_event(
+    mock_ensure_read_permissions_on_user_service_data: None,
     logging_event_handler_observer: None,
     fake_dy_volumes_mount_dir: Path,
     command_template: str,