added cleanup for director-v2's scheduler

Author: Andrei Neagu

services/director-v2/src/simcore_service_director_v2/core/application.py

@@ -34,6 +34,7 @@
     dynamic_services,
     dynamic_sidecar,
     instrumentation,
+    long_running_tasks,
     notifier,
     rabbitmq,
     redis,
@@ -220,6 +221,7 @@ def create_app(  # noqa: C901, PLR0912
         dynamic_sidecar.setup(app)
         socketio.setup(app)
         notifier.setup(app)
+        long_running_tasks.setup(app)
 
     if (
         settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND.COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py

@@ -35,6 +35,9 @@
 )
 from servicelib.utils import limited_gather, logged_gather
 from simcore_postgres_database.models.comp_tasks import NodeClass
+from simcore_service_director_v2.modules.long_running_tasks import (
+    get_client_long_running_manager,
+)
 from tenacity import RetryError, TryAgain
 from tenacity.asyncio import AsyncRetrying
 from tenacity.before_sleep import before_sleep_log
@@ -289,6 +292,16 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes(
         message="finished removing resources", percent=ProgressPercent(1)
     )
 
+    await _cleanup_long_running_tasks(app, scheduler_data.node_uuid)
+
+
+async def _cleanup_long_running_tasks(app: FastAPI, node_id: NodeID) -> None:
+    clinet_long_running_manager = get_client_long_running_manager(app)
+
+    await clinet_long_running_manager.cleanup_store(
+        clinet_long_running_manager.get_sidecar_namespace(node_id)
+    )
+
 
 async def attempt_pod_removal_and_data_saving(
     app: FastAPI, scheduler_data: SchedulerData

services/director-v2/src/simcore_service_director_v2/modules/long_running_tasks.py

@@ -0,0 +1,28 @@
+from fastapi import FastAPI
+from servicelib.long_running_tasks.client_long_running_manager import (
+    ClientLongRunningManager,
+)
+
+
+def setup(app: FastAPI):
+    async def _on_startup() -> None:
+        client_long_running_manager = app.state.client_long_running_manager = (
+            ClientLongRunningManager(redis_settings=app.state.settings.REDIS)
+        )
+        await client_long_running_manager.setup()
+
+    async def _on_shutdown() -> None:
+        client_long_running_manager: ClientLongRunningManager = (
+            app.state.client_long_running_manager
+        )
+        await client_long_running_manager.shutdown()
+
+    app.add_event_handler("startup", _on_startup)
+    app.add_event_handler("shutdown", _on_shutdown)
+
+
+def get_client_long_running_manager(app: FastAPI) -> ClientLongRunningManager:
+    assert isinstance(
+        app.state.client_long_running_manager, ClientLongRunningManager
+    )  # nosec
+    return app.state.client_long_running_manager