♻️ Polished and modernised agent service ⚠️ (ITISFoundation#6452)

Co-authored-by: Andrei Neagu <[email protected]>

Author: 2 people

packages/models-library/src/models_library/api_schemas_directorv2/health.py renamed to packages/models-library/src/models_library/api_schemas__common/health.py

@@ -1,4 +1,4 @@
-from typing import Any, ClassVar
+from typing import Any, ClassVar, Final
 
 from pydantic import BaseModel, Field, validator
 from pydantic.types import ByteSize, NonNegativeInt
@@ -90,3 +90,6 @@ class Config:
                 for node_example in NodeRequirements.Config.schema_extra["examples"]
             ]
         }
+
+
+CHARS_IN_VOLUME_NAME_BEFORE_DIR_NAME: Final[NonNegativeInt] = 89

packages/models-library/src/models_library/api_schemas_directorv2/services.py

@@ -1,8 +1,8 @@
-import logging
+from typing import TypeVar
 
 from fastapi import FastAPI
 
-_logger = logging.getLogger(__name__)
+T = TypeVar("T", bound="SingletonInAppStateMixin")
 
 
 class SingletonInAppStateMixin:
@@ -14,8 +14,8 @@ class SingletonInAppStateMixin:
     frozen: bool = True  # Will raise if set multiple times
 
     @classmethod
-    def get_from_app_state(cls, app: FastAPI):
-        return getattr(app.state, cls.app_state_name)
+    def get_from_app_state(cls: type[T], app: FastAPI) -> T:
+        return getattr(app.state, cls.app_state_name)  # type:ignore[no-any-return]
 
     def set_to_app_state(self, app: FastAPI):
         if (exists := getattr(app.state, self.app_state_name, None)) and self.frozen:
@@ -26,11 +26,11 @@ def set_to_app_state(self, app: FastAPI):
         return self.get_from_app_state(app)
 
     @classmethod
-    def pop_from_app_state(cls, app: FastAPI):
+    def pop_from_app_state(cls: type[T], app: FastAPI) -> T:
         """
         Raises:
             AttributeError: if instance is not in app.state
         """
-        old = getattr(app.state, cls.app_state_name)
+        old = cls.get_from_app_state(app)
         delattr(app.state, cls.app_state_name)
         return old

packages/service-library/src/servicelib/fastapi/app_state.py

@@ -0,0 +1,12 @@
+from pydantic.errors import PydanticErrorMixin
+
+
+class BaseAgentRPCError(PydanticErrorMixin, Exception):
+    ...
+
+
+class NoServiceVolumesFoundRPCError(BaseAgentRPCError):
+    msg_template: str = (
+        "Could not detect any unused volumes after waiting '{period}' seconds for "
+        "volumes to be released after closing all container for service='{node_id}'"
+    )

services/agent/src/simcore_service_agent/modules/__init__.py renamed to packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/__init__.py

@@ -0,0 +1,57 @@
+import logging
+from datetime import timedelta
+from typing import Final
+
+from models_library.projects_nodes_io import NodeID
+from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace
+from pydantic import NonNegativeInt, parse_obj_as
+from servicelib.logging_utils import log_decorator
+from servicelib.rabbitmq import RabbitMQRPCClient
+
+_logger = logging.getLogger(__name__)
+
+_REQUEST_TIMEOUT: Final[NonNegativeInt] = int(timedelta(minutes=60).total_seconds())
+
+
+@log_decorator(_logger, level=logging.DEBUG)
+async def remove_volumes_without_backup_for_service(
+    rabbitmq_rpc_client: RabbitMQRPCClient,
+    *,
+    docker_node_id: str,
+    swarm_stack_name: str,
+    node_id: NodeID,
+) -> None:
+    result = await rabbitmq_rpc_client.request(
+        RPCNamespace.from_entries(
+            {
+                "service": "agent",
+                "docker_node_id": docker_node_id,
+                "swarm_stack_name": swarm_stack_name,
+            }
+        ),
+        parse_obj_as(RPCMethodName, "remove_volumes_without_backup_for_service"),
+        node_id=node_id,
+        timeout_s=_REQUEST_TIMEOUT,
+    )
+    assert result is None  # nosec
+
+
+@log_decorator(_logger, level=logging.DEBUG)
+async def backup_and_remove_volumes_for_all_services(
+    rabbitmq_rpc_client: RabbitMQRPCClient,
+    *,
+    docker_node_id: str,
+    swarm_stack_name: str,
+) -> None:
+    result = await rabbitmq_rpc_client.request(
+        RPCNamespace.from_entries(
+            {
+                "service": "agent",
+                "docker_node_id": docker_node_id,
+                "swarm_stack_name": swarm_stack_name,
+            }
+        ),
+        parse_obj_as(RPCMethodName, "backup_and_remove_volumes_for_all_services"),
+        timeout_s=_REQUEST_TIMEOUT,
+    )
+    assert result is None  # nosec

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/errors.py

@@ -1 +1 @@
-0.0.1
+1.0.0

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/volumes.py

@@ -8,11 +8,12 @@
 # intra-repo required dependencies
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in
+# service-library[fastapi]
+--requirement ../../../packages/service-library/requirements/_base.in
 --requirement ../../../packages/service-library/requirements/_fastapi.in
 
 aiodocker
 fastapi
 packaging
 pydantic
-python-dotenv
 uvicorn