✨ dynamic-sidecar restart policy (⚠️ devops) (#2654)

* added restart policy

* added endpoint to restart containers

* refacto rabbitmq and add ui iframe reload command

* changed defaults rabbit_channels

* fixed .env-devel

* adding rabbitmq parsing

* cleanup

* adding restart containers policy integration

* added restart endpoint

* user is warned he needs to reload the UI

* js codestyle

* fixed pylint

* fixed import order

* fix pylint

* upgraded version

* reformat

* added test case

* refactor description

* fixed description

* removed uncecessary

* using shared client

* refactor using restart command

* renaming

* using namedtuple

* added test to check containers get restarted

* codestyle

* codestyle

* refactor to use project_uuid

* typing enhancements

* typo

* removed unused parameter

* fix routes

* remove files from disk when disconnecting port

* pylint

* refactor

* using asyncio implementation

* changed to NotADirectoryError

* fix pylint

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

.env-devel

@@ -35,7 +35,7 @@ POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
 
-RABBIT_CHANNELS={"log": "comp.backend.channels.log", "progress": "comp.backend.channels.progress", "instrumentation": "comp.backend.channels.instrumentation"}
+RABBIT_CHANNELS={"log": "simcore.services.logs", "progress": "simcore.services.progress", "instrumentation": "simcore.services.instrumentation", "events": "simcore.services.events"}
 RABBIT_HOST=rabbit
 RABBIT_PASSWORD=adminadmin
 RABBIT_PORT=5672

api/specs/webserver/openapi-projects.yaml

@@ -450,6 +450,32 @@ paths:
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 
+  /projects/{project_uuid}/nodes/{node_id}/restart:
+    parameters:
+      - name: project_uuid
+        in: path
+        required: true
+        schema:
+          type: string
+      - name: node_id
+        in: path
+        required: true
+        schema:
+          type: string
+
+    post:
+      tags:
+        - project
+      description: Restarts containers started by the dynamic-sidecar
+      operationId: post_restart
+
+      responses:
+        "204":
+          description: Restarts containers started by the dynamic-sidecar
+
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
   /projects/{study_uuid}/tags/{tag_id}:
     parameters:
       - name: tag_id

api/specs/webserver/openapi.yaml

@@ -203,6 +203,9 @@ paths:
   /projects/{project_id}/nodes/{node_id}:retrieve:
     $ref: "./openapi-projects.yaml#/paths/~1projects~1{project_id}~1nodes~1{node_id}~1retrieve"
 
+  /projects/{project_uuid}/nodes/{node_id}:restart:
+    $ref: "./openapi-projects.yaml#/paths/~1projects~1{project_uuid}~1nodes~1{node_id}~1restart"
+
   /nodes/{nodeInstanceUUID}/outputUi/{outputKey}:
     $ref: "./openapi-node-v0.0.1.yaml#/paths/~1nodes~1{nodeInstanceUUID}~1outputUi~1{outputKey}"
 

packages/models-library/src/models_library/rabbitmq_messages.py

@@ -1,4 +1,5 @@
 from typing import List, Optional, Union
+from enum import Enum
 
 from models_library.projects import ProjectID
 from models_library.projects_nodes import NodeID
@@ -9,6 +10,10 @@
 from simcore_postgres_database.models.comp_tasks import NodeClass
 
 
+class RabbitEventMessageType(str, Enum):
+    RELOAD_IFRAME = "RELOAD_IFRAME"
+
+
 class RabbitMessageBase(BaseModel):
     node_id: NodeID
     user_id: UserID
@@ -19,6 +24,10 @@ class LoggerRabbitMessage(RabbitMessageBase):
     messages: List[str]
 
 
+class EventRabbitMessage(RabbitMessageBase):
+    action: RabbitEventMessageType
+
+
 class ProgressRabbitMessage(RabbitMessageBase):
     progress: NonNegativeFloat
 

packages/models-library/src/models_library/service_settings_labels.py

@@ -1,4 +1,5 @@
 # pylint: disable=unsubscriptable-object
+from enum import Enum
 import json
 from functools import cached_property
 from pathlib import Path
@@ -107,6 +108,11 @@ class Config(_BaseConfig):
 ComposeSpecLabel = Dict[str, Any]
 
 
+class RestartPolicy(str, Enum):
+    NO_RESTART = "no-restart"
+    ON_INPUTS_DOWNLOADED = "on-inputs-downloaded"
+
+
 class DynamicSidecarServiceLabels(BaseModel):
     paths_mapping: Optional[Json[PathMappingsLabel]] = Field(
         None,
@@ -137,6 +143,17 @@ class DynamicSidecarServiceLabels(BaseModel):
         ),
     )
 
+    restart_policy: RestartPolicy = Field(
+        RestartPolicy.NO_RESTART,
+        alias="simcore.service.restart-policy",
+        description=(
+            "the dynamic-sidecar can restart all running containers "
+            "on certain events. Supported events:\n"
+            "- `no-restart` default\n"
+            "- `on-inputs-downloaded` after inputs are downloaded\n"
+        ),
+    )
+
     @cached_property
     def needs_dynamic_sidecar(self) -> bool:
         """if paths mapping is present the service needs to be ran via dynamic-sidecar"""
@@ -204,6 +221,7 @@ class Config(_BaseConfig):
                     "simcore.service.paths-mapping": json.dumps(
                         PathMappingsLabel.Config.schema_extra["example"]
                     ),
+                    "simcore.service.restart-policy": RestartPolicy.NO_RESTART.value,
                 },
                 # dynamic-service with compose spec
                 {
@@ -235,6 +253,7 @@ class Config(_BaseConfig):
                         }
                     ),
                     "simcore.service.container-http-entrypoint": "rt-web",
+                    "simcore.service.restart-policy": RestartPolicy.ON_INPUTS_DOWNLOADED.value,
                 },
             ]
         }

packages/models-library/src/models_library/settings/rabbit.py

@@ -1,10 +1,18 @@
 import warnings
-from typing import Dict
+from typing_extensions import TypedDict
 
 from pydantic import BaseSettings, Extra
 from pydantic.networks import AnyUrl
 from pydantic.types import PositiveInt, SecretStr
 
+
+class Channels(TypedDict):
+    log: str
+    progress: str
+    instrumentation: str
+    events: str
+
+
 warnings.warn(
     "models_library.settings will be mostly replaced by settings_library in future versions. "
     "SEE https://github.com/ITISFoundation/osparc-simcore/pull/2395 for details",
@@ -26,10 +34,11 @@ class RabbitConfig(BaseSettings):
     password: SecretStr = SecretStr("simcore")
 
     # channels
-    channels: Dict[str, str] = {
-        "log": "comp.backend.channels.log",
-        "progress": "comp.backend.channels.progress",
-        "instrumentation": "comp.backend.channels.instrumentation",
+    channels: Channels = {
+        "log": "simcore.services.log",
+        "progress": "simcore.services.progress",
+        "instrumentation": "simcore.services.instrumentation",
+        "events": "simcore.services.events",
     }
 
     @property

packages/models-library/src/models_library/settings/services_common.py

@@ -32,6 +32,9 @@ class ServicesCommonSettings(BaseSettings):
             "allow the service to finish the operation."
         ),
     )
+    restart_containers_timeout: PositiveInt = Field(
+        1 * _MINUTE, description="timeout of containers restart"
+    )
 
     class Config:
         env_prefix = "SERVICES_COMMON_"

packages/models-library/tests/test_service_settings_labels.py

@@ -30,13 +30,13 @@
     ),
     SimcoreServiceExample(
         example=SimcoreServiceLabels.Config.schema_extra["examples"][1],
-        items=2,
+        items=3,
         uses_dynamic_sidecar=True,
         id="dynamic-service",
     ),
     SimcoreServiceExample(
         example=SimcoreServiceLabels.Config.schema_extra["examples"][2],
-        items=4,
+        items=5,
         uses_dynamic_sidecar=True,
         id="dynamic-service-with-compose-spec",
     ),
@@ -130,3 +130,13 @@ def test_simcore_services_labels_compose_spec_null_container_http_entry_provided
     sample_data["simcore.service.compose-spec"] = None
     with pytest.raises(ValidationError):
         SimcoreServiceLabels(**sample_data)
+
+
+def test_raises_error_wrong_restart_policy() -> None:
+    simcore_service_labels: Dict[str, Any] = deepcopy(
+        SimcoreServiceLabels.Config.schema_extra["examples"][2]
+    )
+    simcore_service_labels["simcore.service.restart-policy"] = "__not_a_valid_policy__"
+
+    with pytest.raises(ValueError):
+        SimcoreServiceLabels(**simcore_service_labels)

packages/pytest-simcore/src/pytest_simcore/docker_registry.py

@@ -231,7 +231,7 @@ def jupyter_service(docker_registry: str, node_meta_schema: Dict) -> Dict[str, A
     )
 
 
-@pytest.fixture(scope="session", params=["2.0.2"])
+@pytest.fixture(scope="session", params=["2.0.3"])
 def dy_static_file_server_version(request):
     return request.param
 

packages/service-integration/src/service_integration/osparc_config.py

@@ -14,6 +14,7 @@
 
 from pathlib import Path
 from typing import Any, Dict, List, Literal, Optional
+from models_library.service_settings_labels import RestartPolicy
 
 from models_library.services import (
     COMPUTATIONAL_SERVICE_KEY_FORMAT,
@@ -157,6 +158,8 @@ class RuntimeConfig(BaseModel):
     compose_spec: Optional[ComposeSpecification] = None
     container_http_entrypoint: Optional[str] = None
 
+    restart_policy: RestartPolicy = RestartPolicy.NO_RESTART
+
     paths_mapping: Optional[PathsMapping] = None
 
     settings: List[SettingsItem] = []