🐛 Dynamic services log issues with missing resources when scheduled (ITISFoundation#2566)

* legacy service fails if resources are missing

* refactor and fixed issue with missing resources

* added test for failing to get scheduled

* test is now returning the correct value

* reverting changes & fixing test

* if a pending message is present it will be displyed in the logs

* fix styling

* removed unrequired limits

* reverting

* updated message diplayed to user

* fix camelcase

* fixing linter in CI

* trying to revert

* fixes issue with python linting

* revert & commit

* reverting change

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api.py

@@ -239,16 +239,15 @@ def _make_pending() -> Tuple[ServiceState, str]:
         # it is looking for a service or something with no error message
         return _make_pending()
 
-    task_status = last_task["Status"]
-    service_state, message = extract_task_state(task_status=task_status)
+    service_state, message = extract_task_state(task_status=last_task["Status"])
 
     # to avoid creating confusion for the user, always return the status
     # as pending while the dynamic-sidecar is starting, with
     # FAILED and COMPLETED and RUNNING being the only exceptions
     if service_state not in NO_PENDING_OVERWRITE:
         return ServiceState.PENDING, message
 
-    return extract_task_state(task_status=task_status)
+    return service_state, message
 
 
 async def are_services_missing(

services/director-v2/tests/unit/with_swarm/test_modules_dynamic_sidecar_docker_api.py

@@ -1,6 +1,7 @@
 # pylint: disable=redefined-outer-name
 # pylint: disable=unused-argument
 
+import asyncio
 from asyncio import BaseEventLoop
 from typing import Any, AsyncIterator, Dict, List
 from uuid import UUID, uuid4
@@ -29,6 +30,8 @@
 
 pytestmark = pytest.mark.asyncio
 
+MAX_INT64 = 9223372036854775807
+
 # FIXTURES
 
 
@@ -262,6 +265,12 @@ async def _count_services_in_stack(
     return len(services)
 
 
+def _inject_impossible_resources(dynamic_sidecar_service_spec: Dict[str, Any]) -> None:
+    dynamic_sidecar_service_spec["task_template"]["Resources"] = {
+        "Reservations": {"NanoCPUs": MAX_INT64, "MemoryBytes": MAX_INT64}
+    }
+
+
 # TESTS
 
 
@@ -392,6 +401,28 @@ async def test_dynamic_sidecar_in_running_state_and_node_id_is_recovered(
     assert dynamic_sidecar_state == (ServiceState.RUNNING, "")
 
 
+async def test_dynamic_sidecar_get_dynamic_sidecar_sate_fail_to_schedule(
+    dynamic_sidecar_service_spec: Dict[str, Any],
+    dynamic_sidecar_settings: DynamicSidecarSettings,
+    cleanup_test_dynamic_sidecar_service: None,
+    docker_swarm: None,
+) -> None:
+    _inject_impossible_resources(dynamic_sidecar_service_spec)
+    service_id = await docker_api.create_service_and_get_id(
+        dynamic_sidecar_service_spec
+    )
+    assert service_id
+
+    # wait for the service to get scheduled
+    await asyncio.sleep(0.2)
+
+    dynamic_sidecar_state = await docker_api.get_dynamic_sidecar_state(service_id)
+    assert dynamic_sidecar_state == (
+        ServiceState.PENDING,
+        "no suitable node (insufficient resources on 1 node)",
+    )
+
+
 async def test_are_services_missing(
     node_uuid: UUID,
     dynamic_sidecar_settings: DynamicSidecarSettings,

services/web/client/source/class/osparc/data/model/Node.js

@@ -1019,6 +1019,20 @@ qx.Class.define("osparc.data.model.Node", {
           break;
         }
         case "pending": {
+          if (data["service_message"]) {
+            const serviceId = data["service_uuid"];
+            const serviceName = this.getLabel();
+            const serviceMessage = data["service_message"];
+            const msg = `The service "${serviceName}" is waiting for available ` +
+              `resources. Please inform support and provide the following message ` +
+              `in case this does not resolve in a few minutes: "${serviceId}" ` +
+              `reported "${serviceMessage}"`;
+            const msgData = {
+              nodeId: this.getNodeId(),
+              msg: msg
+            };
+            this.fireDataEvent("showInLogger", msgData);
+          }
           status.setInteractive("pending");
           const interval = 10000;
           qx.event.Timer.once(() => this.__nodeState(), this, interval);