🏗️ ✨ Enhancement/connect director to dask (⚠️ devops) (ITISFoundation#2418)

Author: sanderegg

.env-devel

@@ -23,6 +23,8 @@ DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKE
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
 
+DIRECTOR_V2_DEV_FEATURES_ENABLED=0
+
 DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest}
 
 POSTGRES_DB=simcoredb

.github/workflows/ci-testing-deploy.yml

@@ -32,6 +32,7 @@ env:
 
 jobs:
   unit-test-api:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] api"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -75,6 +76,7 @@ jobs:
         run: ./ci/github/unit-testing/api.bash test
 
   unit-test-api-server:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] api-server"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -131,6 +133,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-catalog:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] catalog"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -193,6 +196,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-datcore-adapter:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] datcore-adapter"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -253,6 +257,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-director:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] director"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -310,6 +315,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-director-v2:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] director-v2"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -370,6 +376,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-sidecar:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] sidecar"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -426,6 +433,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-dask-sidecar:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] dask-sidecar"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -482,6 +490,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-dynamic-sidecar:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] dynamic-sidecar"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -540,6 +549,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-frontend:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] frontend"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -577,6 +587,7 @@ jobs:
     #     flags: unittests #optional
 
   unit-test-python-linting:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] python-linting"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -620,6 +631,7 @@ jobs:
         run: ./ci/github/unit-testing/python-linting.bash test
 
   unit-test-postgres-database:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] postgres-database"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -677,6 +689,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-service-integration:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] service-integration"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -733,6 +746,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-service-library:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] service-library"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -789,6 +803,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-settings-library:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] settings-library"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -845,6 +860,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-models-library:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] models-library"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -901,6 +917,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-simcore-sdk:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] simcore-sdk"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -957,6 +974,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-storage:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] storage"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -1013,6 +1031,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   unit-test-webserver-isolated:
+    timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] webserver isolated"
     runs-on: ${{ matrix.os }}
     strategy:

Makefile

@@ -206,8 +206,9 @@ printf "$$rows" 'Postgres DB' 'http://$(get_my_ip).nip.io:18080/?pgsql=postgres&
 printf "$$rows" Portainer 'http://$(get_my_ip).nip.io:9000' admin adminadmin;\
 printf "$$rows" Redis 'http://$(get_my_ip).nip.io:18081';\
 printf "$$rows" 'Docker Registry' $${REGISTRY_URL} $${REGISTRY_USER} $${REGISTRY_PW};\
-echo "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar"
-echo "⚠️ will not be shown. The frontend accesses them via the uuid.services.YOUR_IP.nip.io:9081"
+printf "$$rows" "Dask Dashboard" "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1).nip.io:8787";
+printf "\n%s\n" "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar";\
+echo "⚠️ will not be shown. The frontend accesses them via the uuid.services.YOUR_IP.nip.io:9081";
 endef
 
 show-endpoints:

packages/pytest-simcore/src/pytest_simcore/docker_registry.py

@@ -39,6 +39,9 @@ def docker_registry(keep_docker_up: bool) -> str:
             name="pytest_registry",
             environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"],
             restart_policy={"Name": "always"},
+            volumes={
+                "pytest_registry_data": {"bind": "/var/lib/registry", "mode": "rw"}
+            },
             detach=True,
         )
 

packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py

@@ -0,0 +1,37 @@
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
+from typing import Any, Dict, Iterator
+
+import pytest
+from distributed import Client
+
+from .helpers.utils_docker import get_service_published_port
+
+
+@pytest.fixture(scope="function")
+async def dask_scheduler_service(simcore_services, monkeypatch) -> Dict[str, Any]:
+    # the dask scheduler has a UI for the dashboard and a secondary port for the API
+    # simcore_services fixture already ensure the dask-scheduler is up and running
+    dask_scheduler_api_port = get_service_published_port(
+        "dask-scheduler", target_ports=[8786]
+    )
+    # override the port
+    monkeypatch.setenv("DASK_SCHEDULER_PORT", f"{dask_scheduler_api_port}")
+    return {"host": "127.0.0.1", "port": dask_scheduler_api_port}
+
+
+@pytest.fixture(scope="function")
+def dask_client(dask_scheduler_service: Dict[str, Any]) -> Iterator[Client]:
+
+    client = Client(
+        f"{dask_scheduler_service['host']}:{dask_scheduler_service['port']}"
+    )
+    yield client
+    client.close()
+
+
+@pytest.fixture(scope="function")
+def dask_sidecar_service(dask_client: Client) -> None:
+    dask_client.wait_for_workers(n_workers=1, timeout=30)

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -15,8 +15,15 @@
 
 log = logging.getLogger(__name__)
 
-SERVICES_TO_SKIP = ["sidecar", "postgres", "redis", "rabbit"]
-SERVICE_HEALTHCHECK_ENTRYPOINT = {"director-v2": "/"}
+SERVICES_TO_SKIP = ["dask-sidecar", "sidecar", "postgres", "redis", "rabbit"]
+SERVICE_PUBLISHED_PORT = {}
+SERVICE_HEALTHCHECK_ENTRYPOINT = {
+    "director-v2": "/",
+    "dask-scheduler": "/health",
+}
+AIOHTTP_BASED_SERVICE_PORT: int = 8080
+FASTAPI_BASED_SERVICE_PORT: int = 8000
+DASK_SCHEDULER_SERVICE_PORT: int = 8787
 
 
 @pytest.fixture(scope="module")
@@ -28,9 +35,9 @@ def services_endpoint(
     stack_name = testing_environ_vars["SWARM_STACK_NAME"]
     for service in core_services_selection:
         assert f"{stack_name}_{service}" in docker_stack["services"]
-        if not service in SERVICES_TO_SKIP:
+        if service not in SERVICES_TO_SKIP:
             endpoint = URL(
-                f"http://127.0.0.1:{get_service_published_port(service, [8080, 8000])}"
+                f"http://127.0.0.1:{get_service_published_port(service, [AIOHTTP_BASED_SERVICE_PORT, FASTAPI_BASED_SERVICE_PORT, DASK_SCHEDULER_SERVICE_PORT])}"
             )
             services_endpoint[service] = endpoint
     return services_endpoint
@@ -42,7 +49,7 @@ async def simcore_services(services_endpoint: Dict[str, URL], monkeypatch) -> No
     # waits for all services to be responsive
     wait_tasks = [
         wait_till_service_responsive(
-            f"{endpoint}{SERVICE_HEALTHCHECK_ENTRYPOINT.get(service, '/v0/')}"
+            URL(f"{endpoint}{SERVICE_HEALTHCHECK_ENTRYPOINT.get(service, '/v0/')}")
         )
         for service, endpoint in services_endpoint.items()
     ]

services/dask-sidecar/docker/boot.sh

@@ -25,7 +25,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then
   pip list | sed 's/^/    /'
 fi
 
-
 # RUNNING application ----------------------------------------
 #
 # - If DASK_START_AS_SCHEDULER is set, then it boots as scheduler otherwise as worker
@@ -52,24 +51,49 @@ else
   DASK_WORKER_VERSION=$(dask-worker --version)
   DASK_SCHEDULER_ADDRESS="tcp://${DASK_SCHEDULER_HOST}:8786"
 
-
+  #
+  # by default a dask worker will use as many threads as there are CPUs on the machine regardless of what limit the
+  # the docker container has set (e.g. if the docker container is limited to 4 CPUs out of 10, dask will still use 10 threads by default)
+  # so for now we lock the number of threads to 1, so that only 1 job is done by 1 sidecar, thus --nthreads 1.
+
+  #
+  # 'daemonic processes are not allowed to have children' arises when running the sidecar.cli
+  # because multi-processing library is used by the sidecar and the nanny does not like it
+  # setting --no-nanny fixes this: see https://github.com/dask/distributed/issues/2142
+  num_gpus=$(python -c "from simcore_service_sidecar.utils import num_available_gpus; print(num_available_gpus());")
+  resources="CPU=1"
+  if [ "$num_gpus" -gt 0 ]; then
+    resources="$resources,GPU=$num_gpus"
+  fi
+  if [ ${TARGET_MPI_NODE_CPU_COUNT+x} ]; then
+    if [ $(nproc) -eq ${TARGET_MPI_NODE_CPU_COUNT} ]; then
+      resources="$resources,MPI=1"
+    fi
+  fi
   echo "$INFO" "Starting as a ${DASK_WORKER_VERSION} -> ${DASK_SCHEDULER_ADDRESS} ..."
+  echo "$INFO" "Worker resources set as: $resources"
   if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]; then
 
     exec watchmedo auto-restart --recursive --pattern="*.py" -- \
       dask-worker "${DASK_SCHEDULER_ADDRESS}" \
-        --local-directory /tmp/dask-sidecar \
-        --preload simcore_service_dask_sidecar.tasks \
-        --reconnect \
-        --dashboard-address 8787
+      --local-directory /tmp/dask-sidecar \
+      --preload simcore_service_dask_sidecar.tasks \
+      --reconnect \
+      --no-nanny \
+      --nthreads 1 \
+      --dashboard-address 8787 \
+      --resources "$resources"
 
   else
 
     exec dask-worker "${DASK_SCHEDULER_ADDRESS}" \
-        --local-directory /tmp/dask-sidecar \
-        --preload simcore_service_dask_sidecar.tasks \
-        --reconnect \
-        --dashboard-address 8787
+      --local-directory /tmp/dask-sidecar \
+      --preload simcore_service_dask_sidecar.tasks \
+      --reconnect \
+      --no-nanny \
+      --nthreads 1 \
+      --dashboard-address 8787 \
+      --resources "$resources"
 
   fi
 fi

services/dask-sidecar/tests/unit/conftest.py

@@ -10,6 +10,8 @@
 pytest_plugins = [
     "pytest_simcore.repository_paths",
     "pytest_simcore.environment_configs",
+    "pytest_simcore.docker_compose",
+    "pytest_simcore.tmp_path_extra",
 ]
 
 

services/dask-sidecar/tests/unit/test_deployment.py

@@ -0,0 +1,10 @@
+from typing import Any, Dict
+
+
+def test_sidecar_service_is_deployed_in_global_mode(
+    simcore_docker_compose: Dict[str, Any]
+):
+    dask_sidecar_deploy_config = simcore_docker_compose["services"]["dask-sidecar"][
+        "deploy"
+    ]
+    assert dask_sidecar_deploy_config["mode"] == "global"

services/director-v2/requirements/_base.in

@@ -3,6 +3,8 @@
 #
 # NOTE: ALL version constraints MUST be commented
 --constraint ../../../requirements/constraints.txt
+# we need EXACTLY the same dask libraries in the client/scheduler/workers
+--requirement ../../../services/dask-sidecar/requirements/_dask-distributed.txt
 
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in