⚗️ Adds dask scheduler and sidecar worker in stack (ITISFoundation#2366)

* ADDED: experimental dask-sidecar (worker) and dask scheduler services
Both worker and scheduler services use the same dask-sidecar image but have different boots
Adds scheduler to both dynamic and computational_services_subnet so notebook can schedule tasks in dask workers
These services are STILL NOT CONNECTED to the director-v2 (next PRs will provide that functionality)

* FIXES: settings initialization models_library.settings.base.BaseCustomSettings

Author: pcrespov

.github/PULL_REQUEST_TEMPLATE.md

@@ -9,10 +9,9 @@ Consider prefix your PR message with an emoticon
   🏗️ maintenance
   📚 documentation
 
-and append (⚠️ devops) if changes in devops configuration required before deploying
+or get your emoji from https://gitmoji.dev/
 
-  SEE https://github.com/dannyfritz/commit-message-emoji
-  SEE https://emojipedia.org
+and append (⚠️ devops) if changes in devops configuration required before deploying
 -->
 
 ## What do these changes do?

.github/workflows/ci-testing-deploy.yml

@@ -314,6 +314,53 @@ jobs:
           name: codeclimate-${{ github.job }}-coverage
           path: codeclimate.${{ github.job }}_coverage.json
 
+  unit-test-dask-sidecar:
+    name: "[unit] dask-sidecar"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.8]
+        os: [ubuntu-20.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+      - name: setup python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-dask-sidecar-${{ hashFiles('services/dask-sidecar/requirements/ci.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-dask-sidecar-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install
+        run: ./ci/github/unit-testing/dask-sidecar.bash install
+      - name: test
+        run: ./ci/github/unit-testing/dask-sidecar.bash test
+      - uses: codecov/codecov-action@v1
+        with:
+          flags: unittests #optional
+      - name: prepare codeclimate coverage file
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
+      - name: upload codeclimate coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: codeclimate-${{ github.job }}-coverage
+          path: codeclimate.${{ github.job }}_coverage.json
+
   unit-test-dynamic-sidecar:
     name: "[unit] dynamic-sidecar"
     runs-on: ${{ matrix.os }}
@@ -1803,6 +1850,7 @@ jobs:
         unit-test-director-v2,
         unit-test-sidecar,
         unit-test-dynamic-sidecar,
+        unit-test-dask-sidecar,
         unit-test-service-integration,
         unit-test-service-library,
         unit-test-models-library,
@@ -1858,6 +1906,7 @@ jobs:
         unit-test-director-v2,
         unit-test-sidecar,
         unit-test-dynamic-sidecar,
+        unit-test-dask-sidecar,
         unit-test-frontend,
         unit-test-python-linting,
         unit-test-service-integration,

.gitignore

@@ -162,3 +162,6 @@ services/**/.codeclimate.yml
 
 # pytest-fixture-tools output
 artifacts
+
+# dask artifacts in devel mode
+dask*-space/

Makefile

@@ -30,14 +30,15 @@ $(if $(IS_WIN),$(error Windows is not supported in all recipes. Use WSL instead.
 SERVICES_LIST := \
 	api-server \
 	catalog \
+	dask-sidecar \
 	director \
 	director-v2 \
 	dynamic-sidecar \
 	migration \
 	sidecar \
+	static-webserver \
 	storage \
-	webserver \
-	static-webserver
+	webserver
 
 CLIENT_WEB_OUTPUT       := $(CURDIR)/services/web/client/source-output
 
@@ -199,7 +200,7 @@ TableWidth=140;\
 printf "%80s| %22s| %12s| %12s\n" Endpoint Name User Password;\
 printf "%.$${TableWidth}s\n" "$$separator";\
 printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9081' 'oSparc platform';\
-printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18080/?pgsql=postgres&username=$${POSTGRES_USER}&db=$${POSTGRES_DB}&ns=public' 'Postgres DB' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
+printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18080/?pgsql=postgres&username='$${POSTGRES_USER}'&db='$${POSTGRES_DB}'&ns=public' 'Postgres DB' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
 printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9000' Portainer admin adminadmin;\
 printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18081' Redis
 endef

ci/github/unit-testing/dask-sidecar.bash

@@ -0,0 +1,29 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+    bash ci/helpers/ensure_python_pip.bash;
+    pushd services/dask-sidecar; pip3 install -r requirements/ci.txt; popd;
+    pip list -v
+}
+
+test() {
+    pytest --cov=simcore_service_dask_sidecar --durations=10 --cov-append \
+          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+          -v services/dask-sidecar/tests/unit
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" > /dev/null
+then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

packages/models-library/src/models_library/projects_state.py

@@ -12,6 +12,11 @@
 
 @unique
 class RunningState(str, Enum):
+    """State of execution of a project's computational workflow
+
+    SEE StateType for task state
+    """
+
     UNKNOWN = "UNKNOWN"
     PUBLISHED = "PUBLISHED"
     NOT_STARTED = "NOT_STARTED"

packages/models-library/src/models_library/settings/base.py

@@ -1,5 +1,5 @@
 from contextlib import suppress
-from typing import List, Tuple
+from typing import List, Tuple, Type
 
 from pydantic import BaseSettings, Extra, SecretStr, ValidationError
 
@@ -15,7 +15,7 @@ class Config:
 
     @classmethod
     def set_defaults_with_default_constructors(
-        cls, default_fields: List[Tuple[str, "BaseCustomSettings"]]
+        cls, default_fields: List[Tuple[str, Type["BaseCustomSettings"]]]
     ):
         # This funcion can set defaults on fields that are BaseSettings as well
         # It is used in control construction of defaults.
@@ -33,3 +33,14 @@ def set_defaults_with_default_constructors(
                 field_obj.default = default
                 field_obj.field_info.default = default
                 field_obj.required = False
+
+    @classmethod
+    def create_from_envs(cls):
+        obj = cls()
+
+        # TODO: perform this check on FieldInfo upon class construction
+        if any(isinstance(field, BaseSettings) for field in obj.__fields__):
+            raise NotImplementedError(
+                "Override in subclass and use set_defaults_with_default_constructors to set sub-settings defaults"
+            )
+        return obj

packages/postgres-database/src/simcore_postgres_database/models/comp_pipeline.py

@@ -10,6 +10,13 @@
 
 
 class StateType(enum.Enum):
+    """Discrete states in a task lifecycle
+
+    NOTE: these states are the exact same ones as the models-library (RunningState for a project's execution state)
+    """
+
+    # TODO: how they map to celery.states?
+    # https://docs.celeryproject.org/en/stable/reference/celery.states.html
     NOT_STARTED = "NOT_STARTED"
     PUBLISHED = "PUBLISHED"
     PENDING = "PENDING"
@@ -26,12 +33,21 @@ def _new_uuid():
 comp_pipeline = sa.Table(
     "comp_pipeline",
     metadata,
-    sa.Column("project_id", sa.String, primary_key=True, default=_new_uuid),
-    sa.Column("dag_adjacency_list", sa.JSON),
+    sa.Column(
+        "project_id",
+        sa.String,
+        primary_key=True,
+        default=_new_uuid,
+        doc="Project ID including this pipeline",
+    ),
+    sa.Column(
+        "dag_adjacency_list", sa.JSON, doc="Adjancey list for the pipeline's graph"
+    ),
     sa.Column(
         "state",
         sa.Enum(StateType),
         nullable=False,
         server_default=StateType.NOT_STARTED.value,
+        doc="Current state of this pipeline",
     ),
 )

packages/postgres-database/src/simcore_postgres_database/models/comp_tasks.py

@@ -18,31 +18,49 @@ class NodeClass(enum.Enum):
 comp_tasks = sa.Table(
     "comp_tasks",
     metadata,
-    # this task db id
-    sa.Column("task_id", sa.Integer, primary_key=True),
-    sa.Column("project_id", sa.String, sa.ForeignKey("comp_pipeline.project_id")),
-    # dag node id and class
-    sa.Column("node_id", sa.String),
-    sa.Column("node_class", sa.Enum(NodeClass)),
-    # celery task id
-    sa.Column("job_id", sa.String),
-    # internal id (better for debugging, nodes from 1 to N)
-    sa.Column("internal_id", sa.Integer),
-    sa.Column("schema", sa.JSON),
-    sa.Column("inputs", sa.JSON),
-    sa.Column("outputs", sa.JSON),
-    sa.Column("run_hash", sa.String, nullable=True),
-    sa.Column("image", sa.JSON),
+    sa.Column(
+        "task_id",
+        sa.Integer,
+        primary_key=True,
+        doc="Primary key, identifies the task in this table",
+    ),
+    sa.Column(
+        "project_id",
+        sa.String,
+        sa.ForeignKey("comp_pipeline.project_id"),
+        doc="Project that contains the node associated to this task",
+    ),
+    sa.Column("node_id", sa.String, doc="Node associated to this task"),
+    sa.Column(
+        "node_class",
+        sa.Enum(NodeClass),
+        doc="Classification of the node associated to this task",
+    ),
+    sa.Column("job_id", sa.String, doc="Celery ID for this task"),
+    sa.Column("internal_id", sa.Integer, doc="DEV: only for development. From 1 to N"),
+    sa.Column("schema", sa.JSON, doc="Schema for inputs and outputs"),
+    sa.Column("inputs", sa.JSON, doc="Input values"),
+    sa.Column("outputs", sa.JSON, doc="Output values"),
+    sa.Column(
+        "run_hash",
+        sa.String,
+        nullable=True,
+        doc="Hashes inputs before run. Used to detect changes in inputs.",
+    ),
+    sa.Column(
+        "image", sa.JSON, doc="Metadata about service image associated to this node"
+    ),
     sa.Column(
         "state",
         sa.Enum(StateType),
         nullable=False,
         server_default=StateType.NOT_STARTED.value,
+        doc="Current state in the task lifecicle",
     ),
     # utc timestamps for submission/start/end
-    sa.Column("submit", sa.DateTime),
-    sa.Column("start", sa.DateTime),
-    sa.Column("end", sa.DateTime),
+    sa.Column("submit", sa.DateTime, doc="UTC timestamp for task submission"),
+    sa.Column("start", sa.DateTime, doc="UTC timestamp when task started"),
+    sa.Column("end", sa.DateTime, doc="UTC timestamp for task completion"),
     sa.UniqueConstraint("project_id", "node_id", name="project_node_uniqueness"),
 )
 

packages/pytest-simcore/setup.py

@@ -6,9 +6,8 @@
     maintainer="pcrespov, sanderegg",
     description="pytest plugin with fixtures and test helpers for osparc-simcore repo modules",
     py_modules=["pytest_simcore"],
-    # WARNING: this is used in frozen services as well
+    # WARNING: this is used in frozen services as well !!!!
     python_requires=">=3.6",
-    # TODO create partial extensions:
     install_requires=["pytest>=3.5.0"],
     extras_require={
         "all": [