Skip to content

Commit aad00a6

Browse files
authored
⚗️ Adds dask scheduler and sidecar worker in stack (ITISFoundation#2366)
* ADDED: experimental dask-sidecar (worker) and dask scheduler services Both worker and scheduler services use the same dask-sidecar image but have different boots Adds scheduler to both dynamic and computational_services_subnet so notebook can schedule tasks in dask workers These services are STILL NOT CONNECTED to the director-v2 (next PRs will provide that functionality) * FIXES: settings initialization models_library.settings.base.BaseCustomSettings
1 parent e212050 commit aad00a6

File tree

57 files changed

+2421
-100
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+2421
-100
lines changed

.github/PULL_REQUEST_TEMPLATE.md

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,9 @@ Consider prefix your PR message with an emoticon
99
🏗️ maintenance
1010
📚 documentation
1111
12-
and append (⚠️ devops) if changes in devops configuration required before deploying
12+
or get your emoji from https://gitmoji.dev/
1313
14-
SEE https://github.com/dannyfritz/commit-message-emoji
15-
SEE https://emojipedia.org
14+
and append (⚠️ devops) if changes in devops configuration required before deploying
1615
-->
1716

1817
## What do these changes do?

.github/workflows/ci-testing-deploy.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,53 @@ jobs:
314314
name: codeclimate-${{ github.job }}-coverage
315315
path: codeclimate.${{ github.job }}_coverage.json
316316

317+
unit-test-dask-sidecar:
318+
name: "[unit] dask-sidecar"
319+
runs-on: ${{ matrix.os }}
320+
strategy:
321+
matrix:
322+
python: [3.8]
323+
os: [ubuntu-20.04]
324+
fail-fast: false
325+
steps:
326+
- uses: actions/checkout@v2
327+
- name: setup docker
328+
run: |
329+
sudo ./ci/github/helpers/setup_docker_compose.bash
330+
./ci/github/helpers/setup_docker_experimental.bash
331+
- name: setup python environment
332+
uses: actions/setup-python@v2
333+
with:
334+
python-version: ${{ matrix.python }}
335+
- name: show system version
336+
run: ./ci/helpers/show_system_versions.bash
337+
- uses: actions/cache@v2
338+
name: getting cached data
339+
with:
340+
path: ~/.cache/pip
341+
key: ${{ runner.os }}-pip-dask-sidecar-${{ hashFiles('services/dask-sidecar/requirements/ci.txt') }}
342+
restore-keys: |
343+
${{ runner.os }}-pip-dask-sidecar-
344+
${{ runner.os }}-pip-
345+
${{ runner.os }}-
346+
- name: install
347+
run: ./ci/github/unit-testing/dask-sidecar.bash install
348+
- name: test
349+
run: ./ci/github/unit-testing/dask-sidecar.bash test
350+
- uses: codecov/codecov-action@v1
351+
with:
352+
flags: unittests #optional
353+
- name: prepare codeclimate coverage file
354+
run: |
355+
curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
356+
chmod +x ./cc-test-reporter && ./cc-test-reporter --version
357+
./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
358+
- name: upload codeclimate coverage
359+
uses: actions/upload-artifact@v2
360+
with:
361+
name: codeclimate-${{ github.job }}-coverage
362+
path: codeclimate.${{ github.job }}_coverage.json
363+
317364
unit-test-dynamic-sidecar:
318365
name: "[unit] dynamic-sidecar"
319366
runs-on: ${{ matrix.os }}
@@ -1803,6 +1850,7 @@ jobs:
18031850
unit-test-director-v2,
18041851
unit-test-sidecar,
18051852
unit-test-dynamic-sidecar,
1853+
unit-test-dask-sidecar,
18061854
unit-test-service-integration,
18071855
unit-test-service-library,
18081856
unit-test-models-library,
@@ -1858,6 +1906,7 @@ jobs:
18581906
unit-test-director-v2,
18591907
unit-test-sidecar,
18601908
unit-test-dynamic-sidecar,
1909+
unit-test-dask-sidecar,
18611910
unit-test-frontend,
18621911
unit-test-python-linting,
18631912
unit-test-service-integration,

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,3 +162,6 @@ services/**/.codeclimate.yml
162162

163163
# pytest-fixture-tools output
164164
artifacts
165+
166+
# dask artifacts in devel mode
167+
dask*-space/

Makefile

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,15 @@ $(if $(IS_WIN),$(error Windows is not supported in all recipes. Use WSL instead.
3030
SERVICES_LIST := \
3131
api-server \
3232
catalog \
33+
dask-sidecar \
3334
director \
3435
director-v2 \
3536
dynamic-sidecar \
3637
migration \
3738
sidecar \
39+
static-webserver \
3840
storage \
39-
webserver \
40-
static-webserver
41+
webserver
4142

4243
CLIENT_WEB_OUTPUT := $(CURDIR)/services/web/client/source-output
4344

@@ -199,7 +200,7 @@ TableWidth=140;\
199200
printf "%80s| %22s| %12s| %12s\n" Endpoint Name User Password;\
200201
printf "%.$${TableWidth}s\n" "$$separator";\
201202
printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9081' 'oSparc platform';\
202-
printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18080/?pgsql=postgres&username=$${POSTGRES_USER}&db=$${POSTGRES_DB}&ns=public' 'Postgres DB' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
203+
printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18080/?pgsql=postgres&username='$${POSTGRES_USER}'&db='$${POSTGRES_DB}'&ns=public' 'Postgres DB' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
203204
printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9000' Portainer admin adminadmin;\
204205
printf "$$rows" 'http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18081' Redis
205206
endef
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
6+
IFS=$'\n\t'
7+
8+
install() {
9+
bash ci/helpers/ensure_python_pip.bash;
10+
pushd services/dask-sidecar; pip3 install -r requirements/ci.txt; popd;
11+
pip list -v
12+
}
13+
14+
test() {
15+
pytest --cov=simcore_service_dask_sidecar --durations=10 --cov-append \
16+
--color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
17+
-v services/dask-sidecar/tests/unit
18+
}
19+
20+
# Check if the function exists (bash specific)
21+
if declare -f "$1" > /dev/null
22+
then
23+
# call arguments verbatim
24+
"$@"
25+
else
26+
# Show a helpful error
27+
echo "'$1' is not a known function name" >&2
28+
exit 1
29+
fi

packages/models-library/src/models_library/projects_state.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@
1212

1313
@unique
1414
class RunningState(str, Enum):
15+
"""State of execution of a project's computational workflow
16+
17+
SEE StateType for task state
18+
"""
19+
1520
UNKNOWN = "UNKNOWN"
1621
PUBLISHED = "PUBLISHED"
1722
NOT_STARTED = "NOT_STARTED"

packages/models-library/src/models_library/settings/base.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from contextlib import suppress
2-
from typing import List, Tuple
2+
from typing import List, Tuple, Type
33

44
from pydantic import BaseSettings, Extra, SecretStr, ValidationError
55

@@ -15,7 +15,7 @@ class Config:
1515

1616
@classmethod
1717
def set_defaults_with_default_constructors(
18-
cls, default_fields: List[Tuple[str, "BaseCustomSettings"]]
18+
cls, default_fields: List[Tuple[str, Type["BaseCustomSettings"]]]
1919
):
2020
# This funcion can set defaults on fields that are BaseSettings as well
2121
# It is used in control construction of defaults.
@@ -33,3 +33,14 @@ def set_defaults_with_default_constructors(
3333
field_obj.default = default
3434
field_obj.field_info.default = default
3535
field_obj.required = False
36+
37+
@classmethod
38+
def create_from_envs(cls):
39+
obj = cls()
40+
41+
# TODO: perform this check on FieldInfo upon class construction
42+
if any(isinstance(field, BaseSettings) for field in obj.__fields__):
43+
raise NotImplementedError(
44+
"Override in subclass and use set_defaults_with_default_constructors to set sub-settings defaults"
45+
)
46+
return obj

packages/postgres-database/src/simcore_postgres_database/models/comp_pipeline.py

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@
1010

1111

1212
class StateType(enum.Enum):
13+
"""Discrete states in a task lifecycle
14+
15+
NOTE: these states are the exact same ones as the models-library (RunningState for a project's execution state)
16+
"""
17+
18+
# TODO: how they map to celery.states?
19+
# https://docs.celeryproject.org/en/stable/reference/celery.states.html
1320
NOT_STARTED = "NOT_STARTED"
1421
PUBLISHED = "PUBLISHED"
1522
PENDING = "PENDING"
@@ -26,12 +33,21 @@ def _new_uuid():
2633
comp_pipeline = sa.Table(
2734
"comp_pipeline",
2835
metadata,
29-
sa.Column("project_id", sa.String, primary_key=True, default=_new_uuid),
30-
sa.Column("dag_adjacency_list", sa.JSON),
36+
sa.Column(
37+
"project_id",
38+
sa.String,
39+
primary_key=True,
40+
default=_new_uuid,
41+
doc="Project ID including this pipeline",
42+
),
43+
sa.Column(
44+
"dag_adjacency_list", sa.JSON, doc="Adjancey list for the pipeline's graph"
45+
),
3146
sa.Column(
3247
"state",
3348
sa.Enum(StateType),
3449
nullable=False,
3550
server_default=StateType.NOT_STARTED.value,
51+
doc="Current state of this pipeline",
3652
),
3753
)

packages/postgres-database/src/simcore_postgres_database/models/comp_tasks.py

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,31 +18,49 @@ class NodeClass(enum.Enum):
1818
comp_tasks = sa.Table(
1919
"comp_tasks",
2020
metadata,
21-
# this task db id
22-
sa.Column("task_id", sa.Integer, primary_key=True),
23-
sa.Column("project_id", sa.String, sa.ForeignKey("comp_pipeline.project_id")),
24-
# dag node id and class
25-
sa.Column("node_id", sa.String),
26-
sa.Column("node_class", sa.Enum(NodeClass)),
27-
# celery task id
28-
sa.Column("job_id", sa.String),
29-
# internal id (better for debugging, nodes from 1 to N)
30-
sa.Column("internal_id", sa.Integer),
31-
sa.Column("schema", sa.JSON),
32-
sa.Column("inputs", sa.JSON),
33-
sa.Column("outputs", sa.JSON),
34-
sa.Column("run_hash", sa.String, nullable=True),
35-
sa.Column("image", sa.JSON),
21+
sa.Column(
22+
"task_id",
23+
sa.Integer,
24+
primary_key=True,
25+
doc="Primary key, identifies the task in this table",
26+
),
27+
sa.Column(
28+
"project_id",
29+
sa.String,
30+
sa.ForeignKey("comp_pipeline.project_id"),
31+
doc="Project that contains the node associated to this task",
32+
),
33+
sa.Column("node_id", sa.String, doc="Node associated to this task"),
34+
sa.Column(
35+
"node_class",
36+
sa.Enum(NodeClass),
37+
doc="Classification of the node associated to this task",
38+
),
39+
sa.Column("job_id", sa.String, doc="Celery ID for this task"),
40+
sa.Column("internal_id", sa.Integer, doc="DEV: only for development. From 1 to N"),
41+
sa.Column("schema", sa.JSON, doc="Schema for inputs and outputs"),
42+
sa.Column("inputs", sa.JSON, doc="Input values"),
43+
sa.Column("outputs", sa.JSON, doc="Output values"),
44+
sa.Column(
45+
"run_hash",
46+
sa.String,
47+
nullable=True,
48+
doc="Hashes inputs before run. Used to detect changes in inputs.",
49+
),
50+
sa.Column(
51+
"image", sa.JSON, doc="Metadata about service image associated to this node"
52+
),
3653
sa.Column(
3754
"state",
3855
sa.Enum(StateType),
3956
nullable=False,
4057
server_default=StateType.NOT_STARTED.value,
58+
doc="Current state in the task lifecicle",
4159
),
4260
# utc timestamps for submission/start/end
43-
sa.Column("submit", sa.DateTime),
44-
sa.Column("start", sa.DateTime),
45-
sa.Column("end", sa.DateTime),
61+
sa.Column("submit", sa.DateTime, doc="UTC timestamp for task submission"),
62+
sa.Column("start", sa.DateTime, doc="UTC timestamp when task started"),
63+
sa.Column("end", sa.DateTime, doc="UTC timestamp for task completion"),
4664
sa.UniqueConstraint("project_id", "node_id", name="project_node_uniqueness"),
4765
)
4866

packages/pytest-simcore/setup.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,8 @@
66
maintainer="pcrespov, sanderegg",
77
description="pytest plugin with fixtures and test helpers for osparc-simcore repo modules",
88
py_modules=["pytest_simcore"],
9-
# WARNING: this is used in frozen services as well
9+
# WARNING: this is used in frozen services as well !!!!
1010
python_requires=">=3.6",
11-
# TODO create partial extensions:
1211
install_requires=["pytest>=3.5.0"],
1312
extras_require={
1413
"all": [

0 commit comments

Comments
 (0)