✨Comp backend: add local prometheus in every cluster to gather metrics (ITISFoundation#5530)

Author: sanderegg

services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml

@@ -4,6 +4,8 @@ services:
     image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG}
     init: true
     hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
+    networks:
+      - cluster
     environment:
       DASK_TLS_CA_FILE: ${DASK_TLS_CA_FILE}
       DASK_TLS_CERT: ${DASK_TLS_CERT}
@@ -16,9 +18,15 @@ services:
       - 8786:8786 # dask-scheduler access
       - 8787:8787 # dashboard
     deploy:
+      labels:
+        prometheus-job: scheduler
+        prometheus-port: 8787
       placement:
         constraints:
           - "node.role==manager"
+      resources:
+        limits:
+          memory: 2048M
     secrets:
       - source: dask_tls_ca
         target: ${DASK_TLS_CA_FILE}
@@ -38,6 +46,8 @@ services:
       - computational_shared_data:${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data}
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro
+    networks:
+      - cluster
     environment:
       DASK_LOG_FORMAT_LOCAL_DEV_ENABLED: 1
       DASK_NPROCS: 1
@@ -54,6 +64,9 @@ services:
       SIDECAR_COMP_SERVICES_SHARED_VOLUME_NAME: computational_shared_data
     deploy:
       mode: global
+      labels:
+        prometheus-job: sidecars
+        prometheus-port: 8787
       placement:
         constraints:
           - "node.role==worker"
@@ -95,11 +108,19 @@ services:
       REDIS_HOST: redis
       REDIS_PORT: 6379
     volumes:
-      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+    networks:
+      - cluster
     deploy:
+      labels:
+        prometheus-job: autoscaling
+        prometheus-port: 8000
       placement:
         constraints:
           - "node.role==manager"
+      resources:
+        limits:
+          memory: 512M
     secrets:
       - source: dask_tls_ca
         target: ${DASK_TLS_CA_FILE}
@@ -123,16 +144,53 @@ services:
       retries: 50
     volumes:
       - redis-data:/data
+    networks:
+      - cluster
     deploy:
       placement:
         constraints:
           - "node.role==manager"
+      resources:
+        limits:
+          memory: 512M
+          cpus: "0.5"
+
+  prometheus:
+    image: prom/prometheus:v2.51.0@sha256:5ccad477d0057e62a7cd1981ffcc43785ac10c5a35522dc207466ff7e7ec845f
+    command:
+      - "--storage.tsdb.retention.size=1GB"
+    ports:
+      - 9090:9090
+    configs:
+      - source: prometheus-config
+        target: /etc/prometheus/prometheus.yml
+    volumes:
+      - prometheus-data:/prometheus
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    user: root # because of docker
+    networks:
+      - cluster
+    deploy:
+      placement:
+        constraints:
+          - "node.role==manager"
+      resources:
+        limits:
+          memory: 1024M
+          cpus: "1.0"
+
+networks:
+  cluster:
+
+configs:
+  prometheus-config:
+    file: ./prometheus.yml
 
 volumes:
   computational_shared_data:
     name: computational_shared_data
   redis-data:
-
+  prometheus-data:
 
 secrets:
   dask_tls_ca:

services/clusters-keeper/src/simcore_service_clusters_keeper/data/prometheus.yml

@@ -0,0 +1,50 @@
+global:
+  scrape_interval: "29s"
+scrape_configs:
+  # Create a job for Docker Swarm containers.
+  - job_name: 'docker nodes'
+    dockerswarm_sd_configs:
+      - host: unix:///var/run/docker.sock
+        role: nodes
+    relabel_configs:
+      # Fetch metrics on port 9323.
+      - source_labels: [__meta_dockerswarm_node_address]
+        target_label: __address__
+        replacement: $1:9323
+      # Set hostname as instance label
+      - source_labels: [__meta_dockerswarm_node_hostname]
+        target_label: instance
+  # Create a job for Docker Swarm containers.
+  - job_name: 'docker tasks'
+    dockerswarm_sd_configs:
+      - host: unix:///var/run/docker.sock
+        role: tasks
+    relabel_configs:
+      # Set hostname as instance label
+      - source_labels: [__meta_dockerswarm_node_hostname]
+        target_label: instance
+      # Only keep containers that should be running.
+      - source_labels: [__meta_dockerswarm_task_desired_state]
+        regex: running
+        action: keep
+      # Only keep tasks with a `prometheus_port` label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_port]
+        regex: .+
+        action: keep
+      # Only keep containers that have a `prometheus-job` label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job]
+        regex: .+
+        action: keep
+      # Use the prometheus-job Swarm label as Prometheus job label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job]
+        target_label: job
+      # Specify the metric path if needed (optional)
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_path]
+        target_label: __metrics_path__
+        regex: (.+)
+      # Use the `prometheus_port` Swarm label to set the __address__ for scraping.
+      - source_labels: [__address__, __meta_dockerswarm_service_label_prometheus_port]
+        target_label: __address__
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        action: replace

services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py

@@ -21,7 +21,9 @@
 from .dask import get_scheduler_url
 
 _DOCKER_COMPOSE_FILE_NAME: Final[str] = "docker-compose.yml"
+_PROMETHEUS_FILE_NAME: Final[str] = "prometheus.yml"
 _HOST_DOCKER_COMPOSE_PATH: Final[Path] = Path(f"/{_DOCKER_COMPOSE_FILE_NAME}")
+_HOST_PROMETHEUS_PATH: Final[Path] = Path(f"/{_PROMETHEUS_FILE_NAME}")
 _HOST_CERTIFICATES_BASE_PATH: Final[Path] = Path("/.dask-sidecar-certificates")
 _HOST_TLS_CA_FILE_PATH: Final[Path] = _HOST_CERTIFICATES_BASE_PATH / "tls_dask_ca.pem"
 _HOST_TLS_CERT_FILE_PATH: Final[Path] = (
@@ -42,6 +44,12 @@ def _docker_compose_yml_base64_encoded() -> str:
     return _base_64_encode(file_path)
 
 
+@functools.lru_cache
+def _prometheus_yml_base64_encoded() -> str:
+    file_path = PACKAGE_DATA_FOLDER / _PROMETHEUS_FILE_NAME
+    return _base_64_encode(file_path)
+
+
 def _prepare_environment_variables(
     app_settings: ApplicationSettings,
     *,
@@ -117,6 +125,7 @@ def create_startup_script(
             # NOTE: https://stackoverflow.com/questions/41203492/solving-redis-warnings-on-overcommit-memory-and-transparent-huge-pages-for-ubunt
             "sysctl vm.overcommit_memory=1",
             f"echo '{_docker_compose_yml_base64_encoded()}' | base64 -d > {_HOST_DOCKER_COMPOSE_PATH}",
+            f"echo '{_prometheus_yml_base64_encoded()}' | base64 -d > {_HOST_PROMETHEUS_PATH}",
             # NOTE: --default-addr-pool is necessary in order to prevent conflicts with AWS node IPs
             "docker swarm init --default-addr-pool 172.20.0.0/14",
             f"{' '.join(environment_variables)} docker stack deploy --with-registry-auth --compose-file={_HOST_DOCKER_COMPOSE_PATH} dask_stack",

services/clusters-keeper/tests/manual/README.md

@@ -34,6 +34,7 @@ flowchart TD
 1. build simcore
 ```bash
 git clone https://github.com/ITISFoundation/osparc-simcore.git
+cd osparc-simcore
 make .env # generate initial .env file
 make build-devel # build for development mode or
 make build # for production mode
@@ -82,6 +83,13 @@ WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION="00:03:00"
 WORKERS_EC2_INSTANCES_CUSTOM_TAGS='{"osparc-tag": "some fun tag value"}'
 ```
 
+4. prepare dask TLS certificates
+NOTE: the dask TLS certificates are in AWS and shall be copied into the local stack such that the director-v2 can access the clusters
+these are defined by PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CA, PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CERT and PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_KEY
+  1. one need to go to the AWS Parameter Store (SSM)
+  2. find these entries then copy their contents into respectively services/dask-sidecar/.dask-certificates/dask-cert.pem and services/dask-sidecar/.dask-certificates/dask-key.pem
+
+
 5. start osparc
 ```bash
 make up-devel # for devel mode

services/clusters-keeper/tests/unit/test_utils_clusters.py

@@ -22,6 +22,7 @@
     NoAuthentication,
     TLSAuthentication,
 )
+from pydantic import ByteSize, parse_obj_as
 from pytest_simcore.helpers.utils_envs import EnvVarsDict
 from simcore_service_clusters_keeper.core.settings import ApplicationSettings
 from simcore_service_clusters_keeper.utils.clusters import (
@@ -140,6 +141,33 @@ def test_create_startup_script(
     )
 
 
+def test_create_startup_script_script_size_below_16kb(
+    disabled_rabbitmq: None,
+    mocked_ec2_server_envs: EnvVarsDict,
+    mocked_redis_server: None,
+    app_settings: ApplicationSettings,
+    cluster_machines_name_prefix: str,
+    clusters_keeper_docker_compose: dict[str, Any],
+    ec2_boot_specs: EC2InstanceBootSpecific,
+):
+    additional_custom_tags = {
+        AWSTagKey("pytest-tag-key"): AWSTagValue("pytest-tag-value")
+    }
+    startup_script = create_startup_script(
+        app_settings,
+        cluster_machines_name_prefix=cluster_machines_name_prefix,
+        ec2_boot_specific=ec2_boot_specs,
+        additional_custom_tags=additional_custom_tags,
+    )
+    script_size_in_bytes = len(startup_script.encode("utf-8"))
+
+    print(
+        f"current script size is {parse_obj_as(ByteSize, script_size_in_bytes).human_readable()}"
+    )
+    # NOTE: EC2 user data cannot be above 16KB, we keep some margin here
+    assert script_size_in_bytes < 15 * 1024
+
+
 def test_startup_script_defines_all_envs_for_docker_compose(
     disabled_rabbitmq: None,
     mocked_ec2_server_envs: EnvVarsDict,