Merge pull request #86 from GabrielSalla/add-scalable-deploy

Add scalable deploy

Author: GabrielSalla

Dockerfile

@@ -3,6 +3,7 @@ ARG PYTHON_VERSION=3.12
 # Base image
 FROM python:${PYTHON_VERSION}-alpine AS base
 
+ARG CONFIGS_FILE
 ENV VIRTUAL_ENV=/opt/venv \
     PATH="/opt/venv/bin:$PATH"
 

Makefile

@@ -1,39 +1,55 @@
+# Development
 build-dev:
-	docker compose -f docker-compose-dev.yaml build sentinela-dev
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml build sentinela-dev
 
 pull-dev:
-	docker compose -f docker-compose-dev.yaml pull --quiet postgres motoserver
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml pull --quiet postgres motoserver
 
 migrate-dev:
-	docker compose -f docker-compose-dev.yaml run --rm --service-ports sentinela-dev alembic upgrade head
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml run --rm --service-ports sentinela-dev alembic upgrade head
 
 run-dev:
-	docker compose -f docker-compose-dev.yaml up sentinela-dev
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml up sentinela-dev
 
 run-shell-dev:
-	docker compose -f docker-compose-dev.yaml run --rm --service-ports sentinela-dev /bin/sh
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml run --rm --service-ports sentinela-dev /bin/sh
 
 test-dev:
-	docker compose -f docker-compose-dev.yaml run --rm sentinela-dev pytest
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml run --rm sentinela-dev pytest
 
 down-dev:
-	docker compose -f docker-compose-dev.yaml down
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml down
 
+# Local development setup
 build-local:
-	docker compose -f docker-compose-local.yaml build sentinela-local
+	docker compose --project-directory . -f ./docker/docker-compose-local.yaml build sentinela-local
 
 migrate-local:
-	docker compose -f docker-compose-local.yaml run --rm --service-ports sentinela-local alembic upgrade head
+	docker compose --project-directory . -f ./docker/docker-compose-local.yaml run --rm --service-ports sentinela-local alembic upgrade head
 
 run-local:
-	docker compose -f docker-compose-local.yaml up sentinela-local
+	docker compose --project-directory . -f ./docker/docker-compose-local.yaml up sentinela-local
 
 down-local:
-	docker compose -f docker-compose-local.yaml down
+	docker compose --project-directory . -f ./docker/docker-compose-local.yaml down
 
+# Scalable setup for production-like environment
+build-scalable:
+	docker compose --project-directory . -f ./docker/docker-compose-scalable.yaml build sentinela-controller
+
+migrate-scalable:
+	docker compose --project-directory . -f ./docker/docker-compose-scalable.yaml run --rm --service-ports sentinela-controller alembic upgrade head
+
+run-scalable:
+	docker compose --project-directory . -f ./docker/docker-compose-scalable.yaml up sentinela-controller sentinela-executor
+
+down-scalable:
+	docker compose --project-directory . -f ./docker/docker-compose-scalable.yaml down
+
+# Development utilities
 linter:
-	docker compose -f docker-compose-dev.yaml run --rm --no-deps sentinela-dev ruff check
-	docker compose -f docker-compose-dev.yaml run --rm --no-deps sentinela-dev ruff format --check --diff
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml run --rm --no-deps sentinela-dev ruff check
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml run --rm --no-deps sentinela-dev ruff format --check --diff
 
 mypy:
-	docker compose -f docker-compose-dev.yaml run --rm --no-deps sentinela-dev mypy --install-types --non-interactive
+	docker compose --project-directory . -f ./docker/docker-compose-dev.yaml run --rm --no-deps sentinela-dev mypy --install-types --non-interactive

configs/configs-scalable.yaml

@@ -0,0 +1,66 @@
+# The documentation for this configuration file can be found at "docs/configuration_file.md"
+plugins:
+  - aws
+  - postgres
+  - slack
+
+load_sample_monitors: true
+sample_monitors_path: sample_monitors
+internal_monitors_path: internal_monitors
+monitors_load_schedule: "*/5 * * * *"
+
+logging:
+  mode: friendly
+  format: "%(asctime)s [%(levelname)s] %(name)s.%(funcName)s.%(lineno)d: %(message)s"
+
+application_database_settings:
+  pool_size: 10
+
+application_queue:
+  type: plugin.aws.sqs
+  name: app
+  url: http://motoserver:5000/123456789012/app
+  region: us-east-1
+  create_queue: true
+  queue_wait_message_time: 2
+  queue_visibility_time: 15
+
+http_server:
+  port: 8000
+
+time_zone: America/Sao_Paulo
+
+controller_process_schedule: "* * * * *"
+controller_concurrency: 5
+controller_procedures:
+  monitors_stuck:
+    schedule: "*/5 * * * *"
+    params:
+      time_tolerance: 10
+  notifications_alert_solved:
+    schedule: "*/5 * * * *"
+
+executor_concurrency: 5
+executor_sleep: 5
+executor_monitor_timeout: 60
+executor_reaction_timeout: 5
+executor_request_timeout: 2
+executor_monitor_heartbeat_time: 5
+
+max_issues_creation: 100
+
+database_default_acquire_timeout: 30
+database_default_query_timeout: 30
+database_close_timeout: 30
+database_log_query_metrics: false
+
+databases_pools_configs:
+  local:
+    min_size: 0
+    max_size: 5
+    timeout: 10
+    max_inactive_connection_lifetime: 120
+    server_settings:
+      application_name: sentinela
+
+log_all_events: false

configs.yaml renamed to configs/configs.yaml

@@ -22,10 +22,13 @@ services:
       context: .
       dockerfile: Dockerfile
       target: sentinela_dev
+      args:
+        CONFIGS_FILE: configs/configs.yaml
     image: sentinela-dev:latest
     ports:
       - 8000:8000
     environment:
+      CONFIGS_FILE: configs/configs.yaml
       SAMPLE_SLACK_CHANNEL: C07NCL94SDT
       SAMPLE_SLACK_MENTION: U07NFGGMB98
       SLACK_WEBSOCKET_ENABLED: true

docker-compose-dev.yaml renamed to docker/docker-compose-dev.yaml

@@ -11,26 +11,26 @@ services:
       context: .
       dockerfile: Dockerfile
       target: sentinela
+      args:
+        CONFIGS_FILE: configs/configs.yaml
     image: sentinela-local:latest
     ports:
       - 8000:8000
     healthcheck:
-      test: curl --fail localhost:8000/status || exit 1
+      test: python tools/health_check.py 8000
       interval: 10s
       timeout: 1s
       retries: 3
       start_period: 2s
     environment:
+      CONFIGS_FILE: configs/configs.yaml
       SAMPLE_SLACK_CHANNEL: C07NCL94SDT
       SAMPLE_SLACK_MENTION: U07NFGGMB98
       SLACK_WEBSOCKET_ENABLED: true
       SLACK_MAIN_CHANNEL: C07NCL94SDT
       SLACK_MAIN_MENTION: U07NFGGMB98
     env_file:
       - .env.secrets
-    volumes:
-      - ./monitors:/app/monitors:Z
-      - ./internal_monitors:/app/internal_monitors:Z
     depends_on:
       - postgres
     command: python3 src/main.py controller executor

docker-compose-local.yaml renamed to docker/docker-compose-local.yaml

@@ -0,0 +1,81 @@
+services:
+  postgres:
+    image: postgres:16.4-alpine
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_PASSWORD=postgres
+
+  motoserver:
+    image: ghcr.io/getmoto/motoserver:latest
+    ports:
+      - 5000:5000
+    healthcheck:
+      test: curl --fail localhost:5000 || exit 1
+      interval: 1s
+      timeout: 1s
+      retries: 5
+      start_period: 1s
+
+  sentinela-controller:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: sentinela
+      args:
+        CONFIGS_FILE: configs/configs-scalable.yaml
+    image: sentinela-scalable:latest
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: python tools/health_check.py 8000
+      interval: 10s
+      timeout: 1s
+      retries: 3
+      start_period: 2s
+    environment:
+      CONFIGS_FILE: configs-scalable.yaml
+      SAMPLE_SLACK_CHANNEL: C07NCL94SDT
+      SAMPLE_SLACK_MENTION: U07NFGGMB98
+      SLACK_WEBSOCKET_ENABLED: true
+      SLACK_MAIN_CHANNEL: C07NCL94SDT
+      SLACK_MAIN_MENTION: U07NFGGMB98
+    env_file:
+      - .env.secrets
+    depends_on:
+      - postgres
+      - motoserver
+    command: python3 src/main.py controller
+
+  sentinela-executor:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: sentinela
+      args:
+        CONFIGS_FILE: configs/configs-scalable.yaml
+    image: sentinela-scalable:latest
+    deploy:
+      mode: replicated
+      replicas: 2
+    ports:
+      - 8001-8100:8000
+    healthcheck:
+      test: python tools/health_check.py 8000
+      interval: 10s
+      timeout: 1s
+      retries: 3
+      start_period: 2s
+    environment:
+      CONFIGS_FILE: configs-scalable.yaml
+      SAMPLE_SLACK_CHANNEL: C07NCL94SDT
+      SAMPLE_SLACK_MENTION: U07NFGGMB98
+      SLACK_WEBSOCKET_ENABLED: true
+      SLACK_MAIN_CHANNEL: C07NCL94SDT
+      SLACK_MAIN_MENTION: U07NFGGMB98
+    env_file:
+      - .env.secrets
+    depends_on:
+      - postgres
+      - motoserver
+    command: python3 src/main.py executor

docker/docker-compose-scalable.yaml

@@ -3,10 +3,10 @@ The basic configs are set through the `configs.yaml` file. This file is read whe
 
 The monitors path is also defined in the `configs.yaml` file. By default, it's set to the `sample_monitors` folder, but it can be changed to another folder if desired. The `configs.yaml` file also have other configurations that can be adjusted.
 
+# Environment variables
 > [!IMPORTANT]
 > Check the documentation for the plugins that are being used to see if they have environment variables of their own.
 
-# Environment variables
 ## `CONFIGS_FILE`
 The application will try to load the configs file through the path defined in the `CONFIGS_FILE` environment variable. If this variable is not defined, it'll look for the file in the root directory of the application.