unify monitoring and telemetry with prometheus (mysql / elastic / fluentbit)

Author: codeyourweb

.env

@@ -8,6 +8,7 @@ SENTINELKIT_GRAFANA_HOSTNAME=grafana.sentinel-kit.local
 SENTINELKIT_DATAMONITOR_SERVER_TOKEN=9561ffd1b6de615286b9e52a9d5bc3226970449700c9461bdbe4225730b47b20
 BACKEND_JWT_PASSPHRASE=f164cfc913d2faf65a1b7bc8ccd4aa8b11b5958bce7c20c8cf159a576f8a75f7
 MYSQL_ROOT_PASSWORD=sentinel-kit_r00tp4ssw0rd
+MYSQL_EXPORTER_PASSWORD=sentinel-kit_3xp0rt3rp4ssw0rd
 MYSQL_USER=sentinel-kit_user
 MYSQL_PASSWORD=sentinel-kit_passwd
 MYSQL_DATABASE=sentinel-kit_db

README.md

@@ -118,7 +118,6 @@ Persistent data are located in the `data/` folder:
 * `data/grafana`: Contains a persistence of your grafana profile if you want to make your own dashboard and customizations.
 * `data/kibana`: Kibana user customizations (dashboard, config...).
 * `data/log_ingest_data`: Is designed to forward logs if you don't want to use fluentbit HTTP forwarder.
-* `data/mysql_data`: Constains a persistence of the web backend database.
 * `data/yara_triage_data`: is used to automatically scan any file placed in this folder.  
 
 ---

clean-user-data.ps1

@@ -6,6 +6,5 @@ Remove-Item ./data/log_ingest_data/auditd/* -Recurse -Force
 Remove-Item ./data/log_ingest_data/evtx/* -Recurse -Force
 Remove-Item ./data/log_ingest_data/json/* -Recurse -Force
 Remove-Item ./data/fluentbit_db/* -Recurse -Force
-Remove-Item ./data/mysql_data/* -Recurse -Force
 Remove-Item ./data/yara_triage_data/* -Recurse -Force
 docker compose down -v

clean-user-data.sh

@@ -6,6 +6,5 @@ rm -rf ./data/log_ingest_data/evtx/*
 rm -rf ./data/log_ingest_data/auditd/*
 rm -rf ./data/log_ingest_data/json/*
 rm -rf ./data/fluentbit_db/*
-rm -rf ./data/mysql_data/*
 rm -rf ./data/yara_triage_data/*
 docker compose down -v

config/caddy_server/Caddyfile

@@ -1,3 +1,12 @@
+{
+	servers {
+    	metrics
+    }
+    admin :2020
+}
+
+
+
 # TLS reverse proxy configuration
 {$SENTINELKIT_BACKEND_HOSTNAME}:443 {
     tls internal
@@ -78,4 +87,4 @@
 
 {$SENTINELKIT_GRAFANA_HOSTNAME}:80 {
     redir https://{$SENTINELKIT_GRAFANA_HOSTNAME}{uri}
-}
+}

config/grafana/datasources/prometheus.yml

@@ -1,7 +1,7 @@
 apiVersion: 1
 
 datasources:
-  - name: Prometheus-Fluentbit
+  - name: Prometheus
     type: prometheus
     url: http://sentinel-kit-utils-prometheus:9090
     access: proxy

config/mysql/init_mysqld_exporter.sh

@@ -0,0 +1,7 @@
+SQL_COMMANDS="
+CREATE USER IF NOT EXISTS 'exporter'@'%' IDENTIFIED BY '$MYSQL_EXPORTER_PASSWORD';
+GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'%';
+FLUSH PRIVILEGES;
+"
+
+echo "$SQL_COMMANDS" | mysql -u root -p"$MYSQL_ROOT_PASSWORD"

config/mysql/my.cnf

@@ -0,0 +1,4 @@
+[client]
+user=exporter
+host=sentinel-kit-db-mysql 
+port=3306

config/prometheus/prometheus.yml

@@ -4,5 +4,16 @@ global:
 scrape_configs:
   - job_name: 'fluentbit'
     static_configs:
-      - targets: ['sentinel-kit-fluentbit-server:2020']
-    metrics_path: /api/v1/metrics/prometheus
+      - targets: ['sentinel-kit-server-fluentbit:2020']
+    metrics_path: /api/v1/metrics/prometheus
+  - job_name: 'caddy hosts'
+    static_configs:
+      - targets: ['sentinel-kit-server-caddy:2020']
+    metrics_path: /metrics
+  - job_name: 'elasticsearch'
+    static_configs:
+      - targets: ['sentinel-kit-utils-elastic-exporter:9114']
+    metrics_path: /metrics
+  - job_name: 'mysql'
+    static_configs:
+      - targets: ['sentinel-kit-utils-mysql-exporter:9104']

docker-compose.yml

@@ -54,6 +54,7 @@ services:
 
   sentinel-kit-server-fluentbit:
     container_name: sentinel-kit-server-fluentbit
+    hostname: sentinel-kit-server-fluentbit
     build:
       context: .
       dockerfile: config/docker-config/Dockerfile.fluentbit
@@ -75,6 +76,7 @@ services:
 
   sentinel-kit-server-sftp:
     container_name: sentinel-kit-server-sftp
+    hostname: sentinel-kit-server-sftp
     image: atmoz/sftp
     restart: on-failure
     command: ${SFTP_USER}:${SFTP_PASSWORD}:1001
@@ -89,18 +91,35 @@ services:
 
   sentinel-kit-db-mysql:
     container_name: sentinel-kit-db-mysql
+    hostname: sentinel-kit-db-mysql
     image: mysql:9.4.0
     restart: on-failure
     environment:
       MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
       MYSQL_DATABASE: ${MYSQL_DATABASE}
       MYSQL_USER: ${MYSQL_USER}
       MYSQL_PASSWORD: ${MYSQL_PASSWORD}
+      MYSQL_EXPORTER_PASSWORD: ${MYSQL_EXPORTER_PASSWORD}
     volumes:
-      - ./data/mysql_data:/var/lib/mysql
+      - sentinel-kit_db_mysql_data:/var/lib/mysql
+      - ./config/mysql/init_mysqld_exporter.sh:/docker-entrypoint-initdb.d/init_mysqld_exporter.sh:ro
     networks:
       - sentinel-kit-network
 
+  sentinel-kit-utils-phpmyadmin:
+    container_name: sentinel-kit-utils-phpmyadmin
+    hostname: ${SENTINELKIT_PMA_HOSTNAME}
+    image: phpmyadmin/phpmyadmin
+    restart: on-failure
+    environment:
+      PMA_HOST: sentinel-kit-db-mysql
+      PMA_PORT: 3306
+    networks:
+      - sentinel-kit-network
+    depends_on:
+      - sentinel-kit-db-mysql
+    profiles: ["phpmyadmin"]
+
   sentinel-kit-server-caddy:
     image: caddy:2.10.2-alpine
     container_name: sentinel-kit-server-caddy
@@ -278,8 +297,6 @@ services:
       - ./config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
     networks:
       - sentinel-kit-network
-    depends_on:
-      - sentinel-kit-server-fluentbit
     profiles: ["internal-monitoring"]
 
   sentinel-kit-utils-grafana:
@@ -298,11 +315,11 @@ services:
       - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD}
       - GF_SERVER_HTTP_PORT=3000
       - ES_PASSWORD=${ELASTICSEARCH_PASSWORD}
+      - MYSQL_DATABASE=${MYSQL_DATABASE}
+      - MYSQL_USER=${MYSQL_USER}
+      - MYSQL_PASSWORD=${MYSQL_PASSWORD}
     networks:
       - sentinel-kit-network
-    depends_on:
-      - sentinel-kit-utils-prometheus
-      - sentinel-kit-db-elasticsearch-es01
     entrypoint: 
       - /bin/sh
       - -c
@@ -312,19 +329,45 @@ services:
         /run.sh
     profiles: ["internal-monitoring"]
 
-  sentinel-kit-utils-phpmyadmin:
-    container_name: sentinel-kit-utils-phpmyadmin
-    hostname: ${SENTINELKIT_PMA_HOSTNAME}
-    image: phpmyadmin/phpmyadmin
+  sentinel-kit-utils-elastic-exporter:
+    hostname: sentinel-kit-utils-elastic-exporter
+    container_name: sentinel-kit-utils-elastic-exporter
+    image: justwatch/elasticsearch_exporter:latest
     restart: on-failure
     environment:
-      PMA_HOST: sentinel-kit-db-mysql
-      PMA_PORT: 3306
+      - ES_URI=https://sentinel-kit-db-elasticsearch-es01:9200 
+      - ES_ALL=true
+      - ES_TIMEOUT=10s
+      - ES_SKIP_TLS_VERIFY=false
+      - ES_CA_PATH=/usr/share/exporter/ca/ca.crt
+      - ES_USERNAME=elastic 
+      - ES_PASSWORD=${ELASTICSEARCH_PASSWORD}
+    volumes:
+      - sentinel-kit_certificates_elasticsearch:/usr/share/exporter/ca
+    networks:
+      - sentinel-kit-network
+    depends_on:
+      sentinel-kit-db-elasticsearch-es01:
+        condition: service_healthy
+    profiles: ["internal-monitoring"]
+
+  sentinel-kit-utils-mysql-exporter:
+    hostname: sentinel-kit-utils-mysql-exporter
+    container_name: sentinel-kit-utils-mysql-exporter
+    image: prom/mysqld-exporter:latest
+    restart: on-failure
+    environment:
+      - MYSQLD_EXPORTER_PASSWORD=${MYSQL_EXPORTER_PASSWORD}
+    volumes:
+      - ./config/mysql/my.cnf:/etc/mysql/my.cnf:ro
+    command:
+      [ "--config.my-cnf=/etc/mysql/my.cnf" ]
     networks:
       - sentinel-kit-network
     depends_on:
       - sentinel-kit-db-mysql
-    profiles: ["phpmyadmin"]
+    profiles: ["internal-monitoring"]
+
 networks:
   sentinel-kit-network:
     driver: bridge
@@ -334,6 +377,7 @@ volumes:
   sentinel-kit_server_caddy_config:
   sentinel-kit_db_elasticsearch_es01_data:
   sentinel-kit_db_elasticsearch_es02_data:
+  sentinel-kit_db_mysql_data:
   sentinel-kit_server_backend_vendor_cache:
   sentinel-kit_server_backend_var_cache:
   sentinel-kit_server_backend_public: