Logging storage service for platform monitoring

.env.cluster

@@ -28,7 +28,7 @@ REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3
 # Reverse Proxy - Nginx
 REVERSE_PROXY_INSTANCES=3
 DOMAIN_NAME=domain
-SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain
+SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain,minio.domain
 STAGING=true
 INSECURE=false
 

infrastructure/ansible/roles/docker/files/docker-daemon.json

@@ -2,6 +2,7 @@
   "log-driver": "json-file",
   "log-opts": {
     "max-size": "10m",
-    "max-file": "3"
+    "max-file": "3",
+    "labels-regex": "^.+"
   }
 }

monitoring/docker-compose.cluster.yml

@@ -29,6 +29,96 @@ services:
       - '--web.console.templates=/etc/prometheus/consoles'
       - '--web.enable-lifecycle'
 
+  minio1:
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+
+  minio2:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio2
+    volumes:
+      - minio2_data1:/data1
+      - minio2_data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+      replicas: 1
+
+  minio3:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio3
+    volumes:
+      - minio3_data1:/data1
+      - minio3_data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-2"
+      replicas: 1
+
+  minio4:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio4
+    volumes:
+      - minio4_data1:/data1
+      - minio4_data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-3"
+      replicas: 1
+
 configs:
   prometheus.yml:
     file: ./prometheus/prometheus.yml
@@ -38,3 +128,9 @@ configs:
 
 volumes:
   prometheus_data_backup:
+  minio2_data1:
+  minio2_data2:
+  minio3_data1:
+  minio3_data2:
+  minio4_data1:
+  minio4_data2:

monitoring/docker-compose.dev.yml

@@ -12,3 +12,15 @@ services:
       - target: 9090
         published: 9090
         mode: host
+
+  loki:
+    ports:
+      - target: 3100
+        published: 3100
+        mode: host
+
+  minio1:
+    ports:
+      - target: 9001
+        published: 9001
+        mode: host

monitoring/docker-compose.yml

@@ -49,6 +49,8 @@ services:
         source: kminion-groups_rev1.json
       - target: /etc/grafana/provisioning/dashboards/applications/kminion-topic_rev1.json
         source: kminion-topic_rev1.json
+      - target: /etc/grafana/provisioning/dashboards/containers/logging-universal-dashboard_rev1.json
+        source: logging-universal-dashboard_rev1.json
 
   prometheus:
     image: prom/prometheus:v2.38.0
@@ -94,6 +96,59 @@ services:
       - KAFKA_BROKER_LIST=kafka:9092
       - KAFKA_COMPRESSION=gzip
 
+  loki:
+    image: grafana/loki:2.6.1
+    volumes:
+      - loki_data:/tmp/loki
+    environment:
+      MO_SECURITY_ADMIN_USER: ${MO_SECURITY_ADMIN_USER}
+      MO_SECURITY_ADMIN_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    configs:
+      - target: /etc/loki/loki-config.yml
+        source: loki-config.yml
+    command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
+    deploy:
+      labels:
+        - prometheus-job-service=loki
+        - prometheus-address=loki:3100
+
+  promtail:
+    image: grafana/promtail:2.6.1
+    volumes:
+      - /var/lib/docker/containers:/host/containers
+      - /var/log:/var/log:ro
+    configs:
+      - target: /etc/promtail/promtail-config.yml
+        source: promtail-config.yml
+    command: -config.file=/etc/promtail/promtail-config.yml
+    deploy:
+      mode: global
+
+  minio1:
+    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio{1...${NUM_MINIO_SERVERS}}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio1
+    volumes:
+      - minio1_data1:/data1
+      - minio1_data2:/data2
+    deploy:
+      replicas: 1
+
 configs:
   grafana.ini:
     file: ./grafana/grafana.ini
@@ -135,12 +190,30 @@ configs:
     name: kminion-topic_rev1.json-${kminion_topic_rev1_json_DIGEST:?err}
     labels:
       name: grafana
+  logging-universal-dashboard_rev1.json:
+    file: ./grafana/dashboards/containers/logging-universal-dashboard_rev1.json
+    name: logging-universal-dashboard_rev1.json-${logging_universal_dashboard_rev1_json_DIGEST:?err}
+    labels:
+      name: grafana
   prometheus.yml:
     file: ./prometheus/prometheus.yml
     name: prometheus.yml-${prometheus_yml_DIGEST:?err}
     labels:
       name: prometheus
+  loki-config.yml:
+    file: ./loki/loki-config.yml
+    name: loki-config.yml-${loki_config_yml_DIGEST:?err}
+    labels:
+      name: loki
+  promtail-config.yml:
+    file: ./promtail/promtail-config.yml
+    name: promtail-config.yml-${promtail_config_yml_DIGEST:?err}
+    labels:
+      name: promtail
 
 volumes:
   prometheus_data:
   grafana_data:
+  loki_data:
+  minio1_data1:
+  minio1_data2: