Add 'loki/promtail' logging

jpraynaud · jpraynaud · commit 1ce356fef95b · 2023-07-04T12:43:46.000+02:00
diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-auth.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-auth.yaml
@@ -16,6 +16,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-aggregator/cardano/config/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-aggregator/cardano/db:/db
@@ -77,6 +79,8 @@ services:
         condition: service_started
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     labels:
       - 'traefik.enable=true'
       - 'traefik.http.routers.mithril-aggregator.rule=Host(`${AGGREGATOR_HOST}`)'
diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-noauth.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-noauth.yaml
@@ -16,6 +16,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-aggregator/cardano/config/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-aggregator/cardano/db:/db
@@ -77,6 +79,8 @@ services:
         condition: service_started
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     labels:
       - 'traefik.enable=true'
       - 'traefik.http.routers.mithril-aggregator.rule=Host(`${AGGREGATOR_HOST}`)'
diff --git a/mithril-infra/assets/docker/docker-compose-monitoring.yaml b/mithril-infra/assets/docker/docker-compose-monitoring.yaml
@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   prometheus:
-    image: prom/prometheus:latest
+    image: prom/prometheus:v2.45.0
     container_name: prometheus
     profiles:
       - tools
@@ -15,6 +15,10 @@ services:
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./prometheus/cardano.json:/etc/prometheus/cardano.json:ro
+    logging:
+      driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     labels:
       - 'traefik.enable=true'
       - 'traefik.http.routers.prometheus.rule=Host(`${PROMETHEUS_HOST}`)'
@@ -40,18 +44,45 @@ services:
       - '--path.rootfs=/rootfs'
       - '--path.sysfs=/host/sys'
       - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+    logging:
+      driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+
+  loki:
+    image: grafana/loki:2.8.0
+    container_name: loki
+    profiles:
+      - tools
+      - monitoring
+      - all
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./loki/local-config.yaml:/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/local-config.yaml
+    logging:
+      driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
   promtail:
-    image: grafana/promtail:latest
+    image: grafana/promtail:2.8.0
     container_name: promtail
     profiles:
       - tools
       - monitoring
       - all
+    ports:
+      - "9080:9080"
     volumes:
       - /var/lib/docker/containers:/var/lib/docker/containers
       - ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml
-    command: -config.file=/etc/promtail/promtail-config.yml
+    command: -config.file=/etc/promtail/promtail-config.yml -config.expand-env=true
+    logging:
+      driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
 networks:
   default:
diff --git a/mithril-infra/assets/docker/docker-compose-reverse-proxy.yaml b/mithril-infra/assets/docker/docker-compose-reverse-proxy.yaml
@@ -17,6 +17,8 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:ro
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
 networks:
   default:
diff --git a/mithril-infra/assets/docker/docker-compose-signer-unverified-alone.yaml b/mithril-infra/assets/docker/docker-compose-signer-unverified-alone.yaml
@@ -30,6 +30,8 @@ services:
       - ../data/${NETWORK}/mithril-aggregator/cardano/ipc:/ipc
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
 networks:
   default:
diff --git a/mithril-infra/assets/docker/docker-compose-signer-unverified-norelay.yaml b/mithril-infra/assets/docker/docker-compose-signer-unverified-norelay.yaml
@@ -18,6 +18,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/full/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/passive:/db
@@ -66,6 +68,8 @@ services:
         condition: service_started
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
 networks:
   default:
diff --git a/mithril-infra/assets/docker/docker-compose-signer-unverified.yaml b/mithril-infra/assets/docker/docker-compose-signer-unverified.yaml
@@ -18,6 +18,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/full/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/passive:/db
@@ -76,6 +78,8 @@ services:
         condition: service_started
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
 networks:
   default:
diff --git a/mithril-infra/assets/docker/docker-compose-signer-verified-norelay.yaml b/mithril-infra/assets/docker/docker-compose-signer-verified-norelay.yaml
@@ -18,6 +18,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/relay/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/relay:/db
@@ -57,6 +59,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/block-producer/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/block-producer:/db
@@ -122,6 +126,8 @@ services:
         condition: service_started
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
   mithril-signer-www:
     image: nginx:latest
@@ -136,6 +142,8 @@ services:
       - "${SIGNER_WWW_PORT}:8080"
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     labels:
       - 'traefik.enable=true'
       - 'traefik.http.routers.mithril-signer-${SIGNER_ID}.rule=Host(`${SIGNER_HOST}`)'
diff --git a/mithril-infra/assets/docker/docker-compose-signer-verified.yaml b/mithril-infra/assets/docker/docker-compose-signer-verified.yaml
@@ -18,6 +18,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/relay/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/relay:/db
@@ -57,6 +59,8 @@ services:
       - all
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     volumes:
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/config/block-producer/${NETWORK}:/config
       - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/db/block-producer:/db
@@ -132,6 +136,8 @@ services:
         condition: service_started
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
   mithril-signer-www:
     image: nginx:latest
@@ -146,6 +152,8 @@ services:
       - "${SIGNER_WWW_PORT}:8080"
     logging:
       driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
     labels:
       - 'traefik.enable=true'
       - 'traefik.http.routers.mithril-signer-${SIGNER_ID}.rule=Host(`${SIGNER_HOST}`)'
diff --git a/mithril-infra/assets/docker/loki/local-config.yaml b/mithril-infra/assets/docker/loki/local-config.yaml
@@ -0,0 +1,50 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false
diff --git a/mithril-infra/assets/docker/promtail/promtail-config.yml b/mithril-infra/assets/docker/promtail/promtail-config.yml
@@ -9,34 +9,44 @@ clients:
   - url: http://loki:3100/loki/api/v1/push
 
 scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log
 
 - job_name: containers
-  entry_parser: raw
-
   static_configs:
   - targets:
       - localhost
     labels:
       job: containerlogs
       __path__: /var/lib/docker/containers/*/*log
 
-  # --log-opt tag="{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
   pipeline_stages:
-
   - json:
       expressions:
+        output: log
         stream: stream
-        attrs: attrs
-        tag: attrs.tag
-
+        attrs:
+  - json:
+      expressions:
+        tag:
+      source: attrs
   - regex:
       expression: (?P<image_name>(?:[^|]*[^|])).(?P<container_name>(?:[^|]*[^|])).(?P<image_id>(?:[^|]*[^|])).(?P<container_id>(?:[^|]*[^|]))
-      source: "tag"
-
+      source: tag
+  - timestamp:
+      format: RFC3339Nano
+      source: time
   - labels:
       tag:
       stream:
       image_name:
       container_name:
       image_id:
       container_id:
+  - output:
+      source: output
diff --git a/mithril-infra/mithril.monitoring.tf b/mithril-infra/mithril.monitoring.tf
@@ -40,6 +40,7 @@ EOT
 
   provisioner "remote-exec" {
     inline = [
+      "export LOGGING_DRIVER='${var.mithril_container_logging_driver}'",
       "export PROMETHEUS_HOST=${local.prometheus_host}",
       "export AUTH_USER_PASSWORD=$(htpasswd -nb ${var.prometheus_auth_username} ${var.prometheus_auth_password})",
       "docker-compose -f /home/curry/docker/docker-compose-monitoring.yaml --profile all up -d",
