Ingest metrics and logs to remote cloud in infra

Author: jpraynaud

mithril-infra/assets/docker/prometheus/prometheus.yml renamed to mithril-infra/assets/docker/prometheus/prometheus-base.yml

@@ -27,5 +27,3 @@ scrape_configs:
     file_sd_configs:
     - files:
       - 'cardano.json'
-
-

mithril-infra/assets/docker/promtail/promtail-config.yml renamed to mithril-infra/assets/docker/promtail/promtail-config-base.yml

@@ -5,9 +5,6 @@ server:
 positions:
   filename: /tmp/positions.yaml
 
-clients:
-  - url: http://loki:3100/loki/api/v1/push
-
 scrape_configs:
 - job_name: system
   static_configs:
@@ -49,4 +46,7 @@ scrape_configs:
       image_id:
       container_id:
   - output:
-      source: output
+      source: output
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push

mithril-infra/mithril.monitoring.tf

@@ -28,6 +28,20 @@ resource "null_resource" "mithril_monitoring" {
       "mkdir -p /home/curry/data/monitoring/prometheus",
       "mkdir -p /home/curry/data/monitoring/loki",
       <<-EOT
+set -e
+# Copy prometheus base configuration
+cp /home/curry/docker/prometheus/prometheus-base.yml /home/curry/docker/prometheus/prometheus.yml
+# Setup prometheus remote write
+if [ -n "${var.prometheus_ingest_host}" ] ; then 
+cat >> /home/curry/docker/prometheus/prometheus.yml << EOF
+
+remote_write:
+- url: https://${var.prometheus_ingest_host}/api/prom/push
+  basic_auth:
+    username: ${var.prometheus_ingest_username}
+    password: ${var.prometheus_ingest_password}
+EOF
+fi
 # Setup prometheus targets configuration for Cardano nodes
 CARDANO_NODES=$(docker ps --format='{{.Names}}:12798,' | grep "cardano-node" | sort | tr -d '\n\t\r ' | sed 's/.$//')
 cat /home/curry/docker/prometheus/cardano.json | jq --arg CARDANO_NODES "$CARDANO_NODES" '. += [{
@@ -38,6 +52,19 @@ cat /home/curry/docker/prometheus/cardano.json | jq --arg CARDANO_NODES "$CARDAN
 }]' | jq '. | map(try(.targets |= split(",")) // .)' > /home/curry/docker/prometheus/cardano.json.new
 rm -f /home/curry/docker/prometheus/cardano.json
 mv /home/curry/docker/prometheus/cardano.json.new docker/prometheus/cardano.json
+EOT
+      ,
+      <<-EOT
+set -e
+# Copy promtail base configuration
+cp /home/curry/docker/promtail/promtail-config-base.yml /home/curry/docker/promtail/promtail-config.yml
+# Setup promtail remote client
+if [ -n "${var.loki_ingest_host}" ] ; then 
+cat >> /home/curry/docker/promtail/promtail-config.yml << EOF
+
+  - url: https://${var.loki_ingest_username}:${var.loki_ingest_password}@${var.loki_ingest_host}/loki/api/v1/push
+EOF
+fi
 EOT
     ]
   }

mithril-infra/variables.tf

@@ -174,25 +174,61 @@ variable "mithril_aggregator_auth_password" {
 
 variable "prometheus_auth_username" {
   type        = string
-  description = "The username for authentication on prometheus"
+  description = "The username for authentication on local prometheus endpoint"
   default     = ""
 }
 
 variable "prometheus_auth_password" {
   type        = string
-  description = "The password for authentication on prometheus"
+  description = "The password for authentication on local prometheus endpoint"
+  default     = ""
+}
+
+variable "prometheus_ingest_host" {
+  type        = string
+  description = "The host to ingest on remote prometheus endpoint"
+  default     = ""
+}
+
+variable "prometheus_ingest_username" {
+  type        = string
+  description = "The username to ingest on remote prometheus endpoint"
+  default     = ""
+}
+
+variable "prometheus_ingest_password" {
+  type        = string
+  description = "The password to ingest on remote prometheus endpoint"
   default     = ""
 }
 
 variable "loki_auth_username" {
   type        = string
-  description = "The username for authentication on loki"
+  description = "The username for authentication on local loki endpoint"
   default     = ""
 }
 
 variable "loki_auth_password" {
   type        = string
-  description = "The password for authentication on loki"
+  description = "The password for authentication on local loki endpoint"
+  default     = ""
+}
+
+variable "loki_ingest_host" {
+  type        = string
+  description = "The host to ingest on remote loki endpoint"
+  default     = ""
+}
+
+variable "loki_ingest_username" {
+  type        = string
+  description = "The username to ingest on remote loki endpoint"
+  default     = ""
+}
+
+variable "loki_ingest_password" {
+  type        = string
+  description = "The password to ingest on remote loki endpoint"
   default     = ""
 }