SMPROD-6724 Updated improved Consul integration (#201)

* SMPROD-6724 Updated improved Consul integration

* SMPROD-6724 Updated Consul prerequisites

* SMPROD-6724 Fix Consul prerequisites

Author: carlosadiegosysdig

resources/consul/INSTALL.md

@@ -1,2 +1,13 @@
 # Prerequisites
 Consul instruments Prometheus metrics and annotates the pods with Prometheus annotations. 
+
+As seen in Consul documentation pages (https://www.consul.io/docs/k8s/helm#v-global-metrics and https://www.consul.io/docs/agent/options#telemetry-prometheus_retention_time), to make Consul expose an endpoint for scraping metrics, you need to enable a few global.metrics configurations.
+You also need to enable the telemetry.disable_hostname "extra configurations" in the Consul Server and Client, so the metrics don't contain the name of the instances.
+
+If you install Consul with Helm, you need to use the following flags:
+```
+--set 'global.metrics.enabled=true'
+--set 'global.metrics.enableAgentMetrics=true'
+--set 'server.extraConfig="{"telemetry": {"disable_hostname": true}}"'
+--set 'client.extraConfig="{"telemetry": {"disable_hostname": true}}"'
+```

resources/consul/alerts.yaml

@@ -74,15 +74,15 @@ configurations:
           description: There are too many elections for leadership."
       - alert: Server cluster unhealthy
         expr: |
-          consul_per_server_autopilot_healthy == 0
+          consul_autopilot_healthy == 0
         for: 5m
         labels:
           severity: high
         annotations:
           description: One or many Consul servers in the cluster are unhealthy.
       - alert: Zero failure tolerance
         expr: |
-          consul_per_server_autopilot_failure_tolerance == 0
+          consul_autopilot_failure_tolerance == 0
         for: 5m
         labels:
           severity: medium
@@ -138,7 +138,7 @@ configurations:
           description: Garbage Collection stop-the-world pauses were greater than 5 seconds per minute.
       - alert: Raft restore duration too high
         expr: |
-          consul_per_server_raft_leader_oldestLogAge < 2* max(consul_raft_fsm_lastRestoreDuration{kube_pod_label_component="server"})
+          consul_raft_leader_oldestLogAge < 2* max(consul_raft_fsm_lastRestoreDuration{kube_pod_label_component="server"})
         for: 5m
         labels:
           severity: medium

resources/consul/include/consul_sysdig.json

@@ -456,7 +456,7 @@
               "unit": "number",
               "yAxis": "auto"
             },
-            "query": "min(consul_per_server_autopilot_healthy{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace})"
+            "query": "min(consul_autopilot_healthy{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace})"
           }
         ],
         "description": "",
@@ -655,7 +655,7 @@
               "unit": "number",
               "yAxis": "auto"
             },
-            "query": "consul_per_server_autopilot_failure_tolerance{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace}"
+            "query": "consul_autopilot_failure_tolerance{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace}"
           }
         ],
         "axesConfiguration": {
@@ -718,7 +718,7 @@
               "unit": "number",
               "yAxis": "auto"
             },
-            "query": "consul_per_server_autopilot_healthy{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace}"
+            "query": "consul_autopilot_healthy{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace}"
           }
         ],
         "axesConfiguration": {
@@ -1433,7 +1433,7 @@
               "unit": "relativeTime",
               "yAxis": "auto"
             },
-            "query": "consul_per_server_raft_leader_oldestLogAge{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace} > 0\n"
+            "query": "consul_raft_leader_oldestLogAge{kube_cluster_name=~$cluster, kube_namespace_name=~$namespace} > 0\n"
           },
           {
             "displayInfo": {

resources/consul/include/sysdig-agent.yaml

@@ -29,6 +29,9 @@ data:
       scrape_interval: 10s
     scrape_configs:
     - job_name: 'consul-envoy-default'
+      metrics_path: '/v1/agent/metrics'
+      params:
+        format: ['prometheus']
       tls_config:
         insecure_skip_verify: true
       kubernetes_sd_configs:
@@ -102,20 +105,9 @@ data:
       - action: keep
         source_labels: [__address__]
         regex: (.*:8500)
-      - action: replace
-        source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-        target_label: __metrics_path__
-        regex: (.+)
-        replacement: '/v1/agent/metrics'
       - action: replace
         source_labels: [__meta_kubernetes_pod_uid]
         target_label: sysdig_k8s_pod_uid
       - action: replace
         source_labels: [__meta_kubernetes_pod_container_name]
-        target_label: sysdig_k8s_pod_container_name
-      metric_relabel_configs:
-      # Change the name of the metric to remove the name of the pod
-      - source_labels: ['__name__']
-        target_label: '__name__'
-        regex: '(consul_)([a-z]+_)+[0-9]+_(.+)'
-        replacement: ${1}per_server_${3}
+        target_label: sysdig_k8s_pod_container_name