Scrape cAdvisor for container-level metrics

This enables the ability to collect metrics similar to what `node_exporter` collects, except these are specific to a particular container (which is what cAdvisor) provides.

As such, monitoring tools can now see what resources a specific pod is using.

In Kubernetes, each Kubelet can run cAdvisor on its node. This commit allows the Postgres Operator to enable this container. This patch also lets each Kubelet use self-signed certificates during scrape as, by default, a Kubelet generates and self-signs the TLS certificate which serves its metrics API. More complete/advanced Kubernetes configurations manage this certificate and sign it with the Kubernetes CA.

We are trusting the Kubernetes CA, which is available to containers at `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt`, but to support simpler configurations we must skip TLS verification of the kubelet API. For more information:

- https://github.com/kubernetes/kubernetes/blob/v1.15.4/staging/src/k8s.io/kubelet/config/v1beta1/types.go#L130-L145
- https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/#certificate-rotation

Author: cbandy

ansible/roles/pgo-metrics/tasks/prometheus.yml

@@ -46,6 +46,7 @@
         dest: "{{ prom_output_dir }}/{{ item | replace('.j2', '') }}"
         mode: '0600'
       with_items: 
+      - prometheus-config.yaml.j2
       - prometheus-pvc.json.j2
       - prometheus-service.json.j2
       - prometheus-deployment.json.j2
@@ -54,6 +55,7 @@
     - name: Create Prometheus Objects
       command: "{{ kubectl_or_oc }} create -f {{ prom_output_dir }}/{{ item }} -n {{ metrics_namespace }}"
       with_items:
+      - prometheus-config.yaml
       - prometheus-pvc.json
       - prometheus-service.json
       - prometheus-deployment.json

ansible/roles/pgo-metrics/templates/prometheus-config.yaml.j2

@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: crunchy-prometheus
+data:
+  prometheus.yml: |-
+    ---
+    global:
+      scrape_interval: 10s
+      scrape_timeout: 10s
+      evaluation_interval: 5s
+
+    scrape_configs:
+    - job_name: cadvisor
+      kubernetes_sd_configs:
+      - role: node
+
+      metrics_path: /metrics/cadvisor
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+      metric_relabel_configs:
+      # Keep only metrics attributed to pods.
+      - action: keep
+        source_labels: [pod,pod_name]
+        separator: ''
+        regex: '.+'
+
+    - job_name: crunchy-collect
+      kubernetes_sd_configs:
+      - role: pod
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_label_crunchy_collect]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_container_port_number]
+        action: drop
+        regex: 5432
+      - source_labels: [__meta_kubernetes_pod_container_port_number]
+        action: drop
+        regex: 10000
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_pod_name]
+        regex: (^[^-]*).*
+        target_label: instance
+        replacement: '$1'
+      - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_name]
+        target_label: job
+        separator: ': '
+        replacement: '$1$2'

ansible/roles/pgo-metrics/templates/prometheus-deployment.json.j2

@@ -48,6 +48,11 @@
                         },
                         "env": [],
                         "volumeMounts": [
+                            {
+                                "mountPath": "/conf",
+                                "name": "prometheusconf",
+                                "readOnly": true
+                            },
                             {
                                 "mountPath": "/data",
                                 "name": "prometheusdata",
@@ -57,6 +62,12 @@
                     }
                 ],
                 "volumes": [
+                    {
+                        "name": "prometheusconf",
+                        "configMap": {
+                            "name": "crunchy-prometheus"
+                        }
+                    },
                     {
                         "name": "prometheusdata",
                         "persistentVolumeClaim": {

ansible/roles/pgo-metrics/templates/prometheus-rbac.json.j2

@@ -10,6 +10,8 @@
                 ""
             ],
             "resources": [
+                "nodes",
+                "nodes/metrics",
                 "pods"
             ],
             "verbs": [