Add alertmanager rules for Docker

Docker has a builtin prometheus exporter that we currently don't have
enabled. This change adds alerts for stopped/paused containers and
failed healthchecks. This patch requires changes to docker's
configuration to export the metrics, and prometheus to consume them.
This means that Kayobe and Kolla-Ansible should both be updated to their
latest versions.

Author: Alex-Welsh

etc/kayobe/inventory/group_vars/all/docker

@@ -0,0 +1,3 @@
+---
+# Address for prometheus metrics endpoint
+docker_metrics_addr: "{{ internal_net_name | net_ip + ':9323'}}"

etc/kayobe/kolla/config/prometheus/docker.rules

@@ -0,0 +1,29 @@
+
+groups:
+- name: Docker
+  rules:
+
+  - alert: DockerContainerStopped
+    expr: 'engine_daemon_container_states_containers{state="stopped"} > 0'
+    labels:
+      severity: warning
+    annotations:
+      summary: "Containers not running (instance {{ $labels.instance }})"
+      description: "One or more container are stopped"
+
+  - alert: DockerContainerPaused
+    expr: 'engine_daemon_container_states_containers{state="paused"} > 0'
+    labels:
+      severity: warning
+    annotations:
+      summary: "Containers not running (instance {{ $labels.instance }})"
+      description: "One or more container are stopped"
+
+  - alert: DockerContainerHealthCheckFail
+    expr: rate(engine_daemon_health_checks_failed_total[1m]) > 1
+    labels:
+      severity: warning
+    annotations:
+      summary: "Containers health check failed (instance {{ $labels.instance }})"
+      description: "One or more container health checks failed"
+

releasenotes/notes/docker-alerts-30e8d870f25e500b.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Added new default alerting rules for containers being unhealthy or stopped.