Switch prometheus to active/passive mode

This uses the same approach as the mariadb role (and others).

Closes-Bug: : 1928193
Co-Authored-By: John Garbutt <[email protected]>
Change-Id: I79a7a8c80327cfd9ef31d17fe71f450a181a638c
(cherry picked from commit 502152290ac5fc2a09176c52a459b9741273b752)
(cherry picked from commit 9ec49a3)

Author: jovial

ansible/roles/prometheus/defaults/main.yml

@@ -13,6 +13,7 @@ prometheus_services:
         mode: "http"
         external: false
         port: "{{ prometheus_port }}"
+        custom_member_list: "{{ prometheus_haproxy_members.split(';') }}"
   prometheus-node-exporter:
     container_name: prometheus_node_exporter
     group: prometheus-node-exporter
@@ -105,6 +106,11 @@ prometheus_services:
 ####################
 prometheus_mysql_exporter_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}prometheus{% endif %}"
 
+####################
+# HAProxy
+####################
+prometheus_haproxy_members: "{% for host in groups['prometheus'] %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ prometheus_port }} check inter 2000 rise 2 fall 5{% if not loop.first %} backup{% endif %};{% endfor %}"
+
 ####################
 # Blackbox
 ####################

releasenotes/notes/prometheus-active-passive-be4fb033493e958f.yaml

@@ -0,0 +1,10 @@
+---
+upgrade:
+  - |
+    Prometheus has been switched to active/passive mode. See bug `1928193
+    <https://bugs.launchpad.net/kolla-ansible/+bug/1928193>`__.
+fixes:
+  - |
+    Fixes an issue with misaligned data points in grafana when loadbalancing
+    over multiple prometheus server instances. See bug `1928193
+    <https://bugs.launchpad.net/kolla-ansible/+bug/1928193>`__.