Merge pull request #99 from trento-project/prometheus-ssl-termination

Prometheus ssl termination

Author: arbulu89

.github/workflows/ci.yaml

@@ -111,6 +111,10 @@ jobs:
                   hosts:
                     server:
                       ansible_host: ${{ env.TEST_HOST_IP }}
+                prometheus_hosts:
+                  hosts:
+                    server:
+                      ansible_host: ${{ env.TEST_HOST_IP }}
                 agents:
                   hosts:
                     agent:
@@ -160,6 +164,10 @@ jobs:
                   hosts:
                     server:
                       ansible_host: ${{ env.TEST_HOST_IP }}
+                prometheus_hosts:
+                  hosts:
+                    server:
+                      ansible_host: ${{ env.TEST_HOST_IP }}
           options: |
             --extra-vars "web_postgres_password='trento' \
             wanda_postgres_password='wanda' \

README.adoc

@@ -56,11 +56,12 @@ The nginx configuration acts as a reverse proxy for all the components.
 `server.yml` playbook installs Trento server components (Web and
 Wanda) along with the supporting third-party application dependencies.
 
-This playbook supports auto-discovery of the hosts for RabbitMQ and
-Postgres in Web and Wanda. This feature is enabled when
-`provision_postgres` and `provision_rabbitmq` flags are true. When
-auto-discovery is active, `*_host` variables in `app` role are
-ignored. If you want to use external services for these dependencies,
+This playbook supports auto-discovery of the hosts for RabbitMQ, Postgres and Prometheus
+in Web and Wanda. This feature is enabled when `provision_postgres`,
+`provision_rabbitmq` and `provision_prometheus` flags are true. When
+auto-discovery is active, `*_host` variables (in `trento` role
+and any other role using them) are ignored.
+If you want to use external services for these dependencies,
 turn off the respective flag and specify the `*_host` variable in your
 inventory.
 
@@ -401,6 +402,24 @@ per-role basis if appropriate.
 
 | trento_rabbitmq_vhost | The rabbitmq vhost used for the current
   deployment | "trento"
+
+| trento_prometheus_host | The host where Prometheus is
+  located. Used as input for calculating
+  `trento_prometheus_effective_host` variable that is the default for
+  `*_prometheus_host` variables in some dependent roles.  This variable
+  is ignored when `trento_prometheus_host_group` is non-empty string. |
+  localhost
+
+| trento_prometheus_port | The port where Prometheus is exposed. | 9090
+
+| trento_prometheus_host_group | Name of the host group where Prometheus
+  is located. If this value is not empty string, activates
+  auto-discovery of the Prometheus host by searching it in the specified
+  group. Used as input for calculating
+  `trento_prometheus_effective_host` variable that is the default for
+  `*_prometheus_host` variables in some dependent roles. When
+  auto-discovery is active, the value of `trento_prometheus_host` is
+  ignored. | ""
 |===
 
 *Web role*
@@ -707,7 +726,7 @@ More detailed info about usage and defaults available link:https://github.com/tr
   The target address of the node exporter to scrape metrics from, in case `agent_prometheus_mode` is set to "pull". | 
   Defaults to the lowest discovered IP address with a default port number (9100).
 
-| agent_prometheus_remote_write_url | The remote write URL of the Prometheus server, in case `agent_prometheus_mode` is set to "push". | currently undefined. Needs to be improved for autodiscovery
+| agent_prometheus_remote_write_url | The remote write URL of the Prometheus server, in case `agent_prometheus_mode` is set to "push". | "https://<value of `trento_server_name`>/prometheus/api/v1/write"
 
 | agent_prometheus_scrape_interval | Scrape interval for Prometheus to scrape the node exporter, in case `agent_prometheus_mode` is set to "push". | "15s"
 
@@ -836,6 +855,14 @@ More detailed info about usage and defaults available link:https://github.com/tr
 
 | rproxy_ssl_key_as_base64 | Whether SSL key is provided as base64
   string | false
+
+| rproxy_provision_prometheus | Whether to provision Prometheus
+  proxy configuration in the reverse proxy | false
+
+| rproxy_prometheus_host | The host where Prometheus is located. | <value of `trento_prometheus_effective_host`>
+
+| rproxy_prometheus_port | The port where Prometheus is exposed. | <value of `trento_prometheus_port`>
+
 |===
 
 == Clean up

roles/agent/vars/main.yml

@@ -7,7 +7,7 @@ agent_prometheus_mode_push: "{{ agent_resolved_prometheus_mode == 'push' }}"
 agent_resolved_prometheus_node_exporter_target: "{{ agent_prometheus_mode_pull | ternary(agent_prometheus_node_exporter_target, '') }}"
 agent_resolved_prometheus_scrape_interval: "{{ agent_prometheus_mode_push | ternary(agent_prometheus_scrape_interval, '') }}"
 
-__agent_discovered_prometheus_remote_write_url: "{{ agent_prometheus_remote_write_url | default('http://{}:{}{}'.format(trento_server_name, 9090, '/api/v1/write'), true) }}"
+__agent_discovered_prometheus_remote_write_url: "{{ agent_prometheus_remote_write_url | default('https://{}{}'.format(trento_server_name, '/prometheus/api/v1/write'), true) }}"
 agent_resolved_prometheus_remote_write_url: "{{ agent_prometheus_mode_push | ternary(__agent_discovered_prometheus_remote_write_url, '') }}"
 
 agent_resolved_prometheus_auth: "{{ agent_prometheus_mode_push | ternary(agent_prometheus_auth, '') }}"

roles/prometheus/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-prometheus_port: 9090
+prometheus_port: "{{ trento_prometheus_port }}"
 prometheus_web_url: "{{ web_host | default('http://{}'.format(trento_server_name)) }}" # `web_host` is for backwards-compatibility
 prometheus_web_listen_port: "{{ web_listen_port | default(trento_web_listen_port) }}" # `web_listen_port` for backwards-compatibility
 prometheus_enable_remote_write_receiver: true

roles/rproxy/defaults/main/main.yml

@@ -14,3 +14,6 @@ rproxy_ssl_cert: "{{ nginx_ssl_cert }}"
 rproxy_ssl_cert_as_base64: "{{ nginx_ssl_cert_as_base64 }}"
 rproxy_ssl_key: "{{ nginx_ssl_key }}"
 rproxy_ssl_key_as_base64: "{{ nginx_ssl_key_as_base64 }}"
+rproxy_provision_prometheus: false
+rproxy_prometheus_host: "{{ trento_prometheus_effective_host }}"
+rproxy_prometheus_port: "{{ trento_prometheus_port }}"

roles/rproxy/templates/trento.conf.j2

@@ -11,6 +11,12 @@ upstream {{ wanda_upstream }} {
   server 127.0.0.1:{{ wanda_port }} max_fails=5 fail_timeout=60s;
 }
 
+{% if rproxy_provision_prometheus -%}
+upstream prometheus {
+  server {{ rproxy_prometheus_host }}:{{ rproxy_prometheus_port }} max_fails=5 fail_timeout=60s;
+}
+{% endif %}
+
 server {
     # Redirect HTTP to HTTPS
     listen {{ http_listen_port }};
@@ -30,6 +36,23 @@ server {
   ssl_prefer_server_ciphers on;
   ssl_session_cache shared:SSL:10m;
 
+{% if rproxy_provision_prometheus %}
+  # Prometheus rule
+  location /prometheus/  {
+    allow all;
+
+    # Proxy Headers
+    proxy_http_version 1.1;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Cluster-Client-Ip $remote_addr;
+
+    # Add final slash to replace the prometheus_location value by the value in proxy_pass
+    # https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass
+    proxy_pass http://prometheus/;
+  }
+{% endif %}
+
   # Wanda rule
   location {{ wanda_location }}/  {
     allow all;

roles/trento/defaults/main.yml

@@ -35,3 +35,6 @@ trento_rabbitmq_host_group: ""
 trento_rabbitmq_username: trento
 trento_rabbitmq_password: "{{ undef(hint='Password for RabbitMQ trento user is mandatory') }}"
 trento_rabbitmq_vhost: trento
+trento_prometheus_host_group: ""
+trento_prometheus_host: localhost
+trento_prometheus_port: 9090

roles/trento/tasks/main.yml

@@ -18,3 +18,13 @@
       Inventory ansible host is {{ __trento_rabbitmq_ansible_host | default('undefined') }}.
       User value is {{ trento_rabbitmq_host }}
     verbosity: 1
+
+- name: Show host discovery results for Prometheus
+  ansible.builtin.debug:
+    msg: |
+      Effective Prometheus host is {{ trento_prometheus_effective_host }}.
+      Discovery host is {{ __trento_prometheus_discovery_host | default('undefined') }}.
+      Inventory host is {{ __trento_prometheus_inventory_host | default('undefined') }}.
+      Inventory ansible host is {{ __trento_prometheus_ansible_host | default('undefined') }}.
+      User value is {{ trento_prometheus_host }}
+    verbosity: 1

roles/trento/vars/main.yml

@@ -36,3 +36,13 @@ __trento_rabbitmq_discovery_host: "{{ (inventory_hostname in groups[trento_rabbi
 trento_rabbitmq_effective_host: "{{ trento_rabbitmq_host_group is truthy
                                     | ternary(__trento_rabbitmq_discovery_host,
                                               trento_rabbitmq_host) }}"
+
+# Prometheus auto-discovery
+__trento_prometheus_inventory_host: "{{ groups[trento_prometheus_host_group][0] }}"
+__trento_prometheus_ansible_host: "{{ hostvars[__trento_prometheus_inventory_host].ansible_host }}"
+__trento_prometheus_discovery_host: "{{ (inventory_hostname in groups[trento_prometheus_host_group])
+                                     | ternary('127.0.0.1',
+                                               __trento_prometheus_ansible_host | default(__trento_prometheus_inventory_host)) }}"
+trento_prometheus_effective_host: "{{ trento_prometheus_host_group is truthy
+                                    | ternary(__trento_prometheus_discovery_host,
+                                              trento_prometheus_host) }}"

server.yml

@@ -44,9 +44,11 @@
     # Explicitly call the trento role with the passed parameters to
     # please the linter.
     - role: trento
+      run_once: true
       vars:
         trento_postgres_host_group: "{{ 'postgres_hosts' if (provision_postgres | bool) else '' }}"
         trento_rabbitmq_host_group: "{{ 'rabbitmq_hosts' if (provision_rabbitmq | bool) else '' }}"
+        trento_prometheus_host_group: "{{ 'prometheus_hosts' if (provision_prometheus | bool) else '' }}"
 
     - role: web
       become: true
@@ -57,6 +59,8 @@
     - role: rproxy
       become: true
       when: provision_proxy | bool
+      vars:
+        rproxy_provision_prometheus: "{{ provision_prometheus | bool }}"
 
     - role: firewall
       become: true