Support monitoring Fluentd with Prometheus

This patch adds support for integrating Prometheus with Fluentd.
This can be used to extract useful information about the status
of Fluentd, such as output buffer capacity and logging rate,
and also to extract metrics from logs via custom Fluentd
configuration. More information can be found here in [1].

[1] https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus

Change-Id: I233d6dd744848ef1f1589a462dbf272ed0f3aaae
(cherry picked from commit b692ce7)

Author: dougszumski

ansible/group_vars/all.yml

@@ -457,6 +457,7 @@ prometheus_memcached_exporter_port: "9150"
 prometheus_rabbitmq_exporter_port: "15692"
 # Default cadvisor port of 8080 already in use
 prometheus_cadvisor_port: "18080"
+prometheus_fluentd_integration_port: "24231"
 
 # Prometheus alertmanager ports
 prometheus_alertmanager_port: "9093"
@@ -1106,6 +1107,7 @@ enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
 enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}"
 enable_prometheus_memcached_exporter: "{{ enable_memcached | bool }}"
 enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}"
+enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable_fluentd | bool }}"
 enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}"
 enable_prometheus_ceph_mgr_exporter: "no"
 enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}"

ansible/roles/common/tasks/config.yml

@@ -114,17 +114,29 @@
          ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }}
     fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
     # Inputs
-    fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
+    fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}"
+    default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}"
     default_input_files:
-      - "conf/input/00-global.conf.j2"
-      - "conf/input/01-syslog.conf.j2"
-      - "conf/input/02-mariadb.conf.j2"
-      - "conf/input/03-rabbitmq.conf.j2"
-      - "conf/input/04-openstack-wsgi.conf.j2"
-      - "conf/input/05-libvirt.conf.j2"
-      - "conf/input/06-zookeeper.conf.j2"
-      - "conf/input/07-kafka.conf.j2"
-      - "conf/input/09-monasca.conf.j2"
+      - name: "conf/input/00-global.conf.j2"
+        enabled: true
+      - name: "conf/input/01-syslog.conf.j2"
+        enabled: true
+      - name: "conf/input/02-mariadb.conf.j2"
+        enabled: true
+      - name: "conf/input/03-rabbitmq.conf.j2"
+        enabled: true
+      - name: "conf/input/04-openstack-wsgi.conf.j2"
+        enabled: true
+      - name: "conf/input/05-libvirt.conf.j2"
+        enabled: true
+      - name: "conf/input/06-zookeeper.conf.j2"
+        enabled: true
+      - name: "conf/input/07-kafka.conf.j2"
+        enabled: true
+      - name: "conf/input/08-prometheus.conf.j2"
+        enabled: "{{ enable_prometheus_fluentd_integration | bool }}"
+      - name: "conf/input/09-monasca.conf.j2"
+        enabled: true
     customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}"
     # Filters
     fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"

ansible/roles/common/templates/conf/input/08-prometheus.conf.j2

@@ -0,0 +1,14 @@
+<source>
+  @type prometheus
+  bind {{ api_interface_address }}
+  port {{ prometheus_fluentd_integration_port }}
+  metrics_path /metrics
+</source>
+
+<source>
+  @type prometheus_output_monitor
+  interval 10
+  <labels>
+    Hostname ${hostname}
+  </labels>
+</source>

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -74,6 +74,15 @@ scrape_configs:
 {% endfor %}
 {% endif %}
 
+{% if enable_prometheus_fluentd_integration | bool %}
+  - job_name: fluentd
+    static_configs:
+      - targets:
+{% for host in groups['fluentd'] %}
+        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
+{% endfor %}
+{% endif %}
+
 {% if enable_prometheus_ceph_mgr_exporter | bool %}
   - job_name: ceph_mgr_exporter
     honor_labels: true

etc/kolla/globals.yml

@@ -711,6 +711,7 @@
 #enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
 #enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}"
 #enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}"
+#enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable fluentd | bool }}"
 #enable_prometheus_memcached: "{{ enable_prometheus | bool }}"
 #enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}"
 #enable_prometheus_ceph_mgr_exporter: "no"

releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds support for integrating Fluentd metrics into Prometheus. By
+    default this is now enabled when Prometheus is enabled. This behaviour
+    can be overridden via the `enable_prometheus_fluentd_integration` flag. By
+    default the integration provides metrics relating to the processing of logs
+    by Fluentd. These metrics can be useful for monitoring the status of the
+    Fluentd service. Additional metrics can also be extracted from logs via
+    custom Fluentd config.