Merge pull request #184 from stackhpc/wallaby-backport/fluentd-monitoring

Wallaby: Support monitoring Fluentd with Prometheus

Author: cityofships

ansible/group_vars/all.yml

@@ -457,6 +457,7 @@ prometheus_memcached_exporter_port: "9150"
 prometheus_rabbitmq_exporter_port: "15692"
 # Default cadvisor port of 8080 already in use
 prometheus_cadvisor_port: "18080"
+prometheus_fluentd_integration_port: "24231"
 
 # Prometheus alertmanager ports
 prometheus_alertmanager_port: "9093"
@@ -1106,6 +1107,7 @@ enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
 enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}"
 enable_prometheus_memcached_exporter: "{{ enable_memcached | bool }}"
 enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}"
+enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable_fluentd | bool }}"
 enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}"
 enable_prometheus_ceph_mgr_exporter: "no"
 enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}"

ansible/roles/common/tasks/config.yml

@@ -114,17 +114,29 @@
          ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }}
     fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
     # Inputs
-    fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
+    fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}"
+    default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}"
     default_input_files:
-      - "conf/input/00-global.conf.j2"
-      - "conf/input/01-syslog.conf.j2"
-      - "conf/input/02-mariadb.conf.j2"
-      - "conf/input/03-rabbitmq.conf.j2"
-      - "conf/input/04-openstack-wsgi.conf.j2"
-      - "conf/input/05-libvirt.conf.j2"
-      - "conf/input/06-zookeeper.conf.j2"
-      - "conf/input/07-kafka.conf.j2"
-      - "conf/input/09-monasca.conf.j2"
+      - name: "conf/input/00-global.conf.j2"
+        enabled: true
+      - name: "conf/input/01-syslog.conf.j2"
+        enabled: true
+      - name: "conf/input/02-mariadb.conf.j2"
+        enabled: true
+      - name: "conf/input/03-rabbitmq.conf.j2"
+        enabled: true
+      - name: "conf/input/04-openstack-wsgi.conf.j2"
+        enabled: true
+      - name: "conf/input/05-libvirt.conf.j2"
+        enabled: true
+      - name: "conf/input/06-zookeeper.conf.j2"
+        enabled: true
+      - name: "conf/input/07-kafka.conf.j2"
+        enabled: true
+      - name: "conf/input/08-prometheus.conf.j2"
+        enabled: "{{ enable_prometheus_fluentd_integration | bool }}"
+      - name: "conf/input/09-monasca.conf.j2"
+        enabled: true
     customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}"
     # Filters
     fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"

ansible/roles/common/templates/conf/input/08-prometheus.conf.j2

@@ -0,0 +1,14 @@
+<source>
+  @type prometheus
+  bind {{ api_interface_address }}
+  port {{ prometheus_fluentd_integration_port }}
+  metrics_path /metrics
+</source>
+
+<source>
+  @type prometheus_output_monitor
+  interval 10
+  <labels>
+    Hostname ${hostname}
+  </labels>
+</source>

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -74,6 +74,15 @@ scrape_configs:
 {% endfor %}
 {% endif %}
 
+{% if enable_prometheus_fluentd_integration | bool %}
+  - job_name: fluentd
+    static_configs:
+      - targets:
+{% for host in groups['fluentd'] %}
+        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
+{% endfor %}
+{% endif %}
+
 {% if enable_prometheus_ceph_mgr_exporter | bool %}
   - job_name: ceph_mgr_exporter
     honor_labels: true

etc/kolla/globals.yml

@@ -711,6 +711,7 @@
 #enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}"
 #enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}"
 #enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}"
+#enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable fluentd | bool }}"
 #enable_prometheus_memcached: "{{ enable_prometheus | bool }}"
 #enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}"
 #enable_prometheus_ceph_mgr_exporter: "no"

releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds support for integrating Fluentd metrics into Prometheus. By
+    default this is now enabled when Prometheus is enabled. This behaviour
+    can be overridden via the `enable_prometheus_fluentd_integration` flag. By
+    default the integration provides metrics relating to the processing of logs
+    by Fluentd. These metrics can be useful for monitoring the status of the
+    Fluentd service. Additional metrics can also be extracted from logs via
+    custom Fluentd config.