Merge pull request #181 from stackhpc/wallaby_cadvisor

Wallaby: Monitoring backports

Author: cityofships

ansible/roles/grafana/tasks/config.yml

@@ -139,34 +139,18 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
 
-- name: Create custom Grafana dashboards folder
-  become: true
-  vars:
-    service: "{{ grafana_services['grafana'] }}"
-  file:
-    dest: "{{ node_config_directory }}/grafana/dashboards/"
-    state: "directory"
-    mode: "0770"
-  when:
-    - grafana_custom_dashboards_folder.stat.exists
-    - grafana_custom_dashboards_folder.stat.isdir
-    - inventory_hostname in groups[service.group]
-    - service.enabled | bool
-
 - name: Copying over custom dashboards
   vars:
     service: "{{ grafana_services['grafana'] }}"
   copy:
-    src: "{{ item }}"
-    dest: "{{ node_config_directory }}/grafana/dashboards/"
+    src: "{{ node_custom_config }}/grafana/dashboards"
+    dest: "{{ node_config_directory }}/grafana/"
     mode: "0660"
   become: true
   when:
     - grafana_custom_dashboards_folder.stat.exists
     - grafana_custom_dashboards_folder.stat.isdir
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
-  with_fileglob:
-    - "{{ node_custom_config }}/grafana/dashboards/*.json"
   notify:
     - Restart grafana container

ansible/roles/prometheus/defaults/main.yml

@@ -257,3 +257,5 @@ prometheus_openstack_exporter_disabled_dns: "{{ '--disable-service.dns' if not e
 prometheus_openstack_exporter_disabled_object: "{{ '--disable-service.object-store' if not enable_swift | bool else '' }}"
 prometheus_openstack_exporter_disabled_lb: "{{ '--disable-service.load-balancer --disable-metric=neutron-loadbalancers --disable-metric=neutron-loadbalancers_not_active' if not enable_octavia | bool else '' }}"
 prometheus_openstack_exporter_disabled_items: "{{ [prometheus_openstack_exporter_disabled_volume, prometheus_openstack_exporter_disabled_dns, prometheus_openstack_exporter_disabled_object, prometheus_openstack_exporter_disabled_lb|trim]|join(' ')|trim }}"
+
+prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process"

ansible/roles/prometheus/templates/prometheus-cadvisor.json.j2

@@ -1,5 +1,5 @@
 {
-    "command": "/opt/cadvisor --port={{ prometheus_cadvisor_port }} --log_dir=/var/log/kolla/prometheus",
+    "command": "/opt/cadvisor --port={{ prometheus_cadvisor_port }} --log_dir=/var/log/kolla/prometheus {{ prometheus_cadvisor_cmdline_extras }}",
     "config_files": [],
     "permissions": [
         {

doc/source/reference/logging-and-monitoring/grafana-guide.rst

@@ -35,9 +35,11 @@ Custom dashboards provisioning
 
 Kolla Ansible sets custom dashboards provisioning using `Dashboard provider <https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards>`_.
 
-Dashboard's JSON files should be placed into
-``{{ node_custom_config }}/grafana/dashboards/`` folder.
-Dashboards will be imported to Grafana dashboards General Folder.
+Dashboard JSON files should be placed into the
+``{{ node_custom_config }}/grafana/dashboards/`` folder. The use of
+sub-folders is also supported when using a custom ``provisioning.yaml``
+file. Dashboards will be imported into the Grafana dashboards 'General'
+folder by default.
 
 Grafana provisioner config can be altered by placing ``provisioning.yaml`` to
 ``{{ node_custom_config }}/grafana/`` folder.

etc/kolla/globals.yml

@@ -716,6 +716,10 @@
 # List of extra parameters passed to prometheus. You can add as many to the list.
 #prometheus_cmdline_extras:
 
+# List of extra parameters passed to cAdvisor. By default system cgroups
+# and container labels are not exposed to reduce time series cardinality.
+#prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process"
+
 # Example of setting endpoints for prometheus ceph mgr exporter.
 # You should add all ceph mgr's in your external ceph deployment.
 #prometheus_ceph_mgr_exporter_endpoints:

releasenotes/notes/reduce-cadvisor-cardinality-1213854b9fe0c828.yaml

@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    Adds support for passing extra runtime options to cAdvisor via
+    ``prometheus_cadvisor_cmdline_extras`` new variable. By default
+    system cgroups' metrics are disabled, plus container labels
+    don't get exposed to Prometheus. Expensive metrics that usually
+    should not be exported are also enforced to be disabled - consult
+    `<https://github.com/google/cadvisor/blob/master/docs/runtime_options.md#metrics>`_
+    for a list. These defaults create savings in resources usage by both
+    cAdvisor and Prometheus.
+upgrade:
+  - |
+    cAdvisor has now reduced number of Prometheus metrics and labels
+    exported by default. This means that corresponding timeseries will
+    no longer be created. If existing setup relies on these, eg. for
+    visualisation or alerting, they could be explicitly enabled prior to
+    upgrading with the ``prometheus_cadvisor_cmdline_extras`` new variable.
+    Reference for the possible options: `<https://github.com/google/cadvisor/blob/master/docs/runtime_options.md#metrics>`_.