Refactored monitoring config and removed redundant groups

Author: wtripp180901

ansible/roles/kube_prometheus_stack/defaults/main/helm.yml

@@ -73,7 +73,7 @@ kube_prometheus_stack_release_defaults:
               - ReadWriteOnce
             resources:
               requests:
-                storage: "{{ kube_prometheus_stack_volume_size }}"
+                storage: "{{ prometheus_volume_size }}"
       retention: "{{ prometheus_storage_retention }}"
       retentionSize: "{{ prometheus_storage_retention_size }}"
       additionalAlertRelabelConfigs: "{{ prometheus_alert_relabel_configs }}"
@@ -102,7 +102,7 @@ kube_prometheus_stack_release_defaults:
     serviceMonitor:
       enabled: false
     ingress:
-      path: "/node/{{ groups['grafana'].0 }}/{{ grafana_port }}"
+      path: "/node/{{ groups['prometheus'].0 }}/{{ grafana_port }}"
     sidecar:
       dashboards:
         searchNamespace: ALL
@@ -113,7 +113,7 @@ kube_prometheus_stack_release_defaults:
         serve_from_sub_path: true
       auth: "{{ grafana_auth }}"
       auth.anonymous:
-        enabled: "{{ grafana_anonymous_auth }}"
+        enabled: "{{ grafana_auth_anonymous }}"
       analytics: "{{ grafana_analytics }}"
       smtp: "{{ grafana_smtp }}"
       log: "{{ grafana_log }}"

ansible/roles/kube_prometheus_stack/defaults/main/main.yml

@@ -16,9 +16,9 @@ login_ip: "{{ hostvars[groups['openondemand'][0]]['ansible_host'] }}" # probably
 control_ip: "{{ ansible_default_ipv4.address| default(ansible_all_ipv4_addresses[0]) }}"
 control_sslip: "{{ control_ip | regex_replace('\\.', '-') }}.sslip.io" 
 
-grafana_claim_size: 10Gi
+grafana_volume_size: 10Gi
 
-grafana_anonymous_auth: true
+grafana_auth_anonymous: true
 
 slack_integration:
   channel: "#alerts"
@@ -64,7 +64,7 @@ prometheus_storage_retention: "30d"
 # supported: KB, MB, GB, TB, PB.
 prometheus_storage_retention_size: "40GB"
 
-kube_prometheus_stack_volume_size: 40Gi
+prometheus_volume_size: 40Gi
 
 prometheus_config_flags_extra: {}
 # prometheus_config_flags_extra:
@@ -126,138 +126,13 @@ prometheus_scrape_configs:
 #   - prometheus/targets/*.yml
 #   - prometheus/targets/*.json
 
+prometheus_extra_alert_rules: []
+
 prometheus_alert_rules:
   appliance-rules:
     groups:
     - name: all
-      rules:
-      - alert: Watchdog
-        expr: vector(1)
-        for: 10m
-        labels:
-          severity: warning
-          alertname: Watchdog
-        annotations:
-          description: "This is an alert meant to ensure that the entire alerting pipeline is functional.\nThis alert is always firing, therefore it should always be firing in Alertmanager\nand always fire against a receiver. There are integrations with various notification\nmechanisms that send a notification when this alert is not firing. For example the\n\"DeadMansSnitch\" integration in PagerDuty."
-          summary: 'Ensure entire alerting pipeline is functional'
-      - alert: InstanceDown
-        expr: 'up == 0'
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          description: '{% raw %}{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.{% endraw %}'
-          summary: '{% raw %}Instance {{ $labels.instance }} down{% endraw %}'
-      - alert: RebootRequired
-        expr: 'node_reboot_required > 0'
-        labels:
-          severity: warning
-        annotations:
-          description: '{% raw %}{{ $labels.instance }} requires a reboot.{% endraw %}'
-          summary: '{% raw %}Instance {{ $labels.instance }} - reboot required{% endraw %}'
-      - alert: NodeFilesystemSpaceFillingUp
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.{% endraw %}'
-          summary: 'Filesystem is predicted to run out of space within the next 24 hours.'
-        expr: "(\n  node_filesystem_avail_bytes{job=\"node\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",fstype!=\"\"} * 100 < 40\nand\n  predict_linear(node_filesystem_avail_bytes{job=\"node\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: warning
-      - alert: NodeFilesystemSpaceFillingUp
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.{% endraw %}'
-          summary: 'Filesystem is predicted to run out of space within the next 4 hours.'
-        expr: "(\n  node_filesystem_avail_bytes{job=\"node\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",fstype!=\"\"} * 100 < 20\nand\n  predict_linear(node_filesystem_avail_bytes{job=\"node\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: critical
-      - alert: NodeFilesystemAlmostOutOfSpace
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.{% endraw %}'
-          summary: 'Filesystem has less than 5% space left.'
-        expr: "(\n  node_filesystem_avail_bytes{job=\"node\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",fstype!=\"\"} * 100 < 5\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: warning
-      - alert: NodeFilesystemAlmostOutOfSpace
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.{% endraw %}'
-          summary: 'Filesystem has less than 3% space left.'
-        expr: "(\n  node_filesystem_avail_bytes{job=\"node\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node\",fstype!=\"\"} * 100 < 3\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: critical
-      - alert: NodeFilesystemFilesFillingUp
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.{% endraw %}'
-          summary: 'Filesystem is predicted to run out of inodes within the next 24 hours.'
-        expr: "(\n  node_filesystem_files_free{job=\"node\",fstype!=\"\"} / node_filesystem_files{job=\"node\",fstype!=\"\"} * 100 < 40\nand\n  predict_linear(node_filesystem_files_free{job=\"node\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: warning
-      - alert: NodeFilesystemFilesFillingUp
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.{% endraw %}'
-          summary: 'Filesystem is predicted to run out of inodes within the next 4 hours.'
-        expr: "(\n  node_filesystem_files_free{job=\"node\",fstype!=\"\"} / node_filesystem_files{job=\"node\",fstype!=\"\"} * 100 < 20\nand\n  predict_linear(node_filesystem_files_free{job=\"node\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: critical
-      - alert: NodeFilesystemAlmostOutOfFiles
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.{% endraw %}'
-          summary: 'Filesystem has less than 5% inodes left.'
-        expr: "(\n  node_filesystem_files_free{job=\"node\",fstype!=\"\"} / node_filesystem_files{job=\"node\",fstype!=\"\"} * 100 < 5\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: warning
-      - alert: NodeFilesystemAlmostOutOfFiles
-        annotations:
-          description: '{% raw %}Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.{% endraw %}'
-          summary: 'Filesystem has less than 3% inodes left.'
-        expr: "(\n  node_filesystem_files_free{job=\"node\",fstype!=\"\"} / node_filesystem_files{job=\"node\",fstype!=\"\"} * 100 < 3\nand\n  node_filesystem_readonly{job=\"node\",fstype!=\"\"} == 0\n)\n"
-        for: 1h
-        labels:
-          severity: critical
-      - alert: NodeNetworkReceiveErrs
-        annotations:
-          description: '{% raw %}{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.{% endraw %}'
-          summary: 'Network interface is reporting many receive errors.'
-        expr: "increase(node_network_receive_errs_total[2m]) > 10\n"
-        for: 1h
-        labels:
-          severity: warning
-      - alert: NodeNetworkTransmitErrs
-        annotations:
-          description: '{% raw %}{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.{% endraw %}'
-          summary: 'Network interface is reporting many transmit errors.'
-        expr: "increase(node_network_transmit_errs_total[2m]) > 10\n"
-        for: 1h
-        labels:
-          severity: warning
-      - alert: NodeHighNumberConntrackEntriesUsed
-        annotations:
-          description: '{% raw %}{{ $value | humanizePercentage }} of conntrack entries are used{% endraw %}'
-          summary: 'Number of conntrack are getting close to the limit'
-        expr: "(node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75\n"
-        labels:
-          severity: warning
-      - alert: NodeClockSkewDetected
-        annotations:
-          message: '{% raw %}Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.{% endraw %}'
-          summary: 'Clock skew detected.'
-        expr: "(\n  node_timex_offset_seconds > 0.05\nand\n  deriv(node_timex_offset_seconds[5m]) >= 0\n)\nor\n(\n  node_timex_offset_seconds < -0.05\nand\n  deriv(node_timex_offset_seconds[5m]) <= 0\n)\n"
-        for: 10m
-        labels:
-          severity: warning
-      - alert: NodeClockNotSynchronising
-        annotations:
-          message: '{% raw %}Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.{% endraw %}'
-          summary: 'Clock not synchronising.'
-        expr: "min_over_time(node_timex_sync_status[5m]) == 0\n"
-        for: 10m
-        labels:
-          severity: warning
+      rules: "{{ prometheus_extra_alert_rules }}" 
 
 # ------------------------------------------------------------------------------------------
 

ansible/roles/kube_prometheus_stack/tasks/main.yml

@@ -46,7 +46,7 @@
           app.kubernetes.io/name: prometheus-dir
       spec:
         capacity:
-          storage: "{{ kube_prometheus_stack_volume_size }}"
+          storage: "{{ prometheus_volume_size }}"
         accessModes:
         - ReadWriteOnce
         hostPath:
@@ -65,7 +65,7 @@
           app.kubernetes.io/name: grafana-dir
       spec:
         capacity:
-          storage: "{{ grafana_claim_size }}"
+          storage: "{{ grafana_volume_size }}"
         accessModes:
         - ReadWriteOnce
         hostPath:
@@ -85,7 +85,7 @@
         - ReadWriteOnce
         resources:
           requests:
-            storage: "{{ grafana_claim_size }}"
+            storage: "{{ grafana_volume_size }}"
         volumeMode: Filesystem
         volumeName: grafana-dir
 

environments/common/inventory/group_vars/all/alertmanager.yml

@@ -0,0 +1,26 @@
+alertmanager_replicas: 1
+alertmanager_port: 30002 # Must be within K3s' reserved port range (default 30000-32767)
+
+# Add receivers here, uncomment below and add Slack bot app creds for Slack integration
+alertmanager_config:
+  route:
+    group_by: ['...']
+    # receiver: slack-receiver
+  global:
+    resolve_timeout: "{{ prometheus_config_flags_extra.alertmanager.timeout | default( '5m' ) }}"
+  receivers:
+    - name: 'null'
+    # - name: slack-receiver
+    #   slack_configs:
+    #     - channel: "{{ slack_integration.channel }}"
+    #       api_url: https://slack.com/api/chat.postMessage
+    #       http_config:
+    #         authorization:
+    #           credentials: "{{ slack_integration.app_creds }}"
+    #       text: "{{ '{{' }} .GroupLabels.alertname {{ '}}' }} : {{ '{{' }}  .CommonAnnotations.description {{ '}}' }}"
+    #       title_link: "http://{{ control_ip }}/alertmanager/#/alerts?receiver=slack-receiver"
+    #       send_resolved: true
+
+# slack_integration:
+#   channel: '#alerts'
+#   app_creds:

environments/common/inventory/group_vars/all/defaults.yml

@@ -18,7 +18,7 @@ api_address: "{{ inventory_hostname }}"
 opensearch_address: "127.0.0.1"
 prometheus_address: "{{ hostvars[groups['prometheus'].0].api_address }}"
 openondemand_address: "{{ hostvars[groups['openondemand'].0].api_address if groups['openondemand'] | count > 0 else '' }}"
-grafana_address: "{{ hostvars[groups['grafana'].0].api_address }}"
+grafana_address: "{{ hostvars[groups['prometheus'].0].api_address }}"
 
 ############################# bootstrap: local user configuration #########################
 
@@ -50,29 +50,6 @@ appliances_local_users_default:
         shell: /sbin/nologin
         uid: 202
         system: true
-    
-    - group:
-        name: prometheus
-        gid: 976
-      user:
-        name: prometheus
-        uid: 981
-        home: "{{ prometheus_db_dir }}"
-        shell: /usr/sbin/nologin
-        system: true
-      enable: "{{ 'prometheus' in group_names }}"
-    
-    - group:
-        name: grafana
-        gid: 979
-      user:
-        name: grafana
-        comment: grafana user
-        uid: 984
-        home: /usr/share/grafana
-        shell: /sbin/nologin
-        system: true
-      enable: "{{ 'grafana' in group_names }}"
 
 # Overide this to add extra users whilst keeping the defaults.
 appliances_local_users_extra: [] # see format of appliances_local_users_default above

environments/common/inventory/group_vars/all/grafana.yml

@@ -1,24 +1,15 @@
----
-
-# See: https://github.com/cloudalchemy/ansible-grafana
-# for variable definitions.
-grafana_version: '9.5.21'
-
-# need to copy some role defaults here so we can use in inventory:
-grafana_port: 30001
+grafana_image_tag: '11.2.2'
+grafana_port: 30001 # Must be within K3s' reserved port range (default 30000-32767)
 
 # Define where state is stored
 grafana_data_dir: "{{ appliances_state_dir | default('/var/lib') }}/grafana"
 
-# Configure internal address & URL - note "api" means "internal" to cloudalchemy.grafana but "external" to appliance:
-grafana_api_address: "{{ hostvars[groups['grafana'].0].internal_address }}"
-grafana_api_url: "http://{{ grafana_api_address }}:{{ grafana_port }}"
-
 # Configure external address, with external URL depending on whether we are using Open Ondemand as a proxy
+grafana_api_address: "{{ hostvars[groups['prometheus'].0].internal_address }}"
+grafana_api_url: "http://{{ grafana_api_address }}:{{ grafana_port }}"
 grafana_url_direct: "http://{{ grafana_address }}:{{ grafana_port }}"
-grafana_url_openondemand_proxy: "https://{{ openondemand_servername | default('') }}/node/{{ groups['grafana'].0 }}/{{ grafana_port }}"
+grafana_url_openondemand_proxy: "https://{{ openondemand_servername | default('') }}/node/{{ groups['prometheus'].0 }}/{{ grafana_port }}"
 grafana_url: "{{ grafana_url_openondemand_proxy if groups['openondemand'] | count > 0 else grafana_url_direct }}"
-grafana_serve_from_sub_path: "{{ groups['openondemand'] | count > 0 }}"
 
 grafana_dashboards_default:
   # node exporter slurm:
@@ -49,7 +40,6 @@ grafana_dashboards_default:
       - placeholder: DS_PROMETHEUS
         replacement: prometheus
     revision_id: 3
-
 grafana_dashboards: "{{ grafana_dashboards_default + (openondemand_dashboard if groups.get('openondemand') else []) }}"
 
 grafana_security:
@@ -58,10 +48,6 @@ grafana_security:
     allow_embedding: true
 
 grafana_datasources:
-  # - name: prometheus
-  #   type: prometheus
-  #   url: "http://{{ prometheus_address }}:9090" # default prometheus port
-  #   editable: true
   - name: slurmstats
     # see https://github.com/grafana/opensearch-datasource#configure-the-data-source-with-provisioning
     type: grafana-opensearch-datasource
@@ -81,27 +67,11 @@ grafana_datasources:
       flavor: elasticsearch
     editable: true
     # readOnly: false
-
 grafana_plugins:
   - grafana-opensearch-datasource 2.8.1
 
-# want to set grafana_server.serve_from_sub_path if have Open Ondemand to proxy:
-grafana_server:
-  # role defaults:
-  protocol: http
-  enforce_domain: false
-  socket: ""
-  cert_key: ""
-  cert_file: ""
-  enable_gzip: false
-  static_root_path: public
-  router_logging: false
-  # appliance specific:
-  serve_from_sub_path: "{{ grafana_serve_from_sub_path }}"
-
-
-grafana_auth_anonymous: false # Enable anonymous View-only login - see implications: https://grafana.com/docs/grafana/latest/administration/security/#implications-of-enabling-anonymous-access-to-dashboards
-
+grafana_auth_anonymous: true # Enable anonymous View-only login - see implications: https://grafana.com/docs/grafana/latest/administration/security/#implications-of-enabling-anonymous-access-to-dashboards
+grafana_volume_size: 10Gi
 _grafana_auth_anon_cfg:
   anonymous:
     org_name: "Main Org."

environments/common/inventory/group_vars/all/monitoring.yml

@@ -0,0 +1,6 @@
+kube_prometheus_stack_chart_version: 59.1.0
+kube_prometheus_stack_release_namespace: monitoring-system
+kube_prometheus_stack_release_name: kube-prometheus-stack
+kube_prometheus_stack_wait_timeout: 5m
+
+# See prometheus.yml, grafana.yml and alertmanager.yml for config of individual monitoring services