Refactored and fixed slack integration

Author: wtripp180901

environments/common/inventory/group_vars/all/alertmanager.yml

@@ -1,26 +1,27 @@
-alertmanager_replicas: 1
-alertmanager_port: 30002 # Must be within K3s' reserved port range (default 30000-32767)
-
-# Add receivers here, uncomment below and add Slack bot app creds for Slack integration
 alertmanager_config:
   route:
     group_by: ['...']
-    # receiver: slack-receiver
-  global:
-    resolve_timeout: "{{ prometheus_config_flags_extra.alertmanager.timeout | default( '5m' ) }}"
-  receivers:
-    - name: 'null'
-    # - name: slack-receiver
-    #   slack_configs:
-    #     - channel: "{{ slack_integration.channel }}"
-    #       api_url: https://slack.com/api/chat.postMessage
-    #       http_config:
-    #         authorization:
-    #           credentials: "{{ slack_integration.app_creds }}"
-    #       text: "{{ '{{' }} .GroupLabels.alertname {{ '}}' }} : {{ '{{' }}  .CommonAnnotations.description {{ '}}' }}"
-    #       title_link: "http://{{ control_ip }}/alertmanager/#/alerts?receiver=slack-receiver"
-    #       send_resolved: true
+    receiver: "{{ 'slack-receiver' if alertmanager_slack_integration is defined else 'null' }}"
+  receivers: "{{ alertmanager_default_receivers  + alertmanager_extra_receivers }}"
+
+alertmanager_default_receivers:
+  - name: 'null'
+
+alertmanager_extra_receivers: "{{ [alertmanager_slack_receiver] if alertmanager_slack_integration is defined else [] }}"
+
+alertmanager_slack_receiver:
+  name: slack-receiver
+  slack_configs:
+    - channel: "{{ alertmanager_slack_integration.channel | default('none') }}"
+      api_url: https://slack.com/api/chat.postMessage
+      http_config:
+        authorization:
+          credentials: "{{ alertmanager_slack_integration.app_creds | default('none') }}"
+      text: "{{ '{{' }} .GroupLabels.alertname {{ '}}' }} : {{ '{{' }}  .CommonAnnotations.description {{ '}}' }}"
+      title_link: "http://{{ prometheus_address }}/alertmanager/#/alerts?receiver=slack-receiver"
+      send_resolved: true
 
-# slack_integration:
+# Uncomment below and add Slack bot app creds for Slack integration
+# alertmanager_slack_integration:
 #   channel: '#alerts'
 #   app_creds:

environments/common/inventory/group_vars/all/defaults.yml

@@ -16,7 +16,7 @@ api_address: "{{ inventory_hostname }}"
 
 # Service endpoints
 opensearch_address: "127.0.0.1"
-prometheus_address: "{{ hostvars[groups['prometheus'].0].api_address }}"
+prometheus_address: "{{ hostvars[groups['prometheus'].0].api_address }}:{{ prometheus_port }}"
 openondemand_address: "{{ hostvars[groups['openondemand'].0].api_address if groups['openondemand'] | count > 0 else '' }}"
 grafana_address: "{{ hostvars[groups['prometheus'].0].api_address }}"
 

environments/common/inventory/group_vars/all/prometheus.yml

@@ -26,7 +26,7 @@ prometheus_scrape_configs: "{{ prometheus_scrape_configs_default + (openondemand
 prometheus_extra_rules:
   - alert: SlurmNodeDown
     annotations:
-      message: '{% raw %}{{ $value }} Slurm nodes are in down status.{% endraw %}'
+      description: '{% raw %}{{ $value }} Slurm nodes are in down status.{% endraw %}'
       summary: 'At least one Slurm node is down.'
     expr: "slurm_nodes_down > 0\n"
     labels:
@@ -39,7 +39,7 @@ prometheus_extra_rules:
     expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode="iowait",job="node-exporter"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node-exporter"}[60s])))
   - record: node_cpu_other_seconds:record
     expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode!="idle",mode!="user",mode!="system",mode!="iowait",job="node-exporter"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node-exporter"}[60s])))
-  - record: node_cpu_scaling_frequency_hertz_avg:record # frequency rules aren't working
+  - record: node_cpu_scaling_frequency_hertz_avg:record # Warning: frequency rules will not work when deploying appliance on VMs
     expr: avg by (instance) (node_cpu_scaling_frequency_hertz)
   - record: node_cpu_scaling_frequency_hertz_min:record
     expr: min by (instance) (node_cpu_scaling_frequency_hertz)