get slack integration working, with node down alert

Author: sjpb

ansible/roles/alertmanager/README.md

@@ -4,3 +4,71 @@
 notes:
 - HA is not supported
 - state ("notification state and configured silences") is not preserved across rebuild
+- not used for caas
+- no dashboard
+
+
+
+## Role variables
+
+The following variables are equivalent to similarly-named arguments to the
+`alertmanager` binary. See `man alertmanager` for more info:
+
+- TODO:
+
+The following variables are templated into the alertmanager configuration file:
+
+- TODO:
+
+Other variables:
+- TODO:
+
+
+## TODO
+
+memory usage looks a bit close:
+
+```
+[root@RL9-control rocky]# free -h
+               total        used        free      shared  buff/cache   available
+Mem:           3.6Gi       2.4Gi       168Mi        11Mi       1.5Gi       1.2Gi
+Swap:             0B          0B          0B
+```
+
+
+
+## Slack Integration
+
+1. Create an app with a bot token:
+
+- Go to https://api.slack.com/apps
+- select "Create an App"
+- select "From scratch"
+- Set app name and workspacef fields, select "Create"
+- Fill out "Short description" and "Background color" fields, select "Save changes"
+- Select "OAuth & Permissions" on left menu
+- Under "Scopes : Bot Token Scopes", select "Add an OAuth Scope", add
+  `chat:write` and select "Save changes"
+- Select "Install App" on left menu, select "Install to your-workspace", select Allow
+- Copy the Bot User OAuth token shown
+
+2. Add the bot token into the config and enable Slurm integration
+
+- Open `environments/site/inventory/group_vars/all/vault_alertmanager.yml`
+- Uncomment `vault_alertmanager_slack_integration_app_creds` and add the token
+- Vault-encrypt that file:
+
+    ansible-vault encrypt environments/$ENV/inventory/group_vars/all/vault_alertmanager.yml
+
+- Open `environments/site/inventory/group_vars/all/alertmanager.yml`
+- Uncomment the config and set your alert channel name
+
+3. Invite the bot to your alerts channel
+- In the appropriate Slack channel type:
+
+    /invite @YOUR_BOT_NAME
+
+
+## Adding Rules
+
+TODO: describe how prom config works

ansible/roles/alertmanager/defaults/main.yml

@@ -7,12 +7,18 @@ alertmanager_enabled: true
 
 alertmanager_system_user: alertmanager
 alertmanager_system_group: alertmanager
-alertmanager_config_path: /etc/alertmanager/alertmanager.yml
-alertmanager_storage_dir: /var/lib/alertmanager
-alertmanager_web_listen_addresses:
-  - ':9100'
-alertmanager_web_external_url: http://localhost:9093/
-alertmanager_config_flags: {}
+alertmanager_config_file: /etc/alertmanager/alertmanager.yml # --config.file: Alertmanager configuration file name
+alertmanager_storage_path: /var/lib/alertmanager # --storage.path: Base path for data storage
+
+alertmanager_port: '9093'
+alertmanager_web_listen_addresses: # elements of --web.listen-address
+  - ":{{ alertmanager_port }}"
+alertmanager_web_external_url: "http://localhost:{{ alertmanager_port}}/" # --web.external-url: The URL under which Alertmanager is externally reachable (for example, if Alertmanager is served via a reverse proxy). Used for generating relative and absolute links back to Alertmanager itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Alertmanager. If omitted, relevant URL components will be derived automatically
+# TODO: work out how we proxy this through ondemand
+
+alertmanager_data_retention: '120h' # --data.retention # How long to keep data for
+alertmanager_data_maintenance_interval: '15m' # --data.maintenance-interval: Interval between garbage collection and snapshotting to disk of the silences and the notification logs
+alertmanager_config_flags: {} # other command-line parameters as shown by `man alertmanager`
 # TODO: data retention?
 alertmanager_config_template: alertmanager.yml.j2
 
@@ -28,7 +34,7 @@ alertmanager_config_template: alertmanager.yml.j2
 alertmanager_default_receivers:
   - name: 'null'
 
-alertmanager_slack_receiver: {} # really defined in common as it needs prometheus_address
+alertmanager_slack_receiver: {} # defined in common env as it needs prometheus_address
 
 alertmanager_extra_receivers: "{{ [alertmanager_slack_receiver] if alertmanager_slack_integration is defined else [] }}"
 

ansible/roles/alertmanager/tasks/configure.yml

@@ -6,8 +6,8 @@
     group: "{{ alertmanager_system_group }}"
     mode: u=rwX,go=rX
   loop:
-    - "{{ alertmanager_config_path | dirname }}"
-    - "{{ alertmanager_storage_dir }}"
+    - "{{ alertmanager_config_file | dirname }}"
+    - "{{ alertmanager_storage_path }}"
 
 # TODO: selinux?
 
@@ -25,7 +25,7 @@
 - name: Template alertmanager config
   ansible.builtin.template:
     src: "{{ alertmanager_config_template }}"
-    dest: "{{ alertmanager_config_path }}"
+    dest: "{{ alertmanager_config_file }}"
     owner: "{{ alertmanager_system_user }}"
     group: "{{ alertmanager_system_group }}"
     mode: u=rw,go=r # TODO: check there are no sensitive things in here!

ansible/roles/alertmanager/templates/alertmanager.service.j2

@@ -16,8 +16,10 @@ Group={{ alertmanager_system_group }}
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart={{ alertmanager_binary_dir }}/alertmanager \
   --cluster.listen-address='' \
-  --config.file={{ alertmanager_config_path }} \
-  --storage.path={{ alertmanager_storage_dir }} \
+  --config.file={{ alertmanager_config_file }} \
+  --storage.path={{ alertmanager_storage_path }} \
+  --data.retention={{ alertmanager_data_retention }} \
+  --data.maintenance-interval={{ alertmanager_data_maintenance_interval }} \
 {% for address in alertmanager_web_listen_addresses %}
   --web.listen-address={{ address }} \
 {% endfor %}
@@ -36,7 +38,7 @@ NoNewPrivileges=true
 MemoryDenyWriteExecute=true
 PrivateTmp=true
 ProtectHome=true
-ReadWriteDirectories={{ alertmanager_storage_dir }}
+ReadWriteDirectories={{ alertmanager_storage_path }}
 RemoveIPC=true
 RestrictSUIDSGID=true
 

ansible/roles/alertmanager/templates/alertmanager.yml.j2

@@ -1,4 +1,4 @@
 {{ ansible_managed | comment }}
 
-{{ alertmanager_config_default }}
-{{ alertmanager_config_extra }}
+{{ alertmanager_config_default | to_nice_yaml }}
+{{ alertmanager_config_extra | to_nice_yaml if alertmanager_config_extra | length > 0 else '' }}

environments/common/files/prometheus/rules/slurm.rules

@@ -0,0 +1,11 @@
+
+groups:
+- name: Slurm
+  rules:
+  - alert: SlurmNodeDown
+    annotations:
+      description: '{{ $value }} Slurm nodes are in down status'
+      summary: 'At least one Slurm node is down.'
+    expr: "slurm_nodes_down > 0\n"
+    labels:
+      severity: critical

environments/common/inventory/group_vars/all/alertmanager.yml

@@ -1,4 +1,7 @@
-alertmanager_slack_receiver:
+
+alertmanager_port: '9093' # defined here as required for prometheus
+
+alertmanager_slack_receiver: # defined here as needs prometheus address
   name: slack-receiver
   slack_configs:
     - channel: "{{ alertmanager_slack_integration.channel | default('none') }}"

environments/common/inventory/group_vars/all/defaults.yml

@@ -22,7 +22,7 @@ prometheus_address: "{{ hostvars[groups['prometheus'].0].api_address }}"
 openondemand_address: "{{ hostvars[groups['openondemand'].0].api_address if groups['openondemand'] | count > 0 else '' }}"
 grafana_address: "{{ hostvars[groups['grafana'].0].api_address }}"
 k3s_server_name: "{{ hostvars[groups['k3s_server'] | first].ansible_host }}"
-
+alertmanager_address: "{{ hostvars[groups['alertmanager'].0].api_address }}"
 ############################# bootstrap: local user configuration #########################
 
 # Note RockyLinux 8.5 defines system user/groups in range 201-999

environments/common/inventory/group_vars/all/prometheus.yml

@@ -9,10 +9,16 @@ prometheus_storage_retention: "31d"
 prometheus_storage_retention_size: "100GB"
 prometheus_db_dir: "{{ appliances_state_dir | default('/var/lib') }}/prometheus"
 
-prometheus_alertmanager_config: []
-
-prometheus_alert_rules_files:
-- "{{ appliances_repository_root }}/environments/common/files/prometheus/rules/*.rules"
+prometheus_alertmanager_config_default:
+    - static_configs:
+      - targets:
+        - "{{ alertmanager_address }}:{{ alertmanager_port }}"
+prometheus_alertmanager_config: "{{ prometheus_alertmanager_config_default if groups['alertmanager'] else {} }}"
+
+# by default, use rule files from the following path relative to current and all parent environment inventory directories:
+prometheus_alert_rules_files_inventory_glob: ../files/prometheus/rules/*.rules
+prometheus_alert_rules_files: "{{ ansible_inventory_sources | product([prometheus_alert_rules_files_inventory_glob]) | map('join', '/') | map('realpath') }}"
+# TODO: find a way to include/exclude files?
 
 prometheus_alert_rules: []
 

environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/alertmanager.yml

@@ -0,0 +1,6 @@
+# Uncomment below and add Slack bot app creds in the adjacent file
+# vault_alertmanager.ym for Slack integration:
+#
+# alertmanager_slack_integration:
+#   channel: '#alerts'
+#   app_creds: "{{ vault_alertmanager_slack_integration_app_creds }}"