Merge remote-tracking branch 'origin/main' into HEAD

Conflicts:
environments/.stackhpc/tofu/cluster_image.auto.tfvars.json

Author: JohnGarbutt

.github/workflows/nightly-cleanup.yml

@@ -56,42 +56,13 @@ jobs:
           fi
         shell: bash
 
-      - name: Delete clusters if control node not tagged with keep
+      - name: Delete CI clusters
         run: |
           . venv/bin/activate
-          if [[ -z ${ci_clusters} ]]; then
+          if [[ -z "${ci_clusters}" ]]; then
             echo "No clusters to delete."
             exit 0
           fi
-          
-          for cluster_prefix in ${ci_clusters}
-          do
-            echo "Processing cluster: $cluster_prefix"
-            # Get all servers with the matching name for control node
-            CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
-            SERVER_COUNT=$(echo "$CONTROL_SERVERS" | jq length)
-
-            if [[ $SERVER_COUNT -gt 1 ]]; then
-              echo "Multiple servers found for control node '${cluster_prefix}-control'. Checking tags for each..."
-              
-              for server in $(echo "$CONTROL_SERVERS" | jq -r '.[].ID'); do
-                # Get tags for each control node
-                TAGS=$(openstack server show "$server" --column tags --format value)
-                
-                if [[ $TAGS =~ "keep" ]]; then
-                  echo "Skipping ${cluster_prefix} (server ${server}) - control instance is tagged as keep"
-                else
-                  ./dev/delete-cluster.py ${cluster_prefix} --force
-                fi
-              done
-            else
-              # If only one server, extract its tags and proceed
-              TAGS=$(echo "$CONTROL_SERVERS" | jq -r '.[0].Tags')
-              if [[ $TAGS =~ "keep" ]]; then
-                echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
-              else
-                ./dev/delete-cluster.py ${cluster_prefix} --force
-              fi
-            fi
-          done
+          echo "Deleting clusters: ${ci_clusters}"
+          ./dev/delete-cluster.py ${ci_clusters} --force
         shell: bash

.github/workflows/stackhpc.yml

@@ -109,7 +109,6 @@ jobs:
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          ansible-playbook ansible/adhoc/generate-passwords.yml
           echo vault_demo_user_password: "$DEMO_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
@@ -135,6 +134,7 @@ jobs:
           . venv/bin/activate
           . environments/.stackhpc/activate
           ansible all -m wait_for_connection
+          ansible-playbook ansible/adhoc/generate-passwords.yml
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
 
@@ -170,6 +170,7 @@ jobs:
           . venv/bin/activate
           . environments/.stackhpc/activate
           ansible all -m wait_for_connection
+          ansible-playbook ansible/adhoc/generate-passwords.yml
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
 

ansible/.gitignore

@@ -88,3 +88,5 @@ roles/*
 !roles/slurm_tools/**
 !roles/gateway/
 !roles/gateway/**
+!roles/alertmanager/
+!roles/alertmanager/**

ansible/fatimage.yml

@@ -178,6 +178,12 @@
         slurm_exporter_state: stopped
       when: "'slurm_exporter' in group_names"
 
+    - name: Install alertmanager
+      include_role:
+        name: alertmanager
+        tasks_from: install.yml
+      when: "'alertmanager' in group_names"
+
 - hosts: prometheus
   become: yes
   gather_facts: yes

ansible/filter_plugins/utils.py

@@ -11,16 +11,26 @@
 import os.path
 import re
 
-def prometheus_node_exporter_targets(hosts, env):
+def prometheus_node_exporter_targets(hosts, hostvars, env_key, group):
+    """ Return a mapping in cloudalchemy.nodeexporter prometheus_targets
+        format.
+
+        hosts: list of inventory_hostnames
+        hostvars: Ansible hostvars variable
+        env_key: key to lookup in each host's hostvars to add as label 'env' (default: 'ungrouped')
+        group: string to add as label 'group'
+    """
     result = []
     per_env = defaultdict(list)
     for host in hosts:
-        per_env[env].append(host)
+        host_env = hostvars[host].get(env_key, 'ungrouped')
+        per_env[host_env].append(host)
     for env, hosts in per_env.items():
         target = {
-            "targets": ["{target}:9100".format(target=target) for target in hosts],
+            "targets": [f"{target}:9100" for target in hosts],
             "labels": {
-                "env": env
+                'env': env,
+                'group': group
             }
         }
         result.append(target)

ansible/monitoring.yml

@@ -86,3 +86,14 @@
         grafana_dashboards: []
     - import_role: # done in same play so it can use handlers from cloudalchemy.grafana
         name: grafana-dashboards
+
+- name: Deploy alertmanager
+  hosts: alertmanager
+  tags: alertmanager
+  become: yes
+  gather_facts: false
+  tasks:
+    - name: Configure alertmanager
+      include_role:
+        name: alertmanager
+        tasks_from: configure.yml

ansible/roles/alertmanager/README.md

@@ -0,0 +1,97 @@
+# alertmanager
+
+Deploy [alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)
+to route Prometheus alerts to a receiver. Currently Slack is the only supported
+receiver.
+
+Note that:
+- HA configuration is not supported
+- Alertmanager state is not preserved when the node it runs on (by default,
+  control node) is reimaged, so any alerts silenced via the GUI will reoccur.
+- No Grafana dashboard for alerts is currently provided.
+
+Alertmanager is enabled by default on the `control` node in the
+[everything](../../../environments/common/layouts/everything) template which
+`cookiecutter` uses for a new environment's `inventory/groups` file.
+
+In general usage may only require:
+- Adding the `control` node into the `alertmanager` group in `environments/site/groups`
+  if upgrading an existing environment.
+- Enabling the Slack integration (see section below).
+- Possibly setting `alertmanager_web_external_url`.
+
+The web UI is available on `alertmanager_web_external_url`.
+
+## Role variables
+
+All variables are optional. See [defaults/main.yml](defaults/main.yml) for
+all default values.
+
+General variables:
+- `alertmanager_version`: String, version (no leading 'v')
+- `alertmanager_download_checksum`: String, checksum for relevant version from
+  [prometheus.io download page](https://prometheus.io/download/), in format
+  `type:value`.
+- `alertmanager_download_dest`: String, path of temporary directory used for
+  download. Must exist.
+- `alertmanager_binary_dir`: String, path of directory to install alertmanager
+  binary to. Must exist.
+- `alertmanager_started`: Bool, whether the alertmanager service should be started.
+- `alertmanager_enabled`: Bool, whether the alertmanager service should be enabled.
+- `alertmanager_system_user`: String, name of user to run alertmanager as. Will be created.
+- `alertmanager_system_group`: String, name of group of alertmanager user.
+- `alertmanager_port`: Port to listen on.
+
+The following variables are equivalent to similarly-named arguments to the
+`alertmanager` binary. See `man alertmanager` for more info:
+
+- `alertmanager_config_file`: String, path the main alertmanager config file
+  will be written to. Parent directory will be created if necessary. 
+- `alertmanager_web_config_file`: String, path alertmanager web config file
+  will be written to. Parent directory will be created if necessary. 
+- `alertmanager_storage_path`: String, base path for data storage.
+- `alertmanager_web_listen_addresses`: List of strings, defining addresses to listeen on.
+- `alertmanager_web_external_url`: String, the URL under which Alertmanager is
+   externally reachable - defaults to host IP address and `alertmanager_port`.
+   See man page for more details if proxying alertmanager.
+- `alertmanager_data_retention`: String, how long to keep data for
+- `alertmanager_data_maintenance_interval`: String, interval between garbage
+  collection and snapshotting to disk of the silences and the notification logs.
+- `alertmanager_config_flags`: Mapping. Keys/values in here are written to the
+  alertmanager commandline as `--{{ key }}={{ value }}`.
+- `alertmanager_default_receivers`:
+
+The following variables are templated into the alertmanager [main configuration](https://prometheus.io/docs/alerting/latest/configuration/):
+- `alertmanager_config_template`: String, path to configuration template. The default
+  is to template in `alertmanager_config_default` and `alertmanager_config_extra`.
+- `alertmanager_config_default`: Mapping with default configuration for the
+  top-level `route` and `receivers` keys. The default is to send all alerts to
+  the Slack receiver, if that has been enabled (see below).
+- `alertmanager_receivers`: A list of [receiver](https://prometheus.io/docs/alerting/)
+  mappings to define under the top-level `receivers` configuration key. This
+  will contain the Slack receiver if that has been enabled (see below).
+- `alertmanager_extra_receivers`: A list of additional [receiver](https://prometheus.io/docs/alerting/),
+  mappings to add, by default empty.
+- `alertmanager_slack_receiver`: Mapping defining the [Slack receiver](https://prometheus.io/docs/alerting/latest/configuration/#slack_config). Note the default configuration for this is in
+`environments/common/inventory/group_vars/all/alertmanager.yml`.
+- `alertmanager_slack_receiver_name`: String, name for the above Slack reciever.
+- `alertmanager_slack_receiver_send_resolved`: Bool, whether to send resolved alerts via the above Slack reciever.
+- `alertmanager_null_receiver`:  Mapping defining a `null` [receiver](https://prometheus.io/docs/alerting/latest/configuration/#receiver) so a receiver is always defined.
+- `alertmanager_config_extra`: Mapping with additional configuration. Keys in
+  this become top-level keys in the configuration. E.g this might be:
+    ```yaml
+    alertmanager_config_extra:
+    global:
+        smtp_from: smtp.example.org:587
+    time_intervals:
+    - name: monday-to-friday
+        time_intervals:
+        - weekdays: ['monday:friday']
+    ```
+  Note that `route` and `receivers` keys should not be added here.
+
+The following variables are templated into the alertmanager [web configuration](https://prometheus.io/docs/alerting/latest/https/):
+- `alertmanager_web_config_default`: Mapping with default configuration for
+  `basic_auth_users` providing the default web user.
+- `alertmanager_alertmanager_web_config_extra`: Mapping with additional web
+  configuration. Keys in this become top-level keys in the web configuration.

ansible/roles/alertmanager/defaults/main.yml

@@ -0,0 +1,50 @@
+alertmanager_version: '0.28.1'
+alertmanager_download_checksum: 'sha256:5ac7ab5e4b8ee5ce4d8fb0988f9cb275efcc3f181b4b408179fafee121693311'
+alertmanager_download_dest: /tmp/alertmanager.tar.gz
+alertmanager_binary_dir: /usr/local/bin
+alertmanager_started: true
+alertmanager_enabled: true
+
+alertmanager_system_user: alertmanager
+alertmanager_system_group: "{{ alertmanager_system_user }}"
+alertmanager_config_file: /etc/alertmanager/alertmanager.yml
+alertmanager_web_config_file: /etc/alertmanager/alertmanager-web.yml
+alertmanager_storage_path: /var/lib/alertmanager
+
+alertmanager_port: '9093'
+alertmanager_web_listen_addresses:
+  - ":{{ alertmanager_port }}"
+alertmanager_web_external_url: '' # defined in environments/common/inventory/group_vars/all/alertmanager.yml for visibility
+
+alertmanager_data_retention: '120h'
+alertmanager_data_maintenance_interval: '15m'
+alertmanager_config_flags: {} # other command-line parameters as shown by `man alertmanager`
+alertmanager_config_template: alertmanager.yml.j2
+alertmanager_web_config_template: alertmanager-web.yml.j2
+
+alertmanager_web_config_default:
+  basic_auth_users:
+    alertmanager: "{{ vault_alertmanager_admin_password | password_hash('bcrypt', '1234567890123456789012', ident='2b') }}"
+alertmanager_alertmanager_web_config_extra: {} # top-level only
+
+# Variables below are interpolated into alertmanager_config_default:
+
+# Uncomment below and add Slack bot app creds for Slack integration
+# alertmanager_slack_integration:
+#   channel: '#alerts'
+#   app_creds:
+
+alertmanager_null_receiver:
+  name: 'null'
+alertmanager_slack_receiver: {} # defined in environments/common/inventory/group_vars/all/alertmanager.yml as it needs prometheus_address
+alertmanager_extra_receivers: []
+alertmanager_default_receivers: "{{ [alertmanager_null_receiver] + ([alertmanager_slack_receiver] if alertmanager_slack_integration is defined else []) }}"
+alertmanager_receivers: "{{ alertmanager_default_receivers  + alertmanager_extra_receivers }}"
+
+alertmanager_config_default:
+  route:
+    group_by: ['...']
+    receiver: "{{ alertmanager_slack_receiver_name if alertmanager_slack_integration is defined else 'null' }}"
+  receivers: "{{ alertmanager_receivers }}"
+
+alertmanager_config_extra: {} # top-level only

ansible/roles/alertmanager/handlers/main.yml

@@ -0,0 +1,6 @@
+- name: Restart alertmanager
+  systemd:
+    name: alertmanager
+    state: restarted
+    daemon_reload: "{{ _alertmanager_service.changed | default(false) }}"
+  when: alertmanager_started | bool

ansible/roles/alertmanager/tasks/configure.yml

@@ -0,0 +1,47 @@
+- name: Create alertmanager directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ alertmanager_system_user }}"
+    group: "{{ alertmanager_system_group }}"
+    mode: u=rwX,go=rX
+  loop:
+    - "{{ alertmanager_config_file | dirname }}"
+    - "{{ alertmanager_web_config_file | dirname }}"
+    - "{{ alertmanager_storage_path }}"
+
+- name: Create alertmanager service file with immutable options
+  template:
+    src: alertmanager.service.j2
+    dest: /usr/lib/systemd/system/alertmanager.service
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  register: _alertmanager_service
+  notify: Restart alertmanager
+
+- name: Template alertmanager config
+  ansible.builtin.template:
+    src: "{{ alertmanager_config_template }}"
+    dest: "{{ alertmanager_config_file }}"
+    owner: "{{ alertmanager_system_user }}"
+    group: "{{ alertmanager_system_group }}"
+    mode: u=rw,go=
+  notify: Restart alertmanager
+
+- name: Template alertmanager web config
+  ansible.builtin.template:
+    src: "{{ alertmanager_web_config_template }}"
+    dest: "{{ alertmanager_web_config_file }}"
+    owner: "{{ alertmanager_system_user }}"
+    group: "{{ alertmanager_system_group }}"
+    mode: u=rw,go=
+  notify: Restart alertmanager
+
+- meta: flush_handlers
+
+- name: Ensure alertmanager service state
+  systemd:
+    name: alertmanager
+    state: "{{ 'started' if alertmanager_started | bool else 'stopped' }}"
+    enabled: "{{ alertmanager_enabled | bool }}"