WIP: add alertmanager

Author: sjpb

ansible/.gitignore

@@ -88,3 +88,5 @@ roles/*
 !roles/slurm_tools/**
 !roles/gateway/
 !roles/gateway/**
+!roles/alertmanager/
+!roles/alertmanager/**

ansible/monitoring.yml

@@ -86,3 +86,19 @@
         grafana_dashboards: []
     - import_role: # done in same play so it can use handlers from cloudalchemy.grafana
         name: grafana-dashboards
+
+- name: Deploy alertmanager
+  hosts: alertmanager
+  tags: alertmanager
+  become: yes
+  gather_facts: false
+  tasks:
+    # TODO: move elsewhere, still needs become
+    - name: Install alertmanager
+      include_role:
+        name: alertmanager
+        tasks_from: install.yml
+    - name: Configure alertmanager
+      include_role:
+        name: alertmanager
+        tasks_from: configure.yml

ansible/roles/alertmanager/README.md

@@ -0,0 +1,6 @@
+# alertmanager
+
+
+notes:
+- HA is not supported
+- state ("notification state and configured silences") is not preserved across rebuild

ansible/roles/alertmanager/defaults/main.yml

@@ -0,0 +1,45 @@
+alertmanager_version: '0.28.1'
+alertmanager_download_checksum: 'sha256:5ac7ab5e4b8ee5ce4d8fb0988f9cb275efcc3f181b4b408179fafee121693311'
+alertmanager_download_dest: /tmp/alertmanager.tar.gz
+alertmanager_binary_dir: /usr/local/bin
+alertmanager_started: true
+alertmanager_enabled: true
+
+alertmanager_system_user: alertmanager
+alertmanager_system_group: alertmanager
+alertmanager_config_path: /etc/alertmanager/alertmanager.yml
+alertmanager_storage_dir: /var/lib/alertmanager
+alertmanager_web_listen_addresses:
+  - ':9100'
+alertmanager_web_external_url: http://localhost:9093/
+alertmanager_config_flags: {}
+# TODO: data retention?
+alertmanager_config_template: alertmanager.yml.j2
+
+
+# everything below here is interpolated into alertmanager_config_default:
+
+# Uncomment below and add Slack bot app creds for Slack integration
+# alertmanager_slack_integration:
+#   channel: '#alerts'
+#   app_creds:
+
+
+alertmanager_default_receivers:
+  - name: 'null'
+
+alertmanager_slack_receiver: {} # really defined in common as it needs prometheus_address
+
+alertmanager_extra_receivers: "{{ [alertmanager_slack_receiver] if alertmanager_slack_integration is defined else [] }}"
+
+alertmanager_config_default:
+  route:
+    group_by: ['...']
+    receiver: "{{ 'slack-receiver' if alertmanager_slack_integration is defined else 'null' }}"
+  receivers: "{{ alertmanager_default_receivers  + alertmanager_extra_receivers }}"
+
+alertmanager_config_extra: {} # top-level only
+
+
+# TODO: routes??
+# TODO: see PR with additional alerts

ansible/roles/alertmanager/handlers/main.yml

@@ -0,0 +1,6 @@
+- name: Restart alertmanager
+  systemd:
+    name: alertmanager
+    state: restarted
+    daemon_reload: "{{ _alertmanager_service.changed | default(false) }}"
+  when: alertmanager_started | bool

ansible/roles/alertmanager/tasks/configure.yml

@@ -0,0 +1,40 @@
+- name: Create alertmanager directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ alertmanager_system_user }}"
+    group: "{{ alertmanager_system_group }}"
+    mode: u=rwX,go=rX
+  loop:
+    - "{{ alertmanager_config_path | dirname }}"
+    - "{{ alertmanager_storage_dir }}"
+
+# TODO: selinux?
+
+- name: Create alertmanager service file with immutable options
+  template:
+    src: alertmanager.service.j2
+    dest: /usr/lib/systemd/system/alertmanager.service
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  register: _alertmanager_service
+  notify: Restart alertmanager
+  # TODO: how do we cope with the binary changing?
+
+- name: Template alertmanager config
+  ansible.builtin.template:
+    src: "{{ alertmanager_config_template }}"
+    dest: "{{ alertmanager_config_path }}"
+    owner: "{{ alertmanager_system_user }}"
+    group: "{{ alertmanager_system_group }}"
+    mode: u=rw,go=r # TODO: check there are no sensitive things in here!
+  notify: Restart alertmanager
+
+- meta: flush_handlers
+
+- name: Ensure alertmanager service state
+  systemd:
+    name: alertmanager
+    state: "{{ 'started' if alertmanager_started | bool else 'stopped' }}"
+    enabled: "{{ alertmanager_enabled | bool }}"

ansible/roles/alertmanager/tasks/install.yml

@@ -0,0 +1,25 @@
+- name: Create alertmanager system user
+  ansible.builtin.user:
+    name: "{{ alertmanager_system_user }}"
+    system: true
+    create_home: false
+
+- name: Download alertmanager binary
+  ansible.builtin.get_url:
+    url: "https://github.com/prometheus/alertmanager/releases/download/v{{ alertmanager_version }}/alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz"
+    dest: "{{ alertmanager_download_dest }}"
+    owner: root
+    group: root
+    mode: u=rw,go=
+    checksum: "{{ alertmanager_download_checksum }}"
+
+- name: Unpack alertmanager binary
+  ansible.builtin.unarchive:
+    src: "{{ alertmanager_download_dest }}"
+    include: "alertmanager-{{ alertmanager_version }}.linux-amd64/alertmanager"
+    dest: "{{ alertmanager_binary_dir }}"
+    owner: root
+    group: root
+    mode: u=rwx,go=rx
+    remote_src: true
+    extra_opts: ['--strip-components=1', '--show-stored-names']

ansible/roles/alertmanager/tasks/main.yml

@@ -0,0 +1,50 @@
+
+
+
+{{ ansible_managed | comment }}
+[Unit]
+Description=Prometheus Alertmanager
+After=network-online.target
+StartLimitInterval=0
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+PIDFile=/run/alertmanager.pid
+User={{ alertmanager_system_user }}
+Group={{ alertmanager_system_group }}
+ExecReload=/bin/kill -HUP $MAINPID
+ExecStart={{ alertmanager_binary_dir }}/alertmanager \
+  --cluster.listen-address='' \
+  --config.file={{ alertmanager_config_path }} \
+  --storage.path={{ alertmanager_storage_dir }} \
+{% for address in alertmanager_web_listen_addresses %}
+  --web.listen-address={{ address }} \
+{% endfor %}
+  --web.external-url={{ alertmanager_web_external_url }} \
+{% for flag, flag_value in alertmanager_config_flags.items() %}
+  --{{ flag }}={{ flag_value }} \
+{% endfor %}
+
+SyslogIdentifier=alertmanager
+Restart=always
+RestartSec=5
+
+CapabilityBoundingSet=CAP_SET_UID
+LockPersonality=true
+NoNewPrivileges=true
+MemoryDenyWriteExecute=true
+PrivateTmp=true
+ProtectHome=true
+ReadWriteDirectories={{ alertmanager_storage_dir }}
+RemoveIPC=true
+RestrictSUIDSGID=true
+
+PrivateUsers=true
+ProtectControlGroups=true
+ProtectKernelModules=true
+ProtectKernelTunables=yes
+ProtectSystem=strict
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/alertmanager/templates/alertmanager.service.j2

@@ -0,0 +1,4 @@
+{{ ansible_managed | comment }}
+
+{{ alertmanager_config_default }}
+{{ alertmanager_config_extra }}