Merge pull request #216 from stfc/systemd_exporter

ENH: Add systemd exporter to hosts

Author: Dmitry-Popovichev

chatops_deployment/ansible/configure.yml

@@ -1,39 +1,41 @@
 ---
 - name: Configure load balancer
   hosts: loadbalancer
-  remote_user: ubuntu
   roles:
     - role: loadbalancer
       tags:
         - loadbalancer
 
 - name: Configure ChatOps
   hosts: chatops
-  remote_user: ubuntu
   roles:
     - role: chatops
       tags:
         - chatops
 
+- name: Set up systemd exporters
+  hosts: all
+  roles:
+    - role: systemd_exporter
+      tags:
+        - systemd_exporter
+
 - name: Configure Grafana
   hosts: grafana
-  remote_user: ubuntu
   roles:
     - grafana
   tags:
     - grafana
 
 - name: Configure Prometheus
   hosts: prometheus
-  remote_user: ubuntu
   roles:
     - prometheus
   tags:
     - prometheus
 
 - name: Configure Alert Manager
   hosts: prometheus
-  remote_user: ubuntu
   roles:
     - alertmanager
   tags:

chatops_deployment/ansible/group_vars/all/vars.yml

@@ -1,2 +1,4 @@
+ansible_user: ubuntu
 domain: "dev-cloud-chatops.nubes.rl.ac.uk"
 chatops_custom_api_token: "{{ vault_chatops_custom_api_token }}"
+systemd_exporter_version: "0.7.0"

chatops_deployment/ansible/roles/prometheus/files/rules.yml

@@ -9,3 +9,12 @@ groups:
         annotations:
           summary: "ChatOps container is detected down on {{ $labels.instance }}"
           description: "Container has been not running for more than 10 seconds."
+
+      - alert: SystemdServiceDown
+        expr: systemd_unit_state{name=~"grafana-server.service|haproxy.service",state=~"failed|inactive"} == 1
+        for: 30s
+        labels:
+          severity: critical
+        annotations:
+          summary: "Systemd service {{ $labels.name }} on host {{ $labels.instance }} is in state {{ $labels.state }}."
+          description: "Systemd service has been in failed state for 30s."

chatops_deployment/ansible/roles/systemd_exporter/files/systemd-exporter.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Systemd Exporter Service
+Documentation=https://github.com/prometheus-community/systemd_exporter
+After=network-online.target
+
+[Service]
+User=systemd-exporter
+Group=systemd-exporter
+Restart=on-failure
+ExecStart=/opt/systemd-exporter/systemd_exporter --web.telemetry-path="/"
+StandardOutput=append:/var/log/systemd-exporter/systemd-exporter.log
+StandardError=append:/var/log/systemd-exporter/systemd-exporter.log
+
+[Install]
+WantedBy=multi-user.target

chatops_deployment/ansible/roles/systemd_exporter/tasks/main.yml

@@ -0,0 +1,81 @@
+---
+- name: Create systemd-exporter group
+  become: true
+  ansible.builtin.group:
+    name: systemd-exporter
+    state: present
+
+- name: Add ubuntu to systemd-exporter group
+  become: true
+  ansible.builtin.user:
+    name: ubuntu
+    groups: systemd-exporter
+    append: true
+
+- name: Reset connection for group changes
+  ansible.builtin.meta: reset_connection
+
+- name: Create a systemd-exporter user
+  become: true
+  ansible.builtin.user:
+    name: systemd-exporter
+    create_home: false
+    group: systemd-exporter
+    system: true
+
+- name: Download and extract systemd-exporter
+  become: true
+  ansible.builtin.unarchive:
+    src: "
+      https://github.com/prometheus-community/systemd_exporter/releases/download/\
+      v{{ systemd_exporter_version }}/systemd_exporter-{{ systemd_exporter_version }}.linux-amd64.tar.gz
+    "
+    dest: /tmp
+    remote_src: true
+    creates: "/tmp/systemd_exporter-{{ systemd_exporter_version }}.linux-amd64"
+    mode: "0774"
+
+- name: Move systemd-exporter binaries
+  become: true
+  ansible.builtin.copy:
+    src: "/tmp/systemd_exporter-{{ systemd_exporter_version }}.linux-amd64/"
+    dest: "/opt/systemd-exporter"
+    mode: preserve
+    owner: systemd-exporter
+    group: systemd-exporter
+    remote_src: true
+
+- name: Create systemd directory
+  become: true
+  ansible.builtin.file:
+    path: /usr/local/lib/systemd/system
+    state: directory
+    mode: "0755"
+    owner: root
+    group: root
+
+- name: Copy systemd-exporter service file
+  become: true
+  ansible.builtin.copy:
+    src: systemd-exporter.service
+    dest: /usr/local/lib/systemd/system/systemd-exporter.service
+    owner: systemd-exporter
+    group: systemd-exporter
+    mode: "0774"
+
+- name: Create systemd-exporter logging directory
+  become: true
+  ansible.builtin.file:
+    path: /var/log/systemd-exporter
+    state: directory
+    mode: "0774"
+    owner: systemd-exporter
+    group: systemd-exporter
+
+- name: Start systemd-exporter service
+  become: true
+  ansible.builtin.systemd_service:
+    name: systemd-exporter.service
+    state: restarted
+    daemon_reload: true
+    enabled: true

chatops_deployment/terraform/modules/networking/main.tf

@@ -134,3 +134,43 @@ resource "openstack_networking_secgroup_rule_v2" "loadbalancer_prometheus" {
   remote_ip_prefix  = "192.168.100.0/22"
   security_group_id = openstack_networking_secgroup_v2.loadbalancer.id
 }
+
+resource "openstack_networking_secgroup_rule_v2" "systemd_exporter_prometheus" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 9558
+  port_range_max    = 9558
+  remote_ip_prefix  = "192.168.100.0/22"
+  security_group_id = openstack_networking_secgroup_v2.prometheus.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "systemd_exporter_chatops" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 9558
+  port_range_max    = 9558
+  remote_ip_prefix  = "192.168.100.0/22"
+  security_group_id = openstack_networking_secgroup_v2.chatops.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "systemd_exporter_grafana" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 9558
+  port_range_max    = 9558
+  remote_ip_prefix  = "192.168.100.0/22"
+  security_group_id = openstack_networking_secgroup_v2.grafana.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "systemd_exporter_loadbalancer" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 9558
+  port_range_max    = 9558
+  remote_ip_prefix  = "192.168.100.0/22"
+  security_group_id = openstack_networking_secgroup_v2.loadbalancer.id
+}