Monitor all machines with grafana/prometheus

vmcj · vmcj · commit 611a8f6ed2d1 · 2021-11-20T16:18:59.000Z
Default this needs an additional machine so we don't install unless its
an actual choice.

Collectd is superseded by prometheus which runs on localhost so the
variable is not needed anymore.
diff --git a/icpc-wf/ansible/admin.yml b/icpc-wf/ansible/admin.yml
@@ -38,6 +38,9 @@
       tags: clusterssh
     - role: phpstorm
       tags: phpstorm
+    - role: prometheus_target_all
+      tags: prometheus_target_all
+      when: GRAFANA_MONITORING
   handlers:
     - include: handlers.yml
   tasks:
diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml
@@ -36,6 +36,12 @@
       tags: mysql_replication
     - role: keepalived
       tags: keepalived
+    - role: prometheus_target_web
+      tags: prometheus_target_web
+      when: GRAFANA_MONITORING
+    - role: prometheus_target_all
+      tags: prometheus_target_all
+      when: GRAFANA_MONITORING
   handlers:
     - include: handlers.yml
   tasks:
diff --git a/icpc-wf/ansible/grafana.yml b/icpc-wf/ansible/grafana.yml
@@ -5,6 +5,9 @@
     host_type: grafana
   become: yes
   tasks:
+    - role: prometheus_target_all
+      tags: prometheus_target_all
+      when: GRAFANA_MONITORING
     - name: include global variables
       include_vars: variables.yml
     - name: include global (secret) variables
diff --git a/icpc-wf/ansible/group_vars/all/all.yml b/icpc-wf/ansible/group_vars/all/all.yml
@@ -42,3 +42,5 @@ DJ_GIT_REPO_SCRIPTS: "domjudge@{{DJ_GIT_HOST}}:domjudge-scripts-bare"
 
 PHPSTORM_VERSION: 2021.2
 PHPSTORM_FULL_VERSION: 212.5284.49
+
+GRAFANA_MONITORING: false
diff --git a/icpc-wf/ansible/group_vars/all/secret.yml.example b/icpc-wf/ansible/group_vars/all/secret.yml.example
@@ -2,10 +2,6 @@
 # Set this to enable master-master replication between two domservers.
 REPLICATION_PASSWORD: some-replication-password
 
-# Password for the collectd MySQL status user.
-# Set this to enable collectd for system statistics collection.
-COLLECTD_MYSQL_PASSWORD: password-here
-
 # Database user password.
 DB_PASSWORD: some-database-password
 
diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml
@@ -12,6 +12,9 @@
     - role: icpc_fixes
       tags: icpc_fixes
       when: ICPC_IMAGE
+    - role: prometheus_target_all
+      tags: prometheus_target_all
+      when: GRAFANA_MONITORING
     - role: system_fixes
       tags: system_fixes
     - role: hosts
@@ -30,6 +33,9 @@
       tags: ssh
     - role: domlogo
       tags: domlogo
+    - role: prometheus_target_all
+      tags: prometheus_target_all
+      when: GRAFANA_MONITORING
   pre_tasks:
     - pause:
         prompt: "Did you disable all judges"
diff --git a/icpc-wf/ansible/roles/prometheus_target_all/tasks/main.yml b/icpc-wf/ansible/roles/prometheus_target_all/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+# This tasks gathers default system metrics.
+
+- name: install required packages
+  apt:
+    state: present
+    pkg:
+      - prometheus-node-exporter
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/files/.gitignore b/icpc-wf/ansible/roles/prometheus_target_web/files/.gitignore
@@ -0,0 +1 @@
+/php-fpm_exporter*.tar.gz
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/files/README.md b/icpc-wf/ansible/roles/prometheus_target_web/files/README.md
@@ -0,0 +1,2 @@
+Add to this directory:
+php-fpm_exporter.tar.gz (See https://github.com/hipages/php-fpm_exporter/releases)
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/files/nginx-status.conf b/icpc-wf/ansible/roles/prometheus_target_web/files/nginx-status.conf
@@ -0,0 +1,9 @@
+server {
+       listen   8080;
+       listen   [::]:8080;
+       server_name _default_;
+
+       location = /basic_status {
+               stub_status;
+       }
+}
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service b/icpc-wf/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/handlers/main.yml b/icpc-wf/ansible/roles/prometheus_target_web/handlers/main.yml
@@ -0,0 +1,14 @@
+---
+# Define here handlers associated to this role.
+
+- name: restart php-exporter
+  service: name=php-fpm-exporter enabled=yes state=restarted
+
+- name: restart nginx
+  service: name=nginx enabled=yes state=restarted
+
+- name: restart nginx-exporter
+  service: name=prometheus-nginx-exporter enabled=yes state=restarted
+
+- name: restart mysqld-exporter
+  service: name=prometheus-mysqld-exporter enabled=yes state=restarted
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/meta/main.yml b/icpc-wf/ansible/roles/prometheus_target_web/meta/main.yml
@@ -0,0 +1,4 @@
+# Role dependencies
+---
+dependencies:
+  - role: domserver
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/tasks/main.yml b/icpc-wf/ansible/roles/prometheus_target_web/tasks/main.yml
@@ -0,0 +1,51 @@
+---
+# These tasks configure metric collectors
+
+- name: install required packages
+  apt:
+    state: present
+    pkg:
+      - prometheus-mysqld-exporter
+      - prometheus-nginx-exporter
+
+# Gather PHP-FPM statistics
+# The exporter from this is currently not in deb sources
+# so we need to download this from GitHub see the README in files
+- name: Install PHP-fpm exporter binary
+  ansible.builtin.unarchive:
+    src: php-fpm_exporter.tar.gz
+    dest: /usr/bin/
+    exclude:
+      - LICENSE
+      - README.md
+
+- name: Export PHP-FPM metrics
+  synchronize:
+    src=php-fpm-exporter.service
+    dest=/etc/systemd/system/php-fpm-exporter.service
+  notify: restart php-exporter
+
+# Gather NGINX statistics,
+# Observe that we use the observed process itself in the monitoring
+- name: Get NGINX status
+  synchronize:
+    src=nginx-status.conf
+    dest=/etc/nginx/sites-enabled/nginx-status.conf
+  notify: restart nginx
+
+- name: Prometheus nginx exporter
+  lineinfile:
+    dest: /etc/default/prometheus-nginx-exporter
+    state: present
+    regexp: '^ARGS=""'
+    line: 'ARGS="-nginx.scrape-uri=http://localhost:8080/basic_status"'
+  notify: restart nginx-exporter
+
+# Because the scrape happens inside the same machine we reuse the DB password
+# which is also used for the normal installation
+- name: Create MySQL authentication file
+  template:
+    src: mysqld-exporter-authentication.cnf.j2
+    dest: /var/lib/prometheus/.my.cnf
+    owner: prometheus
+  notify: restart mysqld-exporter
diff --git a/icpc-wf/ansible/roles/prometheus_target_web/templates/mysqld-exporter-authentication.cnf.j2 b/icpc-wf/ansible/roles/prometheus_target_web/templates/mysqld-exporter-authentication.cnf.j2
@@ -0,0 +1,5 @@
+[client]
+user = domjudge
+password = {{ DB_PASSWORD }}
+host = localhost
+database = domjudge

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Add to this directory:`
	`2`	`+php-fpm_exporter.tar.gz (See https://github.com/hipages/php-fpm_exporter/releases)`