diff --git a/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 b/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 index d7ba872f..8246bf75 100644 --- a/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 +++ b/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 @@ -5,11 +5,17 @@ global: scrape_configs: - job_name: 'grafana' static_configs: - - targets: ['localhost:{{ grafana_port }}'] + - targets: ['localhost:3000'] - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] - job_name: 'db' + basic_auth: + username: "prometheus" + password: "{{ PROMETHEUS_PASS }}" + tls_config: + insecure_skip_verify: true + scheme: https static_configs: - targets: {% for host in groups["domserver"] %} @@ -137,6 +143,12 @@ scrape_configs: - {{ hostvars[host].ansible_host }}:9113 {% endfor %} - job_name: 'web_fpm_domserver' + basic_auth: + username: "prometheus" + password: "{{ PROMETHEUS_PASS }}" + tls_config: + insecure_skip_verify: true + scheme: https static_configs: - targets: {% for host in groups["domserver"] %} diff --git a/provision-contest/ansible/roles/prometheus_target_all/files/.gitignore b/provision-contest/ansible/roles/prometheus_target_all/files/.gitignore new file mode 100644 index 00000000..275ece24 --- /dev/null +++ b/provision-contest/ansible/roles/prometheus_target_all/files/.gitignore @@ -0,0 +1,2 @@ +*.key +*.crt diff --git a/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml b/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml index a509ef4c..7b2f81dc 100644 --- a/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml +++ b/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml @@ -28,9 +28,18 @@ - name: Get HTPassword delegate_to: localhost + become: false shell: "echo {{ PROMETHEUS_PASS }} | htpasswd -inBC 10 \"\" | tr -d ':\n'" register: htpassd_shell +- name: Store HTPassword for nginx wrapper + copy: + content: "prometheus:{{ htpassd_shell.stdout }}" + dest: /etc/prometheus/.htpasswd + owner: root + group: root + mode: 0644 + - name: Set certificate to encrypt node_exporter traffic template: owner: prometheus diff --git a/provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2 b/provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2 new file mode 100644 index 00000000..4d1a35ac --- /dev/null +++ b/provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2 @@ -0,0 +1,5 @@ +basic_auth_users: + prometheus: {{ htpassd_shell.stdout }} +tls_server_config: + cert_file: /etc/prometheus/node_exporter.crt + key_file: /etc/prometheus/node_exporter.key diff --git a/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service b/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service index 94e5185e..9d6ca255 100644 --- a/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service +++ b/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service @@ -4,7 +4,7 @@ Documentation=https://github.com/hipages/php-fpm_exporter [Service] User=www-data Restart=always -ExecStart=/usr/bin/php-fpm_exporter server --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status +ExecStart=/usr/bin/php-fpm_exporter server --web.listen-address :19253 --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status ExecReload=/bin/kill -HUP $MAINPID TimeoutStopSec=20s SendSIGKILL=no diff --git a/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml b/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml index 069520ae..609c7db8 100644 --- a/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml +++ b/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml @@ -18,11 +18,11 @@ - name: Scrape mysql exporter with TLS encryption lineinfile: - dest: /etc/default/prometheus-mysqld-exporter + dest: /etc/default/prometheus-mysqld-exporter state: present regexp: '^ARGS=""' - line: 'ARGS="--web.config /etc/prometheus/prometheus-authentication.yml"' - notify: Restart mysqld-exporter + line: 'ARGS="--web.config.file /etc/prometheus/prometheus-authentication.yml"' + notify: Restart mysqld-exporter # Gather PHP-FPM statistics # The exporter from this is currently not in deb sources @@ -66,9 +66,10 @@ # Gather NGINX statistics, # Observe that we use the observed process itself in the monitoring - name: Get NGINX status - synchronize: - src: nginx-status.conf + template: + src: nginx-status.conf.j2 dest: /etc/nginx/sites-enabled/nginx-status.conf + mode: 0644 notify: Restart nginx # In the future add: --web.config /etc/prometheus/prometheus-authentication.yml"' @@ -79,7 +80,7 @@ dest: /etc/default/prometheus-nginx-exporter state: present regexp: '^ARGS=""' - line: 'ARGS="-nginx.scrape-uri=http://localhost:8787/basic_status"' + line: 'ARGS="-web.listen-address=127.0.0.1:19113 -nginx.scrape-uri=http://localhost:8787/basic_status"' notify: Restart nginx-exporter - name: Create storage dir for exporter settings diff --git a/provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2 b/provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2 new file mode 100644 index 00000000..e5397fdb --- /dev/null +++ b/provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2 @@ -0,0 +1,34 @@ +server { + listen 127.0.0.1:8787; + server_name _default_; + + location = /basic_status { + stub_status; + } +} + +server { + listen 0.0.0.0:9113 ssl; + ssl_certificate /etc/prometheus/node_exporter.crt; + ssl_certificate_key /etc/prometheus/node_exporter.key; + ssl_protocols TLSv1.3; + + auth_basic "Prometheus scraping"; + auth_basic_user_file /etc/prometheus/.htpasswd; + location / { + proxy_pass http://127.0.0.1:19113; + } +} + +server { + listen 0.0.0.0:9253 ssl; + ssl_certificate /etc/prometheus/node_exporter.crt; + ssl_certificate_key /etc/prometheus/node_exporter.key; + ssl_protocols TLSv1.3; + + auth_basic "Prometheus scraping"; + auth_basic_user_file /etc/prometheus/.htpasswd; + location / { + proxy_pass http://127.0.0.1:19253; + } +}