fix: refactor prometheus provisioning

Prometheus was using the clients dir along with metrics-exporter
service to provision the scrape targets.
Since the directory has been deprecated now the prometheus reads
the list of targets directly from the API server using the authenticated
/prometheus/targets endpoint.

List of targets is updated every 60 seconds.

This change also fixes a previously existing bug:
obsolete targets were not removed from prometheus target list.

Author: gsanchietti

README.md

@@ -77,9 +77,7 @@ The module is composed by the following systemd units:
 - prometheus.service: runs the prometheus container
 - loki.service: runs the loki container
 - grafana.service: runs the grafana container
-- metrics-exporter.path: watch for vpn connections from vpn.service and start metrics-exporter.service; each time a new client connects, the vpn
   container creates a file inside the `prometheus.d/` directory
-- metrics-exporter.service: executes the `metrics_exporter_handler` script to create a new prometheus target for the connected machine
 - webssh.service: runs the webssh container
 
 ### API Server
@@ -114,9 +112,7 @@ Promtail sets the following labels:
 [Prometheus](https://prometheus.io/) is a metrics collector, it scrapes metrics from the connected machines. The configuration is available at `/home/nethsecurity-controller1/.config/state/prometheus.yml` and it's generated every time by the `configure-module` action.
 It has a the following targets:
 - static target with job_name `loki` that scrapes Loki metrics
-- dynamic targets with job_name `node` that scrapes metrics from the connected machines from the `prometheus.d/` directory under the state directory (eg. `/home/nethsecurity-controller1/.config/state/prometheus.d`)
-
-Each dynamic target is created by the `metrics-exporter` and has the following labels:
+- dynamic targets with job_name `node` that scrapes metrics from the connected machines from the API server `http://<prometheus_user>:<prometheus_pass>@127.0.0.1:<API_PORT/prometheus/targets`
 
 - `instance` the VPN IP of the connected machine with the netdata port (eg. `172.19.64.3:19999`)
 - `job` fixed to `node`

imageroot/actions/clone-module/20initialize

@@ -40,5 +40,3 @@ EOF
 # replace the ports in the db.env file
 sed -i "s/^POSTGRES_PORT=.*/POSTGRES_PORT=$db_port/" db.env
 sed -i "s|^\(REPORT_DB_URI=postgres://report:[^@]*@127.0.0.1:\)[0-9]\{1,\}|\1$db_port|" db.env
-
-mkdir -p clients

imageroot/actions/configure-module/20configure

@@ -153,18 +153,6 @@ with open('prometheus.env', 'w') as pfp:
     pfp.write(f"PROMETHEUS_PATH={config['prometheus_path']}\n")
     pfp.write(f"PROMETHEUS_RETENTION={metrics_retention_days}d\n")
 
-with open('prometheus.yml', 'w', encoding='utf-8') as fp:
-    fp.write("global:\n")
-    fp.write("scrape_configs:\n")
-    fp.write('  - job_name: "node"\n')
-    fp.write('    file_sd_configs:\n')
-    fp.write('      - files:\n')
-    fp.write('        - "/prometheus/prometheus.d/*.yml"\n')
-    fp.write('  - job_name: "loki"\n')
-    fp.write('    static_configs:\n')
-    fp.write('      - targets:\n')
-    fp.write(f'        - 127.0.0.1:{ports[5]}\n')
-
 # Grafana configuration
 db = agent.read_envfile('db.env')
 with open('grafana.yml', 'w') as fp:

imageroot/actions/create-module/20initialize

@@ -26,6 +26,7 @@ num=$(echo $MODULE_ID | sed 's/nethsecurity\-controller//')
 jwt_secret=$(uuidgen | sha256sum | awk '{print $1}')
 reg_secret=$(uuidgen | sha256sum | awk '{print $1}')
 db_secret=$(uuidgen | sha256sum | awk '{print $1}')
+prometheus_secret=$(uuidgen | sha256sum | awk '{print $1}')
 encryption_key=$(uuidgen | sha256sum | awk '{print substr($1,1,32)}')
 grafana_postgres_password=$(uuidgen | sha256sum | awk '{print $1}')
 
@@ -46,6 +47,8 @@ EOF
 cat << EOF > secret.env
 REGISTRATION_TOKEN=$reg_secret
 ENCRYPTION_KEY=$encryption_key
+PROMETHEUS_AUTH_USERNAME=prometheus_$RANDOM
+PROMETHEUS_AUTH_PASSWORD=$prometheus_secret
 EOF
 
 cat << EOF > db.env
@@ -64,5 +67,3 @@ EOF
 
 # This will be compiled inside the configure-module
 touch platform.env
-
-mkdir -p clients

imageroot/bin/expand-prometheus-config

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2025 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import agent
+
+network = agent.read_envfile('network.env')
+api_port = network.get('API_PORT')
+
+secret = agent.read_envfile('secret.env')
+prometheus_user = secret.get('PROMETHEUS_AUTH_USERNAME', 'prometheus')
+prometheus_pass = secret.get('PROMETHEUS_AUTH_PASSWORD', 'prometheus')
+
+loki = agent.read_envfile('loki.env')
+loki_port = loki.get('LOKI_HTTP_PORT')
+
+with open('prometheus.yml', 'w') as f:
+    f.write('global:\n')
+    f.write('scrape_configs:\n')
+    f.write('  - job_name: "node"\n')
+    f.write('    http_sd_configs:\n')
+    f.write(f'      - url: "http://127.0.0.1:{api_port}/prometheus/targets"\n')
+    f.write('        refresh_interval: 60s\n')
+    f.write('        basic_auth:\n')
+    f.write(f'          username: "{prometheus_user}"\n')
+    f.write(f'          password: "{prometheus_pass}"\n')
+    f.write('  - job_name: "loki"\n')
+    f.write('    static_configs:\n')
+    f.write('      - targets:\n')
+    f.write(f'        - "127.0.0.1:{loki_port}"\n')

imageroot/bin/metrics_exporter_handler

@@ -2,7 +2,7 @@
 [Unit]
 Description=Podman api.service
 BindsTo=controller.service
-After=controller.service vpn.service
+After=controller.service vpn.service timescale.service
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n

imageroot/systemd/user/api.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Podman controller.service
-Requires=vpn.service api.service ui.service proxy.service promtail.service metrics-exporter.path loki.service prometheus.service grafana.service webssh.service timescale.service
-Before=vpn.service api.service ui.service proxy.service promtail.service metrics-exporter.path loki.service prometheus.service grafana.service webssh.service timescale.service
+Requires=vpn.service api.service ui.service proxy.service promtail.service loki.service prometheus.service grafana.service webssh.service timescale.service
+Before=vpn.service api.service ui.service proxy.service promtail.service loki.service prometheus.service grafana.service webssh.service timescale.service
 ConditionPathExists=%S/state/environment
 ConditionPathExists=%S/state/network.env