More config details including alerts

Author: rcowham

README.md

@@ -21,7 +21,7 @@ Two custom components:
 * p4prometheus - This component.
 * monitor_metrics.sh - [SDP](https://swarm.workshop.perforce.com/projects/perforce-software-sdp) compatible bash script to generate simple supplementary metrics - [monitor_metrics.sh](https://swarm.workshop.perforce.com/files/guest/perforce_software/sdp/dev/Server/Unix/p4/common/site/bin/monitor_metrics.sh)
 
-Check out the [Prometheus architecture](https://prometheus.io/assets/architecture.png) - the custom components are "Prometheus targets".
+Check out the ![Prometheus architecture](https://prometheus.io/assets/architecture.png) - the custom components are "Prometheus targets".
 
 # Grafana Dashboards
 
@@ -85,7 +85,7 @@ Ensure the above has global read access (perforce user will write files, node_ex
 
 Create service file:
 
-```bash
+```ini
 cat << EOF > /etc/systemd/system/node_exporter.service
 [Unit]
 Description=Node Exporter
@@ -156,7 +156,7 @@ As user `root`:
 
 Create service file:
 
-```bash
+```ini
 cat << EOF > /etc/systemd/system/p4prometheus.service
 [Unit]
 Description=P4prometheus
@@ -189,3 +189,153 @@ Check that metrics are being written:
 
     cat /hxlogs/metrics/p4_cmds.prom
 
+# Alerting
+
+Done via alertmanager
+
+Setup is very similar to the above.
+
+Sample `/etc/systemd/system/alertmanager.service`:
+
+```ini
+[Unit]
+Description=Alertmanager
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=alertmanager
+Group=alertmanager
+Type=simple
+ExecStart=/usr/local/bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yml --storage.path=/var/lib/alertmanager --log.level=debug
+
+[Install]
+WantedBy=multi-user.target
+```
+
+* create alertmanager user
+* create /etc/alertmanager directory
+
+
+## Prometheus config
+
+```yaml
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+        - localhost:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  - "perforce_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+    - targets: ['localhost:9090']
+
+  - job_name: 'node_exporter'
+    static_configs:
+    - targets: ['p4hms:9100', 'p4main:9100', 'p4_ha:9100']
+
+```
+
+## Alerting rules
+
+This is an example, assuming simple email and local postfix or equivalent setup.
+
+```yaml
+groups:
+- name: alert.rules
+  rules:
+  - alert: NoLogs
+    expr: 100 > rate(p4_prom_log_lines_read{sdpinst="1",serverid="master"}[1m])
+    for: 1m
+    labels:
+      severity: "critical"
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} too few log lines"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been below target for more than 1 minutes."
+  - alert: Replication Slow HA
+    expr: p4_replica_curr_pos{instance="p4master:9100",job="node_exporter",sdpinst="1",servername="master"} - ignoring(serverid,servername) p4_replica_curr_pos{instance="p4master:9100",job="node_exporter",sdpinst="1",servername="p4d_ha_bos"} > 5e+7
+    for: 10m
+    labels:
+      severity: "warning"
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} replication warning"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been above target for more than 1 minutes."
+  - alert: Replication Slow London
+    expr: p4_replica_curr_pos{instance="p4master:9100",job="node_exporter",sdpinst="1",servername="master"} - ignoring(serverid,servername) p4_replica_curr_pos{instance="p4master:9100",job="node_exporter",sdpinst="1",servername="p4d_fr_lon"} > 5e+7
+    for: 10m
+    labels:
+      severity: "warning"
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} replication warning"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been above target for more than 1 minutes."
+  - alert: Checkpoint slow
+    expr: p4_sdp_checkpoint_duration{sdpinst="1",serverid="master"} > 50 * 60
+    for: 5m
+    labels:
+      severity: "warning"
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} checkpoint job duration longer than expected"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been above target for more than 1 minutes."
+  - alert: Checkpoint not taken 
+    expr: time() - p4_sdp_checkpoint_log_time{sdpinst="1",serverid="master"} > 25 * 60 * 60
+    for: 5m
+    labels:
+      severity: "warning"
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} checkpoint not taken in 25 hours warning"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been above target for more than 1 minutes."
+  - alert: P4D service not running
+    expr: node_systemd_unit_state{state="active",name="p4d_1.service"} != 1
+    for: 5m
+    labels:
+      severity: "warning"
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} p4d service not running"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for 5 minutes."
+  - alert: DiskspaceLow
+    expr: node_filesystem_free_bytes{mountpoint=~"/hx.*"} / node_filesystem_size_bytes{mountpoint=~"/hx.*"} * 100 < 10
+    for: 5m
+    labels:
+      severity: "warning"
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} disk space below 10%"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been below limit for 5 minutes."
+```
+
+## Alertmanager config
+
+This is an example, assuming simple email and local postfix or equivalent setup - `/etc/alertmanager/alertmanager.yml`
+
+```yaml
+global:
+  smtp_from: alertmanager@perforce.com
+  smtp_smarthost: localhost:25
+  smtp_require_tls: false
+  # Hello is the local machine name
+  smtp_hello: p4hms
+
+route:
+  group_by: ['alertname']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 60m
+  receiver: mail
+
+receivers:
+- name: mail
+  email_configs:
+  - to: p4-group@perforce.com
+```