feat: migrate from deadmanswitch to Sentry monitors

pillar/dev/secrets/sentry.sls

@@ -0,0 +1,5 @@
+secrets:
+  sentry:
+    project_id: 0123456789012345
+    project_key: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef
+    ingest_url: deadbeef.ingest

pillar/dev/top.sls

@@ -8,6 +8,7 @@ base:
     - tls
     - users.*
     - postgres.clusters
+    - secrets.sentry
 
   'backup-server':
     - match: nodegroup

salt/base/auto-highstate.sls

@@ -1,18 +1,24 @@
-{% set dms_token = salt["pillar.get"]("deadmanssnitch:token") %}
+{% set sentry_enabled = salt["pillar.get"]("secrets:sentry:project_id") and salt["pillar.get"]("secrets:sentry:project_key") and salt["pillar.get"]("secrets:sentry:ingest_url") %}
+
+curl:
+  pkg.installed
+
+/usr/local/bin/sentry-checkin.sh:
+  file.managed:
+    - source: salt://base/scripts/sentry-checkin.sh
+    - mode: '0755'
+    - user: root
+    - group: root
 
-{% if dms_token %}
-15m-interval-highstate:
-  cron.present:
-    - identifier: 15m-interval-highstate
-    - name: "timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1; curl https://nosnch.in/{{ dms_token }} &> /dev/null"
-    - minute: '*/15'
-{% else %}
 15m-interval-highstate:
   cron.present:
     - identifier: 15m-interval-highstate
-    - name: "timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1"
+    - name: "{% if sentry_enabled %}/usr/local/bin/sentry-checkin.sh {% endif %}timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1"
     - minute: '*/15'
-{% endif %}
+    {% if sentry_enabled %}
+    - require:
+      - file: /usr/local/bin/sentry-checkin.sh
+    {% endif %}
 
 /etc/logrotate.d/salt:
   {% if grains["oscodename"] == "xenial" %}

salt/base/scripts/sentry-checkin.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+MINION_ID=$(salt-call --local grains.get id --out=newline_values_only)
+SENTRY_INGEST_URL=$(salt-call pillar.get secrets:sentry:ingest_url --out=newline_values_only)
+SENTRY_PROJECT_ID=$(salt-call pillar.get secrets:sentry:project_id --out=newline_values_only)
+SENTRY_PROJECT_KEY=$(salt-call pillar.get secrets:sentry:project_key --out=newline_values_only)
+
+MONITOR_SLUG="salt-highstate-${MINION_ID//./}"
+
+if [ -n "$SENTRY_INGEST_URL" ] && [ -n "$SENTRY_PROJECT_ID" ] && [ -n "$SENTRY_PROJECT_KEY" ]; then
+    curl -X POST "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/" \
+        --header 'Content-Type: application/json' \
+        --data-raw '{"monitor_config": {"schedule": {"type": "crontab", "value": "*/15 * * * *"}, "checkin_margin": 5, "max_runtime": 30, "timezone": "UTC"}, "status": "in_progress"}' &> /dev/null
+
+    "$@"
+    COMMAND_EXIT=$?
+
+    if [ $COMMAND_EXIT -eq 0 ]; then
+        curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=ok" &> /dev/null
+    else
+        curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=error" &> /dev/null
+    fi
+
+    exit $COMMAND_EXIT
+else
+    exit 1
+fi