From 813055a33bfb1cb9c06194566db948d62bd8e899 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Mon, 2 Jun 2025 17:30:59 -0500 Subject: [PATCH 01/20] feat: replace dms with sentry monitors Closes #562 --- conf/vagrant/master.conf | 1 + pillar/base/sentry.sls | 3 + pillar/dev/secrets/sentry.sls | 3 + pillar/dev/top.sls | 2 + salt/_extensions/pillar/sentry_cron.py | 96 ++++++++++++++++++++++++++ salt/base/auto-highstate.sls | 20 +++--- 6 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 pillar/base/sentry.sls create mode 100644 pillar/dev/secrets/sentry.sls create mode 100644 salt/_extensions/pillar/sentry_cron.py diff --git a/conf/vagrant/master.conf b/conf/vagrant/master.conf index 2395c4c1..cf075669 100644 --- a/conf/vagrant/master.conf +++ b/conf/vagrant/master.conf @@ -27,3 +27,4 @@ ext_pillar: key_path: /var/lib/consul/encryption_keys/primary.key acl_path: /var/lib/consul/acl_tokens/ - backup_ssh: {} + - sentry_cron: {} diff --git a/pillar/base/sentry.sls b/pillar/base/sentry.sls new file mode 100644 index 00000000..8cac0f7d --- /dev/null +++ b/pillar/base/sentry.sls @@ -0,0 +1,3 @@ +sentry: + org_slug: python-software-foundation + project_slug: salt-highstate-monitors diff --git a/pillar/dev/secrets/sentry.sls b/pillar/dev/secrets/sentry.sls new file mode 100644 index 00000000..55d3f093 --- /dev/null +++ b/pillar/dev/secrets/sentry.sls @@ -0,0 +1,3 @@ +secrets: + sentry: + token: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef \ No newline at end of file diff --git a/pillar/dev/top.sls b/pillar/dev/top.sls index de9057ec..a346d187 100644 --- a/pillar/dev/top.sls +++ b/pillar/dev/top.sls @@ -8,6 +8,8 @@ base: - tls - users.* - postgres.clusters + - sentry + - secrets.sentry 'backup-server': - match: nodegroup diff --git a/salt/_extensions/pillar/sentry_cron.py b/salt/_extensions/pillar/sentry_cron.py new file mode 100644 index 00000000..52bf376e --- /dev/null +++ b/salt/_extensions/pillar/sentry_cron.py @@ -0,0 +1,96 @@ +"""Salt extenstion to replace dms.py (Dead Man's Snitch) by using Sentry's cron monitors. + +It grabs the pillar secrets bearer token and upserts a new monitor for the minion it runs on +and stores the monitor ID in a file in /etc/sentry-cron/. + +The schedule is every 15 minutes to match our highstate check interval. + +You can call it via: + - sudo salt-call -l debug pillar.get sentry_cron + - sudo salt-call pillar.get sentry + - sudo salt-call pillar.get secrets + +View below print output via: + - sudo journalctl -u salt-master +""" +import pathlib + +try: + import requests + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False + + +def ext_pillar(minion_id: str, pillar: dict, base_path: str = "/etc/sentry-cron/") -> dict: + """Upsert a new cron monitor for a minion. + + Will print log to stderr (sudo journalctl -u salt-master) + + Args: + minion_id: The minion ID (provided by salt) + pillar: The pillar data for the minion (provided by salt) + base_path: Where the monitor ID is stored for a minion + + Returns: + dict: The pillar data for the minion, if any. + """ + if not (sentry_token := pillar.get("secrets", {}).get("sentry", {}).get("token")): + print("No Sentry token provided") + return {} + + org_slug = pillar.get("sentry", {}).get("org_slug") + project_slug = pillar.get("sentry", {}).get("project_slug") + if not org_slug or not project_slug: + print("Missing org_slug or project_slug in pillar data") + return {} + + base_path = pathlib.Path(base_path) + base_path.mkdir(parents=True, exist_ok=True) + + minion_path = base_path / minion_id + if minion_path.exists(): + if monitor_id := minion_path.read_text(): + print(f"Found existing monitor ID: {monitor_id}") + return {"sentry_cron": {"monitor_id": monitor_id}} + + headers = { + "Authorization": f"Bearer {sentry_token}", + "Content-Type": "application/json", + } + + try: + monitor = requests.post( + f"https://sentry.io/api/0/organizations/{org_slug}/monitors/", + headers=headers, + json={ + "name": f"salt-highstate {minion_id}", + "type": "cron_job", + "config": { + "schedule": { + "type": "crontab", + "value": "*/15 * * * *", + }, + "checkin_margin": 5, + "max_runtime": 30, + "timezone": "UTC", + }, + "project": project_slug, + "status": "active", + }, + timeout=10, + ) + monitor.raise_for_status() + except Exception as e: + print(f"Failed to create monitor: {e}") + return {} + + print(f"Monitor creation response: {monitor.status_code}") + if monitor.status_code == 201: + monitor_id = monitor.json()["id"] + minion_path.write_text(monitor_id) + print(f"Created monitor with ID: {monitor_id}") + return {"sentry_cron": {"monitor_id": monitor_id}} + + print(f"Failed to create monitor: {monitor.text}") + return {} \ No newline at end of file diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index e33d7b00..2ba4f579 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -1,18 +1,18 @@ -{% set dms_token = salt["pillar.get"]("deadmanssnitch:token") %} +{% set sentry_monitor_id = salt["pillar.get"]("sentry_cron:monitor_id") %} -{% if dms_token %} 15m-interval-highstate: cron.present: - identifier: 15m-interval-highstate - - name: "timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1; curl https://nosnch.in/{{ dms_token }} &> /dev/null" + - name: | + timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1 + {% if sentry_monitor_id %} + curl -X POST \ + "https://sentry.io/api/0/organizations/python-software-foundation/monitors/{{ sentry_monitor_id }}/checkins/" \ + -H "Authorization: Bearer {{ pillar.get('sentry', {}).get('token') }}" \ + -H "Content-Type: application/json" \ + -d '{"status": "ok"}' &> /dev/null + {% endif %} - minute: '*/15' -{% else %} -15m-interval-highstate: - cron.present: - - identifier: 15m-interval-highstate - - name: "timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1" - - minute: '*/15' -{% endif %} /etc/logrotate.d/salt: {% if grains["oscodename"] == "xenial" %} From ec0cda041661a525ba7dfe03bdcfff2eff537c94 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 08:56:09 -0500 Subject: [PATCH 02/20] chore: add more helpful debugging if error --- salt/_extensions/pillar/sentry_cron.py | 69 +++++++++++++++----------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/salt/_extensions/pillar/sentry_cron.py b/salt/_extensions/pillar/sentry_cron.py index 52bf376e..da0a72c7 100644 --- a/salt/_extensions/pillar/sentry_cron.py +++ b/salt/_extensions/pillar/sentry_cron.py @@ -5,22 +5,19 @@ The schedule is every 15 minutes to match our highstate check interval. -You can call it via: - - sudo salt-call -l debug pillar.get sentry_cron - - sudo salt-call pillar.get sentry - - sudo salt-call pillar.get secrets +You can get pillar data via: + sudo salt-call -l debug pillar.get sentry_cron -View below print output via: - - sudo journalctl -u salt-master +View print output via: + sudo journalctl -u salt-master """ + +import contextlib import pathlib +import json -try: +with contextlib.suppress(ImportError): import requests - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - def ext_pillar(minion_id: str, pillar: dict, base_path: str = "/etc/sentry-cron/") -> dict: """Upsert a new cron monitor for a minion. @@ -59,33 +56,49 @@ def ext_pillar(minion_id: str, pillar: dict, base_path: str = "/etc/sentry-cron/ "Content-Type": "application/json", } + request_data = { + "name": f"salt-highstate {minion_id}", + "type": "cron_job", + "config": { + "schedule": { + "type": "crontab", + "value": "*/15 * * * *", + }, + "checkin_margin": 5, + "max_runtime": 30, + "timezone": "UTC", + }, + "project": project_slug, + "status": "active", + } + try: + url = f"https://sentry.io/api/0/organizations/{org_slug}/monitors/" monitor = requests.post( - f"https://sentry.io/api/0/organizations/{org_slug}/monitors/", + url, headers=headers, - json={ - "name": f"salt-highstate {minion_id}", - "type": "cron_job", - "config": { - "schedule": { - "type": "crontab", - "value": "*/15 * * * *", - }, - "checkin_margin": 5, - "max_runtime": 30, - "timezone": "UTC", - }, - "project": project_slug, - "status": "active", - }, + json=request_data, timeout=10, ) + + if monitor.status_code != 201: + print(f"Error response from Sentry API: {monitor.text}") + try: + error_data = monitor.json() + print(f"Error details: {json.dumps(error_data, indent=2)}") + except Exception as e: + print(f"Could not parse error response as JSON: {e}") + return {} + monitor.raise_for_status() + except requests.exceptions.HTTPError as e: + print(f"HTTP Error: {e}") + return {} except Exception as e: print(f"Failed to create monitor: {e}") return {} - print(f"Monitor creation response: {monitor.status_code}") + # print(f"Monitor creation response: {monitor.status_code}") if monitor.status_code == 201: monitor_id = monitor.json()["id"] minion_path.write_text(monitor_id) From e86175f110ec078818f1d900f013dcbfb92f252f Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 12:27:08 -0500 Subject: [PATCH 03/20] fix: remove fqdn separators, use key and ids instead of names --- pillar/base/sentry.sls | 3 +++ salt/base/auto-highstate.sls | 14 ++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pillar/base/sentry.sls b/pillar/base/sentry.sls index 8cac0f7d..c888458b 100644 --- a/pillar/base/sentry.sls +++ b/pillar/base/sentry.sls @@ -1,3 +1,6 @@ sentry: org_slug: python-software-foundation project_slug: salt-highstate-monitors + project_id: 4509407540346880 + project_key: 054ea50c6aac11071b72b66fe9085e0d + ingest_url: o20004.ingest.us.sentry.io diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index 2ba4f579..c0999b97 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -1,16 +1,18 @@ {% set sentry_monitor_id = salt["pillar.get"]("sentry_cron:monitor_id") %} +{% set sentry_token = salt["pillar.get"]("secrets:sentry:token") %} +{% set org_slug = salt["pillar.get"]("sentry:org_slug") %} +{% set project_slug = salt["pillar.get"]("sentry:project_slug") %} +{% set project_id = salt["pillar.get"]("sentry:project_id") %} +{% set project_key = salt["pillar.get"]("sentry:project_key") %} +{% set ingest_url = salt["pillar.get"]("sentry:ingest_url") %} 15m-interval-highstate: cron.present: - identifier: 15m-interval-highstate - name: | timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1 - {% if sentry_monitor_id %} - curl -X POST \ - "https://sentry.io/api/0/organizations/python-software-foundation/monitors/{{ sentry_monitor_id }}/checkins/" \ - -H "Authorization: Bearer {{ pillar.get('sentry', {}).get('token') }}" \ - -H "Content-Type: application/json" \ - -d '{"status": "ok"}' &> /dev/null + {% if sentry_monitor_id and sentry_token and org_slug and project_id and project_key and ingest_url %} + curl "https://{{ ingest_url }}/api/{{ project_id }}/cron/salt-highstate-{{ grains['id'] | replace('.', '') }}/{{ project_key }}/?status=in_progress" &> /dev/null && curl "https://{{ ingest_url }}/api/{{ project_id }}/cron/salt-highstate-{{ grains['id'] | replace('.', '') }}/{{ project_key }}/?status=ok" &> /dev/null {% endif %} - minute: '*/15' From 602a03ddc503d870dc2790377d040fa62aff53d9 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:08:02 -0500 Subject: [PATCH 04/20] feat: use script to run highstate if sentry key present adds check in at highstate start and resulting curl at end so we can track highstate times --- salt/base/auto-highstate.sls | 25 +++++++++++++------------ salt/base/scripts/sentry-checkin.sh | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 salt/base/scripts/sentry-checkin.sh diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index c0999b97..2bd5f411 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -1,20 +1,21 @@ -{% set sentry_monitor_id = salt["pillar.get"]("sentry_cron:monitor_id") %} -{% set sentry_token = salt["pillar.get"]("secrets:sentry:token") %} -{% set org_slug = salt["pillar.get"]("sentry:org_slug") %} -{% set project_slug = salt["pillar.get"]("sentry:project_slug") %} -{% set project_id = salt["pillar.get"]("sentry:project_id") %} -{% set project_key = salt["pillar.get"]("sentry:project_key") %} -{% set ingest_url = salt["pillar.get"]("sentry:ingest_url") %} +{% set sentry_enabled = salt["pillar.get"]("secrets:sentry:token") %} + +/usr/local/bin/sentry-checkin.sh: + file.managed: + - source: salt://base/scripts/sentry-checkin.sh + - mode: '0755' + - user: root + - group: root 15m-interval-highstate: cron.present: + - name: {{ '/usr/local/bin/sentry-checkin.sh' if sentry_enabled else 'timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1' }} - identifier: 15m-interval-highstate - - name: | - timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1 - {% if sentry_monitor_id and sentry_token and org_slug and project_id and project_key and ingest_url %} - curl "https://{{ ingest_url }}/api/{{ project_id }}/cron/salt-highstate-{{ grains['id'] | replace('.', '') }}/{{ project_key }}/?status=in_progress" &> /dev/null && curl "https://{{ ingest_url }}/api/{{ project_id }}/cron/salt-highstate-{{ grains['id'] | replace('.', '') }}/{{ project_key }}/?status=ok" &> /dev/null - {% endif %} - minute: '*/15' + {% if sentry_enabled %} + - require: + - file: /usr/local/bin/sentry-checkin.sh + {% endif %} /etc/logrotate.d/salt: {% if grains["oscodename"] == "xenial" %} diff --git a/salt/base/scripts/sentry-checkin.sh b/salt/base/scripts/sentry-checkin.sh new file mode 100644 index 00000000..1100fcf4 --- /dev/null +++ b/salt/base/scripts/sentry-checkin.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +MINION_ID=$(salt-call --local grains.get id --out=newline_values_only) +SENTRY_INGEST_URL=$(salt-call --local pillar.get sentry:ingest_url --out=newline_values_only) +SENTRY_PROJECT_ID=$(salt-call --local pillar.get sentry:project_id --out=newline_values_only) +SENTRY_PROJECT_KEY=$(salt-call --local pillar.get sentry:project_key --out=newline_values_only) + +MONITOR_SLUG="salt-highstate-${MINION_ID//./}" + +if [ -n "$SENTRY_INGEST_URL" ] && [ -n "$SENTRY_PROJECT_ID" ] && [ -n "$SENTRY_PROJECT_KEY" ]; then + curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=in_progress" &> /dev/null + + timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1 + HIGHSTATE_EXIT=$? + + if [ $HIGHSTATE_EXIT -eq 0 ]; then + curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=ok" &> /dev/null + else + curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=error" &> /dev/null + fi + + exit $HIGHSTATE_EXIT +fi \ No newline at end of file From b9da2e2a492436e07620a577d16fe6bb500f6abd Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:09:19 -0500 Subject: [PATCH 05/20] chore: less of a git diff --- pillar/dev/secrets/sentry.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/dev/secrets/sentry.sls b/pillar/dev/secrets/sentry.sls index 55d3f093..68feabaf 100644 --- a/pillar/dev/secrets/sentry.sls +++ b/pillar/dev/secrets/sentry.sls @@ -1,3 +1,3 @@ secrets: sentry: - token: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef \ No newline at end of file + token: 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 \ No newline at end of file From cd93ea13d295b4a180c1d07814fa7ba672368ff8 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:14:27 -0500 Subject: [PATCH 06/20] fix: dont use local --- salt/base/scripts/sentry-checkin.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/base/scripts/sentry-checkin.sh b/salt/base/scripts/sentry-checkin.sh index 1100fcf4..bfaa5b54 100644 --- a/salt/base/scripts/sentry-checkin.sh +++ b/salt/base/scripts/sentry-checkin.sh @@ -1,9 +1,9 @@ #!/bin/bash MINION_ID=$(salt-call --local grains.get id --out=newline_values_only) -SENTRY_INGEST_URL=$(salt-call --local pillar.get sentry:ingest_url --out=newline_values_only) -SENTRY_PROJECT_ID=$(salt-call --local pillar.get sentry:project_id --out=newline_values_only) -SENTRY_PROJECT_KEY=$(salt-call --local pillar.get sentry:project_key --out=newline_values_only) +SENTRY_INGEST_URL=$(salt-call pillar.get sentry:ingest_url --out=newline_values_only) +SENTRY_PROJECT_ID=$(salt-call pillar.get sentry:project_id --out=newline_values_only) +SENTRY_PROJECT_KEY=$(salt-call pillar.get sentry:project_key --out=newline_values_only) MONITOR_SLUG="salt-highstate-${MINION_ID//./}" From 0bca8dbc070e736a9bf1932bd193e8b5d002a547 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:14:33 -0500 Subject: [PATCH 07/20] chore: less of a git diff --- salt/base/auto-highstate.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index 2bd5f411..311d26e8 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -9,8 +9,8 @@ 15m-interval-highstate: cron.present: - - name: {{ '/usr/local/bin/sentry-checkin.sh' if sentry_enabled else 'timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1' }} - identifier: 15m-interval-highstate + - name: {{ '/usr/local/bin/sentry-checkin.sh' if sentry_enabled else 'timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1' }} - minute: '*/15' {% if sentry_enabled %} - require: From b62711abfaef8403ac79e32c7c1ddba7931eff54 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:15:23 -0500 Subject: [PATCH 08/20] chore: undo (inactive) token commit --- pillar/dev/secrets/sentry.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/dev/secrets/sentry.sls b/pillar/dev/secrets/sentry.sls index 68feabaf..55d3f093 100644 --- a/pillar/dev/secrets/sentry.sls +++ b/pillar/dev/secrets/sentry.sls @@ -1,3 +1,3 @@ secrets: sentry: - token: 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 \ No newline at end of file + token: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef \ No newline at end of file From 62984d332e2331494521b54c13108ff1b17e5187 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:16:42 -0500 Subject: [PATCH 09/20] docs: notate what scopes are needed for monitor upserts --- salt/_extensions/pillar/sentry_cron.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/_extensions/pillar/sentry_cron.py b/salt/_extensions/pillar/sentry_cron.py index da0a72c7..cf36d28d 100644 --- a/salt/_extensions/pillar/sentry_cron.py +++ b/salt/_extensions/pillar/sentry_cron.py @@ -3,6 +3,8 @@ It grabs the pillar secrets bearer token and upserts a new monitor for the minion it runs on and stores the monitor ID in a file in /etc/sentry-cron/. +Note: The bearer token will need the `alerts:read, alerts:write, project:read` scopes. + The schedule is every 15 minutes to match our highstate check interval. You can get pillar data via: From 61eb5d6869eb5f353a2ac575019b39e0e8386ec0 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:31:02 -0500 Subject: [PATCH 10/20] fix: resolve issue with duplicate monitor creation by locking file --- salt/_extensions/pillar/sentry_cron.py | 124 +++++++++++++------------ 1 file changed, 66 insertions(+), 58 deletions(-) diff --git a/salt/_extensions/pillar/sentry_cron.py b/salt/_extensions/pillar/sentry_cron.py index cf36d28d..cee022af 100644 --- a/salt/_extensions/pillar/sentry_cron.py +++ b/salt/_extensions/pillar/sentry_cron.py @@ -17,6 +17,9 @@ import contextlib import pathlib import json +import os +import tempfile +import fcntl with contextlib.suppress(ImportError): import requests @@ -46,66 +49,71 @@ def ext_pillar(minion_id: str, pillar: dict, base_path: str = "/etc/sentry-cron/ base_path = pathlib.Path(base_path) base_path.mkdir(parents=True, exist_ok=True) - minion_path = base_path / minion_id - if minion_path.exists(): - if monitor_id := minion_path.read_text(): - print(f"Found existing monitor ID: {monitor_id}") - return {"sentry_cron": {"monitor_id": monitor_id}} - - headers = { - "Authorization": f"Bearer {sentry_token}", - "Content-Type": "application/json", - } - - request_data = { - "name": f"salt-highstate {minion_id}", - "type": "cron_job", - "config": { - "schedule": { - "type": "crontab", - "value": "*/15 * * * *", + lock_path = base_path / f"{minion_id}.lock" + + with open(lock_path, "w") as lockfile: + fcntl.flock(lockfile, fcntl.LOCK_EX) + if minion_path.exists(): + if monitor_id := minion_path.read_text(): + print(f"Found existing monitor ID (locked): {monitor_id}") + return {"sentry_cron": {"monitor_id": monitor_id}} + + headers = { + "Authorization": f"Bearer {sentry_token}", + "Content-Type": "application/json", + } + + request_data = { + "name": f"salt-highstate {minion_id}", + "type": "cron_job", + "config": { + "schedule": { + "type": "crontab", + "value": "*/15 * * * *", + }, + "checkin_margin": 5, + "max_runtime": 30, + "timezone": "UTC", }, - "checkin_margin": 5, - "max_runtime": 30, - "timezone": "UTC", - }, - "project": project_slug, - "status": "active", - } - - try: - url = f"https://sentry.io/api/0/organizations/{org_slug}/monitors/" - monitor = requests.post( - url, - headers=headers, - json=request_data, - timeout=10, - ) - - if monitor.status_code != 201: - print(f"Error response from Sentry API: {monitor.text}") - try: - error_data = monitor.json() - print(f"Error details: {json.dumps(error_data, indent=2)}") - except Exception as e: - print(f"Could not parse error response as JSON: {e}") + "project": project_slug, + "status": "active", + } + + try: + url = f"https://sentry.io/api/0/organizations/{org_slug}/monitors/" + monitor = requests.post( + url, + headers=headers, + json=request_data, + timeout=10, + ) + + if monitor.status_code != 201: + print(f"Error response from Sentry API: {monitor.text}") + try: + error_data = monitor.json() + print(f"Error details: {json.dumps(error_data, indent=2)}") + except Exception as e: + print(f"Could not parse error response as JSON: {e}") + return {} + + monitor.raise_for_status() + except requests.exceptions.HTTPError as e: + print(f"HTTP Error: {e}") + return {} + except Exception as e: + print(f"Failed to create monitor: {e}") return {} - monitor.raise_for_status() - except requests.exceptions.HTTPError as e: - print(f"HTTP Error: {e}") - return {} - except Exception as e: - print(f"Failed to create monitor: {e}") - return {} - - # print(f"Monitor creation response: {monitor.status_code}") - if monitor.status_code == 201: - monitor_id = monitor.json()["id"] - minion_path.write_text(monitor_id) - print(f"Created monitor with ID: {monitor_id}") - return {"sentry_cron": {"monitor_id": monitor_id}} + if monitor.status_code == 201: + monitor_id = monitor.json()["id"] + with tempfile.NamedTemporaryFile('w', dir=str(base_path), delete=False) as tf: + tf.write(monitor_id) + tempname = tf.name + os.replace(tempname, minion_path) + print(f"Created monitor with ID: {monitor_id}") + return {"sentry_cron": {"monitor_id": monitor_id}} - print(f"Failed to create monitor: {monitor.text}") - return {} \ No newline at end of file + print(f"Failed to create monitor: {monitor.text}") + return {} \ No newline at end of file From cb11aa28d569620cc984e62d37a99b0751337b46 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 3 Jun 2025 14:58:24 -0500 Subject: [PATCH 11/20] fix: the script needs curl but its not on vagrant --- salt/base/auto-highstate.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index 311d26e8..abe966ba 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -1,5 +1,8 @@ {% set sentry_enabled = salt["pillar.get"]("secrets:sentry:token") %} +curl: + pkg.installed + /usr/local/bin/sentry-checkin.sh: file.managed: - source: salt://base/scripts/sentry-checkin.sh From 1456a6540f403be2ceff8c11ae9555e96db87b9c Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 4 Jun 2025 12:02:31 -0500 Subject: [PATCH 12/20] feat: simplify --- conf/vagrant/master.conf | 1 - pillar/base/sentry.sls | 6 -- pillar/dev/secrets/sentry.sls | 4 +- pillar/dev/top.sls | 1 - salt/_extensions/pillar/sentry_cron.py | 119 ------------------------- salt/base/auto-highstate.sls | 4 +- salt/base/scripts/sentry-checkin.sh | 20 +++-- 7 files changed, 17 insertions(+), 138 deletions(-) delete mode 100644 pillar/base/sentry.sls delete mode 100644 salt/_extensions/pillar/sentry_cron.py diff --git a/conf/vagrant/master.conf b/conf/vagrant/master.conf index cf075669..2395c4c1 100644 --- a/conf/vagrant/master.conf +++ b/conf/vagrant/master.conf @@ -27,4 +27,3 @@ ext_pillar: key_path: /var/lib/consul/encryption_keys/primary.key acl_path: /var/lib/consul/acl_tokens/ - backup_ssh: {} - - sentry_cron: {} diff --git a/pillar/base/sentry.sls b/pillar/base/sentry.sls deleted file mode 100644 index c888458b..00000000 --- a/pillar/base/sentry.sls +++ /dev/null @@ -1,6 +0,0 @@ -sentry: - org_slug: python-software-foundation - project_slug: salt-highstate-monitors - project_id: 4509407540346880 - project_key: 054ea50c6aac11071b72b66fe9085e0d - ingest_url: o20004.ingest.us.sentry.io diff --git a/pillar/dev/secrets/sentry.sls b/pillar/dev/secrets/sentry.sls index 55d3f093..9202ef79 100644 --- a/pillar/dev/secrets/sentry.sls +++ b/pillar/dev/secrets/sentry.sls @@ -1,3 +1,5 @@ secrets: sentry: - token: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef \ No newline at end of file + project_id: 0123456789012345 + project_key: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef + ingest_url: deadbeef.ingest diff --git a/pillar/dev/top.sls b/pillar/dev/top.sls index a346d187..0c6ad138 100644 --- a/pillar/dev/top.sls +++ b/pillar/dev/top.sls @@ -8,7 +8,6 @@ base: - tls - users.* - postgres.clusters - - sentry - secrets.sentry 'backup-server': diff --git a/salt/_extensions/pillar/sentry_cron.py b/salt/_extensions/pillar/sentry_cron.py deleted file mode 100644 index cee022af..00000000 --- a/salt/_extensions/pillar/sentry_cron.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Salt extenstion to replace dms.py (Dead Man's Snitch) by using Sentry's cron monitors. - -It grabs the pillar secrets bearer token and upserts a new monitor for the minion it runs on -and stores the monitor ID in a file in /etc/sentry-cron/. - -Note: The bearer token will need the `alerts:read, alerts:write, project:read` scopes. - -The schedule is every 15 minutes to match our highstate check interval. - -You can get pillar data via: - sudo salt-call -l debug pillar.get sentry_cron - -View print output via: - sudo journalctl -u salt-master -""" - -import contextlib -import pathlib -import json -import os -import tempfile -import fcntl - -with contextlib.suppress(ImportError): - import requests - -def ext_pillar(minion_id: str, pillar: dict, base_path: str = "/etc/sentry-cron/") -> dict: - """Upsert a new cron monitor for a minion. - - Will print log to stderr (sudo journalctl -u salt-master) - - Args: - minion_id: The minion ID (provided by salt) - pillar: The pillar data for the minion (provided by salt) - base_path: Where the monitor ID is stored for a minion - - Returns: - dict: The pillar data for the minion, if any. - """ - if not (sentry_token := pillar.get("secrets", {}).get("sentry", {}).get("token")): - print("No Sentry token provided") - return {} - - org_slug = pillar.get("sentry", {}).get("org_slug") - project_slug = pillar.get("sentry", {}).get("project_slug") - if not org_slug or not project_slug: - print("Missing org_slug or project_slug in pillar data") - return {} - - base_path = pathlib.Path(base_path) - base_path.mkdir(parents=True, exist_ok=True) - minion_path = base_path / minion_id - lock_path = base_path / f"{minion_id}.lock" - - with open(lock_path, "w") as lockfile: - fcntl.flock(lockfile, fcntl.LOCK_EX) - if minion_path.exists(): - if monitor_id := minion_path.read_text(): - print(f"Found existing monitor ID (locked): {monitor_id}") - return {"sentry_cron": {"monitor_id": monitor_id}} - - headers = { - "Authorization": f"Bearer {sentry_token}", - "Content-Type": "application/json", - } - - request_data = { - "name": f"salt-highstate {minion_id}", - "type": "cron_job", - "config": { - "schedule": { - "type": "crontab", - "value": "*/15 * * * *", - }, - "checkin_margin": 5, - "max_runtime": 30, - "timezone": "UTC", - }, - "project": project_slug, - "status": "active", - } - - try: - url = f"https://sentry.io/api/0/organizations/{org_slug}/monitors/" - monitor = requests.post( - url, - headers=headers, - json=request_data, - timeout=10, - ) - - if monitor.status_code != 201: - print(f"Error response from Sentry API: {monitor.text}") - try: - error_data = monitor.json() - print(f"Error details: {json.dumps(error_data, indent=2)}") - except Exception as e: - print(f"Could not parse error response as JSON: {e}") - return {} - - monitor.raise_for_status() - except requests.exceptions.HTTPError as e: - print(f"HTTP Error: {e}") - return {} - except Exception as e: - print(f"Failed to create monitor: {e}") - return {} - - if monitor.status_code == 201: - monitor_id = monitor.json()["id"] - with tempfile.NamedTemporaryFile('w', dir=str(base_path), delete=False) as tf: - tf.write(monitor_id) - tempname = tf.name - os.replace(tempname, minion_path) - print(f"Created monitor with ID: {monitor_id}") - return {"sentry_cron": {"monitor_id": monitor_id}} - - print(f"Failed to create monitor: {monitor.text}") - return {} \ No newline at end of file diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index abe966ba..db3fdd57 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -1,4 +1,4 @@ -{% set sentry_enabled = salt["pillar.get"]("secrets:sentry:token") %} +{% set sentry_enabled = salt["pillar.get"]("secrets:sentry:project_id") and salt["pillar.get"]("secrets:sentry:project_key") and salt["pillar.get"]("secrets:sentry:ingest_url") %} curl: pkg.installed @@ -13,7 +13,7 @@ curl: 15m-interval-highstate: cron.present: - identifier: 15m-interval-highstate - - name: {{ '/usr/local/bin/sentry-checkin.sh' if sentry_enabled else 'timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1' }} + - name: "{% if sentry_enabled %}/usr/local/bin/sentry-checkin.sh {% endif %}timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1" - minute: '*/15' {% if sentry_enabled %} - require: diff --git a/salt/base/scripts/sentry-checkin.sh b/salt/base/scripts/sentry-checkin.sh index bfaa5b54..d32840ef 100644 --- a/salt/base/scripts/sentry-checkin.sh +++ b/salt/base/scripts/sentry-checkin.sh @@ -1,23 +1,27 @@ #!/bin/bash MINION_ID=$(salt-call --local grains.get id --out=newline_values_only) -SENTRY_INGEST_URL=$(salt-call pillar.get sentry:ingest_url --out=newline_values_only) -SENTRY_PROJECT_ID=$(salt-call pillar.get sentry:project_id --out=newline_values_only) -SENTRY_PROJECT_KEY=$(salt-call pillar.get sentry:project_key --out=newline_values_only) +SENTRY_INGEST_URL=$(salt-call pillar.get secrets:sentry:ingest_url --out=newline_values_only) +SENTRY_PROJECT_ID=$(salt-call pillar.get secrets:sentry:project_id --out=newline_values_only) +SENTRY_PROJECT_KEY=$(salt-call pillar.get secrets:sentry:project_key --out=newline_values_only) MONITOR_SLUG="salt-highstate-${MINION_ID//./}" if [ -n "$SENTRY_INGEST_URL" ] && [ -n "$SENTRY_PROJECT_ID" ] && [ -n "$SENTRY_PROJECT_KEY" ]; then - curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=in_progress" &> /dev/null + curl -X POST "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/" \ + --header 'Content-Type: application/json' \ + --data-raw '{"monitor_config": {"schedule": {"type": "crontab", "value": "*/15 * * * *"}, "checkin_margin": 5, "max_runtime": 30, "timezone": "UTC"}, "status": "in_progress"}' &> /dev/null - timeout 5m salt-call state.highstate >> /var/log/salt/cron-highstate.log 2>&1 - HIGHSTATE_EXIT=$? + "$@" + COMMAND_EXIT=$? - if [ $HIGHSTATE_EXIT -eq 0 ]; then + if [ $COMMAND_EXIT -eq 0 ]; then curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=ok" &> /dev/null else curl "https://${SENTRY_INGEST_URL}/api/${SENTRY_PROJECT_ID}/cron/${MONITOR_SLUG}/${SENTRY_PROJECT_KEY}/?status=error" &> /dev/null fi - exit $HIGHSTATE_EXIT + exit $COMMAND_EXIT +else + exit 1 fi \ No newline at end of file From 4351d0ebc474e8691a7d7731560e0fc7f1131e8b Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 13:48:59 -0500 Subject: [PATCH 13/20] chore: uv run tox -e lint --- pillar/dev/secrets/sentry.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/dev/secrets/sentry.sls b/pillar/dev/secrets/sentry.sls index 9202ef79..27ffaf1c 100644 --- a/pillar/dev/secrets/sentry.sls +++ b/pillar/dev/secrets/sentry.sls @@ -1,5 +1,5 @@ secrets: sentry: - project_id: 0123456789012345 + project_id: 123456789012345 project_key: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef ingest_url: deadbeef.ingest From 130b8073359ae81259a615fb46399115b3d57d13 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 13:57:42 -0500 Subject: [PATCH 14/20] feat: use tempalte instead of making 4 salt-calls! --- salt/base/auto-highstate.sls | 3 ++- .../{sentry-checkin.sh => sentry-checkin.sh.jinja} | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) rename salt/base/scripts/{sentry-checkin.sh => sentry-checkin.sh.jinja} (72%) diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index db3fdd57..865b9704 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -5,7 +5,8 @@ curl: /usr/local/bin/sentry-checkin.sh: file.managed: - - source: salt://base/scripts/sentry-checkin.sh + - source: salt://base/scripts/sentry-checkin.sh.jinja + - template: jinja - mode: '0755' - user: root - group: root diff --git a/salt/base/scripts/sentry-checkin.sh b/salt/base/scripts/sentry-checkin.sh.jinja similarity index 72% rename from salt/base/scripts/sentry-checkin.sh rename to salt/base/scripts/sentry-checkin.sh.jinja index d32840ef..77edde9d 100644 --- a/salt/base/scripts/sentry-checkin.sh +++ b/salt/base/scripts/sentry-checkin.sh.jinja @@ -1,9 +1,9 @@ #!/bin/bash -MINION_ID=$(salt-call --local grains.get id --out=newline_values_only) -SENTRY_INGEST_URL=$(salt-call pillar.get secrets:sentry:ingest_url --out=newline_values_only) -SENTRY_PROJECT_ID=$(salt-call pillar.get secrets:sentry:project_id --out=newline_values_only) -SENTRY_PROJECT_KEY=$(salt-call pillar.get secrets:sentry:project_key --out=newline_values_only) +MINION_ID="{{ grains['id'] }}" +SENTRY_INGEST_URL="{{ pillar.get('secrets:sentry:ingest_url', '') }}" +SENTRY_PROJECT_ID="{{ pillar.get('secrets:sentry:project_id', '') }}" +SENTRY_PROJECT_KEY="{{ pillar.get('secrets:sentry:project_key', '') }}" MONITOR_SLUG="salt-highstate-${MINION_ID//./}" From 034cb2605fc5d91d5440d2ed8bd6003497c09838 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 13:59:26 -0500 Subject: [PATCH 15/20] chore: removing nesting in pillar data --- pillar/dev/secrets/sentry.sls | 8 +++----- salt/base/auto-highstate.sls | 2 +- salt/base/scripts/sentry-checkin.sh.jinja | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pillar/dev/secrets/sentry.sls b/pillar/dev/secrets/sentry.sls index 27ffaf1c..660b8adc 100644 --- a/pillar/dev/secrets/sentry.sls +++ b/pillar/dev/secrets/sentry.sls @@ -1,5 +1,3 @@ -secrets: - sentry: - project_id: 123456789012345 - project_key: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef - ingest_url: deadbeef.ingest +project_id: 123456789012345 +project_key: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef +ingest_url: deadbeef.ingest diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index 865b9704..c2484eee 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -1,4 +1,4 @@ -{% set sentry_enabled = salt["pillar.get"]("secrets:sentry:project_id") and salt["pillar.get"]("secrets:sentry:project_key") and salt["pillar.get"]("secrets:sentry:ingest_url") %} +{% set sentry_enabled = salt["pillar.get"]("project_id") and salt["pillar.get"]("project_key") and salt["pillar.get"]("ingest_url") %} curl: pkg.installed diff --git a/salt/base/scripts/sentry-checkin.sh.jinja b/salt/base/scripts/sentry-checkin.sh.jinja index 77edde9d..220b011d 100644 --- a/salt/base/scripts/sentry-checkin.sh.jinja +++ b/salt/base/scripts/sentry-checkin.sh.jinja @@ -1,9 +1,9 @@ #!/bin/bash MINION_ID="{{ grains['id'] }}" -SENTRY_INGEST_URL="{{ pillar.get('secrets:sentry:ingest_url', '') }}" -SENTRY_PROJECT_ID="{{ pillar.get('secrets:sentry:project_id', '') }}" -SENTRY_PROJECT_KEY="{{ pillar.get('secrets:sentry:project_key', '') }}" +SENTRY_INGEST_URL="{{ pillar.get('ingest_url', '') }}" +SENTRY_PROJECT_ID="{{ pillar.get('project_id', '') }}" +SENTRY_PROJECT_KEY="{{ pillar.get('project_key', '') }}" MONITOR_SLUG="salt-highstate-${MINION_ID//./}" From 83d93cf077e2924b0c686c115bc7f1453ce65d57 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 14:02:41 -0500 Subject: [PATCH 16/20] fix: make sure we dont run sentry things if disabled --- salt/base/auto-highstate.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/base/auto-highstate.sls b/salt/base/auto-highstate.sls index c2484eee..459cae3e 100644 --- a/salt/base/auto-highstate.sls +++ b/salt/base/auto-highstate.sls @@ -1,5 +1,6 @@ {% set sentry_enabled = salt["pillar.get"]("project_id") and salt["pillar.get"]("project_key") and salt["pillar.get"]("ingest_url") %} +{% if sentry_enabled %} curl: pkg.installed @@ -10,6 +11,7 @@ curl: - mode: '0755' - user: root - group: root +{% endif %} 15m-interval-highstate: cron.present: From fc7c48fadc4ca64844bd8d5339ce1be1cefa422f Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 14:09:33 -0500 Subject: [PATCH 17/20] chore: clean up DMS --- docs/guides/migration-recipe.md | 2 +- salt/_extensions/pillar/dms.py | 48 --------------------------------- 2 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 salt/_extensions/pillar/dms.py diff --git a/docs/guides/migration-recipe.md b/docs/guides/migration-recipe.md index 7b0aedc5..5f8befad 100644 --- a/docs/guides/migration-recipe.md +++ b/docs/guides/migration-recipe.md @@ -155,7 +155,7 @@ index 68387c9..7a8ace1 100644 sudo service nginx stop ``` ```{note} - Don't forget to pause service checks for both the old and new hosts in things like Dead Man's Snitch, Pingdom, etc. + Don't forget to pause service checks for both the old and new hosts in things like Sentry monitors, Pingdom, etc. ``` 4. Ensure that any additional volumes are mounted and in the correct location: - Check what disks are currently mounted and where: `df` diff --git a/salt/_extensions/pillar/dms.py b/salt/_extensions/pillar/dms.py deleted file mode 100644 index 47861001..00000000 --- a/salt/_extensions/pillar/dms.py +++ /dev/null @@ -1,48 +0,0 @@ -import pathlib - -try: - import requests - from requests.auth import HTTPBasicAuth - - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - - -def ext_pillar(minion_id, pillar, api_key=None, base_path="/etc/deadmanssnitch/"): - base_path = pathlib.Path(base_path) - # Ensure base path exists - base_path.mkdir(parents=True, exist_ok=True) - - minion_path = base_path / minion_id - - if minion_path.exists(): - token = minion_path.read_text() - if token: - return {"deadmanssnitch": {"token": token}} - - snitches = requests.get( - "https://api.deadmanssnitch.com/v1/snitches", - params={"tags": "salt-master"}, - auth=HTTPBasicAuth(api_key, ""), - ) - - for snitch in snitches.json(): - if snitch["name"] == f"salt-highstate {minion_id}": - token = snitch["token"] - minion_path.write_text(token) - return {"deadmanssnitch": {"token": token}} - - snitch = requests.post( - "https://api.deadmanssnitch.com/v1/snitches", - auth=HTTPBasicAuth(api_key, ""), - json={ - "name": f"salt-highstate {minion_id}", - "interval": "hourly", - "alert_type": "basic", - "tags": ["salt-master"], - }, - ) - token = snitch.json()["token"] - minion_path.write_text(token) - return {"deadmanssnitch": {"token": token}} From 40030346d13c0f08511c7676869e785163cc6054 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 14:12:29 -0500 Subject: [PATCH 18/20] docs: add sentry --- docs/overview.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/overview.rst b/docs/overview.rst index 8e2ff8b2..db9e1bb4 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -63,6 +63,11 @@ Pingdom `Pingdom `_ provides monitoring and complains to us when services are down. +Sentry + `Sentry `_ is used for error reporting and monitoring of + many services. It also provides Salt highstate cron monitoring, which + notifies us when Salt highstate runs fail over a certain threshold. + PagerDuty `PagerDuty `_ is used for on-call rotation for PSF Infrastructure employees on the front-line, and volunteers as backup. From 9a736fe7aa14def2ab9157d646b90b474ae1d7ee Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 14:12:57 -0500 Subject: [PATCH 19/20] fix(docs): trim wording --- docs/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/overview.rst b/docs/overview.rst index db9e1bb4..c637c286 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -66,7 +66,7 @@ Pingdom Sentry `Sentry `_ is used for error reporting and monitoring of many services. It also provides Salt highstate cron monitoring, which - notifies us when Salt highstate runs fail over a certain threshold. + notifies us when runs fail over a certain threshold. PagerDuty `PagerDuty `_ is used for on-call rotation for PSF From 9368b07b3029015f5c7c77ef81c21ee81e9621b6 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Tue, 10 Jun 2025 16:14:58 -0500 Subject: [PATCH 20/20] chore: turn off secrets.sentry if in dev by default, add note --- pillar/dev/top.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/dev/top.sls b/pillar/dev/top.sls index 0c6ad138..fceb9a85 100644 --- a/pillar/dev/top.sls +++ b/pillar/dev/top.sls @@ -8,7 +8,7 @@ base: - tls - users.* - postgres.clusters - - secrets.sentry + # - secrets.sentry # Uncomment and update sentry secrets if you want to work in dev 'backup-server': - match: nodegroup