infra(alerts): add alerts for aggregation mode (#1906)

Co-authored-by: Mauro Toscano <[email protected]>

Author: JuArce

alerts/.env.example

@@ -32,3 +32,16 @@ NETWORK=<NETWORK>
 PRIVATE_KEY=<SENDER_PRIVATE_KEY>
 VERIFICATION_WAIT_TIME=<TIME_TO_WAIT_FOR_VERIFICATION>
 LOGS_BLOCK_RANGE=<LOGS_BLOCK_RANGE>
+
+# Variables for aggregation_mode_alerts.sh
+CONTRACT_ADDRESS=<YOUR_CONTRACT_ADDRESS>
+AGGREGATED_PROOF_VERIFIED_TOPIC=0xfe3e9e971000ab9c80c7e06aba2933aae5419d0e44693e3046913e9e58053f62
+RPC_URL=<YOUR_RPC_URL>
+LOGS_BLOCK_RANGE=7500
+SLEEP_TIME=3600
+PAGER_DUTY_KEY=<YOUR_PAGER_DUTY_KEY>
+PAGER_DUTY_EMAIL=<YOUR_PAGER_DUTY_EMAIL>
+PAGER_DUTY_SERVICE_ID=<YOUR_PAGER_DUTY_SERVICE_ID>
+SLACK_WEBHOOK_URL=<YOUR_SLACK_WEBHOOK_URL>
+TELEGRAM_BOT_TOKEN=<YOUR_TELEGRAM_BOT_TOKEN>
+TELEGRAM_CHAT_ID=<YOUR_TELEGRAM_CHAT_ID>

alerts/aggregation_mode_alerts.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# ENV_VARIABLES
+#
+# - CONTRACT_ADDRESS
+#   - Holesky:  0xe84CD4084d8131841CE6DC265361f81F4C59a1d4
+#   - Stage:    0x7Eace34A8d4C4CacE633946C6F7CF4BeF3F33513
+# - AGGREGATED_PROOF_VERIFIED_TOPIC (0xfe3e9e971000ab9c80c7e06aba2933aae5419d0e44693e3046913e9e58053f62)
+# - RPC_URL
+# - LOGS_BLOCK_RANGE (25hs -> 7500 blocks)
+# - SLEEP_TIME
+# - PAGER_DUTY_KEY
+# - PAGER_DUTY_EMAIL
+# - PAGER_DUTY_SERVICE_ID
+# - SLACK_WEBHOOK_URL
+# - TELEGRAM_CHAT_ID
+# - TELEGRAM_BOT_TOKEN
+
+# Load env file from $1 path
+source "$1"
+
+# Function to send slack message
+# @param message
+function send_slack_message() {
+  . alerts/slack.sh "$1"
+}
+
+# Function to send telegram message
+# @param message
+function send_telegram_message() {
+  . alerts/telegram.sh "$1"
+}
+
+# Function to send PagerDuty alert
+# @param message
+function send_pagerduty_alert() {
+  . alerts/pagerduty.sh "$1"
+}
+
+# Flags to avoid sending multiple alerts
+no_new_aggregation_alert=false
+
+while :
+do
+  last_block=$(cast block --rpc-url $RPC_URL -f number)
+  printf "Last block: %s\n" $last_block
+
+  from_block=$(($last_block - $LOGS_BLOCK_RANGE))
+
+  new_aggregated_proofs_logs=$(cast logs --rpc-url $RPC_URL --from-block $from_block --address $CONTRACT_ADDRESS $AGGREGATED_PROOF_VERIFIED_TOPIC)
+  if [ -z "$new_aggregated_proofs_logs" ]; then
+    printf "No new aggregated proofs logs found\n"
+    if [ "$no_new_aggregation_alert" = false ]; then
+      message="🚨 $NETWORK ALERT Aggregation Mode: No new aggregated proofs since block $from_block"
+      printf "$message\n"
+      send_slack_message "$message"
+      send_telegram_message "$message"
+      send_pagerduty_alert "$message"
+    fi
+    no_new_aggregation_alert=true
+  else
+    printf "New aggregated proofs logs found\n"
+    if [ "$no_new_aggregation_alert" = true ]; then
+      message="🟩 $NETWORK INFO Aggregation Mode: Aggregated proofs creation resumed since block $from_block"
+      printf "$message\n"
+      send_slack_message "$message"
+      send_telegram_message "$message"
+    fi
+    no_new_aggregation_alert=false
+  fi
+
+  sleep $SLEEP_TIME
+done

alerts/telegram.sh

@@ -0,0 +1,7 @@
+# Function to send telegram message
+# @param message
+curl -s -X POST https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage \
+    -d chat_id=$TELEGRAM_CHAT_ID \
+    -d text="$1" \
+    -d disable_notification=true
+