chore: fix next net alerts and add devnet alerts (#18120)

Fixes a typo in the nextnet regex, adds mutetimings for the nightly
deployment and removes 'location' making the mute timing use UTC.

Also adds devnet alerts

Author: alexghr

spartan/metrics/grafana/alerts/contactpoints.yaml

@@ -412,3 +412,46 @@ contactPoints:
               with (index .CommonLabels "network")
             {{ "}}" }}{{ "{{" }} urlquery . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown{{ "{{" }} end {{ "}}" }}{{ "{{" }} end {{ "}}" }}
         disableResolveMessage: false
+
+  - orgId: 1
+    name: "Slack #alerts-devnet by namespace"
+    receivers:
+      - uid: alertsdevnetbynamespace
+        type: slack
+        settings:
+          url: $SLACK_WEBHOOK_DEVNET_URL
+          text: |-
+            {{ "{{" }} if gt (len .Alerts) 0 {{ "}}" }}
+            *Alerts:*
+            {{ "{{" }} range .Alerts {{ "}}" }}
+            - {{ "{{" }} with (index .Labels "k8s_namespace_name") {{ "}}" }}{{ "{{" }} . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown-namespace{{ "{{" }} end {{ "}}" }}: {{ "{{" }} with (index .Annotations "summary") {{ "}}" }}{{ "{{" }} . {{ "}}" }}{{ "{{" }} else {{ "}}" }}(no summary){{ "{{" }} end {{ "}}" }}
+            {{ "{{" }} end {{ "}}" }}
+            {{ "{{" }} else {{ "}}" }}
+            *Alerts:*
+            - {{ "{{" }} with (index .CommonAnnotations "summary") {{ "}}" }}{{ "{{" }} . {{ "}}" }}{{ "{{" }} else {{ "}}" }}(no summary){{ "{{" }} end {{ "}}" }}
+            {{ "{{" }} end {{ "}}" }}
+
+            *Grafana overview:*
+            {{ "{{" }} .ExternalURL {{ "}}" }}d/aztec-network/network-overview?orgId=1&refresh=30s&var-data_source=default&var-namespace={{ "{{" }}
+              if gt (len .Alerts) 0
+            {{ "}}" }}{{ "{{" }}
+              with (index (index .Alerts 0).Labels "k8s_namespace_name")
+            {{ "}}" }}{{ "{{" }} urlquery . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown{{ "{{" }} end {{ "}}" }}{{ "{{" }} else {{ "}}" }}{{ "{{" }}
+              with (index .CommonLabels "k8s_namespace_name")
+            {{ "}}" }}{{ "{{" }} urlquery . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown{{ "{{" }} end {{ "}}" }}{{ "{{" }} end {{ "}}" }}{{ "{{" }} printf "\n" {{ "}}" }}
+
+            *GKE workloads:*
+            https://console.cloud.google.com/kubernetes/workload/overview?project={{ "{{" }}
+              if gt (len .Alerts) 0
+            {{ "}}" }}{{ "{{" }}
+              with (index (index .Alerts 0).Labels "gcp_project")
+            {{ "}}" }}{{ "{{" }} . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown{{ "{{" }} end {{ "}}" }}{{ "{{" }} else {{ "}}" }}{{ "{{" }}
+              with (index .CommonLabels "gcp_project")
+            {{ "}}" }}{{ "{{" }} . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown{{ "{{" }} end {{ "}}" }}{{ "{{" }} end {{ "}}" }}&supportedpurview=project&pageState=%28%22savedViews%22%3A%28%22n%22%3A%5B%22{{ "{{" }}
+              if gt (len .Alerts) 0
+            {{ "}}" }}{{ "{{" }}
+              with (index (index .Alerts 0).Labels "k8s_namespace_name")
+            {{ "}}" }}{{ "{{" }} urlquery . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown{{ "{{" }} end {{ "}}" }}{{ "{{" }} else {{ "}}" }}{{ "{{" }}
+              with (index .CommonLabels "k8s_namespace_name")
+            {{ "}}" }}{{ "{{" }} urlquery . {{ "}}" }}{{ "{{" }} else {{ "}}" }}unknown{{ "{{" }} end {{ "}}" }}{{ "{{" }} end {{ "}}" }}%22%5D%29%29
+        disableResolveMessage: false

spartan/metrics/grafana/alerts/mutetimes.yaml

@@ -4,7 +4,6 @@ muteTimes:
       name: nightly_deployment
       time_intervals:
         - times:
-            - start_time: "05:00"
-              end_time: "05:30"
+            - start_time: "04:00"
+              end_time: "05:00"
           weekdays: [monday, tuesday, wednesday, thursday, friday, saturday, sunday]
-          location: Europe/London

spartan/metrics/grafana/alerts/policies.yaml

@@ -45,6 +45,15 @@ policies:
           - - k8s_namespace_name
             - =~
             - $NEXT_NET_REGEX
+        mute_time_intervals:
+          - nightly_deployment
+
+      # Devnet - only alert based on Kubernetes namespace
+      - receiver: "Slack #alerts-devnet by namespace"
+        object_matchers:
+          - - k8s_namespace_name
+            - =~
+            - $DEVNET_NAMESPACES_REGEX
 
       # Testnet - by namespace
       - receiver: "Slack #alerts-testnet by namespace"
@@ -74,17 +83,3 @@ policies:
             - =~
             - $MAINNET_NAMESPACES_REGEX
 
-      # Production - by namespace
-      - receiver: "Slack #network-alerts channel by namespace"
-        object_matchers:
-          - - k8s_namespace_name
-            - =~
-            - $PRODUCTION_NAMESPACES_REGEX
-      # Nightly - by namespace
-      - receiver: "Slack #network-alerts channel by namespace"
-        object_matchers:
-          - - k8s_namespace_name
-            - =~
-            - $NIGHTLY_NAMESPACES_REGEX
-        mute_time_intervals:
-          - nightly_deployment

spartan/metrics/values.tmp.yaml

@@ -117,20 +117,21 @@ grafana:
       domain:
   env:
     PRODUCTION_NAMESPACES_REGEX: "v2-testnet|staging-public|staging-ignition"
-    NIGHTLY_NAMESPACES_REGEX: "next-rc-1"
     STAGING_PUBLIC_REGEX: "staging-public"
     STAGING_IGNITION_REGEX: "staging-ignition|ignition-fisherman-sepolia"
     NEXT_SCENARIO_REGEX: "v[0-9]+-scenario|next-scenario"
-    NEXT_NEXT_REGEX: "next-net"
+    NEXT_NET_REGEX: "next-net"
     TESTNET_NAMESPACES_REGEX: "testnet|v[0-9]+-testnet"
     MAINNET_NAMESPACES_REGEX: "mainnet|v[0-9]+-mainnet|ignition"
+    DEVNET_NAMESPACES_REGEX: "devnet"
     SLACK_WEBHOOK_URL: "http://127.0.0.1" # dummy value
     SLACK_WEBHOOK_STAGING_PUBLIC_URL: "http://127.0.0.1" # dummy value
     SLACK_WEBHOOK_STAGING_IGNITION_URL: "http://127.0.0.1" # dummy value
     SLACK_WEBHOOK_NEXT_SCENARIO_URL: "http://127.0.0.1" # dummy value
     SLACK_WEBHOOK_NEXT_NET_URL: "http://127.0.0.1" # dummy value
     SLACK_WEBHOOK_TESTNET_URL: "http://127.0.0.1" # dummy value
     SLACK_WEBHOOK_MAINNET_URL: "http://127.0.0.1" # dummy value
+    SLACK_WEBHOOK_DEVNET_URL: "http://127.0.0.1" # dummy value
   datasources:
     datasources.yaml:
       apiVersion: 1

spartan/terraform/deploy-metrics/main.tf

@@ -95,6 +95,11 @@ data "google_secret_manager_secret_version" "slack_webhook_next_net" {
   project = var.project
 }
 
+data "google_secret_manager_secret_version" "slack_webhook_devnet" {
+  secret  = var.SLACK_WEBHOOK_DEVNET_SECRET_NAME
+  project = var.project
+}
+
 data "google_secret_manager_secret_version" "slack_webhook_testnet" {
   secret  = var.SLACK_WEBHOOK_TESTNET_SECRET_NAME
   project = var.project
@@ -183,6 +188,11 @@ resource "helm_release" "aztec-gke-cluster" {
     value = data.google_secret_manager_secret_version.slack_webhook_next_net.secret_data
   }
 
+  set {
+    name  = "grafana.env.SLACK_WEBHOOK_DEVNET_URL"
+    value = data.google_secret_manager_secret_version.slack_webhook_devnet.secret_data
+  }
+
   set {
     name  = "grafana.env.SLACK_WEBHOOK_TESTNET_URL"
     value = data.google_secret_manager_secret_version.slack_webhook_testnet.secret_data

spartan/terraform/deploy-metrics/variables.tf

@@ -52,6 +52,12 @@ variable "SLACK_WEBHOOK_NEXT_NET_SECRET_NAME" {
   default     = "slack-webhook-next-net-url"
 }
 
+variable "SLACK_WEBHOOK_DEVNET_SECRET_NAME" {
+  description = "Webhook for devnet alerts"
+  type        = string
+  default     = "slack-webhook-devnet-url"
+}
+
 variable "SLACK_WEBHOOK_TESTNET_SECRET_NAME" {
   description = "Webhook for testnet alerts"
   type        = string