add alerts (#66)

Author: ssb-jnk

.github/workflows/alert-deploy.yaml

@@ -0,0 +1,39 @@
+name: Deploy alerts
+run-name: Deploy alerts for dapla-statbank-authenticator to test and prod
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '.nais/alerts.yaml'
+      - '.github/workflows/alert-deploy.yml'
+permissions:
+  id-token: write
+
+jobs:
+  test-deploy:
+    name: Deploy alerts to test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Deploy to test
+        uses: nais/deploy/actions/deploy@v2
+        env:
+          CLUSTER: test
+          RESOURCE: .nais/alerts.yaml
+          DEPLOY_SERVER: deploy.ssb.cloud.nais.io:443
+
+  prod-deploy:
+    name: Deploy alerts to prod
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Deploy to prod
+        uses: nais/deploy/actions/deploy@v2
+        env:
+          CLUSTER: prod
+          RESOURCE: .nais/alerts.yaml
+          DEPLOY_SERVER: deploy.ssb.cloud.nais.io:443

.nais/alerts.yaml

@@ -0,0 +1,75 @@
+apiVersion: "monitoring.coreos.com/v1"
+kind: PrometheusRule
+metadata:
+  name: alert-dapla-statbank-authenticator
+  namespace: dapla-stat
+  labels:
+    team: dapla-stat
+spec:
+  groups:
+    - name: dapla-stat
+      rules:
+        # This alert checks if no replicas of dapla-statbank-authenticator are available, indicating the service is unavailable.
+        - alert: StatbankAuthenticatorUnavailable
+          expr: kube_deployment_status_replicas_available{deployment="dapla-statbank-authenticator"} == 0
+          for: 1m
+          annotations:
+            title: "dapla-statbank-authenticator is unavailable"
+            consequence: "The service is unavailable to users. Immediate investigation required."
+            action: "Check the deployment status and logs for issues."
+          labels:
+            service: dapla-statbank-authenticator
+            namespace: dapla-stat
+            severity: critical
+
+        # This alert detects high CPU usage by calculating the CPU time used over 5 minutes.
+        - alert: HighCPUUsage
+          expr: rate(process_cpu_seconds_total{app="dapla-statbank-authenticator"}[5m]) > 0.8
+          for: 5m
+          annotations:
+            title: "High CPU usage detected"
+            consequence: "The service might experience performance degradation."
+            action: "Investigate the cause of high CPU usage and optimize if necessary."
+          labels:
+            service: dapla-statbank-authenticator
+            namespace: dapla-stat
+            severity: warning
+
+        # This alert checks if memory usage exceeds 90% of the 12GB limit, which could cause instability.
+        - alert: HighMemoryUsage
+          expr: sum by (namespace, pod) (container_memory_working_set_bytes{namespace="dapla-stat", pod=~"dapla-statbank-authenticator-.*"}) > 0.9 * sum by (namespace, pod) (kube_pod_container_resource_limits_memory_bytes{namespace="dapla-stat", pod=~"dapla-statbank-authenticator-.*"})
+          for: 5m
+          annotations:
+            title: "High memory usage detected"
+            consequence: "The service might experience instability due to high memory usage."
+            action: "Check memory utilization and consider increasing resources or optimizing the service."
+          labels:
+            service: dapla-statbank-authenticator
+            namespace: dapla-stat
+            severity: warning
+
+        # This alert detects a high number of error logs in dapla-statbank-authenticator.
+        - alert: HighNumberOfErrors
+          expr: (100 * sum by (app, namespace) (rate(log_messages_errors{app="dapla-statbank-authenticator", level=~"Error"}[3m])) / sum by (app, namespace) (rate(log_messages_total{app="dapla-statbank-authenticator"}[3m]))) > 10
+          for: 3m
+          annotations:
+            title: "High number of errors logged in dapla-statbank-authenticator"
+            consequence: "The application is logging a significant number of errors."
+            action: "Check the service logs for errors and address the root cause."
+          labels:
+            service: dapla-statbank-authenticator
+            namespace: dapla-stat
+            severity: critical
+
+        # This alert monitors the number of pod restarts for dapla-statbank-authenticator and triggers if more than 3 restarts occur within 15 minutes.
+        - alert: HighPodRestarts
+          expr: increase(kube_pod_container_status_restarts_total{namespace="dapla-stat", app="dapla-statbank-authenticator"}[15m]) > 3
+          for: 15m
+          annotations:
+            title: "High number of pod restarts"
+            consequence: "The service may be unstable or misconfigured."
+            action: "Investigate the cause of pod restarts and fix configuration or resource issues."
+          labels:
+            service: dapla-statbank-authenticator
+            namespace: dapla-stat
+            severity: warning