diff --git a/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml b/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml index b33423403e67e5..3fa3eadce358e6 100644 --- a/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml +++ b/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml @@ -45,3 +45,23 @@ spec: sum by(cluster) (avg_over_time(gitpod_workspace_regular_not_active_percentage_mk2[1m]) > 0) AND sum by(cluster) (rate(gitpod_ws_manager_mk2_workspace_startup_seconds_sum{type="Regular"}[1m])) == 0 + - alert: GitpodWsManagerMk2BackupFailureError + labels: + severity: error + team: engine + annotations: + runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/WorkspaceBackupFailures.md + summary: Workspace backups failed recently in cluster {{ $labels.cluster }} + description: This can happen when a single node has failed in the cloud provider + expr: | + sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) <= 16 + - alert: GitpodWsManagerMk2BackupFailureCritical + labels: + severity: critical + team: engine + annotations: + runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/WorkspaceBackupFailures.md + summary: Workspace backups failed recently in cluster {{ $labels.cluster }} + description: This can be an indicator of two or more nodes failing in a cloud provider + expr: | + sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) > 16