[observability] Introduce "ReplicasUnavailable" alert (as warning for now)

Author: geropl

operations/observability/mixins/workspace/rules/central/image-builder.yaml

@@ -51,3 +51,15 @@ spec:
         expr: |
           kube_deployment_spec_replicas{deployment="image-builder-mk3", cluster!~"ephemeral.*"} != kube_deployment_status_replicas_available{deployment="image-builder-mk3", cluster!~"ephemeral.*"}
         for: 3m
+      - alert: GitpodImageBuilderMk3ReplicasUnavailable
+        labels:
+          # TODO(gpl): warning for now, to set it up and fine-tune it
+          severity: warning
+          dedicated: included
+        annotations:
+          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md
+          summary: image-builder-mk3 replicas are unavailable in cluster {{ $labels.cluster }}
+          description: 'image-builder-mk3 pods are unavailable in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }}'
+        expr: |
+          kube_deployment_status_replicas_unavailable{deployment="image-builder-mk3", cluster!~"ephemeral.*"} > 0
+        for: 10m

operations/observability/mixins/workspace/rules/central/node-labeler.yaml

@@ -35,3 +35,15 @@ spec:
         expr: |
           kube_deployment_spec_replicas{deployment="node-labeler", cluster!~"ephemeral.*"} != kube_deployment_status_replicas_available{deployment="node-labeler", cluster!~"ephemeral.*"}
         for: 3m
+      - alert: GitpodNodeLabelerReplicasUnavailable
+        labels:
+          # TODO(gpl): warning for now, to set it up and fine-tune it
+          severity: warning
+          dedicated: included
+        annotations:
+          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md
+          summary: node-labeler replicas are unavailable in cluster {{ $labels.cluster }}
+          description: 'node-labeler pods are unavailable in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }}'
+        expr: |
+          kube_deployment_status_replicas_unavailable{deployment="node-labeler", cluster!~"ephemeral.*"} > 0
+        for: 10m

operations/observability/mixins/workspace/rules/central/ws-manager.yaml

@@ -35,3 +35,15 @@ spec:
         expr: |
           kube_deployment_spec_replicas{deployment="ws-manager-mk2", cluster!~"ephemeral.*"} != kube_deployment_status_replicas_available{deployment="ws-manager-mk2", cluster!~"ephemeral.*"}
         for: 3m
+      - alert: GitpodWsManagerMk2ReplicasUnavailable
+        labels:
+          # TODO(gpl): warning for now, to set it up and fine-tune it
+          severity: warning
+          dedicated: included
+        annotations:
+          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md
+          summary: ws-manager-mk2 replicas are unavailable in cluster {{ $labels.cluster }}
+          description: 'ws-manager-mk2 pods are unavailable in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }}'
+        expr: |
+          kube_deployment_status_replicas_unavailable{deployment="ws-manager-mk2", cluster!~"ephemeral.*"} > 0
+        for: 10m

operations/observability/mixins/workspace/rules/central/ws-proxy.yaml

@@ -35,3 +35,15 @@ spec:
         expr: |
           kube_deployment_spec_replicas{deployment="ws-proxy", cluster!~"ephemeral.*"} != kube_deployment_status_replicas_available{deployment="ws-proxy", cluster!~"ephemeral.*"}
         for: 3m
+      - alert: GitpodWsProxyMk2ReplicasUnavailable
+        labels:
+          # TODO(gpl): warning for now, to set it up and fine-tune it
+          severity: warning
+          dedicated: included
+        annotations:
+          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md
+          summary: ws-proxy replicas are unavailable in cluster {{ $labels.cluster }}
+          description: 'ws-proxy pods are unavailable in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }}'
+        expr: |
+          kube_deployment_status_replicas_unavailable{deployment="ws-proxy", cluster!~"ephemeral.*"} > 0
+        for: 10m