Remove prometheus alerts (#298)

Signed-off-by: ArthurSens <[email protected]>

Signed-off-by: ArthurSens <[email protected]>
Co-authored-by: Aleksandar Aleksandrov <[email protected]>

Author: ArthurSens

lib/alert-filter.libsonnet

@@ -77,6 +77,30 @@ local unwatedAlerts = [
   'ConfigReloaderSidecarErrors',  // Re-added to platform-mixin
   'PrometheusOperatorRejectedResources',
   'PrometheusOperatorSyncFailed',
+
+  // From Prometheus
+  'PrometheusBadConfig',  // Re-added to platform-mixin
+  'PrometheusNotificationQueueRunningFull',
+  'PrometheusErrorSendingAlertsToSomeAlertmanagers',
+  'PrometheusNotConnectedToAlertmanagers',
+  'PrometheusTSDBReloadsFailing',
+  'PrometheusTSDBCompactionsFailing',
+  'PrometheusNotIngestingSamples',
+  'PrometheusDuplicateTimestamps',
+  'PrometheusOutOfOrderTimestamps',
+  'PrometheusRemoteStorageFailures',  // Re-added to platform-mixin
+  'PrometheusRemoteWriteBehind',
+  'PrometheusRemoteWriteDesiredShards',
+  'PrometheusRuleFailures',  // Re-added to platform-mixin
+  'PrometheusMissingRuleEvaluations',
+  'PrometheusTargetLimitHit',
+  'PrometheusLabelLimitHit',
+  'PrometheusScrapeBodySizeLimitHit',
+  'PrometheusScrapeSampleLimitHit',
+  'PrometheusTargetSyncFailure',
+  'PrometheusHighQueryLoad',
+  'PrometheusErrorSendingAlertsToAnyAlertmanager',
+
 ];
 
 {

monitoring-satellite/manifests/kube-prometheus-rules/rules.yaml

@@ -1183,270 +1183,7 @@ spec:
         )
       record: instance:node_network_transmit_drop_excluding_lo:rate5m
   - name: prometheus
-    rules:
-    - alert: PrometheusBadConfig
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusBadConfig.md
-        summary: Failed Prometheus configuration reload.
-      expr: |
-        # Without max_over_time, failed scrapes could create false negatives, see
-        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
-        max_over_time(prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) == 0
-      for: 10m
-      labels:
-        severity: critical
-    - alert: PrometheusNotificationQueueRunningFull
-      annotations:
-        description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusNotificationQueueRunningFull.md
-        summary: Prometheus alert notification queue predicted to run full in less than 30m.
-      expr: |
-        # Without min_over_time, failed scrapes could create false negatives, see
-        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
-        (
-          predict_linear(prometheus_notifications_queue_length{job="prometheus-k8s",namespace="monitoring-satellite"}[5m], 60 * 30)
-        >
-          min_over_time(prometheus_notifications_queue_capacity{job="prometheus-k8s",namespace="monitoring-satellite"}[5m])
-        )
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
-      annotations:
-        description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusErrorSendingAlertsToSomeAlertmanagers.md
-        summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
-      expr: |
-        (
-          rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m])
-        /
-          rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m])
-        )
-        * 100
-        > 1
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusNotConnectedToAlertmanagers
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusNotConnectedToAlertmanagers.md
-        summary: Prometheus is not connected to any Alertmanagers.
-      expr: |
-        # Without max_over_time, failed scrapes could create false negatives, see
-        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
-        max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) < 1
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusTSDBReloadsFailing
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusTSDBReloadsFailing.md
-        summary: Prometheus has issues reloading blocks from disk.
-      expr: |
-        increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring-satellite"}[3h]) > 0
-      for: 4h
-      labels:
-        severity: warning
-    - alert: PrometheusTSDBCompactionsFailing
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusTSDBCompactionsFailing.md
-        summary: Prometheus has issues compacting blocks.
-      expr: |
-        increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring-satellite"}[3h]) > 0
-      for: 4h
-      labels:
-        severity: warning
-    - alert: PrometheusNotIngestingSamples
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusNotIngestingSamples.md
-        summary: Prometheus is not ingesting samples.
-      expr: |
-        (
-          rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) <= 0
-        and
-          (
-            sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-k8s",namespace="monitoring-satellite"}) > 0
-          or
-            sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-k8s",namespace="monitoring-satellite"}) > 0
-          )
-        )
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusDuplicateTimestamps
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with different values but duplicated timestamp.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusDuplicateTimestamps.md
-        summary: Prometheus is dropping samples with duplicate timestamps.
-      expr: |
-        rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusOutOfOrderTimestamps
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with timestamps arriving out of order.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusOutOfOrderTimestamps.md
-        summary: Prometheus drops samples with out-of-order timestamps.
-      expr: |
-        rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusRemoteStorageFailures
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusRemoteStorageFailures.md
-        summary: Prometheus fails to send samples to remote storage.
-      expr: |
-        (
-          (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]))
-        /
-          (
-            (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]))
-          +
-            (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]))
-          )
-        )
-        * 100
-        > 1
-      for: 15m
-      labels:
-        severity: critical
-    - alert: PrometheusRemoteWriteBehind
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusRemoteWriteBehind.md
-        summary: Prometheus remote write is behind.
-      expr: |
-        # Without max_over_time, failed scrapes could create false negatives, see
-        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
-        (
-          max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-k8s",namespace="monitoring-satellite"}[5m])
-        - ignoring(remote_name, url) group_right
-          max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-k8s",namespace="monitoring-satellite"}[5m])
-        )
-        > 120
-      for: 15m
-      labels:
-        severity: critical
-    - alert: PrometheusRemoteWriteDesiredShards
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-k8s",namespace="monitoring-satellite"}` $labels.instance | query | first | value }}.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusRemoteWriteDesiredShards.md
-        summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
-      expr: |
-        # Without max_over_time, failed scrapes could create false negatives, see
-        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
-        (
-          max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-k8s",namespace="monitoring-satellite"}[5m])
-        >
-          max_over_time(prometheus_remote_storage_shards_max{job="prometheus-k8s",namespace="monitoring-satellite"}[5m])
-        )
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusRuleFailures
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusRuleFailures.md
-        summary: Prometheus is failing rule evaluations.
-      expr: |
-        increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusMissingRuleEvaluations
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusMissingRuleEvaluations.md
-        summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
-      expr: |
-        increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusTargetLimitHit
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusTargetLimitHit.md
-        summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit.
-      expr: |
-        increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusLabelLimitHit
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusLabelLimitHit.md
-        summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit.
-      expr: |
-        increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusScrapeBodySizeLimitHit
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusScrapeBodySizeLimitHit.md
-        summary: Prometheus has dropped some targets that exceeded body size limit.
-      expr: |
-        increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusScrapeSampleLimitHit
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusScrapeSampleLimitHit.md
-        summary: Prometheus has failed scrapes that have exceeded the configured sample limit.
-      expr: |
-        increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusTargetSyncFailure
-      annotations:
-        description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} have failed to sync because invalid configuration was supplied.'
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusTargetSyncFailure.md
-        summary: Prometheus has failed to sync targets.
-      expr: |
-        increase(prometheus_target_sync_failed_total{job="prometheus-k8s",namespace="monitoring-satellite"}[30m]) > 0
-      for: 5m
-      labels:
-        severity: critical
-    - alert: PrometheusHighQueryLoad
-      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has less than 20% available capacity in its query engine for the last 15 minutes.
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusHighQueryLoad.md
-        summary: Prometheus is reaching its maximum capacity serving concurrent requests.
-      expr: |
-        avg_over_time(prometheus_engine_queries{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0.8
-      for: 15m
-      labels:
-        severity: warning
-    - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
-      annotations:
-        description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/PrometheusErrorSendingAlertsToAnyAlertmanager.md
-        summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
-      expr: |
-        min without (alertmanager) (
-          rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring-satellite",alertmanager!~``}[5m])
-        /
-          rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring-satellite",alertmanager!~``}[5m])
-        )
-        * 100
-        > 3
-      for: 15m
-      labels:
-        severity: critical
+    rules: []
   - name: prometheus-operator
     rules: []
   - name: config-reloaders