chore(jsonnet): use prometheus_remote_storage_queue_highest_timestamp_in_seconds in PrometheusRemoteWriteBehind

This metric was introduced in openshift/prometheus#262 and related PRs.

Dashboard expressions are not changed, since updating them may be more complex. Fixing the alert is more important and we can always revisit that if it causes confusion.

On main, dashboards will be adjusted later once the jsonnet dependencies are updated.

Author: machine424

assets/prometheus-k8s/prometheus-rule.yaml

@@ -166,11 +166,12 @@ spec:
         description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
         summary: Prometheus remote write is behind.
       expr: |
+        # Use the metric added in https://github.com/openshift/prometheus/pull/262 and related PRs.
         # Without max_over_time, failed scrapes could create false negatives, see
         # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
         (
-          max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
-        - ignoring(remote_name, url) group_right
+          max_over_time(prometheus_remote_storage_queue_highest_timestamp_seconds{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
+        -
           max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
         )
         > 120

jsonnet/utils/sanitize-rules.libsonnet

@@ -409,6 +409,17 @@ local patchedRules = [
         labels: {
           severity: 'info',
         },
+        expr: |||
+          # Use the metric added in https://github.com/openshift/prometheus/pull/262 and related PRs.
+          # Without max_over_time, failed scrapes could create false negatives, see
+          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+          (
+            max_over_time(prometheus_remote_storage_queue_highest_timestamp_seconds{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
+          -
+            max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
+          )
+          > 120
+        |||,
       },
     ],
   },