infra(prometheus): update rules to support multiple batchers and aggregators (#2209)

Co-authored-by: Mauro Toscano <[email protected]>

Author: JuArce

infra/ansible/playbooks/files/prometheus/rules.yml

@@ -17,7 +17,7 @@ groups:
 
   - alert: TaskDifferenceAggregatorBatcher
     # Condition for alerting
-    expr: floor(increase(aligned_aggregator_received_tasks{job="aligned-aggregator"}[15m])) - on() floor(increase(sent_batches{job="aligned-batcher"}[15m])) > 1
+    expr: floor(max(increase(aligned_aggregator_received_tasks{job="aligned-aggregator"}[15m]))) - on() floor(sum(increase(sent_batches{job="aligned-batcher"}[15m]))) > 1
     for: 30s
     # Annotation - additional informational labels to store more information
     annotations:
@@ -30,7 +30,7 @@ groups:
 
   - alert: TaskDifferenceAggregator
     # Condition for alerting
-    expr: floor(increase(aligned_aggregator_received_tasks{job="aligned-aggregator"}[15m])) - floor(increase(aligned_aggregated_responses{job="aligned-aggregator"}[15m])) > 1
+    expr: floor(max(increase(aligned_aggregator_received_tasks{job="aligned-aggregator"}[15m]))) - floor(sum(increase(aligned_aggregated_responses{job="aligned-aggregator"}[15m]))) > 1
     for: 30s
     # Annotation - additional informational labels to store more information
     annotations: