alerts: update names

DifferentialOrange · DifferentialOrange · commit de3764230737 · 2022-08-16T18:36:15.000+03:00
Update some alert names to be more readable and consistent. Fix alert descriptions. Change some alerts level. Part of #64
diff --git a/example_cluster/prometheus/alerts.yml b/example_cluster/prometheus/alerts.yml
@@ -14,8 +14,8 @@ groups:
 
 - name: tarantool-common
   rules:
-  # Warning for any instance that uses too Lua runtime memory.
-  - alert: LuaRuntimeWarning
+  # Warning for any instance that uses too much Lua runtime memory.
+  - alert: HighLuaMemoryWarning
     expr: tnt_info_memory_lua >= (512 * 1024 * 1024) 
     for: 1m
     labels:
@@ -25,8 +25,8 @@ groups:
       description: "'{{ $labels.alias }}' instance of job '{{ $labels.job }}' uses too much Lua memory
         and may hit threshold soon."
 
-  # Alert for any instance that uses too Lua runtime memory.
-  - alert: LuaRuntimeAlert
+  # Alert for any instance that uses too much Lua runtime memory.
+  - alert: HighLuaMemory
     expr: tnt_info_memory_lua >= (1024 * 1024 * 1024) 
     for: 1m
     labels:
@@ -37,7 +37,7 @@ groups:
         and likely to hit threshold soon."
 
   # Warning for any instance that have low remaining arena memory.
-  - alert: MemtxArenaWarning
+  - alert: LowMemtxArenaRemainingWarning
     expr: (tnt_slab_quota_used_ratio >= 80) and (tnt_slab_arena_used_ratio >= 80)
     for: 1m
     labels:
@@ -48,7 +48,7 @@ groups:
         Consider increasing memtx_memory or number of storages in case of sharded data."
 
   # Alert for any instance that have low remaining arena memory.
-  - alert: MemtxArenaAlert
+  - alert: LowMemtxArenaRemaining
     expr: (tnt_slab_quota_used_ratio >= 90) and (tnt_slab_arena_used_ratio >= 90)
     for: 1m
     labels:
@@ -60,7 +60,7 @@ groups:
       It is strongly recommended to increase memtx_memory or number of storages in case of sharded data."
 
   # Warning for any instance that have low remaining items memory.
-  - alert: MemtxItemsWarning
+  - alert: LowMemtxItemsRemainingWarning
     expr: (tnt_slab_quota_used_ratio >= 80) and (tnt_slab_items_used_ratio >= 80)
     for: 1m
     labels:
@@ -71,7 +71,7 @@ groups:
         Consider increasing memtx_memory or number of storages in case of sharded data."
 
   # Alert for any instance that have low remaining arena memory.
-  - alert: MemtxItemsAlert
+  - alert: LowMemtxItemsRemaining
     expr: (tnt_slab_quota_used_ratio >= 90) and (tnt_slab_items_used_ratio >= 90)
     for: 1m
     labels:
@@ -178,7 +178,7 @@ groups:
 - name: tarantool-crud
   rules:
   # Alert for CRUD module request errors.
-  - alert: CRUDHighErrorRate
+  - alert: HighCRUDErrorRate
     expr: rate(tnt_crud_stats_count{ job="tarantool", status="error" }[5m]) > 0.1
     for: 1m
     labels:
@@ -189,7 +189,7 @@ groups:
       '{{ $labels.alias }}' instance of job '{{ $labels.job }}' get module error responses."
 
   # Warning for CRUD module requests too long responses.
-  - alert: CRUDHighLatency
+  - alert: HighCRUDLatency
     expr: tnt_crud_stats{ job="tarantool", quantile="0.99" } > 0.1
     for: 1m
     labels:
@@ -200,7 +200,7 @@ groups:
       '{{ $labels.alias }}' instance of job '{{ $labels.job }}' are processed too long."
 
   # Warning for too many map reduce CRUD module requests.
-  - alert: CRUDHighMapReduceRate
+  - alert: HighCRUDMapReduceRate
     expr: rate(tnt_crud_map_reduces{ job="tarantool" }[5m]) > 0.1
     for: 1m
     labels:
@@ -218,7 +218,7 @@ groups:
   # Beware that metric name depends on name of the collector you use in HTTP metrics middleware
   # and request depends on type of this collector.
   # This example based on summary collector with default name.
-  - alert: HTTPHighLatency
+  - alert: HighHTTPLatency
     expr: http_server_request_latency{ job="tarantool", quantile="0.99" } > 0.1
     for: 5m
     labels:
@@ -228,43 +228,43 @@ groups:
       description: "Some {{ $labels.method }} requests to {{ $labels.path }} path with {{ $labels.status }} response status
         on '{{ $labels.alias }}' instance of job '{{ $labels.job }}' are processed too long."
 
-  # Warning for any endpoint of an instance in tarantool job that sends too much 4xx responses.
+  # Alert for any endpoint of an instance in tarantool job that sends too much 4xx responses.
   # Beware that metric name depends on name of the collector you use in HTTP metrics middleware
   # and request depends on type of this collector.
   # This example based on summary collector with default name.
-  - alert: HTTPHighClientErrorRateInstance
+  - alert: HighInstanceHTTPClientErrorRate
     expr: sum by (job, instance, method, path, alias) (rate(http_server_request_latency_count{ job="tarantool", status=~"^4\\d{2}$" }[5m])) > 10
     for: 1m
     labels:
-      severity: warning
+      severity: page
     annotations:
       summary: "Instance '{{ $labels.alias }}' ('{{ $labels.job }}') high rate of client error responses"
       description: "Too many {{ $labels.method }} requests to {{ $labels.path }} path 
         on '{{ $labels.alias }}' instance of job '{{ $labels.job }}' get client error (4xx) responses."
 
-  # Warning for any endpoint in tarantool job that sends too much 4xx responses (cluster overall).
+  # Alert for any endpoint in tarantool job that sends too much 4xx responses (cluster overall).
   # Beware that metric name depends on name of the collector you use in HTTP metrics middleware
   # and request depends on type of this collector.
   # This example based on summary collector with default name.
-  - alert: HTTPHighClientErrorRate
+  - alert: HighHTTPClientErrorRate
     expr: sum by (job, method, path) (rate(http_server_request_latency_count{ job="tarantool", status=~"^4\\d{2}$" }[5m])) > 20
     for: 1m
     labels:
-      severity: warning
+      severity: page
     annotations:
       summary: "Job '{{ $labels.job }}' high rate of client error responses"
       description: "Too many {{ $labels.method }} requests to {{ $labels.path }} path
         on instances of job '{{ $labels.job }}' get client error (4xx) responses."
 
-  # Warning for any endpoint of an instance in tarantool job that sends 5xx responses.
+  # Alert for any endpoint of an instance in tarantool job that sends 5xx responses.
   # Beware that metric name depends on name of the collector you use in HTTP metrics middleware
   # and request depends on type of this collector.
   # This example based on summary collector with default name.
-  - alert: HTTPServerErrors
+  - alert: HighHTTPServerErrorRate
     expr: sum by (job, instance, method, path, alias) (rate(http_server_request_latency_count{ job="tarantool", status=~"^5\\d{2}$" }[5m])) > 0
     for: 1m
     labels:
-      severity: warning
+      severity: page
     annotations:
       summary: "Instance '{{ $labels.alias }}' ('{{ $labels.job }}') server error responses"
       description: "Some {{ $labels.method }} requests to {{ $labels.path }} path 
@@ -274,7 +274,7 @@ groups:
   # Beware that metric name depends on name of the collector you use in HTTP metrics middleware
   # and request depends on type of this collector.
   # This example based on summary collector with default name.
-  - alert: HTTPLowRequestRateRouter
+  - alert: LowRouterHTTPRequestRate
     expr: sum by (job, instance, alias) (rate(http_server_request_latency_count{ job="tarantool", alias=~"^.*router.*$" }[5m])) < 10
     for: 5m
     labels:
diff --git a/example_cluster/prometheus/test_alerts.yml b/example_cluster/prometheus/test_alerts.yml
@@ -32,7 +32,7 @@ tests:
         values: '209715200+104857600x8' # 200 Mb + 100 Mb each interval
     alert_rule_test:
       - eval_time: 2m
-        alertname: LuaRuntimeWarning
+        alertname: HighLuaMemoryWarning
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -44,7 +44,7 @@ tests:
               description: "'tnt_router' instance of job 'tarantool' uses too much Lua memory
                 and may hit threshold soon."
       - eval_time: 2m
-        alertname: LuaRuntimeAlert
+        alertname: HighLuaMemory
         exp_alerts: # no alert firing
 
 
@@ -54,7 +54,7 @@ tests:
         values: '419430400+209715200x8' # 400 Mb + 200 Mb each interval
     alert_rule_test:
       - eval_time: 2m
-        alertname: LuaRuntimeWarning
+        alertname: HighLuaMemoryWarning
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -66,7 +66,7 @@ tests:
               description: "'tnt_router' instance of job 'tarantool' uses too much Lua memory
                 and may hit threshold soon."
       - eval_time: 2m
-        alertname: LuaRuntimeAlert
+        alertname: HighLuaMemory
         exp_alerts:
           - exp_labels:
               severity: page
@@ -87,10 +87,10 @@ tests:
         values: '92+0x2 76+0x8'
     alert_rule_test:
       - eval_time: 2m
-        alertname: MemtxArenaWarning
+        alertname: LowMemtxArenaRemainingWarning
         exp_alerts: # no alert firing
       - eval_time: 2m
-        alertname: MemtxArenaAlert
+        alertname: LowMemtxArenaRemaining
         exp_alerts: # no alert firing
 
 
@@ -102,7 +102,7 @@ tests:
         values: '92+0x2 82+0x8'
     alert_rule_test:
       - eval_time: 2m
-        alertname: MemtxArenaWarning
+        alertname: LowMemtxArenaRemainingWarning
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -114,7 +114,7 @@ tests:
               description: "Low arena memory (tuples and indexes) remaining for 'tnt_router' instance of job 'tarantool'.
                 Consider increasing memtx_memory or number of storages in case of sharded data."
       - eval_time: 2m
-        alertname: MemtxArenaAlert
+        alertname: LowMemtxArenaRemaining
         exp_alerts: # no alert firing
 
 
@@ -126,7 +126,7 @@ tests:
         values: '92+0x2 91+0x8'
     alert_rule_test:
       - eval_time: 2m
-        alertname: MemtxArenaWarning
+        alertname: LowMemtxArenaRemainingWarning
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -138,7 +138,7 @@ tests:
               description: "Low arena memory (tuples and indexes) remaining for 'tnt_router' instance of job 'tarantool'.
                 Consider increasing memtx_memory or number of storages in case of sharded data."
       - eval_time: 2m
-        alertname: MemtxArenaAlert
+        alertname: LowMemtxArenaRemaining
         exp_alerts:
           - exp_labels:
               severity: page
@@ -160,10 +160,10 @@ tests:
         values: '95+0x2 79+0x8'
     alert_rule_test:
       - eval_time: 2m
-        alertname: MemtxItemsWarning
+        alertname: LowMemtxItemsRemainingWarning
         exp_alerts: # no alert firing
       - eval_time: 2m
-        alertname: MemtxItemsAlert
+        alertname: LowMemtxItemsRemaining
         exp_alerts: # no alert firing
 
 
@@ -175,7 +175,7 @@ tests:
         values: '92+0x2 82+0x8'
     alert_rule_test:
       - eval_time: 2m
-        alertname: MemtxItemsWarning
+        alertname: LowMemtxItemsRemainingWarning
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -187,7 +187,7 @@ tests:
               description: "Low items memory (tuples) remaining for 'tnt_router' instance of job 'tarantool'.
                 Consider increasing memtx_memory or number of storages in case of sharded data."
       - eval_time: 2m
-        alertname: MemtxItemsAlert
+        alertname: LowMemtxItemsRemaining
         exp_alerts: # no alert firing
 
 
@@ -396,7 +396,7 @@ tests:
         values: '0+100x100'
     alert_rule_test:
       - eval_time: 5m
-        alertname: CRUDHighErrorRate
+        alertname: HighCRUDErrorRate
         exp_alerts:
           - exp_labels:
               severity: critical
@@ -418,7 +418,7 @@ tests:
         values: '0.11+0x0'
     alert_rule_test:
       - eval_time: 2m
-        alertname: CRUDHighLatency
+        alertname: HighCRUDLatency
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -441,7 +441,7 @@ tests:
         values: '0+100x100'
     alert_rule_test:
       - eval_time: 5m
-        alertname: CRUDHighMapReduceRate
+        alertname: HighCRUDMapReduceRate
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -471,7 +471,7 @@ tests:
           values: '0.11+0x60'
     alert_rule_test:
       - eval_time: 10m
-        alertname: HTTPHighLatency
+        alertname: HighHTTPLatency
         exp_alerts:
           - exp_labels:
               severity: warning
@@ -502,10 +502,10 @@ tests:
           values: '0.02+0x100'
     alert_rule_test:
       - eval_time: 5m
-        alertname: HTTPHighClientErrorRateInstance
+        alertname: HighInstanceHTTPClientErrorRate
         exp_alerts:
           - exp_labels:
-              severity: warning
+              severity: page
               instance: app:8081
               alias: tnt_router
               job: tarantool
@@ -551,13 +551,13 @@ tests:
           values: '0.02+0x100'
     alert_rule_test:
       - eval_time: 5m
-        alertname: HTTPHighClientErrorRateInstance
+        alertname: HighHTTPClientErrorRateInstance
         exp_alerts: # no alert firing
       - eval_time: 5m
-        alertname: HTTPHighClientErrorRate
+        alertname: HighHTTPClientErrorRate
         exp_alerts:
           - exp_labels:
-              severity: warning
+              severity: page
               job: tarantool
               path: /hell0
               method: GET
@@ -580,10 +580,10 @@ tests:
           values: '0+0x10 0.01+0x100'
     alert_rule_test:
       - eval_time: 5m
-        alertname: HTTPServerErrors
+        alertname: HighHTTPServerErrorRate
         exp_alerts:
           - exp_labels:
-              severity: warning
+              severity: page
               instance: app:8081
               alias: tnt_router
               job: tarantool
@@ -608,7 +608,7 @@ tests:
           values: '0+0x10 0.01+0x100'
     alert_rule_test:
       - eval_time: 15m
-        alertname: HTTPLowRequestRateRouter
+        alertname: LowRouterHTTPRequestRate
         exp_alerts:
           - exp_labels:
               severity: warning
