Merge pull request #70 from tarantool/55-cluster-issues

Add Cartridge issues panels and Prometheus alerts

Author: DifferentialOrange

CHANGELOG.md

@@ -7,14 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 
 ## Added
-- Prometheus example alert rules (instance state, memory usage, HTTP load and latency rule examples)
+- Prometheus example alert rules (instance state, memory usage, HTTP load and latency rule examples, etc)
 - Test Prometheus example alert rules with promtool
 - Cartridge issues metrics labels to Telegraf configuration
+- Cartridge issues panels and "Cluster overview" row
 
 ## Changed
 - Update metrics version to 0.9.0
 - Separate app cluster and load generator in example docker stand
 - Use cartridge-cli to run and setup example app cluster instead of luatest
+- Group Prometheus cluster overview panels into "Cluster overview" row
 
 ## Fixed
 - Add missing space and replication metrics labels to Telegraf configuration

example/prometheus/alerts.yml

@@ -79,8 +79,31 @@ groups:
     annotations:
       summary: "Instance {{ $labels.alias }} low items memory remaining"
       description: "Low items memory (tuples) remaining for {{ $labels.alias }} instance of job {{ $labels.job }}.
-      You are likely to hit limit soon.
-      It is strongly recommended to increase memtx_memory or number of storages in case of sharded data."
+        You are likely to hit limit soon.
+        It is strongly recommended to increase memtx_memory or number of storages in case of sharded data."
+
+   # Warning for Cartridge warning issues.
+  - alert: CartridgeWarningIssues
+    expr: tnt_cartridge_issues{level="warning"} > 0
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Instance {{ $labels.alias }} have 'warning'-level Cartridge issues"
+      description: "Possible reasons: high replication lag, replication long idle,
+        failover or switchover issues, clock issues, memory fragmentation,
+        configuration issues, alien members."
+
+  # Alert for Cartridge critical issues.
+  - alert: CartridgeCriticalIssues
+    expr: tnt_cartridge_issues{level="critical"} > 0
+    for: 1m
+    labels:
+      severity: page
+    annotations:
+      summary: "Instance {{ $labels.alias }} have 'critical'-level Cartridge issues"
+      description: "Possible reasons: replication process critical fail,
+        running out of available memory."
 
 - name: tarantool-business
   rules:

example/prometheus/test_alerts.yml

@@ -191,6 +191,68 @@ tests:
         exp_alerts: # no alert firing
 
 
+  - interval: 15s
+    input_series:
+      - series: 'tnt_cartridge_issues{job="tarantool_app", instance="app:8081", alias="tnt_router", level="warning"}'
+        values: '0+0x2 1+0x8'
+      - series: 'tnt_cartridge_issues{job="tarantool_app", instance="app:8081", alias="tnt_router", level="critical"}'
+        values: '0+0x10'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: CartridgeWarningIssues
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              level: warning
+              instance: app:8081
+              alias: tnt_router
+              job: tarantool_app
+            exp_annotations:
+              summary: "Instance tnt_router have 'warning'-level Cartridge issues"
+              description: "Possible reasons: high replication lag, replication long idle,
+                failover or switchover issues, clock issues, memory fragmentation,
+                configuration issues, alien members."
+      - eval_time: 2m
+        alertname: CartridgeCriticalIssues
+        exp_alerts: # no alert firing
+
+
+  - interval: 15s
+    input_series:
+      - series: 'tnt_cartridge_issues{job="tarantool_app", instance="app:8081", alias="tnt_router", level="warning"}'
+        values: '0+0x2 2+0x8'
+      - series: 'tnt_cartridge_issues{job="tarantool_app", instance="app:8081", alias="tnt_router", level="critical"}'
+        values: '1+0x10'
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: CartridgeWarningIssues
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              level: warning
+              instance: app:8081
+              alias: tnt_router
+              job: tarantool_app
+            exp_annotations:
+              summary: "Instance tnt_router have 'warning'-level Cartridge issues"
+              description: "Possible reasons: high replication lag, replication long idle,
+                failover or switchover issues, clock issues, memory fragmentation,
+                configuration issues, alien members."
+      - eval_time: 2m
+        alertname: CartridgeCriticalIssues
+        exp_alerts:
+          - exp_labels:
+              severity: page
+              level: critical
+              instance: app:8081
+              alias: tnt_router
+              job: tarantool_app
+            exp_annotations:
+              summary: "Instance tnt_router have 'critical'-level Cartridge issues"
+              description: "Possible reasons: replication process critical fail,
+                running out of available memory."
+
+
   - interval: 15s
     input_series:
         - series: http_server_request_latency_count{job="tarantool_app",instance="app:8081",path="/hello",method="GET",status="200",alias="tnt_router"}

tarantool/cluster.libsonnet

@@ -1,7 +1,9 @@
 local grafana = import 'grafonnet/grafana.libsonnet';
 
+local graph = grafana.graphPanel;
 local statPanel = grafana.statPanel;
 local tablePanel = grafana.tablePanel;
+local influxdb = grafana.influxdb;
 local prometheus = grafana.prometheus;
 
 {
@@ -286,4 +288,86 @@ local prometheus = grafana.prometheus;
     unit='reqps',
     expr=std.format('sum(rate(http_server_request_latency_count{job=~"%s"}[%s]))', [job, rate_time_range]),
   ),
+
+  local cartridge_issues(
+    title,
+    description,
+    datasource,
+    policy,
+    measurement,
+    job,
+    level,
+  ) = graph.new(
+    title=title,
+    description=description,
+    datasource=datasource,
+
+    format='none',
+    fill=0,
+    decimals=0,
+    sort='decreasing',
+    legend_alignAsTable=true,
+    legend_current=true,
+    legend_values=true,
+    legend_sort='current',
+    legend_sortDesc=true,
+  ).addTarget(
+    if datasource == '${DS_PROMETHEUS}' then
+      prometheus.target(
+        expr=std.format('tnt_cartridge_issues{job=~"%s",level="%s"}', [job, level]),
+        legendFormat='{{alias}}',
+      )
+    else if datasource == '${DS_INFLUXDB}' then
+      influxdb.target(
+        policy=policy,
+        measurement=measurement,
+        group_tags=['label_pairs_alias'],
+        alias='$tag_label_pairs_alias',
+      ).where('metric_name', '=', 'tnt_cartridge_issues').where('label_pairs_level', '=', level)
+      .selectField('value').addConverter('last')
+  ),
+
+  cartridge_warning_issues(
+    title='Cartridge warning issues',
+    description=|||
+      Number of "warning" issues on each cluster instance.
+      "warning" issues includes high replication lag, replication long idle,
+      failover and switchover issues, clock issues, memory fragmentation,
+      configuration issues and alien members warnings.
+    |||,
+    datasource=null,
+    policy=null,
+    measurement=null,
+    job=null,
+  ):: cartridge_issues(
+    title=title,
+    description=description,
+    datasource=datasource,
+    policy=policy,
+    measurement=measurement,
+    job=job,
+    level='warning',
+  ),
+
+  cartridge_critical_issues(
+    title='Cartridge critical issues',
+    description=|||
+      Number of "critical" issues on each cluster instance.
+      "critical" issues includes replication process critical fails and
+      running out of available memory.
+    |||,
+    datasource=null,
+    policy=null,
+    measurement=null,
+    job=null,
+  ):: cartridge_issues(
+    title=title,
+    description=description,
+    datasource=datasource,
+    policy=policy,
+    measurement=measurement,
+    job=job,
+    level='critical',
+  ),
+
 }