Cortex writes dashboard: add panels for KV activity

For HA-tracker, show which tenants are changing election.
For ingester, show how many are active, leaving, etc.

Author: bboreham

CHANGELOG.md

@@ -23,6 +23,7 @@
 * [CHANGE] Increased `CortexIngesterReachingSeriesLimit` critical alert threshold from 80% to 85%. #363
 * [CHANGE] Decreased `-server.grpc-max-concurrent-streams` from 100k to 10k. #369
 * [CHANGE] Decreased blocks storage ingesters graceful termination period from 80m to 20m. #369
+* [ENHANCEMENT] Writes dashboard: fix HA-tracker KV panels; add elections panel and ingester state panel. #371
 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. #317
 * [ENHANCEMENT] Cortex-mixin: Include `cortex-gw-internal` naming variation in default `gateway` job names. #328
 * [ENHANCEMENT] Ruler dashboard: added object storage metrics. #354

cortex-mixin/dashboards/writes.libsonnet

@@ -110,6 +110,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])
       )
+      .addPanel(
+        $.panel('Elected replica changes / min') +
+        $.queryPanel([
+          'max by(exported_cluster, user)(increase(cortex_ha_tracker_elected_replica_changes_total{%s}[1m])) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{user}}/{{exported_cluster}}',
+        ]) +
+        $.stack + {
+          yaxes: $.yaxes('cpm'),
+        },
       )
     )
     .addRow(
@@ -138,8 +148,15 @@ local utils = import 'mixin-utils/utils.libsonnet';
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester)+ [utils.selector.eq('kv_name', 'ingester-lifecycler')])
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('kv_name', 'ingester-lifecycler')])
       )
+      .addPanel(
+        $.panel('Ingester status') +
+        $.queryPanel([
+          'max by (state)(cortex_ring_members{%s}) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{state}}',
+        ])
       )
     )
     .addRowIf(