Added per pod metrics to read dashboard

Signed-off-by: Joe Elliott <[email protected]>

Author: joe-elliott

cortex-mixin/config.libsonnet

@@ -44,6 +44,6 @@
     resources_dashboards_enabled: false,
 
     // If supplied adds additional panels that are broken down per instance.  i.e. 'pod' in a kubernetes install
-    per_instance_label: '',
+    per_instance_label: 'pod',
   },
 }

cortex-mixin/dashboards/reads.libsonnet

@@ -14,6 +14,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.gateway) + [utils.selector.re('route', 'api_prom_api_v1_.+')])
       )
+      .addPanelIf(
+        $._config.per_instance_label != '',
+        $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"api_prom_api_v1_.+"}[$__interval])))' % [$._config.per_instance_label, $.jobMatcherEquality($._config.job_names.gateway)], ''
+        ) +
+        { yaxes: $.yaxes('s') }
+      )
     )
     .addRow(
       $.row('Query Frontend')
@@ -25,6 +33,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.query_frontend) + [utils.selector.re('route', 'api_prom_api_v1_.+')])
       )
+      .addPanelIf(
+        $._config.per_instance_label != '',
+        $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"api_prom_api_v1_.+"}[$__interval])))' % [$._config.per_instance_label, $.jobMatcherEquality($._config.job_names.query_frontend)], ''
+        ) +
+        { yaxes: $.yaxes('s') }
+      )
     )
     .addRow(
       $.row('Cache - Query Results')
@@ -47,6 +63,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.querier) + [utils.selector.re('route', 'api_prom_api_v1_.+')])
       )
+      .addPanelIf(
+        $._config.per_instance_label != '',
+        $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"api_prom_api_v1_.+"}[$__interval])))' % [$._config.per_instance_label, $.jobMatcherEquality($._config.job_names.querier)], ''
+        ) +
+        { yaxes: $.yaxes('s') }
+      )
     )
     .addRow(
       $.row('Ingester')
@@ -58,6 +82,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.re('route', '/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata')])
       )
+      .addPanelIf(
+        $._config.per_instance_label != '',
+        $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata"}[$__interval])))' % [$._config.per_instance_label, $.jobMatcherEquality($._config.job_names.ingester)], ''
+        ) +
+        { yaxes: $.yaxes('s') }
+      )
     )
     .addRowIf(
       std.member($._config.storage_engine, 'blocks'),
@@ -70,6 +102,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.store_gateway) + [utils.selector.re('route', '/gatewaypb.StoreGateway/.*')])
       )
+      .addPanelIf(
+        $._config.per_instance_label != '',
+        $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"/gatewaypb.StoreGateway/.*"}[$__interval])))' % [$._config.per_instance_label, $.jobMatcherEquality($._config.job_names.store_gateway)], ''
+        ) +
+        { yaxes: $.yaxes('s') }
+      )
     )
     .addRowIf(
       std.member($._config.storage_engine, 'chunks'),