Added per pod writes to distributors and ingesters

Signed-off-by: Joe Elliott <[email protected]>

Author: joe-elliott

cortex-mixin/dashboards/writes.libsonnet

@@ -59,6 +59,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.re('route', '/httpgrpc.*|api_prom_push')])
       )
+      .addPanelIf(
+        $._config.per_instance_label != '',
+        $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"/httpgrpc.*|api_prom_push"}[$__interval])))' % [$._config.per_instance_label, $.jobMatcherEquality($._config.job_names.distributor)], ''
+        ) +
+        { yaxes: $.yaxes('s') }
+      )
     )
     .addRow(
       $.row('KV Store (HA Dedupe)')
@@ -81,6 +89,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('route', '/cortex.Ingester/Push')])
       )
+      .addPanelIf(
+        $._config.per_instance_label != '',
+        $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route="/cortex.Ingester/Push"}[$__interval])))' % [$._config.per_instance_label, $.jobMatcherEquality($._config.job_names.ingester)], ''
+        ) +
+        { yaxes: $.yaxes('s') }
+      )
     )
     .addRow(
       $.row('KV Store (Ring)')