Merge pull request #48 from tarantool/46-prometheus-rate-2

Make Prometheus rps graphs rate() time range configurable

Author: DifferentialOrange

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## Changed
+- Make Prometheus rps graphs rate() time range configurable
+
 ## [0.2.1] - 2020-12-11
 Grafana revisions: [Prometheus revision 2](https://grafana.com/api/dashboards/13054/revisions/2/download)
 

README.md

@@ -20,7 +20,7 @@ Guide on Tarantool project prerequirements and metrics collectors configuration
     * Paste link to dashboard: https://grafana.com/grafana/dashboards/12567 for InfluxDB, https://grafana.com/grafana/dashboards/13054 for Prometheus;
     * Upload json file or paste json file contents: download it from Grafana page ([InfluxDB](https://grafana.com/grafana/dashboards/12567), [Prometheus](https://grafana.com/grafana/dashboards/13054)) or build it by yourself.
 
-1. Set dashboard name, folder, uid (if needed), and metrics database parameters (i.e. InfluxDB source, measurement and policy or Prometheus source and job).
+1. Set dashboard name, folder, uid (if needed), and database-related query parameters (InfluxDB source, measurement and policy or Prometheus source, job and rate() time range).
 <br/><br/>![Grafana import setup in v6.6.0 for InfluxDB](./docs/grafana_import_setup_v6.png)
 
 # How to build

example/prometheus/prometheus.yml

@@ -1,7 +1,7 @@
 # my global config
 global:
-  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  scrape_interval: 1m # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 1m # Evaluate rules every 15 seconds. The default is every 1 minute.
   # scrape_timeout is set to the global default (10s).
 
 # Alertmanager configuration

tarantool/cluster.libsonnet

@@ -230,6 +230,7 @@ local prometheus = grafana.prometheus;
     datasource=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: overview_stat(
     title=title,
     description=(
@@ -251,7 +252,7 @@ local prometheus = grafana.prometheus;
     stat_title='Overall space load:',
     decimals=2,
     unit='ops',
-    expr=std.format('sum(rate(tnt_stats_op_total{job=~"%s"}[1m]))', job),
+    expr=std.format('sum(rate(tnt_stats_op_total{job=~"%s"}[%s]))', [job, rate_time_range]),
   ),
 
   http_rps_stat(
@@ -261,6 +262,7 @@ local prometheus = grafana.prometheus;
     datasource=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: overview_stat(
     title=title,
     description=(
@@ -282,6 +284,6 @@ local prometheus = grafana.prometheus;
     stat_title='Overall HTTP load:',
     decimals=2,
     unit='reqps',
-    expr=std.format('sum(rate(http_server_request_latency_count{job=~"%s"}[1m]))', job),
+    expr=std.format('sum(rate(http_server_request_latency_count{job=~"%s"}[%s]))', [job, rate_time_range]),
   ),
 }

tarantool/dashboard.libsonnet

@@ -12,6 +12,7 @@ local row = grafana.row;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
     offset=0
   )::
     dashboard
@@ -43,6 +44,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 0, y: 1 + offset }
     )
@@ -52,6 +54,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 8, y: 1 + offset },
     )
@@ -61,6 +64,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 16, y: 1 + offset },
     )
@@ -190,6 +194,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 0, y: 46 + offset },
     )
@@ -199,6 +204,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 8, y: 46 + offset },
     )
@@ -208,6 +214,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 16, y: 46 + offset },
     )
@@ -217,6 +224,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 0, y: 46 + offset },
     )
@@ -226,6 +234,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 8, y: 46 + offset },
     )
@@ -235,6 +244,7 @@ local row = grafana.row;
         policy=policy,
         measurement=measurement,
         job=job,
+        rate_time_range=rate_time_range,
       ),
       { w: 8, h: 8, x: 16, y: 46 + offset },
     ),

tarantool/http.libsonnet

@@ -12,6 +12,7 @@ local prometheus = grafana.prometheus;
     policy,
     measurement,
     job,
+    rate_time_range,
     metric_name,
     status_regex,
   ) = graph.new(
@@ -35,7 +36,8 @@ local prometheus = grafana.prometheus;
   ).addTarget(
     if datasource == '${DS_PROMETHEUS}' then
       prometheus.target(
-        expr=std.format('rate(%s{job=~"%s",status=~"%s"}[1m])', [metric_name, job, std.strReplace(status_regex, '\\', '\\\\')]),
+        expr=std.format('rate(%s{job=~"%s",status=~"%s"}[%s])',
+                        [metric_name, job, std.strReplace(status_regex, '\\', '\\\\'), rate_time_range]),
         legendFormat='{{alias}} — {{method}} {{path}} (code {{status}})',
       )
     else if datasource == '${DS_INFLUXDB}' then
@@ -59,6 +61,7 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
     metric_name='http_server_request_latency_count',
   ):: rps_graph(
     title=title,
@@ -67,6 +70,7 @@ local prometheus = grafana.prometheus;
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     metric_name=metric_name,
     status_regex='^2\\d{2}$',
   ),
@@ -82,6 +86,7 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
     metric_name='http_server_request_latency_count',
   ):: rps_graph(
     title=title,
@@ -90,6 +95,7 @@ local prometheus = grafana.prometheus;
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     metric_name=metric_name,
     status_regex='^4\\d{2}$',
   ),
@@ -105,6 +111,7 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
     metric_name='http_server_request_latency_count',
   ):: rps_graph(
     title=title,
@@ -113,6 +120,7 @@ local prometheus = grafana.prometheus;
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     metric_name=metric_name,
     status_regex='^5\\d{2}$',
   ),

tarantool/prometheus_dashboard.jsonnet

@@ -15,6 +15,7 @@ local raw_dashboard = grafana.dashboard.new(
 );
 
 local datasource = '${DS_PROMETHEUS}';
+local rate_time_range = '$rate_time_range';
 local job = '[[job]]';
 
 dashboard.build(
@@ -35,6 +36,13 @@ dashboard.build(
     pluginName=null,
     description='Prometheus Tarantool metrics job'
   )
+  .addInput(
+    name='PROMETHEUS_RATE_TIME_RANGE',
+    label='Rate time range',
+    type='constant',
+    value='2m',
+    description='Time range for computing rps graphs with rate(). At the very minimum it should be two times the scrape interval.'
+  )
   .addRequired(
     type='datasource',
     id='prometheus',
@@ -61,6 +69,15 @@ dashboard.build(
       hide='variable',
       label='Prometheus job',
     ),
+  )
+  .addTemplate(
+    grafana.template.custom(
+      name='rate_time_range',
+      query='${PROMETHEUS_RATE_TIME_RANGE}',
+      current='${PROMETHEUS_RATE_TIME_RANGE}',
+      hide='variable',
+      label='rate() time range',
+    ),
   ).addPanel(
     cluster.health_overview_table(
       datasource=datasource,
@@ -89,16 +106,19 @@ dashboard.build(
     cluster.space_ops_stat(
       datasource=datasource,
       job=job,
+      rate_time_range=rate_time_range,
     ),
     { w: 4, h: 4, x: 20, y: 0 }
   ).addPanel(
     cluster.http_rps_stat(
       datasource=datasource,
       job=job,
+      rate_time_range=rate_time_range,
     ),
     { w: 4, h: 4, x: 20, y: 4 }
   ),
   datasource,
   job=job,
+  rate_time_range=rate_time_range,
   offset=8,
 )

tarantool/space.libsonnet

@@ -12,6 +12,7 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
     operation=null,
   ) = graph.new(
     title=(if title != null then title else std.format('%s requests', std.asciiUpper(operation))),
@@ -39,7 +40,7 @@ local prometheus = grafana.prometheus;
   ).addTarget(
     if datasource == '${DS_PROMETHEUS}' then
       prometheus.target(
-        expr=std.format('rate(tnt_stats_op_total{job=~"%s",operation="%s"}[1m])', [job, operation]),
+        expr=std.format('rate(tnt_stats_op_total{job=~"%s",operation="%s"}[%s])', [job, operation, rate_time_range]),
         legendFormat='{{alias}}'
       )
     else if datasource == '${DS_INFLUXDB}' then
@@ -59,13 +60,15 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: operation_rps(
     title=title,
     description=description,
     datasource=datasource,
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     operation='select'
   ),
 
@@ -76,13 +79,15 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: operation_rps(
     title=title,
     description=description,
     datasource=datasource,
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     operation='insert'
   ),
 
@@ -93,13 +98,15 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: operation_rps(
     title=title,
     description=description,
     datasource=datasource,
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     operation='replace'
   ),
 
@@ -110,13 +117,15 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: operation_rps(
     title=title,
     description=description,
     datasource=datasource,
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     operation='upsert'
   ),
 
@@ -127,13 +136,15 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: operation_rps(
     title=title,
     description=description,
     datasource=datasource,
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     operation='update'
   ),
 
@@ -144,13 +155,15 @@ local prometheus = grafana.prometheus;
     policy=null,
     measurement=null,
     job=null,
+    rate_time_range=null,
   ):: operation_rps(
     title=title,
     description=description,
     datasource=datasource,
     policy=policy,
     measurement=measurement,
     job=job,
+    rate_time_range=rate_time_range,
     operation='delete'
   ),
 }