Skip to content
This repository was archived by the owner on Apr 28, 2025. It is now read-only.

Commit 2ba24fe

Browse files
pracuccipracucci
authored
Merge pull request #272 from grafana/add-slow-queries-dashboard
Add slow queries dashboard
2 parents d2c553f + 99d607f commit 2ba24fe

File tree

3 files changed

+187
-0
lines changed

3 files changed

+187
-0
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
git fetch origin
99
git branch -u origin/main main
1010
```
11+
* [FEATURE] Added "Cortex / Slow queries" dashboard based on Loki logs. #271
1112
* [ENHANCEMENT] Add `EtcdAllocatingTooMuchMemory` alert for monitoring etcd memory usage. #261
1213
* [ENHANCEMENT] Sort legend descending in the CPU/memory panels. #271
1314
* [BUGFIX] Fixed `CortexQuerierHighRefetchRate` alert. #268

‎cortex-mixin/dashboards.libsonnet

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
(import 'dashboards/alertmanager.libsonnet') +
88
(import 'dashboards/scaling.libsonnet') +
99
(import 'dashboards/writes.libsonnet') +
10+
(import 'dashboards/slow-queries.libsonnet') +
1011

1112
(if std.member($._config.storage_engine, 'blocks')
1213
then

‎cortex-mixin/dashboards/slow-queries.libsonnet

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
local utils = import 'mixin-utils/utils.libsonnet';
2+
3+
(import 'dashboard-utils.libsonnet') {
4+
'cortex-slow-queries.json':
5+
($.dashboard('Cortex / Slow Queries') + { uid: 'e6f3091e29d2636e3b8393447e925668' })
6+
.addClusterSelectorTemplates(false)
7+
.addRow(
8+
$.row('')
9+
.addPanel(
10+
{
11+
title: 'Slow queries',
12+
type: 'table',
13+
datasource: '${lokidatasource}',
14+
15+
// Query logs from Loki.
16+
targets: [
17+
{
18+
// Filter out the remote read endpoint.
19+
expr: '{cluster=~"$cluster",namespace=~"$namespace",name="query-frontend"} |= "query stats" != "/api/v1/read" | logfmt | org_id=~"${tenant_id}" | response_time > ${min_duration}',
20+
instant: false,
21+
legendFormat: '',
22+
range: true,
23+
refId: 'A',
24+
},
25+
],
26+
27+
// Use Grafana transformations to display fields in a table.
28+
transformations: [
29+
{
30+
// Convert labels to fields.
31+
id: 'labelsToFields',
32+
options: {},
33+
},
34+
{
35+
// Compute the query time range.
36+
id: 'calculateField',
37+
options: {
38+
alias: 'Time range',
39+
mode: 'binary',
40+
binary: {
41+
left: 'param_end',
42+
operator: '-',
43+
reducer: 'sum',
44+
right: 'param_start',
45+
},
46+
reduce: { reducer: 'sum' },
47+
replaceFields: false,
48+
},
49+
},
50+
{
51+
id: 'organize',
52+
options: {
53+
// Hide fields we don't care.
54+
local hiddenFields = ['caller', 'cluster', 'container', 'host', 'id', 'job', 'level', 'line', 'method', 'msg', 'name', 'namespace', 'param_end', 'param_start', 'param_time', 'path', 'pod', 'pod_template_hash', 'query_wall_time_seconds', 'stream', 'traceID', 'tsNs'],
55+
56+
excludeByName: {
57+
[field]: true
58+
for field in hiddenFields
59+
},
60+
61+
// Order fields.
62+
local orderedFields = ['ts', 'org_id', 'param_query', 'Time range', 'param_step', 'response_time'],
63+
64+
indexByName: {
65+
[orderedFields[i]]: i
66+
for i in std.range(0, std.length(orderedFields) - 1)
67+
},
68+
69+
// Rename fields.
70+
renameByName: {
71+
org_id: 'Tenant ID',
72+
param_query: 'Query',
73+
param_step: 'Step',
74+
response_time: 'Duration',
75+
},
76+
},
77+
},
78+
],
79+
80+
fieldConfig: {
81+
// Configure overrides to nicely format field values.
82+
overrides: [
83+
{
84+
matcher: { id: 'byName', options: 'Time range' },
85+
properties: [
86+
{
87+
id: 'mappings',
88+
value: [
89+
{
90+
from: '',
91+
id: 1,
92+
text: 'Instant query',
93+
to: '',
94+
type: 1,
95+
value: '0',
96+
},
97+
],
98+
},
99+
{ id: 'unit', value: 's' },
100+
],
101+
},
102+
{
103+
matcher: { id: 'byName', options: 'Step' },
104+
properties: [{ id: 'unit', value: 's' }],
105+
},
106+
],
107+
},
108+
},
109+
)
110+
)
111+
+ {
112+
templating+: {
113+
list+: [
114+
// Add the Loki datasource.
115+
{
116+
type: 'datasource',
117+
name: 'lokidatasource',
118+
label: 'Logs datasource',
119+
query: 'loki',
120+
hide: 0,
121+
includeAll: false,
122+
multi: false,
123+
},
124+
// Add a variable to configure the min duration.
125+
{
126+
local defaultValue = '5s',
127+
128+
type: 'textbox',
129+
name: 'min_duration',
130+
label: 'Min duration',
131+
hide: 0,
132+
options: [
133+
{
134+
selected: true,
135+
text: defaultValue,
136+
value: defaultValue,
137+
},
138+
],
139+
current: {
140+
// Default value.
141+
selected: true,
142+
text: defaultValue,
143+
value: defaultValue,
144+
},
145+
query: defaultValue,
146+
},
147+
// Add a variable to configure the tenant to filter on.
148+
{
149+
local defaultValue = '.*',
150+
151+
type: 'textbox',
152+
name: 'tenant_id',
153+
label: 'Tenant ID',
154+
hide: 0,
155+
options: [
156+
{
157+
selected: true,
158+
text: defaultValue,
159+
value: defaultValue,
160+
},
161+
],
162+
current: {
163+
// Default value.
164+
selected: true,
165+
text: defaultValue,
166+
value: defaultValue,
167+
},
168+
query: defaultValue,
169+
},
170+
],
171+
},
172+
} + {
173+
templating+: {
174+
list: [
175+
// Do not allow to include all clusters/namespaces otherwise this dashboard
176+
// risks to explode because it shows resources per pod.
177+
l + (if (l.name == 'cluster' || l.name == 'namespace') then { includeAll: false } else {})
178+
for l in super.list
179+
],
180+
},
181+
} + {
182+
// No auto-refresh by default.
183+
refresh: '',
184+
},
185+
}

0 commit comments

Comments
 (0)