Skip to content
This repository was archived by the owner on Apr 28, 2025. It is now read-only.

Commit 903f1b5

Browse files
pracuccipracucci
committed
Add slow queries dashboard
Signed-off-by: Marco Pracucci <[email protected]>
1 parent d2c553f commit 903f1b5

File tree

3 files changed

+186
-0
lines changed

3 files changed

+186
-0
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
git fetch origin
99
git branch -u origin/main main
1010
```
11+
* [FEATURE] Added "Cortex / Slow queries" dashboard based on Loki logs. #271
1112
* [ENHANCEMENT] Add `EtcdAllocatingTooMuchMemory` alert for monitoring etcd memory usage. #261
1213
* [ENHANCEMENT] Sort legend descending in the CPU/memory panels. #271
1314
* [BUGFIX] Fixed `CortexQuerierHighRefetchRate` alert. #268

‎cortex-mixin/dashboards.libsonnet

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
(import 'dashboards/alertmanager.libsonnet') +
88
(import 'dashboards/scaling.libsonnet') +
99
(import 'dashboards/writes.libsonnet') +
10+
(import 'dashboards/slow-queries.libsonnet') +
1011

1112
(if std.member($._config.storage_engine, 'blocks')
1213
then

‎cortex-mixin/dashboards/slow-queries.libsonnet

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
local utils = import 'mixin-utils/utils.libsonnet';
2+
3+
(import 'dashboard-utils.libsonnet') {
4+
'cortex-slow-queries.json':
5+
($.dashboard('Cortex / Slow Queries') + { uid: 'e6f3091e29d2636e3b8393447e925668' })
6+
.addClusterSelectorTemplates(false)
7+
.addRow(
8+
$.row('')
9+
.addPanel(
10+
{
11+
title: 'Slow queries',
12+
type: 'table',
13+
datasource: '${lokidatasource}',
14+
15+
// Query logs from Loki.
16+
targets: [
17+
{
18+
// Filter out the remote read endpoint.
19+
expr: '{cluster=~"$cluster",namespace=~"$namespace",name="query-frontend"} |= "query stats" != "/api/v1/read" | logfmt | org_id=~"${tenant_id}" | response_time > ${min_duration}',
20+
instant: false,
21+
legendFormat: '',
22+
range: true,
23+
refId: 'A',
24+
},
25+
],
26+
27+
// Use Grafana transformations to display fields in a table.
28+
transformations: [
29+
{
30+
// Convert labels to fields.
31+
id: 'labelsToFields',
32+
options: {},
33+
},
34+
{
35+
// Compute the query time range.
36+
id: 'calculateField',
37+
options: {
38+
alias: 'Time range',
39+
mode: 'binary',
40+
binary: {
41+
left: 'param_end',
42+
operator: '-',
43+
reducer: 'sum',
44+
right: 'param_start',
45+
},
46+
reduce: { reducer: 'sum' },
47+
replaceFields: false,
48+
},
49+
},
50+
{
51+
id: 'organize',
52+
options: {
53+
// Hide fields we don't care.
54+
local hiddenFields = ['caller', 'cluster', 'container', 'host', 'id', 'job', 'level', 'line', 'method', 'msg', 'name', 'namespace', 'org_id', 'param_end', 'param_start', 'param_time', 'path', 'pod', 'pod_template_hash', 'query_wall_time_seconds', 'stream', 'traceID', 'tsNs'],
55+
56+
excludeByName: {
57+
[field]: true
58+
for field in hiddenFields
59+
},
60+
61+
// Order fields.
62+
local orderedFields = ['ts', 'param_query', 'Time range', 'param_step', 'response_time'],
63+
64+
indexByName: {
65+
[orderedFields[i]]: i
66+
for i in std.range(0, std.length(orderedFields) - 1)
67+
},
68+
69+
// Rename fields.
70+
renameByName: {
71+
param_query: 'Query',
72+
param_step: 'Step',
73+
response_time: 'Duration',
74+
},
75+
},
76+
},
77+
],
78+
79+
fieldConfig: {
80+
// Configure overrides to nicely format field values.
81+
overrides: [
82+
{
83+
matcher: { id: 'byName', options: 'Time range' },
84+
properties: [
85+
{
86+
id: 'mappings',
87+
value: [
88+
{
89+
from: '',
90+
id: 1,
91+
text: 'Instant query',
92+
to: '',
93+
type: 1,
94+
value: '0',
95+
},
96+
],
97+
},
98+
{ id: 'unit', value: 's' },
99+
],
100+
},
101+
{
102+
matcher: { id: 'byName', options: 'Step' },
103+
properties: [{ id: 'unit', value: 's' }],
104+
},
105+
],
106+
},
107+
},
108+
)
109+
)
110+
+ {
111+
templating+: {
112+
list+: [
113+
// Add the Loki datasource.
114+
{
115+
type: 'datasource',
116+
name: 'lokidatasource',
117+
label: 'Logs datasource',
118+
query: 'loki',
119+
hide: 0,
120+
includeAll: false,
121+
multi: false,
122+
},
123+
// Add a variable to configure the min duration.
124+
{
125+
local defaultValue = '5s',
126+
127+
type: 'textbox',
128+
name: 'min_duration',
129+
label: 'Min duration',
130+
hide: 0,
131+
options: [
132+
{
133+
selected: true,
134+
text: defaultValue,
135+
value: defaultValue,
136+
},
137+
],
138+
current: {
139+
// Default value.
140+
selected: true,
141+
text: defaultValue,
142+
value: defaultValue,
143+
},
144+
query: defaultValue,
145+
},
146+
// Add a variable to configure the tenant to filter on.
147+
{
148+
local defaultValue = '.*',
149+
150+
type: 'textbox',
151+
name: 'tenant_id',
152+
label: 'Tenant ID',
153+
hide: 0,
154+
options: [
155+
{
156+
selected: true,
157+
text: defaultValue,
158+
value: defaultValue,
159+
},
160+
],
161+
current: {
162+
// Default value.
163+
selected: true,
164+
text: defaultValue,
165+
value: defaultValue,
166+
},
167+
query: defaultValue,
168+
},
169+
],
170+
},
171+
} + {
172+
templating+: {
173+
list: [
174+
// Do not allow to include all clusters/namespaces otherwise this dashboard
175+
// risks to explode because it shows resources per pod.
176+
l + (if (l.name == 'cluster' || l.name == 'namespace') then { includeAll: false } else {})
177+
for l in super.list
178+
],
179+
},
180+
} + {
181+
// No auto-refresh by default.
182+
refresh: '',
183+
},
184+
}

0 commit comments

Comments
 (0)