Skip to content

Commit d5a21db

Browse files
alexey-milovidovalexey-milovidov
authored
Merge pull request ClickHouse#79978 from ClickHouse/query-latency-analyzer
Add tool for query latency analyzing
2 parents 654dc82 + 2bd9484 commit d5a21db

File tree

4 files changed

+768
-0
lines changed

4 files changed

+768
-0
lines changed

‎utils/query-latency-analyzer/index.html‎

Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
6+
<title>Latency Analyzer</title>
7+
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-4Q6Gf2aSP4eDXB8Miphtr37CMZZQ5oXLH2yaXMJ2w8e2ZtHTl7GptT4jmndRuHDT" crossorigin="anonymous">
8+
<style>
9+
.matrix-legend {
10+
margin-left: 20px;
11+
margin-right: 20px;
12+
}
13+
.cell {
14+
stroke: #ccc;
15+
}
16+
.cell.unlight {
17+
opacity: 0.2;
18+
}
19+
.axis text {
20+
font-size: 12px;
21+
}
22+
.matrix-tooltip {
23+
position: absolute;
24+
background: rgba(0, 0, 0, 0.7);
25+
color: white;
26+
padding: 5px;
27+
border-radius: 5px;
28+
z-index: 2000;
29+
}
30+
31+
#hourglass
32+
{
33+
display: none;
34+
font-size: 110%;
35+
color: #888;
36+
animation: hourglass-animation 2s linear infinite;
37+
}
38+
39+
@keyframes hourglass-animation {
40+
0% { transform: none; }
41+
25% { transform: rotate(180deg); }
42+
50% { transform: rotate(180deg); }
43+
75% { transform: rotate(360deg); }
44+
100% { transform: rotate(360deg); }
45+
}
46+
</style>
47+
</head>
48+
<body>
49+
<script src="https://d3js.org/d3.v7.min.js"></script>
50+
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" integrity="sha384-j1CDi7MgGQ12Z7Qab0qlWQ/Qqz24Gc6BM0thvEMVjHnfYGF0rmFCozFSxQBxwHKO" crossorigin="anonymous"></script>
51+
<script src="https://cdn.jsdelivr.net/npm/[email protected]/sorttable.min.js"></script>
52+
<script language="javascript" type="text/javascript" src="js/query-clickhouse.js"></script>
53+
<script language="javascript" type="text/javascript" src="js/data-visualization.js"></script>
54+
<script language="javascript" type="text/javascript" src="js/data-processing.js"></script>
55+
56+
<nav class="navbar navbar-expand-lg navbar-light bg-light">
57+
<div class="container-fluid">
58+
<a class="navbar-brand" href="#">ClickHouse query-latency-analyzer</a>
59+
<button class="navbar-toggler" type="button"
60+
data-bs-toggle="collapse"
61+
data-bs-target="#bs-example-navbar-collapse-1"
62+
aria-controls="bs-example-navbar-collapse-1"
63+
aria-expanded="false"
64+
aria-label="Toggle navigation">
65+
<span class="navbar-toggler-icon"></span>
66+
</button>
67+
<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
68+
<form class="d-flex">
69+
<button type="button" class="btn btn-primary"
70+
id="toolbar-load"
71+
data-bs-toggle="modal"
72+
data-bs-target="#loadModal">
73+
Load
74+
</button>
75+
</form>
76+
<p class="navbar-text ms-auto" id="status-text"></p>
77+
</div>
78+
</div>
79+
</nav>
80+
81+
<div class="modal fade" id="loadModal" tabindex="-1" aria-labelledby="loadModalLabel" aria-hidden="true">
82+
<div class="modal-dialog modal-lg">
83+
<div class="modal-content">
84+
<div class="modal-header">
85+
<h5 class="modal-title" id="loadModalLabel">Load Data</h5>
86+
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
87+
</div>
88+
<div class="modal-body">
89+
<ul class="nav nav-tabs" id="loadModalTabs" role="tablist">
90+
<li class="nav-item" role="presentation">
91+
<button class="nav-link active" id="upload-tab" data-bs-toggle="tab"
92+
data-bs-target="#tabFileUpload" type="button" role="tab">Upload JSON</button>
93+
</li>
94+
<li class="nav-item" role="presentation">
95+
<button class="nav-link" id="clickhouse-tab" data-bs-toggle="tab"
96+
data-bs-target="#tabClickhouse" type="button" role="tab">From ClickHouse</button>
97+
</li>
98+
</ul>
99+
100+
<div class="tab-content mt-3">
101+
<div class="tab-pane fade show active" id="tabFileUpload" role="tabpanel">
102+
<input type="file" id="loadFiles" class="form-control" />
103+
</div>
104+
<div class="tab-pane fade" id="tabClickhouse" role="tabpanel">
105+
<form id="params">
106+
<div id="connection-params" class="mb-3">
107+
<input spellcheck="false" id="url" type="text" value="http://localhost:8123"
108+
placeholder="URL" class="form-control mb-2" />
109+
<input spellcheck="false" id="user" type="text" value="" placeholder="user"
110+
class="form-control mb-2" />
111+
<input spellcheck="false" id="password" type="password" placeholder="password" value=""
112+
class="form-control mb-2" />
113+
</div>
114+
<textarea spellcheck="false" id="query" rows="15" class="form-control"
115+
placeholder="SQL Query">SELECT
116+
normalized_query_hash as group,
117+
query_duration_ms * 1000 as _duration,
118+
intDiv(ProfileEvents['LoggerElapsedNanoseconds'], 1000) as logger,
119+
ProfileEvents['OSCPUWaitMicroseconds'] as cpu_wait,
120+
ProfileEvents['OSIOWaitMicroseconds'] as io_wait,
121+
ProfileEvents['UserTimeMicroseconds'] as user_exec,
122+
ProfileEvents['SystemTimeMicroseconds'] as system_exec
123+
FROM system.query_log
124+
WHERE event_time >= now() - INTERVAL 1 HOUR
125+
AND type = 2
126+
SETTINGS output_format_json_named_tuples_as_objects = 1, skip_unavailable_shards = 1</textarea>
127+
<div class="form-check mt-3">
128+
<input class="form-check-input" type="checkbox" id="flushLogsCheckbox">
129+
<label class="form-check-label" for="flushLogsCheckbox">
130+
Run SYSTEM FLUSH LOGS
131+
</label>
132+
</div>
133+
<div id="query-error" class="text-danger mt-2" style="display: none;"></div>
134+
</form>
135+
</div>
136+
</div>
137+
</div>
138+
<div class="modal-footer">
139+
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
140+
<button type="button" class="btn btn-primary" id="btnDoLoad">Load</button>
141+
<button type="button" class="btn btn-success" id="btnQueryClickhouse" style="display: none;">Query</button>
142+
<div id="hourglass">⧗</div>
143+
</div>
144+
</div>
145+
</div>
146+
</div>
147+
148+
<div id="legend-container"></div>
149+
<div id="table-container"></div>
150+
<div class="matrix-tooltip" style="visibility: hidden;"></div>
151+
</body>
152+
</html>
153+
154+
<script>
155+
156+
// Show correct button depending on tab
157+
document.addEventListener('DOMContentLoaded', function () {
158+
const loadModal = document.getElementById('loadModal');
159+
loadModal.addEventListener('shown.bs.modal', function () {
160+
const tabButtons = document.querySelectorAll('#loadModalTabs button[data-bs-toggle="tab"]');
161+
tabButtons.forEach(button => {
162+
button.addEventListener('shown.bs.tab', function (e) {
163+
const target = e.target.getAttribute('data-bs-target');
164+
if (target === '#tabClickhouse') {
165+
document.getElementById('btnDoLoad').style.display = 'none';
166+
document.getElementById('btnQueryClickhouse').style.display = 'inline-block';
167+
} else {
168+
document.getElementById('btnDoLoad').style.display = 'inline-block';
169+
document.getElementById('btnQueryClickhouse').style.display = 'none';
170+
}
171+
});
172+
});
173+
});
174+
});
175+
176+
// Handle ClickHouse query
177+
let queryController = null;
178+
document.getElementById('btnQueryClickhouse').addEventListener('click', async function () {
179+
if (queryController === null) {
180+
let rows = [];
181+
let error = '';
182+
const errorDiv = document.getElementById('query-error');
183+
errorDiv.style.display = 'none';
184+
errorDiv.textContent = '';
185+
document.getElementById('hourglass').style.display = 'inline-block';
186+
this.textContent = "Stop";
187+
188+
if (document.getElementById('flushLogsCheckbox').checked) {
189+
queryController = new AbortController();
190+
await queryClickHouse({
191+
host: document.getElementById('url').value,
192+
user: document.getElementById('user').value,
193+
password: document.getElementById('password').value,
194+
query: "SYSTEM FLUSH LOGS",
195+
for_each_row: () => {},
196+
on_error: (errorMsg) => error = errorMsg,
197+
controller: queryController
198+
});
199+
}
200+
201+
if (error === '') {
202+
queryController = new AbortController();
203+
await queryClickHouse({
204+
host: document.getElementById('url').value,
205+
user: document.getElementById('user').value,
206+
password: document.getElementById('password').value,
207+
query: document.getElementById('query').value,
208+
for_each_row: (data) => rows.push(data),
209+
on_error: (errorMsg) => error = errorMsg,
210+
controller: queryController
211+
});
212+
}
213+
214+
queryController = null;
215+
this.textContent = "Query";
216+
document.getElementById('hourglass').style.display = 'none';
217+
218+
if (error !== '') {
219+
errorDiv.textContent = error;
220+
errorDiv.style.display = 'block';
221+
} else {
222+
renderTable(parseClickHouseData(rows));
223+
const loadModal = bootstrap.Modal.getInstance(document.getElementById('loadModal'));
224+
loadModal.hide();
225+
}
226+
} else { // Cancel query
227+
queryController.abort();
228+
}
229+
});
230+
231+
// Handle uploaded JSON
232+
document.getElementById('btnDoLoad').addEventListener('click', function () {
233+
const element = document.getElementById('loadFiles');
234+
const files = element.files;
235+
if (files.length <= 0) {
236+
return false;
237+
}
238+
239+
const fr = new FileReader();
240+
fr.onload = function (e) {
241+
document.getElementById('errmsg').style.display = 'none';
242+
renderTable(parseClickHouseData(JSON.parse(e.target.result).data));
243+
};
244+
fr.readAsText(files.item(0));
245+
element.value = '';
246+
const loadModal = bootstrap.Modal.getInstance(document.getElementById('loadModal'));
247+
loadModal.hide();
248+
});
249+
250+
function parseClickHouseData(rows) {
251+
let result = [];
252+
for (let i = 0; i < rows.length; i++) {
253+
let row = rows[i];
254+
for (let key in row) {
255+
if (key !== 'group' && !key.startsWith('_'))
256+
row[key] = +row[key];
257+
}
258+
result.push(row);
259+
}
260+
return processData(result);
261+
}
262+
263+
function renderTable(data) {
264+
d3.select("#table-container").selectAll("table").remove();
265+
266+
const table = d3.select("#table-container")
267+
.append("table")
268+
.attr("class", "table sortable");
269+
270+
const thead = table.append("thead");
271+
const tbody = table.append("tbody");
272+
273+
// Get labels strting with underscore
274+
console.log(data.labels);
275+
const extraColumns = data.labels.filter(label => label.startsWith("_")).map(label => label.substring(1));
276+
console.log(extraColumns);
277+
278+
const columns = ["Group", "Count", ...extraColumns, "Covariance Matrix & Distribution"];
279+
thead.append("tr")
280+
.selectAll("th")
281+
.data(columns)
282+
.enter()
283+
.append("th")
284+
.text(d => d);
285+
286+
const rows = tbody.selectAll("tr")
287+
.data(Object.entries(data.groups))
288+
.enter()
289+
.append("tr");
290+
291+
// Put group name and count in the first two columns
292+
rows.append("td").text(([name]) => name);
293+
rows.append("td").text(([, stats]) => stats.count);
294+
295+
// Iterate over extra columns and add them to the table
296+
extraColumns.forEach(col => {
297+
const index = data.labels.findIndex(label => label.substring(1) === col);
298+
rows.append("td").text(([, stats]) => stats.avg[index].toFixed(2));
299+
});
300+
301+
// Add covariance matrix in the third column
302+
rows.append("td")
303+
.append("div")
304+
.attr("class", "matrix")
305+
.each(function ([, stats]) {
306+
addVisualization(this, data.labels, stats);
307+
});
308+
309+
// Add sorting to the table
310+
sorttable.makeSortable(table.node());
311+
}
312+
313+
drawLegend("#legend-container");
314+
315+
const raw_data = generateTestData();
316+
const data = processData(raw_data);
317+
renderTable(data);
318+
319+
</script>

0 commit comments

Comments
 (0)