Skip to content

Commit 4fd4881

Browse files
Craig Corneliusecheran
Craig Cornelius
and
echeran
authored
Add comparison of test results across platforms and ICU versions (#450)
* Adding ICU4C running collation tests - first try * Cache ICU4C binaries in GH and locally, only if they don't exist * Install JSON-C dependency if not installed at beginning of CI or e2e script * Fix bash if condition syntax * Making compare report files * Compare UI prototype * Compare: Moving toward comparison with hash_id * more preparation for comparing test sets * Update testreport for comparing test groups * Removing unneeded changes * More reverting --------- Co-authored-by: Elango Cheran <[email protected]>
1 parent 4636c7c commit 4fd4881

File tree

5 files changed

+446
-3
lines changed

5 files changed

+446
-3
lines changed

‎verifier/compare_template.html‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
<html>
2+
<!--
3+
Build a comparison page for test results on the same test suite,
4+
including graphics and other tools to better understand differences
5+
in results between platforms and different test data versions.
6+
E.g., NodeJS v 21.6.0 number format with ICU 74 vs.
7+
ICU4C 74.2 with ICU74 test data.
8+
9+
Started 19-Jan-2024, cwc.
10+
11+
https://www.freecodecamp.org/news/interactive-heatmap-in-javascript/
12+
-->
13+
<head>
14+
<title>Heatmap in JavaScript</title>
15+
<script src="https://cdn.anychart.com/releases/8.11.0/js/anychart-core.min.js"></script>
16+
<script src="https://cdn.anychart.com/releases/8.11.0/js/anychart-heatmap.min.js"></script>
17+
18+
19+
<!-- JQuery -->
20+
<script src="https://code.jquery.com/jquery-3.6.4.min.js" integrity="sha256-oP6HI9z1XaZNBrJURtCoUT5SUnxFr8s3BzRl+cbzUq8=" crossorigin="anonymous"></script>
21+
22+
<script>
23+
24+
// Pointers to the data directories
25+
const data_dirs = $data_dirs;
26+
const test_names = $test_names;
27+
let compare_directories = ['', '']; // Two data directories for comparison
28+
29+
let results = {};
30+
31+
let heat_map_data = {
32+
};
33+
34+
let states = ['pass', 'fail', 'error', 'unsupported'];
35+
36+
for (let state1 of states) {
37+
for (let state2 of states) {
38+
let data_item = {x:state1, y:state2, heat:0}
39+
}
40+
}
41+
function load_and_update() {
42+
// Get the values from the selectors.
43+
const checked1 = document.querySelector(
44+
"input[type='radio'][name='select1']:checked");
45+
if (checked1 == null) {
46+
alert('Pick 2 items');
47+
return;
48+
}
49+
50+
const checked2 = document.querySelector(
51+
"input[type='radio'][name='select2']:checked");
52+
if (checked2 == null) {
53+
alert('Pick 2 items');
54+
return;
55+
}
56+
57+
// loadRawData for each.
58+
const dir1 = checked1.value;
59+
const dir2 = checked2.value;
60+
61+
let results1 = loadRawData(dir1);
62+
63+
let results2 = loadRawData(dir2);
64+
65+
// update the heat map.
66+
Promise.all([results1, results2])
67+
.then(([results1, results2]) => {
68+
reloadHeatMap(results1, results2);
69+
}
70+
)
71+
}
72+
73+
function loadRawData(data_dir) {
74+
// Get the pass, fail, etc. JSON data from this directory.
75+
// Via fetch
76+
// TODO: Open the files and the JSON data for each status.
77+
78+
let load_results = {};
79+
// Return these data pointers
80+
let p1 = fetch(data_dir + '/pass.json');
81+
let p2 = fetch(data_dir + '/failing_tests.json');
82+
let p3 = fetch(data_dir + '/test_errors.json');
83+
let p4 = fetch(data_dir + '/unsupported.json');
84+
85+
// Synchronize all the data loading and charts / pagination
86+
87+
// Wait for results, then return data
88+
let all_promises = Promise.all([
89+
p1.then((response) => response.json())
90+
.then((data) => {
91+
load_results['pass'] = data}),
92+
p2.then((response) => response.json())
93+
.then((data) => {
94+
load_results['fail'] = data}),
95+
p3.then((response) => response.json())
96+
.then((data) => {
97+
load_results['error'] = data}),
98+
p4.then((response) => response.json())
99+
.then((data) => {
100+
load_results['unsupported'] = data}),
101+
new Promise((resolve, reject) => {
102+
$(document).ready(resolve);
103+
})
104+
]).
105+
then(([p1, p2, p3, p4, _ready]) => {
106+
return load_results;
107+
}
108+
)
109+
return all_promises;
110+
}
111+
112+
function setOfIdentifiers(test_list) {
113+
let result = new Set();
114+
for (let test of test_list) {
115+
let input_data = test['input_data'];
116+
result.add(input_data['hash_id']);
117+
}
118+
return result;
119+
}
120+
121+
function reloadHeatMap(result1, result2) {
122+
// Compute intersections of test identifiers
123+
// Redo the heat map.
124+
heat_map_data = [];
125+
for (let state1 of states) {
126+
let id_set1 = setOfIdentifiers(result1[state1]);
127+
128+
for (let state2 of states) {
129+
let id_set2 = setOfIdentifiers(result2[state2]);
130+
131+
let inter_set =
132+
new Set([...id_set1].filter((x) => id_set2.has(x)));
133+
let data_item = {x:state1, y:state2, heat:inter_set.size}
134+
heat_map_data.push(data_item);
135+
136+
}
137+
}
138+
139+
draw_heat_map();
140+
}
141+
142+
function build_page() {
143+
build_radio_buttons('select1', 'select1');
144+
build_radio_buttons('select2', 'select2');
145+
}
146+
147+
function build_radio_buttons(div_id, rb_name) {
148+
const div = document.getElementById(div_id);
149+
for (const index in test_names) {
150+
const name = test_names[index];
151+
const data_dir = data_dirs[index];
152+
let rb = document.createElement("INPUT");
153+
rb.setAttribute("type", "radio");
154+
rb.name = rb_name;
155+
rb.value = data_dir;
156+
157+
let html_label = document.createElement("label");
158+
html_label.appendChild(document.createTextNode(name));
159+
160+
div.appendChild(html_label);
161+
html_label.appendChild(rb);
162+
}
163+
}
164+
165+
166+
// Global chart;
167+
let chart = null;
168+
169+
// Set up a heat map
170+
function draw_heat_map() {
171+
// anychart.onDocumentReady(function () {
172+
// create a heatmap
173+
const heat_data = getData();
174+
chart = anychart.heatMap(heat_data);
175+
// set a custom color scale
176+
var colorScale = anychart.scales.ordinalColor();
177+
// TODO: Compute max and scale the values as greyscale
178+
let max_val = 0
179+
for (let item of heat_data) {
180+
max_val = Math.max(item.heat, max_val);
181+
}
182+
183+
colorScale.ranges([
184+
{ less: max_val / 100, color: "#B0D8A4" },
185+
{ from: max_val / 100, to: max_val / 20, color: "#FEE191" },
186+
{ from: max_val / 20, to: max_val * 0.90, color: "#FD8060" },
187+
{ greater: max_val * 0.90, color: "#CC333F" }
188+
]);
189+
chart.colorScale(colorScale);
190+
191+
var customColorScale = anychart.scales.linearColor();
192+
customColorScale.colors(["#222222", "#eeeeee"]);
193+
194+
// set the color scale as the color scale of the chart
195+
chart.colorScale(customColorScale);
196+
// Not defined? chart.palette(anychart.palettes.monochrome);
197+
198+
199+
// style the coloring in the hovered state
200+
chart
201+
.hovered()
202+
.fill(function () {
203+
return anychart.color.darken(this.sourceColor, 0.25);
204+
});
205+
206+
// TODO: decide on this hide the item labels
207+
chart.labels(true);
208+
// customize the axes
209+
chart.xAxis().stroke(null);
210+
chart.yAxis().stroke(null);
211+
chart.yAxis().labels().padding([0, 10, 0, 0]);
212+
chart.xAxis().labels().padding([0, 0, 10, 0]);
213+
// set the tooltip
214+
chart.tooltip().title().useHtml(true);
215+
chart
216+
.tooltip()
217+
.useHtml(true)
218+
.titleFormat(function () {
219+
return this.x + "," + this.y + ': ' +this.heat;
220+
})
221+
.format(function () {
222+
return (
223+
'<span style="color: #CECECE">Test 1: </span>' +
224+
this.x +
225+
"<br/>" +
226+
'<span style="color: #CECECE">Test 2:: </span>' +
227+
this.y
228+
);
229+
});
230+
// name the heatmap
231+
chart
232+
.title()
233+
.enabled(true)
234+
.text('$title')
235+
.padding([0, 0, 20, 0]);
236+
// set the container for the heatmap
237+
chart.container("heat_map_container");
238+
// draw the heatmap
239+
chart.draw();
240+
// });
241+
242+
}
243+
244+
// All the code for the JS heatmap will come here
245+
// add the data
246+
function getData() {
247+
// TODO: Fill this in
248+
return heat_map_data;
249+
}
250+
251+
252+
</script>
253+
<style type="text/css">
254+
html, body, #head_map_container_ container {
255+
width: 80%; height: 80%; margin: 0; padding: 0;
256+
}
257+
</style>
258+
<style>
259+
.main div {
260+
float: left;
261+
clear: none;
262+
}
263+
</style>
264+
</head>
265+
266+
<body onLoad="build_page()">
267+
<h2>Compare test results for $test_type</h2>
268+
<div if="instance_selection">
269+
<p>Instances for test $test_type. Pick 2!</p>
270+
<div id="instance_selectors">
271+
<div id="select1"><label for='select1: '>Select 1:</label></div>
272+
<div id="select2"><label for='select2: '>Select 2: </label></div>
273+
<div id="choose_items">
274+
<button type="button" onclick="load_and_update();">Load data</button>
275+
</div>
276+
</div>
277+
</div>
278+
<div id="heat_map_container"></div>
279+
280+
</body>
281+
</html>

‎verifier/compare_template.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Templates for creating comparison pages in verifier
2+
import glob
3+
import json
4+
import logging
5+
import logging.config
6+
from string import Template
7+
import sys
8+
9+
class compareTemplate():
10+
def __init__(self):
11+
logging.config.fileConfig("../logging.conf")
12+
13+
# Read the template data
14+
compare_template = ''
15+
filename = 'compare_template.html'
16+
try:
17+
template_file = open(filename, mode='r')
18+
compare_template = template_file.read()
19+
template_file.close()
20+
except:
21+
logging.error('Cannot open compare_template %s', filename)
22+
23+
self.html_template = Template(compare_template)
24+
25+
# Template for picking tests - will be replaced by html generated in detail_template.html
26+
self.checkbox_test_template = Template(
27+
'<div id="$id_div"><input type=checkbox class="test_id" id="$id" name="$name" value="$value" onclick="checkboxChanged(this);"</input><label for="$id">$test_name</div>'
28+
)
29+
30+
def reportOutline(self):
31+
return self.html_template

‎verifier/summary_template.html‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -239,8 +239,19 @@
239239
let background_color;
240240
for (const test_type of test_types) {
241241
tr = table.insertRow();
242-
td = tr.insertCell();
243-
td.innerHTML = test_type
242+
243+
test_type_td = tr.insertCell();
244+
245+
// Create a link to the page to compare
246+
const a_ref = document.createElement('a');
247+
const link = document.createTextNode('Compare ' + test_type);
248+
a_ref.title = "Compare " + test_type;
249+
test_type.innerHTML = test_type;
250+
// Assemble the data for the instances
251+
let compare_list = [];
252+
let test_names = [];
253+
254+
<!-- put link to comparison selection for the test type -->
244255

245256
const tests = exec_summary_json[test_type];
246257
for (const exec of executed_platforms) {
@@ -307,6 +318,7 @@
307318
details.push(link);
308319
}
309320
}
321+
310322
td = tr.insertCell();
311323
if (reports.length > 0) {
312324
// Create the data for the reports
@@ -344,7 +356,13 @@
344356
}
345357
td = tr.insertCell();
346358
td.innerHTML = details.join('');
347-
}
359+
}
360+
// Set the compare link for this test type.
361+
const compare_link_text = "compare_" + test_type + ".html";
362+
test_type_td.innerHTML = "<a href=\'" + compare_link_text + "'" +
363+
" target='_blank'>" + test_type + "</a>";
364+
365+
348366
}
349367
}
350368
</script>

0 commit comments

Comments
 (0)