Skip to content

Commit 5ab4f9a

Browse files
florystfloryst
committed
chore: generate HTML test reports
This replaces karma-tap and karma-tap-pretty-reporter with a custom tape-specific adapter and reporter.
1 parent 92a23e3 commit 5ab4f9a

File tree

10 files changed

+600
-33
lines changed

10 files changed

+600
-33
lines changed

‎Sources/Testing/setupTestEnv.js‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import test from 'tape-catch';
2+
3+
/**
4+
* Buffers written objects until a reader attaches,
5+
* after which all writes go to the reader.
6+
*
7+
* Only supports a single reader.
8+
*
9+
*/
10+
function BufferedObjectPipe() {
11+
const buffer = [];
12+
let closed = false;
13+
let reader = null;
14+
let flushTimeout = null;
15+
16+
const scheduleFlush = () => {
17+
if (reader && flushTimeout === null) {
18+
flushTimeout = setTimeout(() => {
19+
flushTimeout = null;
20+
while (buffer.length) {
21+
reader.onData(buffer.shift());
22+
}
23+
if (closed) {
24+
reader.onClose();
25+
}
26+
}, 0);
27+
}
28+
};
29+
30+
const write = (data) => {
31+
if (!closed) {
32+
buffer.push(data);
33+
scheduleFlush();
34+
}
35+
};
36+
37+
const end = () => {
38+
closed = true;
39+
scheduleFlush();
40+
};
41+
42+
const setReader = (r) => {
43+
reader = r;
44+
};
45+
46+
return {
47+
write,
48+
end,
49+
setReader,
50+
};
51+
}
52+
53+
// pipe tape objects to karma adapter in tap-object-stream
54+
const pipe = BufferedObjectPipe();
55+
test.createStream({ objectMode: true }).on('data', (row) => {
56+
pipe.write(row);
57+
});
58+
test.onFinish(() => pipe.end());
59+
60+
window.__TapeEnv__ = {
61+
pipe,
62+
};

‎Sources/Testing/testUtils.js‎

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@ function createCanvasContext() {
1010

1111
function getImageDataFromURI(imageDataURI) {
1212
return new Promise((resolve, reject) => {
13-
const { context } = createCanvasContext();
13+
const { canvas, context } = createCanvasContext();
1414
const img = new Image();
1515
img.addEventListener('load', () => {
16+
canvas.width = img.width;
17+
canvas.height = img.height;
1618
context.drawImage(img, 0, 0);
1719
resolve(context.getImageData(0, 0, img.width, img.height));
1820
});
@@ -21,61 +23,91 @@ function getImageDataFromURI(imageDataURI) {
2123
});
2224
}
2325

26+
/**
27+
* Compares two images
28+
* @param image the image under test
29+
* @param baselines an array of baseline images
30+
* @param tapeContext tape testing context
31+
* @param opts if number: mismatch tolerance. if object: tolerance and pixel threshold
32+
*/
2433
async function compareImages(
2534
image,
2635
baselines,
2736
testName,
2837
tapeContext,
29-
threshold = 0.5,
30-
nextCallback = null
38+
opts,
39+
nextCallback
3140
) {
41+
// defaults
42+
let pixelThreshold = 0.1;
43+
let mismatchTolerance = 5; // percent
44+
45+
if (typeof opts === 'number') {
46+
mismatchTolerance = opts;
47+
} else {
48+
pixelThreshold = opts?.pixelThreshold ?? pixelThreshold;
49+
mismatchTolerance = opts?.mismatchTolerance ?? mismatchTolerance;
50+
}
51+
3252
let minDelta = 100;
53+
let minRawCount = 0;
3354
let minDiff = '';
34-
let isSameDimensions = false;
3555
let minIndex = 0;
56+
let isSameDimensions = false;
3657

3758
const imageUnderTest = await getImageDataFromURI(image);
3859
const baselineImages = await Promise.all(
3960
baselines.map((baseline) => getImageDataFromURI(baseline))
4061
);
4162

4263
baselineImages.forEach((baseline, idx) => {
43-
const { canvas, context } = createCanvasContext();
64+
const diff = createCanvasContext();
4465
const { width, height } = baseline;
45-
const diff = context.createImageData(width, height);
66+
diff.canvas.width = width;
67+
diff.canvas.height = height;
68+
69+
const diffImage = diff.context.createImageData(width, height);
4670
const mismatched = pixelmatch(
4771
imageUnderTest.data,
4872
baseline.data,
49-
diff.data,
73+
diffImage.data,
5074
width,
5175
height,
5276
{
5377
alpha: 0.5,
5478
includeAA: false,
55-
threshold,
79+
threshold: pixelThreshold,
5680
}
5781
);
58-
const percentage = (mismatched / (width * height)) * 100;
59-
if (minDelta >= percentage) {
82+
83+
const percentage = (100 * mismatched) / (width * height);
84+
if (percentage < minDelta) {
6085
minDelta = percentage;
61-
minDiff = canvas.toDataURL();
86+
minRawCount = mismatched;
87+
diff.context.putImageData(diffImage, 0, 0);
88+
minDiff = diff.canvas.toDataURL();
89+
minIndex = idx;
6290
isSameDimensions =
6391
width === imageUnderTest.width && height === imageUnderTest.height;
64-
minIndex = idx;
6592
}
6693
});
6794

95+
tapeContext.ok(isSameDimensions, 'Image match resolution');
6896
tapeContext.ok(
69-
minDelta < threshold,
70-
`Matching image - delta ${minDelta.toFixed(2)}%`
97+
minDelta < mismatchTolerance,
98+
`[${testName}]` +
99+
` Matching image - delta ${minDelta.toFixed(2)}%` +
100+
` (count: ${minRawCount})`,
101+
{
102+
operator: 'imagediff',
103+
actual: {
104+
outputImage: image,
105+
expectedImage: baselines[minIndex],
106+
diffImage: minDiff,
107+
},
108+
expected: mismatchTolerance,
109+
}
71110
);
72-
tapeContext.ok(isSameDimensions, 'Image match resolution');
73-
if (minDelta >= threshold) {
74-
tapeContext.comment(
75-
`new image <img src="${image}" /> vs baseline <img src="${baselines[minIndex]}" /> === <img src="${minDiff}" />`
76-
);
77-
tapeContext.fail(`for ${testName} the images were different`);
78-
}
79111

80112
if (nextCallback) {
81113
nextCallback();

‎Utilities/Karma/reporting-template.html‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
<!doctype html>
2+
<html>
3+
<head>
4+
<style type="text/css">
5+
* {
6+
margin: 0;
7+
}
8+
body {
9+
padding: 4px;
10+
}
11+
figcaption {
12+
text-align: center;
13+
}
14+
15+
.container {
16+
display: flex;
17+
flex-flow: column;
18+
}
19+
.line {
20+
border: 1px solid grey;
21+
font-family: sans-serif;
22+
font-size: 1em;
23+
padding: 6px 8px;
24+
display: flex;
25+
flex-flow: row;
26+
align-items: center;
27+
}
28+
.browser {
29+
display: block;
30+
background: white;
31+
font-size: 1.25em;
32+
}
33+
.browser li {
34+
font-size: 0.8em;
35+
}
36+
.test {
37+
margin-left: 32px;
38+
font-size: 1.1em;
39+
}
40+
.test.success {
41+
background: #98ff98;
42+
}
43+
.test.fail {
44+
background: #ee0000;
45+
}
46+
.spec {
47+
margin-left: 64px;
48+
}
49+
.spec.success {
50+
background: #98ff9888;
51+
}
52+
.spec.fail {
53+
background: #ee000088;
54+
}
55+
.skip {
56+
background: #efefef;
57+
}
58+
.badge {
59+
font-size: 0.9rem;
60+
border-radius: 4px;
61+
border: 1px solid #333;
62+
background-color: rgba(255, 255, 255, 0.4);
63+
padding: 4px;
64+
}
65+
.details {
66+
margin-left: 96px;
67+
padding: 8px;
68+
background: #eee;
69+
}
70+
.imagediff {
71+
display: flex;
72+
flex-flow: row;
73+
}
74+
.ml-8 {
75+
margin-left: 8px;
76+
}
77+
.hidden {
78+
display: none;
79+
}
80+
#controls {
81+
padding: 8px;
82+
}
83+
</style>
84+
<script type="text/javascript">
85+
let showPassing = false;
86+
87+
function togglePassingVisible(forceFlag = undefined) {
88+
showPassing = forceFlag === undefined ? !showPassing : forceFlag;
89+
Array.from(document.querySelectorAll('.skip, .success')).forEach((el) => {
90+
if (showPassing) {
91+
el.classList.remove('hidden');
92+
} else {
93+
el.classList.add('hidden');
94+
}
95+
});
96+
document.getElementById('passingVisibleBtn').innerText = showPassing
97+
? 'hide passing tests'
98+
: 'show passing tests';
99+
}
100+
101+
window.onload = () => {
102+
togglePassingVisible(true);
103+
};
104+
</script>
105+
</head>
106+
<body>
107+
<h1>vtk.js Test Results</h1>
108+
<div id="controls">
109+
<button id="passingVisibleBtn" onclick="togglePassingVisible()">loading</button>
110+
</div>
111+
{{#each browsers}}
112+
<div class="line browser">
113+
<div>{{name}}</div>
114+
<ul>
115+
<li>Failed: {{summary.failed}}</li>
116+
<li>Passed: {{summary.passed}}</li>
117+
<li>Skipped: {{summary.skipped}}</li>
118+
<li>Total: {{summary.total}}</li>
119+
</ul>
120+
</div>
121+
{{#each tests}}
122+
<div class="line test {{#if success}}success{{else}}fail{{/if}}">
123+
{{name}}
124+
{{#if success}}
125+
<div class="ml-8 badge">passing</div>
126+
{{else}}
127+
<div class="ml-8 badge">failing</div>
128+
{{/if}}
129+
</div>
130+
{{#each specs}}
131+
<div class="line spec {{#if skipped}}skip{{else if success}}success{{else}}fail{{/if}}">
132+
<div>{{description}}</div>
133+
{{#if skipped}}
134+
<div class="ml-8 badge">skipped</div>
135+
{{else if success}}
136+
<div class="ml-8 badge">passing</div>
137+
{{else}}
138+
<div class="ml-8 badge">failing</div>
139+
{{/if}}
140+
</div>
141+
{{#unless success}}
142+
<div class="details">
143+
{{#with details}}
144+
{{#if (equals operator "imagediff")}}
145+
{{! Image diff structure: see testUtils.compareImage }}
146+
<div>Mismatch count tolerance: {{expected}}%</div>
147+
<div class="imagediff">
148+
<figure><img src="{{actual.outputImage}}"><figcaption>Output</figcaption></figure>
149+
<figure><img src="{{actual.expectedImage}}"><figcaption>Expected</figcaption></figure>
150+
<figure><img src="{{actual.diffImage}}"><figcaption>Difference</figcaption></figure>
151+
</div>
152+
{{else}}
153+
<div>Output: <code>{{details.actual}}</code></div>
154+
<div>Expected: <code>{{details.expected}}</code></div>
155+
{{/if}}
156+
{{/with}}
157+
</div>
158+
{{/unless}}
159+
{{/each}}
160+
{{/each}}
161+
{{/each}}
162+
</body>
163+
</html>

0 commit comments

Comments
 (0)