Skip to content

Commit 0115863

Browse files
simzersimzer
authored
Merge pull request #257 from veghdev/integrationtest_warnings
Integration test option to treat warnings as error
2 parents 8fa5d0b + 34fc39a commit 0115863

File tree

5 files changed

+166
-120
lines changed

5 files changed

+166
-120
lines changed

‎project/js/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@
1414
"dependencies": {
1515
"check-dts": "*",
1616
"eslint": "*",
17-
"prettier": "*",
17+
"prettier": "^2.8.8",
1818
"rollup": "*",
1919
"rollup-plugin-terser": "*",
2020
"typescript": "^4.9.4",
2121
"yaml": "*",
22-
"js-yaml": "*"
22+
"js-yaml": "^4.1.0"
2323
}
2424
}

‎test/integration/modules/integration-test/client/index.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ try {
146146
testData.description =
147147
"ref hash does not exist (hash: " + testData.hash + ")";
148148
testData.result = "WARNING";
149+
testData.warning = "noref";
149150
}
150151
if (typeof window.testData === "undefined") {
151152
window.testData = testData;

‎test/integration/modules/integration-test/test-case/test-case-result.js‎

Lines changed: 144 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,8 @@ class TestCaseResult {
6464
return resolve(
6565
this.#createTestCaseResultPassed(this.#testData.hash)
6666
);
67-
} else if (this.#testData.result === "WARNING") {
68-
return resolve(this.#createTestCaseResultWarning());
69-
} else if (this.#testData.result === "FAILED") {
70-
return resolve(this.#createTestCaseResultFailed());
67+
} else if (this.#testData.result === "WARNING" || this.#testData.result === "FAILED") {
68+
return resolve(this.#createTestCaseResultFailure());
7169
} else {
7270
return resolve(this.#createTestCaseResultError());
7371
}
@@ -77,35 +75,107 @@ class TestCaseResult {
7775
}
7876

7977
#createTestCaseResultPassed(msg) {
78+
if (this.#testCaseObj.createImages === "ALL") {
79+
this.#createImage(this.#testData, "-1new");
80+
}
8081
this.#testCaseObj.testSuiteResults.PASSED.push(
8182
this.#testCaseObj.testCase.testName
8283
);
8384
this.#cnsl.log(
8485
("[ " + "PASSED".padEnd(this.#cnsl.getTestStatusPad(), " ") + " ] ")
8586
.success +
86-
"[ " +
87-
String(++this.#testCaseObj.testSuiteResults.FINISHED).padEnd(
88-
this.#cnsl.getTestNumberPad(),
89-
" "
90-
) +
91-
" ] " +
92-
"[ " +
93-
msg +
94-
" ] " +
95-
path.relative(
96-
TestEnv.getTestSuitePath(),
97-
path.join(
98-
TestEnv.getWorkspacePath(),
99-
this.#testCaseObj.testCase.testName
100-
)
87+
"[ " +
88+
String(++this.#testCaseObj.testSuiteResults.FINISHED).padEnd(
89+
this.#cnsl.getTestNumberPad(),
90+
" "
91+
) +
92+
" ] " +
93+
"[ " +
94+
msg +
95+
" ] " +
96+
path.relative(
97+
TestEnv.getTestSuitePath(),
98+
path.join(
99+
TestEnv.getWorkspacePath(),
100+
this.#testCaseObj.testCase.testName
101101
)
102+
)
102103
);
103-
if (this.#testCaseObj.createImages === "ALL") {
104-
this.#createImage(this.#testData, "-1new");
105-
}
106104
}
107105

108-
#createTestCaseResultWarning() {
106+
#createTestCaseResultFailure() {
107+
return new Promise((resolve, reject) => {
108+
if (this.#testCaseObj.createImages !== "DISABLED") {
109+
this.#createImage(this.#testData, "-1new");
110+
}
111+
if (this.#testData.result === "WARNING") {
112+
if (this.#testData.warning === "noref" && this.#testCaseObj.Werror.includes("noref")) {
113+
this.#createTestCaseResultFailed();
114+
return resolve();
115+
}
116+
this.#createTestCaseResultWarning();
117+
return resolve();
118+
} else {
119+
if (
120+
this.#testCaseObj.createImages !== "DISABLED" &&
121+
!this.#vizzuUrl.includes(VizzuUrl.getRemoteStableBucket())
122+
) {
123+
let testCaseObj = Object.assign({}, this.#testCaseObj);
124+
testCaseObj.createImages = "ALL";
125+
this.#runTestCaseRef(testCaseObj, this.#browserChrome).then(
126+
(testDataRef) => {
127+
let failureMsgs = [];
128+
this.#createImage(testDataRef, "-2ref");
129+
this.#createDifImage(this.#testData, testDataRef);
130+
let diff = false;
131+
for (let i = 0; i < (this.#testData.hashes?.length ?? 0); i++) {
132+
for (let j = 0; j < (this.#testData.hashes?.[i]?.length ?? 0); j++) {
133+
if (this.#testData.hashes[i][j] != testDataRef.hashes[i][j]) {
134+
failureMsgs.push(
135+
"".padEnd(this.#cnsl.getTestStatusPad() + 5, " ") +
136+
"[ " +
137+
"step: " +
138+
i +
139+
". - seek: " +
140+
this.#testData.seeks[i][j] +
141+
" - hash: " +
142+
this.#testData.hashes[i][j].substring(0, 7) +
143+
" " +
144+
"(ref: " +
145+
testDataRef.hashes[i][j].substring(0, 7) +
146+
")" +
147+
" ]"
148+
);
149+
diff = true;
150+
}
151+
}
152+
}
153+
if (!diff) {
154+
failureMsgs.push(
155+
"".padEnd(this.#cnsl.getTestStatusPad() + 5, " ") +
156+
"[ the currently counted hashes are the same, the difference is probably caused by the environment ]"
157+
);
158+
this.#testData.warning = "sameref";
159+
}
160+
return failureMsgs;
161+
}
162+
).then(failureMsgs => {
163+
if (this.#testData.warning === "sameref" && !this.#testCaseObj.Werror.includes("sameref")) {
164+
this.#createTestCaseResultWarning(failureMsgs);
165+
return resolve();
166+
}
167+
this.#createTestCaseResultFailed(failureMsgs);
168+
return resolve();
169+
});
170+
} else {
171+
this.#createTestCaseResultFailed();
172+
return resolve();
173+
}
174+
}
175+
});
176+
}
177+
178+
#createTestCaseResultWarning(failureMsgs) {
109179
this.#testCaseObj.testSuiteResults.WARNING.push(
110180
this.#testCaseObj.testCase.testName
111181
);
@@ -125,78 +195,34 @@ class TestCaseResult {
125195
this.#testData.description +
126196
" ] "
127197
).warn +
128-
path.relative(
129-
TestEnv.getTestSuitePath(),
130-
path.join(
131-
TestEnv.getWorkspacePath(),
132-
this.#testCaseObj.testCase.testName
133-
)
198+
path.relative(
199+
TestEnv.getTestSuitePath(),
200+
path.join(
201+
TestEnv.getWorkspacePath(),
202+
this.#testCaseObj.testCase.testName
134203
)
204+
)
135205
);
136-
if (this.#testCaseObj.createImages !== "DISABLED") {
137-
this.#createImage(this.#testData, "-1new");
206+
if (failureMsgs) {
207+
failureMsgs.forEach(failureMsg => {
208+
this.#cnsl.log(failureMsg);
209+
});
138210
}
139211
}
140212

141-
#createTestCaseResultFailed() {
142-
return new Promise((resolve, reject) => {
143-
this.#testCaseObj.testSuiteResults.FAILED.push(
144-
this.#testCaseObj.testCase.testName
145-
);
146-
this.#testCaseObj.testSuiteResults.MANUAL.push(
147-
this.#testCaseObj.testCase
148-
);
149-
if (this.#testCaseObj.createImages !== "DISABLED") {
150-
this.#createImage(this.#testData, "-1new");
151-
}
152-
if (
153-
this.#testCaseObj.createImages !== "DISABLED" &&
154-
!this.#vizzuUrl.includes(VizzuUrl.getRemoteStableBucket())
155-
) {
156-
let testCaseObj = Object.assign({}, this.#testCaseObj);
157-
testCaseObj.createImages = "ALL";
158-
this.#runTestCaseRef(testCaseObj, this.#browserChrome).then(
159-
(testDataRef) => {
160-
this.#createImage(testDataRef, "-2ref");
161-
this.#createDifImage(this.#testData, testDataRef);
162-
this.#createTestCaseResultErrorMsg();
163-
let diff = false;
164-
for (let i = 0; i < (this.#testData.hashes?.length ?? 0); i++) {
165-
for (let j = 0; j < (this.#testData.hashes?.[i]?.length ?? 0); j++) {
166-
if (this.#testData.hashes[i][j] != testDataRef.hashes[i][j]) {
167-
this.#cnsl.log(
168-
"".padEnd(this.#cnsl.getTestStatusPad() + 5, " ") +
169-
"[ " +
170-
"step: " +
171-
i +
172-
". - seek: " +
173-
this.#testData.seeks[i][j] +
174-
" - hash: " +
175-
this.#testData.hashes[i][j].substring(0, 7) +
176-
" " +
177-
"(ref: " +
178-
testDataRef.hashes[i][j].substring(0, 7) +
179-
")" +
180-
" ]"
181-
);
182-
diff = true;
183-
}
184-
}
185-
}
186-
if (!diff) {
187-
this.#cnsl.log(
188-
"".padEnd(this.#cnsl.getTestStatusPad() + 5, " ") +
189-
"[ the currently counted hashes are the same, the difference is probably caused by the environment ]"
190-
);
191-
}
192-
return resolve();
193-
}
194-
);
195-
} else {
196-
this.#createTestCaseResultErrorMsg();
197-
return resolve();
198-
}
199-
});
213+
#createTestCaseResultFailed(failureMsgs) {
214+
this.#testCaseObj.testSuiteResults.FAILED.push(
215+
this.#testCaseObj.testCase.testName
216+
);
217+
this.#testCaseObj.testSuiteResults.MANUAL.push(
218+
this.#testCaseObj.testCase
219+
);
220+
this.#createTestCaseResultErrorMsg();
221+
if (failureMsgs) {
222+
failureMsgs.forEach(failureMsg => {
223+
this.#cnsl.log(failureMsg);
224+
});
225+
}
200226
}
201227

202228
#createTestCaseResultError() {
@@ -229,13 +255,13 @@ class TestCaseResult {
229255
errParts[0] +
230256
" ] "
231257
).error +
232-
path.relative(
233-
TestEnv.getTestSuitePath(),
234-
path.join(
235-
TestEnv.getWorkspacePath(),
236-
this.#testCaseObj.testCase.testName
237-
)
258+
path.relative(
259+
TestEnv.getTestSuitePath(),
260+
path.join(
261+
TestEnv.getWorkspacePath(),
262+
this.#testCaseObj.testCase.testName
238263
)
264+
)
239265
);
240266
if (errParts.length > 1) {
241267
errParts.slice(1).forEach((item) => {
@@ -291,17 +317,17 @@ class TestCaseResult {
291317
}
292318
fs.writeFile(
293319
testCaseResultPath +
294-
"/" +
295-
path.basename(testCaseResultPath) +
296-
"_" +
297-
i.toString().padStart(3, "0") +
298-
"_" +
299-
seek[0].padStart(3, "0") +
300-
"." +
301-
seek[1].padEnd(3, "0") +
302-
"%" +
303-
fileAdd +
304-
".png",
320+
"/" +
321+
path.basename(testCaseResultPath) +
322+
"_" +
323+
i.toString().padStart(3, "0") +
324+
"_" +
325+
seek[0].padStart(3, "0") +
326+
"." +
327+
seek[1].padEnd(3, "0") +
328+
"%" +
329+
fileAdd +
330+
".png",
305331
data.images[i][j].substring(22),
306332
"base64",
307333
(err) => {
@@ -353,17 +379,17 @@ class TestCaseResult {
353379
if (difference) {
354380
fs.writeFile(
355381
testCaseResultPath +
356-
"/" +
357-
path.basename(testCaseResultPath) +
358-
"_" +
359-
i.toString().padStart(3, "0") +
360-
"_" +
361-
seek[0].padStart(3, "0") +
362-
"." +
363-
seek[1].padEnd(3, "0") +
364-
"%" +
365-
"-3diff" +
366-
".png",
382+
"/" +
383+
path.basename(testCaseResultPath) +
384+
"_" +
385+
i.toString().padStart(3, "0") +
386+
"_" +
387+
seek[0].padStart(3, "0") +
388+
"." +
389+
seek[1].padEnd(3, "0") +
390+
"%" +
391+
"-3diff" +
392+
".png",
367393
pngjs.PNG.sync.write(diff),
368394
(err) => {
369395
if (err) {

‎test/integration/modules/integration-test/test-suite.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ class TestSuite {
2828
#workspaceHostReady;
2929
#workspaceHostServerPort;
3030

31+
#Werror;
3132
#createImages;
3233
#createHashes;
3334

@@ -53,6 +54,7 @@ class TestSuite {
5354
browsersNum,
5455
browserGui,
5556
vizzuUrl,
57+
Werror,
5658
createImages,
5759
createHashes
5860
) {
@@ -68,6 +70,7 @@ class TestSuite {
6870

6971
this.#vizzuUrl = vizzuUrl;
7072

73+
this.#Werror = Werror;
7174
this.#createImages = createImages;
7275
this.#createHashes = createHashes;
7376

@@ -127,6 +130,7 @@ class TestSuite {
127130
testSuiteResults: this.#testSuiteResults,
128131
workspaceHostServerPort: this.#workspaceHostServerPort,
129132
browsersChrome: this.#browsersChrome,
133+
Werror: this.#Werror,
130134
createImages: this.#createImages,
131135
animTimeout: this.#browsersChrome.getTimeout(),
132136
cnsl: this.#cnsl,

0 commit comments

Comments
 (0)