integration test: add error validation

veghdev · veghdev · commit dec7bc3d7d7a · 2021-12-02T17:58:06.000+01:00
diff --git a/test/integration/modules/integration-test/test-case/test-case-result.js b/test/integration/modules/integration-test/test-case/test-case-result.js
@@ -41,23 +41,40 @@ class TestCaseResult {
                 deleteTestCaseResultReady = this.#deleteTestCaseResult();
             }
             deleteTestCaseResultReady.then(() => {
-                if (this.#testData.result == "PASSED") {
-                    return resolve(this.#createTestCaseResultPassed());
-                } else if(this.#testData.result == "WARNING") {
-                    return resolve(this.#createTestCaseResultWarning());
-                } else if(this.#testData.result == "FAILED") {
-                    return resolve(this.#createTestCaseResultFailed());
+
+                if (this.#testCaseObj.testCase in this.#testCaseObj.testCasesConfig.tests && 
+                    "err" in this.#testCaseObj.testCasesConfig.tests[this.#testCaseObj.testCase]) {
+                    let err = this.#testCaseObj.testCasesConfig.tests[this.#testCaseObj.testCase]["err"];
+                    if (this.#testData.result === "ERROR") {
+                        if(this.#testData.description.includes(err)) {
+                            return resolve(this.#createTestCaseResultPassed(err));
+                        } else {
+                            return resolve(this.#createTestCaseResultError());
+                        }
+                    } else {
+                        this.#testData.result = "ERROR";
+                        this.#testData.description = "did not occur " + err;
+                        return resolve(this.#createTestCaseResultError());
+                    }
                 } else {
-                    return resolve(this.#createTestCaseResultError());
+                    if (this.#testData.result === "PASSED") {
+                        return resolve(this.#createTestCaseResultPassed(this.#testData.hash));
+                    } else if(this.#testData.result === "WARNING") {
+                        return resolve(this.#createTestCaseResultWarning());
+                    } else if(this.#testData.result === "FAILED") {
+                        return resolve(this.#createTestCaseResultFailed());
+                    } else {
+                        return resolve(this.#createTestCaseResultError());
+                    }
                 }
             });
         });
     }
 
 
-    #createTestCaseResultPassed() {
+    #createTestCaseResultPassed(msg) {
         this.#testCaseObj.testSuiteResults.PASSED.push(this.#testCaseObj.testCase);
-        this.#cnsl.log(("[ " + this.#testData.result.padEnd(this.#cnsl.getTestStatusPad(), " ") + " ] ").success + "[ " + String(++this.#testCaseObj.testSuiteResults.FINISHED).padEnd(this.#cnsl.getTestNumberPad(), " ") + " ] " + "[ " + this.#testData.hash + " ] " + path.relative(TestEnv.getTestSuitePath(), path.join(TestEnv.getWorkspacePath(), this.#testCaseObj.testCase)));
+        this.#cnsl.log(("[ " + "PASSED".padEnd(this.#cnsl.getTestStatusPad(), " ") + " ] ").success + "[ " + String(++this.#testCaseObj.testSuiteResults.FINISHED).padEnd(this.#cnsl.getTestNumberPad(), " ") + " ] " + "[ " + msg + " ] " + path.relative(TestEnv.getTestSuitePath(), path.join(TestEnv.getWorkspacePath(), this.#testCaseObj.testCase)));
         if (this.#testCaseObj.createImages === "ALL") {
             this.#createImage(this.#testData, '-1new');
         }
@@ -66,7 +83,7 @@ class TestCaseResult {
 
     #createTestCaseResultWarning() {
         this.#testCaseObj.testSuiteResults.WARNING.push(this.#testCaseObj.testCase);
-        this.#cnsl.log(("[ " + this.#testData.result.padEnd(this.#cnsl.getTestStatusPad(), " ") + " ] " + "[ " + String(++this.#testCaseObj.testSuiteResults.FINISHED).padEnd(this.#cnsl.getTestNumberPad(), " ") + " ] " + "[ " + this.#testData.description + " ] ").warn + path.relative(TestEnv.getTestSuitePath(), path.join(TestEnv.getWorkspacePath(), this.#testCaseObj.testCase)));
+        this.#cnsl.log(("[ " + "WARNING".padEnd(this.#cnsl.getTestStatusPad(), " ") + " ] " + "[ " + String(++this.#testCaseObj.testSuiteResults.FINISHED).padEnd(this.#cnsl.getTestNumberPad(), " ") + " ] " + "[ " + this.#testData.description + " ] ").warn + path.relative(TestEnv.getTestSuitePath(), path.join(TestEnv.getWorkspacePath(), this.#testCaseObj.testCase)));
         if (this.#testCaseObj.createImages !== "DISABLED") {
             this.#createImage(this.#testData, '-1new');
         }
@@ -108,7 +125,7 @@ class TestCaseResult {
 
     #createTestCaseResultError() {
         this.#testCaseObj.testSuiteResults.FAILED.push(this.#testCaseObj.testCase);
-        this.#createTestCaseResultErrorMsg();
+        this.#createTestCaseResultErrorMsg();        
     }
 
 
diff --git a/test/integration/modules/integration-test/test-case/test-case.js b/test/integration/modules/integration-test/test-case/test-case.js
@@ -38,6 +38,7 @@ class TestCase {
         return new Promise((resolve, reject) => {
             let refHash = [];
             let animStep = { step: testCaseObj.animStep, default: true };
+            let errMsg;
             if (testCaseObj.testCase in testCaseObj.testCasesConfig.tests) {
                 if ("animstep" in testCaseObj.testCasesConfig.tests[testCaseObj.testCase]) {
                     animStep.step = testCaseObj.testCasesConfig.tests[testCaseObj.testCase]["animstep"].replace("%", "");
@@ -46,6 +47,9 @@ class TestCase {
                 if ("refs" in testCaseObj.testCasesConfig.tests[testCaseObj.testCase]) {
                     refHash = testCaseObj.testCasesConfig.tests[testCaseObj.testCase]["refs"];
                 }
+                if ("err" in testCaseObj.testCasesConfig.tests[testCaseObj.testCase]) {
+                    errMsg = testCaseObj.testCasesConfig.tests[testCaseObj.testCase]["err"];
+                }
             }
             if (vizzuUrl.startsWith("/")) {
                 vizzuUrl = "/" + path.relative(TestEnv.getWorkspacePath(), vizzuUrl);
@@ -64,6 +68,9 @@ class TestCase {
                             if (!animStep.default) {
                                 testData["animstep"] = animStep.step;
                             }
+                            if (errMsg) {
+                                testData["err"] = errMsg;
+                            }
                             return resolve(testData);
                         })
                     }).catch(err => {
diff --git a/test/integration/modules/integration-test/test-case/test-cases-config.js b/test/integration/modules/integration-test/test-case/test-cases-config.js
@@ -102,6 +102,9 @@ class TestCasesConfig {
                                     },
                                     animstep: {
                                         type: "string"
+                                    },
+                                    err: {
+                                        type: "string"
                                     }
                                 },
                                 additionalProperties: false
@@ -149,6 +152,9 @@ class TestCasesConfig {
                                         },
                                         animstep: {
                                             type: "string"
+                                        },
+                                        err: {
+                                            type: "string"
                                         }
                                     },
                                     additionalProperties: false
@@ -169,6 +175,9 @@ class TestCasesConfig {
                             },
                             animstep: {
                                 type: "string"
+                            },
+                            err: {
+                                type: "string"
                             }
                         },
                         additionalProperties: false
diff --git a/test/integration/modules/integration-test/test-suite-result.js b/test/integration/modules/integration-test/test-suite-result.js
@@ -76,7 +76,11 @@ class TestSuiteResult {
                             if(this.#testSuiteResults.RESULTS[key]["animstep"]) {
                                 testData["animstep"] = this.#testSuiteResults.RESULTS[key]["animstep"] + "%";
                             }
-                            testData["refs"] = [this.#testSuiteResults.RESULTS[key]['hash']]
+                            if (this.#testSuiteResults.RESULTS[key]["err"]) {
+                                testData["err"] = this.#testSuiteResults.RESULTS[key]['err'];
+                            } else {
+                                testData["refs"] = [this.#testSuiteResults.RESULTS[key]['hash']];
+                            }
                             testCasesConfig[suite].test[path.relative(suite, key)] = testData;
                         }
                     });
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_digit_1.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_digit_1.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("#ABCDEG")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_digit_2.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_digit_2.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("#00000!")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_digit_3.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_digit_3.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("#ABCDEFFG")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_1.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_1.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("#")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_2.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_2.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("#000")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_3.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_3.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("#00000")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_4.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-hex_length_4.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("#000000000")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-string.mjs b/test/integration/test_options/style/plot/backgroundColor/error/invalid_color-string.mjs
@@ -0,0 +1,9 @@
+import Chart from "/test/integration/test_options/style/plot/backgroundColor/chart.js";
+
+
+const testSteps = [
+    Chart.static("red")
+]
+
+
+export default testSteps
diff --git a/test/integration/test_options/test_options.json b/test/integration/test_options/test_options.json
@@ -61,6 +61,30 @@
                 ""
             ]
         },
+        "style/plot/backgroundColor/error/invalid_color-hex_digit_1": {
+            "err": "error: invalid hexadecimal digit"
+        },
+        "style/plot/backgroundColor/error/invalid_color-hex_digit_2": {
+            "err": "error: invalid hexadecimal digit"
+        },
+        "style/plot/backgroundColor/error/invalid_color-hex_digit_3": {
+            "err": "error: invalid hexadecimal digit"
+        },
+        "style/plot/backgroundColor/error/invalid_color-hex_length_1": {
+            "err": "error: invalid color string"
+        },
+        "style/plot/backgroundColor/error/invalid_color-hex_length_2": {
+            "err": "error: invalid color string"
+        },
+        "style/plot/backgroundColor/error/invalid_color-hex_length_3": {
+            "err": "error: invalid color string"
+        },
+        "style/plot/backgroundColor/error/invalid_color-hex_length_4": {
+            "err": "error: invalid color string"
+        },
+        "style/plot/backgroundColor/error/invalid_color-string": {
+            "err": "error: invalid color string"
+        },
         "style/plot/backgroundColor/static/hex/black": {
             "refs": [
                 "8d2e451"

Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,11 @@ class TestSuiteResult {`
`76`	`76`	`if(this.#testSuiteResults.RESULTS[key]["animstep"]) {`
`77`	`77`	`testData["animstep"] = this.#testSuiteResults.RESULTS[key]["animstep"] + "%";`
`78`	`78`	`}`
`79`		`- testData["refs"] = [this.#testSuiteResults.RESULTS[key]['hash']]`
	`79`	`+ if (this.#testSuiteResults.RESULTS[key]["err"]) {`
	`80`	`+ testData["err"] = this.#testSuiteResults.RESULTS[key]['err'];`
	`81`	`+ } else {`
	`82`	`+ testData["refs"] = [this.#testSuiteResults.RESULTS[key]['hash']];`
	`83`	`+ }`
`80`	`84`	`testCasesConfig[suite].test[path.relative(suite, key)] = testData;`
`81`	`85`	`}`
`82`	`86`	`});`