Add support to enable skipped tests. (#228)

JSRemoteTest-DCO-1.0-Signed-off-by: Roland Takacs [email protected]

Author: rtakacs

jstest/testrunner/skiplist/iotjs-skiplist.json

@@ -0,0 +1,32 @@
+{
+  "rpi2": {
+    "skip": {
+      "testsets": [],
+      "testfiles": []
+    },
+    "enable": []
+  },
+  "stm32f4dis": {
+    "skip": {
+      "testsets": [],
+      "testfiles": []
+    },
+    "enable": []
+  },
+  "artik053": {
+    "skip": {
+      "testsets": [],
+      "testfiles": [
+        { "name": "test_tls.js", "reason": "Error: SSL setup failed (no enough memory for the test)" }
+      ]
+    },
+    "enable": []
+  },
+  "artik530": {
+    "skip": {
+      "testsets": [],
+      "testfiles": []
+    },
+    "enable": []
+  }
+}

jstest/testrunner/skiplist/iotjs-test-descriptor.json

@@ -0,0 +1,47 @@
+{
+  "rpi2": {
+    "skip": {
+      "testsets": [
+        { "name": "es5.1", "reason": "es5.1 profile is not enabled." }
+      ],
+      "testfiles": [
+        { "name": "N.compact-profile-error.js", "reason": "Compact profile is not enabled." }
+      ]
+    },
+    "enable": []
+  },
+  "stm32f4dis": {
+    "skip": {
+      "testsets": [
+        { "name": "es5.1", "reason": "es5.1 profile is not enabled." }
+      ],
+      "testfiles": [
+        { "name": "parser-oom.js", "reason": "Requires exactly 512KB memory." },
+        { "name": "N.compact-profile-error.js", "reason": "Compact profile is not enabled." },
+        { "name": "regression-test-issue-1555.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-1997.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-2451.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-2452.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-2453.js", "reason": "There's not enough memory to run." }
+      ]
+    },
+    "enable": []
+  },
+  "artik053": {
+    "skip": {
+      "testsets": [
+        { "name": "es5.1", "reason": "es5.1 profile is not enabled." }
+      ],
+      "testfiles": [
+        { "name": "parser-oom.js", "reason": "Requires exactly 512KB memory." },
+        { "name": "N.compact-profile-error.js", "reason": "Compact profile is not enabled." },
+        { "name": "regression-test-issue-1555.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-1997.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-2451.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-2452.js", "reason": "There's not enough memory to run." },
+        { "name": "regression-test-issue-2453.js", "reason": "There's not enough memory to run." }
+      ]
+    },
+    "enable": []
+  }
+}

jstest/testrunner/skiplist/jerryscript-skiplist.json

@@ -30,8 +30,9 @@ def __init__(self, env, device):
             self.builtin_features = buildinfo[1]
             self.stability = buildinfo[2]
 
-        # Read the local skiplist.
-        self.skiplist = self._read_skiplist()
+        # Read the test_descriptor file that defines which tests
+        # should be enabled or skipped.
+        self.test_descriptor = self._read_test_descriptor()
 
     def contains(self, testset, test):
         '''
@@ -76,29 +77,32 @@ def _skip_iotjs_test(self, test):
 
         return False
 
-    def _read_skiplist(self):
+    def _read_test_descriptor(self):
         '''
         Read the local skiplists.
         '''
-        skiplists = {
-           'iotjs': 'iotjs-skiplist.json',
-           'jerryscript': 'jerryscript-skiplist.json'
+        descriptors = {
+           'iotjs': 'iotjs-test-descriptor.json',
+           'jerryscript': 'jerryscript-test-descriptor.json'
         }
 
-        skipfile = utils.join(paths.SKIPLIST_PATH, skiplists[self.app])
-        skiplist = utils.read_json_file(skipfile)
+        descriptor_file = utils.join(paths.SKIPLIST_PATH, descriptors[self.app])
+        descriptor_info = utils.read_json_file(descriptor_file)
 
-        return skiplist[self.device_type]
+        return descriptor_info[self.device_type]
 
     def _find_in_skiplist(self, testset, test):
         '''
         Find element in the skiplist.
         '''
-        for obj in self.skiplist['testsets']:
+        if test['name'] in self.test_descriptor['enable']:
+            return None
+
+        for obj in self.test_descriptor['skip']['testsets']:
             if obj['name'] == testset:
                 return obj
 
-        for obj in self.skiplist['testfiles']:
+        for obj in self.test_descriptor['skip']['testfiles']:
             if obj['name'] == test['name']:
                 return obj