Improve testing skip features.

- Improve skip logging.
- Add `skip` and `only` flags in manifests.
- Add `VERBOSE_SKIP=true` env var to debug skipping.

Author: davidlehn

CHANGELOG.md

@@ -1,7 +1,14 @@
 # jsonld ChangeLog
 
 ### Fixed
-- Use explicit id or description test skipping regexes.
+- Testing: Use explicit id and description skipping regexes.
+
+### Changed
+- Testing: Improve skip logging.
+
+### Added
+- Testing: `skip` and `only` flags in manifests.
+- Testing: `VERBOSE_SKIP=true` env var to debug skipping.
 
 ## 1.5.4 - 2019-02-28
 

tests/test-common.js

@@ -326,8 +326,8 @@ function _testsToMocha(tests) {
     }
     describe(suite.title, () => {
       suite.tests.forEach(test => {
-        if(test.skip) {
-          it.skip(test.title);
+        if(test.only) {
+          it.only(test.title, test.f);
           return;
         }
         it(test.title, test.f);
@@ -433,48 +433,89 @@ function addTest(manifest, test, tests) {
   test.manifest = manifest;
   const description = test_id + ' ' + (test.purpose || test.name);
 
-  tests.push({
+  const _test = {
     title: description,
     f: makeFn()
-  });
+  };
+  // only based on test manifest
+  // skip handled via skip()
+  if('only' in test) {
+    _test.only = test.only;
+  }
+  tests.push(_test);
 
   function makeFn() {
     return async function() {
       const self = this;
       self.timeout(5000);
       const testInfo = TEST_TYPES[getJsonLdTestType(test)];
 
-      // skip unknown and explicitly skipped test types
+      // skip based on test manifest
+      if('skip' in test && test.skip) {
+        if(options.verboseSkip) {
+          console.log('Skipping test due to manifest:',
+            {id: test['@id'], name: test.name});
+        }
+        self.skip();
+      }
+
+      // skip based on unknown test type
       const testTypes = Object.keys(TEST_TYPES);
-      if(!isJsonLdType(test, testTypes) || isJsonLdType(test, SKIP_TESTS)) {
-        //const type = [].concat(
-        //  getJsonLdValues(test, '@type'),
-        //  getJsonLdValues(test, 'type')
-        //);
-        //console.log('Skipping test "' + test.name + '" of type: ' + type);
+      if(!isJsonLdType(test, testTypes)) {
+        if(options.verboseSkip) {
+          const type = [].concat(
+            getJsonLdValues(test, '@type'),
+            getJsonLdValues(test, 'type')
+          );
+          console.log('Skipping test due to unknown type:',
+            {id: test['@id'], name: test.name, type});
+        }
         self.skip();
       }
 
+      // skip based on test type
+      if(isJsonLdType(test, SKIP_TESTS)) {
+        if(options.verboseSkip) {
+          const type = [].concat(
+            getJsonLdValues(test, '@type'),
+            getJsonLdValues(test, 'type')
+          );
+          console.log('Skipping test due to test type:',
+            {id: test['@id'], name: test.name, type});
+        }
+        self.skip();
+      }
+
+      // skip based on type info
       if(testInfo.skip && testInfo.skip.type) {
-        //console.log('Skipping test "' + test.name + '" of type: ' + type);
+        if(options.verboseSkip) {
+          console.log('Skipping test due to type info:',
+            {id: test['@id'], name: test.name});
+        }
         self.skip();
       }
 
+      // skip based on id regex
       if(testInfo.skip && testInfo.skip.idRegex) {
         testInfo.skip.idRegex.forEach(function(re) {
           if(re.test(test['@id'])) {
-            //console.log('Skipping test due to id:',
-            //  {id: test['@id']});
+            if(options.verboseSkip) {
+              console.log('Skipping test due to id:',
+                {id: test['@id']});
+            }
             self.skip();
           }
         });
       }
 
+      // skip based on description regex
       if(testInfo.skip && testInfo.skip.descriptionRegex) {
         testInfo.skip.descriptionRegex.forEach(function(re) {
           if(re.test(description)) {
-            //console.log('Skipping test due to description:',
-            //  {id: test['@id'], name: test.name, description});
+            if(options.verboseSkip) {
+              console.log('Skipping test due to description:',
+                {id: test['@id'], name: test.name, description});
+            }
             self.skip();
           }
         });
@@ -490,8 +531,10 @@ function addTest(manifest, test, tests) {
             skipModes = testInfo.skip.processingMode;
           }
           if(skipModes.indexOf(pm) !== -1) {
-            //console.log('Skipping test due to processingMode:',
-            //  {id: test['@id'], name: test.name, processingMode: pm});
+            if(options.verboseSkip) {
+              console.log('Skipping test due to processingMode:',
+                {id: test['@id'], name: test.name, processingMode: pm});
+            }
             self.skip();
           }
         });
@@ -505,8 +548,10 @@ function addTest(manifest, test, tests) {
             skipVersions = testInfo.skip.specVersion;
           }
           if(skipVersions.indexOf(sv) !== -1) {
-            //console.log('Skipping test due to specVersion:',
-            //  {id: test['@id'], name: test.name, specVersion: sv});
+            if(options.verboseSkip) {
+              console.log('Skipping test due to specVersion:',
+                {id: test['@id'], name: test.name, specVersion: sv});
+            }
             self.skip();
           }
         });

tests/test-karma.js

@@ -9,6 +9,8 @@
  *   EARL=filename
  * Bail with tests fail:
  *   BAIL=true
+ * Verbose skip reasons:
+ *   VERBOSE_SKIP=true
  * Benchmark mode:
  *   Basic:
  *   JSONLD_BENCHMARK=1
@@ -106,6 +108,7 @@ const options = {
     id: 'browser',
     filename: process.env.EARL
   },
+  verboseSkip: process.env.VERBOSE_SKIP === 'true',
   bailOnError: process.env.BAIL === 'true',
   entries,
   benchmark,

tests/test.js

@@ -9,6 +9,8 @@
  *   EARL=filename
  * Bail with tests fail:
  *   BAIL=true
+ * Verbose skip reasons:
+ *   VERBOSE_SKIP=true
  * Benchmark mode:
  *   Basic:
  *   JSONLD_BENCHMARK=1
@@ -119,6 +121,7 @@ const options = {
     id: 'node.js',
     filename: process.env.EARL
   },
+  verboseSkip: process.env.VERBOSE_SKIP === 'true',
   bailOnError: process.env.BAIL === 'true',
   entries,
   benchmark,