Core: Add QUnit.config.testFilter to programmatically filter tests

Author: gbudjeakp

src/core/config.js

@@ -29,6 +29,8 @@ const config = {
   // Select by pattern or case-insensitive substring match against "moduleName: testName"
   filter: undefined,
 
+  testFilter: null,
+
   // TODO: Make explicit in QUnit 3.
   // fixture: undefined,
 

src/test.js

@@ -881,15 +881,34 @@ Test.prototype = {
     }
 
     const filter = config.filter;
-    if (!filter) {
-      return true;
+    if (filter) {
+      const regexFilter = /^(!?)\/([\w\W]*)\/(i?$)/.exec(filter);
+      const fullName = (this.module.name + ': ' + this.testName);
+      if (regexFilter) {
+        if (!this.regexFilter(!!regexFilter[1], regexFilter[2], regexFilter[3], fullName)) {
+          return false;
+        }
+      } else if (!this.stringFilter(filter, fullName)) {
+        return false;
+      }
+    }
+
+    if (typeof config.testFilter === 'function') {
+      const testInfo = {
+        testId: this.testId,
+        testName: this.testName,
+        module: this.module.name,
+        skip: !!this.skip
+      };
+      try {
+        return !!config.testFilter(testInfo);
+      } catch (error) {
+        Logger.warn('Error in QUnit.config.testFilter callback: ' + (error.message || error));
+        return false;
+      }
     }
 
-    const regexFilter = /^(!?)\/([\w\W]*)\/(i?$)/.exec(filter);
-    const fullName = (this.module.name + ': ' + this.testName);
-    return regexFilter
-      ? this.regexFilter(!!regexFilter[1], regexFilter[2], regexFilter[3], fullName)
-      : this.stringFilter(filter, fullName);
+    return true;
   },
 
   regexFilter: function (exclude, pattern, flags, fullName) {

test/cli/fixtures/config-testFilter.js

@@ -0,0 +1,25 @@
+const quarantineList = ['flaky test', 'broken test'];
+
+QUnit.config.testFilter = function (testInfo) {
+  return !quarantineList.some(function (pattern) {
+    return testInfo.testName.indexOf(pattern) !== -1;
+  });
+};
+
+QUnit.module('testFilter demo');
+
+QUnit.test('stable test', function (assert) {
+  assert.true(true, 'this test should run');
+});
+
+QUnit.test('flaky test', function (assert) {
+  assert.true(false, 'this test should be filtered out');
+});
+
+QUnit.test('broken test', function (assert) {
+  assert.true(false, 'this test should also be filtered out');
+});
+
+QUnit.test('another stable test', function (assert) {
+  assert.true(true, 'this test should also run');
+});

test/cli/fixtures/config-testFilter.tap.txt

@@ -0,0 +1,11 @@
+# name: config.testFilter
+# command: ["qunit", "config-testFilter.js"]
+
+TAP version 13
+ok 1 testFilter demo > stable test
+ok 2 testFilter demo > another stable test
+1..2
+# pass 2
+# skip 0
+# todo 0
+# fail 0