Improve data-driven testing

Author: ehmicky

test/colors.js

@@ -1,13 +1,14 @@
 'use strict'
 
+const test = require('ava')
 const hasAnsi = require('has-ansi')
 const supportsColor = require('supports-color')
 
-const { forEachEvent, startLogging } = require('./helpers')
+const { repeatEvents, startLogging } = require('./helpers')
 
 /* eslint-disable max-nested-callbacks */
-forEachEvent(({ eventName, emitEvent, test }) => {
-  test('should colorize default opts.getMessage()', async t => {
+repeatEvents((prefix, { eventName, emitEvent }) => {
+  test(`${prefix} should colorize default opts.getMessage()`, async t => {
     const { stopLogging, log } = startLogging({ log: 'spy', eventName })
 
     await emitEvent()
@@ -18,7 +19,7 @@ forEachEvent(({ eventName, emitEvent, test }) => {
     stopLogging()
   })
 
-  test('should allow forcing colorizing default opts.getMessage()', async t => {
+  test(`${prefix} should allow forcing colorizing default opts.getMessage()`, async t => {
     const { stopLogging, log } = startLogging({
       log: 'spy',
       colors: true,
@@ -33,7 +34,7 @@ forEachEvent(({ eventName, emitEvent, test }) => {
     stopLogging()
   })
 
-  test('should allow disabling colorizing default opts.getMessage()', async t => {
+  test(`${prefix} should allow disabling colorizing default opts.getMessage()`, async t => {
     const { stopLogging, log } = startLogging({
       log: 'spy',
       colors: false,

test/emit.js

@@ -6,7 +6,7 @@ const {
   ALL_EVENTS: { all },
 } = require('../helpers')
 
-const { forEachEvent, startLogging } = require('./helpers')
+const { repeatEvents, startLogging } = require('./helpers')
 
 test('[all] events emitters should not throw', async t => {
   const { stopLogging } = startLogging()
@@ -17,13 +17,12 @@ test('[all] events emitters should not throw', async t => {
 })
 
 /* eslint-disable max-nested-callbacks */
-// eslint-disable-next-line no-shadow
-forEachEvent(({ emitEvent, test }) => {
-  test('events emitters should exist', t => {
+repeatEvents((prefix, { emitEvent }) => {
+  test(`${prefix} events emitters should exist`, t => {
     t.is(typeof emitEvent, 'function')
   })
 
-  test('events emitters should not throw', async t => {
+  test(`${prefix} events emitters should not throw`, async t => {
     const { stopLogging } = startLogging()
 
     await t.notThrowsAsync(emitEvent)

test/exit.js

@@ -2,13 +2,14 @@
 
 const process = require('process')
 
+const test = require('ava')
 const sinon = require('sinon')
 const lolex = require('lolex')
 
 // eslint-disable-next-line import/no-internal-modules
 const { EXIT_TIMEOUT } = require('../src/exit')
 
-const { forEachEvent, startLogging } = require('./helpers')
+const { repeatEvents, startLogging } = require('./helpers')
 
 // Stub `process.exit()`
 const stubProcessExit = function() {
@@ -28,8 +29,8 @@ const emitEventAndWait = async function(timeout, { clock, emitEvent }) {
 }
 
 /* eslint-disable max-nested-callbacks */
-forEachEvent(({ eventName, emitEvent, test }) => {
-  test('should call process.exit(1) if inside opts.exitOn', async t => {
+repeatEvents((prefix, { eventName, emitEvent }) => {
+  test(`${prefix} should call process.exit(1) if inside opts.exitOn`, async t => {
     const { clock, processExit } = stubProcessExit()
 
     const exitOn = [eventName]
@@ -45,7 +46,7 @@ forEachEvent(({ eventName, emitEvent, test }) => {
     unstubProcessExit({ clock, processExit })
   })
 
-  test('should not call process.exit(1) if not inside opts.exitOn', async t => {
+  test(`${prefix} should not call process.exit(1) if not inside opts.exitOn`, async t => {
     const { clock, processExit } = stubProcessExit()
 
     const exitOn = []
@@ -60,7 +61,7 @@ forEachEvent(({ eventName, emitEvent, test }) => {
     unstubProcessExit({ clock, processExit })
   })
 
-  test('should delay process.exit(1)', async t => {
+  test(`${prefix} should delay process.exit(1)`, async t => {
     const { clock, processExit } = stubProcessExit()
 
     const { stopLogging } = startLogging({ exitOn: [eventName], eventName })

test/helpers/data_driven.js

@@ -0,0 +1,5 @@
+'use strict'
+
+module.exports = {
+  ...require('./main'),
+}

test/helpers/data_driven/index.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const { getPrefixes } = require('./prefix')
+const { cartesianProduct } = require('./utils')
+
+// Repeat a function with a combination of arguments.
+// Meant for test-driven development.
+// One of the design goals is to be separate from the test framework:
+//  - this can be applied to any test library, not just ava
+//  - this does not require patching test framework functions, which usually
+//    gets in the way of linting such as `eslint-plugin-ava`
+// TODO: extract this library
+//  - check other data-driven test libraries for features
+//  - allow values to be generating functions
+const repeat = function(...args) {
+  const arrays = args.slice(0, -1)
+  const func = args[args.length - 1]
+
+  const arraysA = cartesianProduct(...arrays)
+
+  const prefixes = getPrefixes(arraysA)
+
+  return arraysA.map((values, index) => func(prefixes[index], ...values))
+}
+
+module.exports = {
+  repeat,
+}

test/helpers/data_driven/main.js

@@ -0,0 +1,62 @@
+'use strict'
+
+const { hasDuplicates } = require('./utils')
+
+// Retrieve unique prefix for each combination of values
+const getPrefixes = function(arrays) {
+  const prefixes = arrays.map(getPrefix)
+  const prefixesA = fixDuplicates({ prefixes })
+  return prefixesA
+}
+
+const getPrefix = function(values) {
+  return values
+    .map(getValuePrefix)
+    .filter(Boolean)
+    .join(' ')
+}
+
+const getValuePrefix = function(value) {
+  const valueA = serializeValue(value)
+
+  if (valueA === undefined) {
+    return
+  }
+
+  const valueB = valueA.slice(0, MAX_PREFIX_LENGTH)
+  return `[${valueB}]`
+}
+
+const MAX_PREFIX_LENGTH = 20
+
+// Try to serialize value.
+// Objects must have a `name` member.
+const serializeValue = function(value) {
+  if (value === null || typeof value !== 'object') {
+    return String(value)
+  }
+
+  if (typeof value.name === 'string') {
+    return value.name
+  }
+}
+
+// Add an incrementing counter if some prefixes are duplicates
+// TODO: use an incrementing counter for each value instead of for all values
+// at once
+const fixDuplicates = function({ prefixes }) {
+  if (!hasDuplicates(prefixes)) {
+    return prefixes
+  }
+
+  return prefixes.map(addPrefixIndex)
+}
+
+const addPrefixIndex = function(prefix, index) {
+  const space = prefix.length === 0 ? '' : ' '
+  return `[${index}]${space}${prefix}`
+}
+
+module.exports = {
+  getPrefixes,
+}

test/helpers/data_driven/prefix.js

@@ -0,0 +1,31 @@
+'use strict'
+
+// Does a cartesian product on several arrays
+const cartesianProduct = function(combs, array, ...arrays) {
+  const combsA = combs.map(comb => (Array.isArray(comb) ? comb : [comb]))
+
+  if (!Array.isArray(array)) {
+    return combsA
+  }
+
+  const combsB = combsA.flatMap(comb => arrayProduct({ comb, array }))
+  return cartesianProduct(combsB, ...arrays)
+}
+
+const arrayProduct = function({ comb, array }) {
+  return array.map(value => [...comb, value])
+}
+
+// Check if an array has duplicate items
+const hasDuplicates = function(array) {
+  return array.some((valueA, index) => isDuplicate({ array, valueA, index }))
+}
+
+const isDuplicate = function({ array, valueA, index }) {
+  return array.slice(index + 1).some(valueB => valueB === valueA)
+}
+
+module.exports = {
+  cartesianProduct,
+  hasDuplicates,
+}

test/helpers/data_driven/utils.js

@@ -6,4 +6,5 @@ module.exports = {
   ...require('./stack'),
   ...require('./several'),
   ...require('./data_driven'),
+  ...require('./repeat'),
 }

test/helpers/index.js

@@ -0,0 +1,29 @@
+'use strict'
+
+// eslint-disable-next-line import/no-internal-modules
+const { LEVELS } = require('../../src/level')
+const { EVENTS } = require('../../helpers')
+
+const { repeat } = require('./data_driven')
+
+const getEvents = function() {
+  return Object.entries(EVENTS).map(([eventName, emitEvent]) => ({
+    eventName,
+    emitEvent,
+    name: eventName,
+  }))
+}
+
+const getLevels = function() {
+  return Object.keys(LEVELS)
+}
+
+const repeatEvents = repeat.bind(null, getEvents())
+const repeatLevels = repeat.bind(null, getLevels())
+const repeatEventsLevels = repeat.bind(null, getEvents(), getLevels())
+
+module.exports = {
+  repeatEvents,
+  repeatLevels,
+  repeatEventsLevels,
+}