Tests: Make CLI test cases easier to debug with normalised stack

Krinkle · web-flow · commit e295106eb56e · 2021-03-13T20:42:32.000Z
Remove use of regexes and instead strictly compare exact strings,
but normalise two things:

* collapse any Node internals into a single frame called "internal".
* strip line/column numbers of our own build artefact.
diff --git a/test/cli/fixtures/expected/tap-outputs.js b/test/cli/fixtures/expected/tap-outputs.js
@@ -60,14 +60,13 @@ not ok 1 Throws match > bad
   severity: failed
   actual  : "Error: Match me with a pattern"
   expected: "/incorrect pattern/"
-  stack: .*
+  stack:     at Object.<anonymous> (/qunit/test/cli/fixtures/fail/throws-match.js:3:10)
   ...
 1..1
 # pass 0
 # skip 0
 # todo 0
-# fail 1
-`,
+# fail 1`,
 
 	"qunit test single.js 'glob/**/*-test.js'":
 `TAP version 13
@@ -140,10 +139,9 @@ not ok 2 global failure
 # pass 0
 # skip 0
 # todo 0
-# fail 2
-`,
+# fail 2`,
 
-	// Ignore the last frame about Node processing ticks (differs between Node 10 ad 12+)
+	// The last frame differs between Node 10 and 12+ (changes in processing of ticks)
 	"qunit no-tests":
 `TAP version 13
 not ok 1 global failure
@@ -153,18 +151,17 @@ not ok 1 global failure
   actual  : {}
   expected: undefined
   stack: Error: No tests were run.
-    at done (.*)
-    at advanceTestQueue (.*)
-    at Object.advance (.*)
-    at unblockAndAdvanceQueue (.*)(\n    at <anonymous>)?
-    at .*
+    at done (/qunit/qunit/qunit.js)
+    at advanceTestQueue (/qunit/qunit/qunit.js)
+    at Object.advance (/qunit/qunit/qunit.js)
+    at unblockAndAdvanceQueue (/qunit/qunit/qunit.js)
+    at internal
   ...
 1..1
 # pass 0
 # skip 0
 # todo 0
-# fail 1
-`,
+# fail 1`,
 
 	"qunit sourcemap/source.js":
 `TAP version 13
@@ -181,8 +178,7 @@ not ok 2 Example > bad
 # pass 1
 # skip 0
 # todo 0
-# fail 1
-`,
+# fail 1`,
 
 	"NODE_OPTIONS='--enable-source-maps' qunit sourcemap/source.min.js":
 `TAP version 13
@@ -193,19 +189,18 @@ not ok 2 Example > bad
   severity: failed
   actual  : false
   expected: true
-  stack:     at .* \\(.*source.min.js:1:.*\\)
-        -> .*source.js:7:10
+  stack:     at Object.<anonymous> (/qunit/test/cli/fixtures/sourcemap/source.min.js:1:133)
+        -> /qunit/test/cli/fixtures/sourcemap/sourcemap/source.js:7:10
   ...
 1..2
 # pass 1
 # skip 0
 # todo 0
-# fail 1
-`,
+# fail 1`,
 
 	"qunit ../../es2018/esm.mjs":
 `TAP version 13
-ok 1 ESM test suite > sum\\(\\)
+ok 1 ESM test suite > sum()
 1..1
 # pass 1
 # skip 0
@@ -220,16 +215,14 @@ not ok 1 timeout > first
   severity: failed
   actual  : null
   expected: undefined
-  stack:     at .* (.*timers.js.*)
-    at .*(\n    at .*)?(\n    at .*)?
+  stack:     at internal
   ...
 ok 2 timeout > second
 1..2
 # pass 1
 # skip 0
 # todo 0
-# fail 1
-`,
+# fail 1`,
 
 	"qunit zero-assertions.js":
 `TAP version 13
@@ -250,18 +243,17 @@ not ok 1 global failure
   actual  : {}
   expected: undefined
   stack: Error: No tests matched the filter "no matches".
-    at done (.*)
-    at advanceTestQueue (.*)
-    at Object.advance (.*)
-    at unblockAndAdvanceQueue (.*)(\n    at <anonymous>)?
-    at .*
+    at done (/qunit/qunit/qunit.js)
+    at advanceTestQueue (/qunit/qunit/qunit.js)
+    at Object.advance (/qunit/qunit/qunit.js)
+    at unblockAndAdvanceQueue (/qunit/qunit/qunit.js)
+    at internal
   ...
 1..1
 # pass 0
 # skip 0
 # todo 0
-# fail 1
-`,
+# fail 1`,
 
 	"qunit single.js --require require-dep --require './node_modules/require-dep/module.js'":
 `required require-dep/index.js
@@ -302,7 +294,7 @@ not ok 1 # TODO module B > Only this module should run > a todo test
   severity: todo
   actual  : false
   expected: true
-  stack: .*
+  stack:     at Object.<anonymous> (/qunit/test/cli/fixtures/only/module.js:17:15)
   ...
 ok 2 # SKIP module B > Only this module should run > implicitly skipped test
 ok 3 module B > Only this module should run > normal test
@@ -323,7 +315,7 @@ not ok 1 # TODO module B > test B
   severity: todo
   actual  : false
   expected: true
-  stack: .*
+  stack:     at Object.<anonymous> (/qunit/test/cli/fixtures/only/module-flat.js:9:13)
   ...
 ok 2 # SKIP module B > test C
 ok 3 module B > test D
diff --git a/test/cli/helpers/execute.js b/test/cli/helpers/execute.js
@@ -2,18 +2,48 @@
 
 const path = require( "path" );
 const exec = require( "execa" ).shell;
+const reEscape = /([\\{}()|.?*+\-^$[\]])/g;
+
+// Apply light normalization to CLI output to allow strict string
+// comparison across Node versions and OS platforms against the
+// expected output in fixtures/.
+function normalize( actual ) {
+	const dir = path.join( __dirname, "..", "..", ".." );
+	const reDir = new RegExp( dir.replace( reEscape, "\\$1" ), "g" );
+
+	return actual
+		.replace( reDir, "/qunit" )
+		.replace( /(\/qunit\/qunit\/qunit\.js):\d+:\d+\)/g, "$1)" )
+		.replace( / at .+\([^/)][^)]*\)/g, " at internal" )
+
+		// merge successive lines after initial frame
+		.replace( /(\n\s+at internal)+/g, "$1" )
+
+		// merge successive line with initial frame
+		.replace( /(at internal)\n\s+at internal/g, "$1" );
+}
 
 // Executes the provided command from within the fixtures directory
 // The execaOptions parameter is used by test/cli/watch.js to
 // control the stdio stream.
-module.exports = function execute( command, execaOptions ) {
+module.exports = async function execute( command, execaOptions, hook ) {
 	const cwd = process.cwd();
 	process.chdir( path.join( __dirname, "..", "fixtures" ) );
 
 	command = command.replace( /(^| )qunit\b/, "$1../../../bin/qunit.js" );
 	const execution = exec( command, execaOptions );
+	if ( hook ) {
+		hook( execution );
+	}
 
 	process.chdir( cwd );
 
-	return execution;
+	try {
+		const result = await execution;
+		result.stdout = normalize( String( result.stdout ).trimEnd() );
+		return result;
+	} catch ( e ) {
+		e.stdout = normalize( String( e.stdout ).trimEnd() );
+		throw e;
+	}
 };
diff --git a/test/cli/main.js b/test/cli/main.js
@@ -4,15 +4,6 @@ const expectedOutput = require( "./fixtures/expected/tap-outputs" );
 const execute = require( "./helpers/execute" );
 const semver = require( "semver" );
 
-QUnit.assert.matches = function( actual, expected, message ) {
-	this.pushResult( {
-		result: expected.test( actual ),
-		actual,
-		expected: expected.toString(),
-		message
-	} );
-};
-
 QUnit.module( "CLI Main", function() {
 	QUnit.test( "defaults to running tests in 'test' directory", async function( assert ) {
 		const command = "qunit";
@@ -93,8 +84,7 @@ QUnit.module( "CLI Main", function() {
 		} catch ( e ) {
 			assert.equal( e.code, 1 );
 			assert.equal( e.stderr, "" );
-			const re = new RegExp( expectedOutput[ command ] );
-			assert.equal( re.test( e.stdout ), true );
+			assert.equal( e.stdout, expectedOutput[ command ] );
 		}
 	} );
 
@@ -113,8 +103,7 @@ QUnit.module( "CLI Main", function() {
 		} catch ( e ) {
 			assert.equal( e.code, 1 );
 			assert.equal( e.stderr, "" );
-			const re = new RegExp( expectedOutput[ command ] );
-			assert.equal( re.test( e.stdout ), true );
+			assert.equal( e.stdout, expectedOutput[ command ] );
 		}
 	} );
 
@@ -158,11 +147,7 @@ QUnit.module( "CLI Main", function() {
 				assert.equal( execution.stderr, "" );
 			}
 
-			const re = new RegExp( expectedOutput[ command ] );
-			assert.equal( re.test( execution.stdout ), true );
-			if ( !re.test( execution.stdout ) ) {
-				assert.equal( execution.stdout, expectedOutput[ command ] );
-			}
+			assert.equal( execution.stdout, expectedOutput[ command ] );
 		} );
 	}
 
@@ -188,11 +173,7 @@ QUnit.module( "CLI Main", function() {
 			} catch ( e ) {
 				assert.equal( e.code, 1 );
 				assert.equal( e.stderr, "" );
-				const re = new RegExp( expectedOutput[ command ] );
-				assert.equal( re.test( e.stdout ), true );
-				if ( !re.test( e.stdout ) ) {
-					assert.equal( e.stdout, expectedOutput[ command ] );
-				}
+				assert.equal( e.stdout, expectedOutput[ command ] );
 			}
 		} );
 	}
@@ -204,8 +185,7 @@ QUnit.module( "CLI Main", function() {
 		} catch ( e ) {
 			assert.equal( e.code, 1 );
 			assert.equal( e.stderr, "" );
-			const re = new RegExp( expectedOutput[ command ] );
-			assert.equal( re.test( e.stdout ), true );
+			assert.equal( e.stdout, expectedOutput[ command ] );
 		}
 	} );
 
@@ -249,8 +229,7 @@ QUnit.module( "CLI Main", function() {
 			} catch ( e ) {
 				assert.equal( e.code, 1 );
 				assert.equal( e.stderr, "" );
-				const re = new RegExp( expectedOutput[ command ] );
-				assert.equal( re.test( e.stdout ), true );
+				assert.equal( e.stdout, expectedOutput[ command ] );
 			}
 		} );
 	} );
@@ -323,8 +302,7 @@ QUnit.module( "CLI Main", function() {
 
 			assert.equal( execution.code, 0 );
 			assert.equal( execution.stderr, "" );
-			const re = new RegExp( expectedOutput[ command ] );
-			assert.matches( execution.stdout, re );
+			assert.equal( execution.stdout, expectedOutput[ command ] );
 		} );
 
 		QUnit.test( "flat modules", async function( assert ) {
@@ -334,8 +312,7 @@ QUnit.module( "CLI Main", function() {
 
 			assert.equal( execution.code, 0 );
 			assert.equal( execution.stderr, "" );
-			const re = new RegExp( expectedOutput[ command ] );
-			assert.matches( execution.stdout, re );
+			assert.equal( execution.stdout, expectedOutput[ command ] );
 		} );
 	} );
 } );
diff --git a/test/cli/watch.js b/test/cli/watch.js
