Core: Ensuring semaphores are balanced when timeout occurs (#1376)

* Core: Ensuring semaphores are balanced when timeout occurs

* Test: Loosening regex in tap-output for testing

Author: scalvert

src/test.js

@@ -728,6 +728,7 @@ export function only( testName, callback ) {
 
 // Put a hold on processing and return a function that will release it.
 export function internalStop( test ) {
+	let released = false;
 	test.semaphore += 1;
 	config.blocking = true;
 
@@ -748,13 +749,13 @@ export function internalStop( test ) {
 					`Test took longer than ${timeoutDuration}ms; test timed out.`,
 					sourceFromStacktrace( 2 )
 				);
+				released = true;
 				internalRecover( test );
 			}, timeoutDuration );
 		}
 
 	}
 
-	let released = false;
 	return function resume() {
 		if ( released ) {
 			return;

test/cli/fixtures/expected/tap-outputs.js

@@ -164,6 +164,26 @@ not ok 1 global failure
 # skip 0
 # todo 0
 # fail 1
+`,
+
+	"qunit timeout":
+`TAP version 13
+not ok 1 timeout > first
+  ---
+  message: "Test took longer than 10ms; test timed out."
+  severity: failed
+  actual: null
+  expected: undefined
+  stack:     at ontimeout (.*)
+    at tryOnTimeout (.*)
+    (.*)\\s*(.*)?
+  ...
+ok 2 timeout > second
+1..2
+# pass 1
+# skip 0
+# todo 0
+# fail 1
 `,
 
 	// in node 8, the stack trace includes 'at <anonymous>. But not in node 6 or 10.

test/cli/fixtures/timeout/index.js

@@ -0,0 +1,14 @@
+QUnit.module( "timeout", function() {
+	QUnit.test( "first", function( assert ) {
+		assert.timeout( 10 );
+
+		return new Promise( resolve => setTimeout( resolve, 20 ) );
+	} );
+
+	QUnit.test( "second", function( assert ) {
+		return new Promise( resolve => setTimeout( resolve, 20 ) )
+			.then( () => {
+				assert.ok( true, "This promise resolved" );
+			} );
+	} );
+} );

test/cli/main.js

@@ -137,6 +137,18 @@ QUnit.module( "CLI Main", function() {
 		}
 	} ) );
 
+	QUnit.test( "timeouts correctly recover", co.wrap( function* ( assert ) {
+		const command = "qunit timeout";
+		try {
+			yield execute( command );
+		} catch ( e ) {
+			assert.equal( e.code, 1 );
+			assert.equal( e.stderr, "" );
+			const re = new RegExp( expectedOutput[ command ] );
+			assert.equal( re.test( e.stdout ), true );
+		}
+	} ) );
+
 	if ( semver.gte( process.versions.node, "9.0.0" ) ) {
 		QUnit.test( "callbacks and hooks from modules are properly released for garbage collection", co.wrap( function* ( assert ) {
 			const command = "node --expose-gc --allow-natives-syntax ../../../bin/qunit.js memory-leak/*.js";