Make all tests pass on node.js trunk

Make sure that we stringify all BigInts as well as all arrays before
passing them to "output" as node.js and d8 print them differently.
Also test both result.output as well as result.error as node.js prints
errors on stderr and not on stdout.

Change-Id: I91ee2c1033cd01cca8c46729f488b438a30af539
Reviewed-on: https://chrome-internal-review.googlesource.com/c/v8/fuzzilli/+/8457616
Reviewed-by: Pawel Krawczyk <[email protected]>
Commit-Queue: Matthias Liedtke <[email protected]>

Author: Liedtke

Sources/Fuzzilli/Base/ProgramBuilder.swift

@@ -4033,6 +4033,15 @@ public class ProgramBuilder {
         emit(WasmResolveForwardReference(), withInputs: [forwardReference, to])
     }
 
+    // Converts an array to a string separating elements by comma. This is used for testing only.
+    func arrayToStringForTesting(_ array: Variable) -> Variable {
+        let stringified = callMethod("map", on: array,
+                withArgs: [buildArrowFunction(with: .parameters(n: 1)) { args in
+            doReturn(callMethod("toString", on: args[0]))
+        }])
+        return callMethod("join", on: stringified, withArgs: [loadString(",")])
+    }
+
     /// Returns the next free variable.
     func nextVariable() -> Variable {
         assert(numVariables < Code.maxNumberOfVariables, "Too many variables")

Tests/FuzzilliTests/LiveTests.swift

@@ -217,7 +217,7 @@ class LiveTests: XCTestCase {
             if result.isFailure {
                 var signature: String? = nil
 
-                for line in result.output.split(separator: "\n") {
+                for line in (result.output + result.error).split(separator: "\n") {
                     if line.contains("Error:") {
                         // Remove anything after a potential 2nd ":", which is usually testcase dependent content, e.g. "SyntaxError: Invalid regular expression: /ep{}[]Z7/: Incomplete quantifier"
                         signature = line.split(separator: ":")[0...1].joined(separator: ":")

Tests/FuzzilliTests/WasmAtomicsTests.swift

@@ -63,7 +63,7 @@ class WasmAtomicsTests: XCTestCase {
         for (i, result) in expectedResults.enumerated() {
             let w = b.getProperty("w\(i)", of: exports)
             let r = b.callFunction(w)
-            b.callFunction(outputFunc, withArgs: [r])
+            b.callFunction(outputFunc, withArgs: [b.callMethod("toString", on: r)])
             expectedOutput += result + "\n"
         }
 
@@ -121,7 +121,7 @@ class WasmAtomicsTests: XCTestCase {
         for (i, result) in expectedResults.enumerated() {
             let w = b.getProperty("w\(i)", of: exports)
             let r = b.callFunction(w)
-            b.callFunction(outputFunc, withArgs: [r])
+            b.callFunction(outputFunc, withArgs: [b.callMethod("toString", on: r)])
             expectedOutput += result + "\n"
         }
 

Tests/FuzzilliTests/WasmTests.swift

@@ -39,7 +39,7 @@ func testForOutputRegex(program: String, runner: JavaScriptExecutor, outputPatte
 
 func testForErrorOutput(program: String, runner: JavaScriptExecutor, errorMessageContains errormsg: String) {
     let result = testExecuteScript(program: program, runner: runner)
-    XCTAssert(result.output.contains(errormsg), "Error messages don't match, got:\n" + result.output)
+    XCTAssert(result.output.contains(errormsg) || result.error.contains(errormsg), "Error messages don't match, got stdout:\n\(result.output)\nstderr:\n\(result.error)")
 }
 
 class WasmSignatureConversionTests: XCTestCase {
@@ -487,7 +487,7 @@ class WasmFoundationTests: XCTestCase {
     }
 
     func testGlobalExnRef() throws {
-        let runner = try GetJavaScriptExecutorOrSkipTest(type: .user, withArguments: ["--experimental-wasm-exnref"])
+        let runner = try GetJavaScriptExecutorOrSkipTest(type: .any, withArguments: ["--experimental-wasm-exnref"])
         let liveTestConfig = Configuration(logLevel: .error, enableInspection: true)
 
         let fuzzer = makeMockFuzzer(config: liveTestConfig, environment: JavaScriptEnvironment())
@@ -1021,7 +1021,7 @@ class WasmFoundationTests: XCTestCase {
         let wasmFunction = b.getProperty(module.getExportedMethod(at: 1), of: exports)
         let result = b.callFunction(wasmFunction, withArgs: [b.loadInt(42)])
         let outputFunc = b.createNamedVariable(forBuiltin: "output")
-        b.callFunction(outputFunc, withArgs: [result])
+        b.callFunction(outputFunc, withArgs: [b.arrayToStringForTesting(result)])
 
         let prog = b.finalize()
         let jsProg = fuzzer.lifter.lift(prog)
@@ -2418,7 +2418,7 @@ class WasmFoundationTests: XCTestCase {
         }
 
         for (i, _) in testCases.enumerated() {
-            let print = b.createNamedVariable(forBuiltin: "print")
+            let print = b.createNamedVariable(forBuiltin: "output")
             let number = b.createNamedVariable(forBuiltin: "Number")
 
             let setFormat = b.buildArrowFunction(with: .parameters(n: 1)) { args in
@@ -3468,7 +3468,7 @@ class WasmFoundationTests: XCTestCase {
     }
 
     func testTryTableNoCatch() throws {
-        let runner = try GetJavaScriptExecutorOrSkipTest(type: .user, withArguments: ["--experimental-wasm-exnref"])
+        let runner = try GetJavaScriptExecutorOrSkipTest(type: .any, withArguments: ["--experimental-wasm-exnref"])
         let liveTestConfig = Configuration(logLevel: .error, enableInspection: true)
         let fuzzer = makeMockFuzzer(config: liveTestConfig, environment: JavaScriptEnvironment())
         let b = fuzzer.makeBuilder()
@@ -4263,7 +4263,7 @@ class WasmGCTests: XCTestCase {
     }
 
     func testRefNullAbstractTypes() throws {
-        let runner = try GetJavaScriptExecutorOrSkipTest()
+        let runner = try GetJavaScriptExecutorOrSkipTest(type: .any, withArguments: ["--experimental-wasm-exnref"])
         let liveTestConfig = Configuration(logLevel: .error, enableInspection: true)
         let fuzzer = makeMockFuzzer(config: liveTestConfig, environment: JavaScriptEnvironment())
         let b = fuzzer.makeBuilder()
@@ -4326,8 +4326,8 @@ class WasmGCTests: XCTestCase {
 
         let positiveResults = b.callMethod(module.getExportedMethod(at: 1), on: exports, withArgs: [positiveI31])
         let negativeResults = b.callMethod(module.getExportedMethod(at: 1), on: exports, withArgs: [negativeI31])
-        b.callFunction(outputFunc, withArgs: [positiveResults])
-        b.callFunction(outputFunc, withArgs: [negativeResults])
+        b.callFunction(outputFunc, withArgs: [b.arrayToStringForTesting(positiveResults)])
+        b.callFunction(outputFunc, withArgs: [b.arrayToStringForTesting(negativeResults)])
 
         let prog = b.finalize()
         let jsProg = fuzzer.lifter.lift(prog)