diff --git a/instrumentation/CoverageInstru.cpp b/instrumentation/CoverageInstru.cpp index afb07c8..6b13482 100644 --- a/instrumentation/CoverageInstru.cpp +++ b/instrumentation/CoverageInstru.cpp @@ -1,4 +1,7 @@ #include "CoverageInstru.hpp" +#include +#include +#include namespace wasmInstrumentation { void CoverageInstru::innerAnalysis(BasicBlockAnalysis &basicBlockAnalysis) const noexcept { @@ -30,16 +33,14 @@ void CoverageInstru::innerAnalysis(BasicBlockAnalysis &basicBlockAnalysis) const InstrumentationResponse CoverageInstru::instrument() const noexcept { if (config->fileName.empty() || config->reportFunction.empty() || config->sourceMap.empty() || - config->targetName.empty() || config->expectInfoOutputFilePath.empty() - ) { + config->targetName.empty() || config->expectInfoOutputFilePath.empty()) { std::cout << *config << std::endl; return InstrumentationResponse::CONFIG_ERROR; // config error } - std::filesystem::path filePath(config->fileName); - std::filesystem::path targetFilePath(config->targetName); - std::filesystem::path sourceMapPath(config->sourceMap); - if ((!std::filesystem::exists(filePath)) || - (!std::filesystem::exists(sourceMapPath)) || + const std::filesystem::path filePath(config->fileName); + const std::filesystem::path targetFilePath(config->targetName); + const std::filesystem::path sourceMapPath(config->sourceMap); + if ((!std::filesystem::exists(filePath)) || (!std::filesystem::exists(sourceMapPath)) || (!std::filesystem::exists(targetFilePath.parent_path()))) { std::cout << *config << std::endl; return InstrumentationResponse::CONFIG_FILEPATH_ERROR; // config file path error @@ -127,12 +128,15 @@ InstrumentationResponse CoverageInstru::instrument() const noexcept { MockInstrumentationWalker mockWalker(&module); mockWalker.mockWalk(); + const std::string targetSourceMapPath = std::string{this->config->targetName} + ".map"; + BinaryenSetDebugInfo(true); const BinaryenModuleAllocateAndWriteResult result = - BinaryenModuleAllocateAndWrite(&module, nullptr); + BinaryenModuleAllocateAndWrite(&module, targetSourceMapPath.c_str()); std::ofstream wasmFileStream(this->config->targetName.data(), std::ios::trunc | std::ios::binary); wasmFileStream.write(static_cast(result.binary), static_cast(result.binaryBytes)); - wasmFileStream.close(); + std::ofstream sourceMapFileStream(targetSourceMapPath, std::ios::trunc | std::ios::binary); + sourceMapFileStream << result.sourceMap << std::flush; free(result.binary); free(result.sourceMap); if (wasmFileStream.fail() || wasmFileStream.bad()) { diff --git a/package-lock.json b/package-lock.json index 17d8aad..45edb47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "glob": "^11.0.0", "ignore": "^7.0.3", "semver": "^7.5.3", + "source-map": "^0.7.4", "wasmparser": "5.11.1" }, "bin": { @@ -7848,6 +7849,16 @@ "node": ">=10" } }, + "node_modules/istanbul-lib-source-maps/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/istanbul-reports": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", @@ -11034,13 +11045,12 @@ } }, "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", "license": "BSD-3-Clause", "engines": { - "node": ">=0.10.0" + "node": ">= 8" } }, "node_modules/source-map-js": { @@ -11064,6 +11074,16 @@ "source-map": "^0.6.0" } }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/space-separated-tokens": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", diff --git a/package.json b/package.json index cf785c2..dcc7536 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "glob": "^11.0.0", "ignore": "^7.0.3", "semver": "^7.5.3", + "source-map": "^0.7.4", "wasmparser": "5.11.1" }, "peerDependencies": { diff --git a/src/core/execute.ts b/src/core/execute.ts index 9eb00f0..2ceb7db 100644 --- a/src/core/execute.ts +++ b/src/core/execute.ts @@ -2,14 +2,15 @@ import { WASI } from "node:wasi"; import { promises } from "node:fs"; import { ensureDirSync } from "fs-extra"; import { instantiate, Imports as ASImports } from "@assemblyscript/loader"; -import { ExecutionResult } from "../executionResult.js"; +import { ExecutionResultSummary } from "../executionResult.js"; import { Imports, ImportsArgument, InstrumentResult } from "../interface.js"; import { mockInstrumentFunc } from "../utils/import.js"; import { supplyDefaultFunction } from "../utils/index.js"; import { parseImportFunctionInfo } from "../utils/wasmparser.js"; -import { ExecutionRecorder, SingleExecutionResult } from "./executionRecorder.js"; +import { ExecutionRecorder, ExecutionResult } from "./executionRecorder.js"; import { CoverageRecorder } from "./covRecorder.js"; import assert from "node:assert"; +import { ExecutionError, handleWebAssemblyError } from "../utils/errorTraceHandler.js"; const readFile = promises.readFile; @@ -18,7 +19,7 @@ async function nodeExecutor( outFolder: string, matchedTestNames: string[], imports?: Imports -): Promise { +): Promise { const wasi = new WASI({ args: ["node", instrumentResult.baseName], env: process.env, @@ -48,36 +49,51 @@ async function nodeExecutor( importsArg.module = ins.module; importsArg.instance = ins.instance; importsArg.exports = ins.exports; + + let isCrashed = false; // we don't want to crash any code after crash. AS' heap may be broken. + + const exceptionHandler = async (error: unknown) => { + if (error instanceof WebAssembly.RuntimeError) { + isCrashed = true; + const errorMessage: ExecutionError = await handleWebAssemblyError(error, instrumentResult.instrumentedWasm); + executionRecorder.notifyTestCrash(errorMessage); + return; + } + // unrecoverable error, rethrow + if (error instanceof Error) { + console.error(error.stack); + } + throw new Error("node executor abort"); + }; + try { + executionRecorder.startTestFunction(`${instrumentResult.baseName} - init`); wasi.start(ins); - const execTestFunction = ins.exports["executeTestFunction"]; - assert(typeof execTestFunction === "function"); - if (matchedTestNames.length === 0) { - // By default, all testcases are executed - for (const functionInfo of executionRecorder.registerFunctions) { - const [testCaseName, functionIndex] = functionInfo; - executionRecorder.startTestFunction(testCaseName); + } catch (error) { + await exceptionHandler(error); + } + executionRecorder.finishTestFunction(); + + const execTestFunction = ins.exports["executeTestFunction"]; + assert(typeof execTestFunction === "function"); + + for (const functionInfo of executionRecorder.registerFunctions) { + if (isCrashed) { + break; + } + const [testCaseName, functionIndex] = functionInfo; + if (matchedTestNames.length === 0 || matchedTestNames.includes(testCaseName)) { + executionRecorder.startTestFunction(testCaseName); + try { (execTestFunction as (a: number) => void)(functionIndex); - executionRecorder.finishTestFunction(); - mockInstrumentFunc["mockFunctionStatus.clear"](); + } catch (error) { + await exceptionHandler(error); } - } else { - for (const functionInfo of executionRecorder.registerFunctions) { - const [testCaseName, functionIndex] = functionInfo; - if (matchedTestNames.includes(testCaseName)) { - executionRecorder.startTestFunction(testCaseName); - (execTestFunction as (a: number) => void)(functionIndex); - executionRecorder.finishTestFunction(); - mockInstrumentFunc["mockFunctionStatus.clear"](); - } - } - } - } catch (error) { - if (error instanceof Error) { - console.error(error.stack); + executionRecorder.finishTestFunction(); + mockInstrumentFunc["mockFunctionStatus.clear"](); } - throw new Error("node executor abort."); } + coverageRecorder.outputTrace(instrumentResult.traceFile); return executionRecorder.result; } @@ -87,12 +103,12 @@ export async function execWasmBinaries( instrumentResults: InstrumentResult[], matchedTestNames: string[], imports?: Imports -): Promise { - const assertRes = new ExecutionResult(); +): Promise { + const assertRes = new ExecutionResultSummary(); ensureDirSync(outFolder); await Promise.all( instrumentResults.map(async (instrumentResult): Promise => { - const result: SingleExecutionResult = await nodeExecutor(instrumentResult, outFolder, matchedTestNames, imports); + const result: ExecutionResult = await nodeExecutor(instrumentResult, outFolder, matchedTestNames, imports); await assertRes.merge(result, instrumentResult.expectInfo); }) ); diff --git a/src/core/executionRecorder.ts b/src/core/executionRecorder.ts index 1e7288f..b49241a 100644 --- a/src/core/executionRecorder.ts +++ b/src/core/executionRecorder.ts @@ -1,11 +1,14 @@ +import chalk from "chalk"; import { UnitTestFramework, ImportsArgument, AssertFailMessage, AssertMessage, - IAssertResult, + IExecutionResult, FailedLogMessages, + CrashInfo, } from "../interface.js"; +import { ExecutionError } from "../utils/errorTraceHandler.js"; class LogRecorder { #currentTestLogMessages: string[] = []; @@ -18,7 +21,7 @@ class LogRecorder { this.#isTestFailed = true; } - onStartTest(): void { + reset(): void { this.#currentTestLogMessages = []; this.#isTestFailed = false; } @@ -33,16 +36,16 @@ class LogRecorder { } } -export class SingleExecutionResult implements IAssertResult { +export class ExecutionResult implements IExecutionResult { total: number = 0; fail: number = 0; failedInfo: AssertFailMessage = {}; + crashInfo: CrashInfo = new Set(); failedLogMessages: FailedLogMessages = {}; } -// to do: split execution environment and recorder export class ExecutionRecorder implements UnitTestFramework { - result = new SingleExecutionResult(); + result = new ExecutionResult(); registerFunctions: [string, number][] = []; #currentTestDescriptions: string[] = []; @@ -63,9 +66,10 @@ export class ExecutionRecorder implements UnitTestFramework { const testCaseFullName = this.#currentTestDescriptions.join(" "); this.registerFunctions.push([testCaseFullName, fncIndex]); } + startTestFunction(testCaseFullName: string): void { this.#testCaseFullName = testCaseFullName; - this.logRecorder.onStartTest(); + this.logRecorder.reset(); } finishTestFunction(): void { const logMessages: string[] | null = this.logRecorder.onFinishTest(); @@ -76,11 +80,22 @@ export class ExecutionRecorder implements UnitTestFramework { } } + notifyTestCrash(error: ExecutionError): void { + this.logRecorder.addLog(`Reason: ${chalk.red(error.message)}`); + this.logRecorder.addLog( + error.stacks + .map((stack) => ` at ${stack.functionName} (${stack.fileName}:${stack.lineNumber}:${stack.columnNumber})`) + .join("\n") + ); + this.result.crashInfo.add(this.#testCaseFullName); + this.result.total++; // fake test cases + this.#increaseFailureCount(); + } + collectCheckResult(result: boolean, codeInfoIndex: number, actualValue: string, expectValue: string): void { this.result.total++; if (!result) { - this.logRecorder.markTestFailed(); - this.result.fail++; + this.#increaseFailureCount(); const testCaseFullName = this.#testCaseFullName; const assertMessage: AssertMessage = [codeInfoIndex.toString(), actualValue, expectValue]; this.result.failedInfo[testCaseFullName] = this.result.failedInfo[testCaseFullName] || []; @@ -115,4 +130,9 @@ export class ExecutionRecorder implements UnitTestFramework { }, }; } + + #increaseFailureCount(): void { + this.result.fail++; + this.logRecorder.markTestFailed(); + } } diff --git a/src/executionResult.ts b/src/executionResult.ts index 7149890..7fb51eb 100644 --- a/src/executionResult.ts +++ b/src/executionResult.ts @@ -1,39 +1,81 @@ import { promises } from "node:fs"; import { json2map } from "./utils/index.js"; -import { FailedInfoMap, AssertMessage, ExpectInfo, IAssertResult } from "./interface.js"; +import { + FailedInfoMap, + AssertMessage, + ExpectInfo, + IExecutionResult, + AssertFailMessage, + TestCaseName, + FailedInfo, + FailedLogMessages, +} from "./interface.js"; import chalk from "chalk"; const { readFile, writeFile } = promises; -export class ExecutionResult { +export class ExecutionResultSummary { fail = 0; total = 0; failedInfos: FailedInfoMap = new Map(); - async merge(result: IAssertResult, expectInfoFilePath: string) { + #prepareFailedInfos(testcaseName: TestCaseName): FailedInfo { + if (this.failedInfos.has(testcaseName)) { + return this.failedInfos.get(testcaseName)!; + } + const failedInfo: FailedInfo = { + hasCrash: false, + assertMessages: [], + logMessages: [], + }; + this.failedInfos.set(testcaseName, failedInfo); + return failedInfo; + } + + #processAssertInfo(failedInfo: AssertFailMessage, expectInfo: ExpectInfo) { + for (const [testcaseName, value] of json2map(failedInfo)) { + const errorMsgs: string[] = []; + for (const msg of value) { + const [index, actualValue, expectValue] = msg; + const debugLocation = expectInfo[index]; + let errorMsg = `${debugLocation ?? ""} value: ${actualValue} expect: ${expectValue}`; + if (errorMsg.length > 160) { + errorMsg = `${debugLocation ?? ""}\nvalue: \n ${actualValue}\nexpect: \n ${expectValue}`; + } + errorMsgs.push(errorMsg); + } + this.#prepareFailedInfos(testcaseName).assertMessages = + this.#prepareFailedInfos(testcaseName).assertMessages.concat(errorMsgs); + } + } + + #processCrashInfo(crashInfo: Set) { + for (const testcaseName of crashInfo) { + this.#prepareFailedInfos(testcaseName).hasCrash = true; + } + } + + /** + * It should be called after other error processed to append log messages. + */ + #processLogMessages(failedLogMessages: FailedLogMessages) { + for (const [testcaseName, failedInfo] of this.failedInfos) { + if (failedLogMessages[testcaseName] !== undefined) { + failedInfo.logMessages = failedInfo.logMessages.concat(failedLogMessages[testcaseName]); + } + } + } + + async merge(result: IExecutionResult, expectInfoFilePath: string) { this.fail += result.fail; this.total += result.total; if (result.fail > 0) { - let expectInfo; try { const expectContent = await readFile(expectInfoFilePath, { encoding: "utf8" }); - expectInfo = json2map(JSON.parse(expectContent) as ExpectInfo); - for (const [testcaseName, value] of json2map(result.failedInfo)) { - const errorMsgs: string[] = []; - for (const msg of value) { - const [index, actualValue, expectValue] = msg; - const debugLocation = expectInfo.get(index); - let errorMsg = `${debugLocation ?? ""} value: ${actualValue} expect: ${expectValue}`; - if (errorMsg.length > 160) { - errorMsg = `${debugLocation ?? ""}\nvalue: \n ${actualValue}\nexpect: \n ${expectValue}`; - } - errorMsgs.push(errorMsg); - } - this.failedInfos.set(testcaseName, { - assertMessages: errorMsgs, - logMessages: result.failedLogMessages[testcaseName], - }); - } + const expectInfo = JSON.parse(expectContent) as ExpectInfo; + this.#processAssertInfo(result.failedInfo, expectInfo); + this.#processCrashInfo(result.crashInfo); + this.#processLogMessages(result.failedLogMessages); } catch (error) { if (error instanceof Error) { console.error(error.stack); @@ -54,15 +96,28 @@ export class ExecutionResult { this.total.toString(); log(`\ntest case: ${rate} (success/total)\n`); if (this.fail !== 0) { - log(chalk.red("Error Message: ")); - for (const [testcaseName, { assertMessages, logMessages }] of this.failedInfos.entries()) { - log(` ${testcaseName}: `); - for (const assertMessage of assertMessages) { - log(" " + chalk.yellow(assertMessage)); - } - for (const logMessage of logMessages ?? []) { - log(chalk.gray(logMessage)); - } + this.#printErrorMessage(log); + } + } + + #printErrorMessage(log: (msg: string) => void): void { + log(chalk.red("Error Message: ")); + // sort failedInfos by testcaseName to keep stability for e2e testing + const failedInfosArray = Array.from(this.failedInfos.entries()).sort((a, b) => a[0].localeCompare(b[0])); + for (const [testcaseName, { hasCrash, assertMessages, logMessages }] of failedInfosArray) { + log(` ${testcaseName}: `); + for (const assertMessage of assertMessages) { + log(" " + chalk.yellow(assertMessage)); + } + if (hasCrash) { + log(" " + chalk.red("Test Crashed!")); + } + for (const logMessage of logMessages ?? []) { + log(chalk.gray(logMessage)); + } + if (logMessages.length > 0) { + // empty line to separate test + log(""); } } } diff --git a/src/interface.ts b/src/interface.ts index 5706f77..8bd2443 100644 --- a/src/interface.ts +++ b/src/interface.ts @@ -5,10 +5,14 @@ import { Type } from "wasmparser"; import { ASUtil } from "@assemblyscript/loader"; +import path from "node:path"; // instrumented file information export class InstrumentResult { - constructor(public baseName: string) {} + baseName: string; + constructor(baseName: string) { + this.baseName = path.relative(process.cwd(), baseName); + } get sourceWasm() { return this.baseName.concat(".wasm"); } @@ -69,17 +73,19 @@ export type AssertExpectValue = string; export type AssertActualValue = string; export type AssertMessage = [ExpectInfoIndex, AssertActualValue, AssertExpectValue]; export type AssertFailMessage = Record; - +export type CrashInfo = Set; export type FailedLogMessages = Record; -export type FailedInfoMap = Map; +export type FailedInfo = { hasCrash: boolean; assertMessages: string[]; logMessages: string[] }; +export type FailedInfoMap = Map; export type ExpectInfoDebugLocation = string; export type ExpectInfo = Record; -export interface IAssertResult { +export interface IExecutionResult { fail: number; total: number; + crashInfo: CrashInfo; failedInfo: AssertFailMessage; failedLogMessages: FailedLogMessages; } diff --git a/src/utils/errorTraceHandler.ts b/src/utils/errorTraceHandler.ts new file mode 100644 index 0000000..57abe47 --- /dev/null +++ b/src/utils/errorTraceHandler.ts @@ -0,0 +1,138 @@ +// ref: https://v8.dev/docs/stack-trace-api + +import { readFile } from "node:fs/promises"; +import { parseSourceMapPath } from "./wasmparser.js"; +import { BasicSourceMapConsumer, IndexedSourceMapConsumer, SourceMapConsumer } from "source-map"; +import chalk from "chalk"; + +export interface WebAssemblyCallSite { + functionName: string; + fileName: string; + lineNumber: number; + columnNumber: number; +} + +interface WebAssemblyModuleInfo { + wasmPath: string; + sourceMapConsumer: SourceMapHandler | null; +} + +type SourceMapHandler = BasicSourceMapConsumer | IndexedSourceMapConsumer; + +interface SourceLocation { + fileName: string; + lineNumber: number; + columnNumber: number; +} + +function getOriginLocationWithSourceMap( + line: number | null, + column: number | null, + sourceMapConsumer: SourceMapHandler | null +): SourceLocation | null { + if (sourceMapConsumer === null || line === null || column === null) { + return null; + } + const originPosition = sourceMapConsumer.originalPositionFor({ + line: line, + column: column, + }); + if (originPosition.source === null || originPosition.line === null || originPosition.column === null) { + return null; + } + return { + fileName: originPosition.source, + lineNumber: originPosition.line, + columnNumber: originPosition.column, + }; +} + +function getWebAssemblyFunctionName(callSite: NodeJS.CallSite): string { + return callSite.getFunctionName() ?? `wasm-function[${callSite.getFunction()?.toString() ?? "unknown"}]`; +} + +function createWebAssemblyCallSite( + callSite: NodeJS.CallSite, + moduleInfo: WebAssemblyModuleInfo +): WebAssemblyCallSite | null { + if (!callSite.getFileName()?.startsWith("wasm")) { + // ignore non-wasm call sites + return null; + } + const line: number | null = callSite.getLineNumber(); + const column: number | null = callSite.getColumnNumber(); + const originalPosition: SourceLocation | null = getOriginLocationWithSourceMap( + line, + column, + moduleInfo.sourceMapConsumer + ); + if (originalPosition) { + return { + fileName: originalPosition.fileName, + functionName: getWebAssemblyFunctionName(callSite), + lineNumber: originalPosition.lineNumber, + columnNumber: originalPosition.columnNumber, + }; + } + // fallback to the original call site + return { + fileName: moduleInfo.wasmPath, + functionName: getWebAssemblyFunctionName(callSite), + lineNumber: line || -1, + columnNumber: column || -1, + }; +} + +export interface ExecutionError { + message: string; + stacks: WebAssemblyCallSite[]; +} + +async function getSourceMapConsumer(sourceMapPath: string | null): Promise { + if (sourceMapPath === null) { + return null; + } + const sourceMapContent: string | null = await (async () => { + try { + return await readFile(sourceMapPath, "utf8"); + } catch (error) { + if (error instanceof Error) { + console.log(chalk.yellow(`Failed to read source map file: ${sourceMapPath} due to ${error}`)); + } + return null; + } + })(); + if (sourceMapContent === null) { + return null; + } + return await new SourceMapConsumer(sourceMapContent, undefined); +} + +export async function handleWebAssemblyError( + error: WebAssembly.RuntimeError, + wasmPath: string +): Promise { + let stackTrace: NodeJS.CallSite[] = []; + // eslint-disable-next-line @typescript-eslint/unbound-method + const originalPrepareStackTrace = Error.prepareStackTrace; + Error.prepareStackTrace = (_: Error, structuredStackTrace: NodeJS.CallSite[]) => { + stackTrace = structuredStackTrace; + }; + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + error.stack; // trigger prepareStackTrace + Error.prepareStackTrace = originalPrepareStackTrace; + + const wasmBuffer = await readFile(wasmPath); + const sourceMapPath = parseSourceMapPath( + wasmBuffer.buffer.slice(wasmBuffer.byteOffset, wasmBuffer.byteLength) as ArrayBuffer + ); + const sourceMapConsumer: SourceMapHandler | null = await getSourceMapConsumer(sourceMapPath); + const stacks = stackTrace + .map((callSite) => createWebAssemblyCallSite(callSite, { wasmPath, sourceMapConsumer })) + .filter((callSite) => callSite !== null); + sourceMapConsumer?.destroy(); // clean up the source map consumer + return { + message: error.message, + stacks, + }; +} diff --git a/src/utils/wasmparser.ts b/src/utils/wasmparser.ts index 0532ca5..3483152 100644 --- a/src/utils/wasmparser.ts +++ b/src/utils/wasmparser.ts @@ -6,6 +6,7 @@ import { ISectionInformation, SectionCode, ITypeEntry, + ISourceMappingURL, } from "wasmparser"; import { ImportFunctionInfo } from "../interface.js"; import assert from "node:assert"; @@ -75,3 +76,22 @@ export function parseImportFunctionInfo(buf: ArrayBuffer) { } } } + +export function parseSourceMapPath(buf: ArrayBuffer): string | null { + const reader = new BinaryReader(); + reader.setData(buf, 0, buf.byteLength); + while (true) { + if (!reader.read()) { + return null; + } + if (reader.state === BinaryReaderState.BEGIN_SECTION) { + const sectionInfo = reader.result as ISectionInformation; + if (sectionInfo.id !== SectionCode.Custom) { + reader.skipSection(); + } + } else if (reader.state === BinaryReaderState.SOURCE_MAPPING_URL) { + const sectionInfo = reader.result as ISourceMappingURL; + return new TextDecoder("utf8").decode(sectionInfo.url); + } + } +} diff --git a/tests/e2e/assertFailed/assertOnInit.test.ts b/tests/e2e/assertFailed/assertOnInit.test.ts new file mode 100644 index 0000000..fc7dea4 --- /dev/null +++ b/tests/e2e/assertFailed/assertOnInit.test.ts @@ -0,0 +1 @@ +assert(false); diff --git a/tests/e2e/assertFailed/assertOnTest.test.ts b/tests/e2e/assertFailed/assertOnTest.test.ts new file mode 100644 index 0000000..8d4415e --- /dev/null +++ b/tests/e2e/assertFailed/assertOnTest.test.ts @@ -0,0 +1,7 @@ +import { test, expect } from "../../../assembly"; +import { log } from "./env"; + +test("assert on test", () => { + log("This test will fail due to an assertion error"); + assert(false, "This assertion is expected to fail"); +}); diff --git a/tests/e2e/assertFailed/source.test.ts b/tests/e2e/assertFailed/source.test.ts deleted file mode 100644 index 52cc3ae..0000000 --- a/tests/e2e/assertFailed/source.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { test, expect } from "../../../assembly"; -import { log } from "./env"; - -test("failed test", () => { - log("This is a log message for the failed test."); - assert(false, "This is AS assert failure"); -}); - -test("succeed test", () => { - log("This is a log message for the succeed test."); - expect(1 + 1).equal(2); -}); diff --git a/tests/e2e/assertFailed/stdout.txt b/tests/e2e/assertFailed/stdout.txt index 0adbf7c..a9ed498 100644 --- a/tests/e2e/assertFailed/stdout.txt +++ b/tests/e2e/assertFailed/stdout.txt @@ -1,3 +1,21 @@ code analysis: OK compile testcases: OK instrument: OK +execute testcases: OK + +test case: 0/2 (success/total) + +Error Message: + assert on test: + Test Crashed! +This test will fail due to an assertion error +Reason: unreachable + at start:tests/e2e/assertFailed/assertOnTest.test~anonymous|0 (tests/e2e/assertFailed/assertOnTest.test.ts:6:2) + at executeTestFunction (tests/e2e/assertFailed/tmp/assertOnTest.test.instrumented.wasm:1:675) + + tests/e2e/assertFailed/tmp/assertOnInit.test - init: + Test Crashed! +Reason: unreachable + at start:tests/e2e/assertFailed/assertOnInit.test (tests/e2e/assertFailed/assertOnInit.test.ts:1:0) + at ~start (tests/e2e/assertFailed/tmp/assertOnInit.test.instrumented.wasm:1:244) + diff --git a/tests/e2e/printLogInFailedInfo/as-test.config.js b/tests/e2e/printLogInFailedInfo/as-test.config.js index 6a4e6f8..8dac18a 100644 --- a/tests/e2e/printLogInFailedInfo/as-test.config.js +++ b/tests/e2e/printLogInFailedInfo/as-test.config.js @@ -2,7 +2,6 @@ import os from "node:os"; import path from "node:path"; import { URL } from "node:url"; -const tmpFolder = path.join(os.tmpdir(), "as-test-e2e"); const __dirname = path.dirname(new URL(import.meta.url).pathname); export default { diff --git a/tests/e2e/printLogInFailedInfo/stdout.txt b/tests/e2e/printLogInFailedInfo/stdout.txt index 34bd3c2..807520a 100644 --- a/tests/e2e/printLogInFailedInfo/stdout.txt +++ b/tests/e2e/printLogInFailedInfo/stdout.txt @@ -9,3 +9,4 @@ Error Message: failed test: tests/e2e/printLogInFailedInfo/source.test.ts:6:2 value: 2 expect: = 3 This is a log message for the failed test. + diff --git a/tests/e2e/run.js b/tests/e2e/run.js index 0dc5ae1..f37774c 100644 --- a/tests/e2e/run.js +++ b/tests/e2e/run.js @@ -1,16 +1,17 @@ import assert from "node:assert"; import { exec } from "node:child_process"; -import { diffChars } from "diff"; +import { diffLines } from "diff"; import chalk from "chalk"; import { argv } from "node:process"; import { readFileSync } from "node:fs"; function getDiff(s1, s2) { - const handleEscape = (c) => { - if (c === "\n") return "\n'\\n'"; - return c; - }; - return diffChars(s1, s2) + const handleEscape = (c) => + c + .split("\n") + .map((l) => (l.length === 0 ? "\xB6" : l)) + .join("\n"); + return diffLines(s1, s2) .map((part) => { if (part.added) { return chalk.bgGreen(handleEscape(part.value)); diff --git a/tests/ts/fixture/withSourceMapURL.wasm b/tests/ts/fixture/withSourceMapURL.wasm new file mode 100644 index 0000000..4f409ea Binary files /dev/null and b/tests/ts/fixture/withSourceMapURL.wasm differ diff --git a/tests/ts/test/__snapshots__/executionResult.test.ts.snap b/tests/ts/test/__snapshots__/executionResult.test.ts.snap index 9549f42..0446997 100644 --- a/tests/ts/test/__snapshots__/executionResult.test.ts.snap +++ b/tests/ts/test/__snapshots__/executionResult.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`print 1`] = ` +exports[`print assert failed 1`] = ` " test case: 27/28 (success/total) @@ -9,10 +9,11 @@ Error Message: tests/as/comparison.test.ts:10:20 value: 100 expect: = 200 log message 1 log message 2 -log message 3" +log message 3 +" `; -exports[`print 2`] = ` +exports[`print assert failed 2`] = ` " test case: 27/28 (success/total) @@ -21,5 +22,32 @@ test case: 27/28 (success/total) tests/as/comparison.test.ts:10:20 value: 100 expect: = 200 log message 1 log message 2 -log message 3" +log message 3 +" +`; + +exports[`print crash 1`] = ` +" +test case: 27/28 (success/total) + +Error Message: + A: + Test Crashed! +log message 1 +log message 2 +log message 3 +" +`; + +exports[`print crash 2`] = ` +" +test case: 27/28 (success/total) + +Error Message:  + A: + Test Crashed! +log message 1 +log message 2 +log message 3 +" `; diff --git a/tests/ts/test/core/instrument.test.ts b/tests/ts/test/core/instrument.test.ts index 4cae7fa..b604841 100644 --- a/tests/ts/test/core/instrument.test.ts +++ b/tests/ts/test/core/instrument.test.ts @@ -1,12 +1,12 @@ import fs from "fs-extra"; -import { join } from "node:path"; +import { join, relative } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath, URL } from "node:url"; import { compile } from "../../../../src/core/compile.js"; import { instrument } from "../../../../src/core/instrument.js"; const fixturePath = join(fileURLToPath(new URL(".", import.meta.url)), "..", "..", "fixture", "constructor.ts"); -const outputDir = join(tmpdir(), "assemblyscript-unittest-framework"); +const outputDir = relative(process.cwd(), join(tmpdir(), "assemblyscript-unittest-framework")); test("Instrument", async () => { await compile([fixturePath], outputDir, "--memoryBase 16 --exportTable"); diff --git a/tests/ts/test/executionResult.test.ts b/tests/ts/test/executionResult.test.ts index cf99f88..915f4e5 100644 --- a/tests/ts/test/executionResult.test.ts +++ b/tests/ts/test/executionResult.test.ts @@ -1,16 +1,17 @@ import { join } from "node:path"; import { fileURLToPath, URL } from "node:url"; -import { FailedInfoMap, IAssertResult } from "../../../src/interface.js"; -import { ExecutionResult } from "../../../src/executionResult.js"; +import { FailedInfoMap, IExecutionResult } from "../../../src/interface.js"; +import { ExecutionResultSummary } from "../../../src/executionResult.js"; import chalk from "chalk"; const __dirname = fileURLToPath(new URL(".", import.meta.url)); test("no failedInfo merge", async () => { - const executionResult = new ExecutionResult(); - const testcaseA: IAssertResult = { + const executionResult = new ExecutionResultSummary(); + const testcaseA: IExecutionResult = { fail: 0, total: 28, + crashInfo: new Set(), failedInfo: {}, failedLogMessages: {}, }; @@ -21,13 +22,14 @@ test("no failedInfo merge", async () => { expect(executionResult.failedInfos).toEqual(new Map()); }); -test("equal failed", async () => { - const executionResult = new ExecutionResult(); +test("equal assert failed", async () => { + const executionResult = new ExecutionResultSummary(); const actualString = "A long sentence for testing errorMsg.length > 160 in executionResult.ts merge function"; const expectString = "= A long sentence for testing errorMsg.length > 160 in executionResult.ts merge function "; - const testcaseA: IAssertResult = { + const testcaseA: IExecutionResult = { fail: 1, total: 28, + crashInfo: new Set(), failedInfo: { A: [ ["1", "100", "= 200"], @@ -44,6 +46,7 @@ test("equal failed", async () => { await executionResult.merge(testcaseA, expectInfoFIlePath); const expectFailedInfo: FailedInfoMap = new Map(); expectFailedInfo.set("A", { + hasCrash: false, assertMessages: [ "tests/as/comparison.test.ts:10:20 value: 100 expect: = 200", "tests/as/comparison.test.ts:15:27 value: [10] expect: = [1]", @@ -57,31 +60,88 @@ test("equal failed", async () => { expect(executionResult.failedInfos).toEqual(expectFailedInfo); }); -test("print", async () => { - const executionResult = new ExecutionResult(); - const testcaseA: IAssertResult = { +test("equal crash", async () => { + const executionResult = new ExecutionResultSummary(); + const testcaseA: IExecutionResult = { fail: 1, - total: 28, - failedInfo: { - A: [["1", "100", "= 200"]], - }, + total: 1, + crashInfo: new Set(), + failedInfo: {}, failedLogMessages: { A: ["log message 1", "log message 2", "log message 3"], }, }; + testcaseA.crashInfo.add("A"); const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); await executionResult.merge(testcaseA, expectInfoFIlePath); + const expectFailedInfo: FailedInfoMap = new Map(); + expectFailedInfo.set("A", { + hasCrash: true, + assertMessages: [], + logMessages: ["log message 1", "log message 2", "log message 3"], + }); + expect(executionResult.fail).toEqual(1); + expect(executionResult.total).toEqual(1); + expect(executionResult.failedInfos).toEqual(expectFailedInfo); +}); + +describe("print", () => { + test("assert failed", async () => { + const executionResult = new ExecutionResultSummary(); + const testcaseA: IExecutionResult = { + fail: 1, + total: 28, + crashInfo: new Set(), + failedInfo: { + A: [["1", "100", "= 200"]], + }, + failedLogMessages: { + A: ["log message 1", "log message 2", "log message 3"], + }, + }; + const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); + await executionResult.merge(testcaseA, expectInfoFIlePath); - { - const outputs: string[] = []; - chalk.level = 0; // disable color - executionResult.print((msg) => outputs.push(msg)); - expect(outputs.join("\n")).toMatchSnapshot(); - } - { - const outputs: string[] = []; - chalk.level = 1; // force enable color - executionResult.print((msg) => outputs.push(msg)); - expect(outputs.join("\n")).toMatchSnapshot(); - } + { + const outputs: string[] = []; + chalk.level = 0; // disable color + executionResult.print((msg) => outputs.push(msg)); + expect(outputs.join("\n")).toMatchSnapshot(); + } + { + const outputs: string[] = []; + chalk.level = 1; // force enable color + executionResult.print((msg) => outputs.push(msg)); + expect(outputs.join("\n")).toMatchSnapshot(); + } + }); + + test("crash", async () => { + const executionResult = new ExecutionResultSummary(); + const testcaseA: IExecutionResult = { + fail: 1, + total: 28, + crashInfo: new Set(), + failedInfo: {}, + failedLogMessages: { + A: ["log message 1", "log message 2", "log message 3"], + }, + }; + testcaseA.crashInfo.add("A"); + const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); + await executionResult.merge(testcaseA, expectInfoFIlePath); + + { + const outputs: string[] = []; + chalk.level = 0; // disable color + executionResult.print((msg) => outputs.push(msg)); + expect(outputs.join("\n")).toMatchSnapshot(); + } + { + const outputs: string[] = []; + chalk.level = 1; // force enable color + executionResult.print((msg) => outputs.push(msg)); + expect(outputs.join("\n")).toMatchSnapshot(); + } + }); }); diff --git a/tests/ts/test/utils/wasmparser.test.ts b/tests/ts/test/utils/wasmparser.test.ts index 2f57c79..329469c 100644 --- a/tests/ts/test/utils/wasmparser.test.ts +++ b/tests/ts/test/utils/wasmparser.test.ts @@ -1,4 +1,4 @@ -import { parseImportFunctionInfo } from "../../../../src/utils/wasmparser.js"; +import { parseImportFunctionInfo, parseSourceMapPath } from "../../../../src/utils/wasmparser.js"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath, URL } from "node:url"; @@ -51,3 +51,12 @@ test("parseImportFunctionInfo", () => { expect(parseImportFunctionInfo(arrayBuffer)).toEqual(expectedInfo); // Pass ArrayBuffer }); + +test("parseSourceMapURL", () => { + const fp = join(__dirname, "..", "..", "fixture", "withSourceMapURL.wasm"); + const buf = readFileSync(fp); + const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); // Convert Buffer to ArrayBuffer + console.log(arrayBuffer); + + expect(parseSourceMapPath(arrayBuffer)).toBe("./release.wasm.map"); +});