Skip to content

Commit 15edee1

Browse files
HerrCai0907HerrCai0907
committed
wip
1 parent 5c1e940 commit 15edee1

File tree

10 files changed

+267
-85
lines changed

10 files changed

+267
-85
lines changed

‎instrumentation/CoverageInstru.cpp

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#include "CoverageInstru.hpp"
2+
#include <fstream>
3+
#include <ostream>
24
namespace wasmInstrumentation {
35

46
void CoverageInstru::innerAnalysis(BasicBlockAnalysis &basicBlockAnalysis) const noexcept {
@@ -30,16 +32,14 @@ void CoverageInstru::innerAnalysis(BasicBlockAnalysis &basicBlockAnalysis) const
3032

3133
InstrumentationResponse CoverageInstru::instrument() const noexcept {
3234
if (config->fileName.empty() || config->reportFunction.empty() || config->sourceMap.empty() ||
33-
config->targetName.empty() || config->expectInfoOutputFilePath.empty()
34-
) {
35+
config->targetName.empty() || config->expectInfoOutputFilePath.empty()) {
3536
std::cout << *config << std::endl;
3637
return InstrumentationResponse::CONFIG_ERROR; // config error
3738
}
38-
std::filesystem::path filePath(config->fileName);
39-
std::filesystem::path targetFilePath(config->targetName);
40-
std::filesystem::path sourceMapPath(config->sourceMap);
41-
if ((!std::filesystem::exists(filePath)) ||
42-
(!std::filesystem::exists(sourceMapPath)) ||
39+
const std::filesystem::path filePath(config->fileName);
40+
const std::filesystem::path targetFilePath(config->targetName);
41+
const std::filesystem::path sourceMapPath(config->sourceMap);
42+
if ((!std::filesystem::exists(filePath)) || (!std::filesystem::exists(sourceMapPath)) ||
4343
(!std::filesystem::exists(targetFilePath.parent_path()))) {
4444
std::cout << *config << std::endl;
4545
return InstrumentationResponse::CONFIG_FILEPATH_ERROR; // config file path error
@@ -127,12 +127,14 @@ InstrumentationResponse CoverageInstru::instrument() const noexcept {
127127
MockInstrumentationWalker mockWalker(&module);
128128
mockWalker.mockWalk();
129129

130+
const std::string targetSourceMapPath = std::string{this->config->targetName} + ".map";
130131
const BinaryenModuleAllocateAndWriteResult result =
131-
BinaryenModuleAllocateAndWrite(&module, nullptr);
132+
BinaryenModuleAllocateAndWrite(&module, targetSourceMapPath.c_str());
132133
std::ofstream wasmFileStream(this->config->targetName.data(), std::ios::trunc | std::ios::binary);
133134
wasmFileStream.write(static_cast<char *>(result.binary),
134135
static_cast<std::streamsize>(result.binaryBytes));
135-
wasmFileStream.close();
136+
std::ofstream sourceMapFileStream(targetSourceMapPath, std::ios::trunc | std::ios::binary);
137+
sourceMapFileStream << result.sourceMap << std::flush;
136138
free(result.binary);
137139
free(result.sourceMap);
138140
if (wasmFileStream.fail() || wasmFileStream.bad()) {

‎src/core/execute.ts

Lines changed: 47 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@ import { WASI } from "node:wasi";
22
import { promises } from "node:fs";
33
import { ensureDirSync } from "fs-extra";
44
import { instantiate, Imports as ASImports } from "@assemblyscript/loader";
5-
import { ExecutionResult } from "../executionResult.js";
5+
import { ExecutionResultSummary } from "../executionResult.js";
66
import { Imports, ImportsArgument, InstrumentResult } from "../interface.js";
77
import { mockInstrumentFunc } from "../utils/import.js";
88
import { supplyDefaultFunction } from "../utils/index.js";
99
import { parseImportFunctionInfo } from "../utils/wasmparser.js";
10-
import { ExecutionRecorder, SingleExecutionResult } from "./executionRecorder.js";
10+
import { ExecutionRecorder, ExecutionResult } from "./executionRecorder.js";
1111
import { CoverageRecorder } from "./covRecorder.js";
1212
import assert from "node:assert";
13+
import { ExecutionError, handleWebAssemblyError } from "../utils/errorTraceHandler.js";
1314

1415
const readFile = promises.readFile;
1516

@@ -18,7 +19,7 @@ async function nodeExecutor(
1819
outFolder: string,
1920
matchedTestNames?: string[],
2021
imports?: Imports
21-
): Promise<SingleExecutionResult> {
22+
): Promise<ExecutionResult> {
2223
const wasi = new WASI({
2324
args: ["node", instrumentResult.baseName],
2425
env: process.env,
@@ -48,36 +49,52 @@ async function nodeExecutor(
4849
importsArg.module = ins.module;
4950
importsArg.instance = ins.instance;
5051
importsArg.exports = ins.exports;
52+
let isCrashed = false; // we don't want to crash any code after crash. AS' heap may be broken.
5153
try {
54+
executionRecorder.startTestFunction(`${instrumentResult.baseName} - init`);
5255
wasi.start(ins);
53-
const execTestFunction = ins.exports["executeTestFunction"];
54-
assert(typeof execTestFunction === "function");
55-
if (matchedTestNames === undefined) {
56-
// By default, all testcases are executed
57-
for (const functionInfo of executionRecorder.registerFunctions) {
58-
const [testCaseName, functionIndex] = functionInfo;
59-
executionRecorder.startTestFunction(testCaseName);
60-
(execTestFunction as (a: number) => void)(functionIndex);
61-
executionRecorder.finishTestFunction();
62-
mockInstrumentFunc["mockFunctionStatus.clear"]();
63-
}
56+
} catch (error) {
57+
if (error instanceof WebAssembly.RuntimeError) {
58+
isCrashed = true;
59+
const errorMessage: ExecutionError = await handleWebAssemblyError(error, instrumentResult.instrumentedWasm);
60+
executionRecorder.notifyTestCrash(errorMessage);
6461
} else {
65-
for (const functionInfo of executionRecorder.registerFunctions) {
66-
const [testCaseName, functionIndex] = functionInfo;
67-
if (matchedTestNames.includes(testCaseName)) {
68-
executionRecorder.startTestFunction(testCaseName);
69-
(execTestFunction as (a: number) => void)(functionIndex);
70-
executionRecorder.finishTestFunction();
71-
mockInstrumentFunc["mockFunctionStatus.clear"]();
72-
}
62+
// unrecoverable error, rethrow
63+
if (error instanceof Error) {
64+
console.error(error.stack);
7365
}
66+
throw new Error("node executor abort");
7467
}
75-
} catch (error) {
76-
if (error instanceof Error) {
77-
console.error(error.stack);
78-
}
79-
throw new Error("node executor abort.");
8068
}
69+
executionRecorder.finishTestFunction();
70+
// try {
71+
// const execTestFunction = ins.exports["executeTestFunction"];
72+
// assert(typeof execTestFunction === "function");
73+
// if (matchedTestNames === undefined) {
74+
// // By default, all testcases are executed
75+
// for (const functionInfo of executionRecorder.registerFunctions) {
76+
// const [testCaseName, functionIndex] = functionInfo;
77+
// executionRecorder.startTestFunction(testCaseName);
78+
// (execTestFunction as (a: number) => void)(functionIndex);
79+
// executionRecorder.finishTestFunction();
80+
// mockInstrumentFunc["mockFunctionStatus.clear"]();
81+
// }
82+
// } else {
83+
// for (const functionInfo of executionRecorder.registerFunctions) {
84+
// const [testCaseName, functionIndex] = functionInfo;
85+
// if (matchedTestNames.includes(testCaseName)) {
86+
// executionRecorder.startTestFunction(testCaseName);
87+
// (execTestFunction as (a: number) => void)(functionIndex);
88+
// executionRecorder.finishTestFunction();
89+
// mockInstrumentFunc["mockFunctionStatus.clear"]();
90+
// }
91+
// }
92+
// }
93+
// } catch (error) {
94+
// if (error instanceof Error) {
95+
// console.error(error.stack);
96+
// }
97+
// }
8198
coverageRecorder.outputTrace(instrumentResult.traceFile);
8299
return executionRecorder.result;
83100
}
@@ -87,12 +104,12 @@ export async function execWasmBinaries(
87104
instrumentResults: InstrumentResult[],
88105
matchedTestNames?: string[],
89106
imports?: Imports
90-
): Promise<ExecutionResult> {
91-
const assertRes = new ExecutionResult();
107+
): Promise<ExecutionResultSummary> {
108+
const assertRes = new ExecutionResultSummary();
92109
ensureDirSync(outFolder);
93110
await Promise.all<void>(
94111
instrumentResults.map(async (instrumentResult): Promise<void> => {
95-
const result: SingleExecutionResult = await nodeExecutor(instrumentResult, outFolder, matchedTestNames, imports);
112+
const result: ExecutionResult = await nodeExecutor(instrumentResult, outFolder, matchedTestNames, imports);
96113
await assertRes.merge(result, instrumentResult.expectInfo);
97114
})
98115
);

‎src/core/executionRecorder.ts

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ import {
33
ImportsArgument,
44
AssertFailMessage,
55
AssertMessage,
6-
IAssertResult,
6+
IExecutionResult,
77
FailedLogMessages,
8+
CrashInfo,
89
} from "../interface.js";
10+
import { ExecutionError } from "../utils/errorTraceHandler.js";
911

1012
class LogRecorder {
1113
#currentTestLogMessages: string[] = [];
@@ -18,7 +20,7 @@ class LogRecorder {
1820
this.#isTestFailed = true;
1921
}
2022

21-
onStartTest(): void {
23+
reset(): void {
2224
this.#currentTestLogMessages = [];
2325
this.#isTestFailed = false;
2426
}
@@ -33,16 +35,16 @@ class LogRecorder {
3335
}
3436
}
3537

36-
export class SingleExecutionResult implements IAssertResult {
38+
export class ExecutionResult implements IExecutionResult {
3739
total: number = 0;
3840
fail: number = 0;
3941
failedInfo: AssertFailMessage = {};
42+
crashInfo: CrashInfo = new Set();
4043
failedLogMessages: FailedLogMessages = {};
4144
}
4245

43-
// to do: split execution environment and recorder
4446
export class ExecutionRecorder implements UnitTestFramework {
45-
result = new SingleExecutionResult();
47+
result = new ExecutionResult();
4648

4749
registerFunctions: [string, number][] = [];
4850
#currentTestDescriptions: string[] = [];
@@ -63,9 +65,10 @@ export class ExecutionRecorder implements UnitTestFramework {
6365
const testCaseFullName = this.#currentTestDescriptions.join(" ");
6466
this.registerFunctions.push([testCaseFullName, fncIndex]);
6567
}
68+
6669
startTestFunction(testCaseFullName: string): void {
6770
this.#testCaseFullName = testCaseFullName;
68-
this.logRecorder.onStartTest();
71+
this.logRecorder.reset();
6972
}
7073
finishTestFunction(): void {
7174
const logMessages: string[] | null = this.logRecorder.onFinishTest();
@@ -76,11 +79,22 @@ export class ExecutionRecorder implements UnitTestFramework {
7679
}
7780
}
7881

82+
notifyTestCrash(error: ExecutionError): void {
83+
this.logRecorder.addLog(`Reason: ${error.message}`);
84+
this.logRecorder.addLog(
85+
error.stacks
86+
.map((stack) => ` at ${stack.functionName} (${stack.fileName}:${stack.lineNumber}:${stack.columnNumber})`)
87+
.join("\n")
88+
);
89+
this.result.crashInfo.add(this.#testCaseFullName);
90+
this.result.total++; // fake test cases
91+
this.#increaseFailureCount();
92+
}
93+
7994
collectCheckResult(result: boolean, codeInfoIndex: number, actualValue: string, expectValue: string): void {
8095
this.result.total++;
8196
if (!result) {
82-
this.logRecorder.markTestFailed();
83-
this.result.fail++;
97+
this.#increaseFailureCount();
8498
const testCaseFullName = this.#testCaseFullName;
8599
const assertMessage: AssertMessage = [codeInfoIndex.toString(), actualValue, expectValue];
86100
this.result.failedInfo[testCaseFullName] = this.result.failedInfo[testCaseFullName] || [];
@@ -115,4 +129,9 @@ export class ExecutionRecorder implements UnitTestFramework {
115129
},
116130
};
117131
}
132+
133+
#increaseFailureCount(): void {
134+
this.result.fail++;
135+
this.logRecorder.markTestFailed();
136+
}
118137
}

‎src/executionResult.ts

Lines changed: 67 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,81 @@
11
import { promises } from "node:fs";
22
import { json2map } from "./utils/index.js";
3-
import { FailedInfoMap, AssertMessage, ExpectInfo, IAssertResult } from "./interface.js";
3+
import {
4+
FailedInfoMap,
5+
AssertMessage,
6+
ExpectInfo,
7+
IExecutionResult,
8+
AssertFailMessage,
9+
TestCaseName,
10+
FailedInfo,
11+
FailedLogMessages,
12+
} from "./interface.js";
413
import chalk from "chalk";
514

615
const readFile = promises.readFile;
716

8-
export class ExecutionResult {
17+
export class ExecutionResultSummary {
918
fail = 0;
1019
total = 0;
1120
failedInfos: FailedInfoMap = new Map();
1221

13-
async merge(result: IAssertResult, expectInfoFilePath: string) {
22+
#prepareFailedInfos(testcaseName: TestCaseName): FailedInfo {
23+
if (this.failedInfos.has(testcaseName)) {
24+
return this.failedInfos.get(testcaseName)!;
25+
}
26+
const failedInfo: FailedInfo = {
27+
hasCrash: false,
28+
assertMessages: [],
29+
logMessages: [],
30+
};
31+
this.failedInfos.set(testcaseName, failedInfo);
32+
return failedInfo;
33+
}
34+
35+
#processAssertInfo(failedInfo: AssertFailMessage, expectInfo: ExpectInfo) {
36+
for (const [testcaseName, value] of json2map<AssertMessage[]>(failedInfo)) {
37+
const errorMsgs: string[] = [];
38+
for (const msg of value) {
39+
const [index, actualValue, expectValue] = msg;
40+
const debugLocation = expectInfo[index];
41+
let errorMsg = `${debugLocation ?? ""} value: ${actualValue} expect: ${expectValue}`;
42+
if (errorMsg.length > 160) {
43+
errorMsg = `${debugLocation ?? ""}\nvalue: \n ${actualValue}\nexpect: \n ${expectValue}`;
44+
}
45+
errorMsgs.push(errorMsg);
46+
}
47+
this.#prepareFailedInfos(testcaseName).assertMessages =
48+
this.#prepareFailedInfos(testcaseName).assertMessages.concat(errorMsgs);
49+
}
50+
}
51+
52+
#processCrashInfo(crashInfo: Set<TestCaseName>) {
53+
for (const testcaseName of crashInfo) {
54+
this.#prepareFailedInfos(testcaseName).hasCrash = true;
55+
}
56+
}
57+
58+
/**
59+
* It should be called after other error processed to append log messages.
60+
*/
61+
#processLogMessages(failedLogMessages: FailedLogMessages) {
62+
for (let [testcaseName, failedInfo] of this.failedInfos) {
63+
if (failedLogMessages[testcaseName] !== undefined) {
64+
failedInfo.logMessages = failedInfo.logMessages.concat(failedLogMessages[testcaseName]);
65+
}
66+
}
67+
}
68+
69+
async merge(result: IExecutionResult, expectInfoFilePath: string) {
1470
this.fail += result.fail;
1571
this.total += result.total;
1672
if (result.fail > 0) {
17-
let expectInfo;
1873
try {
1974
const expectContent = await readFile(expectInfoFilePath, { encoding: "utf8" });
20-
expectInfo = json2map(JSON.parse(expectContent) as ExpectInfo);
21-
for (const [testcaseName, value] of json2map<AssertMessage[]>(result.failedInfo)) {
22-
const errorMsgs: string[] = [];
23-
for (const msg of value) {
24-
const [index, actualValue, expectValue] = msg;
25-
const debugLocation = expectInfo.get(index);
26-
let errorMsg = `${debugLocation ?? ""} value: ${actualValue} expect: ${expectValue}`;
27-
if (errorMsg.length > 160) {
28-
errorMsg = `${debugLocation ?? ""}\nvalue: \n ${actualValue}\nexpect: \n ${expectValue}`;
29-
}
30-
errorMsgs.push(errorMsg);
31-
}
32-
this.failedInfos.set(testcaseName, {
33-
assertMessages: errorMsgs,
34-
logMessages: result.failedLogMessages[testcaseName],
35-
});
36-
}
75+
const expectInfo = JSON.parse(expectContent) as ExpectInfo;
76+
this.#processAssertInfo(result.failedInfo, expectInfo);
77+
this.#processCrashInfo(result.crashInfo);
78+
this.#processLogMessages(result.failedLogMessages);
3779
} catch (error) {
3880
if (error instanceof Error) {
3981
console.error(error.stack);
@@ -51,11 +93,14 @@ export class ExecutionResult {
5193
log(`\ntest case: ${rate} (success/total)\n`);
5294
if (this.fail !== 0) {
5395
log(chalk.red("Error Message: "));
54-
for (const [testcaseName, { assertMessages, logMessages }] of this.failedInfos.entries()) {
96+
for (const [testcaseName, { hasCrash, assertMessages, logMessages }] of this.failedInfos.entries()) {
5597
log(` ${testcaseName}: `);
5698
for (const assertMessage of assertMessages) {
5799
log(" " + chalk.yellow(assertMessage));
58100
}
101+
if (hasCrash) {
102+
log(" " + chalk.red("Test case crashed!"));
103+
}
59104
for (const logMessage of logMessages ?? []) {
60105
log(chalk.gray(logMessage));
61106
}

0 commit comments

Comments
 (0)