diff --git a/.gitignore b/.gitignore index a0239ff..441b859 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode +.cache /node_modules /dist diff --git a/a.ts b/a.ts deleted file mode 100644 index e69de29..0000000 diff --git a/assembly/env.ts b/assembly/env.ts index dc578ea..f2a2e2a 100644 --- a/assembly/env.ts +++ b/assembly/env.ts @@ -12,10 +12,6 @@ export namespace assertResult { export declare function registerTestFunction(index: u32): void; - @external("__unittest_framework_env","finishTestFunction") - export declare function finishTestFunction(): void; - - @external("__unittest_framework_env","collectCheckResult") export declare function collectCheckResult( result: bool, diff --git a/assembly/implement.ts b/assembly/implement.ts index 2c68465..adab134 100644 --- a/assembly/implement.ts +++ b/assembly/implement.ts @@ -9,13 +9,10 @@ export function describeImpl( testsFunction(); assertResult.removeDescription(); } -export function testImpl(description: string, testFunction: () => void): void { - assertResult.addDescription(description); +export function testImpl(name: string, testFunction: () => void): void { + assertResult.addDescription(name); assertResult.registerTestFunction(testFunction.index); - testFunction(); - assertResult.finishTestFunction(); assertResult.removeDescription(); - mockFunctionStatus.clear(); } export function mockImpl( diff --git a/assembly/index.ts b/assembly/index.ts index 14d5ad0..c1696ae 100644 --- a/assembly/index.ts +++ b/assembly/index.ts @@ -20,11 +20,11 @@ export function describe(description: string, testsFunction: () => void): void { /** * run a test - * @param description test description + * @param name test name * @param testFunction main function of test */ -export function test(description: string, testFunction: () => void): void { - testImpl(description, testFunction); +export function test(name: string, testFunction: () => void): void { + testImpl(name, testFunction); } /** diff --git a/bin/cli.js b/bin/cli.js index fcfbbab..0fdbd3f 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -12,21 +12,17 @@ import { validatArgument, start_unit_test } from "../dist/index.js"; const program = new Command(); program .option("--config ", "path of config file", "as-test.config.js") - .option("--testcase ", "only run specified test cases") .option("--temp ", "test template file folder") .option("--output ", "coverage report output folder") - .option("--mode ", "test result output format") + .option("--mode ", "coverage report output format") .option("--coverageLimit [error warning...]", "set warn(yellow) and error(red) upper limit in coverage report") - .option("--testNamePattern ", "run only tests with a name that matches the regex pattern"); + .option("--testcase ", "run only specified test cases") + .option("--testNamePattern ", "run only tests with a name that matches the regex pattern") + .option("--collectCoverage ", "whether to collect coverage information and report"); program.parse(process.argv); const options = program.opts(); -if (options.config === undefined) { - console.error(chalk.redBright("Miss config file") + "\n"); - console.error(program.helpInformation()); - exit(-1); -} const configPath = resolve(".", options.config); if (!fs.pathExistsSync(configPath)) { console.error(chalk.redBright("Miss config file") + "\n"); @@ -35,33 +31,36 @@ if (!fs.pathExistsSync(configPath)) { } const config = (await import(pathToFileURL(configPath))).default; -let includes = config.include; +const includes = config.include; if (includes === undefined) { console.error(chalk.redBright("Miss include in config file") + "\n"); exit(-1); } -let excludes = config.exclude || []; -let testcases = options.testcase; - -let flags = config.flags || ""; -let imports = config.imports || null; +const excludes = config.exclude || []; +validatArgument(includes, excludes); -let mode = options.mode || config.mode || "table"; +// if enabled testcase or testNamePattern, disable collectCoverage by default +const collectCoverage = + Boolean(options.collectCoverage) || config.collectCoverage || (!options.testcase && !options.testNamePattern); -let tempFolder = options.temp || config.temp || "coverage"; -let outputFolder = options.output || config.output || "coverage"; +const testOption = { + includes, + excludes, + testcases: options.testcase, + testNamePattern: options.testNamePattern, + collectCoverage, -let errorLimit = options.coverageLimit?.at(0); -let warnLimit = options.coverageLimit?.at(1); + flags: config.flags || "", + imports: config.imports || undefined, -let testNamePattern = options.testNamePattern; + tempFolder: options.temp || config.temp || "coverage", + outputFolder: options.output || config.output || "coverage", + mode: options.mode || config.mode || "table", + warnLimit: Number(options.coverageLimit?.at(1)), + errorLimit: Number(options.coverageLimit?.at(0)), +}; -validatArgument(includes, excludes); -start_unit_test( - { includes, excludes, testcases, testNamePattern }, - { flags, imports }, - { tempFolder, outputFolder, mode, warnLimit, errorLimit } -) +start_unit_test(testOption) .then((success) => { if (!success) { console.error(chalk.redBright("Test Failed") + "\n"); diff --git a/docs/api-documents.md b/docs/api-documents.md index 8490474..e46877d 100644 --- a/docs/api-documents.md +++ b/docs/api-documents.md @@ -2,4 +2,4 @@ A comprehensive AssemblyScript testing solution, offering developers a robust su - Function Mocking - Coverage statistics -- Expectations +- Matchers diff --git a/docs/api-documents/mock-function.md b/docs/api-documents/mock-function.md index 87f3862..f84034a 100644 --- a/docs/api-documents/mock-function.md +++ b/docs/api-documents/mock-function.md @@ -31,7 +31,6 @@ test("getTime error handle", () => { expect(getTime()).equal(false); // success expect(fn.calls).equal(1); // success }); -endTest(); ``` mock API can temporary change the behavior of function, effective scope is each test. diff --git a/docs/api-documents/options.md b/docs/api-documents/options.md index 4293fae..c038708 100644 --- a/docs/api-documents/options.md +++ b/docs/api-documents/options.md @@ -33,7 +33,7 @@ There are several ways to run partial test cases: #### Partial Test Files -Providing file path to `--testcase`. it can specify a certain group of files for testing. +Providing file path to `--testcase`, it can specify a certain group of files for testing. ::: tip `--testcase` can accept multiple file paths. @@ -57,20 +57,49 @@ run `as-test --testcase a.test.ts b.test.ts` will match all tests in `a.test.ts` #### Partial Tests -Providing regex which can match targeted test name to `--testNamePattern`. it can specify a certain group of tests for testing. +Providing regex which can match targeted test name to `--testNamePattern`, it can specify a certain group of tests for testing. ::: details ``` -- a.test.ts -|- case_1 -|- case_2 -- b.test.ts -|- case_A -- c.test.ts -|- case_4 +describe("groupA", () => { + test("case_1", () => { + ... + }); + test("case_2", () => { + ... + }); + test("case_3", () => { + ... + }); +}); + +describe("groupB", () => { + test("case_A", () => { + ... + }); + test("case_B", () => { + ... + }); + test("case_C", () => { + ... + }); +}); ``` -run `as-test --testNamePattern "case_\d"` will match `case 1`, `case 2`, `case 4` +run `as-test --testNamePattern "groupA case_\d"` will run `case_1`, `case_2`, `case_3`. + +::: tip +The framework join `DescriptionName` and `TestName` with `" "` by default, e.g. `groupA case_1` is the fullTestCaseName of `case_1`. ::: + +### Whether collect coverage information + +``` + --collectCoverage whether to collect coverage information and report +``` + +The framework collects coverage and generates reports by default, but it will be disablea while running partial test cases by `--testcase` or `--testNamePattern`. + +You can control the coverage collection manually with `--collectCoverage` option. diff --git a/docs/quick-start.md b/docs/quick-start.md index 94a9b39..1034d9c 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -19,14 +19,13 @@ export function add(a: i32, b: i32): i32 { Then, create a file named `tests/sum.test.ts`. This will contain our actual test: ```Typescript -import { test, expect, endTest } from "assemblyscript-unittest-framework/assembly"; +import { test, expect } from "assemblyscript-unittest-framework/assembly"; import { add } from "../source/sum"; test("sum", () => { expect(add(1, 2)).equal(3); expect(add(1, 1)).equal(3); }); -endTest(); // Don't forget it! ``` Create a config file in project root `as-test.config.js`: @@ -60,16 +59,29 @@ Add the following section to your `package.json` Finally, run `npm run test` and as-test will print this message: ``` -transform source/sum.ts => build/source/sum.ts.cov -transform build/source/sum.ts.cov => build/source/sum.ts -transform tests/sum.test.ts => build/tests/sum.test.ts -(node:489815) ExperimentalWarning: WASI is an experimental feature. This feature could change at any time +> as-test@1.0.0 test +> as-test + +(node:144985) ExperimentalWarning: WASI is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) +code analysis: OK +compile testcases: OK +instrument: OK +execute testcases: OK test case: 1/2 (success/total) Error Message: - sum: - tests/sum.test.ts:6:3 (6:3, 6:29) + sum: + tests/sum.test.ts:6:2 value: 2 expect: = 3 +---------|---------|----------|---------|-------- +File | % Stmts | % Branch | % Funcs | % Lines +---------|---------|----------|---------|-------- +source | 100 | 100 | 100 | 100 + sum.ts | 100 | 100 | 100 | 100 +---------|---------|----------|---------|-------- + +Test Failed ``` You can also use `npx as-test -h` for more information to control detail configurations diff --git a/instrumentation/CoverageInstru.cpp b/instrumentation/CoverageInstru.cpp index d56d496..afb07c8 100644 --- a/instrumentation/CoverageInstru.cpp +++ b/instrumentation/CoverageInstru.cpp @@ -30,17 +30,15 @@ void CoverageInstru::innerAnalysis(BasicBlockAnalysis &basicBlockAnalysis) const InstrumentationResponse CoverageInstru::instrument() const noexcept { if (config->fileName.empty() || config->reportFunction.empty() || config->sourceMap.empty() || - config->targetName.empty() || config->expectInfoOutputFilePath.empty() || - config->debugInfoOutputFilePath.empty()) { + config->targetName.empty() || config->expectInfoOutputFilePath.empty() + ) { std::cout << *config << std::endl; return InstrumentationResponse::CONFIG_ERROR; // config error } std::filesystem::path filePath(config->fileName); std::filesystem::path targetFilePath(config->targetName); - std::filesystem::path debugInfoPath(config->debugInfoOutputFilePath); std::filesystem::path sourceMapPath(config->sourceMap); if ((!std::filesystem::exists(filePath)) || - (!std::filesystem::exists(debugInfoPath.parent_path())) || (!std::filesystem::exists(sourceMapPath)) || (!std::filesystem::exists(targetFilePath.parent_path()))) { std::cout << *config << std::endl; @@ -49,69 +47,82 @@ InstrumentationResponse CoverageInstru::instrument() const noexcept { wasm::Module module; wasm::ModuleReader reader; - + Json::StreamWriterBuilder jsonBuilder; + jsonBuilder["indentation"] = ""; reader.read(std::string(config->fileName), module, std::string(config->sourceMap)); BasicBlockAnalysis basicBlockAnalysis = BasicBlockAnalysis(); innerAnalysis(basicBlockAnalysis); - BasicBlockWalker basicBlockWalker = BasicBlockWalker(&module, basicBlockAnalysis); - basicBlockWalker.basicBlockWalk(); - const std::unordered_map &results = - basicBlockWalker.getResults(); - Json::Value json; - Json::Value debugInfoJson; - Json::Value debugFileJson; - for (auto &[function, result] : results) { - Json::Value innerJson; - innerJson["index"] = result.functionIndex; - Json::Value branchInfoArray(Json::ValueType::arrayValue); - for (const auto &branchInfo : result.branchInfo) { - Json::Value inner_array; - inner_array.append(branchInfo.first); - inner_array.append(branchInfo.second); - branchInfoArray.append(std::move(inner_array)); + + if (config->collectCoverage) { + if (config->debugInfoOutputFilePath.empty()) { + std::cout << *config << std::endl; + return InstrumentationResponse::CONFIG_ERROR; // config error + } + std::filesystem::path debugInfoPath(config->debugInfoOutputFilePath); + if ((!std::filesystem::exists(debugInfoPath.parent_path()))) { + std::cout << *config << std::endl; + return InstrumentationResponse::CONFIG_FILEPATH_ERROR; // config file path error } - innerJson["branchInfo"] = branchInfoArray; - Json::Value debugLineJson; - for (const auto &basicBlock : result.basicBlocks) { - if (basicBlock.basicBlockIndex != static_cast(-1)) { - Json::Value debugLineItemJsonArray(Json::ValueType::arrayValue); - for (const auto &debugLine : basicBlock.debugLocations) { - Json::Value debugInfo; - debugInfo.append(debugLine.fileIndex); - debugInfo.append(debugLine.lineNumber); - debugInfo.append(debugLine.columnNumber); - debugLineItemJsonArray.append(std::move(debugInfo)); + + BasicBlockWalker basicBlockWalker = BasicBlockWalker(&module, basicBlockAnalysis); + basicBlockWalker.basicBlockWalk(); + const std::unordered_map &results = + basicBlockWalker.getResults(); + Json::Value json; + Json::Value debugInfoJson; + Json::Value debugFileJson; + for (auto &[function, result] : results) { + Json::Value innerJson; + innerJson["index"] = result.functionIndex; + Json::Value branchInfoArray(Json::ValueType::arrayValue); + for (const auto &branchInfo : result.branchInfo) { + Json::Value inner_array; + inner_array.append(branchInfo.first); + inner_array.append(branchInfo.second); + branchInfoArray.append(std::move(inner_array)); + } + innerJson["branchInfo"] = branchInfoArray; + Json::Value debugLineJson; + for (const auto &basicBlock : result.basicBlocks) { + if (basicBlock.basicBlockIndex != static_cast(-1)) { + Json::Value debugLineItemJsonArray(Json::ValueType::arrayValue); + for (const auto &debugLine : basicBlock.debugLocations) { + Json::Value debugInfo; + debugInfo.append(debugLine.fileIndex); + debugInfo.append(debugLine.lineNumber); + debugInfo.append(debugLine.columnNumber); + debugLineItemJsonArray.append(std::move(debugInfo)); + } + debugLineJson[basicBlock.basicBlockIndex] = debugLineItemJsonArray; } - debugLineJson[basicBlock.basicBlockIndex] = debugLineItemJsonArray; } + innerJson["lineInfo"] = debugLineJson; + debugInfoJson[function.data()] = innerJson; } - innerJson["lineInfo"] = debugLineJson; - debugInfoJson[function.data()] = innerJson; - } - for (const std::string &debugInfoFileName : module.debugInfoFileNames) { - debugFileJson.append(debugInfoFileName); - } - json["debugInfos"] = debugInfoJson; - json["debugFiles"] = debugFileJson; - std::ofstream jsonWriteStream(config->debugInfoOutputFilePath.data(), std::ios::trunc); - Json::StreamWriterBuilder jsonBuilder; - jsonBuilder["indentation"] = ""; - std::unique_ptr jsonWriter(jsonBuilder.newStreamWriter()); - if (jsonWriter->write(json, &jsonWriteStream) != 0) { - // Hard to control IO error - // LCOV_EXCL_START - return InstrumentationResponse::DEBUG_INFO_GENERATION_ERROR; // debug info json write failed - // LCOV_EXCL_STOP - } - jsonWriteStream.close(); - if (jsonWriteStream.fail() || jsonWriteStream.bad()) { - // Hard to control IO error - // LCOV_EXCL_START - return InstrumentationResponse::DEBUG_INFO_GENERATION_ERROR; // debug info json write failed - // LCOV_EXCL_STOP + for (const std::string &debugInfoFileName : module.debugInfoFileNames) { + debugFileJson.append(debugInfoFileName); + } + json["debugInfos"] = debugInfoJson; + json["debugFiles"] = debugFileJson; + std::ofstream jsonWriteStream(config->debugInfoOutputFilePath.data(), std::ios::trunc); + + std::unique_ptr jsonWriter(jsonBuilder.newStreamWriter()); + if (jsonWriter->write(json, &jsonWriteStream) != 0) { + // Hard to control IO error + // LCOV_EXCL_START + return InstrumentationResponse::DEBUG_INFO_GENERATION_ERROR; // debug info json write failed + // LCOV_EXCL_STOP + } + jsonWriteStream.close(); + if (jsonWriteStream.fail() || jsonWriteStream.bad()) { + // Hard to control IO error + // LCOV_EXCL_START + return InstrumentationResponse::DEBUG_INFO_GENERATION_ERROR; // debug info json write failed + // LCOV_EXCL_STOP + } + CovInstrumentationWalker covWalker(&module, config->reportFunction.data(), basicBlockWalker); + covWalker.covWalk(); } - CovInstrumentationWalker covWalker(&module, config->reportFunction.data(), basicBlockWalker); - covWalker.covWalk(); MockInstrumentationWalker mockWalker(&module); mockWalker.mockWalk(); @@ -167,7 +178,7 @@ wasm_instrument(char const *const fileName, char const *const targetName, char const *const reportFunction, char const *const sourceMap, char const *const expectInfoOutputFilePath, char const *const debugInfoOutputFilePath, char const *const includes, - char const *const excludes, bool skipLib) noexcept { + char const *const excludes, bool skipLib, bool collectCoverage) noexcept { wasmInstrumentation::InstrumentationConfig config; config.fileName = fileName; @@ -179,6 +190,7 @@ wasm_instrument(char const *const fileName, char const *const targetName, config.includes = includes; config.excludes = excludes; config.skipLib = skipLib; + config.collectCoverage = collectCoverage; wasmInstrumentation::CoverageInstru instrumentor(&config); return instrumentor.instrument(); } diff --git a/instrumentation/CoverageInstru.hpp b/instrumentation/CoverageInstru.hpp index 80e8240..340b1e7 100644 --- a/instrumentation/CoverageInstru.hpp +++ b/instrumentation/CoverageInstru.hpp @@ -44,6 +44,7 @@ class InstrumentationConfig final { std::string_view excludes; ///< function exclude filter std::string_view expectInfoOutputFilePath; ///< exception info output file name bool skipLib = true; ///< if skip lib functions + bool collectCoverage = true; ///< whether collect coverage information /// ///@brief Print information of InstrumentationConfig to output stream @@ -57,7 +58,8 @@ class InstrumentationConfig final { << ", sourceMap: " << instance.sourceMap << ", reportFunction:" << instance.reportFunction << ", includes: " << instance.includes << ", excludes: " << instance.excludes << ", expectInfoOutputFilePath: " << instance.expectInfoOutputFilePath - << ", skipLib: " << std::boolalpha << instance.skipLib << std::endl; + << ", skipLib: " << std::boolalpha << instance.skipLib + << ", collectCoverage: " << std::boolalpha << instance.collectCoverage << std::endl; return out; } }; @@ -119,6 +121,6 @@ wasm_instrument(char const *const fileName, char const *const targetName, char const *const reportFunction, char const *const sourceMap, char const *const expectInfoOutputFilePath, char const *const debugInfoOutputFilePath, char const *const includes = NULL, - char const *const excludes = NULL, bool skipLib = true) noexcept; + char const *const excludes = NULL, bool skipLib = true, bool collectCoverage = true) noexcept; #endif #endif diff --git a/instrumentation/MockInstrumentationWalker.cpp b/instrumentation/MockInstrumentationWalker.cpp index cb991c2..19967be 100644 --- a/instrumentation/MockInstrumentationWalker.cpp +++ b/instrumentation/MockInstrumentationWalker.cpp @@ -1,6 +1,9 @@ #include "MockInstrumentationWalker.hpp" +#include +#include #include #include +#include #include // mock test will be tested with wasm-testing-framework project, escape this class // LCOV_EXCL_START @@ -86,6 +89,24 @@ bool MockInstrumentationWalker::mockFunctionDuplicateImportedCheck() const noexc return checkRepeat; } +void MockInstrumentationWalker::addExecuteTestFunction() noexcept { + std::vector operands{}; + if (module->tables.empty()) { + auto * table = module->addTable(wasm::Builder::makeTable(wasm::Name::fromInt(0))); + table->base = "__indirect_function_table"; + } + BinaryenExpressionRef body = moduleBuilder.makeCallIndirect( + module->tables[0]->name, + BinaryenLocalGet(module, 0, wasm::Type::i32), + operands, + wasm::Signature(wasm::Type::none, wasm::Type::none) + ); + + body->finalize(); + BinaryenAddFunction(module, "executeTestFunction", BinaryenTypeInt32(), BinaryenTypeNone(), {}, 0, body); + BinaryenAddFunctionExport(module, "executeTestFunction", "executeTestFunction"); +} + uint32_t MockInstrumentationWalker::mockWalk() noexcept { if (mockFunctionDuplicateImportedCheck()) { return 1U; // failed @@ -93,6 +114,7 @@ uint32_t MockInstrumentationWalker::mockWalk() noexcept { wasm::ModuleUtils::iterDefinedFunctions(*module, [this](wasm::Function *const func) noexcept { walkFunctionInModule(func, this->module); }); + addExecuteTestFunction(); return 0U; } } diff --git a/instrumentation/MockInstrumentationWalker.hpp b/instrumentation/MockInstrumentationWalker.hpp index f45f009..39db9b8 100644 --- a/instrumentation/MockInstrumentationWalker.hpp +++ b/instrumentation/MockInstrumentationWalker.hpp @@ -130,6 +130,11 @@ class MockInstrumentationWalker final : public wasm::PostWalker { const wasi = new WASI({ args: ["node", instrumentResult.baseName], @@ -31,7 +32,7 @@ async function nodeExecutor( const coverageRecorder = new CoverageRecorder(); const importsArg = new ImportsArgument(executionRecorder); - const userDefinedImportsObject = imports === null ? {} : imports(importsArg); + const userDefinedImportsObject = imports === undefined ? {} : imports!(importsArg); const importObject: ASImports = { wasi_snapshot_preview1: wasi.wasiImport, ...executionRecorder.getCollectionFuncSet(importsArg), @@ -49,6 +50,28 @@ async function nodeExecutor( importsArg.exports = ins.exports; try { wasi.start(ins); + const execTestFunction = ins.exports["executeTestFunction"]; + assert(typeof execTestFunction === "function"); + if (matchedTestNames === undefined) { + // By default, all testcases are executed + for (const functionInfo of executionRecorder.registerFunctions) { + const [testCaseName, functionIndex] = functionInfo; + executionRecorder.startTestFunction(testCaseName); + (execTestFunction as (a: number) => void)(functionIndex); + executionRecorder.finishTestFunction(); + mockInstrumentFunc["mockFunctionStatus.clear"](); + } + } else { + for (const functionInfo of executionRecorder.registerFunctions) { + const [testCaseName, functionIndex] = functionInfo; + if (matchedTestNames.includes(testCaseName)) { + executionRecorder.startTestFunction(testCaseName); + (execTestFunction as (a: number) => void)(functionIndex); + executionRecorder.finishTestFunction(); + mockInstrumentFunc["mockFunctionStatus.clear"](); + } + } + } } catch (error) { if (error instanceof Error) { console.error(error.stack); @@ -62,13 +85,14 @@ async function nodeExecutor( export async function execWasmBinaries( outFolder: string, instrumentResults: InstrumentResult[], - imports: Imports + matchedTestNames?: string[], + imports?: Imports ): Promise { const assertRes = new ExecutionResult(); ensureDirSync(outFolder); await Promise.all( instrumentResults.map(async (instrumentResult): Promise => { - const result: SingleExecutionResult = await nodeExecutor(instrumentResult, outFolder, imports); + const result: SingleExecutionResult = await nodeExecutor(instrumentResult, outFolder, matchedTestNames, imports); await assertRes.merge(result, instrumentResult.expectInfo); }) ); diff --git a/src/core/executionRecorder.ts b/src/core/executionRecorder.ts index 19bd9bf..1e7288f 100644 --- a/src/core/executionRecorder.ts +++ b/src/core/executionRecorder.ts @@ -1,5 +1,11 @@ -import { ImportsArgument, UnitTestFramework } from "../index.js"; -import { AssertFailMessage, AssertMessage, IAssertResult, FailedLogMessages } from "../interface.js"; +import { + UnitTestFramework, + ImportsArgument, + AssertFailMessage, + AssertMessage, + IAssertResult, + FailedLogMessages, +} from "../interface.js"; class LogRecorder { #currentTestLogMessages: string[] = []; @@ -40,10 +46,11 @@ export class ExecutionRecorder implements UnitTestFramework { registerFunctions: [string, number][] = []; #currentTestDescriptions: string[] = []; - #logRecorder = new LogRecorder(); + #testCaseFullName: string = ""; + logRecorder = new LogRecorder(); - get #currentTestDescription(): string { - return this.#currentTestDescriptions.join(" - "); + set testCaseFullName(testCaseFullName: string) { + this.#testCaseFullName = testCaseFullName; } _addDescription(description: string): void { @@ -52,26 +59,29 @@ export class ExecutionRecorder implements UnitTestFramework { _removeDescription(): void { this.#currentTestDescriptions.pop(); } - registerTestFunction(fncIndex: number): void { - this.registerFunctions.push([this.#currentTestDescription, fncIndex]); - this.#logRecorder.onStartTest(); + _registerTestFunction(fncIndex: number): void { + const testCaseFullName = this.#currentTestDescriptions.join(" "); + this.registerFunctions.push([testCaseFullName, fncIndex]); } - _finishTestFunction(): void { - const logMessages: string[] | null = this.#logRecorder.onFinishTest(); + startTestFunction(testCaseFullName: string): void { + this.#testCaseFullName = testCaseFullName; + this.logRecorder.onStartTest(); + } + finishTestFunction(): void { + const logMessages: string[] | null = this.logRecorder.onFinishTest(); if (logMessages !== null) { - const testCaseFullName = this.#currentTestDescription; - this.result.failedLogMessages[testCaseFullName] = (this.result.failedLogMessages[testCaseFullName] || []).concat( - logMessages - ); + this.result.failedLogMessages[this.#testCaseFullName] = ( + this.result.failedLogMessages[this.#testCaseFullName] || [] + ).concat(logMessages); } } collectCheckResult(result: boolean, codeInfoIndex: number, actualValue: string, expectValue: string): void { this.result.total++; if (!result) { - this.#logRecorder.markTestFailed(); + this.logRecorder.markTestFailed(); this.result.fail++; - const testCaseFullName = this.#currentTestDescription; + const testCaseFullName = this.#testCaseFullName; const assertMessage: AssertMessage = [codeInfoIndex.toString(), actualValue, expectValue]; this.result.failedInfo[testCaseFullName] = this.result.failedInfo[testCaseFullName] || []; this.result.failedInfo[testCaseFullName].push(assertMessage); @@ -79,7 +89,7 @@ export class ExecutionRecorder implements UnitTestFramework { } log(msg: string): void { - this.#logRecorder.addLog(msg); + this.logRecorder.addLog(msg); } getCollectionFuncSet(arg: ImportsArgument): Record> { @@ -92,10 +102,7 @@ export class ExecutionRecorder implements UnitTestFramework { this._removeDescription(); }, registerTestFunction: (index: number): void => { - this.registerTestFunction(index); - }, - finishTestFunction: () => { - this._finishTestFunction(); + this._registerTestFunction(index); }, collectCheckResult: (result: number, codeInfoIndex: number, actualValue: number, expectValue: number): void => { this.collectCheckResult( diff --git a/src/core/instrument.ts b/src/core/instrument.ts index c87cb4f..9bada45 100644 --- a/src/core/instrument.ts +++ b/src/core/instrument.ts @@ -1,7 +1,11 @@ import initInstrumenter from "../../build_wasm/bin/wasm-instrumentation.js"; import { InstrumentResult } from "../interface.js"; -export async function instrument(sourceWasms: string[], sourceCodePaths: string[]): Promise { +export async function instrument( + sourceWasms: string[], + sourceCodePaths: string[], + collectCoverage: boolean +): Promise { const includeRegexs = sourceCodePaths.map((path) => { return `(start:)?${path.slice(0, -3)}.*`; }); @@ -22,7 +26,18 @@ export async function instrument(sourceWasms: string[], sourceCodePaths: string[ const expectInfo = instrumenter.allocateUTF8(result.expectInfo); const include = instrumenter.allocateUTF8(includeFilter); - instrumenter._wasm_instrument(source, output, report, sourceMap, expectInfo, debugInfo, include, 0, true); + instrumenter._wasm_instrument( + source, + output, + report, + sourceMap, + expectInfo, + debugInfo, + include, + 0, + true, + collectCoverage + ); for (const ptr of [source, output, report, sourceMap, debugInfo, expectInfo, include]) { instrumenter._free(ptr); } diff --git a/src/core/precompile.ts b/src/core/precompile.ts index c4aea04..03f0eeb 100644 --- a/src/core/precompile.ts +++ b/src/core/precompile.ts @@ -8,20 +8,23 @@ import { join, relative, resolve } from "node:path"; import { getIncludeFiles } from "../utils/pathResolver.js"; import { SourceFunctionInfo, UnittestPackage } from "../interface.js"; import { projectRoot } from "../utils/projectRoot.js"; +import assert from "node:assert"; export async function precompile( includes: string[], excludes: string[], testcases: string[] | undefined, testNamePattern: string | undefined, + collectCoverage: boolean, flags: string ): Promise { // if specify testcases, use testcases for unittest // otherwise, get testcases(*.test.ts) in includes directory const testCodePaths = testcases ?? getRelatedFiles(includes, excludes, (path: string) => path.endsWith(".test.ts")); - const matchedTestNames: string[] = []; if (testNamePattern) { + const matchedTestNames: string[] = []; + const matchedTestFiles = new Set(); const testNameInfos = new Map(); const testNameTransformFunction = join(projectRoot, "transform", "listTestNames.mjs"); for (const testCodePath of testCodePaths) { @@ -30,35 +33,44 @@ export async function precompile( }); } const regexPattern = new RegExp(testNamePattern); - for (const testNames of testNameInfos.values()) { + for (const [fileName, testNames] of testNameInfos) { for (const testName of testNames) { if (regexPattern.test(testName)) { matchedTestNames.push(testName); + matchedTestFiles.add(fileName); } } } + + assert(matchedTestFiles.size > 0, `No matched testname using ${testNamePattern}`); + return { + testCodePaths: Array.from(matchedTestFiles), + matchedTestNames: matchedTestNames, + }; } - const sourceCodePaths = getRelatedFiles(includes, excludes, (path: string) => !path.endsWith(".test.ts")); - const sourceFunctions = new Map(); - const sourceTransformFunction = join(projectRoot, "transform", "listFunctions.mjs"); - // The batchSize = 2 is empirical data after benchmarking - const batchSize = 2; - for (let i = 0; i < sourceCodePaths.length; i += batchSize) { - await Promise.all( - sourceCodePaths.slice(i, i + batchSize).map((sourcePath) => - transform(sourceTransformFunction, sourcePath, flags, () => { - sourceFunctions.set(sourcePath, functionInfos); - }) - ) - ); + if (collectCoverage) { + const sourceFunctions = new Map(); + const sourceCodePaths = getRelatedFiles(includes, excludes, (path: string) => !path.endsWith(".test.ts")); + const sourceTransformFunction = join(projectRoot, "transform", "listFunctions.mjs"); + // The batchSize = 2 is empirical data after benchmarking + const batchSize = 2; + for (let i = 0; i < sourceCodePaths.length; i += batchSize) { + await Promise.all( + sourceCodePaths.slice(i, i + batchSize).map((sourcePath) => + transform(sourceTransformFunction, sourcePath, flags, () => { + sourceFunctions.set(sourcePath, functionInfos); + }) + ) + ); + } + return { + testCodePaths, + sourceFunctions, + }; } - return { - testCodePaths, - matchedTestNames, - sourceFunctions, - }; + return { testCodePaths }; } async function transform(transformFunction: string, codePath: string, flags: string, collectCallback: () => void) { diff --git a/src/generator/index.ts b/src/generator/index.ts index 29b100f..af8ad6b 100644 --- a/src/generator/index.ts +++ b/src/generator/index.ts @@ -1,5 +1,4 @@ -import { OutputMode } from "../index.js"; -import { FileCoverageResult } from "../interface.js"; +import { OutputMode, FileCoverageResult } from "../interface.js"; import { genHtml } from "./html-generator/index.js"; import { genJson } from "./json-generator/index.js"; import { genTable } from "./table-generator/index.js"; diff --git a/src/index.ts b/src/index.ts index 9da861e..19132ce 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,12 @@ import chalk from "chalk"; import { emptydirSync } from "fs-extra"; -import { ASUtil } from "@assemblyscript/loader"; import { Parser } from "./parser/index.js"; import { compile } from "./core/compile.js"; import { precompile } from "./core/precompile.js"; import { instrument } from "./core/instrument.js"; import { execWasmBinaries } from "./core/execute.js"; import { generateReport, reportConfig } from "./generator/index.js"; +import { TestOption } from "./interface.js"; export function validatArgument(includes: unknown, excludes: unknown) { if (!Array.isArray(includes)) { @@ -27,62 +27,45 @@ export function validatArgument(includes: unknown, excludes: unknown) { } } -export abstract class UnitTestFramework { - /** - * function to redirect log message to unittest framework - * @param msg: message to log - */ - abstract log(msg: string): void; -} - -export class ImportsArgument { - module: WebAssembly.Module | null = null; - instance: WebAssembly.Instance | null = null; - exports: (ASUtil & Record) | null = null; - constructor(public framework: UnitTestFramework) {} -} - -export type Imports = ((arg: ImportsArgument) => Record) | null; - -export interface FileOption { - includes: string[]; - excludes: string[]; - testcases: string[] | undefined; - testNamePattern: string | undefined; -} -export interface TestOption { - flags: string; - imports: Imports; -} -export interface OutputOption { - tempFolder: string; - outputFolder: string; - mode: OutputMode | OutputMode[]; - warnLimit?: number; - errorLimit?: number; -} -export type OutputMode = "html" | "json" | "table"; - /** * main function of unit-test, will throw Exception in most condition except job carsh */ -export async function start_unit_test(fo: FileOption, to: TestOption, oo: OutputOption): Promise { - emptydirSync(oo.outputFolder); - emptydirSync(oo.tempFolder); - const unittestPackage = await precompile(fo.includes, fo.excludes, fo.testcases, fo.testNamePattern, to.flags); +export async function start_unit_test(options: TestOption): Promise { + emptydirSync(options.outputFolder); + emptydirSync(options.tempFolder); + const unittestPackage = await precompile( + options.includes, + options.excludes, + options.testcases, + options.testNamePattern, + options.collectCoverage, + options.flags + ); console.log(chalk.blueBright("code analysis: ") + chalk.bold.greenBright("OK")); - const wasmPaths = await compile(unittestPackage.testCodePaths, oo.tempFolder, to.flags); + + const wasmPaths = await compile(unittestPackage.testCodePaths, options.tempFolder, options.flags); console.log(chalk.blueBright("compile testcases: ") + chalk.bold.greenBright("OK")); - const instrumentResult = await instrument(wasmPaths, Array.from(unittestPackage.sourceFunctions.keys())); + + const sourcePaths = unittestPackage.sourceFunctions ? Array.from(unittestPackage.sourceFunctions.keys()) : []; + const instrumentResult = await instrument(wasmPaths, sourcePaths, options.collectCoverage); console.log(chalk.blueBright("instrument: ") + chalk.bold.greenBright("OK")); - const executedResult = await execWasmBinaries(oo.tempFolder, instrumentResult, to.imports); + + const executedResult = await execWasmBinaries( + options.tempFolder, + instrumentResult, + unittestPackage.matchedTestNames, + options.imports + ); console.log(chalk.blueBright("execute testcases: ") + chalk.bold.greenBright("OK")); + executedResult.print(console.log); - const parser = new Parser(); - const fileCoverageInfo = await parser.parse(instrumentResult, unittestPackage.sourceFunctions); - reportConfig.warningLimit = oo.warnLimit ?? reportConfig.warningLimit; - reportConfig.errorLimit = oo.errorLimit ?? reportConfig.errorLimit; - generateReport(oo.mode, oo.outputFolder, fileCoverageInfo); + if (options.collectCoverage) { + const parser = new Parser(); + const fileCoverageInfo = await parser.parse(instrumentResult, unittestPackage.sourceFunctions!); + reportConfig.warningLimit = options.warnLimit || reportConfig.warningLimit; + reportConfig.errorLimit = options.errorLimit || reportConfig.errorLimit; + generateReport(options.mode, options.outputFolder, fileCoverageInfo); + } return executedResult.fail === 0; } diff --git a/src/interface.ts b/src/interface.ts index 19994c6..087e284 100644 --- a/src/interface.ts +++ b/src/interface.ts @@ -4,6 +4,7 @@ // input import { Type } from "wasmparser"; +import { ASUtil } from "@assemblyscript/loader"; // instrumented file information export class InstrumentResult { @@ -167,8 +168,8 @@ export class CodeCoverage { export interface UnittestPackage { readonly testCodePaths: string[]; - readonly matchedTestNames: string[]; - readonly sourceFunctions: Map; + readonly matchedTestNames?: string[]; + readonly sourceFunctions?: Map; } export interface SourceFunctionInfo { @@ -181,5 +182,41 @@ export interface TestNameInfo { testFilePath: string; } +export class ImportsArgument { + module: WebAssembly.Module | null = null; + instance: WebAssembly.Instance | null = null; + exports: (ASUtil & Record) | null = null; + constructor(public framework: UnitTestFramework) {} +} + +export type Imports = ((arg: ImportsArgument) => Record) | null; + +export interface TestOption { + includes: string[]; + excludes: string[]; + testcases?: string[]; + testNamePattern?: string; + collectCoverage: boolean; + + flags: string; + imports?: Imports; + + tempFolder: string; + outputFolder: string; + mode: OutputMode | OutputMode[]; + warnLimit?: number; + errorLimit?: number; +} + +export type OutputMode = "html" | "json" | "table"; + +export abstract class UnitTestFramework { + /** + * function to redirect log message to unittest framework + * @param msg: message to log + */ + abstract log(msg: string): void; +} + export const OrganizationName = "wasm-ecosystem"; export const Repository = "https://github.com/wasm-ecosystem/assemblyscript-unittest-framework"; diff --git a/tests/as/comparison.test.ts b/tests/as/comparison.test.ts index 37a4cde..5391d8f 100644 --- a/tests/as/comparison.test.ts +++ b/tests/as/comparison.test.ts @@ -1,4 +1,4 @@ -import { describe, endTest, expect, test } from "../../assembly"; +import { describe, expect, test } from "../../assembly"; describe("base type equal", () => { test("i32", () => { @@ -177,7 +177,6 @@ describe("single level container type equal", () => { }); test("nullable equal normal", () => {}); }); - // end test }); describe("mutli-level container type equal", () => { @@ -209,5 +208,3 @@ describe("mutli-level container type equal", () => { expect(arr).equal(arr2); }); }); - -endTest(); diff --git a/tests/as/expect.test.ts b/tests/as/expect.test.ts index d3c7d64..bcd375d 100644 --- a/tests/as/expect.test.ts +++ b/tests/as/expect.test.ts @@ -1,4 +1,4 @@ -import { describe, endTest, expect, test } from "../../assembly"; +import { describe, expect, test } from "../../assembly"; describe("expect", () => { test("< = >", () => { @@ -14,5 +14,3 @@ describe("expect", () => { expect("test").notNull(); }); }); - -endTest(); diff --git a/tests/as/format.test.ts b/tests/as/format.test.ts index e2eb156..0962bd0 100644 --- a/tests/as/format.test.ts +++ b/tests/as/format.test.ts @@ -1,4 +1,4 @@ -import { describe, endTest, expect, test } from "../../assembly"; +import { describe, expect, test } from "../../assembly"; import { toJson } from "../../assembly/formatPrint"; class A {} @@ -57,5 +57,3 @@ describe("print", () => { expect(toJson(new A())).equal("[Object A]"); }); }); - -endTest(); diff --git a/tests/as/mock.test.ts b/tests/as/mock.test.ts index 3fa8588..c1992c7 100644 --- a/tests/as/mock.test.ts +++ b/tests/as/mock.test.ts @@ -1,4 +1,4 @@ -import { describe, endTest, expect, mock, remock, test, unmock } from "../../assembly"; +import { describe, expect, mock, remock, test, unmock } from "../../assembly"; import { add, callee, caller, incr, MockClass, call_incr } from "./mockBaseFunc"; const mockReturnValue: i32 = 123; @@ -71,5 +71,3 @@ describe("mock test", () => { expect(call_incr(incr)).equal(180); }); }); - -endTest(); diff --git a/tests/ts/test/core/executionRecorder.test.ts b/tests/ts/test/core/executionRecorder.test.ts index 368af3e..c5a1b79 100644 --- a/tests/ts/test/core/executionRecorder.test.ts +++ b/tests/ts/test/core/executionRecorder.test.ts @@ -5,7 +5,10 @@ describe("execution recorder", () => { test("add single description", () => { const recorder = new ExecutionRecorder(); recorder._addDescription("description"); + recorder._registerTestFunction(1); + expect(recorder.registerFunctions).toEqual([["description", 1]]); + recorder.startTestFunction("description"); recorder.collectCheckResult(false, 0, "", ""); expect(recorder.result.failedInfo).toHaveProperty("description"); }); @@ -13,9 +16,12 @@ describe("execution recorder", () => { const recorder = new ExecutionRecorder(); recorder._addDescription("description1"); recorder._addDescription("description2"); + recorder._registerTestFunction(1); + expect(recorder.registerFunctions).toEqual([["description1 description2", 1]]); + recorder.startTestFunction("description1 description2"); recorder.collectCheckResult(false, 0, "", ""); - expect(recorder.result.failedInfo).toHaveProperty("description1 - description2"); + expect(recorder.result.failedInfo).toHaveProperty("description1 description2"); }); test("remove descriptions", () => { @@ -23,7 +29,10 @@ describe("execution recorder", () => { recorder._addDescription("description1"); recorder._addDescription("description2"); recorder._removeDescription(); + recorder._registerTestFunction(1); + expect(recorder.registerFunctions).toEqual([["description1", 1]]); + recorder.startTestFunction("description1"); recorder.collectCheckResult(false, 0, "", ""); expect(recorder.result.failedInfo).toHaveProperty("description1"); }); diff --git a/tests/ts/test/core/instrument.test.ts b/tests/ts/test/core/instrument.test.ts index 22f8195..4cae7fa 100644 --- a/tests/ts/test/core/instrument.test.ts +++ b/tests/ts/test/core/instrument.test.ts @@ -13,7 +13,7 @@ test("Instrument", async () => { const base = join(outputDir, "constructor"); const wasmPath = join(outputDir, "constructor.wasm"); const sourceCodePath = "tests/ts/fixture/constructor.ts"; - const results = await instrument([wasmPath], [sourceCodePath]); + const results = await instrument([wasmPath], [sourceCodePath], true); expect(results.length).toEqual(1); const result = results[0]!; const instrumentedWasm = join(outputDir, "constructor.instrumented.wasm"); diff --git a/tests/ts/test/core/precompile.test.ts b/tests/ts/test/core/precompile.test.ts index ac5e4e0..72056c7 100644 --- a/tests/ts/test/core/precompile.test.ts +++ b/tests/ts/test/core/precompile.test.ts @@ -1,7 +1,14 @@ import { precompile } from "../../../../src/core/precompile.js"; test("listFunction transform", async () => { - const unittestPackages = await precompile(["tests/ts/fixture/transformFunction.ts"], [], undefined, undefined, ""); + const unittestPackages = await precompile( + ["tests/ts/fixture/transformFunction.ts"], + [], + undefined, + undefined, + true, + "" + ); expect(unittestPackages.testCodePaths).toEqual([]); expect(unittestPackages.sourceFunctions).toMatchSnapshot(); }); diff --git a/transform/listTestNames.mts b/transform/listTestNames.mts index 5d95f03..6dd1fed 100644 --- a/transform/listTestNames.mts +++ b/transform/listTestNames.mts @@ -309,7 +309,7 @@ class SourceFunctionTransform extends Transform { const testName = (node.args[0] as StringLiteralExpression).value; this.currentTestDescriptions.push(testName); if (fncName === "test") { - this.testNames.push(this.currentTestDescriptions.join("")); + this.testNames.push(this.currentTestDescriptions.join(" ")); } this.visitNode(node.expression); for (const arg of node.args) {