diff --git a/bin/cli.js b/bin/cli.js index fb13c70..941539a 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -7,18 +7,22 @@ import { resolve } from "node:path"; import { Command } from "commander"; import { pathToFileURL } from "node:url"; -import { validatArgument, start_unit_test } from "../dist/index.js"; +import { validateArgument, start_unit_test } from "../dist/index.js"; const program = new Command(); program .option("--config ", "path of config file", "as-test.config.js") + .option("--temp ", "test template file folder") .option("--output ", "coverage report output folder") .option("--mode ", "coverage report output format") + .option("--coverageLimit [error warning...]", "set warn(yellow) and error(red) upper limit in coverage report") - .option("--testcase ", "run only specified test cases") - .option("--testNamePattern ", "run only tests with a name that matches the regex pattern") .option("--collectCoverage ", "whether to collect coverage information and report") + + .option("--testcase ", "run only specified test cases deprecated, use --testFiles instead") + .option("--testFiles ", "run only specified test files") + .option("--testNamePattern ", "run only tests with a name that matches the regex pattern") .option("--onlyFailures", "Run tests that failed in the previous"); program.parse(process.argv); @@ -38,7 +42,16 @@ if (includes === undefined) { exit(-1); } const excludes = config.exclude || []; -validatArgument(includes, excludes); +validateArgument(includes, excludes); + +if (options.testcase !== undefined) { + console.log( + chalk.yellowBright( + "Warning: --testcase is deprecated, please use --testFiles instead, --testcase will be removed in next versions" + ) + ); +} +const testFiles = options.testFiles || options.testcase; const onlyFailures = options.onlyFailures || false; @@ -46,12 +59,12 @@ const onlyFailures = options.onlyFailures || false; const collectCoverage = Boolean(options.collectCoverage) || config.collectCoverage || - (!options.testcase && !options.testNamePattern && !onlyFailures); + (testFiles !== undefined && options.testNamePattern !== undefined && !onlyFailures); const testOption = { includes, excludes, - testcases: options.testcase, + testFiles, testNamePattern: options.testNamePattern, collectCoverage, onlyFailures, @@ -74,6 +87,9 @@ start_unit_test(testOption) } }) .catch((e) => { - console.error(chalk.redBright(" Test crash, error message: ") + chalk.yellowBright(`${e.stack}`) + "\n"); + console.error(chalk.redBright("framework crash, error message: ") + chalk.yellowBright(`${e.stack}`) + "\n"); + console.error( + "please submit an issue at https://github.com/wasm-ecosystem/assemblyscript-unittest-framework/issues" + ); exit(255); }); diff --git a/docs/api-documents/options.md b/docs/api-documents/options.md index 73642f5..4d855b8 100644 --- a/docs/api-documents/options.md +++ b/docs/api-documents/options.md @@ -25,7 +25,7 @@ There are command line options which can override the configuration in `as-test. ### Run partial test cases ``` - --testcase only run specified test cases + --testFiles only run specified test cases --testNamePattern run only tests with a name that matches the regex pattern --onlyFailures Run tests that failed in the previous ``` @@ -34,10 +34,10 @@ There are several ways to run partial test cases: #### Run specified test files -Providing file path to `--testcase`, it can specify a certain group of files for testing. +Providing file path to `--testFiles`, it can specify a certain group of files for testing. ::: tip -`--testcase` can accept multiple file paths. +`--testFiles` can accept multiple file paths. ::: ::: details @@ -52,7 +52,7 @@ Providing file path to `--testcase`, it can specify a certain group of files for |- case_4 ``` -run `as-test --testcase a.test.ts b.test.ts` will match all tests in `a.test.ts` and `b.test.ts` +run `as-test --testFiles a.test.ts b.test.ts` will match all tests in `a.test.ts` and `b.test.ts` ::: @@ -91,7 +91,7 @@ describe("groupB", () => { run `as-test --testNamePattern "groupA case_\d"` will run `case_1`, `case_2`, `case_3`. ::: tip -The framework join `DescriptionName` and `TestName` with `" "` by default, e.g. `groupA case_1` is the fullTestCaseName of `case_1`. +The framework join `DescriptionName` and `TestName` with `" "` by default, e.g. `groupA case_1` is the full test case name of `case_1`. ::: @@ -105,6 +105,6 @@ Provides `--onlyFailures` command line option to run the test cases that failed --collectCoverage whether to collect coverage information and report ``` -The framework collects coverage and generates reports by default, but it will be disablea while running partial test cases by `--testcase` or `--testNamePattern`. +The framework collects coverage and generates reports by default, but it will be disablea while running partial test cases by `--testFiles` or `--testNamePattern`. You can control the coverage collection manually with `--collectCoverage` option. diff --git a/docs/quick-start.md b/docs/quick-start.md index 1034d9c..6d2856b 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -65,9 +65,9 @@ Finally, run `npm run test` and as-test will print this message: (node:144985) ExperimentalWarning: WASI is an experimental feature and might change at any time (Use `node --trace-warnings ...` to show where the warning was created) code analysis: OK -compile testcases: OK +compile test files: OK instrument: OK -execute testcases: OK +execute test files: OK test case: 1/2 (success/total) diff --git a/src/core/precompile.ts b/src/core/precompile.ts index e11e0de..f438ffd 100644 --- a/src/core/precompile.ts +++ b/src/core/precompile.ts @@ -14,15 +14,15 @@ import assert from "node:assert"; export async function precompile( includes: string[], excludes: string[], - testcases: string[] | undefined, // this field specifed test file names + testFiles: string[] | undefined, // this field specifed test file names testNamePattern: string | undefined, failedTestNames: string[], collectCoverage: boolean, flags: string ): Promise { - // if specify testcases, use testcases for unittest - // otherwise, get testcases(*.test.ts) in includes directory - const testCodePaths = testcases ?? getRelatedFiles(includes, excludes, (path: string) => path.endsWith(".test.ts")); + // if specify testFiles, use testFiles for unittest + // otherwise, get testFiles(*.test.ts) in includes directory + const testCodePaths = testFiles ?? getRelatedFiles(includes, excludes, (path: string) => path.endsWith(".test.ts")); const matchedTestFiles = new Set(); let matchedTestNames: string[] = []; diff --git a/src/executionResult.ts b/src/executionResult.ts index 7fb51eb..b98f085 100644 --- a/src/executionResult.ts +++ b/src/executionResult.ts @@ -94,8 +94,9 @@ export class ExecutionResultSummary { (this.fail === 0 ? chalk.greenBright(this.total) : chalk.redBright(this.total - this.fail)) + "/" + this.total.toString(); - log(`\ntest case: ${rate} (success/total)\n`); + log(`\ntest case: ${rate} (success/total)`); if (this.fail !== 0) { + log(""); this.#printErrorMessage(log); } } diff --git a/src/index.ts b/src/index.ts index 0dfb0af..b7e4b8d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import { join } from "node:path"; const { readFileSync, emptydirSync } = pkg; -export function validatArgument(includes: unknown, excludes: unknown) { +export function validateArgument(includes: unknown, excludes: unknown) { if (!Array.isArray(includes)) { throw new TypeError("include section illegal"); } @@ -53,7 +53,7 @@ export async function start_unit_test(options: TestOption): Promise { const unittestPackage = await precompile( options.includes, options.excludes, - options.testcases, + options.testFiles, options.testNamePattern, failedTestCases, options.collectCoverage, @@ -62,7 +62,7 @@ export async function start_unit_test(options: TestOption): Promise { console.log(chalk.blueBright("code analysis: ") + chalk.bold.greenBright("OK")); const wasmPaths = await compile(unittestPackage.testCodePaths, options.tempFolder, options.flags); - console.log(chalk.blueBright("compile testcases: ") + chalk.bold.greenBright("OK")); + console.log(chalk.blueBright("compile test files: ") + chalk.bold.greenBright("OK")); const sourcePaths = unittestPackage.sourceFunctions ? Array.from(unittestPackage.sourceFunctions.keys()) : []; const instrumentResult = await instrument(wasmPaths, sourcePaths, options.collectCoverage); @@ -74,7 +74,7 @@ export async function start_unit_test(options: TestOption): Promise { unittestPackage.matchedTestNames, options.imports ); - console.log(chalk.blueBright("execute testcases: ") + chalk.bold.greenBright("OK")); + console.log(chalk.blueBright("execute test files: ") + chalk.bold.greenBright("OK")); await executedResult.writeFailures(failurePath); executedResult.print(console.log); diff --git a/src/interface.ts b/src/interface.ts index 8bd2443..85691ee 100644 --- a/src/interface.ts +++ b/src/interface.ts @@ -200,7 +200,7 @@ export type Imports = ((arg: ImportsArgument) => Record) | null export interface TestOption { includes: string[]; excludes: string[]; - testcases?: string[]; + testFiles?: string[]; testNamePattern?: string; collectCoverage: boolean; onlyFailures: boolean; diff --git a/tests/e2e/assertFailed/stdout.txt b/tests/e2e/assertFailed/stdout.txt index a9ed498..07b3671 100644 --- a/tests/e2e/assertFailed/stdout.txt +++ b/tests/e2e/assertFailed/stdout.txt @@ -1,7 +1,7 @@ code analysis: OK -compile testcases: OK +compile test files: OK instrument: OK -execute testcases: OK +execute test files: OK test case: 0/2 (success/total) @@ -11,11 +11,11 @@ Error Message: This test will fail due to an assertion error Reason: unreachable at start:tests/e2e/assertFailed/assertOnTest.test~anonymous|0 (tests/e2e/assertFailed/assertOnTest.test.ts:6:2) - at executeTestFunction (tests/e2e/assertFailed/tmp/assertOnTest.test.instrumented.wasm:1:675) + at executeTestFunction (tests/e2e/assertFailed/tmp/assertOnTest.test.instrumented.wasm:1:637) tests/e2e/assertFailed/tmp/assertOnInit.test - init: Test Crashed! Reason: unreachable at start:tests/e2e/assertFailed/assertOnInit.test (tests/e2e/assertFailed/assertOnInit.test.ts:1:0) - at ~start (tests/e2e/assertFailed/tmp/assertOnInit.test.instrumented.wasm:1:244) + at ~start (tests/e2e/assertFailed/tmp/assertOnInit.test.instrumented.wasm:1:206) diff --git a/tests/e2e/printLogInFailedInfo/stdout.txt b/tests/e2e/printLogInFailedInfo/stdout.txt index 807520a..7b42bb4 100644 --- a/tests/e2e/printLogInFailedInfo/stdout.txt +++ b/tests/e2e/printLogInFailedInfo/stdout.txt @@ -1,7 +1,7 @@ code analysis: OK -compile testcases: OK +compile test files: OK instrument: OK -execute testcases: OK +execute test files: OK test case: 1/2 (success/total) diff --git a/tests/e2e/run.js b/tests/e2e/run.js index f37774c..426afef 100644 --- a/tests/e2e/run.js +++ b/tests/e2e/run.js @@ -6,11 +6,7 @@ import { argv } from "node:process"; import { readFileSync } from "node:fs"; function getDiff(s1, s2) { - const handleEscape = (c) => - c - .split("\n") - .map((l) => (l.length === 0 ? "\xB6" : l)) - .join("\n"); + const handleEscape = (c) => (/^\n*$/.test(c) ? c.replaceAll("\n", "\xB6\n") : c); return diffLines(s1, s2) .map((part) => { if (part.added) { @@ -32,10 +28,10 @@ function isEnabled(name) { return enabledTests.includes(name); } -function runEndToEndTest(name, handle) { +function runEndToEndTest(name, flags, handle) { if (isEnabled(name)) { console.log(`Running e2e test: ${name}`); - exec(`node ./bin/as-test.js --config tests/e2e/${name}/as-test.config.js`, (error, stdout, stderr) => { + exec(`node ./bin/as-test.js --config tests/e2e/${name}/as-test.config.js ${flags}`, (error, stdout, stderr) => { // standard check const expectStdOut = readFileSync(`tests/e2e/${name}/stdout.txt`, "utf-8"); if (expectStdOut !== stdout) { @@ -51,10 +47,18 @@ function runEndToEndTest(name, handle) { } } -runEndToEndTest("printLogInFailedInfo", (error, stdout, stderr) => { +runEndToEndTest("printLogInFailedInfo", "", (error, stdout, stderr) => { assert(error.code === 255); }); -runEndToEndTest("assertFailed", (error, stdout, stderr) => { +runEndToEndTest("assertFailed", "", (error, stdout, stderr) => { assert(error.code === 255); }); + +runEndToEndTest( + "testFiles", + "--testFiles tests/e2e/testFiles/succeed_0.test.ts tests/e2e/testFiles/succeed_1.test.ts", + (error, stdout, stderr) => { + assert(error === null); + } +); diff --git a/tests/e2e/testFiles/as-test.config.js b/tests/e2e/testFiles/as-test.config.js new file mode 100644 index 0000000..48c78b0 --- /dev/null +++ b/tests/e2e/testFiles/as-test.config.js @@ -0,0 +1,19 @@ +import path from "node:path"; + +const __dirname = path.dirname(new URL(import.meta.url).pathname); + +export default { + include: [__dirname], + imports(runtime) { + return { + env: { + log: (msg) => { + runtime.framework.log(runtime.exports.__getString(msg)); + }, + }, + }; + }, + temp: path.join(__dirname, "tmp"), + output: path.join(__dirname, "tmp"), + mode: [], +}; diff --git a/tests/e2e/testFiles/env.ts b/tests/e2e/testFiles/env.ts new file mode 100644 index 0000000..b8c4440 --- /dev/null +++ b/tests/e2e/testFiles/env.ts @@ -0,0 +1 @@ +export declare function log(msg: string): void; diff --git a/tests/e2e/testFiles/failed.test.ts b/tests/e2e/testFiles/failed.test.ts new file mode 100644 index 0000000..de4df59 --- /dev/null +++ b/tests/e2e/testFiles/failed.test.ts @@ -0,0 +1,7 @@ +import { test, expect } from "../../../assembly"; +import { log } from "./env"; + +test("failed test", () => { + log("This is a log message for the failed test."); + expect(1 + 1).equal(3); +}); diff --git a/tests/e2e/testFiles/stdout.txt b/tests/e2e/testFiles/stdout.txt new file mode 100644 index 0000000..bc6ff06 --- /dev/null +++ b/tests/e2e/testFiles/stdout.txt @@ -0,0 +1,6 @@ +code analysis: OK +compile test files: OK +instrument: OK +execute test files: OK + +test case: 2/2 (success/total) diff --git a/tests/e2e/testFiles/succeed_0.test.ts b/tests/e2e/testFiles/succeed_0.test.ts new file mode 100644 index 0000000..99f45d9 --- /dev/null +++ b/tests/e2e/testFiles/succeed_0.test.ts @@ -0,0 +1,7 @@ +import { test, expect } from "../../../assembly"; +import { log } from "./env"; + +test("succeed test", () => { + log("This is a log message for the succeed test."); + expect(1 + 1).equal(2); +}); diff --git a/tests/e2e/testFiles/succeed_1.test.ts b/tests/e2e/testFiles/succeed_1.test.ts new file mode 100644 index 0000000..99f45d9 --- /dev/null +++ b/tests/e2e/testFiles/succeed_1.test.ts @@ -0,0 +1,7 @@ +import { test, expect } from "../../../assembly"; +import { log } from "./env"; + +test("succeed test", () => { + log("This is a log message for the succeed test."); + expect(1 + 1).equal(2); +}); diff --git a/tests/e2e/testFiles/tsconfig.json b/tests/e2e/testFiles/tsconfig.json new file mode 100644 index 0000000..798b474 --- /dev/null +++ b/tests/e2e/testFiles/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "assemblyscript/std/assembly.json", + "include": ["./**/*.ts"] +} diff --git a/tests/ts/test/executionResult.test.ts b/tests/ts/test/executionResult.test.ts index 915f4e5..9a5ff77 100644 --- a/tests/ts/test/executionResult.test.ts +++ b/tests/ts/test/executionResult.test.ts @@ -7,8 +7,8 @@ import chalk from "chalk"; const __dirname = fileURLToPath(new URL(".", import.meta.url)); test("no failedInfo merge", async () => { - const executionResult = new ExecutionResultSummary(); - const testcaseA: IExecutionResult = { + const resultSummary = new ExecutionResultSummary(); + const executionResult: IExecutionResult = { fail: 0, total: 28, crashInfo: new Set(), @@ -16,17 +16,17 @@ test("no failedInfo merge", async () => { failedLogMessages: {}, }; const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); - await executionResult.merge(testcaseA, expectInfoFIlePath); - expect(executionResult.fail).toEqual(0); - expect(executionResult.total).toEqual(28); - expect(executionResult.failedInfos).toEqual(new Map()); + await resultSummary.merge(executionResult, expectInfoFIlePath); + expect(resultSummary.fail).toEqual(0); + expect(resultSummary.total).toEqual(28); + expect(resultSummary.failedInfos).toEqual(new Map()); }); test("equal assert failed", async () => { - const executionResult = new ExecutionResultSummary(); - const actualString = "A long sentence for testing errorMsg.length > 160 in executionResult.ts merge function"; - const expectString = "= A long sentence for testing errorMsg.length > 160 in executionResult.ts merge function "; - const testcaseA: IExecutionResult = { + const resultSummary = new ExecutionResultSummary(); + const actualString = "A long sentence for testing errorMsg.length > 160 in resultSummary.ts merge function"; + const expectString = "= A long sentence for testing errorMsg.length > 160 in resultSummary.ts merge function "; + const executionResult: IExecutionResult = { fail: 1, total: 28, crashInfo: new Set(), @@ -43,7 +43,7 @@ test("equal assert failed", async () => { }, }; const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); - await executionResult.merge(testcaseA, expectInfoFIlePath); + await resultSummary.merge(executionResult, expectInfoFIlePath); const expectFailedInfo: FailedInfoMap = new Map(); expectFailedInfo.set("A", { hasCrash: false, @@ -55,14 +55,14 @@ test("equal assert failed", async () => { ], logMessages: ["log message 1", "log message 2", "log message 3"], }); - expect(executionResult.fail).toEqual(1); - expect(executionResult.total).toEqual(28); - expect(executionResult.failedInfos).toEqual(expectFailedInfo); + expect(resultSummary.fail).toEqual(1); + expect(resultSummary.total).toEqual(28); + expect(resultSummary.failedInfos).toEqual(expectFailedInfo); }); test("equal crash", async () => { - const executionResult = new ExecutionResultSummary(); - const testcaseA: IExecutionResult = { + const resultSummary = new ExecutionResultSummary(); + const executionResult: IExecutionResult = { fail: 1, total: 1, crashInfo: new Set(), @@ -71,24 +71,24 @@ test("equal crash", async () => { A: ["log message 1", "log message 2", "log message 3"], }, }; - testcaseA.crashInfo.add("A"); + executionResult.crashInfo.add("A"); const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); - await executionResult.merge(testcaseA, expectInfoFIlePath); + await resultSummary.merge(executionResult, expectInfoFIlePath); const expectFailedInfo: FailedInfoMap = new Map(); expectFailedInfo.set("A", { hasCrash: true, assertMessages: [], logMessages: ["log message 1", "log message 2", "log message 3"], }); - expect(executionResult.fail).toEqual(1); - expect(executionResult.total).toEqual(1); - expect(executionResult.failedInfos).toEqual(expectFailedInfo); + expect(resultSummary.fail).toEqual(1); + expect(resultSummary.total).toEqual(1); + expect(resultSummary.failedInfos).toEqual(expectFailedInfo); }); describe("print", () => { test("assert failed", async () => { - const executionResult = new ExecutionResultSummary(); - const testcaseA: IExecutionResult = { + const resultSummary = new ExecutionResultSummary(); + const executionResult: IExecutionResult = { fail: 1, total: 28, crashInfo: new Set(), @@ -100,25 +100,25 @@ describe("print", () => { }, }; const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); - await executionResult.merge(testcaseA, expectInfoFIlePath); + await resultSummary.merge(executionResult, expectInfoFIlePath); { const outputs: string[] = []; chalk.level = 0; // disable color - executionResult.print((msg) => outputs.push(msg)); + resultSummary.print((msg) => outputs.push(msg)); expect(outputs.join("\n")).toMatchSnapshot(); } { const outputs: string[] = []; chalk.level = 1; // force enable color - executionResult.print((msg) => outputs.push(msg)); + resultSummary.print((msg) => outputs.push(msg)); expect(outputs.join("\n")).toMatchSnapshot(); } }); test("crash", async () => { - const executionResult = new ExecutionResultSummary(); - const testcaseA: IExecutionResult = { + const resultSummary = new ExecutionResultSummary(); + const executionResult: IExecutionResult = { fail: 1, total: 28, crashInfo: new Set(), @@ -127,20 +127,20 @@ describe("print", () => { A: ["log message 1", "log message 2", "log message 3"], }, }; - testcaseA.crashInfo.add("A"); + executionResult.crashInfo.add("A"); const expectInfoFIlePath = join(__dirname, "..", "fixture", "assertResultTest.expectInfo.json"); - await executionResult.merge(testcaseA, expectInfoFIlePath); + await resultSummary.merge(executionResult, expectInfoFIlePath); { const outputs: string[] = []; chalk.level = 0; // disable color - executionResult.print((msg) => outputs.push(msg)); + resultSummary.print((msg) => outputs.push(msg)); expect(outputs.join("\n")).toMatchSnapshot(); } { const outputs: string[] = []; chalk.level = 1; // force enable color - executionResult.print((msg) => outputs.push(msg)); + resultSummary.print((msg) => outputs.push(msg)); expect(outputs.join("\n")).toMatchSnapshot(); } });