feat: support --onlyFailures (#53)

Co-authored-by: Congcong Cai <[email protected]>

Author: XMadrid

bin/as-test.js

@@ -14,7 +14,7 @@ if (lt(version, "12.16.0")) {
   exit(-1);
 }
 const argv = [];
-argv.push("--experimental-wasi-unstable-preview1");
+argv.push("--no-warnings");
 if (lt(version, "15.0.0")) {
   argv.push("--experimental-wasm-bigint");
 }

bin/cli.js

@@ -18,7 +18,8 @@ program
   .option("--coverageLimit [error warning...]", "set warn(yellow) and error(red) upper limit in coverage report")
   .option("--testcase <testcases...>", "run only specified test cases")
   .option("--testNamePattern <test name pattern>", "run only tests with a name that matches the regex pattern")
-  .option("--collectCoverage <boolean>", "whether to collect coverage information and report");
+  .option("--collectCoverage <boolean>", "whether to collect coverage information and report")
+  .option("--onlyFailures", "Run tests that failed in the previous");
 
 program.parse(process.argv);
 const options = program.opts();
@@ -39,16 +40,21 @@ if (includes === undefined) {
 const excludes = config.exclude || [];
 validatArgument(includes, excludes);
 
-// if enabled testcase or testNamePattern, disable collectCoverage by default
+const onlyFailures = options.onlyFailures || false;
+
+// if enabled testcase or testNamePattern or onlyFailures, disable collectCoverage by default
 const collectCoverage =
-  Boolean(options.collectCoverage) || config.collectCoverage || (!options.testcase && !options.testNamePattern);
+  Boolean(options.collectCoverage) ||
+  config.collectCoverage ||
+  (!options.testcase && !options.testNamePattern && !onlyFailures);
 
 const testOption = {
   includes,
   excludes,
   testcases: options.testcase,
   testNamePattern: options.testNamePattern,
   collectCoverage,
+  onlyFailures,
 
   flags: config.flags || "",
   imports: config.imports || undefined,

docs/api-documents/options.md

@@ -27,11 +27,12 @@ There are command line options which can override the configuration in `as-test.
 ```
   --testcase <testcases...>              only run specified test cases
   --testNamePattern <test name pattern>  run only tests with a name that matches the regex pattern
+  --onlyFailures                         Run tests that failed in the previous
 ```
 
 There are several ways to run partial test cases:
 
-#### Partial Test Files
+#### Run specified test files
 
 Providing file path to `--testcase`, it can specify a certain group of files for testing.
 
@@ -55,7 +56,7 @@ run `as-test --testcase a.test.ts b.test.ts` will match all tests in `a.test.ts`
 
 :::
 
-#### Partial Tests
+#### Run partial tests using a regex name pattern
 
 Providing regex which can match targeted test name to `--testNamePattern`, it can specify a certain group of tests for testing.
 
@@ -94,6 +95,10 @@ The framework join `DescriptionName` and `TestName` with `" "` by default, e.g.
 
 :::
 
+#### Run only failures
+
+Provides `--onlyFailures` command line option to run the test cases that failed in the previous test only.
+
 ### Whether collect coverage information
 
 ```

package-lock.json

@@ -29,7 +29,6 @@
   },
   "dependencies": {
     "@assemblyscript/loader": ">=0.25.1",
-    "@assemblyscript/wasi-shim": "^0.1.0",
     "chalk": "^5.2.0",
     "commander": "^8.3.0",
     "cross-spawn": "^7.0.3",

package.json

@@ -16,7 +16,7 @@ const readFile = promises.readFile;
 async function nodeExecutor(
   instrumentResult: InstrumentResult,
   outFolder: string,
-  matchedTestNames?: string[],
+  matchedTestNames: string[],
   imports?: Imports
 ): Promise<SingleExecutionResult> {
   const wasi = new WASI({
@@ -52,7 +52,7 @@ async function nodeExecutor(
     wasi.start(ins);
     const execTestFunction = ins.exports["executeTestFunction"];
     assert(typeof execTestFunction === "function");
-    if (matchedTestNames === undefined) {
+    if (matchedTestNames.length === 0) {
       // By default, all testcases are executed
       for (const functionInfo of executionRecorder.registerFunctions) {
         const [testCaseName, functionIndex] = functionInfo;
@@ -85,7 +85,7 @@ async function nodeExecutor(
 export async function execWasmBinaries(
   outFolder: string,
   instrumentResults: InstrumentResult[],
-  matchedTestNames?: string[],
+  matchedTestNames: string[],
   imports?: Imports
 ): Promise<ExecutionResult> {
   const assertRes = new ExecutionResult();

src/core/execute.ts

@@ -10,47 +10,59 @@ import { SourceFunctionInfo, UnittestPackage } from "../interface.js";
 import { projectRoot } from "../utils/projectRoot.js";
 import assert from "node:assert";
 
+// eslint-disable-next-line sonarjs/cognitive-complexity
 export async function precompile(
   includes: string[],
   excludes: string[],
-  testcases: string[] | undefined,
+  testcases: string[] | undefined, // this field specifed test file names
   testNamePattern: string | undefined,
+  failedTestNames: string[],
   collectCoverage: boolean,
   flags: string
 ): Promise<UnittestPackage> {
   // if specify testcases, use testcases for unittest
   // otherwise, get testcases(*.test.ts) in includes directory
   const testCodePaths = testcases ?? getRelatedFiles(includes, excludes, (path: string) => path.endsWith(".test.ts"));
+  const matchedTestFiles = new Set<string>();
+  let matchedTestNames: string[] = [];
 
-  if (testNamePattern) {
-    const matchedTestNames: string[] = [];
-    const matchedTestFiles = new Set<string>();
+  if (testNamePattern || failedTestNames.length > 0) {
+    // if enabled testNamePattern or enabled onlyFailures, need listTestName transform
     const testNameInfos = new Map<string, string[]>();
     const testNameTransformFunction = join(projectRoot, "transform", "listTestNames.mjs");
     for (const testCodePath of testCodePaths) {
       await transform(testNameTransformFunction, testCodePath, flags, () => {
         testNameInfos.set(testCodePath, testNames);
       });
     }
-    const regexPattern = new RegExp(testNamePattern);
-    for (const [fileName, testNames] of testNameInfos) {
-      for (const testName of testNames) {
-        if (regexPattern.test(testName)) {
-          matchedTestNames.push(testName);
-          matchedTestFiles.add(fileName);
+    if (testNamePattern) {
+      const regexPattern = new RegExp(testNamePattern);
+      for (const [fileName, testNames] of testNameInfos) {
+        for (const testName of testNames) {
+          if (regexPattern.test(testName)) {
+            matchedTestNames.push(testName);
+            matchedTestFiles.add(fileName);
+          }
         }
       }
     }
 
-    assert(matchedTestFiles.size > 0, `No matched testname using ${testNamePattern}`);
-    return {
-      testCodePaths: Array.from(matchedTestFiles),
-      matchedTestNames: matchedTestNames,
-    };
+    if (failedTestNames.length > 0) {
+      matchedTestNames = failedTestNames;
+      for (const [fileName, testNames] of testNameInfos) {
+        for (const testName of testNames) {
+          if (matchedTestNames.includes(testName)) {
+            matchedTestFiles.add(fileName);
+          }
+        }
+      }
+    }
+
+    assert(matchedTestFiles.size > 0, "No matched testname");
   }
 
+  const sourceFunctions = new Map<string, SourceFunctionInfo[]>();
   if (collectCoverage) {
-    const sourceFunctions = new Map<string, SourceFunctionInfo[]>();
     const sourceCodePaths = getRelatedFiles(includes, excludes, (path: string) => !path.endsWith(".test.ts"));
     const sourceTransformFunction = join(projectRoot, "transform", "listFunctions.mjs");
     // The batchSize = 2 is empirical data after benchmarking
@@ -64,13 +76,13 @@ export async function precompile(
         )
       );
     }
-    return {
-      testCodePaths,
-      sourceFunctions,
-    };
   }
 
-  return { testCodePaths };
+  return {
+    testCodePaths: matchedTestFiles.size > 0 ? Array.from(matchedTestFiles) : testCodePaths,
+    matchedTestNames,
+    sourceFunctions,
+  };
 }
 
 async function transform(transformFunction: string, codePath: string, flags: string, collectCallback: () => void) {

src/core/precompile.ts

@@ -3,7 +3,7 @@ import { json2map } from "./utils/index.js";
 import { FailedInfoMap, AssertMessage, ExpectInfo, IAssertResult } from "./interface.js";
 import chalk from "chalk";
 
-const readFile = promises.readFile;
+const { readFile, writeFile } = promises;
 
 export class ExecutionResult {
   fail = 0;
@@ -43,6 +43,10 @@ export class ExecutionResult {
     }
   }
 
+  async writeFailures(failuresPath: string) {
+    await writeFile(failuresPath, JSON.stringify(Array.from(this.failedInfos.keys())));
+  }
+
   print(log: (msg: string) => void): void {
     const rate =
       (this.fail === 0 ? chalk.greenBright(this.total) : chalk.redBright(this.total - this.fail)) +

src/executionResult.ts

@@ -1,12 +1,15 @@
 import chalk from "chalk";
-import { emptydirSync } from "fs-extra";
+import pkg from "fs-extra";
 import { Parser } from "./parser/index.js";
 import { compile } from "./core/compile.js";
 import { precompile } from "./core/precompile.js";
 import { instrument } from "./core/instrument.js";
 import { execWasmBinaries } from "./core/execute.js";
 import { generateReport, reportConfig } from "./generator/index.js";
 import { TestOption } from "./interface.js";
+import { join } from "node:path";
+
+const { readFileSync, emptydirSync } = pkg;
 
 export function validatArgument(includes: unknown, excludes: unknown) {
   if (!Array.isArray(includes)) {
@@ -31,13 +34,28 @@ export function validatArgument(includes: unknown, excludes: unknown) {
  * main function of unit-test, will throw Exception in most condition except job carsh
  */
 export async function start_unit_test(options: TestOption): Promise<boolean> {
+  const failurePath = join(options.outputFolder, "failures.json");
+  let failedTestCases: string[] = [];
+  if (options.onlyFailures) {
+    failedTestCases = JSON.parse(readFileSync(failurePath, "utf8")) as string[];
+    if (failedTestCases.length === 0) {
+      options.collectCoverage = true;
+      console.log(
+        chalk.yellowBright(
+          'Warning: no failed test cases found while enabled "onlyFailures", execute all test cases by default'
+        )
+      );
+    }
+  }
+
   emptydirSync(options.outputFolder);
   emptydirSync(options.tempFolder);
   const unittestPackage = await precompile(
     options.includes,
     options.excludes,
     options.testcases,
     options.testNamePattern,
+    failedTestCases,
     options.collectCoverage,
     options.flags
   );
@@ -58,6 +76,7 @@ export async function start_unit_test(options: TestOption): Promise<boolean> {
   );
   console.log(chalk.blueBright("execute testcases: ") + chalk.bold.greenBright("OK"));
 
+  await executedResult.writeFailures(failurePath);
   executedResult.print(console.log);
   if (options.collectCoverage) {
     const parser = new Parser();

src/index.ts

@@ -168,7 +168,7 @@ export class CodeCoverage {
 
 export interface UnittestPackage {
   readonly testCodePaths: string[];
-  readonly matchedTestNames?: string[];
+  readonly matchedTestNames: string[];
   readonly sourceFunctions?: Map<string, SourceFunctionInfo[]>;
 }
 
@@ -197,6 +197,7 @@ export interface TestOption {
   testcases?: string[];
   testNamePattern?: string;
   collectCoverage: boolean;
+  onlyFailures: boolean;
 
   flags: string;
   imports?: Imports;