fix: proper handling of situations where AS files are not invalid

bin/cli.js

@@ -32,14 +32,14 @@ const configPath = resolve(".", options.config);
 if (!fs.pathExistsSync(configPath)) {
   console.error(chalk.redBright("Miss config file") + "\n");
   console.error(program.helpInformation());
-  exit(-1);
+  exit(3);
 }
 const config = (await import(pathToFileURL(configPath))).default;
 
 const includes = config.include;
 if (includes === undefined) {
   console.error(chalk.redBright("Miss include in config file") + "\n");
-  exit(-1);
+  exit(3);
 }
 const excludes = config.exclude || [];
 validateArgument(includes, excludes);
@@ -80,10 +80,10 @@ const testOption = {
 };
 
 start_unit_test(testOption)
-  .then((success) => {
-    if (!success) {
+  .then((returnCode) => {
+    if (returnCode !== 0) {
       console.error(chalk.redBright("Test Failed") + "\n");
-      exit(255);
+      exit(returnCode);
     }
   })
   .catch((e) => {

docs/.vitepress/config.ts

@@ -34,6 +34,7 @@ export default defineConfig({
               { text: "Matchers", link: "/api-documents/matchers" },
               { text: "Mock Function", link: "/api-documents/mock-function" },
               { text: "Report", link: "/api-documents/coverage-report" },
+              { text: "Return Code", link: "/api-documents/return-code.md" },
             ],
           },
           {

docs/api-documents/return-code.md

@@ -0,0 +1,7 @@
+## Return Code
+
+The platform indicates different error situations by returning different exit codes.
+
+- exit code 1 indicates a test failure.
+- exit code 2 indicates that the input AS file cannot be compiled.
+- exit code 3 or higher indicates an error in the configuration file.

docs/release-note.md

@@ -1,5 +1,19 @@
 # Release Note
 
+## 1.3.1
+
+🚀 Highlight Features
+
+- Defined the meanings of the return values in different situations.
+  - `0` means success.
+  - `1` means test failed.
+  - `2` means invalid AS file.
+  - `>2` means configuration error.
+
+🚀 Improvements
+
+- Proper handling of situations where AS files are not invalid.
+
 ## 1.3.0
 
 🚀 Highlight Features
@@ -13,7 +27,7 @@
   - Expose the framework's `log` function in the configuration file, and the logs redirected to this function will be appended to the final test report.
   - Support test crashes and provide good call stack information.
 
-Improvements
+🚀 Improvements
 
 - Code coverage calculation.
   - Skip type definitions.

src/core/compile.ts

@@ -1,6 +1,6 @@
-import { main } from "assemblyscript/asc";
 import { join, relative } from "node:path";
 import { findRoot } from "../utils/pathResolver.js";
+import { ascMain } from "../utils/ascWrapper.js";
 
 export async function compile(testCodePaths: string[], outputFolder: string, compileFlags: string): Promise<string[]> {
   const wasm: string[] = [];
@@ -25,12 +25,7 @@ export async function compile(testCodePaths: string[], outputFolder: string, com
       const argv = compileFlags.split(" ");
       ascArgv = ascArgv.concat(argv);
     }
-    const { error, stderr } = await main(ascArgv);
-    if (error) {
-      // eslint-disable-next-line @typescript-eslint/no-base-to-string
-      console.error(stderr.toString());
-      throw error;
-    }
+    await ascMain(ascArgv);
   };
 
   // Here, for-await is more efficient and less memory cost than Promise.all()

src/core/precompile.ts

@@ -3,12 +3,12 @@
  */
 
 import ignore from "ignore";
-import { main } from "assemblyscript/asc";
 import { join, relative, resolve } from "node:path";
 import { getIncludeFiles } from "../utils/pathResolver.js";
 import { SourceFunctionInfo, UnittestPackage } from "../interface.js";
 import { projectRoot } from "../utils/projectRoot.js";
 import assert from "node:assert";
+import { ascMain } from "../utils/ascWrapper.js";
 
 // eslint-disable-next-line sonarjs/cognitive-complexity
 export async function precompile(
@@ -91,12 +91,7 @@ async function transform(transformFunction: string, codePath: string, flags: str
     const argv = flags.split(" ");
     ascArgv = ascArgv.concat(argv);
   }
-  const { error, stderr } = await main(ascArgv);
-  if (error) {
-    // eslint-disable-next-line @typescript-eslint/no-base-to-string
-    console.error(stderr.toString());
-    throw error;
-  }
+  await ascMain(ascArgv);
   collectCallback();
 }
 

src/index.ts

@@ -8,6 +8,7 @@ import { execWasmBinaries } from "./core/execute.js";
 import { generateReport, reportConfig } from "./generator/index.js";
 import { TestOption } from "./interface.js";
 import { join } from "node:path";
+import { CompilationError } from "./utils/ascWrapper.js";
 
 const { readFileSync, emptydirSync } = pkg;
 
@@ -30,10 +31,7 @@ export function validateArgument(includes: unknown, excludes: unknown) {
   }
 }
 
-/**
- * main function of unit-test, will throw Exception in most condition except job carsh
- */
-export async function start_unit_test(options: TestOption): Promise<boolean> {
+async function startUniTestImpl(options: TestOption): Promise<number> {
   const failurePath = join(options.outputFolder, "failures.json");
   let failedTestCases: string[] = [];
   if (options.onlyFailures) {
@@ -85,6 +83,18 @@ export async function start_unit_test(options: TestOption): Promise<boolean> {
     reportConfig.errorLimit = options.errorLimit || reportConfig.errorLimit;
     generateReport(options.mode, options.outputFolder, fileCoverageInfo);
   }
+  return executedResult.fail === 0 ? 0 : 1;
+}
 
-  return executedResult.fail === 0;
+export async function start_unit_test(options: TestOption): Promise<number> {
+  try {
+    return await startUniTestImpl(options);
+  } catch (error) {
+    if (error instanceof CompilationError) {
+      console.log(error.message);
+      return 2;
+    }
+    // unknown exception.
+    throw error;
+  }
 }

src/utils/ascWrapper.ts

@@ -0,0 +1,20 @@
+import { createMemoryStream, main } from "assemblyscript/asc";
+
+export const compiler = {
+  compile: main,
+};
+
+export class CompilationError extends Error {
+  constructor(errorMessage: string) {
+    super(errorMessage);
+    this.name = "CompilationError";
+  }
+}
+
+export async function ascMain(ascArgv: string[]) {
+  const stderr = createMemoryStream();
+  const { error } = await compiler.compile(ascArgv.concat("--noColors"), { stderr });
+  if (error) {
+    throw new CompilationError(stderr.toString());
+  }
+}

tests/e2e/compilationFailed/as-test.config.js

@@ -0,0 +1,19 @@
+import path from "node:path";
+
+const __dirname = path.dirname(new URL(import.meta.url).pathname);
+
+export default {
+  include: [__dirname],
+  imports(runtime) {
+    return {
+      env: {
+        log: (msg) => {
+          runtime.framework.log(runtime.exports.__getString(msg));
+        },
+      },
+    };
+  },
+  temp: path.join(__dirname, "tmp"),
+  output: path.join(__dirname, "tmp"),
+  mode: [],
+};

tests/e2e/compilationFailed/incorrect.test.ts

@@ -0,0 +1,5 @@
+import { test, expect } from "../../../assembly";
+
+test("assert on test", () => {
+  f = 1;
+});

tests/e2e/compilationFailed/stdout.txt

@@ -0,0 +1,9 @@
+code analysis: OK
+ERROR TS2304: Cannot find name 'f'.
+   :
+ 4 │ f = 1;
+   │ ~
+   └─ in tests/e2e/compilationFailed/incorrect.test.ts(4,3)
+
+FAILURE 1 compile error(s)
+

tests/e2e/compilationFailed/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "assemblyscript/std/assembly.json",
+  "include": ["./**/*.ts"]
+}

tests/e2e/run.js

@@ -10,9 +10,9 @@ function getDiff(s1, s2) {
   return diffLines(s1, s2)
     .map((part) => {
       if (part.added) {
-        return chalk.bgGreen(handleEscape(part.value));
+        return chalk.bgGreen("+" + handleEscape(part.value));
       } else if (part.removed) {
-        return chalk.bgRed(handleEscape(part.value));
+        return "-" + chalk.bgRed(handleEscape(part.value));
       } else {
         return part.value;
       }
@@ -47,12 +47,16 @@ function runEndToEndTest(name, flags, handle) {
   }
 }
 
-runEndToEndTest("printLogInFailedInfo", "", (error, stdout, stderr) => {
-  assert(error.code === 255);
+runEndToEndTest("assertFailed", "", (error, stdout, stderr) => {
+  assert(error.code === 1);
 });
 
-runEndToEndTest("assertFailed", "", (error, stdout, stderr) => {
-  assert(error.code === 255);
+runEndToEndTest("compilationFailed", "", (error, stdout, stderr) => {
+  assert(error.code === 2);
+});
+
+runEndToEndTest("printLogInFailedInfo", "", (error, stdout, stderr) => {
+  assert(error.code === 1);
 });
 
 runEndToEndTest(

tests/ts/test/core/throwError.test.ts

@@ -1,21 +1,20 @@
 // eslint-disable-next-line n/no-extraneous-import
 import { jest } from "@jest/globals";
+import { precompile } from "../../../../src/core/precompile.js";
+import { compile } from "../../../../src/core/compile.js";
+import { compiler } from "../../../../src/utils/ascWrapper.js";
 
-jest.unstable_mockModule("assemblyscript/asc", () => ({
-  main: jest.fn(() => {
-    return {
-      error: new Error("mock asc.main() error"),
-      stderr: "mock asc.main() error",
-    };
-  }),
-}));
+beforeEach(() => {
+  jest.spyOn(compiler, "compile").mockImplementation(() => {
+    throw new Error("mock asc.main() error");
+  });
+});
 
-const { main } = await import("assemblyscript/asc");
-const { precompile } = await import("../../../../src/core/precompile.js");
-const { compile } = await import("../../../../src/core/compile.js");
+afterEach(() => {
+  jest.clearAllMocks();
+});
 
 test("transform error", async () => {
-  expect(jest.isMockFunction(main)).toBeTruthy();
   await expect(async () => {
     await precompile(["tests/ts/fixture/transformFunction.ts"], [], undefined, undefined, [], true, "");
   }).rejects.toThrow("mock asc.main() error");