Refactor: improve performance and reduce memory cost during precompile and compile stage (#38)

Author: XMadrid

bin/as-test.js

@@ -26,3 +26,10 @@ const testapp = spawn("node", argv, { stdio: "inherit" });
 testapp.on("close", function (code) {
   exit(code);
 });
+
+testapp.on("exit", (_code, signal) => {
+  if (signal === "SIGABRT") {
+    // node heap out of memory
+    exit(-1);
+  }
+});

src/core/compile.ts

@@ -3,7 +3,6 @@ import { join, relative } from "node:path";
 import { findRoot } from "../utils/pathResolver.js";
 
 export async function compile(testCodePaths: string[], outputFolder: string, compileFlags: string): Promise<string[]> {
-  const pms: Promise<void>[] = [];
   const wasm: string[] = [];
   const root = findRoot(testCodePaths);
   const compile = async (testCodePath: string) => {
@@ -33,10 +32,12 @@ export async function compile(testCodePaths: string[], outputFolder: string, com
       throw error;
     }
   };
-  for (const testCodePath of testCodePaths) {
-    pms.push(compile(testCodePath));
+
+  // Here, for-await is more efficient and less memory cost than Promise.all()
+  for (const codePath of testCodePaths) {
+    await compile(codePath);
   }
-  await Promise.all(pms);
+
   return wasm;
 }
 

src/core/precompile.ts

@@ -22,11 +22,14 @@ export async function precompile(
   const testCodePaths = testcases ?? getRelatedFiles(includes, excludes, (path: string) => path.endsWith(".test.ts"));
 
   const sourceCodePaths = getRelatedFiles(includes, excludes, (path: string) => !path.endsWith(".test.ts"));
-  const sourceCodeTransforms: Promise<void>[] = [];
-  for (const sourceCodePath of sourceCodePaths.values()) {
-    sourceCodeTransforms.push(transform(sourceCodePath, transformFunction, flags));
+
+  // The batchSize = 2 is empirical data after benchmarking
+  const batchSize = 2;
+  for (let i = 0; i < sourceCodePaths.length; i += batchSize) {
+    await Promise.all(
+      sourceCodePaths.slice(i, i + batchSize).map((sourcePath) => transform(sourcePath, transformFunction, flags))
+    );
   }
-  await Promise.all(sourceCodeTransforms);
 
   return {
     testCodePaths,