implement bulk transpileJsFiles in native

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js

@@ -1,6 +1,7 @@
 import crypto from 'crypto';
 import vm from 'vm';
 import fs from 'fs';
+import os from 'os';
 import path from 'path';
 import syntaxCheck from 'syntax-error';
 import { parse } from '@babel/parser';
@@ -21,6 +22,20 @@ const moduleFileCache = {};
 
 const JINJA_SYNTAX = /{%|%}|{{|}}/ig;
 
+const getThreadsCount = () => {
+  const envThreads = getEnv('transpilationWorkerThreadsCount');
+  if (envThreads > 0) {
+    return envThreads;
+  }
+
+  const cpuCount = os.cpus()?.length;
+  if (cpuCount) {
+    return Math.max(1, cpuCount - 1);
+  }
+
+  return 3; // Default (like the workerpool do)
+};
+
 export class DataSchemaCompiler {
   constructor(repository, options = {}) {
     this.repository = repository;
@@ -100,6 +115,7 @@ export class DataSchemaCompiler {
 
     const transpilationWorkerThreads = getEnv('transpilationWorkerThreads');
     const transpilationNative = getEnv('transpilationNative');
+    const transpilationNativeThreadsCount = getThreadsCount();
     const { compilerId } = this;
 
     if (!transpilationNative && transpilationWorkerThreads) {
@@ -151,7 +167,26 @@ export class DataSchemaCompiler {
 
         await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage });
 
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId })));
+        const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js'))
+          .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId }));
+
+        const jsFiles = toCompile.filter(file => file.fileName.endsWith('.js'));
+        let jsChunks;
+        if (jsFiles.length < transpilationNativeThreadsCount * transpilationNativeThreadsCount) {
+          jsChunks = [jsFiles];
+        } else {
+          const baseSize = Math.floor(jsFiles.length / transpilationNativeThreadsCount);
+          jsChunks = [];
+          for (let i = 0; i < transpilationNativeThreadsCount; i++) {
+            // For the last part, we take the remaining files so we don't lose the extra ones.
+            const start = i * baseSize;
+            const end = (i === transpilationNativeThreadsCount - 1) ? jsFiles.length : start + baseSize;
+            jsChunks.push(jsFiles.slice(start, end));
+          }
+        }
+        const JsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesBulk(chunk, errorsReport, { transpilerNames, compilerId }));
+
+        results = (await Promise.all([...nonJsFilesTasks, ...JsFilesTasks])).flat();
       } else if (transpilationWorkerThreads) {
         results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
@@ -231,6 +266,44 @@ export class DataSchemaCompiler {
     }
   }
 
+  /**
+   * Right now it is used only for transpilation in native,
+   * so no checks for transpilation type inside this method
+   */
+  async transpileJsFilesBulk(files, errorsReport, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }) {
+    // for bulk processing this data may be optimized even more by passing transpilerNames, compilerId only once for a bulk
+    // but this requires more complex logic to be implemented in the native side.
+    // And comparing to the file content sizes, a few bytes of JSON data is not a big deal here
+    const reqDataArr = files.map(file => ({
+      fileName: file.fileName,
+      fileContent: file.content,
+      transpilers: transpilerNames,
+      compilerId,
+      ...(cubeNames && {
+        metaData: {
+          cubeNames,
+          cubeSymbols,
+          contextSymbols,
+          stage
+        },
+      }),
+    }));
+    const res = await transpileJs(reqDataArr);
+
+    return files.map((file, index) => {
+      errorsReport.inFile(file);
+      if (!res[index]) { // This should not happen in theory but just to be safe
+        errorsReport.error(`No transpilation result received for the file ${file.fileName}.`);
+        return undefined;
+      }
+      errorsReport.addErrors(res[index].errors);
+      errorsReport.addWarnings(res[index].warnings);
+      errorsReport.exitFile();
+
+      return { ...file, content: res[index].code };
+    });
+  }
+
   async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }) {
     try {
       if (getEnv('transpilationNative')) {