refactor: split first and later compile/transpile stages

Author: KSDaemon

packages/cubejs-backend-shared/src/FileRepository.ts

@@ -6,7 +6,6 @@ export interface FileContent {
   content: string;
   readOnly?: boolean;
   isModule?: boolean;
-  convertedToJs?: boolean;
 }
 
 export interface SchemaFileRepository {

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -47,6 +47,24 @@ const getThreadsCount = () => {
   return 3; // Default (like the workerpool do)
 };
 
+const splitFilesToChunks = (files: FileContent[], chunksCount: number): FileContent[][] => {
+  let chunks: FileContent[][];
+  if (files.length < chunksCount * chunksCount) {
+    chunks = [files];
+  } else {
+    const baseSize = Math.floor(files.length / chunksCount);
+    chunks = [];
+    for (let i = 0; i < chunksCount; i++) {
+      // For the last part, we take the remaining files so we don't lose the extra ones.
+      const start = i * baseSize;
+      const end = (i === chunksCount - 1) ? files.length : start + baseSize;
+      chunks.push(files.slice(start, end));
+    }
+  }
+
+  return chunks;
+};
+
 export type DataSchemaCompilerOptions = {
   compilerCache: CompilerCache;
   omitErrors?: boolean;
@@ -258,34 +276,62 @@ export class DataSchemaCompiler {
       );
     }
 
-    const transpile = async (stage: CompileStage): Promise<FileContent[]> => {
+    const transpilePhaseFirst = async (stage: CompileStage): Promise<FileContent[]> => {
       let cubeNames: string[] = [];
       let cubeSymbols: Record<string, Record<string, boolean>> = {};
       let transpilerNames: string[] = [];
       let results: (FileContent | undefined)[];
 
       if (transpilationNative || transpilationWorkerThreads) {
-        cubeNames = Object.keys(this.cubeDictionary.byId);
-        // We need only cubes and all its member names for transpiling.
-        // Cubes doesn't change during transpiling, but are changed during compilation phase,
-        // so we can prepare them once for every phase.
-        // Communication between main and worker threads uses
-        // The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm)
-        // which doesn't allow passing any function objects, so we need to sanitize the symbols.
-        // Communication with native backend also involves deserialization.
-        cubeSymbols = Object.fromEntries(
-          Object.entries(this.cubeSymbols.symbols as Record<string, Record<string, any>>)
-            .map(
-              ([key, value]: [string, Record<string, any>]) => [key, Object.fromEntries(
-                Object.keys(value).map((k) => [k, true]),
-              )],
-            ),
-        );
+        ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols());
+      }
+
+      if (transpilationNative) {
+        const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js'))
+          .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId }));
+
+        const jsFiles = toCompile.filter(file => file.fileName.endsWith('.js'));
+        let jsFilesTasks: Promise<(FileContent | undefined)[]>[] = [];
+
+        if (jsFiles.length > 0) {
+          // Warming up swc compiler cache
+          const dummyFile = {
+            fileName: 'dummy.js',
+            content: ';',
+          };
+
+          await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage });
+
+          const jsChunks = splitFilesToChunks(jsFiles, transpilationNativeThreadsCount);
+          jsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesBulk(chunk, errorsReport, { transpilerNames, compilerId }));
+        }
 
-        // Transpilers are the same for all files within phase.
-        transpilerNames = this.transpilers.map(t => t.constructor.name);
+        results = (await Promise.all([...nonJsFilesTasks, ...jsFilesTasks])).flat();
+      } else if (transpilationWorkerThreads) {
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
+      } else {
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {})));
+      }
+
+      return results.filter(f => !!f) as FileContent[];
+    };
+
+    const transpilePhase = async (stage: CompileStage): Promise<FileContent[]> => {
+      let cubeNames: string[] = [];
+      let cubeSymbols: Record<string, Record<string, boolean>> = {};
+      let transpilerNames: string[] = [];
+      let results: (FileContent | undefined)[];
+
+      if (toCompile.length === 0) {
+        return [];
       }
 
+      if (transpilationNative || transpilationWorkerThreads) {
+        ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols());
+      }
+
+      // After the first phase all files are with JS source code: original or transpiled
+
       if (transpilationNative) {
         // Warming up swc compiler cache
         const dummyFile = {
@@ -295,34 +341,14 @@ export class DataSchemaCompiler {
 
         await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage });
 
-        const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js') && !file.convertedToJs)
-          .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId }));
-
-        const jsFiles = toCompile.filter(file => file.fileName.endsWith('.js') || file.convertedToJs);
-        let JsFilesTasks = [];
+        const jsChunks = splitFilesToChunks(toCompile, transpilationNativeThreadsCount);
+        const jsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesBulk(chunk, errorsReport, { transpilerNames, compilerId }));
 
-        if (jsFiles.length > 0) {
-          let jsChunks;
-          if (jsFiles.length < transpilationNativeThreadsCount * transpilationNativeThreadsCount) {
-            jsChunks = [jsFiles];
-          } else {
-            const baseSize = Math.floor(jsFiles.length / transpilationNativeThreadsCount);
-            jsChunks = [];
-            for (let i = 0; i < transpilationNativeThreadsCount; i++) {
-              // For the last part, we take the remaining files so we don't lose the extra ones.
-              const start = i * baseSize;
-              const end = (i === transpilationNativeThreadsCount - 1) ? jsFiles.length : start + baseSize;
-              jsChunks.push(jsFiles.slice(start, end));
-            }
-          }
-          JsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesBulk(chunk, errorsReport, { transpilerNames, compilerId }));
-        }
-
-        results = (await Promise.all([...nonJsFilesTasks, ...JsFilesTasks])).flat();
+        results = (await Promise.all(jsFilesTasks)).flat();
       } else if (transpilationWorkerThreads) {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
+        results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {})));
+        results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, {})));
       }
 
       return results.filter(f => !!f) as FileContent[];
@@ -335,6 +361,14 @@ export class DataSchemaCompiler {
     let asyncModules: CallableFunction[] = [];
     let transpiledFiles: FileContent[] = [];
 
+    const cleanup = () => {
+      cubes = [];
+      exports = {};
+      contexts = [];
+      compiledFiles = {};
+      asyncModules = [];
+    };
+
     this.compileV8ContextCache = vm.createContext({
       view: (name, cube) => {
         const file = ctxFileStorage.getStore();
@@ -420,26 +454,28 @@ export class DataSchemaCompiler {
       COMPILE_CONTEXT: this.standalone ? this.standaloneCompileContextProxy() : this.cloneCompileContextWithGetterAlias(this.compileContext || {}),
     });
 
-    const compilePhase = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => {
+    const compilePhaseFirst = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => {
       // clear the objects for the next phase
-      cubes = [];
-      exports = {};
-      contexts = [];
-      compiledFiles = {};
-      asyncModules = [];
-      transpiledFiles = await transpile(stage);
+      cleanup();
+      transpiledFiles = await transpilePhaseFirst(stage);
 
-      if (stage === 0) {
-        // We render jinja and transpile yaml only once on first phase and then use resulting JS for these files
-        // afterward avoiding costly YAML/Python parsing again. Original JS files are preserved as is for cache hits.
-        const convertedToJsFiles = transpiledFiles.filter(f => f.convertedToJs);
-        toCompile = [...originalJsFiles, ...convertedToJsFiles];
-      }
+      // We render jinja and transpile yaml only once on first phase and then use resulting JS for these files
+      // afterward avoiding costly YAML/Python parsing again. Original JS files are preserved as is for cache hits.
+      const convertedToJsFiles = transpiledFiles.filter(f => !f.fileName.endsWith('.js'));
+      toCompile = [...originalJsFiles, ...convertedToJsFiles];
+
+      return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
+    };
+
+    const compilePhase = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => {
+      // clear the objects for the next phase
+      cleanup();
+      transpiledFiles = await transpilePhase(stage);
 
       return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
     };
 
-    return compilePhase({ cubeCompilers: this.cubeNameCompilers }, 0)
+    return compilePhaseFirst({ cubeCompilers: this.cubeNameCompilers }, 0)
       .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers.concat([this.viewCompilationGate]) }, 1))
       .then(() => (this.viewCompilationGate.shouldCompileViews() ?
         compilePhase({ cubeCompilers: this.viewCompilers }, 2)
@@ -450,11 +486,7 @@ export class DataSchemaCompiler {
       }, 3))
       .then(() => {
         // Free unneeded resources
-        cubes = [];
-        exports = {};
-        contexts = [];
-        compiledFiles = {};
-        asyncModules = [];
+        cleanup();
         transpiledFiles = [];
         toCompile = [];
 
@@ -508,36 +540,49 @@ export class DataSchemaCompiler {
     });
   }
 
+  private prepareTranspileSymbols() {
+    const cubeNames: string[] = Object.keys(this.cubeDictionary.byId);
+    // We need only cubes and all its member names for transpiling.
+    // Cubes doesn't change during transpiling, but are changed during compilation phase,
+    // so we can prepare them once for every phase.
+    // Communication between main and worker threads uses
+    // The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm)
+    // which doesn't allow passing any function objects, so we need to sanitize the symbols.
+    // Communication with native backend also involves deserialization.
+    const cubeSymbols: Record<string, Record<string, boolean>> = Object.fromEntries(
+      Object.entries(this.cubeSymbols.symbols as Record<string, Record<string, any>>)
+        .map(
+          ([key, value]: [string, Record<string, any>]) => [key, Object.fromEntries(
+            Object.keys(value).map((k) => [k, true]),
+          )],
+        ),
+    );
+
+    // Transpilers are the same for all files within phase.
+    const transpilerNames: string[] = this.transpilers.map(t => t.constructor.name);
+
+    return { cubeNames, cubeSymbols, transpilerNames };
+  }
+
   private async transpileFile(
     file: FileContent,
     errorsReport: ErrorReporter,
     options: TranspileOptions = {}
   ): Promise<(FileContent | undefined)> {
-    if (file.fileName.endsWith('.js') || file.convertedToJs) {
+    if (file.fileName.endsWith('.js')) {
       return this.transpileJsFile(file, errorsReport, options);
     } else if (file.fileName.endsWith('.jinja') ||
       (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml'))
       && file.content.match(JINJA_SYNTAX)
     ) {
-      const transpiledFile = await this.yamlCompiler.compileYamlWithJinjaFile(
+      return this.yamlCompiler.compileYamlWithJinjaFile(
         file,
         errorsReport,
         this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
         this.pythonContext!
       );
-      if (transpiledFile) {
-        transpiledFile.convertedToJs = true;
-      }
-
-      return transpiledFile;
     } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) {
-      const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
-
-      if (transpiledFile) {
-        transpiledFile.convertedToJs = true;
-      }
-
-      return transpiledFile;
+      return this.yamlCompiler.transpileYamlFile(file, errorsReport);
     } else {
       return file;
     }