implement yaml transpilation in worker threads

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -7,6 +7,7 @@ import { DynamicReference } from './DynamicReference';
 import { camelizeCube } from './utils';
 
 import type { ErrorReporter } from './ErrorReporter';
+import { TranspilerSymbolResolver } from './transpilers';
 
 export type ToString = { toString(): string };
 
@@ -193,7 +194,7 @@ export const CONTEXT_SYMBOLS = {
 
 export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE'];
 
-export class CubeSymbols {
+export class CubeSymbols implements TranspilerSymbolResolver{
   public symbols: Record<string | symbol, CubeSymbolsDefinition>;
 
   private builtCubes: Record<string, CubeDefinitionExtended>;

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -605,7 +605,7 @@ export class DataSchemaCompiler {
           cubeSymbols,
         };
 
-        const res = await this.workerPool!.exec('transpile', [data]);
+        const res = await this.workerPool!.exec('transpileJs', [data]);
         errorsReport.addErrors(res.errors);
         errorsReport.addWarnings(res.warnings);
 
@@ -647,23 +647,35 @@ export class DataSchemaCompiler {
     errorsReport: ErrorReporter,
     { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions
   ): Promise<(FileContent | undefined)> {
-    // if (getEnv('transpilationNative')) {
-    //
-    // } else if (getEnv('transpilationWorkerThreads')) {
-    //
-    // } else {
-    const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
+    /* if (getEnv('transpilationNative')) {
+
+    } else */ if (getEnv('transpilationWorkerThreads')) {
+      const data = {
+        fileName: file.fileName,
+        content: file.content,
+        transpilers: [],
+        cubeNames,
+        cubeSymbols,
+      };
+
+      const res = await this.workerPool!.exec('transpileYaml', [data]);
+      errorsReport.addErrors(res.errors);
+      errorsReport.addWarnings(res.warnings);
+
+      return { ...file, content: res.content };
+    } else {
+      const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
+
+      if (transpiledFile) {
+        // We update the yaml file content to the transpiled js content
+        // and raise related flag so it will go JS transpilation flow afterward
+        // avoiding costly YAML/Python parsing again.
+        file.content = transpiledFile.content;
+        file.convertedToJs = true;
+      }
 
-    if (transpiledFile) {
-      // We update the yaml file content to the transpiled js content
-      // and raise related flag so it will go JS transpilation flow afterward
-      // avoiding costly YAML/Python parsing again.
-      file.content = transpiledFile.content;
-      file.convertedToJs = true;
+      return transpiledFile;
     }
-
-    return transpiledFile;
-    // }
   }
 
   private async transpileJinjaFile(

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -8,11 +8,14 @@ import { JinjaEngine, NativeInstance, PythonCtx } from '@cubejs-backend/native';
 import type { FileContent } from '@cubejs-backend/shared';
 
 import { getEnv } from '@cubejs-backend/shared';
-import { CubePropContextTranspiler, transpiledFields, transpiledFieldsPatterns } from './transpilers';
+import {
+  CubePropContextTranspiler,
+  transpiledFields,
+  transpiledFieldsPatterns,
+  TranspilerCubeResolver, TranspilerSymbolResolver
+} from './transpilers';
 import { PythonParser } from '../parser/PythonParser';
-import { CubeSymbols } from './CubeSymbols';
 import { nonStringFields } from './CubeValidator';
-import { CubeDictionary } from './CubeDictionary';
 import { ErrorReporter } from './ErrorReporter';
 import { camelizeCube } from './utils';
 import { CompileContext } from './DataSchemaCompiler';
@@ -28,10 +31,10 @@ export class YamlCompiler {
   protected jinjaEngine: JinjaEngine | null = null;
 
   public constructor(
-    private readonly cubeSymbols: CubeSymbols,
-    private readonly cubeDictionary: CubeDictionary,
+    private readonly cubeSymbols: TranspilerSymbolResolver,
+    private readonly cubeDictionary: TranspilerCubeResolver,
     private readonly nativeInstance: NativeInstance,
-    private readonly viewCompiler: CubeSymbols,
+    private readonly viewCompiler: TranspilerSymbolResolver,
   ) {
   }
 

packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts

@@ -11,6 +11,8 @@ import { ErrorReporter } from '../ErrorReporter';
 import { LightweightSymbolResolver } from './LightweightSymbolResolver';
 import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary';
 import { IIFETranspiler } from './IIFETranspiler';
+import { YamlCompiler } from '../YamlCompiler';
+import { NativeInstance } from '@cubejs-backend/native';
 
 type TransferContent = {
   fileName: string;
@@ -23,6 +25,7 @@ type TransferContent = {
 const cubeDictionary = new LightweightNodeCubeDictionary();
 const cubeSymbols = new LightweightSymbolResolver();
 const errorsReport = new ErrorReporter(null, []);
+const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, new NativeInstance(), cubeSymbols);
 
 const transpilers = {
   ValidationTranspiler: new ValidationTranspiler(),
@@ -32,7 +35,7 @@ const transpilers = {
   IIFETranspiler: new IIFETranspiler(),
 };
 
-const transpile = (data: TransferContent) => {
+const transpileJs = (data: TransferContent) => {
   cubeDictionary.setCubeNames(data.cubeNames);
   cubeSymbols.setSymbols(data.cubeSymbols);
 
@@ -64,6 +67,22 @@ const transpile = (data: TransferContent) => {
   };
 };
 
+const transpileYaml = (data: TransferContent) => {
+  cubeDictionary.setCubeNames(data.cubeNames);
+  cubeSymbols.setSymbols(data.cubeSymbols);
+
+  errorsReport.inFile(data);
+  const transpiledFile = yamlCompiler.transpileYamlFile(data, errorsReport);
+  errorsReport.exitFile();
+
+  return {
+    content: transpiledFile?.content,
+    errors: errorsReport.getErrors(),
+    warnings: errorsReport.getWarnings()
+  };
+};
+
 workerpool.worker({
-  transpile,
+  transpileJs,
+  transpileYaml,
 });