perf(schema-compiler): YAML transpilation in worker threads (#9929)

* extracted yaml and jinja transpilation in separate functions

* implement yaml transpilation in worker threads

* fix to allow use threads and native in parallel

* just code polish

* small fix

* remove speed test as it is flacky

* introduce compiledYamlCache

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -7,6 +7,7 @@ import { DynamicReference } from './DynamicReference';
 import { camelizeCube } from './utils';
 
 import type { ErrorReporter } from './ErrorReporter';
+import { TranspilerSymbolResolver } from './transpilers';
 
 export type ToString = { toString(): string };
 
@@ -193,7 +194,7 @@ export const CONTEXT_SYMBOLS = {
 
 export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE'];
 
-export class CubeSymbols {
+export class CubeSymbols implements TranspilerSymbolResolver {
   public symbols: Record<string | symbol, CubeSymbolsDefinition>;
 
   private builtCubes: Record<string, CubeDefinitionExtended>;

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -88,6 +88,7 @@ export type DataSchemaCompilerOptions = {
   compileContext?: any;
   allowNodeRequire?: boolean;
   compiledScriptCache: LRUCache<string, vm.Script>;
+  compiledYamlCache: LRUCache<string, string>;
 };
 
 export type TranspileOptions = {
@@ -163,6 +164,8 @@ export class DataSchemaCompiler {
 
   private readonly compiledScriptCache: LRUCache<string, vm.Script>;
 
+  private readonly compiledYamlCache: LRUCache<string, string>;
+
   private compileV8ContextCache: vm.Context | null = null;
 
   // FIXME: Is public only because of tests, should be private
@@ -196,6 +199,7 @@ export class DataSchemaCompiler {
     this.workerPool = null;
     this.compilerId = options.compilerId || 'default';
     this.compiledScriptCache = options.compiledScriptCache;
+    this.compiledYamlCache = options.compiledYamlCache;
   }
 
   public compileObjects(compileServices: CompilerInterface[], objects, errorsReport: ErrorReporter) {
@@ -268,7 +272,7 @@ export class DataSchemaCompiler {
     const transpilationNativeThreadsCount = getThreadsCount();
     const { compilerId } = this;
 
-    if (!transpilationNative && transpilationWorkerThreads) {
+    if (transpilationWorkerThreads) {
       const wc = getEnv('transpilationWorkerThreadsCount');
       this.workerPool = workerpool.pool(
         path.join(__dirname, 'transpilers/transpiler_worker'),
@@ -288,7 +292,7 @@ export class DataSchemaCompiler {
 
       if (transpilationNative) {
         const nonJsFilesTasks = [...jinjaTemplatedFiles, ...yamlFiles]
-          .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId }));
+          .map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames, compilerId }));
 
         const jsFiles = originalJsFiles;
         let jsFilesTasks: Promise<(FileContent | undefined)[]>[] = [];
@@ -575,14 +579,9 @@ export class DataSchemaCompiler {
       (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml'))
       && file.content.match(JINJA_SYNTAX)
     ) {
-      return this.yamlCompiler.compileYamlWithJinjaFile(
-        file,
-        errorsReport,
-        this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
-        this.pythonContext!
-      );
+      return this.transpileJinjaFile(file, errorsReport, options);
     } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) {
-      return this.yamlCompiler.transpileYamlFile(file, errorsReport);
+      return this.transpileYamlFile(file, errorsReport, options);
     } else {
       return file;
     }
@@ -668,7 +667,7 @@ export class DataSchemaCompiler {
           cubeSymbols,
         };
 
-        const res = await this.workerPool!.exec('transpile', [data]);
+        const res = await this.workerPool!.exec('transpileJs', [data]);
         errorsReport.addErrors(res.errors);
         errorsReport.addWarnings(res.warnings);
 
@@ -705,6 +704,65 @@ export class DataSchemaCompiler {
     return undefined;
   }
 
+  private async transpileYamlFile(
+    file: FileContent,
+    errorsReport: ErrorReporter,
+    { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions
+  ): Promise<(FileContent | undefined)> {
+    const cacheKey = crypto.createHash('md5').update(JSON.stringify(file.content)).digest('hex');
+
+    if (this.compiledYamlCache.has(cacheKey)) {
+      const content = this.compiledYamlCache.get(cacheKey)!;
+
+      return { ...file, content };
+    }
+
+    /* if (getEnv('transpilationNative')) {
+
+    } else */ if (getEnv('transpilationWorkerThreads')) {
+      const data = {
+        fileName: file.fileName,
+        content: file.content,
+        transpilers: [],
+        cubeNames,
+        cubeSymbols,
+      };
+
+      const res = await this.workerPool!.exec('transpileYaml', [data]);
+      errorsReport.addErrors(res.errors);
+      errorsReport.addWarnings(res.warnings);
+
+      this.compiledYamlCache.set(cacheKey, res.content);
+
+      return { ...file, content: res.content };
+    } else {
+      const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
+
+      this.compiledYamlCache.set(cacheKey, transpiledFile?.content || '');
+
+      return transpiledFile;
+    }
+  }
+
+  private async transpileJinjaFile(
+    file: FileContent,
+    errorsReport: ErrorReporter,
+    { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions
+  ): Promise<(FileContent | undefined)> {
+    // if (getEnv('transpilationNative')) {
+    //
+    // } else if (getEnv('transpilationWorkerThreads')) {
+    //
+    // } else {
+    return this.yamlCompiler.compileYamlWithJinjaFile(
+      file,
+      errorsReport,
+      this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
+      this.pythonContext!
+    );
+    // }
+  }
+
   public withQuery(query, fn) {
     const oldQuery = this.currentQuery;
     this.currentQuery = query;

packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

@@ -37,6 +37,7 @@ export type PrepareCompilerOptions = {
   headCommitId?: string;
   adapter?: string;
   compiledScriptCache?: LRUCache<string, vm.Script>;
+  compiledYamlCache?: LRUCache<string, string>;
 };
 
 export interface CompilerInterface {
@@ -59,6 +60,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
   const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler);
 
   const compiledScriptCache = options.compiledScriptCache || new LRUCache<string, vm.Script>({ max: 250 });
+  const compiledYamlCache = options.compiledYamlCache || new LRUCache<string, string>({ max: 250 });
 
   const transpilers: TranspilerInterface[] = [
     new ValidationTranspiler(),
@@ -79,6 +81,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     transpilers,
     viewCompilationGate,
     compiledScriptCache,
+    compiledYamlCache,
     viewCompilers: [viewCompiler],
     cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer],
     contextCompilers: [contextEvaluator],

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -8,11 +8,14 @@ import { JinjaEngine, NativeInstance, PythonCtx } from '@cubejs-backend/native';
 import type { FileContent } from '@cubejs-backend/shared';
 
 import { getEnv } from '@cubejs-backend/shared';
-import { CubePropContextTranspiler, transpiledFields, transpiledFieldsPatterns } from './transpilers';
+import {
+  CubePropContextTranspiler,
+  transpiledFields,
+  transpiledFieldsPatterns,
+  TranspilerCubeResolver, TranspilerSymbolResolver
+} from './transpilers';
 import { PythonParser } from '../parser/PythonParser';
-import { CubeSymbols } from './CubeSymbols';
 import { nonStringFields } from './CubeValidator';
-import { CubeDictionary } from './CubeDictionary';
 import { ErrorReporter } from './ErrorReporter';
 import { camelizeCube } from './utils';
 import { CompileContext } from './DataSchemaCompiler';
@@ -28,10 +31,10 @@ export class YamlCompiler {
   protected jinjaEngine: JinjaEngine | null = null;
 
   public constructor(
-    private readonly cubeSymbols: CubeSymbols,
-    private readonly cubeDictionary: CubeDictionary,
+    private readonly cubeSymbols: TranspilerSymbolResolver,
+    private readonly cubeDictionary: TranspilerCubeResolver,
     private readonly nativeInstance: NativeInstance,
-    private readonly viewCompiler: CubeSymbols,
+    private readonly viewCompiler: TranspilerSymbolResolver,
   ) {
   }
 
@@ -125,15 +128,14 @@ export class YamlCompiler {
     cubeObj.dimensions = this.yamlArrayToObj(cubeObj.dimensions || [], 'dimension', errorsReport);
     cubeObj.segments = this.yamlArrayToObj(cubeObj.segments || [], 'segment', errorsReport);
     cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations || [], 'preAggregation', errorsReport);
+    cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport);
 
     cubeObj.joins = cubeObj.joins || []; // For edge cases where joins are not defined/null
     if (!Array.isArray(cubeObj.joins)) {
       errorsReport.error('joins must be defined as array');
       cubeObj.joins = [];
     }
 
-    cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport);
-
     return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport);
   }
 

packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts

@@ -3,6 +3,7 @@ import { parse } from '@babel/parser';
 import babelGenerator from '@babel/generator';
 import babelTraverse from '@babel/traverse';
 
+import { NativeInstance } from '@cubejs-backend/native';
 import { ValidationTranspiler } from './ValidationTranspiler';
 import { ImportExportTranspiler } from './ImportExportTranspiler';
 import { CubeCheckDuplicatePropTranspiler } from './CubeCheckDuplicatePropTranspiler';
@@ -11,6 +12,7 @@ import { ErrorReporter } from '../ErrorReporter';
 import { LightweightSymbolResolver } from './LightweightSymbolResolver';
 import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary';
 import { IIFETranspiler } from './IIFETranspiler';
+import { YamlCompiler } from '../YamlCompiler';
 
 type TransferContent = {
   fileName: string;
@@ -23,6 +25,7 @@ type TransferContent = {
 const cubeDictionary = new LightweightNodeCubeDictionary();
 const cubeSymbols = new LightweightSymbolResolver();
 const errorsReport = new ErrorReporter(null, []);
+const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, new NativeInstance(), cubeSymbols);
 
 const transpilers = {
   ValidationTranspiler: new ValidationTranspiler(),
@@ -32,7 +35,7 @@ const transpilers = {
   IIFETranspiler: new IIFETranspiler(),
 };
 
-const transpile = (data: TransferContent) => {
+const transpileJs = (data: TransferContent) => {
   cubeDictionary.setCubeNames(data.cubeNames);
   cubeSymbols.setSymbols(data.cubeSymbols);
 
@@ -64,6 +67,22 @@ const transpile = (data: TransferContent) => {
   };
 };
 
+const transpileYaml = (data: TransferContent) => {
+  cubeDictionary.setCubeNames(data.cubeNames);
+  cubeSymbols.setSymbols(data.cubeSymbols);
+
+  errorsReport.inFile(data);
+  const transpiledFile = yamlCompiler.transpileYamlFile(data, errorsReport);
+  errorsReport.exitFile();
+
+  return {
+    content: transpiledFile?.content || '',
+    errors: errorsReport.getErrors(),
+    warnings: errorsReport.getWarnings()
+  };
+};
+
 workerpool.worker({
-  transpile,
+  transpileJs,
+  transpileYaml,
 });

packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts

@@ -158,49 +158,6 @@ describe('DataSchemaCompiler', () => {
         compiler.throwIfAnyErrors();
       });
     });
-
-    describe('Test perfomance', () => {
-      const schema = `
-        cube('visitors', {
-          sql: 'select * from visitors',
-          measures: {
-            count: {
-              type: 'count',
-              sql: 'id'
-            },
-            duration: {
-              type: 'avg',
-              sql: 'duration'
-            },
-          },
-          dimensions: {
-            date: {
-              type: 'string',
-              sql: 'date'
-            },
-            browser: {
-              type: 'string',
-              sql: 'browser'
-            }
-          }
-        })
-      `;
-
-      it('Should compile 200 schemas in less than 2500ms * 10', async () => {
-        const repeats = 200;
-
-        const compilerWith = prepareJsCompiler(schema, { allowJsDuplicatePropsInSchema: false });
-        const start = new Date().getTime();
-        for (let i = 0; i < repeats; i++) {
-          delete compilerWith.compiler.compilePromise; // Reset compile result
-          await compilerWith.compiler.compile();
-        }
-        const end = new Date().getTime();
-        const time = end - start;
-
-        expect(time).toBeLessThan(2500 * 10);
-      });
-    });
   });
 
   it('calculated metrics', async () => {