add LightweightJoinResolver for thread-based js transpilation + pass JoinResolver to DataSchemaCompiler

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts

@@ -9,7 +9,7 @@ export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface
   // key: cubeName with joins defined
   // 1st level value: join alias
   // 2nd level value: join definition
-  private cubeJoinAliases: Record<string, Record<string, JoinDefinition>>;
+  public cubeJoinAliases: Record<string, Record<string, JoinDefinition>>;
 
   // key: cubeName with joins defined
   // 1st level value: target cube name
@@ -27,7 +27,7 @@ export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface
     super.compile(cubes, errorReporter);
 
     this.cubeList.forEach(cube => {
-      if (!cube.joins) {
+      if (!cube.joins?.length) {
         return;
       }
 

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -22,6 +22,7 @@ import { CompilerInterface } from './PrepareCompiler';
 import { YamlCompiler } from './YamlCompiler';
 import { CubeDictionary } from './CubeDictionary';
 import { CompilerCache } from './CompilerCache';
+import { CubeJoinsResolver } from './CubeJoinsResolver';
 
 const NATIVE_IS_SUPPORTED = isNativeSupported();
 
@@ -53,6 +54,7 @@ export type DataSchemaCompilerOptions = {
   cubeFactory: Function;
   cubeDictionary: CubeDictionary;
   cubeSymbols: CubeSymbols;
+  cubeJoinsResolver: CubeJoinsResolver;
   cubeCompilers?: CompilerInterface[];
   contextCompilers?: CompilerInterface[];
   transpilers?: TranspilerInterface[];
@@ -72,6 +74,7 @@ export type DataSchemaCompilerOptions = {
 export type TranspileOptions = {
   cubeNames?: string[];
   cubeSymbols?: Record<string, Record<string, boolean>>;
+  cubeJoins?: Record<string, Record<string, boolean>>;
   contextSymbols?: Record<string, string>;
   transpilerNames?: string[];
   compilerId?: string;
@@ -108,6 +111,8 @@ export class DataSchemaCompiler {
 
   private readonly cubeSymbols: CubeSymbols;
 
+  private readonly cubeJoinsResolver: CubeJoinsResolver;
+
   // Actually should be something like
   // createCube(cubeDefinition: CubeDefinition): CubeDefinitionExtended
   private readonly cubeFactory: CallableFunction;
@@ -157,6 +162,7 @@ export class DataSchemaCompiler {
     this.extensions = options.extensions || {};
     this.cubeDictionary = options.cubeDictionary;
     this.cubeSymbols = options.cubeSymbols;
+    this.cubeJoinsResolver = options.cubeJoinsResolver;
     this.cubeFactory = options.cubeFactory;
     this.filesToCompile = options.filesToCompile || [];
     this.omitErrors = options.omitErrors || false;
@@ -232,6 +238,7 @@ export class DataSchemaCompiler {
     const transpile = async (stage: CompileStage) => {
       let cubeNames: string[] = [];
       let cubeSymbols: Record<string, Record<string, boolean>> = {};
+      let cubeJoins: Record<string, Record<string, boolean>> = {};
       let transpilerNames: string[] = [];
       let results;
 
@@ -252,6 +259,14 @@ export class DataSchemaCompiler {
               )],
             ),
         );
+        cubeJoins = Object.fromEntries(
+          Object.entries(this.cubeJoinsResolver.cubeJoinAliases as Record<string, Record<string, any>>)
+            .map(
+              ([key, value]: [string, Record<string, any>]) => [key, Object.fromEntries(
+                Object.keys(value).map((k) => [k, true]),
+              )],
+            ),
+        );
 
         // Transpilers are the same for all files within phase.
         transpilerNames = this.transpilers.map(t => t.constructor.name);
@@ -264,7 +279,7 @@ export class DataSchemaCompiler {
           content: ';',
         };
 
-        await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage });
+        await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, cubeJoins, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage });
 
         const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js'))
           .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId }));
@@ -291,7 +306,7 @@ export class DataSchemaCompiler {
 
         results = (await Promise.all([...nonJsFilesTasks, ...JsFilesTasks])).flat();
       } else if (transpilationWorkerThreads) {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, cubeJoins, transpilerNames })));
       } else {
         results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {})));
       }
@@ -405,7 +420,7 @@ export class DataSchemaCompiler {
     });
   }
 
-  private async transpileJsFile(file: FileContent, errorsReport: ErrorReporter, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) {
+  private async transpileJsFile(file: FileContent, errorsReport: ErrorReporter, { cubeNames, cubeSymbols, cubeJoins, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) {
     try {
       if (getEnv('transpilationNative')) {
         const reqData = {
@@ -417,6 +432,7 @@ export class DataSchemaCompiler {
             metaData: {
               cubeNames,
               cubeSymbols: cubeSymbols || {},
+              cubeJoins: cubeJoins || {},
               contextSymbols: contextSymbols || {},
               stage: stage || 0 as CompileStage,
             },
@@ -437,6 +453,7 @@ export class DataSchemaCompiler {
           transpilers: transpilerNames,
           cubeNames,
           cubeSymbols,
+          cubeJoins,
         };
 
         const res = await this.workerPool!.exec('transpile', [data]);

packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

@@ -24,6 +24,7 @@ import { CompilerCache } from './CompilerCache';
 import { YamlCompiler } from './YamlCompiler';
 import { ViewCompilationGate } from './ViewCompilationGate';
 import type { ErrorReporter } from './ErrorReporter';
+import { CubeJoinsResolver } from './CubeJoinsResolver';
 
 export type PrepareCompilerOptions = {
   nativeInstance?: NativeInstance,
@@ -46,6 +47,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
   const nativeInstance = options.nativeInstance || new NativeInstance();
   const cubeDictionary = new CubeDictionary();
   const cubeSymbols = new CubeSymbols();
+  const cubeJoinsResolver = new CubeJoinsResolver();
   const viewCompiler = new CubeSymbols(true);
   const viewCompilationGate = new ViewCompilationGate();
   const cubeValidator = new CubeValidator(cubeSymbols);
@@ -62,7 +64,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
   const transpilers: TranspilerInterface[] = [
     new ValidationTranspiler(),
     new ImportExportTranspiler(),
-    new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler),
+    new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler, cubeJoinsResolver),
   ];
 
   if (!options.allowJsDuplicatePropsInSchema) {
@@ -73,7 +75,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
 
   const compiler = new DataSchemaCompiler(repo, Object.assign({}, {
     cubeNameCompilers: [cubeDictionary],
-    preTranspileCubeCompilers: [cubeSymbols, cubeValidator],
+    preTranspileCubeCompilers: [cubeSymbols, cubeValidator, cubeJoinsResolver],
     transpilers,
     viewCompilationGate,
     compiledScriptCache,
@@ -84,6 +86,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     compilerCache,
     cubeDictionary,
     cubeSymbols,
+    cubeJoinsResolver,
     extensions: {
       Funnels,
       RefreshKeys,

packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts

@@ -0,0 +1,28 @@
+import { TranspilerSymbolResolver } from './transpiler.interface';
+import { CONTEXT_SYMBOLS, CURRENT_CUBE_CONSTANTS } from '../CubeSymbols';
+
+export class LightweightJoinResolver implements TranspilerSymbolResolver {
+  public constructor(private cubeJoinAliases: Record<string, Record<string, boolean>> = {}) {
+  }
+
+  public setJoinAliases(cubeJoinAliases: Record<string, Record<string, boolean>>) {
+    this.cubeJoinAliases = cubeJoinAliases;
+  }
+
+  public isCurrentCube(name: string): boolean {
+    return CURRENT_CUBE_CONSTANTS.indexOf(name) >= 0;
+  }
+
+  public resolveSymbol(cubeName: string, name: string): any {
+    if (name === 'USER_CONTEXT') {
+      throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.');
+    }
+
+    if (CONTEXT_SYMBOLS[name]) {
+      return true;
+    }
+
+    const cube = this.cubeJoinAliases[this.isCurrentCube(name) ? cubeName : name];
+    return !!(cube || this.cubeJoinAliases[cubeName]?.[name]);
+  }
+}

packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts

@@ -10,29 +10,33 @@ import { CubePropContextTranspiler } from './CubePropContextTranspiler';
 import { ErrorReporter } from '../ErrorReporter';
 import { LightweightSymbolResolver } from './LightweightSymbolResolver';
 import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary';
+import { LightweightJoinResolver } from './LightweightJoinResolver';
 
 type TransferContent = {
   fileName: string;
   content: string;
   transpilers: string[];
   cubeNames: string[];
   cubeSymbols: Record<string, Record<string, boolean>>;
+  cubeJoins: Record<string, Record<string, boolean>>;
 };
 
 const cubeDictionary = new LightweightNodeCubeDictionary();
 const cubeSymbols = new LightweightSymbolResolver();
+const cubeJoinsResolver = new LightweightJoinResolver();
 const errorsReport = new ErrorReporter(null, []);
 
 const transpilers = {
   ValidationTranspiler: new ValidationTranspiler(),
   ImportExportTranspiler: new ImportExportTranspiler(),
   CubeCheckDuplicatePropTranspiler: new CubeCheckDuplicatePropTranspiler(),
-  CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols),
+  CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols, cubeJoinsResolver),
 };
 
 const transpile = (data: TransferContent) => {
   cubeDictionary.setCubeNames(data.cubeNames);
   cubeSymbols.setSymbols(data.cubeSymbols);
+  cubeJoinsResolver.setJoinAliases(data.cubeJoins);
 
   const ast = parse(
     data.content,