pass compiler Id to transpilation native

Author: KSDaemon

packages/cubejs-backend-native/js/index.ts

@@ -104,6 +104,7 @@ export type SQLInterfaceOptions = {
 export interface TransformConfig {
   fileName: string;
   transpilers: string[];
+  compilerId: string;
   metaData?: {
     cubeNames: string[];
     cubeSymbols: Record<string, Record<string, boolean>>;

packages/cubejs-backend-native/src/transpilers.rs

@@ -7,7 +7,7 @@ use neon::prelude::{JsPromise, JsResult, JsValue, NeonResult};
 use neon::types::JsString;
 use serde::Deserialize;
 use std::collections::{HashMap, HashSet};
-use std::sync::RwLock;
+use std::sync::{LazyLock, RwLock};
 
 #[derive(Deserialize, Default, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
@@ -22,10 +22,12 @@ pub struct TransformMetaData {
 pub struct TransformRequestConfig {
     pub file_name: String,
     pub transpilers: Vec<Transpilers>,
+    pub compiler_id: String,
     pub meta_data: Option<TransformMetaData>,
 }
 
-static METADATA_CACHE: RwLock<Option<TransformMetaData>> = RwLock::new(None);
+static METADATA_CACHE: LazyLock<RwLock<HashMap<String, TransformMetaData>>> =
+    LazyLock::new(|| RwLock::new(HashMap::new()));
 
 pub fn register_module(cx: &mut ModuleContext) -> NeonResult<()> {
     cx.export_function("transpileJs", transpile_js)?;
@@ -57,17 +59,27 @@ pub fn transpile_js(mut cx: FunctionContext) -> JsResult<JsPromise> {
                             cube_symbols: cache.cube_symbols.clone(),
                             context_symbols: cache.context_symbols.clone(),
                         };
-                        *config_lock = Some(cache);
+                        config_lock.insert(data.compiler_id.clone(), cache);
                         cfg
                     }
                     None => {
-                        let cache = METADATA_CACHE.read().unwrap().clone().unwrap_or_default();
-                        TransformConfig {
-                            file_name: data.file_name,
-                            transpilers: data.transpilers,
-                            cube_names: cache.cube_names.clone(),
-                            cube_symbols: cache.cube_symbols.clone(),
-                            context_symbols: cache.context_symbols.clone(),
+                        let cache = METADATA_CACHE.read().unwrap();
+
+                        match cache.get(&data.compiler_id) {
+                            Some(cached) => TransformConfig {
+                                file_name: data.file_name,
+                                transpilers: data.transpilers,
+                                cube_names: cached.cube_names.clone(),
+                                cube_symbols: cached.cube_symbols.clone(),
+                                context_symbols: cached.context_symbols.clone(),
+                            },
+                            None => TransformConfig {
+                                file_name: data.file_name,
+                                transpilers: data.transpilers,
+                                cube_names: HashSet::new(),
+                                cube_symbols: HashMap::new(),
+                                context_symbols: HashMap::new(),
+                            },
                         }
                     }
                 },

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js

@@ -46,6 +46,7 @@ export class DataSchemaCompiler {
     this.yamlCompiler.dataSchemaCompiler = this;
     this.pythonContext = null;
     this.workerPool = null;
+    this.compilerId = options.compilerId;
   }
 
   compileObjects(compileServices, objects, errorsReport) {
@@ -97,6 +98,7 @@ export class DataSchemaCompiler {
 
     const transpilationWorkerThreads = getEnv('transpilationWorkerThreads');
     const transpilationNative = getEnv('transpilationNative');
+    const { compilerId } = this;
 
     if (!transpilationNative && transpilationWorkerThreads) {
       const wc = getEnv('transpilationWorkerThreadsCount');
@@ -141,9 +143,9 @@ export class DataSchemaCompiler {
           content: ';',
         };
 
-        await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS });
+        await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId });
 
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { transpilerNames })));
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId })));
       } else if (transpilationWorkerThreads) {
         results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
@@ -172,7 +174,11 @@ export class DataSchemaCompiler {
             content: ';',
           };
 
-          return this.transpileJsFile(dummyFile, errorsReport, { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {} });
+          return this.transpileJsFile(
+            dummyFile,
+            errorsReport,
+            { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {}, compilerId: this.compilerId }
+          );
         } else if (transpilationWorkerThreads && this.workerPool) {
           this.workerPool.terminate();
         }
@@ -219,12 +225,13 @@ export class DataSchemaCompiler {
     }
   }
 
-  async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbols, contextSymbols, transpilerNames }) {
+  async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId }) {
     try {
       if (getEnv('transpilationNative')) {
         const reqData = {
           fileName: file.fileName,
           transpilers: transpilerNames,
+          compilerId,
           ...(cubeNames && {
             metaData: {
               cubeNames,

packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

@@ -59,6 +59,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     transpilers.push(new CubeCheckDuplicatePropTranspiler());
   }
 
+  const compilerId = uuidv4();
+
   const compiler = new DataSchemaCompiler(repo, Object.assign({}, {
     cubeNameCompilers: [cubeDictionary],
     preTranspileCubeCompilers: [cubeSymbols, cubeValidator],
@@ -80,6 +82,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     standalone: options.standalone,
     nativeInstance,
     yamlCompiler,
+    compilerId,
   }, options));
 
   return {
@@ -90,7 +93,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     joinGraph,
     compilerCache,
     headCommitId: options.headCommitId,
-    compilerId: uuidv4(),
+    compilerId,
   };
 };