wip

Author: KSDaemon

.github/workflows/push.yml

@@ -9,6 +9,7 @@ on:
       - 'packages/**'
       - 'rust/cubestore/**'
       - 'rust/cubesql/**'
+      - 'rust/cubetranspilers/**'
       - '.eslintrc.js'
       - '.prettierrc'
       - 'package.json'
@@ -24,6 +25,7 @@ on:
       - 'packages/**'
       - 'rust/cubestore/**'
       - 'rust/cubesql/**'
+      - 'rust/cubetranspilers/**'
       - '.eslintrc.js'
       - '.prettierrc'
       - 'package.json'
@@ -43,10 +45,15 @@ jobs:
         # Current docker version + next LTS
         node-version: [20.x, 22.x]
         transpile-worker-threads: [false, true]
+        transpile-native: [false, true]
+        exclude:
+          - transpile-worker-threads: true
+            transpile-native: true
       fail-fast: false
 
     env:
       CUBEJS_TRANSPILATION_WORKER_THREADS: ${{ matrix.transpile-worker-threads }}
+      CUBEJS_TRANSPILATION_NATIVE: ${{ matrix.transpile-native }}
     steps:
       - id: get-tag-out
         run: echo "$OUT"

packages/cubejs-backend-native/Cargo.toml

@@ -18,6 +18,7 @@ crate-type = ["cdylib", "lib"]
 [dependencies]
 cubesqlplanner = { path = "../../rust/cubesqlplanner/cubesqlplanner/" }
 cubenativeutils = { path = "../../rust/cubenativeutils" }
+cubetranspilers = { path = "../../rust/cubetranspilers" }
 async-channel = { version = "2" }
 async-trait = "0.1.36"
 convert_case = "0.6.0"
@@ -34,7 +35,10 @@ minijinja = { version = "1", features = ["json", "loader"] }
 once_cell = "1.10"
 # python
 pyo3 = { version = "0.20.0", features = [], optional = true }
-pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime", "attributes"], optional = true }
+pyo3-asyncio = { version = "0.20.0", features = [
+  "tokio-runtime",
+  "attributes",
+], optional = true }
 serde = { version = "1.0.209", features = ["derive"] }
 serde_json = "1.0.127"
 simple_logger = "1.7.0"

packages/cubejs-backend-native/js/index.ts

@@ -98,6 +98,22 @@ export type SQLInterfaceOptions = {
   gatewayPort?: number,
 };
 
+export interface TransformConfig {
+  fileName: string;
+  transpilers: string[];
+  metaData?: {
+    cubeNames: string[];
+    cubeSymbols: Record<string, Record<string, boolean>>;
+    contextSymbols: Record<string, string>;
+  }
+}
+
+export interface TransformResponse {
+  code: string;
+  errors: string[];
+  warnings: string[];
+}
+
 export function loadNative() {
   // Development version
   if (fs.existsSync(path.join(__dirname, '/../../index.node'))) {
@@ -348,6 +364,12 @@ export const buildSqlAndParams = (cubeEvaluator: any): String => {
   return native.buildSqlAndParams(cubeEvaluator);
 };
 
+export const transpileJs = async (content: String, metadata: TransformConfig): Promise<TransformResponse> => {
+  const native = loadNative();
+
+  return native.transpileJs(content, metadata);
+};
+
 export interface PyConfiguration {
   repositoryFactory?: (ctx: unknown) => Promise<unknown>,
   logger?: (msg: string, params: Record<string, any>) => void,

packages/cubejs-backend-native/src/lib.rs

@@ -16,6 +16,7 @@ pub mod node_obj_serializer;
 pub mod python;
 pub mod stream;
 pub mod template;
+pub mod transpilers;
 pub mod transport;
 pub mod utils;
 

packages/cubejs-backend-native/src/node_export.rs

@@ -515,6 +515,9 @@ pub fn register_module_exports<C: NodeConfiguration + 'static>(
 
     cx.export_function("buildSqlAndParams", build_sql_and_params)?;
 
+    //========= transpilers exports =================
+    crate::transpilers::register_module(&mut cx)?;
+
     crate::template::template_register_module(&mut cx)?;
 
     #[cfg(feature = "python")]

packages/cubejs-backend-native/src/transpilers.rs

@@ -0,0 +1,92 @@
+@@ -0,0 +1,91 @@
+use crate::node_obj_deserializer::JsValueDeserializer;
+use crate::node_obj_serializer::NodeObjSerializer;
+use anyhow::anyhow;
+use cubetranspilers::{run_transpilers, TransformConfig, Transpilers};
+use neon::context::{Context, FunctionContext, ModuleContext};
+use neon::prelude::{JsPromise, JsResult, JsValue, NeonResult};
+use neon::types::JsString;
+use serde::Deserialize;
+use std::collections::{HashMap, HashSet};
+use std::sync::RwLock;
+
+#[derive(Deserialize, Default, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct TransformMetaData {
+    pub cube_names: HashSet<String>,
+    pub cube_symbols: HashMap<String, HashMap<String, bool>>,
+    pub context_symbols: HashMap<String, String>,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct TransformRequestConfig {
+    pub file_name: String,
+    pub transpilers: Vec<Transpilers>,
+    pub meta_data: Option<TransformMetaData>,
+}
+
+static METADATA_CACHE: RwLock<Option<TransformMetaData>> = RwLock::new(None);
+
+pub fn register_module(cx: &mut ModuleContext) -> NeonResult<()> {
+    cx.export_function("transpileJs", transpile_js)?;
+
+    Ok(())
+}
+
+pub fn transpile_js(mut cx: FunctionContext) -> JsResult<JsPromise> {
+    let content = cx.argument::<JsString>(0)?.value(&mut cx);
+    let transform_data_js_object = cx.argument::<JsValue>(1)?;
+    let deserializer = JsValueDeserializer::new(&mut cx, transform_data_js_object);
+    let transform_request_config = TransformRequestConfig::deserialize(deserializer);
+
+    let promise = cx
+        .task(move || {
+            let transform_config: TransformConfig = match transform_request_config {
+                Ok(data) => match data.meta_data {
+                    Some(meta_data) => {
+                        let mut config_lock = METADATA_CACHE.write().unwrap();
+                        let cache = TransformMetaData {
+                            cube_names: meta_data.cube_names,
+                            cube_symbols: meta_data.cube_symbols,
+                            context_symbols: meta_data.context_symbols,
+                        };
+                        let cfg = TransformConfig {
+                            file_name: data.file_name,
+                            transpilers: data.transpilers,
+                            cube_names: cache.cube_names.clone(),
+                            cube_symbols: cache.cube_symbols.clone(),
+                            context_symbols: cache.context_symbols.clone(),
+                        };
+                        *config_lock = Some(cache);
+                        cfg
+                    }
+                    None => {
+                        let cache = METADATA_CACHE.read().unwrap().clone().unwrap_or_default();
+                        TransformConfig {
+                            file_name: data.file_name,
+                            transpilers: data.transpilers,
+                            cube_names: cache.cube_names.clone(),
+                            cube_symbols: cache.cube_symbols.clone(),
+                            context_symbols: cache.context_symbols.clone(),
+                        }
+                    }
+                },
+                Err(err) => return Err(anyhow!("Failed to deserialize input data: {}", err)),
+            };
+
+            run_transpilers(content, transform_config)
+        })
+        .promise(move |mut cx, res| match res {
+            Ok(result) => {
+                let obj = match NodeObjSerializer::serialize(&result, &mut cx) {
+                    Ok(data) => data,
+                    Err(err) => return cx.throw_error(err.to_string()),
+                };
+                Ok(obj)
+            }
+            Err(err) => cx.throw_error(err.to_string()),
+        });
+
+    Ok(promise)
+}

packages/cubejs-backend-shared/src/env.ts

@@ -199,6 +199,10 @@ const variables: Record<string, (...args: any) => any> = {
   transpilationWorkerThreadsCount: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS_COUNT')
     .default('0')
     .asInt(),
+  // This one takes precedence over CUBEJS_TRANSPILATION_WORKER_THREADS
+  transpilationNative: () => get('CUBEJS_TRANSPILATION_NATIVE')
+    .default('false')
+    .asBoolStrict(),
 
   /** ****************************************************************
    * Common db options                                               *

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js

@@ -9,8 +9,10 @@ import R from 'ramda';
 import workerpool from 'workerpool';
 
 import { getEnv, isNativeSupported } from '@cubejs-backend/shared';
+import { transpileJs } from '@cubejs-backend/native';
 import { UserError } from './UserError';
 import { ErrorReporter } from './ErrorReporter';
+import { CONTEXT_SYMBOLS } from './CubeSymbols';
 
 const NATIVE_IS_SUPPORTED = isNativeSupported();
 
@@ -91,7 +93,10 @@ export class DataSchemaCompiler {
     const errorsReport = new ErrorReporter(null, [], this.errorReport);
     this.errorsReport = errorsReport;
 
-    if (getEnv('transpilationWorkerThreads')) {
+    const transpilationWorkerThreads = getEnv('transpilationWorkerThreads');
+    const transpilationNative = getEnv('transpilationNative');
+
+    if (!transpilationNative && transpilationWorkerThreads) {
       const wc = getEnv('transpilationWorkerThreadsCount');
       this.workerPool = workerpool.pool(
         path.join(__dirname, 'transpilers/transpiler_worker'),
@@ -101,26 +106,47 @@ export class DataSchemaCompiler {
 
     const transpile = async () => {
       let cubeNames;
-      let cubeSymbolsNames;
+      let cubeSymbols;
+      let transpilerNames;
+      let results;
 
-      if (getEnv('transpilationWorkerThreads')) {
+      if (transpilationNative || transpilationWorkerThreads) {
         cubeNames = Object.keys(this.cubeDictionary.byId);
         // We need only cubes and all its member names for transpiling.
         // Cubes doesn't change during transpiling, but are changed during compilation phase,
         // so we can prepare them once for every phase.
         // Communication between main and worker threads uses
         // The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm)
         // which doesn't allow passing any function objects, so we need to sanitize the symbols.
-        cubeSymbolsNames = Object.fromEntries(
+        cubeSymbols = Object.fromEntries(
           Object.entries(this.cubeSymbols.symbols)
             .map(
               ([key, value]) => [key, Object.fromEntries(
                 Object.keys(value).map((k) => [k, true]),
               )],
             ),
         );
+
+        // Transpilers are the same for all files within phase.
+        transpilerNames = this.transpilers.map(t => t.constructor.name);
+      }
+
+      if (transpilationNative) {
+        // Warming up swc compiler cache
+        const dummyFile = {
+          fileName: 'dummy.js',
+          content: ';',
+        };
+
+        await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS });
+
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { transpilerNames })));
+      } else if (transpilationWorkerThreads) {
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
+      } else {
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {})));
       }
-      const results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames })));
+
       return results.filter(f => !!f);
     };
 
@@ -133,9 +159,19 @@ export class DataSchemaCompiler {
         contextCompilers: this.contextCompilers,
       }))
       .then(() => {
-        if (this.workerPool) {
+        if (transpilationNative) {
+          // Clean up cache
+          const dummyFile = {
+            fileName: 'terminate.js',
+            content: ';',
+          };
+
+          return this.transpileJsFile(dummyFile, errorsReport, { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {} });
+        } else if (transpilationWorkerThreads && this.workerPool) {
           this.workerPool.terminate();
         }
+
+        return Promise.resolve();
       });
   }
 
@@ -177,15 +213,35 @@ export class DataSchemaCompiler {
     }
   }
 
-  async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbolsNames }) {
+  async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbols, contextSymbols, transpilerNames }) {
     try {
-      if (getEnv('transpilationWorkerThreads')) {
+      if (getEnv('transpilationNative')) {
+        const reqData = {
+          fileName: file.fileName,
+          transpilers: transpilerNames,
+          ...(cubeNames && {
+            metaData: {
+              cubeNames,
+              cubeSymbols,
+              contextSymbols,
+            },
+          }),
+        };
+
+        errorsReport.inFile(file);
+        const res = await transpileJs(file.content, reqData);
+        errorsReport.addErrors(res.errors);
+        errorsReport.addWarnings(res.warnings);
+        errorsReport.exitFile();
+
+        return Object.assign({}, file, { content: res.code });
+      } else if (getEnv('transpilationWorkerThreads')) {
         const data = {
           fileName: file.fileName,
           content: file.content,
-          transpilers: this.transpilers.map(t => t.constructor.name),
+          transpilers: transpilerNames,
           cubeNames,
-          cubeSymbolsNames,
+          cubeSymbols,
         };
 
         const res = await this.workerPool.exec('transpile', [data]);
@@ -203,11 +259,11 @@ export class DataSchemaCompiler {
           },
         );
 
+        errorsReport.inFile(file);
         this.transpilers.forEach((t) => {
-          errorsReport.inFile(file);
           babelTraverse(ast, t.traverseObject(errorsReport));
-          errorsReport.exitFile();
         });
+        errorsReport.exitFile();
 
         const content = babelGenerator(ast, {}, file.content).code;
         return Object.assign({}, file, { content });

packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts

@@ -108,7 +108,7 @@ export class ErrorReporter {
   }
 
   public error(e: any, fileName?: any, lineNumber?: any, position?: any) {
-    const message = `${this.context.length ? `${this.context.join(' -> ')}: ` : ''}${e instanceof UserError ? e.message : (e.stack || e)}`;
+    const message = `${this.context.length ? `${this.context.join(' -> ')}: ` : ''}${e.message ? e.message : (e.stack || e)}`;
     if (this.rootReporter().errors.find(m => (m.message || m) === message)) {
       return;
     }
@@ -142,15 +142,15 @@ export class ErrorReporter {
   }
 
   public addErrors(errors: CompilerErrorInterface[]) {
-    this.rootReporter().errors.push(...errors);
+    errors.forEach((e: any) => { this.error(e); });
   }
 
   public getWarnings() {
     return this.rootReporter().warnings;
   }
 
-  public addWarnings(warnings: SyntaxErrorInterface[]) {
-    this.rootReporter().warnings.push(...warnings);
+  public addWarnings(warnings: any[]) {
+    warnings.forEach((w: any) => { this.warning(w); });
   }
 
   protected rootReporter(): ErrorReporter {