refactor(schema-compiler): Drop support for disabling native workers … (#10196)

This flag is enabled by default for a long time and there are not side effect. I am working on speeding up Cube startup and I've figure out that we load babel, which we don't need in the Cube Cloud Runtime, because it uses native transpilers. So, let's remove this env variable and speedup cloud as side effect.

---------

Co-authored-by: Konstantin Burkalev <[email protected]>

Author: ovr

.github/workflows/drivers-tests.yml

@@ -369,7 +369,6 @@ jobs:
           (contains(env.CLOUD_DATABASES, matrix.database) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
           (!contains(env.CLOUD_DATABASES, matrix.database))
         env:
-          DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS: true
           DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE: false
           DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_tesseract_sql_planner) || matrix.use_tesseract_sql_planner }}
 

docs/pages/product/configuration/reference/environment-variables.mdx

@@ -1467,21 +1467,6 @@ learn more.
 | --------------- | ---------------------- | --------------------- |
 | A valid number  | 86400                  | 86400                 |
 
-## `CUBEJS_TRANSPILATION_WORKER_THREADS`
-
-If `true`, optimizes data model compilation by running critical parts of the
-code in worker threads.
-
-| Possible Values | Default in Development | Default in Production |
-| --------------- | ---------------------- | --------------------- |
-| `true`, `false` | `true`                 | `true`                |
-
-<ReferenceBox>
-
-See [this issue](https://github.com/cube-js/cube/issues/9285) for details.
-
-</ReferenceBox>
-
 ## `CUBEJS_WEB_SOCKETS`
 
 If `true`, then use WebSocket for data fetching.
@@ -1911,4 +1896,4 @@ The port for a Cube deployment to listen to API connections on.
 [ref-mdx-api-locale]: /product/apis-integrations/mdx-api#measure-format
 [ref-time-dimensions]: /product/data-modeling/reference/dimensions#time
 [ref-time-zone]: /product/apis-integrations/queries#time-zone
-[link-tzdb]: https://en.wikipedia.org/wiki/Tz_database
+[link-tzdb]: https://en.wikipedia.org/wiki/Tz_database

packages/cubejs-backend-shared/src/env.ts

@@ -228,9 +228,19 @@ const variables: Record<string, (...args: any) => any> = {
   nativeOrchestrator: () => get('CUBEJS_TESSERACT_ORCHESTRATOR')
     .default('true')
     .asBoolStrict(),
-  transpilationWorkerThreads: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS')
-    .default('true')
-    .asBoolStrict(),
+  transpilationWorkerThreads: () => {
+    const enabled = get('CUBEJS_TRANSPILATION_WORKER_THREADS')
+      .default('true')
+      .asBoolStrict();
+
+    if (!enabled) {
+      console.warn(
+        'Worker thread transpilation is enabled by default and cannot be disabled with CUBEJS_TRANSPILATION_WORKER_THREADS.'
+      );
+    }
+
+    return true;
+  },
   allowNonStrictDateRangeMatching: () => get('CUBEJS_PRE_AGGREGATIONS_ALLOW_NON_STRICT_DATE_RANGE_MATCH')
     .default('true')
     .asBoolStrict(),
@@ -2186,6 +2196,9 @@ export function getEnv<T extends keyof Vars>(key: T, opts?: Parameters<Vars[T]>)
   );
 }
 
+// trigger warning
+getEnv('transpilationWorkerThreads');
+
 export function isDockerImage(): boolean {
   return Boolean(process.env.CUBEJS_DOCKER_IMAGE_TAG);
 }

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -5,9 +5,6 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import syntaxCheck from 'syntax-error';
-import { parse } from '@babel/parser';
-import babelGenerator from '@babel/generator';
-import babelTraverse from '@babel/traverse';
 import R from 'ramda';
 import workerpool from 'workerpool';
 import { LRUCache } from 'lru-cache';
@@ -275,12 +272,11 @@ export class DataSchemaCompiler {
     const errorsReport = new ErrorReporter(null, [], this.errorReportOptions);
     this.errorsReporter = errorsReport;
 
-    const transpilationWorkerThreads = getEnv('transpilationWorkerThreads');
     const transpilationNative = getEnv('transpilationNative');
     const transpilationNativeThreadsCount = getThreadsCount();
     const { compilerId } = this;
 
-    if (transpilationWorkerThreads) {
+    if (!transpilationNative) {
       const wc = getEnv('transpilationWorkerThreadsCount');
       this.workerPool = workerpool.pool(
         path.join(__dirname, 'transpilers/transpiler_worker'),
@@ -292,11 +288,10 @@ export class DataSchemaCompiler {
       let cubeNames: string[] = [];
       let cubeSymbols: Record<string, Record<string, boolean>> = {};
       let transpilerNames: string[] = [];
-      let results: (FileContent | undefined)[];
 
-      if (transpilationNative || transpilationWorkerThreads) {
-        ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols());
-      }
+      ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols());
+
+      let results: (FileContent | undefined)[];
 
       if (transpilationNative) {
         const jsFiles = originalJsFiles;
@@ -325,28 +320,25 @@ export class DataSchemaCompiler {
           .map(f => this.transpileJinjaFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }));
 
         results = (await Promise.all([...jsFilesTasks, ...yamlFilesTasks, ...jinjaFilesTasks])).flat();
-      } else if (transpilationWorkerThreads) {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {})));
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       }
 
       return results.filter(f => !!f) as FileContent[];
     };
 
     const transpilePhase = async (stage: CompileStage): Promise<FileContent[]> => {
-      let cubeNames: string[] = [];
-      let cubeSymbols: Record<string, Record<string, boolean>> = {};
-      let transpilerNames: string[] = [];
       let results: (FileContent | undefined)[];
 
       if (toCompile.length === 0) {
         return [];
       }
 
-      if (transpilationNative || transpilationWorkerThreads) {
-        ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols());
-      }
+      let cubeNames: string[] = [];
+      let cubeSymbols: Record<string, Record<string, boolean>> = {};
+      let transpilerNames: string[] = [];
+
+      ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols());
 
       // After the first phase all files are with JS source code: original or transpiled
 
@@ -363,10 +355,8 @@ export class DataSchemaCompiler {
         const jsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesNativeBulk(chunk, errorsReport, { transpilerNames, compilerId }));
 
         results = (await Promise.all(jsFilesTasks)).flat();
-      } else if (transpilationWorkerThreads) {
-        results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
-        results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, {})));
+        results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       }
 
       return results.filter(f => !!f) as FileContent[];
@@ -520,7 +510,7 @@ export class DataSchemaCompiler {
             errorsReport,
             { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {}, compilerId: this.compilerId, stage: 0 }
           ).then(() => undefined);
-        } else if (transpilationWorkerThreads && this.workerPool) {
+        } else if (this.workerPool) {
           this.workerPool.terminate();
         }
       });
@@ -701,7 +691,7 @@ export class DataSchemaCompiler {
         errorsReport.exitFile();
 
         return { ...file, content: res[0].code };
-      } else if (getEnv('transpilationWorkerThreads')) {
+      } else {
         const data = {
           fileName: file.fileName,
           content: file.content,
@@ -715,25 +705,6 @@ export class DataSchemaCompiler {
         errorsReport.addWarnings(res.warnings);
 
         return { ...file, content: res.content };
-      } else {
-        const ast = parse(
-          file.content,
-          {
-            sourceFilename: file.fileName,
-            sourceType: 'module',
-            plugins: ['objectRestSpread'],
-          },
-        );
-
-        errorsReport.inFile(file);
-        this.transpilers.forEach((t) => {
-          babelTraverse(ast, t.traverseObject(errorsReport));
-        });
-        errorsReport.exitFile();
-
-        const content = babelGenerator(ast, {}, file.content).code;
-
-        return { ...file, content };
       }
     } catch (e: any) {
       if (e.toString().indexOf('SyntaxError') !== -1) {
@@ -778,7 +749,7 @@ export class DataSchemaCompiler {
       this.compiledYamlCache.set(cacheKey, res[0].code);
 
       return { ...file, content: res[0].code };
-    } else if (getEnv('transpilationWorkerThreads')) {
+    } else {
       const data = {
         fileName: file.fileName,
         content: file.content,
@@ -794,12 +765,6 @@ export class DataSchemaCompiler {
       this.compiledYamlCache.set(cacheKey, res.content);
 
       return { ...file, content: res.content };
-    } else {
-      const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
-
-      this.compiledYamlCache.set(cacheKey, transpiledFile?.content || '');
-
-      return transpiledFile;
     }
   }
 

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -17,7 +17,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "ports" : ["4000", "5656"]

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -19,7 +19,6 @@
       "CUBEJS_DB_EXPORT_BUCKET": "cube-open-source-export-bucket",
       "CUBEJS_DB_EXPORT_BUCKET_TYPE": "gcp",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "ports" : ["4000", "5656"]

packages/cubejs-testing-drivers/fixtures/clickhouse.json

@@ -38,7 +38,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["data"],

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -79,7 +79,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "ports" : ["4000", "5656"]

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -13,7 +13,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["data"],

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -14,7 +14,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["data"],