diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 1d52d56fceaa3..8f509a3f92d49 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -369,7 +369,6 @@ jobs: (contains(env.CLOUD_DATABASES, matrix.database) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') || (!contains(env.CLOUD_DATABASES, matrix.database)) env: - DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS: true DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE: false DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_tesseract_sql_planner) || matrix.use_tesseract_sql_planner }} diff --git a/docs/pages/product/configuration/reference/environment-variables.mdx b/docs/pages/product/configuration/reference/environment-variables.mdx index a816f7284d466..09d02e574d282 100644 --- a/docs/pages/product/configuration/reference/environment-variables.mdx +++ b/docs/pages/product/configuration/reference/environment-variables.mdx @@ -1467,21 +1467,6 @@ learn more. | --------------- | ---------------------- | --------------------- | | A valid number | 86400 | 86400 | -## `CUBEJS_TRANSPILATION_WORKER_THREADS` - -If `true`, optimizes data model compilation by running critical parts of the -code in worker threads. - -| Possible Values | Default in Development | Default in Production | -| --------------- | ---------------------- | --------------------- | -| `true`, `false` | `true` | `true` | - - - -See [this issue](https://github.com/cube-js/cube/issues/9285) for details. - - - ## `CUBEJS_WEB_SOCKETS` If `true`, then use WebSocket for data fetching. @@ -1911,4 +1896,4 @@ The port for a Cube deployment to listen to API connections on. [ref-mdx-api-locale]: /product/apis-integrations/mdx-api#measure-format [ref-time-dimensions]: /product/data-modeling/reference/dimensions#time [ref-time-zone]: /product/apis-integrations/queries#time-zone -[link-tzdb]: https://en.wikipedia.org/wiki/Tz_database \ No newline at end of file +[link-tzdb]: https://en.wikipedia.org/wiki/Tz_database diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 2d84e5cf56ee5..79aabbf86f085 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -228,9 +228,19 @@ const variables: Record any> = { nativeOrchestrator: () => get('CUBEJS_TESSERACT_ORCHESTRATOR') .default('true') .asBoolStrict(), - transpilationWorkerThreads: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS') - .default('true') - .asBoolStrict(), + transpilationWorkerThreads: () => { + const enabled = get('CUBEJS_TRANSPILATION_WORKER_THREADS') + .default('true') + .asBoolStrict(); + + if (!enabled) { + console.warn( + 'Worker thread transpilation is enabled by default and cannot be disabled with CUBEJS_TRANSPILATION_WORKER_THREADS.' + ); + } + + return true; + }, allowNonStrictDateRangeMatching: () => get('CUBEJS_PRE_AGGREGATIONS_ALLOW_NON_STRICT_DATE_RANGE_MATCH') .default('true') .asBoolStrict(), @@ -2186,6 +2196,9 @@ export function getEnv(key: T, opts?: Parameters) ); } +// trigger warning +getEnv('transpilationWorkerThreads'); + export function isDockerImage(): boolean { return Boolean(process.env.CUBEJS_DOCKER_IMAGE_TAG); } diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 83fc686ce9bdc..40267faa84a57 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -5,9 +5,6 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import syntaxCheck from 'syntax-error'; -import { parse } from '@babel/parser'; -import babelGenerator from '@babel/generator'; -import babelTraverse from '@babel/traverse'; import R from 'ramda'; import workerpool from 'workerpool'; import { LRUCache } from 'lru-cache'; @@ -275,12 +272,11 @@ export class DataSchemaCompiler { const errorsReport = new ErrorReporter(null, [], this.errorReportOptions); this.errorsReporter = errorsReport; - const transpilationWorkerThreads = getEnv('transpilationWorkerThreads'); const transpilationNative = getEnv('transpilationNative'); const transpilationNativeThreadsCount = getThreadsCount(); const { compilerId } = this; - if (transpilationWorkerThreads) { + if (!transpilationNative) { const wc = getEnv('transpilationWorkerThreadsCount'); this.workerPool = workerpool.pool( path.join(__dirname, 'transpilers/transpiler_worker'), @@ -292,11 +288,10 @@ export class DataSchemaCompiler { let cubeNames: string[] = []; let cubeSymbols: Record> = {}; let transpilerNames: string[] = []; - let results: (FileContent | undefined)[]; - if (transpilationNative || transpilationWorkerThreads) { - ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols()); - } + ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols()); + + let results: (FileContent | undefined)[]; if (transpilationNative) { const jsFiles = originalJsFiles; @@ -325,28 +320,25 @@ export class DataSchemaCompiler { .map(f => this.transpileJinjaFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })); results = (await Promise.all([...jsFilesTasks, ...yamlFilesTasks, ...jinjaFilesTasks])).flat(); - } else if (transpilationWorkerThreads) { - results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }))); } else { - results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {}))); + results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }))); } return results.filter(f => !!f) as FileContent[]; }; const transpilePhase = async (stage: CompileStage): Promise => { - let cubeNames: string[] = []; - let cubeSymbols: Record> = {}; - let transpilerNames: string[] = []; let results: (FileContent | undefined)[]; if (toCompile.length === 0) { return []; } - if (transpilationNative || transpilationWorkerThreads) { - ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols()); - } + let cubeNames: string[] = []; + let cubeSymbols: Record> = {}; + let transpilerNames: string[] = []; + + ({ cubeNames, cubeSymbols, transpilerNames } = this.prepareTranspileSymbols()); // After the first phase all files are with JS source code: original or transpiled @@ -363,10 +355,8 @@ export class DataSchemaCompiler { const jsFilesTasks = jsChunks.map(chunk => this.transpileJsFilesNativeBulk(chunk, errorsReport, { transpilerNames, compilerId })); results = (await Promise.all(jsFilesTasks)).flat(); - } else if (transpilationWorkerThreads) { - results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }))); } else { - results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, {}))); + results = await Promise.all(toCompile.map(f => this.transpileJsFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }))); } return results.filter(f => !!f) as FileContent[]; @@ -520,7 +510,7 @@ export class DataSchemaCompiler { errorsReport, { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {}, compilerId: this.compilerId, stage: 0 } ).then(() => undefined); - } else if (transpilationWorkerThreads && this.workerPool) { + } else if (this.workerPool) { this.workerPool.terminate(); } }); @@ -701,7 +691,7 @@ export class DataSchemaCompiler { errorsReport.exitFile(); return { ...file, content: res[0].code }; - } else if (getEnv('transpilationWorkerThreads')) { + } else { const data = { fileName: file.fileName, content: file.content, @@ -715,25 +705,6 @@ export class DataSchemaCompiler { errorsReport.addWarnings(res.warnings); return { ...file, content: res.content }; - } else { - const ast = parse( - file.content, - { - sourceFilename: file.fileName, - sourceType: 'module', - plugins: ['objectRestSpread'], - }, - ); - - errorsReport.inFile(file); - this.transpilers.forEach((t) => { - babelTraverse(ast, t.traverseObject(errorsReport)); - }); - errorsReport.exitFile(); - - const content = babelGenerator(ast, {}, file.content).code; - - return { ...file, content }; } } catch (e: any) { if (e.toString().indexOf('SyntaxError') !== -1) { @@ -778,7 +749,7 @@ export class DataSchemaCompiler { this.compiledYamlCache.set(cacheKey, res[0].code); return { ...file, content: res[0].code }; - } else if (getEnv('transpilationWorkerThreads')) { + } else { const data = { fileName: file.fileName, content: file.content, @@ -794,12 +765,6 @@ export class DataSchemaCompiler { this.compiledYamlCache.set(cacheKey, res.content); return { ...file, content: res.content }; - } else { - const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport); - - this.compiledYamlCache.set(cacheKey, transpiledFile?.content || ''); - - return transpiledFile; } } diff --git a/packages/cubejs-testing-drivers/fixtures/athena.json b/packages/cubejs-testing-drivers/fixtures/athena.json index 988029d607cbd..271213ac1e52f 100644 --- a/packages/cubejs-testing-drivers/fixtures/athena.json +++ b/packages/cubejs-testing-drivers/fixtures/athena.json @@ -17,7 +17,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "ports" : ["4000", "5656"] diff --git a/packages/cubejs-testing-drivers/fixtures/bigquery.json b/packages/cubejs-testing-drivers/fixtures/bigquery.json index 3f628e10bdda0..5627d85fb3204 100644 --- a/packages/cubejs-testing-drivers/fixtures/bigquery.json +++ b/packages/cubejs-testing-drivers/fixtures/bigquery.json @@ -19,7 +19,6 @@ "CUBEJS_DB_EXPORT_BUCKET": "cube-open-source-export-bucket", "CUBEJS_DB_EXPORT_BUCKET_TYPE": "gcp", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "ports" : ["4000", "5656"] diff --git a/packages/cubejs-testing-drivers/fixtures/clickhouse.json b/packages/cubejs-testing-drivers/fixtures/clickhouse.json index 7649744f2d5e8..0577f89943acb 100644 --- a/packages/cubejs-testing-drivers/fixtures/clickhouse.json +++ b/packages/cubejs-testing-drivers/fixtures/clickhouse.json @@ -38,7 +38,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "depends_on": ["data"], diff --git a/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json b/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json index 891f07703c780..5cdd16f7e6b90 100644 --- a/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json +++ b/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json @@ -79,7 +79,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "ports" : ["4000", "5656"] diff --git a/packages/cubejs-testing-drivers/fixtures/mssql.json b/packages/cubejs-testing-drivers/fixtures/mssql.json index 337e725466bb1..5b233fb5ace72 100644 --- a/packages/cubejs-testing-drivers/fixtures/mssql.json +++ b/packages/cubejs-testing-drivers/fixtures/mssql.json @@ -13,7 +13,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "depends_on": ["data"], diff --git a/packages/cubejs-testing-drivers/fixtures/mysql.json b/packages/cubejs-testing-drivers/fixtures/mysql.json index c0dd56f4690a2..30cbcffa1df4f 100644 --- a/packages/cubejs-testing-drivers/fixtures/mysql.json +++ b/packages/cubejs-testing-drivers/fixtures/mysql.json @@ -14,7 +14,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "depends_on": ["data"], diff --git a/packages/cubejs-testing-drivers/fixtures/postgres.json b/packages/cubejs-testing-drivers/fixtures/postgres.json index 50d09f2052adc..665e4cc4eb676 100644 --- a/packages/cubejs-testing-drivers/fixtures/postgres.json +++ b/packages/cubejs-testing-drivers/fixtures/postgres.json @@ -14,7 +14,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "depends_on": ["data"], diff --git a/packages/cubejs-testing-drivers/fixtures/redshift.json b/packages/cubejs-testing-drivers/fixtures/redshift.json index 6d6f027e3202b..ac1c913d8fbc9 100644 --- a/packages/cubejs-testing-drivers/fixtures/redshift.json +++ b/packages/cubejs-testing-drivers/fixtures/redshift.json @@ -32,7 +32,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "ports" : ["4000", "5656"] diff --git a/packages/cubejs-testing-drivers/fixtures/snowflake.json b/packages/cubejs-testing-drivers/fixtures/snowflake.json index 3ce4909af95c0..1a505f721cffd 100644 --- a/packages/cubejs-testing-drivers/fixtures/snowflake.json +++ b/packages/cubejs-testing-drivers/fixtures/snowflake.json @@ -118,7 +118,6 @@ "CUBEJS_SQL_PASSWORD": "admin_password", "CUBESQL_SQL_PUSH_DOWN": "true", "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}", - "CUBEJS_TRANSPILATION_WORKER_THREADS": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_WORKER_THREADS}", "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}" }, "ports" : ["4000", "5656"]