Skip to content

Commit 6e6a924

Browse files
KSDaemonKSDaemon
authored
feat(schema-compiler): Move transpiling to worker threads (under the … (#9191)
* feat(schema-compiler): Move transpiling to worker threads (under the flag) * update outdated actions
1 parent 2a0a5f2 commit 6e6a924

File tree

17 files changed

+249
-36
lines changed

17 files changed

+249
-36
lines changed

‎.github/workflows/publish.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ jobs:
191191
components: rustfmt
192192
target: ${{ matrix.target }}
193193
- name: Install Python
194-
uses: actions/setup-python@v4
194+
uses: actions/setup-python@v5
195195
if: (matrix.python-version != 'fallback')
196196
with:
197197
python-version: ${{ matrix.python-version }}
@@ -265,7 +265,7 @@ jobs:
265265
rustflags: ""
266266
components: rustfmt
267267
- name: Install Python
268-
uses: actions/setup-python@v4
268+
uses: actions/setup-python@v5
269269
if: (matrix.python-version != 'fallback')
270270
with:
271271
python-version: ${{ matrix.python-version }}
@@ -807,7 +807,7 @@ jobs:
807807
- name: Checkout
808808
uses: actions/checkout@v4
809809
- name: Trigger runtime
810-
uses: actions/github-script@v6
810+
uses: actions/github-script@v7
811811
with:
812812
github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
813813
script: |

‎.github/workflows/push.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,11 @@ jobs:
4242
matrix:
4343
# Current docker version + next LTS
4444
node-version: [20.x, 22.x]
45+
transpile-worker-threads: [false, true]
4546
fail-fast: false
4647

48+
env:
49+
CUBEJS_TRANSPILATION_WORKER_THREADS: ${{ matrix.transpile-worker-threads }}
4750
steps:
4851
- id: get-tag-out
4952
run: echo "$OUT"

‎.github/workflows/rust-cubesql.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ jobs:
245245
components: rustfmt
246246
target: ${{ matrix.target }}
247247
- name: Install Python
248-
uses: actions/setup-python@v4
248+
uses: actions/setup-python@v5
249249
if: (matrix.python-version != 'fallback')
250250
with:
251251
python-version: ${{ matrix.python-version }}
@@ -327,7 +327,7 @@ jobs:
327327
rustflags: ""
328328
components: rustfmt
329329
- name: Install Python
330-
uses: actions/setup-python@v4
330+
uses: actions/setup-python@v5
331331
if: (matrix.python-version != 'fallback')
332332
with:
333333
python-version: ${{ matrix.python-version }}

‎.github/workflows/rust-cubestore-master.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ jobs:
149149
- name: Set up Docker Buildx
150150
uses: docker/setup-buildx-action@v3
151151
- name: Push to Docker Hub
152-
uses: docker/build-push-action@v3
152+
uses: docker/build-push-action@v6
153153
with:
154154
context: ./rust/cubestore
155155
file: ./rust/cubestore/Dockerfile

‎.github/workflows/rust-cubestore.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797
- name: Set up Docker Buildx
9898
uses: docker/setup-buildx-action@v3
9999
- name: Build only
100-
uses: docker/build-push-action@v3
100+
uses: docker/build-push-action@v6
101101
with:
102102
context: ./rust/cubestore/
103103
file: ./rust/cubestore/Dockerfile

‎packages/cubejs-backend-shared/src/env.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,12 @@ const variables: Record<string, (...args: any) => any> = {
193193
.default('1')
194194
.asInt(),
195195
nativeSqlPlanner: () => get('CUBEJS_TESSERACT_SQL_PLANNER').asBool(),
196+
transpilationWorkerThreads: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS')
197+
.default('false')
198+
.asBoolStrict(),
199+
transpilationWorkerThreadsCount: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS_COUNT')
200+
.default('0')
201+
.asInt(),
196202

197203
/** ****************************************************************
198204
* Common db options *

‎packages/cubejs-schema-compiler/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@
5454
"node-dijkstra": "^2.5.0",
5555
"ramda": "^0.27.2",
5656
"syntax-error": "^1.3.0",
57-
"uuid": "^8.3.2"
57+
"uuid": "^8.3.2",
58+
"workerpool": "^9.2.0"
5859
},
5960
"devDependencies": {
6061
"@cubejs-backend/apla-clickhouse": "^1.7.0",

‎packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { camelizeCube } from './utils';
88
import { BaseQuery } from '../adapter';
99

1010
const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/;
11-
const CONTEXT_SYMBOLS = {
11+
export const CONTEXT_SYMBOLS = {
1212
SECURITY_CONTEXT: 'securityContext',
1313
// SECURITY_CONTEXT has been deprecated, however security_context (lowecase)
1414
// is allowed in RBAC policies for query-time attribute matching
@@ -19,7 +19,7 @@ const CONTEXT_SYMBOLS = {
1919
SQL_UTILS: 'sqlUtils'
2020
};
2121

22-
const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE'];
22+
export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE'];
2323

2424
export class CubeSymbols {
2525
constructor(evaluateViews) {

‎packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js‎

Lines changed: 78 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ import { parse } from '@babel/parser';
66
import babelGenerator from '@babel/generator';
77
import babelTraverse from '@babel/traverse';
88
import R from 'ramda';
9+
import workerpool from 'workerpool';
910

10-
import { isNativeSupported } from '@cubejs-backend/shared';
11+
import { getEnv, isNativeSupported } from '@cubejs-backend/shared';
1112
import { UserError } from './UserError';
1213
import { ErrorReporter } from './ErrorReporter';
1314

@@ -26,6 +27,8 @@ export class DataSchemaCompiler {
2627
this.preTranspileCubeCompilers = options.preTranspileCubeCompilers || [];
2728
this.cubeNameCompilers = options.cubeNameCompilers || [];
2829
this.extensions = options.extensions || {};
30+
this.cubeDictionary = options.cubeDictionary;
31+
this.cubeSymbols = options.cubeSymbols;
2932
this.cubeFactory = options.cubeFactory;
3033
this.filesToCompile = options.filesToCompile;
3134
this.omitErrors = options.omitErrors;
@@ -38,6 +41,7 @@ export class DataSchemaCompiler {
3841
this.yamlCompiler = options.yamlCompiler;
3942
this.yamlCompiler.dataSchemaCompiler = this;
4043
this.pythonContext = null;
44+
this.workerPool = null;
4145
}
4246

4347
compileObjects(compileServices, objects, errorsReport) {
@@ -87,17 +91,52 @@ export class DataSchemaCompiler {
8791
const errorsReport = new ErrorReporter(null, [], this.errorReport);
8892
this.errorsReport = errorsReport;
8993

90-
// TODO: required in order to get pre transpile compilation work
91-
const transpile = () => toCompile.map(f => this.transpileFile(f, errorsReport)).filter(f => !!f);
94+
if (getEnv('transpilationWorkerThreads')) {
95+
const wc = getEnv('transpilationWorkerThreadsCount');
96+
this.workerPool = workerpool.pool(
97+
path.join(__dirname, 'transpilers/transpiler_worker'),
98+
wc > 0 ? { maxWorkers: wc } : undefined,
99+
);
100+
}
101+
102+
const transpile = async () => {
103+
let cubeNames;
104+
let cubeSymbolsNames;
105+
106+
if (getEnv('transpilationWorkerThreads')) {
107+
cubeNames = Object.keys(this.cubeDictionary.byId);
108+
// We need only cubes and all its member names for transpiling.
109+
// Cubes doesn't change during transpiling, but are changed during compilation phase,
110+
// so we can prepare them once for every phase.
111+
// Communication between main and worker threads uses
112+
// The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm)
113+
// which doesn't allow passing any function objects, so we need to sanitize the symbols.
114+
cubeSymbolsNames = Object.fromEntries(
115+
Object.entries(this.cubeSymbols.symbols)
116+
.map(
117+
([key, value]) => [key, Object.fromEntries(
118+
Object.keys(value).map((k) => [k, true]),
119+
)],
120+
),
121+
);
122+
}
123+
const results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbolsNames })));
124+
return results.filter(f => !!f);
125+
};
92126

93-
const compilePhase = (compilers) => this.compileCubeFiles(compilers, transpile(), errorsReport);
127+
const compilePhase = async (compilers) => this.compileCubeFiles(compilers, await transpile(), errorsReport);
94128

95129
return compilePhase({ cubeCompilers: this.cubeNameCompilers })
96130
.then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers }))
97131
.then(() => compilePhase({
98132
cubeCompilers: this.cubeCompilers,
99133
contextCompilers: this.contextCompilers,
100-
}));
134+
}))
135+
.then(() => {
136+
if (this.workerPool) {
137+
this.workerPool.terminate();
138+
}
139+
});
101140
}
102141

103142
compile() {
@@ -113,7 +152,7 @@ export class DataSchemaCompiler {
113152
return this.compilePromise;
114153
}
115154

116-
transpileFile(file, errorsReport) {
155+
async transpileFile(file, errorsReport, options) {
117156
if (R.endsWith('.jinja', file.fileName) ||
118157
(R.endsWith('.yml', file.fileName) || R.endsWith('.yaml', file.fileName))
119158
// TODO do Jinja syntax check with jinja compiler
@@ -132,31 +171,47 @@ export class DataSchemaCompiler {
132171
} else if (R.endsWith('.yml', file.fileName) || R.endsWith('.yaml', file.fileName)) {
133172
return file;
134173
} else if (R.endsWith('.js', file.fileName)) {
135-
return this.transpileJsFile(file, errorsReport);
174+
return this.transpileJsFile(file, errorsReport, options);
136175
} else {
137176
return file;
138177
}
139178
}
140179

141-
transpileJsFile(file, errorsReport) {
180+
async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbolsNames }) {
142181
try {
143-
const ast = parse(
144-
file.content,
145-
{
146-
sourceFilename: file.fileName,
147-
sourceType: 'module',
148-
plugins: ['objectRestSpread']
149-
},
150-
);
182+
if (getEnv('transpilationWorkerThreads')) {
183+
const data = {
184+
fileName: file.fileName,
185+
content: file.content,
186+
transpilers: this.transpilers.map(t => t.constructor.name),
187+
cubeNames,
188+
cubeSymbolsNames,
189+
};
190+
191+
const res = await this.workerPool.exec('transpile', [data]);
192+
errorsReport.addErrors(res.errors);
193+
errorsReport.addWarnings(res.warnings);
194+
195+
return Object.assign({}, file, { content: res.content });
196+
} else {
197+
const ast = parse(
198+
file.content,
199+
{
200+
sourceFilename: file.fileName,
201+
sourceType: 'module',
202+
plugins: ['objectRestSpread'],
203+
},
204+
);
151205

152-
this.transpilers.forEach((t) => {
153-
errorsReport.inFile(file);
154-
babelTraverse(ast, t.traverseObject(errorsReport));
155-
errorsReport.exitFile();
156-
});
206+
this.transpilers.forEach((t) => {
207+
errorsReport.inFile(file);
208+
babelTraverse(ast, t.traverseObject(errorsReport));
209+
errorsReport.exitFile();
210+
});
157211

158-
const content = babelGenerator(ast, {}, file.content).code;
159-
return Object.assign({}, file, { content });
212+
const content = babelGenerator(ast, {}, file.content).code;
213+
return Object.assign({}, file, { content });
214+
}
160215
} catch (e) {
161216
if (e.toString().indexOf('SyntaxError') !== -1) {
162217
const line = file.content.split('\n')[e.loc.line - 1];

‎packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,22 @@ export class ErrorReporter {
137137
}
138138
}
139139

140+
public getErrors() {
141+
return this.rootReporter().errors;
142+
}
143+
144+
public addErrors(errors: CompilerErrorInterface[]) {
145+
this.rootReporter().errors.push(...errors);
146+
}
147+
148+
public getWarnings() {
149+
return this.rootReporter().warnings;
150+
}
151+
152+
public addWarnings(warnings: SyntaxErrorInterface[]) {
153+
this.rootReporter().warnings.push(...warnings);
154+
}
155+
140156
protected rootReporter(): ErrorReporter {
141157
return this.parent ? this.parent.rootReporter() : this;
142158
}

0 commit comments

Comments
 (0)