diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index 154c950e1066d..ea230df9fb0b2 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -1,3 +1,4 @@ +import { AsyncLocalStorage } from 'async_hooks'; import crypto from 'crypto'; import vm from 'vm'; import fs from 'fs'; @@ -15,7 +16,7 @@ import { FileContent, getEnv, isNativeSupported, SchemaFileRepository } from '@c import { NativeInstance, PythonCtx, transpileJs } from '@cubejs-backend/native'; import { UserError } from './UserError'; import { ErrorReporter, ErrorReporterOptions, SyntaxErrorInterface } from './ErrorReporter'; -import { CONTEXT_SYMBOLS, CubeSymbols } from './CubeSymbols'; +import { CONTEXT_SYMBOLS, CubeDefinition, CubeSymbols } from './CubeSymbols'; import { ViewCompilationGate } from './ViewCompilationGate'; import { TranspilerInterface } from './transpilers'; import { CompilerInterface } from './PrepareCompiler'; @@ -23,6 +24,8 @@ import { YamlCompiler } from './YamlCompiler'; import { CubeDictionary } from './CubeDictionary'; import { CompilerCache } from './CompilerCache'; +const ctxFileStorage = new AsyncLocalStorage(); + const NATIVE_IS_SUPPORTED = isNativeSupported(); const moduleFileCache = {}; @@ -140,6 +143,8 @@ export class DataSchemaCompiler { private readonly compiledScriptCache: LRUCache; + private compileV8ContextCache: vm.Context | null = null; + // FIXME: Is public only because of tests, should be private public compilePromise: any; @@ -229,11 +234,11 @@ export class DataSchemaCompiler { ); } - const transpile = async (stage: CompileStage) => { + const transpile = async (stage: CompileStage): Promise => { let cubeNames: string[] = []; let cubeSymbols: Record> = {}; let transpilerNames: string[] = []; - let results; + let results: (FileContent | undefined)[]; if (transpilationNative || transpilationWorkerThreads) { cubeNames = Object.keys(this.cubeDictionary.byId); @@ -296,10 +301,113 @@ export class DataSchemaCompiler { results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {}))); } - return results.filter(f => !!f); + return results.filter(f => !!f) as FileContent[]; }; - const compilePhase = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => this.compileCubeFiles(compilers, await transpile(stage), errorsReport); + let cubes: CubeDefinition[] = []; + let exports: Record> = {}; + let contexts: Record[] = []; + let compiledFiles: Record = {}; + let asyncModules: CallableFunction[] = []; + let transpiledFiles: FileContent[] = []; + + this.compileV8ContextCache = vm.createContext({ + view: (name, cube) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + return !cube ? + this.cubeFactory({ ...name, fileName: file.fileName, isView: true }) : + cubes.push({ ...cube, name, fileName: file.fileName, isView: true }); + }, + cube: (name, cube) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + return !cube ? + this.cubeFactory({ ...name, fileName: file.fileName }) : + cubes.push({ ...cube, name, fileName: file.fileName }); + }, + context: (name: string, context) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + return contexts.push({ ...context, name, fileName: file.fileName }); + }, + addExport: (obj) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + exports[file.fileName] = exports[file.fileName] || {}; + exports[file.fileName] = Object.assign(exports[file.fileName], obj); + }, + setExport: (obj) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + exports[file.fileName] = obj; + }, + asyncModule: (fn) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + // We need to run async module code in the context of the original data model file + // where it was defined. So we pass the same file to the async context. + // @see https://nodejs.org/api/async_context.html#class-asynclocalstorage + asyncModules.push(async () => ctxFileStorage.run(file, () => fn())); + }, + require: (extensionName: string) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + + if (this.extensions[extensionName]) { + return new (this.extensions[extensionName])(this.cubeFactory, this, cubes); + } else { + const foundFile = this.resolveModuleFile(file, extensionName, transpiledFiles, errorsReport); + if (!foundFile && this.allowNodeRequire) { + if (extensionName.indexOf('.') === 0) { + extensionName = path.resolve(this.repository.localPath(), extensionName); + } + // eslint-disable-next-line global-require,import/no-dynamic-require + const Extension = require(extensionName); + if (Object.getPrototypeOf(Extension).name === 'AbstractExtension') { + return new Extension(this.cubeFactory, this, cubes); + } + return Extension; + } + this.compileFile( + foundFile, + errorsReport, + compiledFiles, + [], + { doSyntaxCheck: true } + ); + exports[foundFile.fileName] = exports[foundFile.fileName] || {}; + return exports[foundFile.fileName]; + } + }, + COMPILE_CONTEXT: this.standalone ? this.standaloneCompileContextProxy() : this.cloneCompileContextWithGetterAlias(this.compileContext || {}), + }); + + const compilePhase = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => { + // clear the objects for the next phase + cubes = []; + exports = {}; + contexts = []; + compiledFiles = {}; + asyncModules = []; + transpiledFiles = await transpile(stage); + + return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport); + }; return compilePhase({ cubeCompilers: this.cubeNameCompilers }, 0) .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers.concat([this.viewCompilationGate]) }, 1)) @@ -311,6 +419,14 @@ export class DataSchemaCompiler { contextCompilers: this.contextCompilers, }, 3)) .then(() => { + // Free unneeded resources + cubes = []; + exports = {}; + contexts = []; + compiledFiles = {}; + asyncModules = []; + transpiledFiles = []; + if (transpilationNative) { // Clean up cache const dummyFile = { @@ -336,6 +452,7 @@ export class DataSchemaCompiler { this.throwIfAnyErrors(); } // Free unneeded resources + this.compileV8ContextCache = null; this.cubeDictionary.free(); this.cubeSymbols.free(); return res; @@ -345,7 +462,11 @@ export class DataSchemaCompiler { return this.compilePromise; } - private async transpileFile(file: FileContent, errorsReport: ErrorReporter, options: TranspileOptions = {}) { + private async transpileFile( + file: FileContent, + errorsReport: ErrorReporter, + options: TranspileOptions = {} + ): Promise<(FileContent | undefined)> { if (file.fileName.endsWith('.jinja') || (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) // TODO do Jinja syntax check with jinja compiler @@ -374,7 +495,11 @@ export class DataSchemaCompiler { * Right now it is used only for transpilation in native, * so no checks for transpilation type inside this method */ - private async transpileJsFilesBulk(files: FileContent[], errorsReport: ErrorReporter, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) { + private async transpileJsFilesBulk( + files: FileContent[], + errorsReport: ErrorReporter, + { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions + ): Promise<(FileContent | undefined)[]> { // for bulk processing this data may be optimized even more by passing transpilerNames, compilerId only once for a bulk // but this requires more complex logic to be implemented in the native side. // And comparing to the file content sizes, a few bytes of JSON data is not a big deal here @@ -408,7 +533,11 @@ export class DataSchemaCompiler { }); } - private async transpileJsFile(file: FileContent, errorsReport: ErrorReporter, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) { + private async transpileJsFile( + file: FileContent, + errorsReport: ErrorReporter, + { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions + ): Promise<(FileContent | undefined)> { try { if (getEnv('transpilationNative')) { const reqData = { @@ -493,22 +622,20 @@ export class DataSchemaCompiler { return this.currentQuery; } - private async compileCubeFiles(compilers: CompileCubeFilesCompilers, toCompile: FileContent[], errorsReport: ErrorReporter) { - const cubes = []; - const exports = {}; - const contexts = []; - const compiledFiles = {}; - const asyncModules = []; - + private async compileCubeFiles( + cubes: CubeDefinition[], + contexts: Record[], + compiledFiles: Record, + asyncModules: CallableFunction[], + compilers: CompileCubeFilesCompilers, + toCompile: FileContent[], + errorsReport: ErrorReporter + ) { toCompile .forEach((file) => { this.compileFile( file, errorsReport, - cubes, - exports, - contexts, - toCompile, compiledFiles, asyncModules ); @@ -523,7 +650,11 @@ export class DataSchemaCompiler { } private compileFile( - file: FileContent, errorsReport: ErrorReporter, cubes, exports, contexts, toCompile, compiledFiles, asyncModules, { doSyntaxCheck } = { doSyntaxCheck: false } + file: FileContent, + errorsReport: ErrorReporter, + compiledFiles: Record, + asyncModules: CallableFunction[], + { doSyntaxCheck } = { doSyntaxCheck: false } ) { if (compiledFiles[file.fileName]) { return; @@ -532,7 +663,7 @@ export class DataSchemaCompiler { compiledFiles[file.fileName] = true; if (file.fileName.endsWith('.js')) { - this.compileJsFile(file, errorsReport, cubes, contexts, exports, asyncModules, toCompile, compiledFiles, { doSyntaxCheck }); + this.compileJsFile(file, errorsReport, { doSyntaxCheck }); } else if (file.fileName.endsWith('.yml.jinja') || file.fileName.endsWith('.yaml.jinja') || ( file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml') @@ -542,17 +673,11 @@ export class DataSchemaCompiler { asyncModules.push(() => this.yamlCompiler.compileYamlWithJinjaFile( file, errorsReport, - cubes, - contexts, - exports, - asyncModules, - toCompile, - compiledFiles, this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext), this.pythonContext! )); } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) { - this.yamlCompiler.compileYamlFile(file, errorsReport, cubes, contexts, exports, asyncModules, toCompile, compiledFiles); + this.yamlCompiler.compileYamlFile(file, errorsReport); } } @@ -568,7 +693,11 @@ export class DataSchemaCompiler { return script; } - public compileJsFile(file: FileContent, errorsReport: ErrorReporter, cubes, contexts, exports, asyncModules, toCompile, compiledFiles, { doSyntaxCheck } = { doSyntaxCheck: false }) { + public compileJsFile( + file: FileContent, + errorsReport: ErrorReporter, + { doSyntaxCheck } = { doSyntaxCheck: false } + ) { if (doSyntaxCheck) { // There is no need to run syntax check for data model files // because they were checked during transpilation/transformation phase @@ -582,62 +711,12 @@ export class DataSchemaCompiler { try { const script = this.getJsScript(file); - script.runInNewContext({ - view: (name, cube) => ( - !cube ? - this.cubeFactory({ ...name, fileName: file.fileName, isView: true }) : - cubes.push({ ...cube, name, fileName: file.fileName, isView: true }) - ), - cube: - (name, cube) => ( - !cube ? - this.cubeFactory({ ...name, fileName: file.fileName }) : - cubes.push({ ...cube, name, fileName: file.fileName }) - ), - context: (name, context) => contexts.push({ ...context, name, fileName: file.fileName }), - addExport: (obj) => { - exports[file.fileName] = exports[file.fileName] || {}; - exports[file.fileName] = Object.assign(exports[file.fileName], obj); - }, - setExport: (obj) => { - exports[file.fileName] = obj; - }, - asyncModule: (fn) => { - asyncModules.push(fn); - }, - require: (extensionName) => { - if (this.extensions[extensionName]) { - return new (this.extensions[extensionName])(this.cubeFactory, this, cubes); - } else { - const foundFile = this.resolveModuleFile(file, extensionName, toCompile, errorsReport); - if (!foundFile && this.allowNodeRequire) { - if (extensionName.indexOf('.') === 0) { - extensionName = path.resolve(this.repository.localPath(), extensionName); - } - // eslint-disable-next-line global-require,import/no-dynamic-require - const Extension = require(extensionName); - if (Object.getPrototypeOf(Extension).name === 'AbstractExtension') { - return new Extension(this.cubeFactory, this, cubes); - } - return Extension; - } - this.compileFile( - foundFile, - errorsReport, - cubes, - exports, - contexts, - toCompile, - compiledFiles, - [], - { doSyntaxCheck: true } - ); - exports[foundFile.fileName] = exports[foundFile.fileName] || {}; - return exports[foundFile.fileName]; - } - }, - COMPILE_CONTEXT: this.standalone ? this.standaloneCompileContextProxy() : this.cloneCompileContextWithGetterAlias(this.compileContext || {}), - }, { filename: file.fileName, timeout: 15000 }); + // We use AsyncLocalStorage to store the current file context + // so that it can be accessed in the script execution context even within async functions. + // @see https://nodejs.org/api/async_context.html#class-asynclocalstorage + ctxFileStorage.run(file, () => { + script.runInContext(this.compileV8ContextCache!, { timeout: 15000 }); + }); } catch (e) { errorsReport.error(e); } diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 75fc6025929f8..a4cf1a26df922 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -9,6 +9,7 @@ import { DataSchemaCompiler } from './DataSchemaCompiler'; import { CubeCheckDuplicatePropTranspiler, CubePropContextTranspiler, + IIFETranspiler, ImportExportTranspiler, TranspilerInterface, ValidationTranspiler, @@ -63,6 +64,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp new ValidationTranspiler(), new ImportExportTranspiler(), new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler), + new IIFETranspiler(), ]; if (!options.allowJsDuplicatePropsInSchema) { diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index c36b648500244..32e919f8601f2 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -66,30 +66,18 @@ export class YamlCompiler { public async compileYamlWithJinjaFile( file: FileContent, errorsReport: ErrorReporter, - cubes, - contexts, - exports, - asyncModules, - toCompile, - compiledFiles, compileContext, pythonContext: PythonCtx ) { const compiledFile = await this.renderTemplate(file, compileContext, pythonContext); - return this.compileYamlFile( - compiledFile, - errorsReport, - cubes, - contexts, - exports, - asyncModules, - toCompile, - compiledFiles - ); + return this.compileYamlFile(compiledFile, errorsReport); } - public compileYamlFile(file: FileContent, errorsReport: ErrorReporter, cubes, contexts, exports, asyncModules, toCompile, compiledFiles) { + public compileYamlFile( + file: FileContent, + errorsReport: ErrorReporter, + ) { if (!file.content.trim()) { return; } @@ -103,12 +91,12 @@ export class YamlCompiler { if (key === 'cubes') { (yamlObj.cubes || []).forEach(({ name, ...cube }) => { const transpiledFile = this.transpileAndPrepareJsFile(file, 'cube', { name, ...cube }, errorsReport); - this.dataSchemaCompiler?.compileJsFile(transpiledFile, errorsReport, cubes, contexts, exports, asyncModules, toCompile, compiledFiles); + this.dataSchemaCompiler?.compileJsFile(transpiledFile, errorsReport); }); } else if (key === 'views') { (yamlObj.views || []).forEach(({ name, ...cube }) => { const transpiledFile = this.transpileAndPrepareJsFile(file, 'view', { name, ...cube }, errorsReport); - this.dataSchemaCompiler?.compileJsFile(transpiledFile, errorsReport, cubes, contexts, exports, asyncModules, toCompile, compiledFiles); + this.dataSchemaCompiler?.compileJsFile(transpiledFile, errorsReport); }); } else { errorsReport.error(`Unexpected YAML key: ${key}. Only 'cubes' and 'views' are allowed here.`); diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/IIFETranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/IIFETranspiler.ts new file mode 100644 index 0000000000000..e3f562c0dbf70 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/IIFETranspiler.ts @@ -0,0 +1,36 @@ +import * as t from '@babel/types'; +import type { NodePath } from '@babel/traverse'; +import type { TranspilerInterface, TraverseObject } from './transpiler.interface'; +import type { ErrorReporter } from '../ErrorReporter'; + +/** + * IIFETranspiler wraps the entire file content in an Immediately Invoked Function Expression (IIFE). + * This prevents: + * - Variable redeclaration errors when multiple files define the same variables + * - Global scope pollution between data model files + * - Provides isolated execution context for each file + */ +export class IIFETranspiler implements TranspilerInterface { + public traverseObject(_reporter: ErrorReporter): TraverseObject { + return { + Program: (path: NodePath) => { + const { body } = path.node; + + if (body.length > 0) { + // Create an IIFE that wraps all the existing statements + const iife = t.callExpression( + t.functionExpression( + null, // anonymous function + [], + t.blockStatement(body) + ), + [] + ); + + // Replace the program body with the IIFE + path.node.body = [t.expressionStatement(iife)]; + } + } + }; + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/ImportExportTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/ImportExportTranspiler.ts index 43d09f7ad18f0..1eaf6a0321b02 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/ImportExportTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/ImportExportTranspiler.ts @@ -69,9 +69,9 @@ export class ImportExportTranspiler implements TranspilerInterface { path.replaceWithMultiple([ decl.node, - t.callExpression(t.identifier('addExport'), [ + t.expressionStatement(t.callExpression(t.identifier('addExport'), [ t.objectExpression([t.objectProperty(name, name)]) - ]) + ])) ]); return; } @@ -80,12 +80,12 @@ export class ImportExportTranspiler implements TranspilerInterface { if (t.isVariableDeclaration(decl.node)) { path.replaceWithMultiple([ decl.node, - t.callExpression(t.identifier('addExport'), [ + t.expressionStatement(t.callExpression(t.identifier('addExport'), [ t.objectExpression( // @ts-ignore decl.get('declarations').map(d => t.objectProperty(d.get('id').node, d.get('id').node)) ) - ]) + ])) ]); return; } @@ -97,12 +97,12 @@ export class ImportExportTranspiler implements TranspilerInterface { return; } - const addExportCall = t.callExpression(t.identifier('addExport'), [t.objectExpression(declarations)]); + const addExportCall = t.expressionStatement(t.callExpression(t.identifier('addExport'), [t.objectExpression(declarations)])); path.replaceWith(addExportCall); }, ExportDefaultDeclaration(path) { // @ts-ignore - path.replaceWith(t.callExpression(t.identifier('setExport'), [path.get('declaration').node])); + path.replaceWith(t.expressionStatement(t.callExpression(t.identifier('setExport'), [path.get('declaration').node]))); } }; } diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/index.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/index.ts index 8ead359784183..fbcb5c394f991 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/index.ts @@ -2,4 +2,5 @@ export * from './ImportExportTranspiler'; export * from './CubePropContextTranspiler'; export * from './CubeCheckDuplicatePropTranspiler'; export * from './ValidationTranspiler'; +export * from './IIFETranspiler'; export * from './transpiler.interface'; diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts index 93c1c142744f0..ffaeba382ec7a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts @@ -10,6 +10,7 @@ import { CubePropContextTranspiler } from './CubePropContextTranspiler'; import { ErrorReporter } from '../ErrorReporter'; import { LightweightSymbolResolver } from './LightweightSymbolResolver'; import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary'; +import { IIFETranspiler } from './IIFETranspiler'; type TransferContent = { fileName: string; @@ -28,6 +29,7 @@ const transpilers = { ImportExportTranspiler: new ImportExportTranspiler(), CubeCheckDuplicatePropTranspiler: new CubeCheckDuplicatePropTranspiler(), CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols), + IIFETranspiler: new IIFETranspiler(), }; const transpile = (data: TransferContent) => { diff --git a/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts b/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts index 4dcacd3f9ad8d..d0cccbd75d725 100644 --- a/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts @@ -77,7 +77,7 @@ describe('Transpilers', () => { expect(content).toEqual(`const helperFunction = () => 'hello'; addExport({ helperFunction: helperFunction -}) +}); addExport({ alias: helperFunction }); @@ -87,11 +87,11 @@ function requireFilterParam() { } addExport({ requireFilterParam: requireFilterParam -}) +}); const someVar = 42; addExport({ someVar: someVar -})`); +});`); errorsReport.throwIfAny(); // should not throw });