diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index a20e3ab439822..cfe9d97d1a213 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -67,6 +67,7 @@ "@types/babel__traverse": "^7.20.5", "@types/inflection": "^1.5.28", "@types/jest": "^29", + "@types/js-yaml": "^4.0.9", "@types/node": "^20", "@types/node-dijkstra": "^2.5.6", "@types/ramda": "^0.27.34", diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c5de58b620094..4597837948e58 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -22,7 +22,6 @@ import { localTimestampToUtc, timeSeries as timeSeriesBase, timeSeriesFromCustomInterval, - parseSqlInterval, findMinGranularityDimension } from '@cubejs-backend/shared'; @@ -155,99 +154,6 @@ export class BaseQuery { this.initFromOptions(); } - extractDimensionsAndMeasures(filters = []) { - if (!filters) { - return []; - } - let allFilters = []; - filters.forEach(f => { - if (f.and) { - allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.and)); - } else if (f.or) { - allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.or)); - } else if (!f.member && !f.dimension) { - throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); - } else if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { - allFilters.push({ measure: f.member || f.dimension }); - } else { - allFilters.push({ dimension: f.member || f.dimension }); - } - }); - - return allFilters; - } - - keepFilters(filters = [], fn) { - return filters.map(f => { - if (f.and) { - return { and: this.keepFilters(f.and, fn) }; - } else if (f.or) { - return { or: this.keepFilters(f.or, fn) }; - } else if (!f.member && !f.dimension) { - throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); - } else { - return fn(f.member || f.dimension || f.measure) ? f : null; - } - }).filter(f => !!f); - } - - extractFiltersAsTree(filters = []) { - if (!filters) { - return []; - } - - return filters.map(f => { - if (f.and || f.or) { - let operator = 'or'; - if (f.and) { - operator = 'and'; - } - const data = this.extractDimensionsAndMeasures(f[operator]); - const dimension = data.filter(e => !!e.dimension).map(e => e.dimension); - const measure = data.filter(e => !!e.measure).map(e => e.measure); - if (dimension.length && !measure.length) { - return { - values: this.extractFiltersAsTree(f[operator]), - operator, - dimensionGroup: true, - measure: null, - }; - } - if (!dimension.length && measure.length) { - return { - values: this.extractFiltersAsTree(f[operator]), - operator, - dimension: null, - measureGroup: true, - }; - } - if (!dimension.length && !measure.length) { - return { - values: [], - operator, - }; - } - throw new UserError(`You cannot use dimension and measure in same condition: ${JSON.stringify(f)}`); - } - - if (!f.member && !f.dimension) { - throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); - } - - if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { - return Object.assign({}, f, { - dimension: null, - measure: f.member || f.dimension - }); - } - - return Object.assign({}, f, { - measure: null, - dimension: f.member || f.dimension - }); - }); - } - /** * @protected */ @@ -317,7 +223,11 @@ export class BaseQuery { /** @type {import('./BaseTimeDimension').BaseTimeDimension[]} */ this.timeDimensions = (this.options.timeDimensions || []).map(dimension => { if (!dimension.dimension) { - const join = this.joinGraph.buildJoin(this.collectJoinHints(true)); + const { joinHints, joinAliases } = this.collectJoinHints(true); + + // TODO: Use joinAliases for join hints building + + const join = this.joinGraph.buildJoin(joinHints); if (!join) { return undefined; } @@ -356,6 +266,116 @@ export class BaseQuery { this.initUngrouped(); } + initUngrouped() { + this.ungrouped = this.options.ungrouped; + if (this.ungrouped) { + if (!this.options.allowUngroupedWithoutPrimaryKey && this.join) { + const cubes = R.uniq([this.join.root].concat(this.join.joins.map(j => j.originalTo))); + const primaryKeyNames = cubes.flatMap(c => this.primaryKeyNames(c)); + const missingPrimaryKeys = primaryKeyNames.filter(key => !this.dimensions.find(d => d.dimension === key)); + if (missingPrimaryKeys.length) { + throw new UserError(`Ungrouped query requires primary keys to be present in dimensions: ${missingPrimaryKeys.map(k => `'${k}'`).join(', ')}. Pass allowUngroupedWithoutPrimaryKey option to disable this check.`); + } + } + if (this.measureFilters.length) { + throw new UserError('Measure filters aren\'t allowed in ungrouped query'); + } + } + } + + extractDimensionsAndMeasures(filters = []) { + if (!filters) { + return []; + } + let allFilters = []; + filters.forEach(f => { + if (f.and) { + allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.and)); + } else if (f.or) { + allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.or)); + } else if (!f.member && !f.dimension) { + throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); + } else if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { + allFilters.push({ measure: f.member || f.dimension }); + } else { + allFilters.push({ dimension: f.member || f.dimension }); + } + }); + + return allFilters; + } + + keepFilters(filters = [], fn) { + return filters.map(f => { + if (f.and) { + return { and: this.keepFilters(f.and, fn) }; + } else if (f.or) { + return { or: this.keepFilters(f.or, fn) }; + } else if (!f.member && !f.dimension) { + throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); + } else { + return fn(f.member || f.dimension || f.measure) ? f : null; + } + }).filter(f => !!f); + } + + extractFiltersAsTree(filters = []) { + if (!filters) { + return []; + } + + return filters.map(f => { + if (f.and || f.or) { + let operator = 'or'; + if (f.and) { + operator = 'and'; + } + const data = this.extractDimensionsAndMeasures(f[operator]); + const dimension = data.filter(e => !!e.dimension).map(e => e.dimension); + const measure = data.filter(e => !!e.measure).map(e => e.measure); + if (dimension.length && !measure.length) { + return { + values: this.extractFiltersAsTree(f[operator]), + operator, + dimensionGroup: true, + measure: null, + }; + } + if (!dimension.length && measure.length) { + return { + values: this.extractFiltersAsTree(f[operator]), + operator, + dimension: null, + measureGroup: true, + }; + } + if (!dimension.length && !measure.length) { + return { + values: [], + operator, + }; + } + throw new UserError(`You cannot use dimension and measure in same condition: ${JSON.stringify(f)}`); + } + + if (!f.member && !f.dimension) { + throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); + } + + if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { + return Object.assign({}, f, { + dimension: null, + measure: f.member || f.dimension + }); + } + + return Object.assign({}, f, { + measure: null, + dimension: f.member || f.dimension + }); + }); + } + // Temporary workaround to avoid checking for multistage in CubeStoreQuery, since that could lead to errors when HLL functions are present in the query. neverUseSqlPlannerPreaggregation() { return false; @@ -424,9 +444,16 @@ export class BaseQuery { */ get allJoinHints() { if (!this.collectedJoinHints) { - const [rootOfJoin, ...allMembersJoinHints] = this.collectJoinHintsFromMembers(this.allMembersConcat(false)); - const customSubQueryJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery()); - let joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(this.join)); + let { joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.allMembersConcat(false)); + const [rootOfJoin, ...allMembersJoinHints] = joinHints; + + ({ joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery())); + const customSubQueryJoinHints = joinHints; + + ({ joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(this.join))); + let joinMembersJoinHints = joinHints; + + // TODO: Use joinAliases for join hints building // One cube may join the other cube via transitive joined cubes, // members from which are referenced in the join `on` clauses. @@ -485,7 +512,12 @@ export class BaseQuery { while (newJoin?.joins.length > 0 && !isJoinTreesEqual(prevJoins, newJoin) && cnt < 10000) { prevJoins = newJoin; - joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(newJoin)); + + ({ joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(newJoin))); + joinMembersJoinHints = joinHints; + + // TODO: Use joinAliases for join hints building + if (!isOrderPreserved(prevJoinMembersJoinHints, joinMembersJoinHints)) { throw new UserError(`Can not construct joins for the query, potential loop detected: ${prevJoinMembersJoinHints.join('->')} vs ${joinMembersJoinHints.join('->')}`); } @@ -526,23 +558,6 @@ export class BaseQuery { ); } - initUngrouped() { - this.ungrouped = this.options.ungrouped; - if (this.ungrouped) { - if (!this.options.allowUngroupedWithoutPrimaryKey && this.join) { - const cubes = R.uniq([this.join.root].concat(this.join.joins.map(j => j.originalTo))); - const primaryKeyNames = cubes.flatMap(c => this.primaryKeyNames(c)); - const missingPrimaryKeys = primaryKeyNames.filter(key => !this.dimensions.find(d => d.dimension === key)); - if (missingPrimaryKeys.length) { - throw new UserError(`Ungrouped query requires primary keys to be present in dimensions: ${missingPrimaryKeys.map(k => `'${k}'`).join(', ')}. Pass allowUngroupedWithoutPrimaryKey option to disable this check.`); - } - } - if (this.measureFilters.length) { - throw new UserError('Measure filters aren\'t allowed in ungrouped query'); - } - } - } - get subQueryDimensions() { // eslint-disable-next-line no-underscore-dangle if (!this._subQueryDimensions) { @@ -2393,7 +2408,8 @@ export class BaseQuery { ); if (shouldBuildJoinForMeasureSelect) { - const joinHints = this.collectJoinHintsFromMembers(measures); + const { joinHints, joinAliases } = this.collectJoinHintsFromMembers(measures); + // TODO: Use joinAliases for join hints building const measuresJoin = this.joinGraph.buildJoin(joinHints); if (measuresJoin.multiplicationFactor[keyCubeName]) { throw new UserError( @@ -2474,10 +2490,14 @@ export class BaseQuery { const cubes = this.collectFrom(nonViewMembers, this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor'); // Not using `collectJoinHintsFromMembers([measure])` because it would collect too many join hints from view + const { joinHints: collectedJoinHints, joinAliases } = this.collectJoinHintsFromMembers(nonViewMembers); const joinHints = [ measure.joinHint, - ...this.collectJoinHintsFromMembers(nonViewMembers), + ...collectedJoinHints, ]; + + // TODO: Use joinAliases for join hints building + if (R.any(cubeName => keyCubeName !== cubeName, cubes)) { const measuresJoin = this.joinGraph.buildJoin(joinHints); if (measuresJoin.multiplicationFactor[keyCubeName]) { @@ -2658,10 +2678,23 @@ export class BaseQuery { } collectJoinHintsFromMembers(members) { - return [ - ...members.map(m => m.joinHint).filter(h => h?.length > 0), - ...this.collectFrom(members, this.collectJoinHintsFor.bind(this), 'collectJoinHintsFromMembers'), - ]; + const { + joinHints: collectedJoinHints, + joinAliases: collectedJoinAliases, + } = this.collectFrom(members, this.collectJoinHintsFor.bind(this), 'collectJoinHintsFromMembers') + .reduce((acc, { joinHints, joinAliases }) => { + joinHints.forEach(item => acc.joinHints.add(item)); + acc.joinAliases.push(...joinAliases); + return acc; + }, { joinHints: new Set(), joinAliases: [] }); + + return { + joinHints: [ + ...members.map(m => m.joinHint).filter(h => h?.length > 0), + ...Array.from(collectedJoinHints), + ], + joinAliases: collectedJoinAliases, + }; } /** @@ -3067,16 +3100,26 @@ export class BaseQuery { return sql; } + /** + * @param {string} sql + * @returns {string} + */ wrapSegmentForDimensionSelect(sql) { return sql; } + /** + * @param {string} cubeName + */ pushCubeNameForCollectionIfNecessary(cubeName) { if ((this.evaluateSymbolContext || {}).cubeNames && cubeName) { this.evaluateSymbolContext.cubeNames.push(cubeName); } } + /** + * @param {string|Array} joinHints + */ pushJoinHints(joinHints) { if (this.safeEvaluateSymbolContext().joinHints && joinHints) { if (Array.isArray(joinHints) && joinHints.length === 1) { @@ -3086,6 +3129,10 @@ export class BaseQuery { } } + /** + * @param {string} cubeName + * @param {string} name + */ pushMemberNameForCollectionIfNecessary(cubeName, name) { const pathFromArray = this.cubeEvaluator.pathFromArray([cubeName, name]); if (!this.cubeEvaluator.getCubeDefinition(cubeName).isView) { @@ -3372,12 +3419,15 @@ export class BaseQuery { } collectJoinHintsFor(fn) { - const context = { joinHints: [] }; + const context = { joinHints: [], joinAliases: [] }; this.evaluateSymbolSqlWithContext( fn, context ); - return context.joinHints; + return { + joinHints: context.joinHints, + joinAliases: context.joinAliases, + }; } /** diff --git a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.js b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.js deleted file mode 100644 index a3a03b3652cd1..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.js +++ /dev/null @@ -1,35 +0,0 @@ -import R from 'ramda'; - -export class ContextEvaluator { - /** - * @param {import('./CubeEvaluator').CubeEvaluator} cubeEvaluator - */ - constructor(cubeEvaluator) { - this.cubeEvaluator = cubeEvaluator; - this.contextDefinitions = {}; - } - - // eslint-disable-next-line no-unused-vars - compile(contexts, _errorReporter) { - if (contexts.length === 0) { - return; - } - - // TODO: handle duplications, context names must be uniq - this.contextDefinitions = R.compose( - R.fromPairs, - R.map(v => [v.name, this.compileContext(v)]) - )(contexts); - } - - compileContext(context) { - return { - name: context.name, - contextMembers: this.cubeEvaluator.evaluateReferences(null, context.contextMembers) - }; - } - - get contextList() { - return R.keys(this.contextDefinitions); - } -} diff --git a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts new file mode 100644 index 0000000000000..0e8df6ea277e7 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts @@ -0,0 +1,48 @@ +import { CubeEvaluator } from './CubeEvaluator'; +import { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; + +export type ContextDefinition = { + name: string; + contextMembers: string | string[]; +}; + +export class ContextEvaluator implements CompilerInterface { + private cubeEvaluator: CubeEvaluator; + + public contextDefinitions: Record; + + public constructor(cubeEvaluator: CubeEvaluator) { + this.cubeEvaluator = cubeEvaluator; + this.contextDefinitions = {}; + } + + public compile(contexts: any, errorReporter: ErrorReporter) { + if (contexts.length === 0) { + return; + } + + const definitions: Record = {}; + + for (const v of contexts) { + if (definitions[v.name]) { + errorReporter.error(`Duplicate context name found: '${v.name}'`); + } else { + definitions[v.name] = this.compileContext(v); + } + } + + this.contextDefinitions = definitions; + } + + private compileContext(context: any): ContextDefinition { + return { + name: context.name, + contextMembers: this.cubeEvaluator.evaluateReferences(null, context.contextMembers) + }; + } + + public get contextList(): string[] { + return Object.keys(this.contextDefinitions); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.js b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.js deleted file mode 100644 index a3fe63e9ec092..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.js +++ /dev/null @@ -1,16 +0,0 @@ -import R from 'ramda'; - -export class CubeDictionary { - constructor() { - this.byId = {}; - } - - // eslint-disable-next-line no-unused-vars - compile(cubes, errorReporter) { - this.byId = R.fromPairs(cubes.map(v => [v.name, v])); - } - - resolveCube(cubeName) { - return this.byId[cubeName]; - } -} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts new file mode 100644 index 0000000000000..43c248c3ce821 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts @@ -0,0 +1,18 @@ +import { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; + +export class CubeDictionary implements CompilerInterface { + public byId: Record; + + public constructor() { + this.byId = {}; + } + + public compile(cubes: any[], _errorReporter: ErrorReporter) { + this.byId = Object.fromEntries(cubes.map(v => [v.name, v])); + } + + public resolveCube(cubeName: string): any { + return this.byId[cubeName]; + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 3d2e2e3a993ea..4d305d7e2b6b8 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -2,35 +2,50 @@ import R from 'ramda'; import { + AccessPolicyDefinition, CubeDefinitionExtended, CubeSymbols, - HierarchyDefinition, JoinDefinition, - PreAggregationDefinition, PreAggregationDefinitionRollup, + GranularityDefinition, + HierarchyDefinition, + JoinDefinition, + PreAggregationDefinition, + PreAggregationDefinitionRollup, type ToString } from './CubeSymbols'; import { UserError } from './UserError'; import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; import type { CubeValidator } from './CubeValidator'; import type { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; export type SegmentDefinition = { type: string; - sql(): string; + sql?: (...args: any[]) => string; primaryKey?: true; ownedByCube: boolean; fieldType?: string; // TODO should we have it here? multiStage?: boolean; + meta?: any; + description?: string; + aliasMember?: string; }; export type DimensionDefinition = { type: string; - sql(): string; + sql?: (...args: any[]) => string; primaryKey?: true; ownedByCube: boolean; fieldType?: string; multiStage?: boolean; shiftInterval?: string; + meta?: any; + description?: string; + format?: string; + granularities?: Record; + aliasMember?: string; + suggestFilterValues?: boolean; }; export type TimeShiftDefinition = { @@ -49,7 +64,7 @@ export type TimeShiftDefinitionReference = { export type MeasureDefinition = { type: string; - sql(): string; + sql?: (...args: any[]) => string; ownedByCube: boolean; rollingWindow?: any filters?: any @@ -65,6 +80,9 @@ export type MeasureDefinition = { addGroupByReferences?: string[]; timeShiftReferences?: TimeShiftDefinitionReference[]; patchedFrom?: { cubeName: string; name: string }; + meta?: any; + description?: string; + aliasMember?: string; }; export type PreAggregationFilters = { @@ -110,30 +128,6 @@ export type EvaluatedHierarchy = { [key: string]: any; }; -export type Filter = - | { - member: string; - memberReference?: string; - [key: string]: any; - } - | { - and?: Filter[]; - or?: Filter[]; - [key: string]: any; - }; - -export type AccessPolicy = { - rowLevel?: { - filters: Filter[]; - }; - memberLevel?: { - includes?: string | string[]; - excludes?: string | string[]; - includesMembers?: string[]; - excludesMembers?: string[]; - }; -}; - export type EvaluatedFolder = { name: string; includes: (EvaluatedFolder | DimensionDefinition | MeasureDefinition)[]; @@ -141,22 +135,30 @@ export type EvaluatedFolder = { [key: string]: any; }; -export type EvaluatedCube = { +export interface EvaluatedCube { + name: string; + extends?: (...args: Array) => { __cubeName: string }; + sql?: (...args: any[]) => string; + sqlTable?: (...args: any[]) => string; + isView?: boolean; + calendar?: boolean; + meta?: any; + description?: string; + title?: string; measures: Record; dimensions: Record; segments: Record; - joins: Record; + joins: JoinDefinition[]; hierarchies: Record; evaluatedHierarchies: EvaluatedHierarchy[]; + aliasMember?: string; preAggregations: Record; dataSource?: string; folders: EvaluatedFolder[]; - sql?: (...args: any[]) => string; - sqlTable?: (...args: any[]) => string; - accessPolicy?: AccessPolicy[]; -}; + accessPolicy?: AccessPolicyDefinition[]; +} -export class CubeEvaluator extends CubeSymbols { +export class CubeEvaluator extends CubeJoinsResolver implements CompilerInterface { public evaluatedCubes: Record = {}; public primaryKeys: Record = {}; @@ -171,7 +173,7 @@ export class CubeEvaluator extends CubeSymbols { super(true); } - public compile(cubes: any[], errorReporter: ErrorReporter) { + public compile(cubes: CubeDefinitionExtended[], errorReporter: ErrorReporter) { super.compile(cubes, errorReporter); const validCubes = this.cubeList.filter(cube => this.cubeValidator.isCubeValid(cube)).sort((a, b) => { if (a.isView) { @@ -200,7 +202,7 @@ export class CubeEvaluator extends CubeSymbols { ); } - protected prepareCube(cube, errorReporter: ErrorReporter) { + protected prepareCube(cube, errorReporter: ErrorReporter): EvaluatedCube { this.prepareJoins(cube, errorReporter); this.preparePreAggregations(cube, errorReporter); this.prepareMembers(cube.measures, cube, errorReporter); @@ -412,7 +414,7 @@ export class CubeEvaluator extends CubeSymbols { } } - private evaluateMultiStageReferences(cubeName: string, obj: { [key: string]: MeasureDefinition }) { + private evaluateMultiStageReferences(cubeName: string, obj: Record) { if (!obj) { return; } @@ -444,35 +446,39 @@ export class CubeEvaluator extends CubeSymbols { } protected prepareJoins(cube: any, _errorReporter: ErrorReporter) { - if (cube.joins) { - // eslint-disable-next-line no-restricted-syntax - for (const join of Object.values(cube.joins) as any[]) { - // eslint-disable-next-line default-case - switch (join.relationship) { - case 'belongs_to': - case 'many_to_one': - case 'manyToOne': - join.relationship = 'belongsTo'; - break; - case 'has_many': - case 'one_to_many': - case 'oneToMany': - join.relationship = 'hasMany'; - break; - case 'has_one': - case 'one_to_one': - case 'oneToOne': - join.relationship = 'hasOne'; - break; - } - } + if (!cube.joins) { + return; } + + const joins: JoinDefinition[] = Array.isArray(cube.joins) ? cube.joins : Object.values(cube.joins); + + joins.forEach(join => { + // eslint-disable-next-line default-case + switch (join.relationship) { + case 'belongs_to': + case 'many_to_one': + case 'manyToOne': + join.relationship = 'belongsTo'; + break; + case 'has_many': + case 'one_to_many': + case 'oneToMany': + join.relationship = 'hasMany'; + break; + case 'has_one': + case 'one_to_one': + case 'oneToOne': + join.relationship = 'hasOne'; + break; + } + }); } protected preparePreAggregations(cube: any, errorReporter: ErrorReporter) { if (cube.preAggregations) { // eslint-disable-next-line no-restricted-syntax for (const preAggregation of Object.values(cube.preAggregations) as any) { + // preAggregation is actually (PreAggregationDefinitionRollup | PreAggregationDefinitionOriginalSql) if (preAggregation.timeDimension) { preAggregation.timeDimensionReference = preAggregation.timeDimension; delete preAggregation.timeDimension; @@ -574,7 +580,7 @@ export class CubeEvaluator extends CubeSymbols { } } - public cubesByFileName(fileName) { + public cubesByFileName(fileName): CubeDefinitionExtended[] { return this.byFileName[fileName] || []; } @@ -691,7 +697,7 @@ export class CubeEvaluator extends CubeSymbols { return this.preAggregations({ scheduled: true }); } - public cubeNames() { + public cubeNames(): string[] { return Object.keys(this.evaluatedCubes); } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts new file mode 100644 index 0000000000000..c8d89f3ccbec4 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts @@ -0,0 +1,73 @@ +import { CompilerInterface } from './PrepareCompiler'; +import { CubeDefinitionExtended, CubeSymbols, JoinDefinition } from './CubeSymbols'; +import type { ErrorReporter } from './ErrorReporter'; + +export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface { + // key: cubeName with joins defined + private cubeJoins: Record; + + // key: cubeName with joins defined + // 1st level value: join alias + // 2nd level value: join definition + public cubeJoinAliases: Record>; + + // key: cubeName with joins defined + // 1st level value: target cube name + // 2nd level value: join definition + private cubeJoinTargets: Record>; + + public constructor(evaluateViews = false) { + super(evaluateViews); + this.cubeJoins = {}; + this.cubeJoinAliases = {}; + this.cubeJoinTargets = {}; + } + + public compile(cubes: CubeDefinitionExtended[], errorReporter: ErrorReporter) { + super.compile(cubes, errorReporter); + + this.cubeList.forEach(cube => { + if (!cube.joins?.length) { + return; + } + + this.cubeJoins[cube.name] = this.cubeJoins[cube.name] || []; + this.cubeJoinAliases[cube.name] = this.cubeJoinAliases[cube.name] || {}; + this.cubeJoinTargets[cube.name] = this.cubeJoinTargets[cube.name] || {}; + + const er = errorReporter.inContext(`${cube.name} cube`); + + cube.joins.forEach(join => { + this.cubeJoins[cube.name].push(join); + this.cubeJoinTargets[cube.name][join.name] = join; + + if (join.alias) { + if (this.cubeJoinAliases[cube.name][join.alias]) { + er.error( + `Join alias "${join.alias}" is already defined in cube "${cube.name}".`, + cube.fileName + ); + + return; + } + this.cubeJoinAliases[cube.name][join.alias] = join; + } + }); + }); + } + + public resolveSymbol(cubeName: string | null, name: string) { + if (this.isCurrentCube(name) && !cubeName) { + return null; + } + + if (cubeName && this.cubeJoinAliases[cubeName]?.[name]) { + // TODO: Write a full implementation + // Some kind of proxy like in symbols + + return super.resolveSymbol(cubeName, this.cubeJoinAliases[cubeName]?.[name].name); + } + + return super.resolveSymbol(cubeName, name); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index c7acc1ab20304..02b6a9d9816a9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -7,6 +7,8 @@ import { DynamicReference } from './DynamicReference'; import { camelizeCube } from './utils'; import type { ErrorReporter } from './ErrorReporter'; +import { EvaluatedHierarchy } from './CubeEvaluator'; +import { CompilerInterface } from './PrepareCompiler'; export type ToString = { toString(): string }; @@ -26,12 +28,26 @@ export type TimeshiftDefinition = { }; export type CubeSymbolDefinition = { - type?: string; + type: string; sql?: (...args: any[]) => string; primaryKey?: boolean; + description?: string; + meta?: any; granularities?: Record; timeShift?: TimeshiftDefinition[]; format?: string; + aliasMember?: string; +}; + +export type CubeDimensionSymbolDefinition = CubeSymbolDefinition & { + suggestFilterValues?: boolean; +}; + +export type CubeMeasureSymbolDefinition = CubeSymbolDefinition & { + multiStage?: boolean; + groupBy?: (...args: any[]) => ToString; + reduceBy?: (...args: any[]) => ToString; + addGroupBy?: (...args: any[]) => ToString; }; export type HierarchyDefinition = { @@ -100,8 +116,39 @@ export type PreAggregationDefinitionRollup = BasePreAggregationDefinition & { export type PreAggregationDefinition = PreAggregationDefinitionRollup; export type JoinDefinition = { - relationship: string, - sql: (...args: any[]) => string, + name: string; + relationship: string; + sql: (...args: any[]) => string; + alias?: string; +}; + +export type Filter = + | { + member: string; + memberReference?: string; + [key: string]: any; + } + | { + and?: Filter[]; + or?: Filter[]; + [key: string]: any; + }; + +export type AccessPolicyDefinition = { + rowLevel?: { + filters: Filter[]; + }; + memberLevel?: { + includes?: string | string[]; + excludes?: string | string[]; + includesMembers?: string[]; + excludesMembers?: string[]; + }; +}; + +export type FolderDefinition = { + name: string, + members: (string | FolderDefinition)[], }; export interface CubeDefinition { @@ -112,16 +159,19 @@ export interface CubeDefinition { sql_table?: string | ((...args: any[]) => string); sqlTable?: string | ((...args: any[]) => string); dataSource?: string; - measures?: Record; - dimensions?: Record; + meta?: any; + description?: string; + title?: string; + measures?: Record; + dimensions?: Record; segments?: Record; hierarchies?: Record; preAggregations?: Record; // eslint-disable-next-line camelcase pre_aggregations?: Record; - joins?: Record; - accessPolicy?: any[]; - folders?: any[]; + joins?: JoinDefinition[]; + accessPolicy?: AccessPolicyDefinition[]; + folders?: FolderDefinition[]; includes?: any; excludes?: any; cubes?: any; @@ -136,6 +186,10 @@ export interface CubeDefinitionExtended extends CubeDefinition { allDefinitions: (type: string) => Record; rawFolders: () => any[]; rawCubes: () => any[]; + // This is filled by CubeEvaluator later, but because of Javascript, it's been available here too, + // and it's been used in the CubeToMetaTransformer compiler. + // TODO: Think how to make it better. + evaluatedHierarchies?: EvaluatedHierarchy[]; } interface SplitViews { @@ -163,7 +217,7 @@ export const CONTEXT_SYMBOLS = { export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE']; -export class CubeSymbols { +export class CubeSymbols implements CompilerInterface { public symbols: Record; private builtCubes: Record; @@ -219,17 +273,17 @@ export class CubeSymbols { } public createCube(cubeDefinition: CubeDefinition): CubeDefinitionExtended { - let preAggregations: any; - let joins: any; - let measures: any; - let dimensions: any; - let segments: any; - let hierarchies: any; - let accessPolicy: any; - let folders: any; - let cubes: any; - - const cubeObject = Object.assign({ + let preAggregations: CubeDefinition['preAggregations']; + let joins: CubeDefinition['joins']; + let measures: CubeDefinition['measures']; + let dimensions: CubeDefinition['dimensions']; + let segments: CubeDefinition['segments']; + let hierarchies: CubeDefinition['hierarchies']; + let accessPolicy: CubeDefinition['accessPolicy']; + let folders: CubeDefinition['folders']; + let cubes: CubeDefinition['cubes']; + + const cubeObject: CubeDefinitionExtended = Object.assign({ allDefinitions(type: string) { if (cubeDefinition.extends) { return { @@ -295,7 +349,8 @@ export class CubeSymbols { get joins() { if (!joins) { - joins = this.allDefinitions('joins'); + const parentJoins = cubeDefinition.extends ? super.joins : []; + joins = [...parentJoins, ...(cubeDefinition.joins || [])]; } return joins; }, @@ -380,9 +435,10 @@ export class CubeSymbols { return cubeObject; } - protected transform(cubeName: string, errorReporter: ErrorReporter, splitViews: SplitViews) { + protected transform(cubeName: string, errorReporter: ErrorReporter, splitViews: SplitViews): CubeSymbolsDefinition { const cube = this.getCubeDefinition(cubeName); - const duplicateNames = R.compose( + // @ts-ignore + const duplicateNames: string[] = R.compose( R.map((nameToDefinitions: any) => nameToDefinitions[0]), R.toPairs, R.filter((definitionsByName: any) => definitionsByName.length > 1), @@ -394,9 +450,7 @@ export class CubeSymbols { // @ts-ignore )([cube.measures, cube.dimensions, cube.segments, cube.preAggregations, cube.hierarchies]); - // @ts-ignore if (duplicateNames.length > 0) { - // @ts-ignore errorReporter.error(`${duplicateNames.join(', ')} defined more than once`); } @@ -424,23 +478,24 @@ export class CubeSymbols { ...cube.dimensions || {}, ...cube.segments || {}, ...cube.preAggregations || {} - }; + } as CubeSymbolsDefinition; } - private camelCaseTypes(obj: Object | undefined) { + private camelCaseTypes(obj: Object | Array | undefined) { if (!obj) { return; } - // eslint-disable-next-line no-restricted-syntax - for (const member of Object.values(obj)) { + const members = Array.isArray(obj) ? obj : Object.values(obj); + + members.forEach(member => { if (member.type && member.type.indexOf('_') !== -1) { member.type = camelize(member.type, true); } if (member.relationship && member.relationship.indexOf('_') !== -1) { member.relationship = camelize(member.relationship, true); } - } + }); } protected transformPreAggregations(preAggregations: Object) { @@ -545,7 +600,7 @@ export class CubeSymbols { } } - const includeMembers = this.generateIncludeMembers(cubeIncludes, cube.name, type); + const includeMembers = this.generateIncludeMembers(cubeIncludes, type); this.applyIncludeMembers(includeMembers, cube, type, errorReporter); const existing = cube.includedMembers ?? []; @@ -696,11 +751,7 @@ export class CubeSymbols { splitViewDef = splitViews[viewName]; } - const includeMembers = this.generateIncludeMembers( - finalIncludes, - parentCube.name, - type - ); + const includeMembers = this.generateIncludeMembers(finalIncludes, type); this.applyIncludeMembers(includeMembers, splitViewDef, type, errorReporter); } else { for (const member of finalIncludes) { @@ -730,7 +781,7 @@ export class CubeSymbols { return this.symbols[cubeName]?.cubeObj()?.[type]?.[memberName]; } - protected generateIncludeMembers(members: any[], cubeName: string, type: string) { + protected generateIncludeMembers(members: any[], type: string) { return members.map(memberRef => { const path = memberRef.member.split('.'); const resolvedMember = this.getResolvedMember(type, path[path.length - 2], path[path.length - 1]); @@ -811,7 +862,7 @@ export class CubeSymbols { }); } - protected evaluateReferences>( + public evaluateReferences>( cube: string | null, referencesFn: (...args: Array) => T, options: { collectJoinHints?: boolean, originalSorting?: boolean } = {} @@ -867,9 +918,8 @@ export class CubeSymbols { /** * Split join path to member to join hint and member path: `A.B.C.D.E.dim` => `[A, B, C, D, E]` + `E.dim` - * @param path */ - public static joinHintFromPath(path: string): { path: string, joinHint: Array } { + public static joinHintFromPath(path: string): { path: string, joinHint: string[] } { const parts = path.split('.'); if (parts.length > 2) { // Path contains join path @@ -905,7 +955,7 @@ export class CubeSymbols { } } - protected withSymbolsCallContext(func, context) { + protected withSymbolsCallContext(func: Function, context) { const oldContext = this.resolveSymbolsCallContext; this.resolveSymbolsCallContext = context; try { @@ -930,7 +980,7 @@ export class CubeSymbols { return this.funcArgumentsValues[funcDefinition]; } - protected joinHints() { + protected joinHints(): string | string[] | undefined { const { joinHints } = this.resolveSymbolsCallContext || {}; if (Array.isArray(joinHints)) { return R.uniq(joinHints); @@ -1008,10 +1058,10 @@ export class CubeSymbols { } protected filterGroupFunctionDep() { - return (...filterParamArgs) => ''; + return (..._filterParamArgs) => ''; } - public resolveSymbol(cubeName, name) { + public resolveSymbol(cubeName: string | null, name: string) { const { sqlResolveFn, contextSymbols, collectJoinHints, depsResolveFn, currResolveIndexFn } = this.resolveSymbolsCallContext || {}; if (name === 'USER_CONTEXT') { throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.'); @@ -1025,20 +1075,24 @@ export class CubeSymbols { return symbol; } + if (this.isCurrentCube(name) && !cubeName) { + return null; + } + // In proxied subProperty flow `name` will be set to parent dimension|measure name, // so there will be no cube = this.symbols[cubeName : name] found, but potentially // during cube definition evaluation some other deeper subProperty may be requested. // To distinguish such cases we pass the right now requested property name to // cubeReferenceProxy, so later if subProperty is requested we'll have all the required // information to construct the response. - let cube = this.symbols[this.isCurrentCube(name) ? cubeName : name]; + let cube = this.symbols[this.isCurrentCube(name) ? cubeName! : name]; if (sqlResolveFn) { if (cube) { cube = this.cubeReferenceProxy( this.isCurrentCube(name) ? cubeName : name, collectJoinHints ? [] : undefined ); - } else if (this.symbols[cubeName]?.[name]) { + } else if (cubeName && this.symbols[cubeName]?.[name]) { cube = this.cubeReferenceProxy( cubeName, collectJoinHints ? [] : undefined, @@ -1051,12 +1105,12 @@ export class CubeSymbols { const parentIndex = currResolveIndexFn(); cube = this.cubeDependenciesProxy(parentIndex, newCubeName); return cube; - } else if (this.symbols[cubeName]?.[name] && this.symbols[cubeName][name].type === 'time') { + } else if (cubeName && this.symbols[cubeName]?.[name] && this.symbols[cubeName][name].type === 'time') { const parentIndex = currResolveIndexFn(); return this.timeDimDependenciesProxy(parentIndex); } } - return cube || this.symbols[cubeName]?.[name]; + return cube || (cubeName && this.symbols[cubeName]?.[name]); } protected cubeReferenceProxy(cubeName, joinHints?: any[], refProperty?: any) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js deleted file mode 100644 index 740da78e43576..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js +++ /dev/null @@ -1,238 +0,0 @@ -import inflection from 'inflection'; -import R from 'ramda'; -import camelCase from 'camelcase'; - -import { getEnv } from '@cubejs-backend/shared'; -import { CubeSymbols } from './CubeSymbols'; -import { UserError } from './UserError'; -import { BaseMeasure } from '../adapter'; - -export class CubeToMetaTransformer { - /** - * @param {import('./CubeValidator').CubeValidator} cubeValidator - * @param {import('./CubeEvaluator').CubeEvaluator} cubeEvaluator - * @param {import('./ContextEvaluator').ContextEvaluator} contextEvaluator - * @param {import('./JoinGraph').JoinGraph} joinGraph - */ - constructor(cubeValidator, cubeEvaluator, contextEvaluator, joinGraph) { - this.cubeValidator = cubeValidator; - this.cubeSymbols = cubeEvaluator; - this.cubeEvaluator = cubeEvaluator; - this.contextEvaluator = contextEvaluator; - this.joinGraph = joinGraph; - this.cubes = []; - } - - compile(cubes, errorReporter) { - this.cubes = this.cubeSymbols.cubeList - .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) - .map((v) => this.transform(v, errorReporter.inContext(`${v.name} cube`))) - .filter(Boolean); - - /** - * @deprecated - * @protected - */ - this.queries = this.cubes; - } - - /** - * @protected - */ - transform(cube) { - const cubeTitle = cube.title || this.titleize(cube.name); - - const isCubeVisible = this.isVisible(cube, true); - - const flatFolderSeparator = getEnv('nestedFoldersDelimiter'); - const flatFolders = []; - - const processFolder = (folder, path = [], mergedMembers = []) => { - const flatMembers = []; - const nestedMembers = folder.includes.map(member => { - if (member.type === 'folder') { - return processFolder(member, [...path, folder.name], flatMembers); - } - const memberName = `${cube.name}.${member.name}`; - flatMembers.push(memberName); - - return memberName; - }); - - if (flatFolderSeparator !== '') { - flatFolders.push({ - name: [...path, folder.name].join(flatFolderSeparator), - members: flatMembers, - }); - } else if (path.length > 0) { - mergedMembers.push(...flatMembers); - } else { // We're at the root level - flatFolders.push({ - name: folder.name, - members: [...new Set(flatMembers)], - }); - } - - return { - name: folder.name, - members: nestedMembers, - }; - }; - - const nestedFolders = (cube.folders || []).map(f => processFolder(f)); - - return { - config: { - name: cube.name, - type: cube.isView ? 'view' : 'cube', - title: cubeTitle, - isVisible: isCubeVisible, - public: isCubeVisible, - description: cube.description, - connectedComponent: this.joinGraph.connectedComponents()[cube.name], - meta: cube.meta, - measures: R.compose( - R.map((nameToMetric) => ({ - ...this.measureConfig(cube.name, cubeTitle, nameToMetric), - isVisible: isCubeVisible ? this.isVisible(nameToMetric[1], true) : false, - public: isCubeVisible ? this.isVisible(nameToMetric[1], true) : false, - })), - R.toPairs - )(cube.measures || {}), - dimensions: R.compose( - R.map((nameToDimension) => ({ - name: `${cube.name}.${nameToDimension[0]}`, - title: this.title(cubeTitle, nameToDimension), - type: nameToDimension[1].type, - description: nameToDimension[1].description, - shortTitle: this.title(cubeTitle, nameToDimension, true), - suggestFilterValues: - nameToDimension[1].suggestFilterValues == null - ? true - : nameToDimension[1].suggestFilterValues, - format: nameToDimension[1].format, - meta: nameToDimension[1].meta, - isVisible: isCubeVisible - ? this.isVisible(nameToDimension[1], !nameToDimension[1].primaryKey) - : false, - public: isCubeVisible - ? this.isVisible(nameToDimension[1], !nameToDimension[1].primaryKey) - : false, - primaryKey: !!nameToDimension[1].primaryKey, - aliasMember: nameToDimension[1].aliasMember, - granularities: - nameToDimension[1].granularities - ? R.compose(R.map((g) => ({ - name: g[0], - title: this.title(cubeTitle, g, true), - interval: g[1].interval, - offset: g[1].offset, - origin: g[1].origin, - })), R.toPairs)(nameToDimension[1].granularities) - : undefined, - })), - R.toPairs - )(cube.dimensions || {}), - segments: R.compose( - R.map((nameToSegment) => ({ - name: `${cube.name}.${nameToSegment[0]}`, - title: this.title(cubeTitle, nameToSegment), - shortTitle: this.title(cubeTitle, nameToSegment, true), - description: nameToSegment[1].description, - meta: nameToSegment[1].meta, - isVisible: isCubeVisible ? this.isVisible(nameToSegment[1], true) : false, - public: isCubeVisible ? this.isVisible(nameToSegment[1], true) : false, - })), - R.toPairs - )(cube.segments || {}), - hierarchies: (cube.evaluatedHierarchies || []).map((it) => ({ - ...it, - aliasMember: it.aliasMember, - public: it.public ?? true, - name: `${cube.name}.${it.name}`, - })), - folders: flatFolders, - nestedFolders, - }, - }; - } - - queriesForContext(contextId) { - // return All queries if no context pass - if (R.isNil(contextId) || R.isEmpty(contextId)) { - return this.queries; - } - - const context = this.contextEvaluator.contextDefinitions[contextId]; - - // If contextId is wrong - if (R.isNil(context)) { - throw new UserError(`Context ${contextId} doesn't exist`); - } - - // As for now context works on the cubes level - return R.filter( - (query) => R.includes(query.config.name, context.contextMembers) - )(this.queries); - } - - /** - * @protected - */ - isVisible(symbol, defaultValue) { - if (symbol.public != null) { - return symbol.public; - } - - // TODO: Deprecated, should be removed in the future - if (symbol.visible != null) { - return symbol.visible; - } - - // TODO: Deprecated, should be removed in the futur - if (symbol.shown != null) { - return symbol.shown; - } - - return defaultValue; - } - - measureConfig(cubeName, cubeTitle, nameToMetric) { - const name = `${cubeName}.${nameToMetric[0]}`; - // Support both old 'drillMemberReferences' and new 'drillMembers' keys - const drillMembers = nameToMetric[1].drillMembers || nameToMetric[1].drillMemberReferences; - - const drillMembersArray = (drillMembers && this.cubeEvaluator.evaluateReferences( - cubeName, drillMembers, { originalSorting: true } - )) || []; - - const type = CubeSymbols.toMemberDataType(nameToMetric[1].type); - - return { - name, - title: this.title(cubeTitle, nameToMetric), - description: nameToMetric[1].description, - shortTitle: this.title(cubeTitle, nameToMetric, true), - format: nameToMetric[1].format, - cumulativeTotal: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), - cumulative: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), - type, - aggType: nameToMetric[1].aggType || nameToMetric[1].type, - drillMembers: drillMembersArray, - drillMembersGrouped: { - measures: drillMembersArray.filter((member) => this.cubeEvaluator.isMeasure(member)), - dimensions: drillMembersArray.filter((member) => this.cubeEvaluator.isDimension(member)), - }, - meta: nameToMetric[1].meta - }; - } - - title(cubeTitle, nameToDef, short) { - // eslint-disable-next-line prefer-template - return `${short ? '' : cubeTitle + ' '}${nameToDef[1].title || this.titleize(nameToDef[0])}`; - } - - titleize(name) { - return inflection.titleize(inflection.underscore(camelCase(name, { pascalCase: true }))); - } -} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts new file mode 100644 index 0000000000000..f65c23ad612c5 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -0,0 +1,319 @@ +import inflection from 'inflection'; +import R from 'ramda'; +import camelCase from 'camelcase'; + +import { getEnv } from '@cubejs-backend/shared'; +import { CubeDefinitionExtended, CubeSymbols, FolderDefinition } from './CubeSymbols'; +import { UserError } from './UserError'; +import { BaseMeasure } from '../adapter'; +import { JoinGraph } from './JoinGraph'; +import { ContextEvaluator } from './ContextEvaluator'; +import { CubeEvaluator, EvaluatedCube } from './CubeEvaluator'; +import { CubeValidator } from './CubeValidator'; +import { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; + +export interface CubeTransformDefinition { + config: { + name: string; + type: 'cube' | 'view'; + title: string; + isVisible: boolean; + public: boolean; + description?: string; + connectedComponent?: any; + meta?: any; + measures: { + name: string; + title: string; + description?: string; + type: string; + aggType?: string; + cumulative?: boolean; + cumulativeTotal?: boolean; + drillMembers?: string[]; + drillMemberReferences?: string[]; + drillMembersGrouped?: { + [group: string]: string[]; + }; + meta?: any; + isVisible: boolean; + public: boolean; + }[]; + dimensions: { + name: string; + title: string; + shortTitle: string; + description?: string; + type: string; + format?: string; + meta?: any; + isVisible: boolean; + public: boolean; + primaryKey?: boolean; + aliasMember?: string; + granularities?: { + name: string; + title: string; + interval?: string; + offset?: string; + origin?: string; + }[]; + }[]; + segments: { + name: string; + title: string; + shortTitle: string; + description?: string; + meta?: any; + isVisible: boolean; + public: boolean; + }[]; + hierarchies: { + name: string; + levels: string[]; + aliasMember?: string; + public?: boolean; + }[]; + folders: FolderDefinition[]; + nestedFolders: { + name: string; + members: (string | { name: string; members: any })[]; + }[]; + }; +} + +export class CubeToMetaTransformer implements CompilerInterface { + private cubeValidator: CubeValidator; + + private cubeSymbols: CubeEvaluator; + + /** + * Is public just because is used in tests. Should be private. + */ + public cubeEvaluator: CubeEvaluator; + + private contextEvaluator: ContextEvaluator; + + private joinGraph: JoinGraph; + + public cubes: CubeTransformDefinition[]; + + /** + * @deprecated + */ + protected queries: CubeTransformDefinition[]; + + public constructor(cubeValidator: CubeValidator, cubeEvaluator: CubeEvaluator, contextEvaluator: ContextEvaluator, joinGraph: JoinGraph) { + this.cubeValidator = cubeValidator; + this.cubeSymbols = cubeEvaluator; + this.cubeEvaluator = cubeEvaluator; + this.contextEvaluator = contextEvaluator; + this.joinGraph = joinGraph; + this.cubes = []; + this.queries = []; + } + + public compile(cubes: EvaluatedCube[], errorReporter: ErrorReporter) { + this.cubes = this.cubeSymbols.cubeList + .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) + .map((v) => { + errorReporter.inContext(`${v.name} cube`); + return this.transform(v); + }) + .filter(Boolean); + + this.queries = this.cubes; + } + + protected transform(cube: CubeDefinitionExtended): CubeTransformDefinition { + const cubeTitle = cube.title || this.titleize(cube.name); + + const isCubeVisible = this.isVisible(cube, true); + + const flatFolderSeparator = getEnv('nestedFoldersDelimiter'); + const flatFolders: FolderDefinition[] = []; + + const processFolder = (folder, path: string[] = [], mergedMembers: string[] = []) => { + const flatMembers: string[] = []; + const nestedMembers = folder.includes.map(member => { + if (member.type === 'folder') { + return processFolder(member, [...path, folder.name], flatMembers); + } + const memberName = `${cube.name}.${member.name}`; + flatMembers.push(memberName); + + return memberName; + }); + + if (flatFolderSeparator !== '') { + flatFolders.push({ + name: [...path, folder.name].join(flatFolderSeparator), + members: flatMembers, + }); + } else if (path.length > 0) { + mergedMembers.push(...flatMembers); + } else { // We're at the root level + flatFolders.push({ + name: folder.name, + members: [...new Set(flatMembers)], + }); + } + + return { + name: folder.name, + members: nestedMembers, + }; + }; + + const nestedFolders = (cube.folders || []).map(f => processFolder(f)); + + return { + config: { + name: cube.name, + type: cube.isView ? 'view' : 'cube', + title: cubeTitle, + isVisible: isCubeVisible, + public: isCubeVisible, + description: cube.description, + connectedComponent: this.joinGraph.connectedComponents()[cube.name], + meta: cube.meta, + measures: Object.entries(cube.measures || {}).map(([name, definition]) => ({ + ...this.measureConfig(cube.name, cubeTitle, [name, definition]), + isVisible: isCubeVisible ? this.isVisible(definition, true) : false, + public: isCubeVisible ? this.isVisible(definition, true) : false, + })), + dimensions: Object.entries(cube.dimensions || {}).map(([name, def]) => { + const isVisible = isCubeVisible + ? this.isVisible(def, !def.primaryKey) + : false; + + return { + name: `${cube.name}.${name}`, + title: this.title(cubeTitle, [name, def]), + type: def.type, + description: def.description, + shortTitle: this.title(cubeTitle, [name, def], true), + suggestFilterValues: + def.suggestFilterValues == null ? true : def.suggestFilterValues, + format: def.format, + meta: def.meta, + isVisible, + public: isVisible, + primaryKey: !!def.primaryKey, + aliasMember: def.aliasMember, + granularities: def.granularities + ? Object.entries(def.granularities).map(([gName, gDef]) => ({ + name: gName, + title: this.title(cubeTitle, [gName, gDef], true), + interval: gDef.interval, + offset: gDef.offset, + origin: gDef.origin, + })) + : undefined, + }; + }), + segments: Object.entries(cube.segments || {}).map(([name, def]) => { + const isVisible = isCubeVisible ? this.isVisible(def, true) : false; + + return { + name: `${cube.name}.${name}`, + title: this.title(cubeTitle, [name, def]), + shortTitle: this.title(cubeTitle, [name, def], true), + description: def.description, + meta: def.meta, + isVisible, + public: isVisible, + }; + }), + hierarchies: (cube.evaluatedHierarchies || []).map((it) => ({ + ...it, + aliasMember: it.aliasMember, + public: it.public ?? true, + name: `${cube.name}.${it.name}`, + })), + folders: flatFolders, + nestedFolders, + }, + }; + } + + /** + * @deprecated + */ + public queriesForContext(contextId) { + // return All queries if no context pass + if (R.isNil(contextId) || R.isEmpty(contextId)) { + return this.queries; + } + + const context = this.contextEvaluator.contextDefinitions[contextId]; + + // If contextId is wrong + if (R.isNil(context)) { + throw new UserError(`Context ${contextId} doesn't exist`); + } + + // As for now context works on the cubes level + return R.filter( + (query: CubeTransformDefinition) => R.includes(query.config.name, context.contextMembers) + )(this.queries); + } + + protected isVisible(symbol, defaultValue) { + if (symbol.public != null) { + return symbol.public; + } + + // TODO: Deprecated, should be removed in the future + if (symbol.visible != null) { + return symbol.visible; + } + + // TODO: Deprecated, should be removed in the futur + if (symbol.shown != null) { + return symbol.shown; + } + + return defaultValue; + } + + protected measureConfig(cubeName, cubeTitle, nameToMetric) { + const name = `${cubeName}.${nameToMetric[0]}`; + // Support both old 'drillMemberReferences' and new 'drillMembers' keys + const drillMembers = nameToMetric[1].drillMembers || nameToMetric[1].drillMemberReferences; + + const drillMembersArray = (drillMembers && this.cubeEvaluator.evaluateReferences( + cubeName, drillMembers, { originalSorting: true } + )) || []; + + const type = CubeSymbols.toMemberDataType(nameToMetric[1].type); + + return { + name, + title: this.title(cubeTitle, nameToMetric), + description: nameToMetric[1].description, + shortTitle: this.title(cubeTitle, nameToMetric, true), + format: nameToMetric[1].format, + cumulativeTotal: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), + cumulative: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), + type, + aggType: nameToMetric[1].aggType || nameToMetric[1].type, + drillMembers: drillMembersArray, + drillMembersGrouped: { + measures: drillMembersArray.filter((member) => this.cubeEvaluator.isMeasure(member)), + dimensions: drillMembersArray.filter((member) => this.cubeEvaluator.isDimension(member)), + }, + meta: nameToMetric[1].meta + }; + } + + protected title(cubeTitle: string, nameToDef, short: boolean = false): string { + // eslint-disable-next-line prefer-template + return `${short ? '' : cubeTitle + ' '}${nameToDef[1].title || this.titleize(nameToDef[0])}`; + } + + protected titleize(name: string): string { + return inflection.titleize(inflection.underscore(camelCase(name, { pascalCase: true }))); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 32540d4864ac6..c3fcfd569ef4d 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -3,6 +3,7 @@ import cronParser from 'cron-parser'; import type { CubeSymbols, CubeDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; /* ***************************** * ATTENTION: @@ -802,14 +803,26 @@ const baseSchema = { shown: Joi.boolean().strict(), public: Joi.boolean().strict(), meta: Joi.any(), - joins: Joi.object().pattern(identifierRegex, Joi.object().keys({ - sql: Joi.func().required(), - relationship: Joi.any().valid( - 'belongsTo', 'belongs_to', 'many_to_one', 'manyToOne', - 'hasMany', 'has_many', 'one_to_many', 'oneToMany', - 'hasOne', 'has_one', 'one_to_one', 'oneToOne' - ).required() - })), + joins: Joi.alternatives([ + Joi.object().pattern(identifierRegex, Joi.object().keys({ + sql: Joi.func().required(), + relationship: Joi.any().valid( + 'belongsTo', 'belongs_to', 'many_to_one', 'manyToOne', + 'hasMany', 'has_many', 'one_to_many', 'oneToMany', + 'hasOne', 'has_one', 'one_to_one', 'oneToOne' + ).required() + })), + Joi.array().items(Joi.object().keys({ + name: identifier.required(), + sql: Joi.func().required(), + relationship: Joi.any().valid( + 'belongsTo', 'belongs_to', 'many_to_one', 'manyToOne', + 'hasMany', 'has_many', 'one_to_many', 'oneToMany', + 'hasOne', 'has_one', 'one_to_one', 'oneToOne' + ).required(), + alias: identifier, + })) + ]), measures: MeasuresSchema, dimensions: DimensionsSchema, segments: SegmentsSchema, @@ -928,7 +941,7 @@ export function functionFieldsPatterns(): string[] { return Array.from(functionPatterns); } -export class CubeValidator { +export class CubeValidator implements CompilerInterface { protected readonly validCubes: Map = new Map(); public constructor( @@ -936,7 +949,7 @@ export class CubeValidator { ) { } - public compile(cubes, errorReporter: ErrorReporter) { + public compile(_cubes, errorReporter: ErrorReporter) { return this.cubeSymbols.cubeList.map( (v) => this.validate(this.cubeSymbols.getCubeDefinition(v.name), errorReporter.inContext(`${v.name} cube`)) ); diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index e40ad0eb38a2f..a2a94fcc9ccc0 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -22,6 +22,7 @@ import { CompilerInterface } from './PrepareCompiler'; import { YamlCompiler } from './YamlCompiler'; import { CubeDictionary } from './CubeDictionary'; import { CompilerCache } from './CompilerCache'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; const NATIVE_IS_SUPPORTED = isNativeSupported(); @@ -53,6 +54,7 @@ export type DataSchemaCompilerOptions = { cubeFactory: Function; cubeDictionary: CubeDictionary; cubeSymbols: CubeSymbols; + cubeJoinsResolver: CubeJoinsResolver; cubeCompilers?: CompilerInterface[]; contextCompilers?: CompilerInterface[]; transpilers?: TranspilerInterface[]; @@ -72,6 +74,7 @@ export type DataSchemaCompilerOptions = { export type TranspileOptions = { cubeNames?: string[]; cubeSymbols?: Record>; + cubeJoins?: Record>; contextSymbols?: Record; transpilerNames?: string[]; compilerId?: string; @@ -108,6 +111,8 @@ export class DataSchemaCompiler { private readonly cubeSymbols: CubeSymbols; + private readonly cubeJoinsResolver: CubeJoinsResolver; + // Actually should be something like // createCube(cubeDefinition: CubeDefinition): CubeDefinitionExtended private readonly cubeFactory: CallableFunction; @@ -157,6 +162,7 @@ export class DataSchemaCompiler { this.extensions = options.extensions || {}; this.cubeDictionary = options.cubeDictionary; this.cubeSymbols = options.cubeSymbols; + this.cubeJoinsResolver = options.cubeJoinsResolver; this.cubeFactory = options.cubeFactory; this.filesToCompile = options.filesToCompile || []; this.omitErrors = options.omitErrors || false; @@ -232,6 +238,7 @@ export class DataSchemaCompiler { const transpile = async (stage: CompileStage) => { let cubeNames: string[] = []; let cubeSymbols: Record> = {}; + let cubeJoins: Record> = {}; let transpilerNames: string[] = []; let results; @@ -252,6 +259,14 @@ export class DataSchemaCompiler { )], ), ); + cubeJoins = Object.fromEntries( + Object.entries(this.cubeJoinsResolver.cubeJoinAliases as Record>) + .map( + ([key, value]: [string, Record]) => [key, Object.fromEntries( + Object.keys(value).map((k) => [k, true]), + )], + ), + ); // Transpilers are the same for all files within phase. transpilerNames = this.transpilers.map(t => t.constructor.name); @@ -264,7 +279,7 @@ export class DataSchemaCompiler { content: ';', }; - await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage }); + await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, cubeJoins, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage }); const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js')) .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId })); @@ -291,7 +306,7 @@ export class DataSchemaCompiler { results = (await Promise.all([...nonJsFilesTasks, ...JsFilesTasks])).flat(); } else if (transpilationWorkerThreads) { - results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }))); + results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, cubeJoins, transpilerNames }))); } else { results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {}))); } @@ -405,7 +420,7 @@ export class DataSchemaCompiler { }); } - private async transpileJsFile(file: FileContent, errorsReport: ErrorReporter, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) { + private async transpileJsFile(file: FileContent, errorsReport: ErrorReporter, { cubeNames, cubeSymbols, cubeJoins, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) { try { if (getEnv('transpilationNative')) { const reqData = { @@ -417,6 +432,7 @@ export class DataSchemaCompiler { metaData: { cubeNames, cubeSymbols: cubeSymbols || {}, + cubeJoins: cubeJoins || {}, contextSymbols: contextSymbols || {}, stage: stage || 0 as CompileStage, }, @@ -437,6 +453,7 @@ export class DataSchemaCompiler { transpilers: transpilerNames, cubeNames, cubeSymbols, + cubeJoins, }; const res = await this.workerPool!.exec('transpile', [data]); diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 179450a0cfed9..50fbbb30a9e6a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -6,6 +6,7 @@ import type { CubeValidator } from './CubeValidator'; import type { CubeEvaluator, MeasureDefinition } from './CubeEvaluator'; import type { CubeDefinition, JoinDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; type JoinEdge = { join: JoinDefinition, @@ -30,7 +31,7 @@ export type JoinHint = string | string[]; export type JoinHints = JoinHint[]; -export class JoinGraph { +export class JoinGraph implements CompilerInterface { private readonly cubeValidator: CubeValidator; private readonly cubeEvaluator: CubeEvaluator; @@ -60,7 +61,7 @@ export class JoinGraph { this.graph = null; } - public compile(cubes: unknown, errorReporter: ErrorReporter): void { + public compile(_cubes: unknown, errorReporter: ErrorReporter): void { this.edges = R.compose< Array, Array, @@ -76,16 +77,11 @@ export class JoinGraph { // This requires @types/ramda@0.29 or newer // @ts-ignore - this.nodes = R.compose< - Record, - Array<[string, JoinEdge]>, - Array, - Record | undefined>, - Record> - >( + this.nodes = R.compose( // This requires @types/ramda@0.29 or newer // @ts-ignore R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.to, 1]))), + // @ts-ignore R.groupBy((join: JoinEdge) => join.from), R.map(v => v[1]), R.toPairs @@ -109,70 +105,53 @@ export class JoinGraph { } protected buildJoinEdges(cube: CubeDefinition, errorReporter: ErrorReporter): Array<[string, JoinEdge]> { - // @ts-ignore - return R.compose( - // @ts-ignore - R.filter(R.identity), - R.map((join: [string, JoinEdge]) => { - const multipliedMeasures: ((m: Record) => MeasureDefinition[]) = R.compose( - R.filter( - (m: MeasureDefinition): boolean => m.sql && this.cubeEvaluator.funcArguments(m.sql).length === 0 && m.sql() === 'count(*)' || - ['sum', 'avg', 'count', 'number'].indexOf(m.type) !== -1 - ), - R.values as (input: Record) => MeasureDefinition[] - ); - const joinRequired = - (v) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; - if ( - !this.cubeEvaluator.primaryKeys[join[1].from].length && - multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].from)).length > 0 - ) { - errorReporter.error(joinRequired(join[1].from)); - return null; + if (!cube.joins) { + return []; + } + + const getMultipliedMeasures = (cubeName: string): MeasureDefinition[] => { + const measures = this.cubeEvaluator.measuresForCube(cubeName); + return Object.values(measures).filter((m: MeasureDefinition) => (m.sql && + this.cubeEvaluator.funcArguments(m.sql).length === 0 && + m.sql() === 'count(*)') || + ['sum', 'avg', 'count', 'number'].includes(m.type)); + }; + + const joinRequired = + (v: string) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; + + return cube.joins + .filter(join => { + if (!this.cubeEvaluator.cubeExists(join.name)) { + errorReporter.error(`Cube ${join.name} doesn't exist`); + return false; } - if (!this.cubeEvaluator.primaryKeys[join[1].to].length && - multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].to)).length > 0) { - errorReporter.error(joinRequired(join[1].to)); - return null; + + const fromMultipliedMeasures = getMultipliedMeasures(cube.name); + if (!this.cubeEvaluator.primaryKeys[cube.name].length && fromMultipliedMeasures.length > 0) { + errorReporter.error(joinRequired(cube.name)); + return false; } - return join; - }), - R.unnest, - R.map((join: [string, JoinDefinition]): [[string, JoinEdge]] => [ - [`${cube.name}-${join[0]}`, { - join: join[1], + + const toMultipliedMeasures = getMultipliedMeasures(join.name); + if (!this.cubeEvaluator.primaryKeys[join.name].length && toMultipliedMeasures.length > 0) { + errorReporter.error(joinRequired(join.name)); + return false; + } + + return true; + }) + .map(join => { + const joinEdge: JoinEdge = { + join, from: cube.name, - to: join[0], + to: join.name, originalFrom: cube.name, - originalTo: join[0] - }] - ]), - // @ts-ignore - R.filter(R.identity), - R.map((join: [string, JoinDefinition]) => { - if (!this.cubeEvaluator.cubeExists(join[0])) { - errorReporter.error(`Cube ${join[0]} doesn't exist`); - return undefined; - } - return join; - }), - // @ts-ignore - R.toPairs - // @ts-ignore - )(cube.joins || {}); - } + originalTo: join.name + }; - protected buildJoinNode(cube: CubeDefinition): Record { - return R.compose< - Record, - Array<[string, JoinDefinition]>, - Array<[string, 1]>, - Record - >( - R.fromPairs, - R.map(v => [v[0], 1]), - R.toPairs - )(cube.joins || {}); + return [`${cube.name}-${join.name}`, joinEdge] as [string, JoinEdge]; + }); } public buildJoin(cubesToJoin: JoinHints): FinishedJoinTree | null { @@ -214,7 +193,7 @@ export class JoinGraph { return this.builtJoins[key]; } - protected cubeFromPath(cubePath) { + protected cubeFromPath(cubePath: JoinHint): string { if (Array.isArray(cubePath)) { return cubePath[cubePath.length - 1]; } diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 75fc6025929f8..efae115bc2e30 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -24,6 +24,7 @@ import { CompilerCache } from './CompilerCache'; import { YamlCompiler } from './YamlCompiler'; import { ViewCompilationGate } from './ViewCompilationGate'; import type { ErrorReporter } from './ErrorReporter'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -46,6 +47,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const nativeInstance = options.nativeInstance || new NativeInstance(); const cubeDictionary = new CubeDictionary(); const cubeSymbols = new CubeSymbols(); + const cubeJoinsResolver = new CubeJoinsResolver(); const viewCompiler = new CubeSymbols(true); const viewCompilationGate = new ViewCompilationGate(); const cubeValidator = new CubeValidator(cubeSymbols); @@ -55,14 +57,14 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, joinGraph); const { maxQueryCacheSize, maxQueryCacheAge } = options; const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge }); - const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); + const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler, cubeJoinsResolver); const compiledScriptCache = options.compiledScriptCache || new LRUCache({ max: 250 }); const transpilers: TranspilerInterface[] = [ new ValidationTranspiler(), new ImportExportTranspiler(), - new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler), + new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler, cubeJoinsResolver), ]; if (!options.allowJsDuplicatePropsInSchema) { @@ -73,7 +75,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const compiler = new DataSchemaCompiler(repo, Object.assign({}, { cubeNameCompilers: [cubeDictionary], - preTranspileCubeCompilers: [cubeSymbols, cubeValidator], + preTranspileCubeCompilers: [cubeSymbols, cubeValidator, cubeJoinsResolver], transpilers, viewCompilationGate, compiledScriptCache, @@ -84,6 +86,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp compilerCache, cubeDictionary, cubeSymbols, + cubeJoinsResolver, extensions: { Funnels, RefreshKeys, diff --git a/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts index af53f12f30de6..264eccf60fee9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts +++ b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts @@ -1,4 +1,6 @@ -export class ViewCompilationGate { +import { CompilerInterface } from './PrepareCompiler'; + +export class ViewCompilationGate implements CompilerInterface { private shouldCompile: any; public constructor() { @@ -6,10 +8,10 @@ export class ViewCompilationGate { } public compile(cubes: any[]) { - // When developing Data Access Policies feature, we've came across a + // When developing Data Access Policies feature, we've come across a // limitation that Cube members can't be referenced in access policies defined on Views, // because views aren't (yet) compiled at the time of access policy evaluation. - // To workaround this limitation and additional compilation pass is necessary, + // To work around this limitation additional compilation pass is necessary, // however it comes with a significant performance penalty. // This gate check whether the data model contains views with access policies, // and only then allows the additional compilation pass. diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index d66224582cc0e..f1a49037e343c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -16,6 +16,7 @@ import { nonStringFields } from './CubeValidator'; import { CubeDictionary } from './CubeDictionary'; import { ErrorReporter } from './ErrorReporter'; import { camelizeCube } from './utils'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; type EscapeStateStack = { inFormattedStr?: boolean; @@ -34,6 +35,7 @@ export class YamlCompiler { private readonly cubeDictionary: CubeDictionary, private readonly nativeInstance: NativeInstance, private readonly viewCompiler: CubeSymbols, + private readonly cubeJoinsResolver: CubeJoinsResolver, ) { } @@ -116,7 +118,7 @@ export class YamlCompiler { } } - private transpileAndPrepareJsFile(file, methodFn, cubeObj, errorsReport: ErrorReporter) { + private transpileAndPrepareJsFile(file: FileContent, methodFn: ('cube' | 'view'), cubeObj, errorsReport: ErrorReporter): FileContent { const yamlAst = this.transformYamlCubeObj(cubeObj, errorsReport); const cubeOrViewCall = t.callExpression(t.identifier(methodFn), [t.stringLiteral(cubeObj.name), yamlAst]); @@ -135,7 +137,13 @@ export class YamlCompiler { cubeObj.dimensions = this.yamlArrayToObj(cubeObj.dimensions || [], 'dimension', errorsReport); cubeObj.segments = this.yamlArrayToObj(cubeObj.segments || [], 'segment', errorsReport); cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations || [], 'preAggregation', errorsReport); - cubeObj.joins = this.yamlArrayToObj(cubeObj.joins || [], 'join', errorsReport); + + cubeObj.joins = cubeObj.joins || []; // For edge cases where joins are not defined/null + if (!Array.isArray(cubeObj.joins)) { + errorsReport.error('joins must be defined as array'); + cubeObj.joins = []; + } + cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport); return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport); @@ -295,7 +303,8 @@ export class YamlCompiler { resolveSymbol = resolveSymbol || (n => this.viewCompiler.resolveSymbol(cubeName, n) || this.cubeSymbols.resolveSymbol(cubeName, n) || - this.cubeSymbols.isCurrentCube(n)); + this.cubeSymbols.isCurrentCube(n) || + this.cubeJoinsResolver.resolveSymbol(cubeName, n)); const traverseObj = { Program: (babelPath) => { diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 2ffc96058b3ef..7836ede81a753 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -8,8 +8,6 @@ import { TranspilerSymbolResolver, TraverseObject } from './transpiler.interface'; -import type { CubeSymbols } from '../CubeSymbols'; -import type { CubeDictionary } from '../CubeDictionary'; /* this list was generated by getTransformPatterns() with additional variants for snake_case */ export const transpiledFieldsPatterns: Array = [ @@ -47,6 +45,7 @@ export class CubePropContextTranspiler implements TranspilerInterface { protected readonly cubeSymbols: TranspilerSymbolResolver, protected readonly cubeDictionary: TranspilerCubeResolver, protected readonly viewCompiler: TranspilerSymbolResolver, + protected readonly cubeJoinsResolver: TranspilerSymbolResolver, ) { } @@ -96,10 +95,18 @@ export class CubePropContextTranspiler implements TranspilerInterface { protected sqlAndReferencesFieldVisitor(cubeName): TraverseObject { const resolveSymbol = n => this.viewCompiler.resolveSymbol(cubeName, n) || this.cubeSymbols.resolveSymbol(cubeName, n) || - this.cubeSymbols.isCurrentCube(n); + this.cubeSymbols.isCurrentCube(n) || + this.cubeJoinsResolver.resolveSymbol(cubeName, n); return { ObjectProperty: (path) => { + if (path.node.key.type === 'Identifier' && path.node.key.name === 'joins' && t.isObjectExpression(path.node.value)) { + const fullPath = this.fullPath(path); + if (fullPath === 'joins') { + this.convertJoinsObjectToArray(path); + } + } + if (path.node.key.type === 'Identifier' && transpiledFields.has(path.node.key.name)) { const fullPath = this.fullPath(path); // eslint-disable-next-line no-restricted-syntax @@ -114,6 +121,39 @@ export class CubePropContextTranspiler implements TranspilerInterface { }; } + protected convertJoinsObjectToArray(path: NodePath) { + const valuePath = path.get('value'); + if (!valuePath.isObjectExpression()) { + return; + } + + const elements: t.ObjectExpression[] = []; + + for (const prop of valuePath.get('properties')) { + if (!prop.isObjectProperty()) { + return; + } + const keyNode = prop.node.key; + const valueNode = prop.node.value; + + if ( + (t.isIdentifier(keyNode) || t.isStringLiteral(keyNode)) && + t.isObjectExpression(valueNode) + ) { + const nameProp = t.objectProperty( + t.identifier('name'), + t.stringLiteral(t.isIdentifier(keyNode) ? keyNode.name : keyNode.value) + ); + + const newProps = [nameProp, ...valueNode.properties]; + const objectExpr = t.objectExpression(newProps); + elements.push(objectExpr); + } + } + + valuePath.replaceWith(t.arrayExpression(elements)); + } + protected fullPath(path: NodePath): string { // @ts-ignore let fp = path?.node?.key?.name || ''; diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts new file mode 100644 index 0000000000000..de20db130d3fa --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts @@ -0,0 +1,28 @@ +import { TranspilerSymbolResolver } from './transpiler.interface'; +import { CONTEXT_SYMBOLS, CURRENT_CUBE_CONSTANTS } from '../CubeSymbols'; + +export class LightweightJoinResolver implements TranspilerSymbolResolver { + public constructor(private cubeJoinAliases: Record> = {}) { + } + + public setJoinAliases(cubeJoinAliases: Record>) { + this.cubeJoinAliases = cubeJoinAliases; + } + + public isCurrentCube(name: string): boolean { + return CURRENT_CUBE_CONSTANTS.indexOf(name) >= 0; + } + + public resolveSymbol(cubeName: string, name: string): any { + if (name === 'USER_CONTEXT') { + throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.'); + } + + if (CONTEXT_SYMBOLS[name]) { + return true; + } + + const cube = this.cubeJoinAliases[this.isCurrentCube(name) ? cubeName : name]; + return !!(cube || this.cubeJoinAliases[cubeName]?.[name]); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts index 803a1c0fa4082..7885f8e51332c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts @@ -8,10 +8,10 @@ export interface TranspilerInterface { } export interface TranspilerSymbolResolver { - resolveSymbol(cubeName, name): any; - isCurrentCube(name): boolean; + resolveSymbol(cubeName: string | null, name: string): any; + isCurrentCube(name: string): boolean; } export interface TranspilerCubeResolver { - resolveCube(name): boolean; + resolveCube(name: string): boolean; } diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts index 93c1c142744f0..8c5bcb8c8536e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts @@ -10,6 +10,7 @@ import { CubePropContextTranspiler } from './CubePropContextTranspiler'; import { ErrorReporter } from '../ErrorReporter'; import { LightweightSymbolResolver } from './LightweightSymbolResolver'; import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary'; +import { LightweightJoinResolver } from './LightweightJoinResolver'; type TransferContent = { fileName: string; @@ -17,22 +18,25 @@ type TransferContent = { transpilers: string[]; cubeNames: string[]; cubeSymbols: Record>; + cubeJoins: Record>; }; const cubeDictionary = new LightweightNodeCubeDictionary(); const cubeSymbols = new LightweightSymbolResolver(); +const cubeJoinsResolver = new LightweightJoinResolver(); const errorsReport = new ErrorReporter(null, []); const transpilers = { ValidationTranspiler: new ValidationTranspiler(), ImportExportTranspiler: new ImportExportTranspiler(), CubeCheckDuplicatePropTranspiler: new CubeCheckDuplicatePropTranspiler(), - CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols), + CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols, cubeJoinsResolver), }; const transpile = (data: TransferContent) => { cubeDictionary.setCubeNames(data.cubeNames); cubeSymbols.setSymbols(data.cubeSymbols); + cubeJoinsResolver.setJoinAliases(data.cubeJoins); const ast = parse( data.content, diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts index 049e938ba2243..9ed21de944884 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts @@ -412,13 +412,13 @@ view(\`OrdersView3\`, { it('check includes are exposed in meta', async () => { await compiler.compile(); const cube = metaTransformer.cubes.find(c => c.config.name === 'OrdersView'); - expect(cube.config.measures.find((({ name }) => name === 'OrdersView.count')).name).toBe('OrdersView.count'); + expect(cube?.config.measures.find((({ name }) => name === 'OrdersView.count'))?.name).toBe('OrdersView.count'); }); it('orders are hidden', async () => { await compiler.compile(); const cube = metaTransformer.cubes.find(c => c.config.name === 'Orders'); - expect(cube.config.measures.filter((({ isVisible }) => isVisible)).length).toBe(0); + expect(cube?.config.measures.filter((({ isVisible }) => isVisible)).length).toBe(0); }); it('split views', async () => runQueryTest({ diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index a2083b8601b9e..eee70c19d64f3 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -166,7 +166,7 @@ Object { }, "fileName": "custom_calendar.js", "hierarchies": Object {}, - "joins": Object {}, + "joins": Array [], "measures": Object { "count": Object { "ownedByCube": true, @@ -385,7 +385,7 @@ Object { "title": "Retail Calendar Hierarchy", }, }, - "joins": Object {}, + "joins": Array [], "measures": Object { "count": Object { "ownedByCube": true, @@ -585,25 +585,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions): measures 1`] = ` @@ -925,25 +928,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions): measures 1`] = ` @@ -1223,25 +1229,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions): measures 1`] = ` @@ -1563,25 +1572,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions): measures 1`] = ` @@ -1677,6 +1689,73 @@ Object { } `; +exports[`Schema Testing Joins join types (joins as array) 1`] = ` +Array [ + Object { + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + +exports[`Schema Testing Joins join types (joins as object) 1`] = ` +Array [ + Object { + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + +exports[`Schema Testing Joins join aliases (joins as array) 1`] = ` +Array [ + Object { + "alias": "CubeB_alias1", + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "alias": "CubeB_alias2", + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + exports[`Schema Testing Views allows to override \`title\`, \`description\`, \`meta\`, and \`format\` on includes members 1`] = ` Object { "accessPolicy": undefined, @@ -1813,7 +1892,7 @@ Object { }, ], "isView": true, - "joins": Object {}, + "joins": Array [], "measures": Object { "count": Object { "aggType": "count", diff --git a/packages/cubejs-schema-compiler/test/unit/folders.test.ts b/packages/cubejs-schema-compiler/test/unit/folders.test.ts index 22b0bdb89257f..fb8c8fb3b9b5e 100644 --- a/packages/cubejs-schema-compiler/test/unit/folders.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/folders.test.ts @@ -3,6 +3,7 @@ import path from 'path'; import { CubeToMetaTransformer } from 'src/compiler/CubeToMetaTransformer'; import { prepareYamlCompiler } from './PrepareCompiler'; +import { FolderDefinition } from '../../src/compiler/CubeSymbols'; describe('Cube Folders', () => { let metaTransformer: CubeToMetaTransformer; @@ -25,20 +26,20 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view' ); - expect(emptyView.config.folders.length).toBe(2); + expect(emptyView?.config.folders.length).toBe(2); - const folder1 = emptyView.config.folders.find( + const folder1 = emptyView?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view.age', 'test_view.renamed_gender', ]); - const folder2 = emptyView.config.folders.find( + const folder2 = emptyView?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view.age', 'test_view.renamed_gender']) ); }); @@ -48,29 +49,29 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view4' ); - expect(testView.config.folders.length).toBe(3); + expect(testView?.config.folders.length).toBe(3); - const folder1 = testView.config.folders.find( + const folder1 = testView?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view4.users_age', 'test_view4.users_state', 'test_view4.renamed_orders_status', ]); - const folder2 = testView.config.folders.find( + const folder2 = testView?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view4.users_city', 'test_view4.users_renamed_in_view3_gender']) ); - const folder3 = testView.config.folders.find( + const folder3 = testView?.config.folders.find( (it) => it.name === 'folder3' ); - expect(folder3.members.length).toBe(9); - expect(folder3.members).toEqual([ + expect(folder3?.members.length).toBe(9); + expect(folder3?.members).toEqual([ 'test_view4.users_city', 'test_view4.renamed_orders_status', 'test_view4.renamed_orders_count', @@ -99,43 +100,43 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view4' ); - expect(testView.config.folders.length).toBe(5); + expect(testView?.config.folders.length).toBe(5); - const folder1 = testView.config.folders.find( + const folder1 = testView?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view4.users_age', 'test_view4.users_state', 'test_view4.renamed_orders_status', ]); - const folder2 = testView.config.folders.find( + const folder2 = testView?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view4.users_city', 'test_view4.users_renamed_in_view3_gender']) ); - const folder3 = testView.config.folders.find( + const folder3 = testView?.config.folders.find( (it) => it.name === 'folder3' ); - expect(folder3.members.length).toBe(1); - expect(folder3.members).toEqual([ + expect(folder3?.members.length).toBe(1); + expect(folder3?.members).toEqual([ 'test_view4.users_city', ]); - const folder4 = testView.config.folders.find( + const folder4 = testView?.config.folders.find( (it) => it.name === 'folder3/inner folder 4' ); - expect(folder4.members.length).toBe(1); - expect(folder4.members).toEqual(['test_view4.renamed_orders_status']); + expect(folder4?.members.length).toBe(1); + expect(folder4?.members).toEqual(['test_view4.renamed_orders_status']); - const folder5 = testView.config.folders.find( + const folder5 = testView?.config.folders.find( (it) => it.name === 'folder3/inner folder 5' ); - expect(folder5.members.length).toBe(9); - expect(folder5.members).toEqual([ + expect(folder5?.members.length).toBe(9); + expect(folder5?.members).toEqual([ 'test_view4.renamed_orders_count', 'test_view4.renamed_orders_id', 'test_view4.renamed_orders_number', @@ -153,33 +154,33 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view4' ); - expect(testView.config.nestedFolders.length).toBe(3); + expect(testView?.config.nestedFolders.length).toBe(3); - const folder1 = testView.config.nestedFolders.find( + const folder1 = testView?.config.nestedFolders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view4.users_age', 'test_view4.users_state', 'test_view4.renamed_orders_status', ]); - const folder2 = testView.config.nestedFolders.find( + const folder2 = testView?.config.nestedFolders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view4.users_city', 'test_view4.users_renamed_in_view3_gender']) ); - const folder3 = testView.config.nestedFolders.find( + const folder3 = testView?.config.nestedFolders.find( (it) => it.name === 'folder3' ); - expect(folder3.members.length).toBe(3); - expect(folder3.members[1]).toEqual( + expect(folder3?.members.length).toBe(3); + expect(folder3?.members[1]).toEqual( { name: 'inner folder 4', members: ['test_view4.renamed_orders_status'] } ); - expect(folder3.members[2].name).toEqual('inner folder 5'); - expect(folder3.members[2].members).toEqual([ + expect((folder3?.members[2] as FolderDefinition).name).toEqual('inner folder 5'); + expect((folder3?.members[2] as FolderDefinition).members).toEqual([ 'test_view4.renamed_orders_count', 'test_view4.renamed_orders_id', 'test_view4.renamed_orders_number', @@ -200,31 +201,31 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view3' ); - expect(view2.config.folders.length).toBe(1); - expect(view3.config.folders.length).toBe(2); + expect(view2?.config.folders.length).toBe(1); + expect(view3?.config.folders.length).toBe(2); - const folder1 = view2.config.folders.find( + const folder1 = view2?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view2.users_age', 'test_view2.users_state', 'test_view2.renamed_orders_status', ]); - const folder1v3 = view3.config.folders.find( + const folder1v3 = view3?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1v3.members).toEqual([ + expect(folder1v3?.members).toEqual([ 'test_view3.users_age', 'test_view3.users_state', 'test_view3.renamed_orders_status', ]); - const folder2 = view3.config.folders.find( + const folder2 = view3?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view3.users_city', 'test_view3.users_renamed_in_view3_gender']) ); }); @@ -252,10 +253,10 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view2' ); - expect(view.config.folders.length).toBe(1); + expect(view?.config.folders.length).toBe(1); - const folder1 = view.config.folders.find((it) => it.name === 'folder1'); - expect(folder1.members).toEqual([ + const folder1 = view?.config.folders.find((it) => it.name === 'folder1'); + expect(folder1?.members).toEqual([ 'test_view2.users_age', 'test_view2.users_state', 'test_view2.renamed_orders_status', diff --git a/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts b/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts index 0f2566ccfaa7e..afe8685b6cbbb 100644 --- a/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts @@ -15,7 +15,7 @@ describe('Cube hierarchies', () => { const ordersView = metaTransformer.cubes.find( (it) => it.config.name === 'orders_users_view' - ); + )!; expect(ordersView.config.hierarchies.length).toBe(2); expect(ordersView.config.hierarchies).toEqual([ @@ -42,19 +42,19 @@ describe('Cube hierarchies', () => { const ordersIncludesExcludesView = metaTransformer.cubes.find( (it) => it.config.name === 'orders_includes_excludes_view' ); - expect(ordersIncludesExcludesView.config.hierarchies.length).toBe(1); + expect(ordersIncludesExcludesView?.config.hierarchies.length).toBe(1); const emptyView = metaTransformer.cubes.find( (it) => it.config.name === 'empty_view' ); - expect(emptyView.config.hierarchies.length).toBe(0); + expect(emptyView?.config.hierarchies.length).toBe(0); const allHierarchyView = metaTransformer.cubes.find( (it) => it.config.name === 'all_hierarchy_view' ); - expect(allHierarchyView.config.hierarchies.length).toBe(3); + expect(allHierarchyView?.config.hierarchies.length).toBe(3); - const prefixedHierarchy = allHierarchyView.config.hierarchies.find((it) => it.name === 'all_hierarchy_view.users_users_hierarchy'); + const prefixedHierarchy = allHierarchyView?.config.hierarchies.find((it) => it.name === 'all_hierarchy_view.users_users_hierarchy'); expect(prefixedHierarchy).toBeTruthy(); expect(prefixedHierarchy?.aliasMember).toEqual('users.users_hierarchy'); expect(prefixedHierarchy?.levels).toEqual(['all_hierarchy_view.users_age', 'all_hierarchy_view.users_city']); @@ -73,7 +73,7 @@ describe('Cube hierarchies', () => { (it) => it.config.name === 'only_hierarchy_included_view' ); - expect(view1.config.dimensions).toEqual( + expect(view1?.config.dimensions).toEqual( expect.arrayContaining([ expect.objectContaining({ name: 'only_hierarchy_included_view.status' }), expect.objectContaining({ name: 'only_hierarchy_included_view.number' }), @@ -85,8 +85,8 @@ describe('Cube hierarchies', () => { const view2 = metaTransformer.cubes.find( (it) => it.config.name === 'auto_include_view' ); - expect(view2.config.dimensions.length).toEqual(2); - expect(view2.config.dimensions).toEqual( + expect(view2?.config.dimensions.length).toEqual(2); + expect(view2?.config.dimensions).toEqual( expect.arrayContaining([ expect.objectContaining({ name: 'auto_include_view.status' }), expect.objectContaining({ name: 'auto_include_view.number' }), @@ -177,7 +177,7 @@ describe('Cube hierarchies', () => { (it) => it.config.name === 'orders' ); - expect(ordersCube.config.hierarchies).toEqual([ + expect(ordersCube?.config.hierarchies).toEqual([ { aliasMember: undefined, name: 'orders.hello', diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 9fb37a1ca66ed..10951256766b1 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -350,8 +350,8 @@ describe('Schema Testing', () => { expect(dimensions.length).toBeGreaterThan(0); expect(dimensions.every((dimension) => dimension.primaryKey)).toBeDefined(); expect(dimensions.every((dimension) => typeof dimension.primaryKey === 'boolean')).toBe(true); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.id').primaryKey).toBe(true); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.type').primaryKey).toBe(false); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.id')?.primaryKey).toBe(true); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.type')?.primaryKey).toBe(false); }); it('descriptions', async () => { @@ -369,15 +369,15 @@ describe('Schema Testing', () => { expect(dimensions).toBeDefined(); expect(dimensions.length).toBeGreaterThan(0); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.id').description).toBe('id dimension from createCubeSchema'); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.id')?.description).toBe('id dimension from createCubeSchema'); expect(measures).toBeDefined(); expect(measures.length).toBeGreaterThan(0); - expect(measures.find((measure) => measure.name === 'CubeA.count').description).toBe('count measure from createCubeSchema'); + expect(measures.find((measure) => measure.name === 'CubeA.count')?.description).toBe('count measure from createCubeSchema'); expect(segments).toBeDefined(); expect(segments.length).toBeGreaterThan(0); - expect(segments.find((segment) => segment.name === 'CubeA.sfUsers').description).toBe('SF users segment from createCubeSchema'); + expect(segments.find((segment) => segment.name === 'CubeA.sfUsers')?.description).toBe('SF users segment from createCubeSchema'); }); it('custom granularities in meta', async () => { @@ -393,64 +393,143 @@ describe('Schema Testing', () => { const dg = dimensions.find((dimension) => dimension.name === 'orders.createdAt'); expect(dg).toBeDefined(); - expect(dg.granularities).toBeDefined(); - expect(dg.granularities.length).toBeGreaterThan(0); + expect(dg?.granularities).toBeDefined(); + expect(dg?.granularities?.length).toBeGreaterThan(0); // Granularity defined with title - let gr = dg.granularities.find(g => g.name === 'half_year'); + let gr = dg?.granularities?.find(g => g.name === 'half_year'); expect(gr).toBeDefined(); - expect(gr.title).toBe('6 month intervals'); - expect(gr.interval).toBe('6 months'); + expect(gr?.title).toBe('6 month intervals'); + expect(gr?.interval).toBe('6 months'); - gr = dg.granularities.find(g => g.name === 'half_year_by_1st_april'); + gr = dg?.granularities?.find(g => g.name === 'half_year_by_1st_april'); expect(gr).toBeDefined(); - expect(gr.title).toBe('Half year from Apr to Oct'); - expect(gr.interval).toBe('6 months'); - expect(gr.offset).toBe('3 months'); + expect(gr?.title).toBe('Half year from Apr to Oct'); + expect(gr?.interval).toBe('6 months'); + expect(gr?.offset).toBe('3 months'); // // Granularity defined without title -> titlize() - gr = dg.granularities.find(g => g.name === 'half_year_by_1st_june'); + gr = dg?.granularities?.find(g => g.name === 'half_year_by_1st_june'); expect(gr).toBeDefined(); - expect(gr.title).toBe('Half Year By1 St June'); - expect(gr.interval).toBe('6 months'); - expect(gr.origin).toBe('2020-06-01 10:00:00'); + expect(gr?.title).toBe('Half Year By1 St June'); + expect(gr?.interval).toBe('6 months'); + expect(gr?.origin).toBe('2020-06-01 10:00:00'); }); - it('join types', async () => { - const { compiler, cubeEvaluator } = prepareJsCompiler([ - createCubeSchema({ - name: 'CubeA', - joins: `{ - CubeB: { - sql: \`SQL ON clause\`, - relationship: 'one_to_one' - }, - CubeC: { - sql: \`SQL ON clause\`, - relationship: 'one_to_many' - }, - CubeD: { - sql: \`SQL ON clause\`, - relationship: 'many_to_one' - }, - }` - }), - createCubeSchema({ - name: 'CubeB', - }), - createCubeSchema({ - name: 'CubeC', - }), - createCubeSchema({ - name: 'CubeD', - }), - ]); - await compiler.compile(); + describe('Joins', () => { + it('join types (joins as object)', async () => { + const { compiler, cubeEvaluator } = prepareJsCompiler([ + createCubeSchema({ + name: 'CubeA', + joins: `{ + CubeB: { + sql: \`SQL ON clause\`, + relationship: 'one_to_one' + }, + CubeC: { + sql: \`SQL ON clause\`, + relationship: 'one_to_many' + }, + CubeD: { + sql: \`SQL ON clause\`, + relationship: 'many_to_one' + }, + }` + }), + createCubeSchema({ + name: 'CubeB', + }), + createCubeSchema({ + name: 'CubeC', + }), + createCubeSchema({ + name: 'CubeD', + }), + ]); + await compiler.compile(); + + expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); + }); + + it('join types (joins as array)', async () => { + const { compiler, cubeEvaluator } = prepareJsCompiler([ + createCubeSchema({ + name: 'CubeA', + joins: `[ + { + name: 'CubeB', + sql: \`SQL ON clause\`, + relationship: 'one_to_one' + }, + { + name: 'CubeC', + sql: \`SQL ON clause\`, + relationship: 'one_to_many' + }, + { + name: 'CubeD', + sql: \`SQL ON clause\`, + relationship: 'many_to_one' + }, + ]` + }), + createCubeSchema({ + name: 'CubeB', + }), + createCubeSchema({ + name: 'CubeC', + }), + createCubeSchema({ + name: 'CubeD', + }), + ]); + await compiler.compile(); + + expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); + }); + + it('join aliases (joins as array)', async () => { + const { compiler, cubeEvaluator } = prepareJsCompiler([ + createCubeSchema({ + name: 'CubeA', + joins: `[ + { + name: 'CubeB', + sql: \`SQL ON clause1\`, + relationship: 'one_to_one', + alias: 'CubeB_alias1' + }, + { + name: 'CubeB', + sql: \`SQL ON clause2\`, + relationship: 'one_to_one', + alias: 'CubeB_alias2' + }, + { + name: 'CubeC', + sql: \`SQL ON clause\`, + relationship: 'one_to_many' + }, + { + name: 'CubeD', + sql: \`SQL ON clause\`, + relationship: 'many_to_one' + }, + ]` + }), + createCubeSchema({ + name: 'CubeB', + }), + createCubeSchema({ + name: 'CubeC', + }), + createCubeSchema({ + name: 'CubeD', + }), + ]); + await compiler.compile(); - expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchObject({ - CubeB: { relationship: 'hasOne' }, - CubeC: { relationship: 'hasMany' }, - CubeD: { relationship: 'belongsTo' } + expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); }); }); diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 8ff619a34f861..521d10482d77c 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -384,15 +384,15 @@ describe('Yaml Schema Testing', () => { expect(dimensions).toBeDefined(); expect(dimensions.length).toBeGreaterThan(0); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.id').description).toBe('id dimension from YAML test cube'); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.id')?.description).toBe('id dimension from YAML test cube'); expect(measures).toBeDefined(); expect(measures.length).toBeGreaterThan(0); - expect(measures.find((measure) => measure.name === 'CubeA.count').description).toBe('count measure from YAML test cube'); + expect(measures.find((measure) => measure.name === 'CubeA.count')?.description).toBe('count measure from YAML test cube'); expect(segments).toBeDefined(); expect(segments.length).toBeGreaterThan(0); - expect(segments.find((segment) => segment.name === 'CubeA.sfUsers').description).toBe('SF users segment from createCubeSchema'); + expect(segments.find((segment) => segment.name === 'CubeA.sfUsers')?.description).toBe('SF users segment from createCubeSchema'); }); describe('Custom dimension granularities: ', () => { @@ -629,4 +629,51 @@ describe('Yaml Schema Testing', () => { await compiler.compile(); }); + + it('joins with aliases - success', async () => { + const { compiler } = prepareYamlCompiler( + ` + cubes: + - name: CubeA + sql: "select * from tbl" + joins: + - name: CubeB + sql: SQL ON clause1 + relationship: one_to_one + alias: CubeB_alias1 + + - name: CubeB + sql: SQL ON clause2 + relationship: one_to_one + alias: CubeB_alias2 + + - name: CubeC + sql: SQL ON clause + relationship: one_to_many + + - name: CubeD + sql: SQL ON clause + relationship: many_to_one + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: created_at + sql: created_at + type: time + measures: + - name: count + type: count + - name: CubeB + sql: "select * from tbl" + - name: CubeC + sql: "select * from tbl" + - name: CubeD + sql: "select * from tbl" + ` + ); + + await compiler.compile(); + }); }); diff --git a/yarn.lock b/yarn.lock index 4a85bba90af51..3eadf7ee5fafa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7769,6 +7769,11 @@ expect "^29.0.0" pretty-format "^29.0.0" +"@types/js-yaml@^4.0.9": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.9.tgz#cd82382c4f902fed9691a2ed79ec68c5898af4c2" + integrity sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg== + "@types/json-bigint@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/json-bigint/-/json-bigint-1.0.1.tgz#201062a6990119a8cc18023cfe1fed12fc2fc8a7"