From d191a61733151e221df1f6f51608577a0ef3827c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 15:43:43 +0300 Subject: [PATCH 01/37] more types in CubeSymbols --- .../src/compiler/CubeSymbols.ts | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index c7acc1ab20304..bfd171dc6a4b5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -100,6 +100,7 @@ export type PreAggregationDefinitionRollup = BasePreAggregationDefinition & { export type PreAggregationDefinition = PreAggregationDefinitionRollup; export type JoinDefinition = { + name?: string, // TODO: Make it required relationship: string, sql: (...args: any[]) => string, }; @@ -219,17 +220,17 @@ export class CubeSymbols { } public createCube(cubeDefinition: CubeDefinition): CubeDefinitionExtended { - let preAggregations: any; - let joins: any; - let measures: any; - let dimensions: any; - let segments: any; - let hierarchies: any; - let accessPolicy: any; - let folders: any; - let cubes: any; - - const cubeObject = Object.assign({ + let preAggregations: CubeDefinition['preAggregations']; + let joins: CubeDefinition['joins']; + let measures: CubeDefinition['measures']; + let dimensions: CubeDefinition['dimensions']; + let segments: CubeDefinition['segments']; + let hierarchies: CubeDefinition['hierarchies']; + let accessPolicy: CubeDefinition['accessPolicy']; + let folders: CubeDefinition['folders']; + let cubes: CubeDefinition['cubes']; + + const cubeObject: CubeDefinitionExtended = Object.assign({ allDefinitions(type: string) { if (cubeDefinition.extends) { return { @@ -380,9 +381,10 @@ export class CubeSymbols { return cubeObject; } - protected transform(cubeName: string, errorReporter: ErrorReporter, splitViews: SplitViews) { + protected transform(cubeName: string, errorReporter: ErrorReporter, splitViews: SplitViews): CubeSymbolsDefinition { const cube = this.getCubeDefinition(cubeName); - const duplicateNames = R.compose( + // @ts-ignore + const duplicateNames: string[] = R.compose( R.map((nameToDefinitions: any) => nameToDefinitions[0]), R.toPairs, R.filter((definitionsByName: any) => definitionsByName.length > 1), @@ -394,9 +396,7 @@ export class CubeSymbols { // @ts-ignore )([cube.measures, cube.dimensions, cube.segments, cube.preAggregations, cube.hierarchies]); - // @ts-ignore if (duplicateNames.length > 0) { - // @ts-ignore errorReporter.error(`${duplicateNames.join(', ')} defined more than once`); } @@ -424,23 +424,24 @@ export class CubeSymbols { ...cube.dimensions || {}, ...cube.segments || {}, ...cube.preAggregations || {} - }; + } as CubeSymbolsDefinition; } - private camelCaseTypes(obj: Object | undefined) { + private camelCaseTypes(obj: Object | Array | undefined) { if (!obj) { return; } - // eslint-disable-next-line no-restricted-syntax - for (const member of Object.values(obj)) { + const members = Array.isArray(obj) ? obj : Object.values(obj); + + members.forEach(member => { if (member.type && member.type.indexOf('_') !== -1) { member.type = camelize(member.type, true); } if (member.relationship && member.relationship.indexOf('_') !== -1) { member.relationship = camelize(member.relationship, true); } - } + }); } protected transformPreAggregations(preAggregations: Object) { @@ -545,7 +546,7 @@ export class CubeSymbols { } } - const includeMembers = this.generateIncludeMembers(cubeIncludes, cube.name, type); + const includeMembers = this.generateIncludeMembers(cubeIncludes, type); this.applyIncludeMembers(includeMembers, cube, type, errorReporter); const existing = cube.includedMembers ?? []; @@ -696,11 +697,7 @@ export class CubeSymbols { splitViewDef = splitViews[viewName]; } - const includeMembers = this.generateIncludeMembers( - finalIncludes, - parentCube.name, - type - ); + const includeMembers = this.generateIncludeMembers(finalIncludes, type); this.applyIncludeMembers(includeMembers, splitViewDef, type, errorReporter); } else { for (const member of finalIncludes) { @@ -730,7 +727,7 @@ export class CubeSymbols { return this.symbols[cubeName]?.cubeObj()?.[type]?.[memberName]; } - protected generateIncludeMembers(members: any[], cubeName: string, type: string) { + protected generateIncludeMembers(members: any[], type: string) { return members.map(memberRef => { const path = memberRef.member.split('.'); const resolvedMember = this.getResolvedMember(type, path[path.length - 2], path[path.length - 1]); @@ -867,9 +864,8 @@ export class CubeSymbols { /** * Split join path to member to join hint and member path: `A.B.C.D.E.dim` => `[A, B, C, D, E]` + `E.dim` - * @param path */ - public static joinHintFromPath(path: string): { path: string, joinHint: Array } { + public static joinHintFromPath(path: string): { path: string, joinHint: string[] } { const parts = path.split('.'); if (parts.length > 2) { // Path contains join path @@ -905,7 +901,7 @@ export class CubeSymbols { } } - protected withSymbolsCallContext(func, context) { + protected withSymbolsCallContext(func: Function, context) { const oldContext = this.resolveSymbolsCallContext; this.resolveSymbolsCallContext = context; try { @@ -930,7 +926,7 @@ export class CubeSymbols { return this.funcArgumentsValues[funcDefinition]; } - protected joinHints() { + protected joinHints(): string | string[] | undefined { const { joinHints } = this.resolveSymbolsCallContext || {}; if (Array.isArray(joinHints)) { return R.uniq(joinHints); @@ -1011,7 +1007,7 @@ export class CubeSymbols { return (...filterParamArgs) => ''; } - public resolveSymbol(cubeName, name) { + public resolveSymbol(cubeName, name: string) { const { sqlResolveFn, contextSymbols, collectJoinHints, depsResolveFn, currResolveIndexFn } = this.resolveSymbolsCallContext || {}; if (name === 'USER_CONTEXT') { throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.'); From 2afbffeb5c5be08baf51e0ddaf8807aff0705278 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 15:43:58 +0300 Subject: [PATCH 02/37] change joins schema in Validator --- .../src/compiler/CubeValidator.ts | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 32540d4864ac6..a8b99fd77284e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -802,14 +802,25 @@ const baseSchema = { shown: Joi.boolean().strict(), public: Joi.boolean().strict(), meta: Joi.any(), - joins: Joi.object().pattern(identifierRegex, Joi.object().keys({ - sql: Joi.func().required(), - relationship: Joi.any().valid( - 'belongsTo', 'belongs_to', 'many_to_one', 'manyToOne', - 'hasMany', 'has_many', 'one_to_many', 'oneToMany', - 'hasOne', 'has_one', 'one_to_one', 'oneToOne' - ).required() - })), + joins: Joi.alternatives([ + Joi.object().pattern(identifierRegex, Joi.object().keys({ + sql: Joi.func().required(), + relationship: Joi.any().valid( + 'belongsTo', 'belongs_to', 'many_to_one', 'manyToOne', + 'hasMany', 'has_many', 'one_to_many', 'oneToMany', + 'hasOne', 'has_one', 'one_to_one', 'oneToOne' + ).required() + })), + Joi.array().items(Joi.object().keys({ + name: identifier.required(), + sql: Joi.func().required(), + relationship: Joi.any().valid( + 'belongsTo', 'belongs_to', 'many_to_one', 'manyToOne', + 'hasMany', 'has_many', 'one_to_many', 'oneToMany', + 'hasOne', 'has_one', 'one_to_one', 'oneToOne' + ).required() + })) + ]), measures: MeasuresSchema, dimensions: DimensionsSchema, segments: SegmentsSchema, From 8eb3b5aeb7eb779b60a3e2031a3d25c488d3414f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 16:19:06 +0300 Subject: [PATCH 03/37] specify type for accessPolicy --- .../src/compiler/CubeEvaluator.ts | 35 ++++--------------- .../src/compiler/CubeSymbols.ts | 26 +++++++++++++- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 3d2e2e3a993ea..9a0fa62049749 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -2,10 +2,13 @@ import R from 'ramda'; import { + AccessPolicyDefinition, CubeDefinitionExtended, CubeSymbols, - HierarchyDefinition, JoinDefinition, - PreAggregationDefinition, PreAggregationDefinitionRollup, + HierarchyDefinition, + JoinDefinition, + PreAggregationDefinition, + PreAggregationDefinitionRollup, type ToString } from './CubeSymbols'; import { UserError } from './UserError'; @@ -110,30 +113,6 @@ export type EvaluatedHierarchy = { [key: string]: any; }; -export type Filter = - | { - member: string; - memberReference?: string; - [key: string]: any; - } - | { - and?: Filter[]; - or?: Filter[]; - [key: string]: any; - }; - -export type AccessPolicy = { - rowLevel?: { - filters: Filter[]; - }; - memberLevel?: { - includes?: string | string[]; - excludes?: string | string[]; - includesMembers?: string[]; - excludesMembers?: string[]; - }; -}; - export type EvaluatedFolder = { name: string; includes: (EvaluatedFolder | DimensionDefinition | MeasureDefinition)[]; @@ -153,7 +132,7 @@ export type EvaluatedCube = { folders: EvaluatedFolder[]; sql?: (...args: any[]) => string; sqlTable?: (...args: any[]) => string; - accessPolicy?: AccessPolicy[]; + accessPolicy?: AccessPolicyDefinition[]; }; export class CubeEvaluator extends CubeSymbols { @@ -200,7 +179,7 @@ export class CubeEvaluator extends CubeSymbols { ); } - protected prepareCube(cube, errorReporter: ErrorReporter) { + protected prepareCube(cube, errorReporter: ErrorReporter): EvaluatedCube { this.prepareJoins(cube, errorReporter); this.preparePreAggregations(cube, errorReporter); this.prepareMembers(cube.measures, cube, errorReporter); diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index bfd171dc6a4b5..a71b42c9e4c8b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -105,6 +105,30 @@ export type JoinDefinition = { sql: (...args: any[]) => string, }; +export type Filter = + | { + member: string; + memberReference?: string; + [key: string]: any; + } + | { + and?: Filter[]; + or?: Filter[]; + [key: string]: any; + }; + +export type AccessPolicyDefinition = { + rowLevel?: { + filters: Filter[]; + }; + memberLevel?: { + includes?: string | string[]; + excludes?: string | string[]; + includesMembers?: string[]; + excludesMembers?: string[]; + }; +}; + export interface CubeDefinition { name: string; extends?: (...args: Array) => { __cubeName: string }; @@ -121,7 +145,7 @@ export interface CubeDefinition { // eslint-disable-next-line camelcase pre_aggregations?: Record; joins?: Record; - accessPolicy?: any[]; + accessPolicy?: AccessPolicyDefinition[]; folders?: any[]; includes?: any; excludes?: any; From f9f4901766758031daa994835ab005488a8c03f2 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 16:19:23 +0300 Subject: [PATCH 04/37] make prepareJoins() aware of arrays --- .../src/compiler/CubeEvaluator.ts | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 9a0fa62049749..bb10eaf81abdf 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -423,35 +423,39 @@ export class CubeEvaluator extends CubeSymbols { } protected prepareJoins(cube: any, _errorReporter: ErrorReporter) { - if (cube.joins) { - // eslint-disable-next-line no-restricted-syntax - for (const join of Object.values(cube.joins) as any[]) { - // eslint-disable-next-line default-case - switch (join.relationship) { - case 'belongs_to': - case 'many_to_one': - case 'manyToOne': - join.relationship = 'belongsTo'; - break; - case 'has_many': - case 'one_to_many': - case 'oneToMany': - join.relationship = 'hasMany'; - break; - case 'has_one': - case 'one_to_one': - case 'oneToOne': - join.relationship = 'hasOne'; - break; - } - } + if (!cube.joins) { + return; } + + const joins: JoinDefinition[] = Array.isArray(cube.joins) ? cube.joins : Object.values(cube.joins); + + joins.forEach(join => { + // eslint-disable-next-line default-case + switch (join.relationship) { + case 'belongs_to': + case 'many_to_one': + case 'manyToOne': + join.relationship = 'belongsTo'; + break; + case 'has_many': + case 'one_to_many': + case 'oneToMany': + join.relationship = 'hasMany'; + break; + case 'has_one': + case 'one_to_one': + case 'oneToOne': + join.relationship = 'hasOne'; + break; + } + }); } protected preparePreAggregations(cube: any, errorReporter: ErrorReporter) { if (cube.preAggregations) { // eslint-disable-next-line no-restricted-syntax for (const preAggregation of Object.values(cube.preAggregations) as any) { + // preAggregation is actually (PreAggregationDefinitionRollup | PreAggregationDefinitionOriginalSql) if (preAggregation.timeDimension) { preAggregation.timeDimensionReference = preAggregation.timeDimension; delete preAggregation.timeDimension; @@ -553,7 +557,7 @@ export class CubeEvaluator extends CubeSymbols { } } - public cubesByFileName(fileName) { + public cubesByFileName(fileName): CubeDefinitionExtended[] { return this.byFileName[fileName] || []; } @@ -670,7 +674,7 @@ export class CubeEvaluator extends CubeSymbols { return this.preAggregations({ scheduled: true }); } - public cubeNames() { + public cubeNames(): string[] { return Object.keys(this.evaluatedCubes); } From 5a2f978d483cc067b822694af8a6c141d37408d1 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 17:34:22 +0300 Subject: [PATCH 05/37] make joins in CubeSymbols / CubeEvaluator as array instead of hashmap --- .../cubejs-schema-compiler/src/compiler/CubeEvaluator.ts | 2 +- .../cubejs-schema-compiler/src/compiler/CubeSymbols.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index bb10eaf81abdf..dc0f1a2743bca 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -124,7 +124,7 @@ export type EvaluatedCube = { measures: Record; dimensions: Record; segments: Record; - joins: Record; + joins: JoinDefinition[]; hierarchies: Record; evaluatedHierarchies: EvaluatedHierarchy[]; preAggregations: Record; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index a71b42c9e4c8b..4d2e2ff7f2a39 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -100,7 +100,7 @@ export type PreAggregationDefinitionRollup = BasePreAggregationDefinition & { export type PreAggregationDefinition = PreAggregationDefinitionRollup; export type JoinDefinition = { - name?: string, // TODO: Make it required + name: string, relationship: string, sql: (...args: any[]) => string, }; @@ -144,7 +144,7 @@ export interface CubeDefinition { preAggregations?: Record; // eslint-disable-next-line camelcase pre_aggregations?: Record; - joins?: Record; + joins?: JoinDefinition[]; accessPolicy?: AccessPolicyDefinition[]; folders?: any[]; includes?: any; @@ -320,7 +320,8 @@ export class CubeSymbols { get joins() { if (!joins) { - joins = this.allDefinitions('joins'); + const parentJoins = cubeDefinition.extends ? super.joins : []; + joins = [...parentJoins, ...(cubeDefinition.joins || [])]; } return joins; }, From 537dbea9af844bd815bc0e0c4b32205c7b732d2d Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 17:34:51 +0300 Subject: [PATCH 06/37] update JoinGraph to treat joins as array instead of hashmap --- .../src/compiler/JoinGraph.ts | 108 +++++++++--------- 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 179450a0cfed9..39b9acc3296e7 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -109,70 +109,64 @@ export class JoinGraph { } protected buildJoinEdges(cube: CubeDefinition, errorReporter: ErrorReporter): Array<[string, JoinEdge]> { - // @ts-ignore - return R.compose( - // @ts-ignore - R.filter(R.identity), - R.map((join: [string, JoinEdge]) => { - const multipliedMeasures: ((m: Record) => MeasureDefinition[]) = R.compose( - R.filter( - (m: MeasureDefinition): boolean => m.sql && this.cubeEvaluator.funcArguments(m.sql).length === 0 && m.sql() === 'count(*)' || - ['sum', 'avg', 'count', 'number'].indexOf(m.type) !== -1 - ), - R.values as (input: Record) => MeasureDefinition[] - ); - const joinRequired = - (v) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; - if ( - !this.cubeEvaluator.primaryKeys[join[1].from].length && - multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].from)).length > 0 - ) { - errorReporter.error(joinRequired(join[1].from)); - return null; + if (!cube.joins) { + return []; + } + + const getMultipliedMeasures = (cubeName: string): MeasureDefinition[] => { + const measures = this.cubeEvaluator.measuresForCube(cubeName); + return Object.values(measures).filter((m: MeasureDefinition) => (m.sql && + this.cubeEvaluator.funcArguments(m.sql).length === 0 && + m.sql() === 'count(*)') || + ['sum', 'avg', 'count', 'number'].includes(m.type)); + }; + + const joinRequired = + (v) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; + + return cube.joins + .filter(join => { + if (!this.cubeEvaluator.cubeExists(join.name)) { + errorReporter.error(`Cube ${join.name} doesn't exist`); + return false; } - if (!this.cubeEvaluator.primaryKeys[join[1].to].length && - multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].to)).length > 0) { - errorReporter.error(joinRequired(join[1].to)); - return null; + + const fromMultipliedMeasures = getMultipliedMeasures(cube.name); + if (!this.cubeEvaluator.primaryKeys[cube.name].length && fromMultipliedMeasures.length > 0) { + errorReporter.error(joinRequired(cube.name)); + return false; } - return join; - }), - R.unnest, - R.map((join: [string, JoinDefinition]): [[string, JoinEdge]] => [ - [`${cube.name}-${join[0]}`, { - join: join[1], + + const toMultipliedMeasures = getMultipliedMeasures(join.name); + if (!this.cubeEvaluator.primaryKeys[join.name].length && toMultipliedMeasures.length > 0) { + errorReporter.error(joinRequired(join.name)); + return false; + } + + return true; + }) + .map(join => { + const joinEdge: JoinEdge = { + join, from: cube.name, - to: join[0], + to: join.name, originalFrom: cube.name, - originalTo: join[0] - }] - ]), - // @ts-ignore - R.filter(R.identity), - R.map((join: [string, JoinDefinition]) => { - if (!this.cubeEvaluator.cubeExists(join[0])) { - errorReporter.error(`Cube ${join[0]} doesn't exist`); - return undefined; - } - return join; - }), - // @ts-ignore - R.toPairs - // @ts-ignore - )(cube.joins || {}); + originalTo: join.name + }; + + return [`${cube.name}-${join.name}`, joinEdge] as [string, JoinEdge]; + }); } protected buildJoinNode(cube: CubeDefinition): Record { - return R.compose< - Record, - Array<[string, JoinDefinition]>, - Array<[string, 1]>, - Record - >( - R.fromPairs, - R.map(v => [v[0], 1]), - R.toPairs - )(cube.joins || {}); + if (!cube.joins) { + return {}; + } + + return cube.joins.reduce((acc, join) => { + acc[join.name] = 1; + return acc; + }, {} as Record); } public buildJoin(cubesToJoin: JoinHints): FinishedJoinTree | null { From e07a740c494f9b46e4332258d07ec0763e195844 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 17:57:50 +0300 Subject: [PATCH 07/37] fix CubePropContextTranspiler to convert joins hashmap into array --- .../transpilers/CubePropContextTranspiler.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 2ffc96058b3ef..4fb43d08868d8 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -100,6 +100,13 @@ export class CubePropContextTranspiler implements TranspilerInterface { return { ObjectProperty: (path) => { + if (path.node.key.type === 'Identifier' && path.node.key.name === 'joins' && t.isObjectExpression(path.node.value)) { + const fullPath = this.fullPath(path); + if (fullPath === 'joins') { + this.convertJoinsObjectToArray(path); + } + } + if (path.node.key.type === 'Identifier' && transpiledFields.has(path.node.key.name)) { const fullPath = this.fullPath(path); // eslint-disable-next-line no-restricted-syntax @@ -114,6 +121,39 @@ export class CubePropContextTranspiler implements TranspilerInterface { }; } + protected convertJoinsObjectToArray(path: NodePath) { + const valuePath = path.get('value'); + if (!valuePath.isObjectExpression()) { + return; + } + + const elements: t.ObjectExpression[] = []; + + for (const prop of valuePath.get('properties')) { + if (!prop.isObjectProperty()) { + return; + } + const keyNode = prop.node.key; + const valueNode = prop.node.value; + + if ( + (t.isIdentifier(keyNode) || t.isStringLiteral(keyNode)) && + t.isObjectExpression(valueNode) + ) { + const nameProp = t.objectProperty( + t.identifier('name'), + t.stringLiteral(t.isIdentifier(keyNode) ? keyNode.name : keyNode.value) + ); + + const newProps = [nameProp, ...valueNode.properties]; + const objectExpr = t.objectExpression(newProps); + elements.push(objectExpr); + } + } + + valuePath.replaceWith(t.arrayExpression(elements)); + } + protected fullPath(path: NodePath): string { // @ts-ignore let fp = path?.node?.key?.name || ''; From 982e3b479e3efb28d03c80b368d82f7aba5103eb Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 18:21:23 +0300 Subject: [PATCH 08/37] add @types/js-yaml --- packages/cubejs-schema-compiler/package.json | 1 + yarn.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index a20e3ab439822..cfe9d97d1a213 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -67,6 +67,7 @@ "@types/babel__traverse": "^7.20.5", "@types/inflection": "^1.5.28", "@types/jest": "^29", + "@types/js-yaml": "^4.0.9", "@types/node": "^20", "@types/node-dijkstra": "^2.5.6", "@types/ramda": "^0.27.34", diff --git a/yarn.lock b/yarn.lock index 4a85bba90af51..3eadf7ee5fafa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7769,6 +7769,11 @@ expect "^29.0.0" pretty-format "^29.0.0" +"@types/js-yaml@^4.0.9": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.9.tgz#cd82382c4f902fed9691a2ed79ec68c5898af4c2" + integrity sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg== + "@types/json-bigint@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/json-bigint/-/json-bigint-1.0.1.tgz#201062a6990119a8cc18023cfe1fed12fc2fc8a7" From f8c423b399968404aca054819628881bf1312fc7 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 19:29:28 +0300 Subject: [PATCH 09/37] make YamlCompiler aware of joins as array --- packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index d66224582cc0e..49ed2313464b8 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -116,7 +116,7 @@ export class YamlCompiler { } } - private transpileAndPrepareJsFile(file, methodFn, cubeObj, errorsReport: ErrorReporter) { + private transpileAndPrepareJsFile(file: FileContent, methodFn: ('cube' | 'view'), cubeObj, errorsReport: ErrorReporter): FileContent { const yamlAst = this.transformYamlCubeObj(cubeObj, errorsReport); const cubeOrViewCall = t.callExpression(t.identifier(methodFn), [t.stringLiteral(cubeObj.name), yamlAst]); @@ -135,7 +135,7 @@ export class YamlCompiler { cubeObj.dimensions = this.yamlArrayToObj(cubeObj.dimensions || [], 'dimension', errorsReport); cubeObj.segments = this.yamlArrayToObj(cubeObj.segments || [], 'segment', errorsReport); cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations || [], 'preAggregation', errorsReport); - cubeObj.joins = this.yamlArrayToObj(cubeObj.joins || [], 'join', errorsReport); + cubeObj.joins = cubeObj.joins || []; // For edge cases where joins are not defined/null cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport); return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport); From 226f36a1fb96983cad5299934f04ac6942dccec2 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 19:32:20 +0300 Subject: [PATCH 10/37] fix/update snapshots for schema tests --- .../unit/__snapshots__/schema.test.ts.snap | 110 +++++++++++------- .../test/unit/schema.test.ts | 6 +- 2 files changed, 72 insertions(+), 44 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index a2083b8601b9e..303411e0b59c5 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -166,7 +166,7 @@ Object { }, "fileName": "custom_calendar.js", "hierarchies": Object {}, - "joins": Object {}, + "joins": Array [], "measures": Object { "count": Object { "ownedByCube": true, @@ -385,7 +385,7 @@ Object { "title": "Retail Calendar Hierarchy", }, }, - "joins": Object {}, + "joins": Array [], "measures": Object { "count": Object { "ownedByCube": true, @@ -585,25 +585,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions): measures 1`] = ` @@ -925,25 +928,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions): measures 1`] = ` @@ -1223,25 +1229,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions): measures 1`] = ` @@ -1563,25 +1572,28 @@ Object { `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions): joins 1`] = ` -Object { - "order_users": Object { +Array [ + Object { + "name": "order_users", "relationship": "belongsTo", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions): joins 2`] = ` -Object { - "line_items": Object { - "relationship": "hasMany", +Array [ + Object { + "name": "order_users", + "relationship": "belongsTo", "sql": [Function], }, - "order_users": Object { - "relationship": "belongsTo", + Object { + "name": "line_items", + "relationship": "hasMany", "sql": [Function], }, -} +] `; exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions): measures 1`] = ` @@ -1813,7 +1825,7 @@ Object { }, ], "isView": true, - "joins": Object {}, + "joins": Array [], "measures": Object { "count": Object { "aggType": "count", @@ -2078,3 +2090,23 @@ Array [ }, ] `; + +exports[`Schema Testing join types 1`] = ` +Array [ + Object { + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 9fb37a1ca66ed..ac11a6cebb408 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -447,11 +447,7 @@ describe('Schema Testing', () => { ]); await compiler.compile(); - expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchObject({ - CubeB: { relationship: 'hasOne' }, - CubeC: { relationship: 'hasMany' }, - CubeD: { relationship: 'belongsTo' } - }); + expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); }); describe('Access Policies', () => { From 61b2f7dc728105f23ecf7070a12256466cf3a88e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 18 Jul 2025 19:44:01 +0300 Subject: [PATCH 11/37] fix transform yaml joins --- .../cubejs-schema-compiler/src/compiler/YamlCompiler.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 49ed2313464b8..c36b648500244 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -135,7 +135,13 @@ export class YamlCompiler { cubeObj.dimensions = this.yamlArrayToObj(cubeObj.dimensions || [], 'dimension', errorsReport); cubeObj.segments = this.yamlArrayToObj(cubeObj.segments || [], 'segment', errorsReport); cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations || [], 'preAggregation', errorsReport); + cubeObj.joins = cubeObj.joins || []; // For edge cases where joins are not defined/null + if (!Array.isArray(cubeObj.joins)) { + errorsReport.error('joins must be defined as array'); + cubeObj.joins = []; + } + cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport); return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport); From fdebd11538bbfd27e739c1a19b19d5f191a3ada8 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 21 Jul 2025 15:07:07 +0300 Subject: [PATCH 12/37] add schema tests for joins as array/object --- .../unit/__snapshots__/schema.test.ts.snap | 40 +++++++ .../test/unit/schema.test.ts | 102 ++++++++++++------ 2 files changed, 111 insertions(+), 31 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index 303411e0b59c5..83117c413402d 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -2091,6 +2091,46 @@ Array [ ] `; +exports[`Schema Testing join types (joins as array) 1`] = ` +Array [ + Object { + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + +exports[`Schema Testing join types (joins as object) 1`] = ` +Array [ + Object { + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + exports[`Schema Testing join types 1`] = ` Array [ Object { diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index ac11a6cebb408..1b0d3a11474a4 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -416,38 +416,78 @@ describe('Schema Testing', () => { expect(gr.origin).toBe('2020-06-01 10:00:00'); }); - it('join types', async () => { - const { compiler, cubeEvaluator } = prepareJsCompiler([ - createCubeSchema({ - name: 'CubeA', - joins: `{ - CubeB: { - sql: \`SQL ON clause\`, - relationship: 'one_to_one' - }, - CubeC: { - sql: \`SQL ON clause\`, - relationship: 'one_to_many' - }, - CubeD: { - sql: \`SQL ON clause\`, - relationship: 'many_to_one' - }, - }` - }), - createCubeSchema({ - name: 'CubeB', - }), - createCubeSchema({ - name: 'CubeC', - }), - createCubeSchema({ - name: 'CubeD', - }), - ]); - await compiler.compile(); + describe('Joins', () => { + it('join types (joins as object)', async () => { + const { compiler, cubeEvaluator } = prepareJsCompiler([ + createCubeSchema({ + name: 'CubeA', + joins: `{ + CubeB: { + sql: \`SQL ON clause\`, + relationship: 'one_to_one' + }, + CubeC: { + sql: \`SQL ON clause\`, + relationship: 'one_to_many' + }, + CubeD: { + sql: \`SQL ON clause\`, + relationship: 'many_to_one' + }, + }` + }), + createCubeSchema({ + name: 'CubeB', + }), + createCubeSchema({ + name: 'CubeC', + }), + createCubeSchema({ + name: 'CubeD', + }), + ]); + await compiler.compile(); + + expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); + }); + + it('join types (joins as array)', async () => { + const { compiler, cubeEvaluator } = prepareJsCompiler([ + createCubeSchema({ + name: 'CubeA', + joins: `[ + { + name: 'CubeB', + sql: \`SQL ON clause\`, + relationship: 'one_to_one' + }, + { + name: 'CubeC', + sql: \`SQL ON clause\`, + relationship: 'one_to_many' + }, + { + name: 'CubeD', + sql: \`SQL ON clause\`, + relationship: 'many_to_one' + }, + ]` + }), + createCubeSchema({ + name: 'CubeB', + }), + createCubeSchema({ + name: 'CubeC', + }), + createCubeSchema({ + name: 'CubeD', + }), + ]); + await compiler.compile(); + + expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); + }); - expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); }); describe('Access Policies', () => { From b093d749b504f9c68eef641d4130126899985da4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 21 Jul 2025 17:29:37 +0300 Subject: [PATCH 13/37] lint:fix --- packages/cubejs-schema-compiler/test/unit/schema.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 1b0d3a11474a4..43dc8cd6839bc 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -487,7 +487,6 @@ describe('Schema Testing', () => { expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); }); - }); describe('Access Policies', () => { From a00154189e15a80fd0cf415740a16c077b20d0da Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 21 Jul 2025 17:41:09 +0300 Subject: [PATCH 14/37] update snapshots --- .../unit/__snapshots__/schema.test.ts.snap | 100 +++++++----------- 1 file changed, 40 insertions(+), 60 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index 83117c413402d..7f79bc7421f29 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -1689,6 +1689,46 @@ Object { } `; +exports[`Schema Testing Joins join types (joins as array) 1`] = ` +Array [ + Object { + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + +exports[`Schema Testing Joins join types (joins as object) 1`] = ` +Array [ + Object { + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + exports[`Schema Testing Views allows to override \`title\`, \`description\`, \`meta\`, and \`format\` on includes members 1`] = ` Object { "accessPolicy": undefined, @@ -2090,63 +2130,3 @@ Array [ }, ] `; - -exports[`Schema Testing join types (joins as array) 1`] = ` -Array [ - Object { - "name": "CubeB", - "relationship": "hasOne", - "sql": [Function], - }, - Object { - "name": "CubeC", - "relationship": "hasMany", - "sql": [Function], - }, - Object { - "name": "CubeD", - "relationship": "belongsTo", - "sql": [Function], - }, -] -`; - -exports[`Schema Testing join types (joins as object) 1`] = ` -Array [ - Object { - "name": "CubeB", - "relationship": "hasOne", - "sql": [Function], - }, - Object { - "name": "CubeC", - "relationship": "hasMany", - "sql": [Function], - }, - Object { - "name": "CubeD", - "relationship": "belongsTo", - "sql": [Function], - }, -] -`; - -exports[`Schema Testing join types 1`] = ` -Array [ - Object { - "name": "CubeB", - "relationship": "hasOne", - "sql": [Function], - }, - Object { - "name": "CubeC", - "relationship": "hasMany", - "sql": [Function], - }, - Object { - "name": "CubeD", - "relationship": "belongsTo", - "sql": [Function], - }, -] -`; From 6c985ca813877c970d9750eb72a69566244ef8b4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 21 Jul 2025 15:34:00 +0300 Subject: [PATCH 15/37] introduce alias in CubeValidator and cover it with schema tests --- .../src/compiler/CubeValidator.ts | 3 +- .../unit/__snapshots__/schema.test.ts.snap | 27 +++++++++++ .../test/unit/schema.test.ts | 44 +++++++++++++++++ .../test/unit/yaml-schema.test.ts | 47 +++++++++++++++++++ 4 files changed, 120 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index a8b99fd77284e..20033b2bb85f2 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -818,7 +818,8 @@ const baseSchema = { 'belongsTo', 'belongs_to', 'many_to_one', 'manyToOne', 'hasMany', 'has_many', 'one_to_many', 'oneToMany', 'hasOne', 'has_one', 'one_to_one', 'oneToOne' - ).required() + ).required(), + alias: identifier, })) ]), measures: MeasuresSchema, diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index 7f79bc7421f29..eee70c19d64f3 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -1729,6 +1729,33 @@ Array [ ] `; +exports[`Schema Testing Joins join aliases (joins as array) 1`] = ` +Array [ + Object { + "alias": "CubeB_alias1", + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "alias": "CubeB_alias2", + "name": "CubeB", + "relationship": "hasOne", + "sql": [Function], + }, + Object { + "name": "CubeC", + "relationship": "hasMany", + "sql": [Function], + }, + Object { + "name": "CubeD", + "relationship": "belongsTo", + "sql": [Function], + }, +] +`; + exports[`Schema Testing Views allows to override \`title\`, \`description\`, \`meta\`, and \`format\` on includes members 1`] = ` Object { "accessPolicy": undefined, diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 43dc8cd6839bc..a832c49864c91 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -487,6 +487,50 @@ describe('Schema Testing', () => { expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); }); + + it('join aliases (joins as array)', async () => { + const { compiler, cubeEvaluator } = prepareJsCompiler([ + createCubeSchema({ + name: 'CubeA', + joins: `[ + { + name: 'CubeB', + sql: \`SQL ON clause1\`, + relationship: 'one_to_one', + alias: 'CubeB_alias1' + }, + { + name: 'CubeB', + sql: \`SQL ON clause2\`, + relationship: 'one_to_one', + alias: 'CubeB_alias2' + }, + { + name: 'CubeC', + sql: \`SQL ON clause\`, + relationship: 'one_to_many' + }, + { + name: 'CubeD', + sql: \`SQL ON clause\`, + relationship: 'many_to_one' + }, + ]` + }), + createCubeSchema({ + name: 'CubeB', + }), + createCubeSchema({ + name: 'CubeC', + }), + createCubeSchema({ + name: 'CubeD', + }), + ]); + await compiler.compile(); + + expect(cubeEvaluator.cubeFromPath('CubeA').joins).toMatchSnapshot(); + }); }); describe('Access Policies', () => { diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 8ff619a34f861..774c8035e96ba 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -629,4 +629,51 @@ describe('Yaml Schema Testing', () => { await compiler.compile(); }); + + it('joins with aliases - success', async () => { + const { compiler } = prepareYamlCompiler( + ` + cubes: + - name: CubeA + sql: "select * from tbl" + joins: + - name: CubeB + sql: SQL ON clause1 + relationship: one_to_one + alias: CubeB_alias1 + + - name: CubeB + sql: SQL ON clause2 + relationship: one_to_one + alias: CubeB_alias2 + + - name: CubeC + sql: SQL ON clause + relationship: one_to_many + + - name: CubeD + sql: SQL ON clause + relationship: many_to_one + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: created_at + sql: created_at + type: time + measures: + - name: count + type: count + - name: CubeB + sql: "select * from tbl" + - name: CubeC + sql: "select * from tbl" + - name: CubeD + sql: "select * from tbl" + ` + ); + + await compiler.compile(); + }); }); From 9ca17c158327d9b277600f753fe396291e1f59f8 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 21 Jul 2025 17:24:06 +0300 Subject: [PATCH 16/37] Move CubeDictionary to TS --- .../src/compiler/CubeDictionary.js | 16 ---------------- .../src/compiler/CubeDictionary.ts | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 packages/cubejs-schema-compiler/src/compiler/CubeDictionary.js create mode 100644 packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.js b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.js deleted file mode 100644 index a3fe63e9ec092..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.js +++ /dev/null @@ -1,16 +0,0 @@ -import R from 'ramda'; - -export class CubeDictionary { - constructor() { - this.byId = {}; - } - - // eslint-disable-next-line no-unused-vars - compile(cubes, errorReporter) { - this.byId = R.fromPairs(cubes.map(v => [v.name, v])); - } - - resolveCube(cubeName) { - return this.byId[cubeName]; - } -} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts new file mode 100644 index 0000000000000..db1aeb372618d --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts @@ -0,0 +1,17 @@ +import { ErrorReporter } from './ErrorReporter'; + +export class CubeDictionary { + public byId: Record; + + public constructor() { + this.byId = {}; + } + + public compile(cubes: any[], _errorReporter: ErrorReporter) { + this.byId = Object.fromEntries(cubes.map(v => [v.name, v])); + } + + public resolveCube(cubeName: string): any { + return this.byId[cubeName]; + } +} From f64c694415b0049134f8699b3818ca08b7ca54b8 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 21 Jul 2025 17:47:21 +0300 Subject: [PATCH 17/37] fix typos --- .../src/compiler/ViewCompilationGate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts index af53f12f30de6..90a038771f910 100644 --- a/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts +++ b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts @@ -6,10 +6,10 @@ export class ViewCompilationGate { } public compile(cubes: any[]) { - // When developing Data Access Policies feature, we've came across a + // When developing Data Access Policies feature, we've come across a // limitation that Cube members can't be referenced in access policies defined on Views, // because views aren't (yet) compiled at the time of access policy evaluation. - // To workaround this limitation and additional compilation pass is necessary, + // To work around this limitation additional compilation pass is necessary, // however it comes with a significant performance penalty. // This gate check whether the data model contains views with access policies, // and only then allows the additional compilation pass. From fda07c2fed22ae9b80d41959ac00ad0dcef6a371 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 21 Jul 2025 18:36:38 +0300 Subject: [PATCH 18/37] Move ContextEvaluator to TS --- .../src/compiler/ContextEvaluator.js | 35 -------------- .../src/compiler/ContextEvaluator.ts | 47 +++++++++++++++++++ .../src/compiler/CubeSymbols.ts | 2 +- 3 files changed, 48 insertions(+), 36 deletions(-) delete mode 100644 packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.js create mode 100644 packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.js b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.js deleted file mode 100644 index a3a03b3652cd1..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.js +++ /dev/null @@ -1,35 +0,0 @@ -import R from 'ramda'; - -export class ContextEvaluator { - /** - * @param {import('./CubeEvaluator').CubeEvaluator} cubeEvaluator - */ - constructor(cubeEvaluator) { - this.cubeEvaluator = cubeEvaluator; - this.contextDefinitions = {}; - } - - // eslint-disable-next-line no-unused-vars - compile(contexts, _errorReporter) { - if (contexts.length === 0) { - return; - } - - // TODO: handle duplications, context names must be uniq - this.contextDefinitions = R.compose( - R.fromPairs, - R.map(v => [v.name, this.compileContext(v)]) - )(contexts); - } - - compileContext(context) { - return { - name: context.name, - contextMembers: this.cubeEvaluator.evaluateReferences(null, context.contextMembers) - }; - } - - get contextList() { - return R.keys(this.contextDefinitions); - } -} diff --git a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts new file mode 100644 index 0000000000000..d458c09d96213 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts @@ -0,0 +1,47 @@ +import { CubeEvaluator } from './CubeEvaluator'; +import { ErrorReporter } from './ErrorReporter'; + +export type ContextDefinition = { + name: string; + contextMembers: string | string[]; +}; + +export class ContextEvaluator { + private cubeEvaluator: CubeEvaluator; + + private contextDefinitions: Record; + + public constructor(cubeEvaluator: CubeEvaluator) { + this.cubeEvaluator = cubeEvaluator; + this.contextDefinitions = {}; + } + + public compile(contexts: any, errorReporter: ErrorReporter) { + if (contexts.length === 0) { + return; + } + + const definitions: Record = {}; + + for (const v of contexts) { + if (definitions[v.name]) { + errorReporter.error(`Duplicate context name found: '${v.name}'`); + } else { + definitions[v.name] = this.compileContext(v); + } + } + + this.contextDefinitions = definitions; + } + + private compileContext(context: any): ContextDefinition { + return { + name: context.name, + contextMembers: this.cubeEvaluator.evaluateReferences(null, context.contextMembers) + }; + } + + public get contextList(): string[] { + return Object.keys(this.contextDefinitions); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 4d2e2ff7f2a39..1ef19c8728d7c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -833,7 +833,7 @@ export class CubeSymbols { }); } - protected evaluateReferences>( + public evaluateReferences>( cube: string | null, referencesFn: (...args: Array) => T, options: { collectJoinHints?: boolean, originalSorting?: boolean } = {} From 6ab145c51299b9b5648b0cad66c71c79aabb5af9 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 22 Jul 2025 00:13:02 +0300 Subject: [PATCH 19/37] Move CubeToMetaTransformer to TS --- .../src/compiler/ContextEvaluator.ts | 2 +- .../src/compiler/CubeEvaluator.ts | 42 ++- .../src/compiler/CubeSymbols.ts | 35 +- .../src/compiler/CubeToMetaTransformer.js | 238 ------------- .../src/compiler/CubeToMetaTransformer.ts | 318 ++++++++++++++++++ .../integration/postgres/cube-views.test.ts | 4 +- .../test/unit/folders.test.ts | 97 +++--- .../test/unit/hierarchies.test.ts | 18 +- .../test/unit/schema.test.ts | 36 +- .../test/unit/yaml-schema.test.ts | 6 +- 10 files changed, 463 insertions(+), 333 deletions(-) delete mode 100644 packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js create mode 100644 packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts index d458c09d96213..7b5736845de7f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts @@ -9,7 +9,7 @@ export type ContextDefinition = { export class ContextEvaluator { private cubeEvaluator: CubeEvaluator; - private contextDefinitions: Record; + public contextDefinitions: Record; public constructor(cubeEvaluator: CubeEvaluator) { this.cubeEvaluator = cubeEvaluator; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index dc0f1a2743bca..1e55441d872a0 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -3,8 +3,10 @@ import R from 'ramda'; import { AccessPolicyDefinition, - CubeDefinitionExtended, + CubeDefinition, + CubeDefinitionExtended, CubeSymbolDefinition, CubeSymbols, + GranularityDefinition, HierarchyDefinition, JoinDefinition, PreAggregationDefinition, @@ -18,22 +20,31 @@ import type { ErrorReporter } from './ErrorReporter'; export type SegmentDefinition = { type: string; - sql(): string; + sql?: (...args: any[]) => string; primaryKey?: true; ownedByCube: boolean; fieldType?: string; // TODO should we have it here? multiStage?: boolean; + meta?: any; + description?: string; + aliasMember?: string; }; export type DimensionDefinition = { type: string; - sql(): string; + sql?: (...args: any[]) => string; primaryKey?: true; ownedByCube: boolean; fieldType?: string; multiStage?: boolean; shiftInterval?: string; + meta?: any; + description?: string; + format?: string; + granularities?: Record; + aliasMember?: string; + suggestFilterValues?: boolean; }; export type TimeShiftDefinition = { @@ -52,7 +63,7 @@ export type TimeShiftDefinitionReference = { export type MeasureDefinition = { type: string; - sql(): string; + sql?: (...args: any[]) => string; ownedByCube: boolean; rollingWindow?: any filters?: any @@ -68,6 +79,9 @@ export type MeasureDefinition = { addGroupByReferences?: string[]; timeShiftReferences?: TimeShiftDefinitionReference[]; patchedFrom?: { cubeName: string; name: string }; + meta?: any; + description?: string; + aliasMember?: string; }; export type PreAggregationFilters = { @@ -120,20 +134,28 @@ export type EvaluatedFolder = { [key: string]: any; }; -export type EvaluatedCube = { +export interface EvaluatedCube { + name: string; + extends?: (...args: Array) => { __cubeName: string }; + sql?: (...args: any[]) => string; + sqlTable?: (...args: any[]) => string; + isView?: boolean; + calendar?: boolean; + meta?: any; + description?: string; + title?: string; measures: Record; dimensions: Record; segments: Record; joins: JoinDefinition[]; hierarchies: Record; evaluatedHierarchies: EvaluatedHierarchy[]; + aliasMember?: string; preAggregations: Record; dataSource?: string; folders: EvaluatedFolder[]; - sql?: (...args: any[]) => string; - sqlTable?: (...args: any[]) => string; accessPolicy?: AccessPolicyDefinition[]; -}; +} export class CubeEvaluator extends CubeSymbols { public evaluatedCubes: Record = {}; @@ -150,7 +172,7 @@ export class CubeEvaluator extends CubeSymbols { super(true); } - public compile(cubes: any[], errorReporter: ErrorReporter) { + public compile(cubes: CubeDefinitionExtended[], errorReporter: ErrorReporter) { super.compile(cubes, errorReporter); const validCubes = this.cubeList.filter(cube => this.cubeValidator.isCubeValid(cube)).sort((a, b) => { if (a.isView) { @@ -391,7 +413,7 @@ export class CubeEvaluator extends CubeSymbols { } } - private evaluateMultiStageReferences(cubeName: string, obj: { [key: string]: MeasureDefinition }) { + private evaluateMultiStageReferences(cubeName: string, obj: Record) { if (!obj) { return; } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 1ef19c8728d7c..93f5d739b166c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -7,6 +7,7 @@ import { DynamicReference } from './DynamicReference'; import { camelizeCube } from './utils'; import type { ErrorReporter } from './ErrorReporter'; +import { EvaluatedHierarchy } from './CubeEvaluator'; export type ToString = { toString(): string }; @@ -26,12 +27,26 @@ export type TimeshiftDefinition = { }; export type CubeSymbolDefinition = { - type?: string; + type: string; sql?: (...args: any[]) => string; primaryKey?: boolean; + description?: string; + meta?: any; granularities?: Record; timeShift?: TimeshiftDefinition[]; format?: string; + aliasMember?: string; +}; + +export type CubeDimensionSymbolDefinition = CubeSymbolDefinition & { + suggestFilterValues?: boolean; +}; + +export type CubeMeasureSymbolDefinition = CubeSymbolDefinition & { + multiStage?: boolean; + groupBy?: (...args: any[]) => ToString; + reduceBy?: (...args: any[]) => ToString; + addGroupBy?: (...args: any[]) => ToString; }; export type HierarchyDefinition = { @@ -129,6 +144,11 @@ export type AccessPolicyDefinition = { }; }; +export type FolderDefinition = { + name: string, + members: (string | FolderDefinition)[], +}; + export interface CubeDefinition { name: string; extends?: (...args: Array) => { __cubeName: string }; @@ -137,8 +157,11 @@ export interface CubeDefinition { sql_table?: string | ((...args: any[]) => string); sqlTable?: string | ((...args: any[]) => string); dataSource?: string; - measures?: Record; - dimensions?: Record; + meta?: any; + description?: string; + title?: string; + measures?: Record; + dimensions?: Record; segments?: Record; hierarchies?: Record; preAggregations?: Record; @@ -146,7 +169,7 @@ export interface CubeDefinition { pre_aggregations?: Record; joins?: JoinDefinition[]; accessPolicy?: AccessPolicyDefinition[]; - folders?: any[]; + folders?: FolderDefinition[]; includes?: any; excludes?: any; cubes?: any; @@ -161,6 +184,10 @@ export interface CubeDefinitionExtended extends CubeDefinition { allDefinitions: (type: string) => Record; rawFolders: () => any[]; rawCubes: () => any[]; + // This is filled by CubeEvaluator later, but because of Javascript, it's been available here too, + // and it's been used in the CubeToMetaTransformer compiler. + // TODO: Think how to make it better. + evaluatedHierarchies?: EvaluatedHierarchy[]; } interface SplitViews { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js deleted file mode 100644 index 740da78e43576..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js +++ /dev/null @@ -1,238 +0,0 @@ -import inflection from 'inflection'; -import R from 'ramda'; -import camelCase from 'camelcase'; - -import { getEnv } from '@cubejs-backend/shared'; -import { CubeSymbols } from './CubeSymbols'; -import { UserError } from './UserError'; -import { BaseMeasure } from '../adapter'; - -export class CubeToMetaTransformer { - /** - * @param {import('./CubeValidator').CubeValidator} cubeValidator - * @param {import('./CubeEvaluator').CubeEvaluator} cubeEvaluator - * @param {import('./ContextEvaluator').ContextEvaluator} contextEvaluator - * @param {import('./JoinGraph').JoinGraph} joinGraph - */ - constructor(cubeValidator, cubeEvaluator, contextEvaluator, joinGraph) { - this.cubeValidator = cubeValidator; - this.cubeSymbols = cubeEvaluator; - this.cubeEvaluator = cubeEvaluator; - this.contextEvaluator = contextEvaluator; - this.joinGraph = joinGraph; - this.cubes = []; - } - - compile(cubes, errorReporter) { - this.cubes = this.cubeSymbols.cubeList - .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) - .map((v) => this.transform(v, errorReporter.inContext(`${v.name} cube`))) - .filter(Boolean); - - /** - * @deprecated - * @protected - */ - this.queries = this.cubes; - } - - /** - * @protected - */ - transform(cube) { - const cubeTitle = cube.title || this.titleize(cube.name); - - const isCubeVisible = this.isVisible(cube, true); - - const flatFolderSeparator = getEnv('nestedFoldersDelimiter'); - const flatFolders = []; - - const processFolder = (folder, path = [], mergedMembers = []) => { - const flatMembers = []; - const nestedMembers = folder.includes.map(member => { - if (member.type === 'folder') { - return processFolder(member, [...path, folder.name], flatMembers); - } - const memberName = `${cube.name}.${member.name}`; - flatMembers.push(memberName); - - return memberName; - }); - - if (flatFolderSeparator !== '') { - flatFolders.push({ - name: [...path, folder.name].join(flatFolderSeparator), - members: flatMembers, - }); - } else if (path.length > 0) { - mergedMembers.push(...flatMembers); - } else { // We're at the root level - flatFolders.push({ - name: folder.name, - members: [...new Set(flatMembers)], - }); - } - - return { - name: folder.name, - members: nestedMembers, - }; - }; - - const nestedFolders = (cube.folders || []).map(f => processFolder(f)); - - return { - config: { - name: cube.name, - type: cube.isView ? 'view' : 'cube', - title: cubeTitle, - isVisible: isCubeVisible, - public: isCubeVisible, - description: cube.description, - connectedComponent: this.joinGraph.connectedComponents()[cube.name], - meta: cube.meta, - measures: R.compose( - R.map((nameToMetric) => ({ - ...this.measureConfig(cube.name, cubeTitle, nameToMetric), - isVisible: isCubeVisible ? this.isVisible(nameToMetric[1], true) : false, - public: isCubeVisible ? this.isVisible(nameToMetric[1], true) : false, - })), - R.toPairs - )(cube.measures || {}), - dimensions: R.compose( - R.map((nameToDimension) => ({ - name: `${cube.name}.${nameToDimension[0]}`, - title: this.title(cubeTitle, nameToDimension), - type: nameToDimension[1].type, - description: nameToDimension[1].description, - shortTitle: this.title(cubeTitle, nameToDimension, true), - suggestFilterValues: - nameToDimension[1].suggestFilterValues == null - ? true - : nameToDimension[1].suggestFilterValues, - format: nameToDimension[1].format, - meta: nameToDimension[1].meta, - isVisible: isCubeVisible - ? this.isVisible(nameToDimension[1], !nameToDimension[1].primaryKey) - : false, - public: isCubeVisible - ? this.isVisible(nameToDimension[1], !nameToDimension[1].primaryKey) - : false, - primaryKey: !!nameToDimension[1].primaryKey, - aliasMember: nameToDimension[1].aliasMember, - granularities: - nameToDimension[1].granularities - ? R.compose(R.map((g) => ({ - name: g[0], - title: this.title(cubeTitle, g, true), - interval: g[1].interval, - offset: g[1].offset, - origin: g[1].origin, - })), R.toPairs)(nameToDimension[1].granularities) - : undefined, - })), - R.toPairs - )(cube.dimensions || {}), - segments: R.compose( - R.map((nameToSegment) => ({ - name: `${cube.name}.${nameToSegment[0]}`, - title: this.title(cubeTitle, nameToSegment), - shortTitle: this.title(cubeTitle, nameToSegment, true), - description: nameToSegment[1].description, - meta: nameToSegment[1].meta, - isVisible: isCubeVisible ? this.isVisible(nameToSegment[1], true) : false, - public: isCubeVisible ? this.isVisible(nameToSegment[1], true) : false, - })), - R.toPairs - )(cube.segments || {}), - hierarchies: (cube.evaluatedHierarchies || []).map((it) => ({ - ...it, - aliasMember: it.aliasMember, - public: it.public ?? true, - name: `${cube.name}.${it.name}`, - })), - folders: flatFolders, - nestedFolders, - }, - }; - } - - queriesForContext(contextId) { - // return All queries if no context pass - if (R.isNil(contextId) || R.isEmpty(contextId)) { - return this.queries; - } - - const context = this.contextEvaluator.contextDefinitions[contextId]; - - // If contextId is wrong - if (R.isNil(context)) { - throw new UserError(`Context ${contextId} doesn't exist`); - } - - // As for now context works on the cubes level - return R.filter( - (query) => R.includes(query.config.name, context.contextMembers) - )(this.queries); - } - - /** - * @protected - */ - isVisible(symbol, defaultValue) { - if (symbol.public != null) { - return symbol.public; - } - - // TODO: Deprecated, should be removed in the future - if (symbol.visible != null) { - return symbol.visible; - } - - // TODO: Deprecated, should be removed in the futur - if (symbol.shown != null) { - return symbol.shown; - } - - return defaultValue; - } - - measureConfig(cubeName, cubeTitle, nameToMetric) { - const name = `${cubeName}.${nameToMetric[0]}`; - // Support both old 'drillMemberReferences' and new 'drillMembers' keys - const drillMembers = nameToMetric[1].drillMembers || nameToMetric[1].drillMemberReferences; - - const drillMembersArray = (drillMembers && this.cubeEvaluator.evaluateReferences( - cubeName, drillMembers, { originalSorting: true } - )) || []; - - const type = CubeSymbols.toMemberDataType(nameToMetric[1].type); - - return { - name, - title: this.title(cubeTitle, nameToMetric), - description: nameToMetric[1].description, - shortTitle: this.title(cubeTitle, nameToMetric, true), - format: nameToMetric[1].format, - cumulativeTotal: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), - cumulative: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), - type, - aggType: nameToMetric[1].aggType || nameToMetric[1].type, - drillMembers: drillMembersArray, - drillMembersGrouped: { - measures: drillMembersArray.filter((member) => this.cubeEvaluator.isMeasure(member)), - dimensions: drillMembersArray.filter((member) => this.cubeEvaluator.isDimension(member)), - }, - meta: nameToMetric[1].meta - }; - } - - title(cubeTitle, nameToDef, short) { - // eslint-disable-next-line prefer-template - return `${short ? '' : cubeTitle + ' '}${nameToDef[1].title || this.titleize(nameToDef[0])}`; - } - - titleize(name) { - return inflection.titleize(inflection.underscore(camelCase(name, { pascalCase: true }))); - } -} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts new file mode 100644 index 0000000000000..f8cae61e5c604 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -0,0 +1,318 @@ +import inflection from 'inflection'; +import R from 'ramda'; +import camelCase from 'camelcase'; + +import { getEnv } from '@cubejs-backend/shared'; +import { CubeDefinitionExtended, CubeSymbols, FolderDefinition } from './CubeSymbols'; +import { UserError } from './UserError'; +import { BaseMeasure } from '../adapter'; +import { JoinGraph } from './JoinGraph'; +import { ContextEvaluator } from './ContextEvaluator'; +import { CubeEvaluator, EvaluatedCube } from './CubeEvaluator'; +import { CubeValidator } from './CubeValidator'; +import { ErrorReporter } from './ErrorReporter'; + +export interface CubeTransformDefinition { + config: { + name: string; + type: 'cube' | 'view'; + title: string; + isVisible: boolean; + public: boolean; + description?: string; + connectedComponent?: any; + meta?: any; + measures: { + name: string; + title: string; + description?: string; + type: string; + aggType?: string; + cumulative?: boolean; + cumulativeTotal?: boolean; + drillMembers?: string[]; + drillMemberReferences?: string[]; + drillMembersGrouped?: { + [group: string]: string[]; + }; + meta?: any; + isVisible: boolean; + public: boolean; + }[]; + dimensions: { + name: string; + title: string; + shortTitle: string; + description?: string; + type: string; + format?: string; + meta?: any; + isVisible: boolean; + public: boolean; + primaryKey?: boolean; + aliasMember?: string; + granularities?: { + name: string; + title: string; + interval?: string; + offset?: string; + origin?: string; + }[]; + }[]; + segments: { + name: string; + title: string; + shortTitle: string; + description?: string; + meta?: any; + isVisible: boolean; + public: boolean; + }[]; + hierarchies: { + name: string; + levels: string[]; + aliasMember?: string; + public?: boolean; + }[]; + folders: FolderDefinition[]; + nestedFolders: { + name: string; + members: (string | { name: string; members: any })[]; + }[]; + }; +} + +export class CubeToMetaTransformer { + private cubeValidator: CubeValidator; + + private cubeSymbols: CubeEvaluator; + + /** + * Is public just because is used in tests. Should be private. + */ + public cubeEvaluator: CubeEvaluator; + + private contextEvaluator: ContextEvaluator; + + private joinGraph: JoinGraph; + + public cubes: CubeTransformDefinition[]; + + /** + * @deprecated + */ + protected queries: CubeTransformDefinition[]; + + public constructor(cubeValidator: CubeValidator, cubeEvaluator: CubeEvaluator, contextEvaluator: ContextEvaluator, joinGraph: JoinGraph) { + this.cubeValidator = cubeValidator; + this.cubeSymbols = cubeEvaluator; + this.cubeEvaluator = cubeEvaluator; + this.contextEvaluator = contextEvaluator; + this.joinGraph = joinGraph; + this.cubes = []; + this.queries = []; + } + + public compile(cubes: EvaluatedCube[], errorReporter: ErrorReporter) { + this.cubes = this.cubeSymbols.cubeList + .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) + .map((v) => { + errorReporter.inContext(`${v.name} cube`); + return this.transform(v); + }) + .filter(Boolean); + + this.queries = this.cubes; + } + + protected transform(cube: CubeDefinitionExtended): CubeTransformDefinition { + const cubeTitle = cube.title || this.titleize(cube.name); + + const isCubeVisible = this.isVisible(cube, true); + + const flatFolderSeparator = getEnv('nestedFoldersDelimiter'); + const flatFolders: FolderDefinition[] = []; + + const processFolder = (folder, path: string[] = [], mergedMembers: string[] = []) => { + const flatMembers: string[] = []; + const nestedMembers = folder.includes.map(member => { + if (member.type === 'folder') { + return processFolder(member, [...path, folder.name], flatMembers); + } + const memberName = `${cube.name}.${member.name}`; + flatMembers.push(memberName); + + return memberName; + }); + + if (flatFolderSeparator !== '') { + flatFolders.push({ + name: [...path, folder.name].join(flatFolderSeparator), + members: flatMembers, + }); + } else if (path.length > 0) { + mergedMembers.push(...flatMembers); + } else { // We're at the root level + flatFolders.push({ + name: folder.name, + members: [...new Set(flatMembers)], + }); + } + + return { + name: folder.name, + members: nestedMembers, + }; + }; + + const nestedFolders = (cube.folders || []).map(f => processFolder(f)); + + return { + config: { + name: cube.name, + type: cube.isView ? 'view' : 'cube', + title: cubeTitle, + isVisible: isCubeVisible, + public: isCubeVisible, + description: cube.description, + connectedComponent: this.joinGraph.connectedComponents()[cube.name], + meta: cube.meta, + measures: Object.entries(cube.measures || {}).map(([name, definition]) => ({ + ...this.measureConfig(cube.name, cubeTitle, [name, definition]), + isVisible: isCubeVisible ? this.isVisible(definition, true) : false, + public: isCubeVisible ? this.isVisible(definition, true) : false, + })), + dimensions: Object.entries(cube.dimensions || {}).map(([name, def]) => { + const isVisible = isCubeVisible + ? this.isVisible(def, !def.primaryKey) + : false; + + return { + name: `${cube.name}.${name}`, + title: this.title(cubeTitle, [name, def]), + type: def.type, + description: def.description, + shortTitle: this.title(cubeTitle, [name, def], true), + suggestFilterValues: + def.suggestFilterValues == null ? true : def.suggestFilterValues, + format: def.format, + meta: def.meta, + isVisible, + public: isVisible, + primaryKey: !!def.primaryKey, + aliasMember: def.aliasMember, + granularities: def.granularities + ? Object.entries(def.granularities).map(([gName, gDef]) => ({ + name: gName, + title: this.title(cubeTitle, [gName, gDef], true), + interval: gDef.interval, + offset: gDef.offset, + origin: gDef.origin, + })) + : undefined, + }; + }), + segments: Object.entries(cube.segments || {}).map(([name, def]) => { + const isVisible = isCubeVisible ? this.isVisible(def, true) : false; + + return { + name: `${cube.name}.${name}`, + title: this.title(cubeTitle, [name, def]), + shortTitle: this.title(cubeTitle, [name, def], true), + description: def.description, + meta: def.meta, + isVisible, + public: isVisible, + }; + }), + hierarchies: (cube.evaluatedHierarchies || []).map((it) => ({ + ...it, + aliasMember: it.aliasMember, + public: it.public ?? true, + name: `${cube.name}.${it.name}`, + })), + folders: flatFolders, + nestedFolders, + }, + }; + } + + /** + * @deprecated + */ + public queriesForContext(contextId) { + // return All queries if no context pass + if (R.isNil(contextId) || R.isEmpty(contextId)) { + return this.queries; + } + + const context = this.contextEvaluator.contextDefinitions[contextId]; + + // If contextId is wrong + if (R.isNil(context)) { + throw new UserError(`Context ${contextId} doesn't exist`); + } + + // As for now context works on the cubes level + return R.filter( + (query: CubeTransformDefinition) => R.includes(query.config.name, context.contextMembers) + )(this.queries); + } + + protected isVisible(symbol, defaultValue) { + if (symbol.public != null) { + return symbol.public; + } + + // TODO: Deprecated, should be removed in the future + if (symbol.visible != null) { + return symbol.visible; + } + + // TODO: Deprecated, should be removed in the futur + if (symbol.shown != null) { + return symbol.shown; + } + + return defaultValue; + } + + protected measureConfig(cubeName, cubeTitle, nameToMetric) { + const name = `${cubeName}.${nameToMetric[0]}`; + // Support both old 'drillMemberReferences' and new 'drillMembers' keys + const drillMembers = nameToMetric[1].drillMembers || nameToMetric[1].drillMemberReferences; + + const drillMembersArray = (drillMembers && this.cubeEvaluator.evaluateReferences( + cubeName, drillMembers, { originalSorting: true } + )) || []; + + const type = CubeSymbols.toMemberDataType(nameToMetric[1].type); + + return { + name, + title: this.title(cubeTitle, nameToMetric), + description: nameToMetric[1].description, + shortTitle: this.title(cubeTitle, nameToMetric, true), + format: nameToMetric[1].format, + cumulativeTotal: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), + cumulative: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]), + type, + aggType: nameToMetric[1].aggType || nameToMetric[1].type, + drillMembers: drillMembersArray, + drillMembersGrouped: { + measures: drillMembersArray.filter((member) => this.cubeEvaluator.isMeasure(member)), + dimensions: drillMembersArray.filter((member) => this.cubeEvaluator.isDimension(member)), + }, + meta: nameToMetric[1].meta + }; + } + + protected title(cubeTitle: string, nameToDef, short: boolean = false): string { + // eslint-disable-next-line prefer-template + return `${short ? '' : cubeTitle + ' '}${nameToDef[1].title || this.titleize(nameToDef[0])}`; + } + + protected titleize(name: string): string { + return inflection.titleize(inflection.underscore(camelCase(name, { pascalCase: true }))); + } +} diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts index 049e938ba2243..9ed21de944884 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts @@ -412,13 +412,13 @@ view(\`OrdersView3\`, { it('check includes are exposed in meta', async () => { await compiler.compile(); const cube = metaTransformer.cubes.find(c => c.config.name === 'OrdersView'); - expect(cube.config.measures.find((({ name }) => name === 'OrdersView.count')).name).toBe('OrdersView.count'); + expect(cube?.config.measures.find((({ name }) => name === 'OrdersView.count'))?.name).toBe('OrdersView.count'); }); it('orders are hidden', async () => { await compiler.compile(); const cube = metaTransformer.cubes.find(c => c.config.name === 'Orders'); - expect(cube.config.measures.filter((({ isVisible }) => isVisible)).length).toBe(0); + expect(cube?.config.measures.filter((({ isVisible }) => isVisible)).length).toBe(0); }); it('split views', async () => runQueryTest({ diff --git a/packages/cubejs-schema-compiler/test/unit/folders.test.ts b/packages/cubejs-schema-compiler/test/unit/folders.test.ts index 22b0bdb89257f..fb8c8fb3b9b5e 100644 --- a/packages/cubejs-schema-compiler/test/unit/folders.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/folders.test.ts @@ -3,6 +3,7 @@ import path from 'path'; import { CubeToMetaTransformer } from 'src/compiler/CubeToMetaTransformer'; import { prepareYamlCompiler } from './PrepareCompiler'; +import { FolderDefinition } from '../../src/compiler/CubeSymbols'; describe('Cube Folders', () => { let metaTransformer: CubeToMetaTransformer; @@ -25,20 +26,20 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view' ); - expect(emptyView.config.folders.length).toBe(2); + expect(emptyView?.config.folders.length).toBe(2); - const folder1 = emptyView.config.folders.find( + const folder1 = emptyView?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view.age', 'test_view.renamed_gender', ]); - const folder2 = emptyView.config.folders.find( + const folder2 = emptyView?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view.age', 'test_view.renamed_gender']) ); }); @@ -48,29 +49,29 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view4' ); - expect(testView.config.folders.length).toBe(3); + expect(testView?.config.folders.length).toBe(3); - const folder1 = testView.config.folders.find( + const folder1 = testView?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view4.users_age', 'test_view4.users_state', 'test_view4.renamed_orders_status', ]); - const folder2 = testView.config.folders.find( + const folder2 = testView?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view4.users_city', 'test_view4.users_renamed_in_view3_gender']) ); - const folder3 = testView.config.folders.find( + const folder3 = testView?.config.folders.find( (it) => it.name === 'folder3' ); - expect(folder3.members.length).toBe(9); - expect(folder3.members).toEqual([ + expect(folder3?.members.length).toBe(9); + expect(folder3?.members).toEqual([ 'test_view4.users_city', 'test_view4.renamed_orders_status', 'test_view4.renamed_orders_count', @@ -99,43 +100,43 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view4' ); - expect(testView.config.folders.length).toBe(5); + expect(testView?.config.folders.length).toBe(5); - const folder1 = testView.config.folders.find( + const folder1 = testView?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view4.users_age', 'test_view4.users_state', 'test_view4.renamed_orders_status', ]); - const folder2 = testView.config.folders.find( + const folder2 = testView?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view4.users_city', 'test_view4.users_renamed_in_view3_gender']) ); - const folder3 = testView.config.folders.find( + const folder3 = testView?.config.folders.find( (it) => it.name === 'folder3' ); - expect(folder3.members.length).toBe(1); - expect(folder3.members).toEqual([ + expect(folder3?.members.length).toBe(1); + expect(folder3?.members).toEqual([ 'test_view4.users_city', ]); - const folder4 = testView.config.folders.find( + const folder4 = testView?.config.folders.find( (it) => it.name === 'folder3/inner folder 4' ); - expect(folder4.members.length).toBe(1); - expect(folder4.members).toEqual(['test_view4.renamed_orders_status']); + expect(folder4?.members.length).toBe(1); + expect(folder4?.members).toEqual(['test_view4.renamed_orders_status']); - const folder5 = testView.config.folders.find( + const folder5 = testView?.config.folders.find( (it) => it.name === 'folder3/inner folder 5' ); - expect(folder5.members.length).toBe(9); - expect(folder5.members).toEqual([ + expect(folder5?.members.length).toBe(9); + expect(folder5?.members).toEqual([ 'test_view4.renamed_orders_count', 'test_view4.renamed_orders_id', 'test_view4.renamed_orders_number', @@ -153,33 +154,33 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view4' ); - expect(testView.config.nestedFolders.length).toBe(3); + expect(testView?.config.nestedFolders.length).toBe(3); - const folder1 = testView.config.nestedFolders.find( + const folder1 = testView?.config.nestedFolders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view4.users_age', 'test_view4.users_state', 'test_view4.renamed_orders_status', ]); - const folder2 = testView.config.nestedFolders.find( + const folder2 = testView?.config.nestedFolders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view4.users_city', 'test_view4.users_renamed_in_view3_gender']) ); - const folder3 = testView.config.nestedFolders.find( + const folder3 = testView?.config.nestedFolders.find( (it) => it.name === 'folder3' ); - expect(folder3.members.length).toBe(3); - expect(folder3.members[1]).toEqual( + expect(folder3?.members.length).toBe(3); + expect(folder3?.members[1]).toEqual( { name: 'inner folder 4', members: ['test_view4.renamed_orders_status'] } ); - expect(folder3.members[2].name).toEqual('inner folder 5'); - expect(folder3.members[2].members).toEqual([ + expect((folder3?.members[2] as FolderDefinition).name).toEqual('inner folder 5'); + expect((folder3?.members[2] as FolderDefinition).members).toEqual([ 'test_view4.renamed_orders_count', 'test_view4.renamed_orders_id', 'test_view4.renamed_orders_number', @@ -200,31 +201,31 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view3' ); - expect(view2.config.folders.length).toBe(1); - expect(view3.config.folders.length).toBe(2); + expect(view2?.config.folders.length).toBe(1); + expect(view3?.config.folders.length).toBe(2); - const folder1 = view2.config.folders.find( + const folder1 = view2?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1.members).toEqual([ + expect(folder1?.members).toEqual([ 'test_view2.users_age', 'test_view2.users_state', 'test_view2.renamed_orders_status', ]); - const folder1v3 = view3.config.folders.find( + const folder1v3 = view3?.config.folders.find( (it) => it.name === 'folder1' ); - expect(folder1v3.members).toEqual([ + expect(folder1v3?.members).toEqual([ 'test_view3.users_age', 'test_view3.users_state', 'test_view3.renamed_orders_status', ]); - const folder2 = view3.config.folders.find( + const folder2 = view3?.config.folders.find( (it) => it.name === 'folder2' ); - expect(folder2.members).toEqual( + expect(folder2?.members).toEqual( expect.arrayContaining(['test_view3.users_city', 'test_view3.users_renamed_in_view3_gender']) ); }); @@ -252,10 +253,10 @@ describe('Cube Folders', () => { (it) => it.config.name === 'test_view2' ); - expect(view.config.folders.length).toBe(1); + expect(view?.config.folders.length).toBe(1); - const folder1 = view.config.folders.find((it) => it.name === 'folder1'); - expect(folder1.members).toEqual([ + const folder1 = view?.config.folders.find((it) => it.name === 'folder1'); + expect(folder1?.members).toEqual([ 'test_view2.users_age', 'test_view2.users_state', 'test_view2.renamed_orders_status', diff --git a/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts b/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts index 0f2566ccfaa7e..afe8685b6cbbb 100644 --- a/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts @@ -15,7 +15,7 @@ describe('Cube hierarchies', () => { const ordersView = metaTransformer.cubes.find( (it) => it.config.name === 'orders_users_view' - ); + )!; expect(ordersView.config.hierarchies.length).toBe(2); expect(ordersView.config.hierarchies).toEqual([ @@ -42,19 +42,19 @@ describe('Cube hierarchies', () => { const ordersIncludesExcludesView = metaTransformer.cubes.find( (it) => it.config.name === 'orders_includes_excludes_view' ); - expect(ordersIncludesExcludesView.config.hierarchies.length).toBe(1); + expect(ordersIncludesExcludesView?.config.hierarchies.length).toBe(1); const emptyView = metaTransformer.cubes.find( (it) => it.config.name === 'empty_view' ); - expect(emptyView.config.hierarchies.length).toBe(0); + expect(emptyView?.config.hierarchies.length).toBe(0); const allHierarchyView = metaTransformer.cubes.find( (it) => it.config.name === 'all_hierarchy_view' ); - expect(allHierarchyView.config.hierarchies.length).toBe(3); + expect(allHierarchyView?.config.hierarchies.length).toBe(3); - const prefixedHierarchy = allHierarchyView.config.hierarchies.find((it) => it.name === 'all_hierarchy_view.users_users_hierarchy'); + const prefixedHierarchy = allHierarchyView?.config.hierarchies.find((it) => it.name === 'all_hierarchy_view.users_users_hierarchy'); expect(prefixedHierarchy).toBeTruthy(); expect(prefixedHierarchy?.aliasMember).toEqual('users.users_hierarchy'); expect(prefixedHierarchy?.levels).toEqual(['all_hierarchy_view.users_age', 'all_hierarchy_view.users_city']); @@ -73,7 +73,7 @@ describe('Cube hierarchies', () => { (it) => it.config.name === 'only_hierarchy_included_view' ); - expect(view1.config.dimensions).toEqual( + expect(view1?.config.dimensions).toEqual( expect.arrayContaining([ expect.objectContaining({ name: 'only_hierarchy_included_view.status' }), expect.objectContaining({ name: 'only_hierarchy_included_view.number' }), @@ -85,8 +85,8 @@ describe('Cube hierarchies', () => { const view2 = metaTransformer.cubes.find( (it) => it.config.name === 'auto_include_view' ); - expect(view2.config.dimensions.length).toEqual(2); - expect(view2.config.dimensions).toEqual( + expect(view2?.config.dimensions.length).toEqual(2); + expect(view2?.config.dimensions).toEqual( expect.arrayContaining([ expect.objectContaining({ name: 'auto_include_view.status' }), expect.objectContaining({ name: 'auto_include_view.number' }), @@ -177,7 +177,7 @@ describe('Cube hierarchies', () => { (it) => it.config.name === 'orders' ); - expect(ordersCube.config.hierarchies).toEqual([ + expect(ordersCube?.config.hierarchies).toEqual([ { aliasMember: undefined, name: 'orders.hello', diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index a832c49864c91..10951256766b1 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -350,8 +350,8 @@ describe('Schema Testing', () => { expect(dimensions.length).toBeGreaterThan(0); expect(dimensions.every((dimension) => dimension.primaryKey)).toBeDefined(); expect(dimensions.every((dimension) => typeof dimension.primaryKey === 'boolean')).toBe(true); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.id').primaryKey).toBe(true); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.type').primaryKey).toBe(false); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.id')?.primaryKey).toBe(true); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.type')?.primaryKey).toBe(false); }); it('descriptions', async () => { @@ -369,15 +369,15 @@ describe('Schema Testing', () => { expect(dimensions).toBeDefined(); expect(dimensions.length).toBeGreaterThan(0); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.id').description).toBe('id dimension from createCubeSchema'); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.id')?.description).toBe('id dimension from createCubeSchema'); expect(measures).toBeDefined(); expect(measures.length).toBeGreaterThan(0); - expect(measures.find((measure) => measure.name === 'CubeA.count').description).toBe('count measure from createCubeSchema'); + expect(measures.find((measure) => measure.name === 'CubeA.count')?.description).toBe('count measure from createCubeSchema'); expect(segments).toBeDefined(); expect(segments.length).toBeGreaterThan(0); - expect(segments.find((segment) => segment.name === 'CubeA.sfUsers').description).toBe('SF users segment from createCubeSchema'); + expect(segments.find((segment) => segment.name === 'CubeA.sfUsers')?.description).toBe('SF users segment from createCubeSchema'); }); it('custom granularities in meta', async () => { @@ -393,27 +393,27 @@ describe('Schema Testing', () => { const dg = dimensions.find((dimension) => dimension.name === 'orders.createdAt'); expect(dg).toBeDefined(); - expect(dg.granularities).toBeDefined(); - expect(dg.granularities.length).toBeGreaterThan(0); + expect(dg?.granularities).toBeDefined(); + expect(dg?.granularities?.length).toBeGreaterThan(0); // Granularity defined with title - let gr = dg.granularities.find(g => g.name === 'half_year'); + let gr = dg?.granularities?.find(g => g.name === 'half_year'); expect(gr).toBeDefined(); - expect(gr.title).toBe('6 month intervals'); - expect(gr.interval).toBe('6 months'); + expect(gr?.title).toBe('6 month intervals'); + expect(gr?.interval).toBe('6 months'); - gr = dg.granularities.find(g => g.name === 'half_year_by_1st_april'); + gr = dg?.granularities?.find(g => g.name === 'half_year_by_1st_april'); expect(gr).toBeDefined(); - expect(gr.title).toBe('Half year from Apr to Oct'); - expect(gr.interval).toBe('6 months'); - expect(gr.offset).toBe('3 months'); + expect(gr?.title).toBe('Half year from Apr to Oct'); + expect(gr?.interval).toBe('6 months'); + expect(gr?.offset).toBe('3 months'); // // Granularity defined without title -> titlize() - gr = dg.granularities.find(g => g.name === 'half_year_by_1st_june'); + gr = dg?.granularities?.find(g => g.name === 'half_year_by_1st_june'); expect(gr).toBeDefined(); - expect(gr.title).toBe('Half Year By1 St June'); - expect(gr.interval).toBe('6 months'); - expect(gr.origin).toBe('2020-06-01 10:00:00'); + expect(gr?.title).toBe('Half Year By1 St June'); + expect(gr?.interval).toBe('6 months'); + expect(gr?.origin).toBe('2020-06-01 10:00:00'); }); describe('Joins', () => { diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 774c8035e96ba..521d10482d77c 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -384,15 +384,15 @@ describe('Yaml Schema Testing', () => { expect(dimensions).toBeDefined(); expect(dimensions.length).toBeGreaterThan(0); - expect(dimensions.find((dimension) => dimension.name === 'CubeA.id').description).toBe('id dimension from YAML test cube'); + expect(dimensions.find((dimension) => dimension.name === 'CubeA.id')?.description).toBe('id dimension from YAML test cube'); expect(measures).toBeDefined(); expect(measures.length).toBeGreaterThan(0); - expect(measures.find((measure) => measure.name === 'CubeA.count').description).toBe('count measure from YAML test cube'); + expect(measures.find((measure) => measure.name === 'CubeA.count')?.description).toBe('count measure from YAML test cube'); expect(segments).toBeDefined(); expect(segments.length).toBeGreaterThan(0); - expect(segments.find((segment) => segment.name === 'CubeA.sfUsers').description).toBe('SF users segment from createCubeSchema'); + expect(segments.find((segment) => segment.name === 'CubeA.sfUsers')?.description).toBe('SF users segment from createCubeSchema'); }); describe('Custom dimension granularities: ', () => { From 3fe1519e6c13c0dab8f36870eb7b06d033669c9f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 22 Jul 2025 00:17:03 +0300 Subject: [PATCH 20/37] add CompilerInterface --- .../cubejs-schema-compiler/src/compiler/ContextEvaluator.ts | 3 ++- .../cubejs-schema-compiler/src/compiler/CubeDictionary.ts | 3 ++- .../cubejs-schema-compiler/src/compiler/CubeEvaluator.ts | 6 +++--- packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts | 3 ++- .../src/compiler/CubeToMetaTransformer.ts | 3 ++- .../cubejs-schema-compiler/src/compiler/CubeValidator.ts | 5 +++-- packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts | 3 ++- .../src/compiler/ViewCompilationGate.ts | 4 +++- 8 files changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts index 7b5736845de7f..0e8df6ea277e7 100644 --- a/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/ContextEvaluator.ts @@ -1,12 +1,13 @@ import { CubeEvaluator } from './CubeEvaluator'; import { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; export type ContextDefinition = { name: string; contextMembers: string | string[]; }; -export class ContextEvaluator { +export class ContextEvaluator implements CompilerInterface { private cubeEvaluator: CubeEvaluator; public contextDefinitions: Record; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts index db1aeb372618d..43c248c3ce821 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeDictionary.ts @@ -1,6 +1,7 @@ import { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; -export class CubeDictionary { +export class CubeDictionary implements CompilerInterface { public byId: Record; public constructor() { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 1e55441d872a0..f93260a49ffbf 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -3,8 +3,7 @@ import R from 'ramda'; import { AccessPolicyDefinition, - CubeDefinition, - CubeDefinitionExtended, CubeSymbolDefinition, + CubeDefinitionExtended, CubeSymbols, GranularityDefinition, HierarchyDefinition, @@ -17,6 +16,7 @@ import { UserError } from './UserError'; import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; import type { CubeValidator } from './CubeValidator'; import type { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; export type SegmentDefinition = { type: string; @@ -157,7 +157,7 @@ export interface EvaluatedCube { accessPolicy?: AccessPolicyDefinition[]; } -export class CubeEvaluator extends CubeSymbols { +export class CubeEvaluator extends CubeSymbols implements CompilerInterface { public evaluatedCubes: Record = {}; public primaryKeys: Record = {}; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 93f5d739b166c..57404f6432ec9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -8,6 +8,7 @@ import { camelizeCube } from './utils'; import type { ErrorReporter } from './ErrorReporter'; import { EvaluatedHierarchy } from './CubeEvaluator'; +import { CompilerInterface } from './PrepareCompiler'; export type ToString = { toString(): string }; @@ -215,7 +216,7 @@ export const CONTEXT_SYMBOLS = { export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE']; -export class CubeSymbols { +export class CubeSymbols implements CompilerInterface { public symbols: Record; private builtCubes: Record; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index f8cae61e5c604..f65c23ad612c5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -11,6 +11,7 @@ import { ContextEvaluator } from './ContextEvaluator'; import { CubeEvaluator, EvaluatedCube } from './CubeEvaluator'; import { CubeValidator } from './CubeValidator'; import { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; export interface CubeTransformDefinition { config: { @@ -82,7 +83,7 @@ export interface CubeTransformDefinition { }; } -export class CubeToMetaTransformer { +export class CubeToMetaTransformer implements CompilerInterface { private cubeValidator: CubeValidator; private cubeSymbols: CubeEvaluator; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 20033b2bb85f2..c3fcfd569ef4d 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -3,6 +3,7 @@ import cronParser from 'cron-parser'; import type { CubeSymbols, CubeDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; /* ***************************** * ATTENTION: @@ -940,7 +941,7 @@ export function functionFieldsPatterns(): string[] { return Array.from(functionPatterns); } -export class CubeValidator { +export class CubeValidator implements CompilerInterface { protected readonly validCubes: Map = new Map(); public constructor( @@ -948,7 +949,7 @@ export class CubeValidator { ) { } - public compile(cubes, errorReporter: ErrorReporter) { + public compile(_cubes, errorReporter: ErrorReporter) { return this.cubeSymbols.cubeList.map( (v) => this.validate(this.cubeSymbols.getCubeDefinition(v.name), errorReporter.inContext(`${v.name} cube`)) ); diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 39b9acc3296e7..ddde503af1d82 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -6,6 +6,7 @@ import type { CubeValidator } from './CubeValidator'; import type { CubeEvaluator, MeasureDefinition } from './CubeEvaluator'; import type { CubeDefinition, JoinDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; type JoinEdge = { join: JoinDefinition, @@ -30,7 +31,7 @@ export type JoinHint = string | string[]; export type JoinHints = JoinHint[]; -export class JoinGraph { +export class JoinGraph implements CompilerInterface { private readonly cubeValidator: CubeValidator; private readonly cubeEvaluator: CubeEvaluator; diff --git a/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts index 90a038771f910..264eccf60fee9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts +++ b/packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts @@ -1,4 +1,6 @@ -export class ViewCompilationGate { +import { CompilerInterface } from './PrepareCompiler'; + +export class ViewCompilationGate implements CompilerInterface { private shouldCompile: any; public constructor() { From 48788fbd69550e57027dc6ae98f93f7302505042 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 22 Jul 2025 13:07:02 +0300 Subject: [PATCH 21/37] first version of CubeJoinsResolver --- .../src/compiler/CubeJoinsResolver.ts | 58 +++++++++++++++++++ .../src/compiler/CubeSymbols.ts | 7 ++- 2 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts new file mode 100644 index 0000000000000..90cc320c6390b --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts @@ -0,0 +1,58 @@ +import { CompilerInterface } from './PrepareCompiler'; +import { CubeDefinitionExtended, CubeSymbols, JoinDefinition } from './CubeSymbols'; +import type { ErrorReporter } from './ErrorReporter'; + +export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface { + // key: cubeName with joins defined + private cubeJoins: Record; + + // key: cubeName with joins defined + // 1st level value: join alias + // 2nd level value: join definition + private cubeJoinAliases: Record>; + + // key: cubeName with joins defined + // 1st level value: target cube name + // 2nd level value: join definition + private cubeJoinTargets: Record>; + + public constructor() { + super(false); // It seems that we don't need to evaluate views + this.cubeJoins = {}; + this.cubeJoinAliases = {}; + this.cubeJoinTargets = {}; + } + + public compile(cubes: CubeDefinitionExtended[], errorReporter: ErrorReporter) { + super.compile(cubes, errorReporter); + + this.cubeList.forEach(cube => { + if (!cube.joins) { + return; + } + + this.cubeJoins[cube.name] = this.cubeJoins[cube.name] || []; + this.cubeJoinAliases[cube.name] = this.cubeJoinAliases[cube.name] || {}; + this.cubeJoinTargets[cube.name] = this.cubeJoinTargets[cube.name] || {}; + + const er = errorReporter.inContext(`${cube.name} cube`); + + cube.joins.forEach(join => { + this.cubeJoins[cube.name].push(join); + this.cubeJoinTargets[cube.name][join.name] = join; + + if (join.alias) { + if (this.cubeJoinAliases[cube.name][join.alias]) { + er.error( + `Join alias "${join.alias}" is already defined in cube "${cube.name}".`, + cube.fileName + ); + + return; + } + this.cubeJoinAliases[cube.name][join.alias] = join; + } + }); + }); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 57404f6432ec9..d88b18eb3be8d 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -116,9 +116,10 @@ export type PreAggregationDefinitionRollup = BasePreAggregationDefinition & { export type PreAggregationDefinition = PreAggregationDefinitionRollup; export type JoinDefinition = { - name: string, - relationship: string, - sql: (...args: any[]) => string, + name: string; + relationship: string; + sql: (...args: any[]) => string; + alias?: string; }; export type Filter = From 791c93f249e4ed8a559a9f4045d8a3fa4c8a1936 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 22 Jul 2025 13:35:09 +0300 Subject: [PATCH 22/37] more types in CubeSymbols --- .../src/compiler/CubeSymbols.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index d88b18eb3be8d..02b6a9d9816a9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -1058,10 +1058,10 @@ export class CubeSymbols implements CompilerInterface { } protected filterGroupFunctionDep() { - return (...filterParamArgs) => ''; + return (..._filterParamArgs) => ''; } - public resolveSymbol(cubeName, name: string) { + public resolveSymbol(cubeName: string | null, name: string) { const { sqlResolveFn, contextSymbols, collectJoinHints, depsResolveFn, currResolveIndexFn } = this.resolveSymbolsCallContext || {}; if (name === 'USER_CONTEXT') { throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.'); @@ -1075,20 +1075,24 @@ export class CubeSymbols implements CompilerInterface { return symbol; } + if (this.isCurrentCube(name) && !cubeName) { + return null; + } + // In proxied subProperty flow `name` will be set to parent dimension|measure name, // so there will be no cube = this.symbols[cubeName : name] found, but potentially // during cube definition evaluation some other deeper subProperty may be requested. // To distinguish such cases we pass the right now requested property name to // cubeReferenceProxy, so later if subProperty is requested we'll have all the required // information to construct the response. - let cube = this.symbols[this.isCurrentCube(name) ? cubeName : name]; + let cube = this.symbols[this.isCurrentCube(name) ? cubeName! : name]; if (sqlResolveFn) { if (cube) { cube = this.cubeReferenceProxy( this.isCurrentCube(name) ? cubeName : name, collectJoinHints ? [] : undefined ); - } else if (this.symbols[cubeName]?.[name]) { + } else if (cubeName && this.symbols[cubeName]?.[name]) { cube = this.cubeReferenceProxy( cubeName, collectJoinHints ? [] : undefined, @@ -1101,12 +1105,12 @@ export class CubeSymbols implements CompilerInterface { const parentIndex = currResolveIndexFn(); cube = this.cubeDependenciesProxy(parentIndex, newCubeName); return cube; - } else if (this.symbols[cubeName]?.[name] && this.symbols[cubeName][name].type === 'time') { + } else if (cubeName && this.symbols[cubeName]?.[name] && this.symbols[cubeName][name].type === 'time') { const parentIndex = currResolveIndexFn(); return this.timeDimDependenciesProxy(parentIndex); } } - return cube || this.symbols[cubeName]?.[name]; + return cube || (cubeName && this.symbols[cubeName]?.[name]); } protected cubeReferenceProxy(cubeName, joinHints?: any[], refProperty?: any) { From 866dce16ae601fffb4f2c4447c769df4e6f17397 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 22 Jul 2025 15:54:12 +0300 Subject: [PATCH 23/37] extend CubePropContextTranspiler with cubeJoinsResolver --- .../src/compiler/transpilers/CubePropContextTranspiler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 4fb43d08868d8..54206515fa8b3 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -47,6 +47,7 @@ export class CubePropContextTranspiler implements TranspilerInterface { protected readonly cubeSymbols: TranspilerSymbolResolver, protected readonly cubeDictionary: TranspilerCubeResolver, protected readonly viewCompiler: TranspilerSymbolResolver, + protected readonly cubeJoinsResolver: TranspilerSymbolResolver, ) { } @@ -96,7 +97,8 @@ export class CubePropContextTranspiler implements TranspilerInterface { protected sqlAndReferencesFieldVisitor(cubeName): TraverseObject { const resolveSymbol = n => this.viewCompiler.resolveSymbol(cubeName, n) || this.cubeSymbols.resolveSymbol(cubeName, n) || - this.cubeSymbols.isCurrentCube(n); + this.cubeSymbols.isCurrentCube(n) || + this.cubeJoinsResolver.resolveSymbol(cubeName, n); return { ObjectProperty: (path) => { From 5d13b97591028b68d99fa1975ac427a16689a9ca Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 22 Jul 2025 15:56:41 +0300 Subject: [PATCH 24/37] add LightweightJoinResolver for thread-based js transpilation + pass JoinResolver to DataSchemaCompiler --- .../src/compiler/CubeJoinsResolver.ts | 4 +-- .../src/compiler/DataSchemaCompiler.ts | 23 +++++++++++++-- .../src/compiler/PrepareCompiler.ts | 7 +++-- .../transpilers/LightweightJoinResolver.ts | 28 +++++++++++++++++++ .../compiler/transpilers/transpiler_worker.ts | 6 +++- 5 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts index 90cc320c6390b..71ea70482e0ec 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts @@ -9,7 +9,7 @@ export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface // key: cubeName with joins defined // 1st level value: join alias // 2nd level value: join definition - private cubeJoinAliases: Record>; + public cubeJoinAliases: Record>; // key: cubeName with joins defined // 1st level value: target cube name @@ -27,7 +27,7 @@ export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface super.compile(cubes, errorReporter); this.cubeList.forEach(cube => { - if (!cube.joins) { + if (!cube.joins?.length) { return; } diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index e40ad0eb38a2f..a2a94fcc9ccc0 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -22,6 +22,7 @@ import { CompilerInterface } from './PrepareCompiler'; import { YamlCompiler } from './YamlCompiler'; import { CubeDictionary } from './CubeDictionary'; import { CompilerCache } from './CompilerCache'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; const NATIVE_IS_SUPPORTED = isNativeSupported(); @@ -53,6 +54,7 @@ export type DataSchemaCompilerOptions = { cubeFactory: Function; cubeDictionary: CubeDictionary; cubeSymbols: CubeSymbols; + cubeJoinsResolver: CubeJoinsResolver; cubeCompilers?: CompilerInterface[]; contextCompilers?: CompilerInterface[]; transpilers?: TranspilerInterface[]; @@ -72,6 +74,7 @@ export type DataSchemaCompilerOptions = { export type TranspileOptions = { cubeNames?: string[]; cubeSymbols?: Record>; + cubeJoins?: Record>; contextSymbols?: Record; transpilerNames?: string[]; compilerId?: string; @@ -108,6 +111,8 @@ export class DataSchemaCompiler { private readonly cubeSymbols: CubeSymbols; + private readonly cubeJoinsResolver: CubeJoinsResolver; + // Actually should be something like // createCube(cubeDefinition: CubeDefinition): CubeDefinitionExtended private readonly cubeFactory: CallableFunction; @@ -157,6 +162,7 @@ export class DataSchemaCompiler { this.extensions = options.extensions || {}; this.cubeDictionary = options.cubeDictionary; this.cubeSymbols = options.cubeSymbols; + this.cubeJoinsResolver = options.cubeJoinsResolver; this.cubeFactory = options.cubeFactory; this.filesToCompile = options.filesToCompile || []; this.omitErrors = options.omitErrors || false; @@ -232,6 +238,7 @@ export class DataSchemaCompiler { const transpile = async (stage: CompileStage) => { let cubeNames: string[] = []; let cubeSymbols: Record> = {}; + let cubeJoins: Record> = {}; let transpilerNames: string[] = []; let results; @@ -252,6 +259,14 @@ export class DataSchemaCompiler { )], ), ); + cubeJoins = Object.fromEntries( + Object.entries(this.cubeJoinsResolver.cubeJoinAliases as Record>) + .map( + ([key, value]: [string, Record]) => [key, Object.fromEntries( + Object.keys(value).map((k) => [k, true]), + )], + ), + ); // Transpilers are the same for all files within phase. transpilerNames = this.transpilers.map(t => t.constructor.name); @@ -264,7 +279,7 @@ export class DataSchemaCompiler { content: ';', }; - await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage }); + await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, cubeJoins, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage }); const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js')) .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId })); @@ -291,7 +306,7 @@ export class DataSchemaCompiler { results = (await Promise.all([...nonJsFilesTasks, ...JsFilesTasks])).flat(); } else if (transpilationWorkerThreads) { - results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames }))); + results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, cubeJoins, transpilerNames }))); } else { results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {}))); } @@ -405,7 +420,7 @@ export class DataSchemaCompiler { }); } - private async transpileJsFile(file: FileContent, errorsReport: ErrorReporter, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) { + private async transpileJsFile(file: FileContent, errorsReport: ErrorReporter, { cubeNames, cubeSymbols, cubeJoins, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions) { try { if (getEnv('transpilationNative')) { const reqData = { @@ -417,6 +432,7 @@ export class DataSchemaCompiler { metaData: { cubeNames, cubeSymbols: cubeSymbols || {}, + cubeJoins: cubeJoins || {}, contextSymbols: contextSymbols || {}, stage: stage || 0 as CompileStage, }, @@ -437,6 +453,7 @@ export class DataSchemaCompiler { transpilers: transpilerNames, cubeNames, cubeSymbols, + cubeJoins, }; const res = await this.workerPool!.exec('transpile', [data]); diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 75fc6025929f8..7d1b3c473cf22 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -24,6 +24,7 @@ import { CompilerCache } from './CompilerCache'; import { YamlCompiler } from './YamlCompiler'; import { ViewCompilationGate } from './ViewCompilationGate'; import type { ErrorReporter } from './ErrorReporter'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; export type PrepareCompilerOptions = { nativeInstance?: NativeInstance, @@ -46,6 +47,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const nativeInstance = options.nativeInstance || new NativeInstance(); const cubeDictionary = new CubeDictionary(); const cubeSymbols = new CubeSymbols(); + const cubeJoinsResolver = new CubeJoinsResolver(); const viewCompiler = new CubeSymbols(true); const viewCompilationGate = new ViewCompilationGate(); const cubeValidator = new CubeValidator(cubeSymbols); @@ -62,7 +64,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const transpilers: TranspilerInterface[] = [ new ValidationTranspiler(), new ImportExportTranspiler(), - new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler), + new CubePropContextTranspiler(cubeSymbols, cubeDictionary, viewCompiler, cubeJoinsResolver), ]; if (!options.allowJsDuplicatePropsInSchema) { @@ -73,7 +75,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const compiler = new DataSchemaCompiler(repo, Object.assign({}, { cubeNameCompilers: [cubeDictionary], - preTranspileCubeCompilers: [cubeSymbols, cubeValidator], + preTranspileCubeCompilers: [cubeSymbols, cubeValidator, cubeJoinsResolver], transpilers, viewCompilationGate, compiledScriptCache, @@ -84,6 +86,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp compilerCache, cubeDictionary, cubeSymbols, + cubeJoinsResolver, extensions: { Funnels, RefreshKeys, diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts new file mode 100644 index 0000000000000..de20db130d3fa --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/LightweightJoinResolver.ts @@ -0,0 +1,28 @@ +import { TranspilerSymbolResolver } from './transpiler.interface'; +import { CONTEXT_SYMBOLS, CURRENT_CUBE_CONSTANTS } from '../CubeSymbols'; + +export class LightweightJoinResolver implements TranspilerSymbolResolver { + public constructor(private cubeJoinAliases: Record> = {}) { + } + + public setJoinAliases(cubeJoinAliases: Record>) { + this.cubeJoinAliases = cubeJoinAliases; + } + + public isCurrentCube(name: string): boolean { + return CURRENT_CUBE_CONSTANTS.indexOf(name) >= 0; + } + + public resolveSymbol(cubeName: string, name: string): any { + if (name === 'USER_CONTEXT') { + throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.'); + } + + if (CONTEXT_SYMBOLS[name]) { + return true; + } + + const cube = this.cubeJoinAliases[this.isCurrentCube(name) ? cubeName : name]; + return !!(cube || this.cubeJoinAliases[cubeName]?.[name]); + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts index 93c1c142744f0..8c5bcb8c8536e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler_worker.ts @@ -10,6 +10,7 @@ import { CubePropContextTranspiler } from './CubePropContextTranspiler'; import { ErrorReporter } from '../ErrorReporter'; import { LightweightSymbolResolver } from './LightweightSymbolResolver'; import { LightweightNodeCubeDictionary } from './LightweightNodeCubeDictionary'; +import { LightweightJoinResolver } from './LightweightJoinResolver'; type TransferContent = { fileName: string; @@ -17,22 +18,25 @@ type TransferContent = { transpilers: string[]; cubeNames: string[]; cubeSymbols: Record>; + cubeJoins: Record>; }; const cubeDictionary = new LightweightNodeCubeDictionary(); const cubeSymbols = new LightweightSymbolResolver(); +const cubeJoinsResolver = new LightweightJoinResolver(); const errorsReport = new ErrorReporter(null, []); const transpilers = { ValidationTranspiler: new ValidationTranspiler(), ImportExportTranspiler: new ImportExportTranspiler(), CubeCheckDuplicatePropTranspiler: new CubeCheckDuplicatePropTranspiler(), - CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols), + CubePropContextTranspiler: new CubePropContextTranspiler(cubeSymbols, cubeDictionary, cubeSymbols, cubeJoinsResolver), }; const transpile = (data: TransferContent) => { cubeDictionary.setCubeNames(data.cubeNames); cubeSymbols.setSymbols(data.cubeSymbols); + cubeJoinsResolver.setJoinAliases(data.cubeJoins); const ast = parse( data.content, From 318ca38dc22fdde6b323f73be029963fa53b850a Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 22 Jul 2025 15:57:05 +0300 Subject: [PATCH 25/37] pass cubeJoinsResolver to YamlCompiler --- .../cubejs-schema-compiler/src/compiler/PrepareCompiler.ts | 2 +- packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 7d1b3c473cf22..efae115bc2e30 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -57,7 +57,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, joinGraph); const { maxQueryCacheSize, maxQueryCacheAge } = options; const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge }); - const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); + const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler, cubeJoinsResolver); const compiledScriptCache = options.compiledScriptCache || new LRUCache({ max: 250 }); diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index c36b648500244..f1a49037e343c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -16,6 +16,7 @@ import { nonStringFields } from './CubeValidator'; import { CubeDictionary } from './CubeDictionary'; import { ErrorReporter } from './ErrorReporter'; import { camelizeCube } from './utils'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; type EscapeStateStack = { inFormattedStr?: boolean; @@ -34,6 +35,7 @@ export class YamlCompiler { private readonly cubeDictionary: CubeDictionary, private readonly nativeInstance: NativeInstance, private readonly viewCompiler: CubeSymbols, + private readonly cubeJoinsResolver: CubeJoinsResolver, ) { } @@ -301,7 +303,8 @@ export class YamlCompiler { resolveSymbol = resolveSymbol || (n => this.viewCompiler.resolveSymbol(cubeName, n) || this.cubeSymbols.resolveSymbol(cubeName, n) || - this.cubeSymbols.isCurrentCube(n)); + this.cubeSymbols.isCurrentCube(n) || + this.cubeJoinsResolver.resolveSymbol(cubeName, n)); const traverseObj = { Program: (babelPath) => { From 12998b0cf010a718e9809765213f8d1734d94d05 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 23 Jul 2025 13:31:46 +0300 Subject: [PATCH 26/37] inject CubeJoinsResolver into inheritance chain CubeEvaluator extends CubeJoinsResolver extends CubeSymbols --- .../src/compiler/CubeEvaluator.ts | 3 ++- .../src/compiler/CubeJoinsResolver.ts | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index f93260a49ffbf..4d305d7e2b6b8 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -17,6 +17,7 @@ import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; import type { CubeValidator } from './CubeValidator'; import type { ErrorReporter } from './ErrorReporter'; import { CompilerInterface } from './PrepareCompiler'; +import { CubeJoinsResolver } from './CubeJoinsResolver'; export type SegmentDefinition = { type: string; @@ -157,7 +158,7 @@ export interface EvaluatedCube { accessPolicy?: AccessPolicyDefinition[]; } -export class CubeEvaluator extends CubeSymbols implements CompilerInterface { +export class CubeEvaluator extends CubeJoinsResolver implements CompilerInterface { public evaluatedCubes: Record = {}; public primaryKeys: Record = {}; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts index 71ea70482e0ec..c8d89f3ccbec4 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeJoinsResolver.ts @@ -16,8 +16,8 @@ export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface // 2nd level value: join definition private cubeJoinTargets: Record>; - public constructor() { - super(false); // It seems that we don't need to evaluate views + public constructor(evaluateViews = false) { + super(evaluateViews); this.cubeJoins = {}; this.cubeJoinAliases = {}; this.cubeJoinTargets = {}; @@ -55,4 +55,19 @@ export class CubeJoinsResolver extends CubeSymbols implements CompilerInterface }); }); } + + public resolveSymbol(cubeName: string | null, name: string) { + if (this.isCurrentCube(name) && !cubeName) { + return null; + } + + if (cubeName && this.cubeJoinAliases[cubeName]?.[name]) { + // TODO: Write a full implementation + // Some kind of proxy like in symbols + + return super.resolveSymbol(cubeName, this.cubeJoinAliases[cubeName]?.[name].name); + } + + return super.resolveSymbol(cubeName, name); + } } From 9c2915266123425cf13192f8e79ae6096fe3e518 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 23 Jul 2025 14:48:45 +0300 Subject: [PATCH 27/37] more types in Transpiler interfaces --- .../src/compiler/transpilers/transpiler.interface.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts index 803a1c0fa4082..7885f8e51332c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/transpiler.interface.ts @@ -8,10 +8,10 @@ export interface TranspilerInterface { } export interface TranspilerSymbolResolver { - resolveSymbol(cubeName, name): any; - isCurrentCube(name): boolean; + resolveSymbol(cubeName: string | null, name: string): any; + isCurrentCube(name: string): boolean; } export interface TranspilerCubeResolver { - resolveCube(name): boolean; + resolveCube(name: string): boolean; } From 8e7269aca93c58ec88c89d24e0d77d3009ac7c51 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 23 Jul 2025 14:50:10 +0300 Subject: [PATCH 28/37] remove unused import --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c5de58b620094..2667e27dfec57 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -22,7 +22,6 @@ import { localTimestampToUtc, timeSeries as timeSeriesBase, timeSeriesFromCustomInterval, - parseSqlInterval, findMinGranularityDimension } from '@cubejs-backend/shared'; From d061eaa4b2145b614a2e51fe43dbd9fa20835809 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 23 Jul 2025 14:54:23 +0300 Subject: [PATCH 29/37] move initialization to top in baseQuery --- .../src/adapter/BaseQuery.js | 220 +++++++++--------- 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 2667e27dfec57..7b0ee80042e7c 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -154,99 +154,6 @@ export class BaseQuery { this.initFromOptions(); } - extractDimensionsAndMeasures(filters = []) { - if (!filters) { - return []; - } - let allFilters = []; - filters.forEach(f => { - if (f.and) { - allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.and)); - } else if (f.or) { - allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.or)); - } else if (!f.member && !f.dimension) { - throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); - } else if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { - allFilters.push({ measure: f.member || f.dimension }); - } else { - allFilters.push({ dimension: f.member || f.dimension }); - } - }); - - return allFilters; - } - - keepFilters(filters = [], fn) { - return filters.map(f => { - if (f.and) { - return { and: this.keepFilters(f.and, fn) }; - } else if (f.or) { - return { or: this.keepFilters(f.or, fn) }; - } else if (!f.member && !f.dimension) { - throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); - } else { - return fn(f.member || f.dimension || f.measure) ? f : null; - } - }).filter(f => !!f); - } - - extractFiltersAsTree(filters = []) { - if (!filters) { - return []; - } - - return filters.map(f => { - if (f.and || f.or) { - let operator = 'or'; - if (f.and) { - operator = 'and'; - } - const data = this.extractDimensionsAndMeasures(f[operator]); - const dimension = data.filter(e => !!e.dimension).map(e => e.dimension); - const measure = data.filter(e => !!e.measure).map(e => e.measure); - if (dimension.length && !measure.length) { - return { - values: this.extractFiltersAsTree(f[operator]), - operator, - dimensionGroup: true, - measure: null, - }; - } - if (!dimension.length && measure.length) { - return { - values: this.extractFiltersAsTree(f[operator]), - operator, - dimension: null, - measureGroup: true, - }; - } - if (!dimension.length && !measure.length) { - return { - values: [], - operator, - }; - } - throw new UserError(`You cannot use dimension and measure in same condition: ${JSON.stringify(f)}`); - } - - if (!f.member && !f.dimension) { - throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); - } - - if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { - return Object.assign({}, f, { - dimension: null, - measure: f.member || f.dimension - }); - } - - return Object.assign({}, f, { - measure: null, - dimension: f.member || f.dimension - }); - }); - } - /** * @protected */ @@ -355,6 +262,116 @@ export class BaseQuery { this.initUngrouped(); } + initUngrouped() { + this.ungrouped = this.options.ungrouped; + if (this.ungrouped) { + if (!this.options.allowUngroupedWithoutPrimaryKey && this.join) { + const cubes = R.uniq([this.join.root].concat(this.join.joins.map(j => j.originalTo))); + const primaryKeyNames = cubes.flatMap(c => this.primaryKeyNames(c)); + const missingPrimaryKeys = primaryKeyNames.filter(key => !this.dimensions.find(d => d.dimension === key)); + if (missingPrimaryKeys.length) { + throw new UserError(`Ungrouped query requires primary keys to be present in dimensions: ${missingPrimaryKeys.map(k => `'${k}'`).join(', ')}. Pass allowUngroupedWithoutPrimaryKey option to disable this check.`); + } + } + if (this.measureFilters.length) { + throw new UserError('Measure filters aren\'t allowed in ungrouped query'); + } + } + } + + extractDimensionsAndMeasures(filters = []) { + if (!filters) { + return []; + } + let allFilters = []; + filters.forEach(f => { + if (f.and) { + allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.and)); + } else if (f.or) { + allFilters = allFilters.concat(this.extractDimensionsAndMeasures(f.or)); + } else if (!f.member && !f.dimension) { + throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); + } else if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { + allFilters.push({ measure: f.member || f.dimension }); + } else { + allFilters.push({ dimension: f.member || f.dimension }); + } + }); + + return allFilters; + } + + keepFilters(filters = [], fn) { + return filters.map(f => { + if (f.and) { + return { and: this.keepFilters(f.and, fn) }; + } else if (f.or) { + return { or: this.keepFilters(f.or, fn) }; + } else if (!f.member && !f.dimension) { + throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); + } else { + return fn(f.member || f.dimension || f.measure) ? f : null; + } + }).filter(f => !!f); + } + + extractFiltersAsTree(filters = []) { + if (!filters) { + return []; + } + + return filters.map(f => { + if (f.and || f.or) { + let operator = 'or'; + if (f.and) { + operator = 'and'; + } + const data = this.extractDimensionsAndMeasures(f[operator]); + const dimension = data.filter(e => !!e.dimension).map(e => e.dimension); + const measure = data.filter(e => !!e.measure).map(e => e.measure); + if (dimension.length && !measure.length) { + return { + values: this.extractFiltersAsTree(f[operator]), + operator, + dimensionGroup: true, + measure: null, + }; + } + if (!dimension.length && measure.length) { + return { + values: this.extractFiltersAsTree(f[operator]), + operator, + dimension: null, + measureGroup: true, + }; + } + if (!dimension.length && !measure.length) { + return { + values: [], + operator, + }; + } + throw new UserError(`You cannot use dimension and measure in same condition: ${JSON.stringify(f)}`); + } + + if (!f.member && !f.dimension) { + throw new UserError(`member attribute is required for filter ${JSON.stringify(f)}`); + } + + if (this.cubeEvaluator.isMeasure(f.member || f.dimension)) { + return Object.assign({}, f, { + dimension: null, + measure: f.member || f.dimension + }); + } + + return Object.assign({}, f, { + measure: null, + dimension: f.member || f.dimension + }); + }); + } + // Temporary workaround to avoid checking for multistage in CubeStoreQuery, since that could lead to errors when HLL functions are present in the query. neverUseSqlPlannerPreaggregation() { return false; @@ -525,23 +542,6 @@ export class BaseQuery { ); } - initUngrouped() { - this.ungrouped = this.options.ungrouped; - if (this.ungrouped) { - if (!this.options.allowUngroupedWithoutPrimaryKey && this.join) { - const cubes = R.uniq([this.join.root].concat(this.join.joins.map(j => j.originalTo))); - const primaryKeyNames = cubes.flatMap(c => this.primaryKeyNames(c)); - const missingPrimaryKeys = primaryKeyNames.filter(key => !this.dimensions.find(d => d.dimension === key)); - if (missingPrimaryKeys.length) { - throw new UserError(`Ungrouped query requires primary keys to be present in dimensions: ${missingPrimaryKeys.map(k => `'${k}'`).join(', ')}. Pass allowUngroupedWithoutPrimaryKey option to disable this check.`); - } - } - if (this.measureFilters.length) { - throw new UserError('Measure filters aren\'t allowed in ungrouped query'); - } - } - } - get subQueryDimensions() { // eslint-disable-next-line no-underscore-dangle if (!this._subQueryDimensions) { From d72c777551f534cbc9a1f900fd3310ee6534b839 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 23 Jul 2025 15:40:31 +0300 Subject: [PATCH 30/37] remove unused --- .../src/compiler/transpilers/CubePropContextTranspiler.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 54206515fa8b3..7836ede81a753 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -8,8 +8,6 @@ import { TranspilerSymbolResolver, TraverseObject } from './transpiler.interface'; -import type { CubeSymbols } from '../CubeSymbols'; -import type { CubeDictionary } from '../CubeDictionary'; /* this list was generated by getTransformPatterns() with additional variants for snake_case */ export const transpiledFieldsPatterns: Array = [ From 14597fdc511b372289ca81fa953f9ff7ecc22e16 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 23 Jul 2025 17:00:46 +0300 Subject: [PATCH 31/37] more type annotations in baseQuery --- .../src/adapter/BaseQuery.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 7b0ee80042e7c..c32e5a9b4299d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -3066,16 +3066,26 @@ export class BaseQuery { return sql; } + /** + * @param {string} sql + * @returns {string} + */ wrapSegmentForDimensionSelect(sql) { return sql; } + /** + * @param {string} cubeName + */ pushCubeNameForCollectionIfNecessary(cubeName) { if ((this.evaluateSymbolContext || {}).cubeNames && cubeName) { this.evaluateSymbolContext.cubeNames.push(cubeName); } } + /** + * @param {string|Array} joinHints + */ pushJoinHints(joinHints) { if (this.safeEvaluateSymbolContext().joinHints && joinHints) { if (Array.isArray(joinHints) && joinHints.length === 1) { @@ -3085,6 +3095,10 @@ export class BaseQuery { } } + /** + * @param {string} cubeName + * @param {string} name + */ pushMemberNameForCollectionIfNecessary(cubeName, name) { const pathFromArray = this.cubeEvaluator.pathFromArray([cubeName, name]); if (!this.cubeEvaluator.getCubeDefinition(cubeName).isView) { From 294655466fa06b049a1bf5261846d927168b1abb Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 24 Jul 2025 19:16:03 +0300 Subject: [PATCH 32/37] Add joinAliases collection alongside with the joinHints And this doesn't break existing things :) --- .../src/adapter/BaseQuery.js | 63 +++++++++++++++---- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c32e5a9b4299d..4597837948e58 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -223,7 +223,11 @@ export class BaseQuery { /** @type {import('./BaseTimeDimension').BaseTimeDimension[]} */ this.timeDimensions = (this.options.timeDimensions || []).map(dimension => { if (!dimension.dimension) { - const join = this.joinGraph.buildJoin(this.collectJoinHints(true)); + const { joinHints, joinAliases } = this.collectJoinHints(true); + + // TODO: Use joinAliases for join hints building + + const join = this.joinGraph.buildJoin(joinHints); if (!join) { return undefined; } @@ -440,9 +444,16 @@ export class BaseQuery { */ get allJoinHints() { if (!this.collectedJoinHints) { - const [rootOfJoin, ...allMembersJoinHints] = this.collectJoinHintsFromMembers(this.allMembersConcat(false)); - const customSubQueryJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery()); - let joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(this.join)); + let { joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.allMembersConcat(false)); + const [rootOfJoin, ...allMembersJoinHints] = joinHints; + + ({ joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery())); + const customSubQueryJoinHints = joinHints; + + ({ joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(this.join))); + let joinMembersJoinHints = joinHints; + + // TODO: Use joinAliases for join hints building // One cube may join the other cube via transitive joined cubes, // members from which are referenced in the join `on` clauses. @@ -501,7 +512,12 @@ export class BaseQuery { while (newJoin?.joins.length > 0 && !isJoinTreesEqual(prevJoins, newJoin) && cnt < 10000) { prevJoins = newJoin; - joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(newJoin)); + + ({ joinHints, joinAliases } = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(newJoin))); + joinMembersJoinHints = joinHints; + + // TODO: Use joinAliases for join hints building + if (!isOrderPreserved(prevJoinMembersJoinHints, joinMembersJoinHints)) { throw new UserError(`Can not construct joins for the query, potential loop detected: ${prevJoinMembersJoinHints.join('->')} vs ${joinMembersJoinHints.join('->')}`); } @@ -2392,7 +2408,8 @@ export class BaseQuery { ); if (shouldBuildJoinForMeasureSelect) { - const joinHints = this.collectJoinHintsFromMembers(measures); + const { joinHints, joinAliases } = this.collectJoinHintsFromMembers(measures); + // TODO: Use joinAliases for join hints building const measuresJoin = this.joinGraph.buildJoin(joinHints); if (measuresJoin.multiplicationFactor[keyCubeName]) { throw new UserError( @@ -2473,10 +2490,14 @@ export class BaseQuery { const cubes = this.collectFrom(nonViewMembers, this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor'); // Not using `collectJoinHintsFromMembers([measure])` because it would collect too many join hints from view + const { joinHints: collectedJoinHints, joinAliases } = this.collectJoinHintsFromMembers(nonViewMembers); const joinHints = [ measure.joinHint, - ...this.collectJoinHintsFromMembers(nonViewMembers), + ...collectedJoinHints, ]; + + // TODO: Use joinAliases for join hints building + if (R.any(cubeName => keyCubeName !== cubeName, cubes)) { const measuresJoin = this.joinGraph.buildJoin(joinHints); if (measuresJoin.multiplicationFactor[keyCubeName]) { @@ -2657,10 +2678,23 @@ export class BaseQuery { } collectJoinHintsFromMembers(members) { - return [ - ...members.map(m => m.joinHint).filter(h => h?.length > 0), - ...this.collectFrom(members, this.collectJoinHintsFor.bind(this), 'collectJoinHintsFromMembers'), - ]; + const { + joinHints: collectedJoinHints, + joinAliases: collectedJoinAliases, + } = this.collectFrom(members, this.collectJoinHintsFor.bind(this), 'collectJoinHintsFromMembers') + .reduce((acc, { joinHints, joinAliases }) => { + joinHints.forEach(item => acc.joinHints.add(item)); + acc.joinAliases.push(...joinAliases); + return acc; + }, { joinHints: new Set(), joinAliases: [] }); + + return { + joinHints: [ + ...members.map(m => m.joinHint).filter(h => h?.length > 0), + ...Array.from(collectedJoinHints), + ], + joinAliases: collectedJoinAliases, + }; } /** @@ -3385,12 +3419,15 @@ export class BaseQuery { } collectJoinHintsFor(fn) { - const context = { joinHints: [] }; + const context = { joinHints: [], joinAliases: [] }; this.evaluateSymbolSqlWithContext( fn, context ); - return context.joinHints; + return { + joinHints: context.joinHints, + joinAliases: context.joinAliases, + }; } /** From 8a4eb9c5ec1c2acb5be965ba925c930ac189b861 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 25 Jul 2025 15:11:22 +0300 Subject: [PATCH 33/37] remove unused --- .../src/compiler/JoinGraph.ts | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index ddde503af1d82..ddae978b95082 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -61,7 +61,7 @@ export class JoinGraph implements CompilerInterface { this.graph = null; } - public compile(cubes: unknown, errorReporter: ErrorReporter): void { + public compile(_cubes: unknown, errorReporter: ErrorReporter): void { this.edges = R.compose< Array, Array, @@ -159,17 +159,6 @@ export class JoinGraph implements CompilerInterface { }); } - protected buildJoinNode(cube: CubeDefinition): Record { - if (!cube.joins) { - return {}; - } - - return cube.joins.reduce((acc, join) => { - acc[join.name] = 1; - return acc; - }, {} as Record); - } - public buildJoin(cubesToJoin: JoinHints): FinishedJoinTree | null { if (!cubesToJoin.length) { return null; From aa7f2be5fc97c16dfcca8a3e76defe68f68307e0 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 25 Jul 2025 15:17:04 +0300 Subject: [PATCH 34/37] a few more types --- packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index ddae978b95082..8a4cdae10ebd5 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -123,7 +123,7 @@ export class JoinGraph implements CompilerInterface { }; const joinRequired = - (v) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; + (v: string) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; return cube.joins .filter(join => { @@ -198,7 +198,7 @@ export class JoinGraph implements CompilerInterface { return this.builtJoins[key]; } - protected cubeFromPath(cubePath) { + protected cubeFromPath(cubePath: JoinHint): string { if (Array.isArray(cubePath)) { return cubePath[cubePath.length - 1]; } From ebb14099c412d62d65b00a3adbf76066cca17367 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 25 Jul 2025 18:41:03 +0300 Subject: [PATCH 35/37] remove unused props (from, to) --- .../src/compiler/JoinGraph.ts | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 8a4cdae10ebd5..40104a512bff0 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -10,8 +10,6 @@ import { CompilerInterface } from './PrepareCompiler'; type JoinEdge = { join: JoinDefinition, - from: string, - to: string, originalFrom: string, originalTo: string, }; @@ -86,8 +84,8 @@ export class JoinGraph implements CompilerInterface { >( // This requires @types/ramda@0.29 or newer // @ts-ignore - R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.to, 1]))), - R.groupBy((join: JoinEdge) => join.from), + R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.originalTo, 1]))), + R.groupBy((join: JoinEdge) => join.originalFrom), R.map(v => v[1]), R.toPairs // @ts-ignore @@ -96,12 +94,12 @@ export class JoinGraph implements CompilerInterface { // @ts-ignore this.undirectedNodes = R.compose( // @ts-ignore - R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.from, 1]))), + R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.originalFrom, 1]))), // @ts-ignore - R.groupBy(join => join.to), + R.groupBy(join => join.originalTo), R.unnest, // @ts-ignore - R.map(v => [v[1], { from: v[1].to, to: v[1].from }]), + R.map(v => [v[1], { originalFrom: v[1].originalTo, originalTo: v[1].originalFrom }]), R.toPairs // @ts-ignore )(this.edges); @@ -149,8 +147,6 @@ export class JoinGraph implements CompilerInterface { .map(join => { const joinEdge: JoinEdge = { join, - from: cube.name, - to: join.name, originalFrom: cube.name, originalTo: join.name }; @@ -295,9 +291,9 @@ export class JoinGraph implements CompilerInterface { } visited[currentCube] = true; function nextNode(nextJoin: JoinEdge): string { - return nextJoin.from === currentCube ? nextJoin.to : nextJoin.from; + return nextJoin.originalFrom === currentCube ? nextJoin.originalTo : nextJoin.originalFrom; } - const nextJoins = joins.filter(j => j.from === currentCube || j.to === currentCube); + const nextJoins = joins.filter(j => j.originalFrom === currentCube || j.originalTo === currentCube); if (nextJoins.find( nextJoin => self.checkIfCubeMultiplied(currentCube, nextJoin) && !visited[nextNode(nextJoin)] )) { @@ -311,8 +307,8 @@ export class JoinGraph implements CompilerInterface { } protected checkIfCubeMultiplied(cube: string, join: JoinEdge): boolean { - return join.from === cube && join.join.relationship === 'hasMany' || - join.to === cube && join.join.relationship === 'belongsTo'; + return join.originalFrom === cube && join.join.relationship === 'hasMany' || + join.originalTo === cube && join.join.relationship === 'belongsTo'; } protected joinsByPath(path: string[]): JoinEdge[] { From f157025404985d3a75ec1b394e565b99916ce90e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 25 Jul 2025 19:14:13 +0300 Subject: [PATCH 36/37] fix types for ramda composes Just ts-ignore, because it's impossible --- .../cubejs-schema-compiler/src/compiler/JoinGraph.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 40104a512bff0..cf7601fe6adf6 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -75,16 +75,11 @@ export class JoinGraph implements CompilerInterface { // This requires @types/ramda@0.29 or newer // @ts-ignore - this.nodes = R.compose< - Record, - Array<[string, JoinEdge]>, - Array, - Record | undefined>, - Record> - >( + this.nodes = R.compose( // This requires @types/ramda@0.29 or newer // @ts-ignore R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.originalTo, 1]))), + // @ts-ignore R.groupBy((join: JoinEdge) => join.originalFrom), R.map(v => v[1]), R.toPairs From 61cf85227047edb724c3514cf02e33fc10719665 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 25 Jul 2025 19:14:51 +0300 Subject: [PATCH 37/37] Revert "remove unused props (from, to)" This reverts commit ebb14099c412d62d65b00a3adbf76066cca17367. # Conflicts: # packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts --- .../src/compiler/JoinGraph.ts | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index cf7601fe6adf6..50fbbb30a9e6a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -10,6 +10,8 @@ import { CompilerInterface } from './PrepareCompiler'; type JoinEdge = { join: JoinDefinition, + from: string, + to: string, originalFrom: string, originalTo: string, }; @@ -78,9 +80,9 @@ export class JoinGraph implements CompilerInterface { this.nodes = R.compose( // This requires @types/ramda@0.29 or newer // @ts-ignore - R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.originalTo, 1]))), + R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.to, 1]))), // @ts-ignore - R.groupBy((join: JoinEdge) => join.originalFrom), + R.groupBy((join: JoinEdge) => join.from), R.map(v => v[1]), R.toPairs // @ts-ignore @@ -89,12 +91,12 @@ export class JoinGraph implements CompilerInterface { // @ts-ignore this.undirectedNodes = R.compose( // @ts-ignore - R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.originalFrom, 1]))), + R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.from, 1]))), // @ts-ignore - R.groupBy(join => join.originalTo), + R.groupBy(join => join.to), R.unnest, // @ts-ignore - R.map(v => [v[1], { originalFrom: v[1].originalTo, originalTo: v[1].originalFrom }]), + R.map(v => [v[1], { from: v[1].to, to: v[1].from }]), R.toPairs // @ts-ignore )(this.edges); @@ -142,6 +144,8 @@ export class JoinGraph implements CompilerInterface { .map(join => { const joinEdge: JoinEdge = { join, + from: cube.name, + to: join.name, originalFrom: cube.name, originalTo: join.name }; @@ -286,9 +290,9 @@ export class JoinGraph implements CompilerInterface { } visited[currentCube] = true; function nextNode(nextJoin: JoinEdge): string { - return nextJoin.originalFrom === currentCube ? nextJoin.originalTo : nextJoin.originalFrom; + return nextJoin.from === currentCube ? nextJoin.to : nextJoin.from; } - const nextJoins = joins.filter(j => j.originalFrom === currentCube || j.originalTo === currentCube); + const nextJoins = joins.filter(j => j.from === currentCube || j.to === currentCube); if (nextJoins.find( nextJoin => self.checkIfCubeMultiplied(currentCube, nextJoin) && !visited[nextNode(nextJoin)] )) { @@ -302,8 +306,8 @@ export class JoinGraph implements CompilerInterface { } protected checkIfCubeMultiplied(cube: string, join: JoinEdge): boolean { - return join.originalFrom === cube && join.join.relationship === 'hasMany' || - join.originalTo === cube && join.join.relationship === 'belongsTo'; + return join.from === cube && join.join.relationship === 'hasMany' || + join.to === cube && join.join.relationship === 'belongsTo'; } protected joinsByPath(path: string[]): JoinEdge[] {