diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 53b3295d798c3..3a5834d455977 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -387,16 +387,16 @@ export class BaseQuery { } /** - * Is used by native * This function follows the same logic as in this.collectJoinHints() - * @private + * skipQueryJoinMap is used by PreAggregations to build join tree without user's query all members map + * @public * @param {Array<(Array | string)>} hints + * @param { boolean } skipQueryJoinMap * @return {import('../compiler/JoinGraph').FinishedJoinTree} */ - joinTreeForHints(hints) { - const explicitJoinHintMembers = new Set(hints.filter(j => Array.isArray(j)).flat()); - const queryJoinMaps = this.queryJoinMap(); - const newCollectedHints = []; + joinTreeForHints(hints, skipQueryJoinMap = false) { + const queryJoinMaps = skipQueryJoinMap ? {} : this.queryJoinMap(); + let newCollectedHints = []; const constructJH = () => R.uniq(this.enrichHintsWithJoinMap([ ...newCollectedHints, @@ -421,8 +421,12 @@ export class BaseQuery { const iterationCollectedHints = joinMembersJoinHints.filter(j => !allJoinHintsFlatten.has(j)); newJoinHintsCollectedCnt = iterationCollectedHints.length; cnt++; - if (newJoin) { - newCollectedHints.push(...joinMembersJoinHints.filter(j => !explicitJoinHintMembers.has(j))); + if (newJoin && newJoin.joins.length > 0) { + // Even if there is no join tree changes, we still + // push correctly ordered join hints, collected from the resolving of members of join tree + // upfront the all existing query members. This ensures the correct cube join order + // with transitive joins even if they are already presented among query members. + newCollectedHints = this.enrichedJoinHintsFromJoinTree(newJoin, joinMembersJoinHints); } } while (newJoin?.joins.length > 0 && !this.isJoinTreesEqual(prevJoin, newJoin) && cnt < 10000 && newJoinHintsCollectedCnt > 0); @@ -430,7 +434,7 @@ export class BaseQuery { throw new UserError('Can not construct joins for the query, potential loop detected'); } - return newJoin; + return this.joinGraph.buildJoin(constructJH()); } cacheValue(key, fn, { contextPropNames, inputProps, cache } = {}) { @@ -505,6 +509,34 @@ export class BaseQuery { return joinMaps; } + /** + * @private + * @param { import('../compiler/JoinGraph').FinishedJoinTree } joinTree + * @param { string[] } joinHints + * @return { string[][] } + */ + enrichedJoinHintsFromJoinTree(joinTree, joinHints) { + const joinsMap = {}; + + for (const j of joinTree.joins) { + joinsMap[j.to] = j.from; + } + + return joinHints.map(jh => { + let cubeName = jh; + const path = [cubeName]; + while (joinsMap[cubeName]) { + cubeName = joinsMap[cubeName]; + path.push(cubeName); + } + + if (path.length === 1) { + return path[0]; + } + return path.reverse(); + }); + } + /** * @private * @param { (string|string[])[] } hints @@ -2666,10 +2698,9 @@ export class BaseQuery { */ collectJoinHints(excludeTimeDimensions = false) { const allMembersJoinHints = this.collectJoinHintsFromMembers(this.allMembersConcat(excludeTimeDimensions)); - const explicitJoinHintMembers = new Set(allMembersJoinHints.filter(j => Array.isArray(j)).flat()); const queryJoinMaps = this.queryJoinMap(); const customSubQueryJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery()); - const newCollectedHints = []; + let newCollectedHints = []; // One cube may join the other cube via transitive joined cubes, // members from which are referenced in the join `on` clauses. @@ -2703,8 +2734,12 @@ export class BaseQuery { const iterationCollectedHints = joinMembersJoinHints.filter(j => !allJoinHintsFlatten.has(j)); newJoinHintsCollectedCnt = iterationCollectedHints.length; cnt++; - if (newJoin) { - newCollectedHints.push(...joinMembersJoinHints.filter(j => !explicitJoinHintMembers.has(j))); + if (newJoin && newJoin.joins.length > 0) { + // Even if there is no join tree changes, we still + // push correctly ordered join hints, collected from the resolving of members of join tree + // upfront the all existing query members. This ensures the correct cube join order + // with transitive joins even if they are already presented among query members. + newCollectedHints = this.enrichedJoinHintsFromJoinTree(newJoin, joinMembersJoinHints); } } while (newJoin?.joins.length > 0 && !this.isJoinTreesEqual(prevJoin, newJoin) && cnt < 10000 && newJoinHintsCollectedCnt > 0); diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 106319d1ed55f..d7fd3d3479d3f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1,6 +1,7 @@ import R from 'ramda'; import { CubeSymbols, PreAggregationDefinition } from '../compiler/CubeSymbols'; +import { FinishedJoinTree, JoinEdge } from '../compiler/JoinGraph'; import { UserError } from '../compiler/UserError'; import { BaseQuery } from './BaseQuery'; import { @@ -15,8 +16,6 @@ import { BaseGroupFilter } from './BaseGroupFilter'; import { BaseDimension } from './BaseDimension'; import { BaseSegment } from './BaseSegment'; -export type RollupJoin = any; - export type PartitionTimeDimension = { dimension: string; dateRange: [string, string]; @@ -45,6 +44,7 @@ export type PreAggregationForQuery = { references: PreAggregationReferences; preAggregationsToJoin?: PreAggregationForQuery[]; referencedPreAggregations?: PreAggregationForQuery[]; + // eslint-disable-next-line no-use-before-define rollupJoin?: RollupJoin; sqlAlias?: string; }; @@ -66,6 +66,18 @@ export type EvaluateReferencesContext = { export type BaseMember = BaseDimension | BaseMeasure | BaseFilter | BaseGroupFilter | BaseSegment; +export type JoinEdgeWithMembers = JoinEdge & { + fromMembers: string[]; + toMembers: string[]; +}; + +export type RollupJoinItem = JoinEdgeWithMembers & { + fromPreAggObj: PreAggregationForQuery; + toPreAggObj: PreAggregationForQuery; +}; + +export type RollupJoin = RollupJoinItem[]; + export type CanUsePreAggregationFn = (references: PreAggregationReferences) => boolean; /** @@ -155,8 +167,7 @@ export class PreAggregations { let preAggregations: PreAggregationForQuery[] = [foundPreAggregation]; if (foundPreAggregation.preAggregation.type === 'rollupJoin') { preAggregations = foundPreAggregation.preAggregationsToJoin || []; - } - if (foundPreAggregation.preAggregation.type === 'rollupLambda') { + } else if (foundPreAggregation.preAggregation.type === 'rollupLambda') { preAggregations = foundPreAggregation.referencedPreAggregations || []; } @@ -632,10 +643,11 @@ export class PreAggregations { * Determine whether pre-aggregation can be used or not. */ const canUsePreAggregationNotAdditive: CanUsePreAggregationFn = (references: PreAggregationReferences): boolean => { - const refTimeDimensions = backAlias(sortTimeDimensions(references.timeDimensions)); const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; + + const refTimeDimensions = backAlias(sortTimeDimensions(references.timeDimensions)); const backAliasMeasures = backAlias(references.measures); const backAliasDimensions = backAlias(references.dimensions); return (( @@ -654,9 +666,9 @@ export class PreAggregations { transformedQuery.allFiltersWithinSelectedDimensions && R.equals(backAliasDimensions, transformedQuery.sortedDimensions) ) && ( - R.all(m => backAliasMeasures.indexOf(m) !== -1, transformedQuery.measures) || + R.all(m => backAliasMeasures.includes(m), transformedQuery.measures) || // TODO do we need backAlias here? - R.all(m => backAliasMeasures.indexOf(m) !== -1, transformedQuery.leafMeasures) + R.all(m => backAliasMeasures.includes(m), transformedQuery.leafMeasures) )); }; @@ -728,22 +740,10 @@ export class PreAggregations { } } - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - const dimsToMatch = references.rollups.length > 0 ? references.dimensions : references.fullNameDimensions; - - const dimensionsMatch = (dimensions, doBackAlias) => R.all( - d => ( - doBackAlias ? - backAlias(dimsToMatch) : - (dimsToMatch) - ).indexOf(d) !== -1, - dimensions - ); - - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - const timeDimsToMatch = references.rollups.length > 0 ? references.timeDimensions : references.fullNameTimeDimensions; + const dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(references.dimensions) : references.dimensions; + return dimensions.every(d => target.includes(d)); + }; const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass( timeDimensionsList.map( @@ -757,8 +757,8 @@ export class PreAggregations { ) )( doBackAlias ? - backAlias(sortTimeDimensions(timeDimsToMatch)) : - (sortTimeDimensions(timeDimsToMatch)) + backAlias(sortTimeDimensions(references.timeDimensions)) : + (sortTimeDimensions(references.timeDimensions)) ); if (transformedQuery.ungrouped) { @@ -953,19 +953,42 @@ export class PreAggregations { } } + private collectJoinHintsFromRollupReferences(refs: PreAggregationReferences): (string | string[])[] { + if (!refs.joinTree) { + return []; + } + + const hints: (string | string[])[] = [refs.joinTree.root]; + + for (const j of refs.joinTree.joins) { + hints.push([j.from, j.to]); + } + + return hints; + } + // TODO check multiplication factor didn't change private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): RollupJoin { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { - const targetJoins = this.resolveJoinMembers( - // TODO join hints? - this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj)) - ); - const existingJoins = R.unnest(preAggObjsToJoin.map( - // TODO join hints? - p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) - )); + // It's not enough to call buildJoin() directly on cubesFromPreAggregation() + // because transitive joins won't be collected in that case. + const builtJoinTree = this.query.joinTreeForHints(this.cubesHintsFromPreAggregation(preAggObj), true); + + if (!builtJoinTree) { + throw new UserError(`Can't build join tree for pre-aggregation ${preAggObj.cube}.${preAggObj.preAggregationName}`); + } + + const targetJoins = this.resolveJoinMembers(builtJoinTree); + + // TODO join hints? + const existingJoins = preAggObjsToJoin + .map(p => this.resolveJoinMembers( + this.query.joinTreeForHints(this.cubesHintsFromPreAggregation(p), true) + )) + .flat(); + const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( existing => existing.originalFrom === target.originalFrom && existing.originalTo === target.originalTo && @@ -976,8 +999,8 @@ export class PreAggregations { throw new UserError(`Nothing to join in rollup join. Target joins ${JSON.stringify(targetJoins)} are included in existing rollup joins ${JSON.stringify(existingJoins)}`); } return nonExistingJoins.map(join => { - const fromPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.fromMembers, join); - const toPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.toMembers, join); + const fromPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.fromMembers, join, `${preAggObj.cube}.${preAggObj.preAggregationName}`); + const toPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.toMembers, join, `${preAggObj.cube}.${preAggObj.preAggregationName}`); return { ...join, fromPreAggObj, @@ -988,11 +1011,20 @@ export class PreAggregations { ); } - private preAggObjForJoin(preAggObjsToJoin: PreAggregationForQuery[], joinMembers, join): PreAggregationForQuery { + private preAggObjForJoin( + preAggObjsToJoin: PreAggregationForQuery[], + joinMembers: string[], + join: JoinEdgeWithMembers, + rollupJoinPreAggName: string, + ): PreAggregationForQuery { const fromPreAggObj = preAggObjsToJoin .filter(p => joinMembers.every(m => !!p.references.dimensions.find(d => m === d))); if (!fromPreAggObj.length) { - throw new UserError(`No rollups found that can be used for rollup join: ${JSON.stringify(join)}`); + const msg = `No rollups found that can be used for a rollup join from "${ + join.from}" (fromMembers: ${JSON.stringify(join.fromMembers)}) to "${join.to}" (toMembers: ${ + JSON.stringify(join.toMembers)}). Check the "${ + rollupJoinPreAggName}" pre-aggregation definition — you may have forgotten to specify the full dimension paths`; + throw new UserError(msg); } if (fromPreAggObj.length > 1) { throw new UserError( @@ -1002,14 +1034,19 @@ export class PreAggregations { return fromPreAggObj[0]; } - private resolveJoinMembers(join) { + private resolveJoinMembers(join: FinishedJoinTree): JoinEdgeWithMembers[] { + const joinMap = new Set(); + return join.joins.map(j => { + joinMap.add(j.originalFrom); + const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); - const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); + + const invalidMembers = memberPaths.filter(m => !joinMap.has(m[0]) && m[0] !== j.originalTo); if (invalidMembers.length) { throw new UserError(`Members ${invalidMembers.join(', ')} in join from '${j.originalFrom}' to '${j.originalTo}' doesn't reference join cubes`); } - const fromMembers = memberPaths.filter(m => m[0] === j.originalFrom).map(m => m.join('.')); + const fromMembers = memberPaths.filter(m => joinMap.has(m[0])).map(m => m.join('.')); if (!fromMembers.length) { throw new UserError(`From members are not found in [${memberPaths.map(m => m.join('.')).join(', ')}] for join ${JSON.stringify(j)}. Please make sure join fields are referencing dimensions instead of columns.`); } @@ -1017,6 +1054,8 @@ export class PreAggregations { if (!toMembers.length) { throw new UserError(`To members are not found in [${memberPaths.map(m => m.join('.')).join(', ')}] for join ${JSON.stringify(j)}. Please make sure join fields are referencing dimensions instead of columns.`); } + joinMap.add(j.originalTo); + return { ...j, fromMembers, @@ -1025,11 +1064,11 @@ export class PreAggregations { }); } - private cubesFromPreAggregation(preAggObj: PreAggregationForQuery): string[] { + private cubesHintsFromPreAggregation(preAggObj: PreAggregationForQuery): string[][] { return R.uniq( - preAggObj.references.measures.map(m => this.query.cubeEvaluator.parsePath('measures', m)).concat( - preAggObj.references.dimensions.map(m => this.query.cubeEvaluator.parsePathAnyType(m)) - ).map(p => p[0]) + preAggObj.references.measures.concat( + preAggObj.references.dimensions + ).map(p => p.split('.').slice(0, -1)) ); } @@ -1044,7 +1083,9 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - canUsePreAggregation: canUsePreAggregation(references), + // For rollupJoin and rollupLambda we need to enrich references with data + // from the underlying rollups which are collected later; + canUsePreAggregation: preAggregation.type === 'rollup' ? canUsePreAggregation(references) : false, references, preAggregationId: `${cube}.${preAggregationName}` }; @@ -1062,10 +1103,16 @@ export class PreAggregations { ); } ); + preAggregationsToJoin.forEach(preAgg => { + references.rollupsReferences.push(preAgg.references); + }); + const rollupJoin = this.buildRollupJoin(preAggObj, preAggregationsToJoin); + return { ...preAggObj, + canUsePreAggregation: canUsePreAggregation(references), preAggregationsToJoin, - rollupJoin: this.buildRollupJoin(preAggObj, preAggregationsToJoin) + rollupJoin, }; } else if (preAggregation.type === 'rollupLambda') { // TODO evaluation optimizations. Should be cached or moved to compile time. @@ -1110,8 +1157,12 @@ export class PreAggregations { PreAggregations.memberNameMismatchValidation(preAggObj, referencedPreAggregation, 'dimensions'); PreAggregations.memberNameMismatchValidation(preAggObj, referencedPreAggregation, 'timeDimensions'); }); + referencedPreAggregations.forEach(preAgg => { + references.rollupsReferences.push(preAgg.references); + }); return { ...preAggObj, + canUsePreAggregation: canUsePreAggregation(references), referencedPreAggregations, }; } else { @@ -1142,7 +1193,7 @@ export class PreAggregations { if (typeof member !== 'string') { return `${member.dimension.split('.')[1]}.${member.granularity}`; } else { - return member.split('.')[1]; + return member.split('.').at(-1)!; } }); } @@ -1228,11 +1279,10 @@ export class PreAggregations { cube, aggregation ) && - !!references.dimensions.find((d) => { + references.dimensions.some((d) => this.query.cubeEvaluator.dimensionByPath( // `d` can contain full join path, so we should trim it - const trimmedDimension = CubeSymbols.joinHintFromPath(d).path; - return this.query.cubeEvaluator.dimensionByPath(trimmedDimension).primaryKey; - }), + this.query.cubeEvaluator.memberShortNameFromPath(d) + ).primaryKey), }); } @@ -1277,6 +1327,38 @@ export class PreAggregations { .toLowerCase(); } + private enrichMembersCubeJoinPath(cubeName: string, joinsMap: Record): string[] { + const path = [cubeName]; + const parentMap = joinsMap; + while (parentMap[cubeName]) { + cubeName = parentMap[cubeName]; + path.push(cubeName); + } + + return path.reverse(); + } + + private buildMembersFullName(members: string[], joinsMap: Record): string[] { + return members.map(d => { + const [cubeName, ...restPath] = d.split('.'); + const path = this.enrichMembersCubeJoinPath(cubeName, joinsMap); + + return `${path.join('.')}.${restPath.join('.')}`; + }); + } + + private buildTimeDimensionsFullName(members: PreAggregationTimeDimensionReference[], joinsMap: Record): PreAggregationTimeDimensionReference[] { + return members.map(td => { + const [cubeName, ...restPath] = td.dimension.split('.'); + const path = this.enrichMembersCubeJoinPath(cubeName, joinsMap); + + return { + ...td, + dimension: `${path.join('.')}.${restPath.join('.')}`, + }; + }); + } + private evaluateAllReferences(cube: string, aggregation: PreAggregationDefinition, preAggregationName: string | null = null, context: EvaluateReferencesContext = {}): PreAggregationReferences { const evaluateReferences = () => { const references = this.query.cubeEvaluator.evaluatePreAggregationReferences(cube, aggregation); @@ -1287,18 +1369,18 @@ export class PreAggregations { if (preAggQuery) { // We need to build a join tree for all references, so they would always include full join path // even for preaggregation references without join path. It is necessary to be able to match - // query and preaggregation based on full join tree. But we can not update - // references.{dimensions,measures,timeDimensions} directly, because it will break - // evaluation of references in the query on later stages. - // So we store full named members separately and use them in canUsePreAggregation functions. + // query and preaggregation based on full join tree. references.joinTree = preAggQuery.join; - const root = references.joinTree?.root || ''; - references.fullNameMeasures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); - references.fullNameDimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); - references.fullNameTimeDimensions = references.timeDimensions.map(d => ({ - dimension: (d.dimension.startsWith(root) ? d.dimension : `${root}.${d.dimension}`), - granularity: d.granularity, - })); + const joinsMap: Record = {}; + if (references.joinTree) { + for (const j of references.joinTree.joins) { + joinsMap[j.to] = j.from; + } + } + + references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); + references.measures = this.buildMembersFullName(references.measures, joinsMap); + references.timeDimensions = this.buildTimeDimensionsFullName(references.timeDimensions, joinsMap); } } if (aggregation.type === 'rollupLambda') { @@ -1321,7 +1403,15 @@ export class PreAggregations { if (!preAggregationName) { return evaluateReferences(); } - return this.query.cacheValue(['evaluateAllReferences', cube, preAggregationName], evaluateReferences); + + // Using [cube, preAggregationName] alone as cache keys isn’t reliable, + // as different queries can build distinct join graphs during pre-aggregation matching. + // Because the matching logic compares join subgraphs — particularly for 'rollupJoin' and 'rollupLambda' + // pre-aggregations — relying on such keys may cause incorrect results. + return this.query.cacheValue( + ['evaluateAllReferences', cube, preAggregationName, JSON.stringify(this.query.join)], + evaluateReferences + ); } public originalSqlPreAggregationTable(preAggregationDescription: PreAggregationForCube): string { @@ -1407,7 +1497,7 @@ export class PreAggregations { }); if (preAggregationForQuery.preAggregation.type === 'rollupJoin') { - const join = preAggregationForQuery.rollupJoin; + const join = preAggregationForQuery.rollupJoin!; toJoin = [ sqlAndAlias(join[0].fromPreAggObj), @@ -1553,8 +1643,7 @@ export class PreAggregations { private rollupMembers(preAggregationForQuery: PreAggregationForQuery, type: T): PreAggregationReferences[T] { return preAggregationForQuery.preAggregation.type === 'autoRollup' ? - // TODO proper types - (preAggregationForQuery.preAggregation as any)[type] : + preAggregationForQuery.preAggregation[type] : this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation, preAggregationForQuery.preAggregationName)[type]; } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index e540183d3d9a8..b7f85b84bf682 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -89,12 +89,10 @@ export type PreAggregationTimeDimensionReference = { export type PreAggregationReferences = { allowNonStrictDateRangeMatch?: boolean, dimensions: Array, - fullNameDimensions: Array, measures: Array, - fullNameMeasures: Array, timeDimensions: Array, - fullNameTimeDimensions: Array, rollups: Array, + rollupsReferences: Array, multipliedMeasures?: Array, joinTree?: FinishedJoinTree; }; @@ -730,6 +728,18 @@ export class CubeEvaluator extends CubeSymbols { return !!this.evaluatedCubes[cube]; } + public memberShortNameFromPath(path: string | string[]): string { + if (!Array.isArray(path)) { + path = path.split('.'); + } + + if (path.length < 2) { + throw new UserError(`Not full member name provided: ${path[0]}`); + } + + return path.slice(-2).join('.'); + } + public cubeFromPath(path: string): EvaluatedCube { return this.evaluatedCubes[this.cubeNameFromPath(path)]; } @@ -888,9 +898,7 @@ export class CubeEvaluator extends CubeSymbols { timeDimensions, rollups: aggregation.rollupReferences && this.evaluateReferences(cube, aggregation.rollupReferences, { originalSorting: true }) || [], - fullNameDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() - fullNameMeasures: [], // May be filled in PreAggregations.evaluateAllReferences() - fullNameTimeDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() + rollupsReferences: [], // May be filled in PreAggregations.evaluateAllReferences() }; } } diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 39b9acc3296e7..505b10f952bf9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -7,7 +7,7 @@ import type { CubeEvaluator, MeasureDefinition } from './CubeEvaluator'; import type { CubeDefinition, JoinDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; -type JoinEdge = { +export type JoinEdge = { join: JoinDefinition, from: string, to: string, diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 5235d0b98917a..eab0d3a137e5d 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -641,6 +641,593 @@ describe('PreAggregations', () => { } } }); + + cube('cube_1', { + sql: \`SELECT 1 as id, 'dim_1' as dim_1\`, + + joins: { + cube_2: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_1} = \${cube_2.dim_1}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_1: { + sql: 'dim_1', + type: 'string' + }, + }, + + pre_aggregations: { + aaa: { + dimensions: [ + dim_1 + ] + }, + rollupJoin: { + type: 'rollupJoin', + dimensions: [ + dim_1, + CUBE.cube_2.dim_1, + CUBE.cube_2.dim_2 // XXX + ], + rollups: [ + aaa, + cube_2.bbb + ] + } + } + }); + + cube('cube_2', { + sql: \`SELECT 2 as id, 'dim_1' as dim_1, 'dim_2' as dim_2\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_1: { + sql: 'dim_1', + type: 'string' + }, + + dim_2: { + sql: 'dim_2', + type: 'string' + }, + }, + + pre_aggregations: { + bbb: { + dimensions: [ + dim_1, + dim_2, + ] + } + } + }); + + cube('cube_x', { + sql: \`SELECT 1 as id, 'dim_x' as dim_x\`, + + joins: { + cube_y: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_x} = \${cube_y.dim_x}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_x: { + sql: 'dim_x', + type: 'string' + }, + }, + + pre_aggregations: { + xxx: { + dimensions: [ + dim_x + ] + }, + rollupJoinThreeCubes: { + type: 'rollupJoin', + dimensions: [ + dim_x, + cube_y.dim_y, + cube_z.dim_z + ], + rollups: [ + xxx, + cube_y.yyy, + cube_z.zzz + ] + } + } + }); + + cube('cube_y', { + sql: \`SELECT 2 as id, 'dim_x' as dim_x, 'dim_y' as dim_y\`, + + joins: { + cube_z: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_y} = \${cube_z.dim_y}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_x: { + sql: 'dim_x', + type: 'string' + }, + + dim_y: { + sql: 'dim_y', + type: 'string' + }, + }, + + pre_aggregations: { + yyy: { + dimensions: [ + dim_x, + dim_y, + ] + } + } + }); + + cube('cube_z', { + sql: \`SELECT 3 as id, 'dim_y' as dim_y, 'dim_z' as dim_z\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_y: { + sql: 'dim_y', + type: 'string' + }, + + dim_z: { + sql: 'dim_z', + type: 'string' + }, + }, + + pre_aggregations: { + zzz: { + dimensions: [ + dim_y, + dim_z, + ] + } + } + }); + + cube('cube_a', { + sql: \`SELECT 1 as id, 'dim_a' as dim_a\`, + + joins: { + cube_b: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_b.dim_a}\` + }, + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_c.dim_a}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + aaa_rollup: { + dimensions: [ + dim_a + ] + }, + rollupJoinAB: { + type: 'rollupJoin', + dimensions: [ + dim_a, + CUBE.cube_b.dim_b, + CUBE.cube_b.cube_c.dim_c + ], + rollups: [ + aaa_rollup, + cube_b.bbb_rollup + ] + } + } + }); + + cube('cube_b', { + sql: \`SELECT 2 as id, 'dim_a' as dim_a, 'dim_b' as dim_b\`, + + joins: { + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_b} = \${cube_c.dim_b}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + bbb_rollup: { + dimensions: [ + dim_a, + dim_b, + cube_c.dim_c + ] + } + } + }); + + cube('cube_c', { + sql: \`SELECT 3 as id, 'dim_a' as dim_a, 'dim_b' as dim_b, 'dim_c' as dim_c\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + + dim_c: { + sql: 'dim_c', + type: 'string' + }, + } + }); + + view('view_abc', { + cubes: [ + { + join_path: cube_a, + includes: ['dim_a'] + }, + { + join_path: cube_a.cube_b, + includes: ['dim_b'] + }, + { + join_path: cube_a.cube_b.cube_c, + includes: ['dim_c'] + } + ] + }); + + // Cube with not full paths in rollupJoin pre-aggregation + cube('cube_a_to_fail_pre_agg', { + sql: \`SELECT 1 as id, 'dim_a' as dim_a\`, + + joins: { + cube_b: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_b.dim_a}\` + }, + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_c.dim_a}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + aaa_rollup: { + dimensions: [ + dim_a + ] + }, + rollupJoinAB: { + type: 'rollupJoin', + dimensions: [ + dim_a, + cube_b.dim_b, + cube_c.dim_c + ], + rollups: [ + aaa_rollup, + cube_b.bbb_rollup + ] + } + } + }); + + // Models with transitive joins for rollupJoin matching + cube('merchant_dims', { + sql: \` + SELECT 101 AS merchant_sk, 'M1' AS merchant_id + UNION ALL + SELECT 102 AS merchant_sk, 'M2' AS merchant_id + \`, + + dimensions: { + merchant_sk: { + sql: 'merchant_sk', + type: 'number', + primary_key: true + }, + merchant_id: { + sql: 'merchant_id', + type: 'string' + } + } + }); + + cube('product_dims', { + sql: \` + SELECT 201 AS product_sk, 'P1' AS product_id + UNION ALL + SELECT 202 AS product_sk, 'P2' AS product_id + \`, + + dimensions: { + product_sk: { + sql: 'product_sk', + type: 'number', + primary_key: true + }, + product_id: { + sql: 'product_id', + type: 'string' + } + } + }); + + cube('merchant_and_product_dims', { + sql: \` + SELECT 'M1' AS merchant_id, 'P1' AS product_id, 'Organic' AS acquisition_channel, 'SOLD' AS status + UNION ALL + SELECT 'M1' AS merchant_id, 'P2' AS product_id, 'Paid' AS acquisition_channel, 'PAID' AS status + UNION ALL + SELECT 'M2' AS merchant_id, 'P1' AS product_id, 'Referral' AS acquisition_channel, 'RETURNED' AS status + \`, + + dimensions: { + product_id: { + sql: 'product_id', + type: 'string', + primary_key: true + }, + merchant_id: { + sql: 'merchant_id', + type: 'string', + primary_key: true + }, + status: { + sql: 'status', + type: 'string' + }, + acquisition_channel: { + sql: 'acquisition_channel', + type: 'string' + } + }, + + pre_aggregations: { + bridge_rollup: { + dimensions: [ + merchant_id, + product_id, + acquisition_channel, + status + ] + } + } + }); + + cube('other_facts', { + sql: \` + SELECT 1 AS id, 1 AS fact_id, 'OF1' AS fact + UNION ALL + SELECT 2 AS id, 2 AS fact_id, 'OF2' AS fact + UNION ALL + SELECT 3 AS id, 3 AS fact_id, 'OF3' AS fact + \`, + + dimensions: { + other_fact_id: { + sql: 'id', + type: 'number', + primary_key: true + }, + fact_id: { + sql: 'fact_id', + type: 'number' + }, + fact: { + sql: 'fact', + type: 'string' + } + }, + + pre_aggregations: { + bridge_rollup: { + dimensions: [ + fact_id, + fact + ] + } + } + + }); + + cube('test_facts', { + sql: \` + SELECT 1 AS id, 101 AS merchant_sk, 201 AS product_sk, 100 AS amount + UNION ALL + SELECT 2 AS id, 101 AS merchant_sk, 202 AS product_sk, 150 AS amount + UNION ALL + SELECT 3 AS id, 102 AS merchant_sk, 201 AS product_sk, 200 AS amount + \`, + + joins: { + merchant_dims: { + relationship: 'many_to_one', + sql: \`\${CUBE.merchant_sk} = \${merchant_dims.merchant_sk}\` + }, + product_dims: { + relationship: 'many_to_one', + sql: \`\${CUBE.product_sk} = \${product_dims.product_sk}\` + }, + // Transitive join - depends on merchant_dims and product_dims + merchant_and_product_dims: { + relationship: 'many_to_one', + sql: \`\${merchant_dims.merchant_id} = \${merchant_and_product_dims.merchant_id} AND \${product_dims.product_id} = \${merchant_and_product_dims.product_id}\` + }, + other_facts: { + relationship: 'one_to_many', + sql: \`\${CUBE.id} = \${other_facts.fact_id}\` + }, + }, + + dimensions: { + id: { + sql: 'id', + type: 'number', + primary_key: true + }, + merchant_sk: { + sql: 'merchant_sk', + type: 'number' + }, + product_sk: { + sql: 'product_sk', + type: 'number' + }, + acquisition_channel: { + sql: \`\${merchant_and_product_dims.acquisition_channel}\`, + type: 'string' + } + }, + + measures: { + amount_sum: { + sql: 'amount', + type: 'sum' + } + }, + + pre_aggregations: { + facts_rollup: { + dimensions: [ + id, + merchant_sk, + merchant_dims.merchant_sk, + merchant_dims.merchant_id, + merchant_and_product_dims.merchant_id, + product_sk, + product_dims.product_sk, + product_dims.product_id, + merchant_and_product_dims.product_id, + acquisition_channel, + merchant_and_product_dims.status + ] + }, + rollupJoinTransitive: { + type: 'rollupJoin', + dimensions: [ + merchant_sk, + product_sk, + CUBE.merchant_and_product_dims.status, + CUBE.other_facts.fact + ], + rollups: [ + facts_rollup, + other_facts.bridge_rollup + ] + } + } + }); + `); it('simple pre-aggregation', async () => { @@ -2816,38 +3403,211 @@ describe('PreAggregations', () => { expect(loadSql[0]).toMatch(/THEN 1 END `real_time_lambda_visitors__count`/); }); - it('querying proxied to external cube pre-aggregation time-dimension', async () => { + it('rollupJoin pre-aggregation', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - measures: [], - dimensions: [], + dimensions: ['cube_1.dim_1', 'cube_2.dim_2'], timezone: 'America/Los_Angeles', - preAggregationsSchema: '', - timeDimensions: [{ - dimension: 'cube_pre_agg_proxy_b.terminal_date', - granularity: 'day', - }], - order: [], + preAggregationsSchema: '' }); const queryAndParams = query.buildSqlAndParams(); console.log(queryAndParams); - const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); - console.log(JSON.stringify(preAggregationsDescription, null, 2)); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(2); + const aaa = preAggregationsDescription.find(p => p.preAggregationId === 'cube_1.aaa'); + const bbb = preAggregationsDescription.find(p => p.preAggregationId === 'cube_2.bbb'); + expect(aaa).toBeDefined(); + expect(bbb).toBeDefined(); - expect((preAggregationsDescription)[0].loadSql[0]).toMatch(/main/); + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoin'); - const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_1__dim_1: 'dim_1', + cube_2__dim_2: 'dim_2', + }] + ); + }); + }); - console.log(JSON.stringify(queries.concat(queryAndParams))); + it('rollupJoin pre-aggregation with three cubes', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_x.dim_x', 'cube_y.dim_y', 'cube_z.dim_z'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(3); + const xxx = preAggregationsDescription.find(p => p.preAggregationId === 'cube_x.xxx'); + const yyy = preAggregationsDescription.find(p => p.preAggregationId === 'cube_y.yyy'); + const zzz = preAggregationsDescription.find(p => p.preAggregationId === 'cube_z.zzz'); + expect(xxx).toBeDefined(); + expect(yyy).toBeDefined(); + expect(zzz).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinThreeCubes'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_x__dim_x: 'dim_x', + cube_y__dim_y: 'dim_y', + cube_z__dim_z: 'dim_z', + }] + ); + }); + }); + + it('rollupJoin pre-aggregation with nested joins via view (A->B->C)', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['view_abc.dim_a', 'view_abc.dim_b', 'view_abc.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(2); + const aaa = preAggregationsDescription.find(p => p.preAggregationId === 'cube_a.aaa_rollup'); + const bbb = preAggregationsDescription.find(p => p.preAggregationId === 'cube_b.bbb_rollup'); + expect(aaa).toBeDefined(); + expect(bbb).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinAB'); return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { expect(res).toEqual( [{ - cube_pre_agg_proxy_b__terminal_date_day: '2025-10-01T00:00:00.000Z', + view_abc__dim_a: 'dim_a', + view_abc__dim_b: 'dim_b', + view_abc__dim_c: 'dim_c', }] ); }); }); + + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollupJoin pre-aggregation with nested joins via cube (A->B->C)', () => { + // Need to investigate tesseract internals of how pre-aggs members are resolved and how + // rollups are used to construct rollupJoins. + }); + } else { + it('rollupJoin pre-aggregation with nested joins via cube (A->B->C)', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_a.dim_a', 'cube_b.dim_b', 'cube_c.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(0); + + expect(query.preAggregations?.preAggregationForQuery).toBeUndefined(); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_a__dim_a: 'dim_a', + cube_b__dim_b: 'dim_b', + cube_c__dim_c: 'dim_c', + }] + ); + }); + }); + } + + it('rollupJoin pre-aggregation matching with transitive joins', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: [ + 'test_facts.merchant_sk', + 'test_facts.product_sk', + 'merchant_and_product_dims.status', + 'other_facts.fact' + ], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(JSON.stringify(preAggregationsDescription, null, 2)); + + // Verify that both rollups are included in the description + expect(preAggregationsDescription.length).toBe(2); + const factsRollup = preAggregationsDescription.find(p => p.preAggregationId === 'test_facts.facts_rollup'); + const bridgeRollup = preAggregationsDescription.find(p => p.preAggregationId === 'other_facts.bridge_rollup'); + expect(factsRollup).toBeDefined(); + expect(bridgeRollup).toBeDefined(); + + // Verify that the rollupJoin pre-aggregation can be used for the query + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinTransitive'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual([ + { + merchant_and_product_dims__status: 'SOLD', + other_facts__fact: 'OF1', + test_facts__merchant_sk: 101, + test_facts__product_sk: 201, + }, + { + merchant_and_product_dims__status: 'PAID', + other_facts__fact: 'OF2', + test_facts__merchant_sk: 101, + test_facts__product_sk: 202, + }, + { + merchant_and_product_dims__status: 'RETURNED', + other_facts__fact: 'OF3', + test_facts__merchant_sk: 102, + test_facts__product_sk: 201, + }, + ]); + }); + }); + + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollupJoin pre-aggregation with not-full paths should fail', () => { + // Need to investigate tesseract internals of how pre-aggs members are resolved and how + // rollups are used to construct rollupJoins. + }); + } else { + it('rollupJoin pre-aggregation with not-full paths should fail', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_a_to_fail_pre_agg.dim_a', 'cube_b.dim_b', 'cube_c.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + expect(() => query.buildSqlAndParams()).toThrow('No rollups found that can be used for a rollup join'); + }); + } }); diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts index 004fb1de3c4f4..1b2d9f72af826 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts @@ -59,9 +59,7 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], + rollupsReferences: [], }; await compiler.compile(); diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index bea487909c743..784f066ecd2ec 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -69,9 +69,7 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], + rollupsReferences: [], }; await compiler.compile(); diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 1ba8b3622a166..22309182253cc 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -665,9 +665,7 @@ describe('Refresh Scheduler', () => { measures: ['Foo.count'], timeDimensions: [{ dimension: 'Foo.time', granularity: 'hour' }], rollups: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], + rollupsReferences: [], }, refreshKey: { every: '1 hour', updateWindow: '1 day', incremental: true }, },