From 2e9e662067c378dda9881fe8485e4e3ce9e4d313 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 29 Sep 2025 20:04:37 +0300 Subject: [PATCH 01/38] expose joinGraph from transformQueryToCanUseForm() --- packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 106319d1ed55f..e65e77dd4445f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -510,6 +510,7 @@ export class PreAggregations { allValuesEq1(filterDimensionsSingleValueEqual) ? new Set(filterDimensionsSingleValueEqual?.keys()) : null; return { + joinGraph: query.join, sortedDimensions, sortedTimeDimensions, timeDimensions, From 1e867bdedfa08d01acebc3e22ee72106a31f008b Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 15:43:45 +0300 Subject: [PATCH 02/38] get rid of ramda in favor of simple js --- .../src/adapter/PreAggregations.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index e65e77dd4445f..73238c63b354d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -733,14 +733,10 @@ export class PreAggregations { // no connections in the joinTree between cubes from different datasources const dimsToMatch = references.rollups.length > 0 ? references.dimensions : references.fullNameDimensions; - const dimensionsMatch = (dimensions, doBackAlias) => R.all( - d => ( - doBackAlias ? - backAlias(dimsToMatch) : - (dimsToMatch) - ).indexOf(d) !== -1, - dimensions - ); + const dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are // no connections in the joinTree between cubes from different datasources From 59c2e60bb308784e8688b123a70ccaea5e301953 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 15:55:24 +0300 Subject: [PATCH 03/38] preparing dimensionsMatch() --- .../src/adapter/PreAggregations.ts | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 73238c63b354d..5b63b78ec0dd3 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -729,18 +729,29 @@ export class PreAggregations { } } - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - const dimsToMatch = references.rollups.length > 0 ? references.dimensions : references.fullNameDimensions; - - const dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; - return dimensions.every(d => target.includes(d)); - }; + let dimsToMatch: string[]; + let timeDimsToMatch: PreAggregationTimeDimensionReference[]; + let dimensionsMatch: (dimensions: string[], doBackAlias: boolean) => boolean; + + if (references.rollups.length > 0) { + // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are + // no connections in the joinTree between cubes from different datasources + dimsToMatch = references.dimensions; + timeDimsToMatch = references.timeDimensions; + + dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; + } else { + dimsToMatch = references.fullNameDimensions; + timeDimsToMatch = references.fullNameTimeDimensions; - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - const timeDimsToMatch = references.rollups.length > 0 ? references.timeDimensions : references.fullNameTimeDimensions; + dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; + } const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass( timeDimensionsList.map( From 9096d1c15ddab5ec94cf77d75f9a03c200dc5bcc Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 17:26:47 +0300 Subject: [PATCH 04/38] fix(schema-compiler): Fix pre-agg matching for 'rollupJoin' / 'rollupLambda' pre-aggregations --- .../src/adapter/PreAggregations.ts | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 5b63b78ec0dd3..d37124ec60100 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -509,8 +509,18 @@ export class PreAggregations { filterDimensionsSingleValueEqual = allValuesEq1(filterDimensionsSingleValueEqual) ? new Set(filterDimensionsSingleValueEqual?.keys()) : null; + // Build reverse query joins map, which is used for + // rollupLambda and rollupJoin pre-aggs matching later + const joinsMap: Record = {}; + if (query.join) { + for (const j of query.join.joins) { + joinsMap[j.to] = j.from; + } + } + return { - joinGraph: query.join, + joinGraphRoot: query.join?.root, + joinsMap, sortedDimensions, sortedTimeDimensions, timeDimensions, @@ -736,11 +746,33 @@ export class PreAggregations { if (references.rollups.length > 0) { // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are // no connections in the joinTree between cubes from different datasources + // but joinGraph of the query has all the connections, necessary for serving the query, + // so we use this information to complete the full paths of members from the root of the query + // up to the pre-agg cube. dimsToMatch = references.dimensions; timeDimsToMatch = references.timeDimensions; + const buildPath = (cube: string): string[] => { + const path = [cube]; + const parentMap = transformedQuery.joinsMap; + while (parentMap[cube]) { + cube = parentMap[cube]; + path.push(cube); + } + return path.reverse(); + }; + dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + let target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + target = target.map(dim => { + const [cube, field] = dim.split('.'); + if (cube === transformedQuery.joinGraphRoot) { + return dim; + } + const path = buildPath(cube); + return `${path.join('.')}.${field}`; + }); + return dimensions.every(d => target.includes(d)); }; } else { From d8d73bbe129a346f641bfb6cb5d40f30d8d9e58b Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 17:44:00 +0300 Subject: [PATCH 05/38] add tests # Conflicts: # packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts --- .../postgres/pre-aggregations.test.ts | 110 ++++++++++++------ 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 5235d0b98917a..20977a33ce9d8 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -641,6 +641,81 @@ describe('PreAggregations', () => { } } }); + + cube('cube_1', { + sql: \`SELECT 1 as id, 'dim_1' as dim_1\`, + + joins: { + cube_2: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_1} = \${cube_2.dim_1}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_1: { + sql: 'dim_1', + type: 'string' + }, + }, + + pre_aggregations: { + aaa: { + dimensions: [ + dim_1 + ] + }, + rollupJoin: { + type: 'rollupJoin', + dimensions: [ + dim_1, + cube_2.dim_1, + cube_2.dim_2 // XXX + ], + rollups: [ + aaa, + cube_2.bbb + ] + } + } + }); + + cube('cube_2', { + sql: \`SELECT 2 as id, 'dim_1' as dim_1, 'dim_2' as dim_2\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_1: { + sql: 'dim_1', + type: 'string' + }, + + dim_2: { + sql: 'dim_2', + type: 'string' + }, + }, + + pre_aggregations: { + bbb: { + dimensions: [ + dim_1, + dim_2, + ] + } + } + }); `); it('simple pre-aggregation', async () => { @@ -2815,39 +2890,4 @@ describe('PreAggregations', () => { expect(loadSql[0]).not.toMatch(/GROUP BY/); expect(loadSql[0]).toMatch(/THEN 1 END `real_time_lambda_visitors__count`/); }); - - it('querying proxied to external cube pre-aggregation time-dimension', async () => { - await compiler.compile(); - - const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - measures: [], - dimensions: [], - timezone: 'America/Los_Angeles', - preAggregationsSchema: '', - timeDimensions: [{ - dimension: 'cube_pre_agg_proxy_b.terminal_date', - granularity: 'day', - }], - order: [], - }); - - const queryAndParams = query.buildSqlAndParams(); - console.log(queryAndParams); - const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); - console.log(JSON.stringify(preAggregationsDescription, null, 2)); - - expect((preAggregationsDescription)[0].loadSql[0]).toMatch(/main/); - - const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); - - console.log(JSON.stringify(queries.concat(queryAndParams))); - - return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { - expect(res).toEqual( - [{ - cube_pre_agg_proxy_b__terminal_date_day: '2025-10-01T00:00:00.000Z', - }] - ); - }); - }); }); From 145a54d5c4641af2784f72538d0eabad7e484a30 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 14:42:07 +0300 Subject: [PATCH 06/38] add test for 3-cube rollupJoin pre-agg --- .../postgres/pre-aggregations.test.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 20977a33ce9d8..829ec758314da 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -716,6 +716,120 @@ describe('PreAggregations', () => { } } }); + + cube('cube_x', { + sql: \`SELECT 1 as id, 'dim_x' as dim_x\`, + + joins: { + cube_y: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_x} = \${cube_y.dim_x}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_x: { + sql: 'dim_x', + type: 'string' + }, + }, + + pre_aggregations: { + xxx: { + dimensions: [ + dim_x + ] + }, + rollupJoinThreeCubes: { + type: 'rollupJoin', + dimensions: [ + dim_x, + cube_y.dim_y, + cube_z.dim_z + ], + rollups: [ + xxx, + cube_y.yyy, + cube_z.zzz + ] + } + } + }); + + cube('cube_y', { + sql: \`SELECT 2 as id, 'dim_x' as dim_x, 'dim_y' as dim_y\`, + + joins: { + cube_z: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_y} = \${cube_z.dim_y}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_x: { + sql: 'dim_x', + type: 'string' + }, + + dim_y: { + sql: 'dim_y', + type: 'string' + }, + }, + + pre_aggregations: { + yyy: { + dimensions: [ + dim_x, + dim_y, + ] + } + } + }); + + cube('cube_z', { + sql: \`SELECT 3 as id, 'dim_y' as dim_y, 'dim_z' as dim_z\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_y: { + sql: 'dim_y', + type: 'string' + }, + + dim_z: { + sql: 'dim_z', + type: 'string' + }, + }, + + pre_aggregations: { + zzz: { + dimensions: [ + dim_y, + dim_z, + ] + } + } + }); `); it('simple pre-aggregation', async () => { @@ -2890,4 +3004,36 @@ describe('PreAggregations', () => { expect(loadSql[0]).not.toMatch(/GROUP BY/); expect(loadSql[0]).toMatch(/THEN 1 END `real_time_lambda_visitors__count`/); }); + + it('rollupJoin pre-aggregation', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_1.dim_1', 'cube_2.dim_2'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(2); + const aaa = preAggregationsDescription.find(p => p.preAggregationId === 'cube_1.aaa'); + const bbb = preAggregationsDescription.find(p => p.preAggregationId === 'cube_2.bbb'); + expect(aaa).toBeDefined(); + expect(bbb).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoin'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_1__dim_1: 'dim_1', + cube_2__dim_2: 'dim_2', + }] + ); + }); + }); }); From c4c354b2e735adf3f0133cef05862e00158eece4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 18:47:33 +0300 Subject: [PATCH 07/38] use rollupsReferences for 'rollupJoin' / 'rollupLambda' pre-agg matching --- .../src/adapter/PreAggregations.ts | 27 +++++++++++++++---- .../src/compiler/CubeEvaluator.ts | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index d37124ec60100..c259412a43945 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -749,8 +749,15 @@ export class PreAggregations { // but joinGraph of the query has all the connections, necessary for serving the query, // so we use this information to complete the full paths of members from the root of the query // up to the pre-agg cube. - dimsToMatch = references.dimensions; - timeDimsToMatch = references.timeDimensions; + // We use references from the underlying pre-aggregations, filtered with members existing in the root + // pre-aggregation itself. + + dimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameDimensions) + .filter(d => references.dimensions.some(rd => d.endsWith(rd))); + timeDimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameTimeDimensions) + .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); const buildPath = (cube: string): string[] => { const path = [cube]; @@ -765,12 +772,12 @@ export class PreAggregations { dimensionsMatch = (dimensions, doBackAlias) => { let target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; target = target.map(dim => { - const [cube, field] = dim.split('.'); + const [cube, ...restPath] = dim.split('.'); if (cube === transformedQuery.joinGraphRoot) { return dim; } const path = buildPath(cube); - return `${path.join('.')}.${field}`; + return `${path.join('.')}.${restPath.join('.')}`; }); return dimensions.every(d => target.includes(d)); @@ -1084,7 +1091,9 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - canUsePreAggregation: canUsePreAggregation(references), + // For rollupJoin and rollupLambda we need to pass references of the underlying rollups + // to canUsePreAggregation fn, which are collected later; + canUsePreAggregation: preAggregation.type === 'rollup' ? canUsePreAggregation(references) : false, references, preAggregationId: `${cube}.${preAggregationName}` }; @@ -1102,8 +1111,12 @@ export class PreAggregations { ); } ); + preAggregationsToJoin.forEach(preAgg => { + references.rollupsReferences.push(preAgg.references); + }); return { ...preAggObj, + canUsePreAggregation: canUsePreAggregation(references), preAggregationsToJoin, rollupJoin: this.buildRollupJoin(preAggObj, preAggregationsToJoin) }; @@ -1150,8 +1163,12 @@ export class PreAggregations { PreAggregations.memberNameMismatchValidation(preAggObj, referencedPreAggregation, 'dimensions'); PreAggregations.memberNameMismatchValidation(preAggObj, referencedPreAggregation, 'timeDimensions'); }); + referencedPreAggregations.forEach(preAgg => { + references.rollupsReferences.push(preAgg.references); + }); return { ...preAggObj, + canUsePreAggregation: canUsePreAggregation(references), referencedPreAggregations, }; } else { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index e540183d3d9a8..f89ddad1da0db 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -95,6 +95,7 @@ export type PreAggregationReferences = { timeDimensions: Array, fullNameTimeDimensions: Array, rollups: Array, + rollupsReferences: Array, multipliedMeasures?: Array, joinTree?: FinishedJoinTree; }; @@ -891,6 +892,7 @@ export class CubeEvaluator extends CubeSymbols { fullNameDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() fullNameMeasures: [], // May be filled in PreAggregations.evaluateAllReferences() fullNameTimeDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() + rollupsReferences: [], // May be filled in PreAggregations.evaluateAllReferences() }; } } From 1522d406a2d7240f1b4a025cd3bb2abc566f0308 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 18:47:52 +0300 Subject: [PATCH 08/38] fix old tests with new required fields --- .../test/unit/pre-agg-by-filter-match.test.ts | 10 +++++++--- .../test/unit/pre-agg-time-dim-match.test.ts | 10 +++++++--- .../test/unit/RefreshScheduler.test.ts | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts index 004fb1de3c4f4..7253ca3c35720 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts @@ -59,9 +59,13 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], + fullNameDimensions: testPreAgg.segments ? testPreAgg.dimensions.concat(testPreAgg.segments) : testPreAgg.dimensions, + fullNameMeasures: testPreAgg.measures, + fullNameTimeDimensions: [{ + dimension: testPreAgg.timeDimension, + granularity: testPreAgg.granularity, + }], + rollupsReferences: [], }; await compiler.compile(); diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index bea487909c743..56d294701ba6c 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -69,9 +69,13 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], + fullNameDimensions: testPreAgg.dimensions, + fullNameMeasures: testPreAgg.measures, + fullNameTimeDimensions: [{ + dimension: testPreAgg.timeDimension, + granularity: testPreAgg.granularity, + }], + rollupsReferences: [], }; await compiler.compile(); diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 1ba8b3622a166..370dd90d82742 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -665,6 +665,7 @@ describe('Refresh Scheduler', () => { measures: ['Foo.count'], timeDimensions: [{ dimension: 'Foo.time', granularity: 'hour' }], rollups: [], + rollupsReferences: [], fullNameDimensions: [], fullNameMeasures: [], fullNameTimeDimensions: [], From af336cd99609380e24394740a2721fce65c0262a Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 18:48:24 +0300 Subject: [PATCH 09/38] more tests --- .../postgres/pre-aggregations.test.ts | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 829ec758314da..812fdd163f726 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -830,6 +830,143 @@ describe('PreAggregations', () => { } } }); + + cube('cube_a', { + sql: \`SELECT 1 as id, 'dim_a' as dim_a\`, + + joins: { + cube_b: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_b.dim_a}\` + }, + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_c.dim_a}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + aaa_rollup: { + dimensions: [ + dim_a + ] + }, + rollupJoinAB: { + type: 'rollupJoin', + dimensions: [ + dim_a, + cube_b.dim_b, + cube_c.dim_c + ], + rollups: [ + aaa_rollup, + cube_b.bbb_rollup + ] + } + } + }); + + cube('cube_b', { + sql: \`SELECT 2 as id, 'dim_a' as dim_a, 'dim_b' as dim_b\`, + + joins: { + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_b} = \${cube_c.dim_b}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + bbb_rollup: { + dimensions: [ + dim_a, + dim_b, + cube_c.dim_c + ] + } + } + }); + + cube('cube_c', { + sql: \`SELECT 3 as id, 'dim_a' as dim_a, 'dim_b' as dim_b, 'dim_c' as dim_c\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + + dim_c: { + sql: 'dim_c', + type: 'string' + }, + } + }); + + view('view_abc', { + cubes: [ + { + join_path: cube_a, + includes: ['dim_a'] + }, + { + join_path: cube_a.cube_b, + includes: ['dim_b'] + }, + { + join_path: cube_a.cube_b.cube_c, + includes: ['dim_c'] + } + ] + }); + `); it('simple pre-aggregation', async () => { @@ -3036,4 +3173,39 @@ describe('PreAggregations', () => { ); }); }); + + it('rollupJoin pre-aggregation with three cubes', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_x.dim_x', 'cube_y.dim_y', 'cube_z.dim_z'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(3); + const xxx = preAggregationsDescription.find(p => p.preAggregationId === 'cube_x.xxx'); + const yyy = preAggregationsDescription.find(p => p.preAggregationId === 'cube_y.yyy'); + const zzz = preAggregationsDescription.find(p => p.preAggregationId === 'cube_z.zzz'); + expect(xxx).toBeDefined(); + expect(yyy).toBeDefined(); + expect(zzz).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinThreeCubes'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_x__dim_x: 'dim_x', + cube_y__dim_y: 'dim_y', + cube_z__dim_z: 'dim_z', + }] + ); + }); + }); }); From a73fd193b1c284c96ef342877a47059aec8dfc6b Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 20:54:42 +0300 Subject: [PATCH 10/38] implement pre-agg matching using pre-agg join subgraphs --- .../src/adapter/PreAggregations.ts | 106 ++++++++++++------ 1 file changed, 71 insertions(+), 35 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index c259412a43945..0285fc9be75af 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -639,6 +639,16 @@ export class PreAggregations { transformedQuery.allBackAliasMembers[r] || r )); + const buildPath = (cube: string): string[] => { + const path = [cube]; + const parentMap = transformedQuery.joinsMap; + while (parentMap[cube]) { + cube = parentMap[cube]; + path.push(cube); + } + return path.reverse(); + }; + /** * Determine whether pre-aggregation can be used or not. */ @@ -647,8 +657,32 @@ export class PreAggregations { const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; - const backAliasMeasures = backAlias(references.measures); - const backAliasDimensions = backAlias(references.dimensions); + + let dimsToMatch: string[]; + let measToMatch: string[]; + + if (references.rollups.length > 0) { + // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are + // no connections in the joinTree between cubes from different datasources + // but joinGraph of the query has all the connections, necessary for serving the query, + // so we use this information to complete the full paths of members from the root of the query + // up to the pre-agg cube. + // We use references from the underlying pre-aggregations, filtered with members existing in the root + // pre-aggregation itself. + + dimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameDimensions) + .filter(d => references.dimensions.some(rd => d.endsWith(rd))); + measToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameMeasures) + .filter(m => references.measures.some(rm => m.endsWith(rm))); + } else { + dimsToMatch = references.fullNameDimensions; + measToMatch = references.fullNameMeasures; + } + + const backAliasMeasures = backAlias(measToMatch); + const backAliasDimensions = backAlias(dimsToMatch); return (( transformedQuery.hasNoTimeDimensionsWithoutGranularity ) && ( @@ -741,7 +775,6 @@ export class PreAggregations { let dimsToMatch: string[]; let timeDimsToMatch: PreAggregationTimeDimensionReference[]; - let dimensionsMatch: (dimensions: string[], doBackAlias: boolean) => boolean; if (references.rollups.length > 0) { // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are @@ -754,44 +787,28 @@ export class PreAggregations { dimsToMatch = references.rollupsReferences .flatMap(rolRef => rolRef.fullNameDimensions) - .filter(d => references.dimensions.some(rd => d.endsWith(rd))); - timeDimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameTimeDimensions) - .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); - - const buildPath = (cube: string): string[] => { - const path = [cube]; - const parentMap = transformedQuery.joinsMap; - while (parentMap[cube]) { - cube = parentMap[cube]; - path.push(cube); - } - return path.reverse(); - }; - - dimensionsMatch = (dimensions, doBackAlias) => { - let target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; - target = target.map(dim => { - const [cube, ...restPath] = dim.split('.'); + .filter(d => references.dimensions.some(rd => d.endsWith(rd))) + .map(d => { + const [cube, ...restPath] = d.split('.'); if (cube === transformedQuery.joinGraphRoot) { - return dim; + return d; } const path = buildPath(cube); return `${path.join('.')}.${restPath.join('.')}`; }); - - return dimensions.every(d => target.includes(d)); - }; + timeDimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameTimeDimensions) + .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); } else { dimsToMatch = references.fullNameDimensions; timeDimsToMatch = references.fullNameTimeDimensions; - - dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; - return dimensions.every(d => target.includes(d)); - }; } + const dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; + const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass( timeDimensionsList.map( tds => R.anyPass(tds.map((td: [string, string]) => { @@ -1005,10 +1022,28 @@ export class PreAggregations { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { + // It's important to build join graph not only using the pre-agg members, but also + // taking into account all explicit underlying rollup pre-aggregation joins, because + // otherwise the built join tree might differ from the actual pre-aggregation. + const preAggJoinsJoinHints = preAggObj.references.rollupsReferences.map(r => { + if (!r.joinTree) { + return []; + } + + const hints: (string | string[])[] = [r.joinTree.root]; + + for (const j of r.joinTree.joins) { + hints.push([j.from, j.to]); + } + + return hints; + }).flat(); const targetJoins = this.resolveJoinMembers( - // TODO join hints? - this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj)) + this.query.joinGraph.buildJoin( + preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) + ) ); + // const targetJoins = this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj))); const existingJoins = R.unnest(preAggObjsToJoin.map( // TODO join hints? p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) @@ -1114,11 +1149,12 @@ export class PreAggregations { preAggregationsToJoin.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); }); + const canUsePreAggregationResult = canUsePreAggregation(references); return { ...preAggObj, - canUsePreAggregation: canUsePreAggregation(references), + canUsePreAggregation: canUsePreAggregationResult, preAggregationsToJoin, - rollupJoin: this.buildRollupJoin(preAggObj, preAggregationsToJoin) + rollupJoin: canUsePreAggregationResult ? this.buildRollupJoin(preAggObj, preAggregationsToJoin) : null, }; } else if (preAggregation.type === 'rollupLambda') { // TODO evaluation optimizations. Should be cached or moved to compile time. From f7db87bdc3aab26649b676cc903d7e55275f7587 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 21:25:55 +0300 Subject: [PATCH 11/38] fix incorrect cache for pre-aggs --- .../src/adapter/PreAggregations.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 0285fc9be75af..b24094da576b0 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1414,7 +1414,15 @@ export class PreAggregations { if (!preAggregationName) { return evaluateReferences(); } - return this.query.cacheValue(['evaluateAllReferences', cube, preAggregationName], evaluateReferences); + + // Using [cube, preAggregationName] alone as cache keys isn’t reliable, + // as different queries can build distinct join graphs during pre-aggregation matching. + // Because the matching logic compares join subgraphs — particularly for 'rollupJoin' and 'rollupLambda' + // pre-aggregations — relying on such keys may cause incorrect results. + return this.query.cacheValue( + ['evaluateAllReferences', cube, preAggregationName, JSON.stringify(this.query.join)], + evaluateReferences + ); } public originalSqlPreAggregationTable(preAggregationDescription: PreAggregationForCube): string { From 66db2988855a2df9a631a1723dbfd7815000f5c0 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 23:03:36 +0300 Subject: [PATCH 12/38] fix canUsePreAggregationNotAdditive --- .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index b24094da576b0..35332274af29f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -653,13 +653,13 @@ export class PreAggregations { * Determine whether pre-aggregation can be used or not. */ const canUsePreAggregationNotAdditive: CanUsePreAggregationFn = (references: PreAggregationReferences): boolean => { - const refTimeDimensions = backAlias(sortTimeDimensions(references.timeDimensions)); const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; let dimsToMatch: string[]; let measToMatch: string[]; + let timeDimsToMatch: PreAggregationTimeDimensionReference[]; if (references.rollups.length > 0) { // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are @@ -673,14 +673,19 @@ export class PreAggregations { dimsToMatch = references.rollupsReferences .flatMap(rolRef => rolRef.fullNameDimensions) .filter(d => references.dimensions.some(rd => d.endsWith(rd))); + timeDimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameTimeDimensions) + .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); measToMatch = references.rollupsReferences .flatMap(rolRef => rolRef.fullNameMeasures) .filter(m => references.measures.some(rm => m.endsWith(rm))); } else { dimsToMatch = references.fullNameDimensions; + timeDimsToMatch = references.fullNameTimeDimensions; measToMatch = references.fullNameMeasures; } + const refTimeDimensions = backAlias(sortTimeDimensions(timeDimsToMatch)); const backAliasMeasures = backAlias(measToMatch); const backAliasDimensions = backAlias(dimsToMatch); return (( @@ -1043,7 +1048,6 @@ export class PreAggregations { preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) ) ); - // const targetJoins = this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj))); const existingJoins = R.unnest(preAggObjsToJoin.map( // TODO join hints? p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) From 9c15e1a3e717e402982cc73ea44b8d5af6083d4e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 21 Oct 2025 19:39:05 +0300 Subject: [PATCH 13/38] skip test for tesseract --- .../postgres/pre-aggregations.test.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 812fdd163f726..9cd294756659d 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -3208,4 +3208,72 @@ describe('PreAggregations', () => { ); }); }); + + it('rollupJoin pre-aggregation with nested joins via view (A->B->C)', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['view_abc.dim_a', 'view_abc.dim_b', 'view_abc.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(2); + const aaa = preAggregationsDescription.find(p => p.preAggregationId === 'cube_a.aaa_rollup'); + const bbb = preAggregationsDescription.find(p => p.preAggregationId === 'cube_b.bbb_rollup'); + expect(aaa).toBeDefined(); + expect(bbb).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinAB'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + view_abc__dim_a: 'dim_a', + view_abc__dim_b: 'dim_b', + view_abc__dim_c: 'dim_c', + }] + ); + }); + }); + + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollupJoin pre-aggregation with nested joins via cube (A->B->C)', () => { + // Need to investigate tesseract internals of how pre-aggs members are resolved and how + // rollups are used to construct rollupJoins. + }); + } else { + it('rollupJoin pre-aggregation with nested joins via cube (A->B->C)', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_a.dim_a', 'cube_b.dim_b', 'cube_c.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(0); + + expect(query.preAggregations?.preAggregationForQuery).toBeUndefined(); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_a__dim_a: 'dim_a', + cube_b__dim_b: 'dim_b', + cube_c__dim_c: 'dim_c', + }] + ); + }); + }); + } }); From c9991e73e361e150f83f889a68fbc86b9ab4ba61 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 12:55:30 +0300 Subject: [PATCH 14/38] export type --- packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 39b9acc3296e7..505b10f952bf9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -7,7 +7,7 @@ import type { CubeEvaluator, MeasureDefinition } from './CubeEvaluator'; import type { CubeDefinition, JoinDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; -type JoinEdge = { +export type JoinEdge = { join: JoinDefinition, from: string, to: string, From 1ff6d7d1d68816e7a1cc32f460ac3ec3db81d01d Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 12:57:13 +0300 Subject: [PATCH 15/38] more types --- .../src/adapter/PreAggregations.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 35332274af29f..a826eb4866ed2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1,6 +1,7 @@ import R from 'ramda'; import { CubeSymbols, PreAggregationDefinition } from '../compiler/CubeSymbols'; +import { FinishedJoinTree, JoinEdge } from '../compiler/JoinGraph'; import { UserError } from '../compiler/UserError'; import { BaseQuery } from './BaseQuery'; import { @@ -15,8 +16,6 @@ import { BaseGroupFilter } from './BaseGroupFilter'; import { BaseDimension } from './BaseDimension'; import { BaseSegment } from './BaseSegment'; -export type RollupJoin = any; - export type PartitionTimeDimension = { dimension: string; dateRange: [string, string]; @@ -45,6 +44,7 @@ export type PreAggregationForQuery = { references: PreAggregationReferences; preAggregationsToJoin?: PreAggregationForQuery[]; referencedPreAggregations?: PreAggregationForQuery[]; + // eslint-disable-next-line no-use-before-define rollupJoin?: RollupJoin; sqlAlias?: string; }; @@ -66,6 +66,18 @@ export type EvaluateReferencesContext = { export type BaseMember = BaseDimension | BaseMeasure | BaseFilter | BaseGroupFilter | BaseSegment; +export type JoinEdgeWithMembers = JoinEdge & { + fromMembers: string[]; + toMembers: string[]; +}; + +export type RollupJoinItem = JoinEdgeWithMembers & { + fromPreAggObj: PreAggregationForQuery; + toPreAggObj: PreAggregationForQuery; +}; + +export type RollupJoin = RollupJoinItem[]; + export type CanUsePreAggregationFn = (references: PreAggregationReferences) => boolean; /** @@ -1050,7 +1062,7 @@ export class PreAggregations { ); const existingJoins = R.unnest(preAggObjsToJoin.map( // TODO join hints? - p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) + p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!) )); const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( existing => existing.originalFrom === target.originalFrom && @@ -1088,7 +1100,7 @@ export class PreAggregations { return fromPreAggObj[0]; } - private resolveJoinMembers(join) { + private resolveJoinMembers(join: FinishedJoinTree): JoinEdgeWithMembers[] { return join.joins.map(j => { const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); @@ -1512,7 +1524,7 @@ export class PreAggregations { }); if (preAggregationForQuery.preAggregation.type === 'rollupJoin') { - const join = preAggregationForQuery.rollupJoin; + const join = preAggregationForQuery.rollupJoin!; toJoin = [ sqlAndAlias(join[0].fromPreAggObj), From 0582006759eb74b06690262c479b78c6ec07b8d4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 13:38:10 +0300 Subject: [PATCH 16/38] build fullNames for rollupJoin/Lambda in the evaluatedPreAggregationObj() --- .../src/adapter/PreAggregations.ts | 182 ++++++++---------- 1 file changed, 83 insertions(+), 99 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index a826eb4866ed2..3467b4e9db5a2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -92,6 +92,11 @@ export type FullPreAggregationDescription = any; */ export type TransformedQuery = any; +type BuildRollupJoinResult = { + rollupJoin: RollupJoin; + existingJoins: JoinEdgeWithMembers[]; +}; + export class PreAggregations { private readonly query: BaseQuery; @@ -521,18 +526,7 @@ export class PreAggregations { filterDimensionsSingleValueEqual = allValuesEq1(filterDimensionsSingleValueEqual) ? new Set(filterDimensionsSingleValueEqual?.keys()) : null; - // Build reverse query joins map, which is used for - // rollupLambda and rollupJoin pre-aggs matching later - const joinsMap: Record = {}; - if (query.join) { - for (const j of query.join.joins) { - joinsMap[j.to] = j.from; - } - } - return { - joinGraphRoot: query.join?.root, - joinsMap, sortedDimensions, sortedTimeDimensions, timeDimensions, @@ -651,16 +645,6 @@ export class PreAggregations { transformedQuery.allBackAliasMembers[r] || r )); - const buildPath = (cube: string): string[] => { - const path = [cube]; - const parentMap = transformedQuery.joinsMap; - while (parentMap[cube]) { - cube = parentMap[cube]; - path.push(cube); - } - return path.reverse(); - }; - /** * Determine whether pre-aggregation can be used or not. */ @@ -669,37 +653,9 @@ export class PreAggregations { ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; - let dimsToMatch: string[]; - let measToMatch: string[]; - let timeDimsToMatch: PreAggregationTimeDimensionReference[]; - - if (references.rollups.length > 0) { - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - // but joinGraph of the query has all the connections, necessary for serving the query, - // so we use this information to complete the full paths of members from the root of the query - // up to the pre-agg cube. - // We use references from the underlying pre-aggregations, filtered with members existing in the root - // pre-aggregation itself. - - dimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameDimensions) - .filter(d => references.dimensions.some(rd => d.endsWith(rd))); - timeDimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameTimeDimensions) - .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); - measToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameMeasures) - .filter(m => references.measures.some(rm => m.endsWith(rm))); - } else { - dimsToMatch = references.fullNameDimensions; - timeDimsToMatch = references.fullNameTimeDimensions; - measToMatch = references.fullNameMeasures; - } - - const refTimeDimensions = backAlias(sortTimeDimensions(timeDimsToMatch)); - const backAliasMeasures = backAlias(measToMatch); - const backAliasDimensions = backAlias(dimsToMatch); + const refTimeDimensions = backAlias(sortTimeDimensions(references.fullNameTimeDimensions)); + const backAliasMeasures = backAlias(references.fullNameMeasures); + const backAliasDimensions = backAlias(references.fullNameDimensions); return (( transformedQuery.hasNoTimeDimensionsWithoutGranularity ) && ( @@ -716,9 +672,9 @@ export class PreAggregations { transformedQuery.allFiltersWithinSelectedDimensions && R.equals(backAliasDimensions, transformedQuery.sortedDimensions) ) && ( - R.all(m => backAliasMeasures.indexOf(m) !== -1, transformedQuery.measures) || + R.all(m => backAliasMeasures.includes(m), transformedQuery.measures) || // TODO do we need backAlias here? - R.all(m => backAliasMeasures.indexOf(m) !== -1, transformedQuery.leafMeasures) + R.all(m => backAliasMeasures.includes(m), transformedQuery.leafMeasures) )); }; @@ -790,36 +746,8 @@ export class PreAggregations { } } - let dimsToMatch: string[]; - let timeDimsToMatch: PreAggregationTimeDimensionReference[]; - - if (references.rollups.length > 0) { - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - // but joinGraph of the query has all the connections, necessary for serving the query, - // so we use this information to complete the full paths of members from the root of the query - // up to the pre-agg cube. - // We use references from the underlying pre-aggregations, filtered with members existing in the root - // pre-aggregation itself. - - dimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameDimensions) - .filter(d => references.dimensions.some(rd => d.endsWith(rd))) - .map(d => { - const [cube, ...restPath] = d.split('.'); - if (cube === transformedQuery.joinGraphRoot) { - return d; - } - const path = buildPath(cube); - return `${path.join('.')}.${restPath.join('.')}`; - }); - timeDimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameTimeDimensions) - .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); - } else { - dimsToMatch = references.fullNameDimensions; - timeDimsToMatch = references.fullNameTimeDimensions; - } + const dimsToMatch = references.fullNameDimensions; + const timeDimsToMatch = references.fullNameTimeDimensions; const dimensionsMatch = (dimensions, doBackAlias) => { const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; @@ -1035,7 +963,7 @@ export class PreAggregations { } // TODO check multiplication factor didn't change - private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): RollupJoin { + private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): BuildRollupJoinResult { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { @@ -1055,15 +983,22 @@ export class PreAggregations { return hints; }).flat(); - const targetJoins = this.resolveJoinMembers( - this.query.joinGraph.buildJoin( - preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) - ) + + const builtJoinTree = this.query.joinGraph.buildJoin( + preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) ); - const existingJoins = R.unnest(preAggObjsToJoin.map( - // TODO join hints? - p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!) - )); + + if (!builtJoinTree) { + throw new UserError(`Can't build join tree for pre-aggregation ${preAggObj.cube}.${preAggObj.preAggregationName}`); + } + + const targetJoins = this.resolveJoinMembers(builtJoinTree); + + // TODO join hints? + const existingJoins = preAggObjsToJoin + .map(p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!)) + .flat(); + const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( existing => existing.originalFrom === target.originalFrom && existing.originalTo === target.originalTo && @@ -1073,7 +1008,7 @@ export class PreAggregations { if (!nonExistingJoins.length) { throw new UserError(`Nothing to join in rollup join. Target joins ${JSON.stringify(targetJoins)} are included in existing rollup joins ${JSON.stringify(existingJoins)}`); } - return nonExistingJoins.map(join => { + const rollupJoin = nonExistingJoins.map(join => { const fromPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.fromMembers, join); const toPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.toMembers, join); return { @@ -1082,6 +1017,11 @@ export class PreAggregations { toPreAggObj }; }); + + return { + rollupJoin, + existingJoins, + }; } ); } @@ -1142,8 +1082,8 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - // For rollupJoin and rollupLambda we need to pass references of the underlying rollups - // to canUsePreAggregation fn, which are collected later; + // For rollupJoin and rollupLambda we need to enrich references with data + // from the underlying rollups which are collected later; canUsePreAggregation: preAggregation.type === 'rollup' ? canUsePreAggregation(references) : false, references, preAggregationId: `${cube}.${preAggregationName}` @@ -1165,12 +1105,53 @@ export class PreAggregations { preAggregationsToJoin.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); }); - const canUsePreAggregationResult = canUsePreAggregation(references); + const { rollupJoin, existingJoins } = this.buildRollupJoin(preAggObj, preAggregationsToJoin); + + const joinsMap: Record = {}; + for (const j of rollupJoin) { + joinsMap[j.to] = j.from; + } + for (const j of existingJoins) { + joinsMap[j.to] = j.from; + } + + const buildPath = (cubeName: string): string[] => { + const path = [cubeName]; + const parentMap = joinsMap; + while (parentMap[cubeName]) { + cubeName = parentMap[cubeName]; + path.push(cubeName); + } + return path.reverse(); + }; + + references.fullNameDimensions = references.dimensions.map(d => { + const [cubeName, ...restPath] = d.split('.'); + const path = buildPath(cubeName); + + return `${path.join('.')}.${restPath.join('.')}`; + }); + references.fullNameMeasures = references.measures.map(m => { + const [cubeName, ...restPath] = m.split('.'); + const path = buildPath(cubeName); + + return `${path.join('.')}.${restPath.join('.')}`; + }); + references.fullNameTimeDimensions = references.timeDimensions.map(td => { + const [cubeName, ...restPath] = td.dimension.split('.'); + const path = buildPath(cubeName); + + return { + ...td, + dimension: `${path.join('.')}.${restPath.join('.')}`, + }; + }); + return { ...preAggObj, - canUsePreAggregation: canUsePreAggregationResult, + canUsePreAggregation: canUsePreAggregation(references), preAggregationsToJoin, - rollupJoin: canUsePreAggregationResult ? this.buildRollupJoin(preAggObj, preAggregationsToJoin) : null, + rollupJoin, }; } else if (preAggregation.type === 'rollupLambda') { // TODO evaluation optimizations. Should be cached or moved to compile time. @@ -1217,6 +1198,9 @@ export class PreAggregations { }); referencedPreAggregations.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); + references.fullNameDimensions.push(...preAgg.references.fullNameDimensions); + references.fullNameMeasures.push(...preAgg.references.fullNameMeasures); + references.fullNameTimeDimensions.push(...preAgg.references.fullNameTimeDimensions); }); return { ...preAggObj, From 84bde20baf1c99ee25d0ad70f1229b06e64a9c80 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 16:20:14 +0300 Subject: [PATCH 17/38] add memberShortNameFromPath() to Evaluator --- .../src/compiler/CubeEvaluator.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index f89ddad1da0db..6acda3f3e256b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -731,6 +731,18 @@ export class CubeEvaluator extends CubeSymbols { return !!this.evaluatedCubes[cube]; } + public memberShortNameFromPath(path: string | string[]): string { + if (!Array.isArray(path)) { + path = path.split('.'); + } + + if (path.length < 2) { + throw new UserError(`Not full member name provided: ${path[0]}`); + } + + return `${path.at(-2)}.${path.at(-1)}`; + } + public cubeFromPath(path: string): EvaluatedCube { return this.evaluatedCubes[this.cubeNameFromPath(path)]; } From c90ec16183b7fb23446b3612e05fede294323849 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 16:20:37 +0300 Subject: [PATCH 18/38] fix type --- packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 3467b4e9db5a2..57ff7e5d4dbfc 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1654,8 +1654,7 @@ export class PreAggregations { private rollupMembers(preAggregationForQuery: PreAggregationForQuery, type: T): PreAggregationReferences[T] { return preAggregationForQuery.preAggregation.type === 'autoRollup' ? - // TODO proper types - (preAggregationForQuery.preAggregation as any)[type] : + preAggregationForQuery.preAggregation[type] : this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation, preAggregationForQuery.preAggregationName)[type]; } From 8233f28b560d2f7f61fe0dbeb09c80028cc9f442 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 17:02:30 +0300 Subject: [PATCH 19/38] get rid of references.fullName* in favor of fullpath-members --- .../src/adapter/PreAggregations.ts | 36 ++++++++----------- .../src/compiler/CubeEvaluator.ts | 6 ---- .../test/unit/pre-agg-by-filter-match.test.ts | 6 ---- .../test/unit/pre-agg-time-dim-match.test.ts | 6 ---- .../test/unit/RefreshScheduler.test.ts | 3 -- 5 files changed, 15 insertions(+), 42 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 57ff7e5d4dbfc..8a8bc44913bcd 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -653,9 +653,9 @@ export class PreAggregations { ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; - const refTimeDimensions = backAlias(sortTimeDimensions(references.fullNameTimeDimensions)); - const backAliasMeasures = backAlias(references.fullNameMeasures); - const backAliasDimensions = backAlias(references.fullNameDimensions); + const refTimeDimensions = backAlias(sortTimeDimensions(references.timeDimensions)); + const backAliasMeasures = backAlias(references.measures); + const backAliasDimensions = backAlias(references.dimensions); return (( transformedQuery.hasNoTimeDimensionsWithoutGranularity ) && ( @@ -746,11 +746,8 @@ export class PreAggregations { } } - const dimsToMatch = references.fullNameDimensions; - const timeDimsToMatch = references.fullNameTimeDimensions; - const dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + const target = doBackAlias ? backAlias(references.dimensions) : references.dimensions; return dimensions.every(d => target.includes(d)); }; @@ -766,8 +763,8 @@ export class PreAggregations { ) )( doBackAlias ? - backAlias(sortTimeDimensions(timeDimsToMatch)) : - (sortTimeDimensions(timeDimsToMatch)) + backAlias(sortTimeDimensions(references.timeDimensions)) : + (sortTimeDimensions(references.timeDimensions)) ); if (transformedQuery.ungrouped) { @@ -1065,8 +1062,8 @@ export class PreAggregations { private cubesFromPreAggregation(preAggObj: PreAggregationForQuery): string[] { return R.uniq( - preAggObj.references.measures.map(m => this.query.cubeEvaluator.parsePath('measures', m)).concat( - preAggObj.references.dimensions.map(m => this.query.cubeEvaluator.parsePathAnyType(m)) + preAggObj.references.measures.map(m => this.query.cubeEvaluator.parsePath('measures', this.query.cubeEvaluator.memberShortNameFromPath(m))).concat( + preAggObj.references.dimensions.map(m => this.query.cubeEvaluator.parsePathAnyType(this.query.cubeEvaluator.memberShortNameFromPath(m))) ).map(p => p[0]) ); } @@ -1125,19 +1122,19 @@ export class PreAggregations { return path.reverse(); }; - references.fullNameDimensions = references.dimensions.map(d => { + references.dimensions = references.dimensions.map(d => { const [cubeName, ...restPath] = d.split('.'); const path = buildPath(cubeName); return `${path.join('.')}.${restPath.join('.')}`; }); - references.fullNameMeasures = references.measures.map(m => { + references.measures = references.measures.map(m => { const [cubeName, ...restPath] = m.split('.'); const path = buildPath(cubeName); return `${path.join('.')}.${restPath.join('.')}`; }); - references.fullNameTimeDimensions = references.timeDimensions.map(td => { + references.timeDimensions = references.timeDimensions.map(td => { const [cubeName, ...restPath] = td.dimension.split('.'); const path = buildPath(cubeName); @@ -1198,9 +1195,6 @@ export class PreAggregations { }); referencedPreAggregations.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); - references.fullNameDimensions.push(...preAgg.references.fullNameDimensions); - references.fullNameMeasures.push(...preAgg.references.fullNameMeasures); - references.fullNameTimeDimensions.push(...preAgg.references.fullNameTimeDimensions); }); return { ...preAggObj, @@ -1235,7 +1229,7 @@ export class PreAggregations { if (typeof member !== 'string') { return `${member.dimension.split('.')[1]}.${member.granularity}`; } else { - return member.split('.')[1]; + return member.split('.').at(-1)!; } }); } @@ -1386,9 +1380,9 @@ export class PreAggregations { // So we store full named members separately and use them in canUsePreAggregation functions. references.joinTree = preAggQuery.join; const root = references.joinTree?.root || ''; - references.fullNameMeasures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); - references.fullNameDimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); - references.fullNameTimeDimensions = references.timeDimensions.map(d => ({ + references.measures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); + references.dimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); + references.timeDimensions = references.timeDimensions.map(d => ({ dimension: (d.dimension.startsWith(root) ? d.dimension : `${root}.${d.dimension}`), granularity: d.granularity, })); diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 6acda3f3e256b..396640004034b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -89,11 +89,8 @@ export type PreAggregationTimeDimensionReference = { export type PreAggregationReferences = { allowNonStrictDateRangeMatch?: boolean, dimensions: Array, - fullNameDimensions: Array, measures: Array, - fullNameMeasures: Array, timeDimensions: Array, - fullNameTimeDimensions: Array, rollups: Array, rollupsReferences: Array, multipliedMeasures?: Array, @@ -901,9 +898,6 @@ export class CubeEvaluator extends CubeSymbols { timeDimensions, rollups: aggregation.rollupReferences && this.evaluateReferences(cube, aggregation.rollupReferences, { originalSorting: true }) || [], - fullNameDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() - fullNameMeasures: [], // May be filled in PreAggregations.evaluateAllReferences() - fullNameTimeDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() rollupsReferences: [], // May be filled in PreAggregations.evaluateAllReferences() }; } diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts index 7253ca3c35720..1b2d9f72af826 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts @@ -59,12 +59,6 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: testPreAgg.segments ? testPreAgg.dimensions.concat(testPreAgg.segments) : testPreAgg.dimensions, - fullNameMeasures: testPreAgg.measures, - fullNameTimeDimensions: [{ - dimension: testPreAgg.timeDimension, - granularity: testPreAgg.granularity, - }], rollupsReferences: [], }; diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index 56d294701ba6c..784f066ecd2ec 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -69,12 +69,6 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: testPreAgg.dimensions, - fullNameMeasures: testPreAgg.measures, - fullNameTimeDimensions: [{ - dimension: testPreAgg.timeDimension, - granularity: testPreAgg.granularity, - }], rollupsReferences: [], }; diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 370dd90d82742..22309182253cc 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -666,9 +666,6 @@ describe('Refresh Scheduler', () => { timeDimensions: [{ dimension: 'Foo.time', granularity: 'hour' }], rollups: [], rollupsReferences: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], }, refreshKey: { every: '1 hour', updateWindow: '1 day', incremental: true }, }, From 623e4fb74660218b65dc549a8339e3665720fb09 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 19:25:57 +0300 Subject: [PATCH 20/38] some refactoring to avoid copy/paste --- .../src/adapter/PreAggregations.ts | 128 +++++++++--------- .../src/compiler/CubeEvaluator.ts | 2 +- 2 files changed, 68 insertions(+), 62 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 8a8bc44913bcd..019223af7449f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -172,8 +172,7 @@ export class PreAggregations { let preAggregations: PreAggregationForQuery[] = [foundPreAggregation]; if (foundPreAggregation.preAggregation.type === 'rollupJoin') { preAggregations = foundPreAggregation.preAggregationsToJoin || []; - } - if (foundPreAggregation.preAggregation.type === 'rollupLambda') { + } else if (foundPreAggregation.preAggregation.type === 'rollupLambda') { preAggregations = foundPreAggregation.referencedPreAggregations || []; } @@ -959,6 +958,20 @@ export class PreAggregations { } } + private collectJoinHintsFromRollupReferences(refs: PreAggregationReferences): (string | string[])[] { + if (!refs.joinTree) { + return []; + } + + const hints: (string | string[])[] = [refs.joinTree.root]; + + for (const j of refs.joinTree.joins) { + hints.push([j.from, j.to]); + } + + return hints; + } + // TODO check multiplication factor didn't change private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): BuildRollupJoinResult { return this.query.cacheValue( @@ -967,19 +980,9 @@ export class PreAggregations { // It's important to build join graph not only using the pre-agg members, but also // taking into account all explicit underlying rollup pre-aggregation joins, because // otherwise the built join tree might differ from the actual pre-aggregation. - const preAggJoinsJoinHints = preAggObj.references.rollupsReferences.map(r => { - if (!r.joinTree) { - return []; - } - - const hints: (string | string[])[] = [r.joinTree.root]; - - for (const j of r.joinTree.joins) { - hints.push([j.from, j.to]); - } - - return hints; - }).flat(); + const preAggJoinsJoinHints = preAggObj.references.rollupsReferences.map( + this.collectJoinHintsFromRollupReferences + ).flat(); const builtJoinTree = this.query.joinGraph.buildJoin( preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) @@ -1112,37 +1115,9 @@ export class PreAggregations { joinsMap[j.to] = j.from; } - const buildPath = (cubeName: string): string[] => { - const path = [cubeName]; - const parentMap = joinsMap; - while (parentMap[cubeName]) { - cubeName = parentMap[cubeName]; - path.push(cubeName); - } - return path.reverse(); - }; - - references.dimensions = references.dimensions.map(d => { - const [cubeName, ...restPath] = d.split('.'); - const path = buildPath(cubeName); - - return `${path.join('.')}.${restPath.join('.')}`; - }); - references.measures = references.measures.map(m => { - const [cubeName, ...restPath] = m.split('.'); - const path = buildPath(cubeName); - - return `${path.join('.')}.${restPath.join('.')}`; - }); - references.timeDimensions = references.timeDimensions.map(td => { - const [cubeName, ...restPath] = td.dimension.split('.'); - const path = buildPath(cubeName); - - return { - ...td, - dimension: `${path.join('.')}.${restPath.join('.')}`, - }; - }); + references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); + references.measures = this.buildMembersFullName(references.measures, joinsMap); + references.timeDimensions = this.buildTimeDimensionsFullName(references.timeDimensions, joinsMap); return { ...preAggObj, @@ -1315,11 +1290,10 @@ export class PreAggregations { cube, aggregation ) && - !!references.dimensions.find((d) => { + references.dimensions.some((d) => this.query.cubeEvaluator.dimensionByPath( // `d` can contain full join path, so we should trim it - const trimmedDimension = CubeSymbols.joinHintFromPath(d).path; - return this.query.cubeEvaluator.dimensionByPath(trimmedDimension).primaryKey; - }), + this.query.cubeEvaluator.memberShortNameFromPath(d) + ).primaryKey), }); } @@ -1364,6 +1338,38 @@ export class PreAggregations { .toLowerCase(); } + private enrichMembersCubeJoinPath(cubeName: string, joinsMap: Record): string[] { + const path = [cubeName]; + const parentMap = joinsMap; + while (parentMap[cubeName]) { + cubeName = parentMap[cubeName]; + path.push(cubeName); + } + + return path.reverse(); + } + + private buildMembersFullName(members: string[], joinsMap: Record): string[] { + return members.map(d => { + const [cubeName, ...restPath] = d.split('.'); + const path = this.enrichMembersCubeJoinPath(cubeName, joinsMap); + + return `${path.join('.')}.${restPath.join('.')}`; + }); + } + + private buildTimeDimensionsFullName(members: PreAggregationTimeDimensionReference[], joinsMap: Record): PreAggregationTimeDimensionReference[] { + return members.map(td => { + const [cubeName, ...restPath] = td.dimension.split('.'); + const path = this.enrichMembersCubeJoinPath(cubeName, joinsMap); + + return { + ...td, + dimension: `${path.join('.')}.${restPath.join('.')}`, + }; + }); + } + private evaluateAllReferences(cube: string, aggregation: PreAggregationDefinition, preAggregationName: string | null = null, context: EvaluateReferencesContext = {}): PreAggregationReferences { const evaluateReferences = () => { const references = this.query.cubeEvaluator.evaluatePreAggregationReferences(cube, aggregation); @@ -1374,18 +1380,18 @@ export class PreAggregations { if (preAggQuery) { // We need to build a join tree for all references, so they would always include full join path // even for preaggregation references without join path. It is necessary to be able to match - // query and preaggregation based on full join tree. But we can not update - // references.{dimensions,measures,timeDimensions} directly, because it will break - // evaluation of references in the query on later stages. - // So we store full named members separately and use them in canUsePreAggregation functions. + // query and preaggregation based on full join tree. references.joinTree = preAggQuery.join; - const root = references.joinTree?.root || ''; - references.measures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); - references.dimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); - references.timeDimensions = references.timeDimensions.map(d => ({ - dimension: (d.dimension.startsWith(root) ? d.dimension : `${root}.${d.dimension}`), - granularity: d.granularity, - })); + const joinsMap: Record = {}; + if (references.joinTree) { + for (const j of references.joinTree.joins) { + joinsMap[j.to] = j.from; + } + } + + references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); + references.measures = this.buildMembersFullName(references.measures, joinsMap); + references.timeDimensions = this.buildTimeDimensionsFullName(references.timeDimensions, joinsMap); } } if (aggregation.type === 'rollupLambda') { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 396640004034b..b7f85b84bf682 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -737,7 +737,7 @@ export class CubeEvaluator extends CubeSymbols { throw new UserError(`Not full member name provided: ${path[0]}`); } - return `${path.at(-2)}.${path.at(-1)}`; + return path.slice(-2).join('.'); } public cubeFromPath(path: string): EvaluatedCube { From b1e4d236429ccd7cbb53f547c2bdbf80ed7be726 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 17:09:34 +0300 Subject: [PATCH 21/38] add 'rollupJoin pre-aggregation matching with transitive joins' test --- .../postgres/pre-aggregations.test.ts | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 9cd294756659d..ceb594faba42a 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -967,6 +967,213 @@ describe('PreAggregations', () => { ] }); + // Models with transitive joins for rollupJoin matching + cube('merchant_dims', { + sql: \` + SELECT 101 AS merchant_sk, 'M1' AS merchant_id + UNION ALL + SELECT 102 AS merchant_sk, 'M2' AS merchant_id + \`, + + dimensions: { + merchant_sk: { + sql: 'merchant_sk', + type: 'number', + primary_key: true + }, + merchant_id: { + sql: 'merchant_id', + type: 'string' + } + } + }); + + cube('product_dims', { + sql: \` + SELECT 201 AS product_sk, 'P1' AS product_id + UNION ALL + SELECT 202 AS product_sk, 'P2' AS product_id + \`, + + dimensions: { + product_sk: { + sql: 'product_sk', + type: 'number', + primary_key: true + }, + product_id: { + sql: 'product_id', + type: 'string' + } + } + }); + + cube('merchant_and_product_dims', { + sql: \` + SELECT 'M1' AS merchant_id, 'P1' AS product_id, 'Organic' AS acquisition_channel, 'SOLD' AS status + UNION ALL + SELECT 'M1' AS merchant_id, 'P2' AS product_id, 'Paid' AS acquisition_channel, 'PAID' AS status + UNION ALL + SELECT 'M2' AS merchant_id, 'P1' AS product_id, 'Referral' AS acquisition_channel, 'RETURNED' AS status + \`, + + dimensions: { + product_id: { + sql: 'product_id', + type: 'string', + primary_key: true + }, + merchant_id: { + sql: 'merchant_id', + type: 'string', + primary_key: true + }, + status: { + sql: 'status', + type: 'string' + }, + acquisition_channel: { + sql: 'acquisition_channel', + type: 'string' + } + }, + + pre_aggregations: { + bridge_rollup: { + dimensions: [ + merchant_id, + product_id, + acquisition_channel, + status + ] + } + } + }); + + cube('other_facts', { + sql: \` + SELECT 1 AS id, 1 AS fact_id, 'OF1' AS fact + UNION ALL + SELECT 2 AS id, 2 AS fact_id, 'OF2' AS fact + UNION ALL + SELECT 3 AS id, 3 AS fact_id, 'OF3' AS fact + \`, + + dimensions: { + other_fact_id: { + sql: 'id', + type: 'number', + primary_key: true + }, + fact_id: { + sql: 'fact_id', + type: 'number' + }, + fact: { + sql: 'fact', + type: 'string' + } + }, + + pre_aggregations: { + bridge_rollup: { + dimensions: [ + fact_id, + fact + ] + } + } + + }); + + cube('test_facts', { + sql: \` + SELECT 1 AS id, 101 AS merchant_sk, 201 AS product_sk, 100 AS amount + UNION ALL + SELECT 2 AS id, 101 AS merchant_sk, 202 AS product_sk, 150 AS amount + UNION ALL + SELECT 3 AS id, 102 AS merchant_sk, 201 AS product_sk, 200 AS amount + \`, + + joins: { + merchant_dims: { + relationship: 'many_to_one', + sql: \`\${CUBE.merchant_sk} = \${merchant_dims.merchant_sk}\` + }, + product_dims: { + relationship: 'many_to_one', + sql: \`\${CUBE.product_sk} = \${product_dims.product_sk}\` + }, + // Transitive join - depends on merchant_dims and product_dims + merchant_and_product_dims: { + relationship: 'many_to_one', + sql: \`\${merchant_dims.merchant_id} = \${merchant_and_product_dims.merchant_id} AND \${product_dims.product_id} = \${merchant_and_product_dims.product_id}\` + }, + other_facts: { + relationship: 'one_to_many', + sql: \`\${CUBE.id} = \${other_facts.fact_id}\` + }, + }, + + dimensions: { + id: { + sql: 'id', + type: 'number', + primary_key: true + }, + merchant_sk: { + sql: 'merchant_sk', + type: 'number' + }, + product_sk: { + sql: 'product_sk', + type: 'number' + }, + acquisition_channel: { + sql: \`\${merchant_and_product_dims.acquisition_channel}\`, + type: 'string' + } + }, + + measures: { + amount_sum: { + sql: 'amount', + type: 'sum' + } + }, + + pre_aggregations: { + facts_rollup: { + dimensions: [ + id, + merchant_sk, + merchant_dims.merchant_sk, + merchant_dims.merchant_id, + merchant_and_product_dims.merchant_id, + product_sk, + product_dims.product_sk, + product_dims.product_id, + merchant_and_product_dims.product_id, + acquisition_channel, + merchant_and_product_dims.status + ] + }, + rollupJoinTransitive: { + type: 'rollupJoin', + dimensions: [ + merchant_sk, + product_sk, + merchant_and_product_dims.status, + other_facts.fact + ], + rollups: [ + facts_rollup, + other_facts.bridge_rollup + ] + } + } + }); + `); it('simple pre-aggregation', async () => { @@ -3276,4 +3483,58 @@ describe('PreAggregations', () => { }); }); } + + it('rollupJoin pre-aggregation matching with transitive joins', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: [ + 'test_facts.merchant_sk', + 'test_facts.product_sk', + 'merchant_and_product_dims.status', + 'other_facts.fact' + ], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(JSON.stringify(preAggregationsDescription, null, 2)); + + // Verify that both rollups are included in the description + expect(preAggregationsDescription.length).toBe(2); + const factsRollup = preAggregationsDescription.find(p => p.preAggregationId === 'test_facts.facts_rollup'); + const bridgeRollup = preAggregationsDescription.find(p => p.preAggregationId === 'other_facts.bridge_rollup'); + expect(factsRollup).toBeDefined(); + expect(bridgeRollup).toBeDefined(); + + // Verify that the rollupJoin pre-aggregation can be used for the query + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinTransitive'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual([ + { + merchant_and_product_dims__status: 'SOLD', + other_facts__fact: 'OF1', + test_facts__merchant_sk: 101, + test_facts__product_sk: 201, + }, + { + merchant_and_product_dims__status: 'PAID', + other_facts__fact: 'OF2', + test_facts__merchant_sk: 101, + test_facts__product_sk: 202, + }, + { + merchant_and_product_dims__status: 'RETURNED', + other_facts__fact: 'OF3', + test_facts__merchant_sk: 102, + test_facts__product_sk: 201, + }, + ]); + }); + }); }); From 26cb0c910f1df5c64d502dd0fdd0b890457e44d6 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 17:10:59 +0300 Subject: [PATCH 22/38] fix buildRollupJoin --- .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 019223af7449f..983f8e06523ec 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -996,7 +996,11 @@ export class PreAggregations { // TODO join hints? const existingJoins = preAggObjsToJoin - .map(p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!)) + .map(p => this.resolveJoinMembers( + this.query.joinGraph.buildJoin( + this.collectJoinHintsFromRollupReferences(p.references).concat(this.cubesFromPreAggregation(p)) + )! + )) .flat(); const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( From 4f4a00755fda6ddb0d11e48de95d59d644ec6c94 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 17:13:13 +0300 Subject: [PATCH 23/38] fix resolveJoinMembers() --- .../src/adapter/PreAggregations.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 983f8e06523ec..715792c803728 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1045,13 +1045,18 @@ export class PreAggregations { } private resolveJoinMembers(join: FinishedJoinTree): JoinEdgeWithMembers[] { + const joinMap = new Set(); + return join.joins.map(j => { + joinMap.add(j.originalFrom); + const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); - const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); + + const invalidMembers = memberPaths.filter(m => !joinMap.has(m[0]) && m[0] !== j.originalTo); if (invalidMembers.length) { throw new UserError(`Members ${invalidMembers.join(', ')} in join from '${j.originalFrom}' to '${j.originalTo}' doesn't reference join cubes`); } - const fromMembers = memberPaths.filter(m => m[0] === j.originalFrom).map(m => m.join('.')); + const fromMembers = memberPaths.filter(m => joinMap.has(m[0])).map(m => m.join('.')); if (!fromMembers.length) { throw new UserError(`From members are not found in [${memberPaths.map(m => m.join('.')).join(', ')}] for join ${JSON.stringify(j)}. Please make sure join fields are referencing dimensions instead of columns.`); } @@ -1059,6 +1064,8 @@ export class PreAggregations { if (!toMembers.length) { throw new UserError(`To members are not found in [${memberPaths.map(m => m.join('.')).join(', ')}] for join ${JSON.stringify(j)}. Please make sure join fields are referencing dimensions instead of columns.`); } + joinMap.add(j.originalTo); + return { ...j, fromMembers, From 8e4307ea613ecae6162e7dad612bb8eb5ad9948c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 18:40:26 +0300 Subject: [PATCH 24/38] implement sortMembersByJoinTree() --- .../src/adapter/PreAggregations.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 715792c803728..d9741756d76b0 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -669,7 +669,10 @@ export class PreAggregations { references.dimensions.length === filterDimensionsSingleValueEqual.size && R.all(d => filterDimensionsSingleValueEqual.has(d), backAliasDimensions) || transformedQuery.allFiltersWithinSelectedDimensions && - R.equals(backAliasDimensions, transformedQuery.sortedDimensions) + // references.dimensions might be reordered because of joinTree join order, + // so we need to compare without order here. + backAliasDimensions.length === transformedQuery.sortedDimensions.length && + R.equals(new Set(backAliasDimensions), new Set(transformedQuery.sortedDimensions)) ) && ( R.all(m => backAliasMeasures.includes(m), transformedQuery.measures) || // TODO do we need backAlias here? @@ -1398,6 +1401,32 @@ export class PreAggregations { for (const j of references.joinTree.joins) { joinsMap[j.to] = j.from; } + + // As full-path references may be passed to query options, + // it is important to sort them based on join tree order, + // because full-path names work as explicit join hints, + // and JoinGraph will take them as granted in the order of + // occurrence. But that might be incorrect for transitive-join cases. + const sortMembersByJoinTree = (members: string[]) => { + const joinOrder: Record = {}; + joinOrder[references.joinTree!.root] = 0; + for (const join of references.joinTree!.joins) { + const index = references.joinTree!.joins.indexOf(join); + joinOrder[join.to] = index + 1; + } + + members.sort((a, b) => { + const cubeA = a.split('.')[0]; + const cubeB = b.split('.')[0]; + const orderA = joinOrder[cubeA] ?? Infinity; + const orderB = joinOrder[cubeB] ?? Infinity; + + return orderA - orderB; + }); + }; + + sortMembersByJoinTree(references.dimensions); + sortMembersByJoinTree(references.measures); } references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); From 09ac100d58e83104631269dfe9b5373283cbe631 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 15:18:46 +0200 Subject: [PATCH 25/38] fix joint hints collection for transitive joins --- .../src/adapter/BaseQuery.js | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 53b3295d798c3..d1092cdd5a221 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -505,6 +505,34 @@ export class BaseQuery { return joinMaps; } + /** + * @private + * @param { import('../compiler/JoinGraph').FinishedJoinTree } joinTree + * @param { string[] } joinHints + * @return { string[][] } + */ + enrichedJoinHintsFromJoinTree(joinTree, joinHints) { + const joinsMap = {}; + + for (const j of joinTree.joins) { + joinsMap[j.to] = j.from; + } + + return joinHints.map(jh => { + let cubeName = jh; + const path = [cubeName]; + while (joinsMap[cubeName]) { + cubeName = joinsMap[cubeName]; + path.push(cubeName); + } + + if (path.length === 1) { + return path[0]; + } + return path.reverse(); + }); + } + /** * @private * @param { (string|string[])[] } hints @@ -2669,7 +2697,7 @@ export class BaseQuery { const explicitJoinHintMembers = new Set(allMembersJoinHints.filter(j => Array.isArray(j)).flat()); const queryJoinMaps = this.queryJoinMap(); const customSubQueryJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery()); - const newCollectedHints = []; + let newCollectedHints = []; // One cube may join the other cube via transitive joined cubes, // members from which are referenced in the join `on` clauses. @@ -2703,8 +2731,12 @@ export class BaseQuery { const iterationCollectedHints = joinMembersJoinHints.filter(j => !allJoinHintsFlatten.has(j)); newJoinHintsCollectedCnt = iterationCollectedHints.length; cnt++; - if (newJoin) { - newCollectedHints.push(...joinMembersJoinHints.filter(j => !explicitJoinHintMembers.has(j))); + if (newJoin && newJoin.joins.length > 0) { + // Even if there is no join tree changes, we still + // push correctly ordered join hints, collected from the resolving of members of join tree + // upfront the all existing query members. This ensures the correct cube join order + // with transitive joins even if they are already presented among query members. + newCollectedHints = this.enrichedJoinHintsFromJoinTree(newJoin, joinMembersJoinHints); } } while (newJoin?.joins.length > 0 && !this.isJoinTreesEqual(prevJoin, newJoin) && cnt < 10000 && newJoinHintsCollectedCnt > 0); From d2332b328984311571de21a283fa484cc83a97ce Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 16:45:53 +0200 Subject: [PATCH 26/38] remove obsolete --- .../src/adapter/PreAggregations.ts | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index d9741756d76b0..c2004a9c15f0f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1401,32 +1401,6 @@ export class PreAggregations { for (const j of references.joinTree.joins) { joinsMap[j.to] = j.from; } - - // As full-path references may be passed to query options, - // it is important to sort them based on join tree order, - // because full-path names work as explicit join hints, - // and JoinGraph will take them as granted in the order of - // occurrence. But that might be incorrect for transitive-join cases. - const sortMembersByJoinTree = (members: string[]) => { - const joinOrder: Record = {}; - joinOrder[references.joinTree!.root] = 0; - for (const join of references.joinTree!.joins) { - const index = references.joinTree!.joins.indexOf(join); - joinOrder[join.to] = index + 1; - } - - members.sort((a, b) => { - const cubeA = a.split('.')[0]; - const cubeB = b.split('.')[0]; - const orderA = joinOrder[cubeA] ?? Infinity; - const orderB = joinOrder[cubeB] ?? Infinity; - - return orderA - orderB; - }); - }; - - sortMembersByJoinTree(references.dimensions); - sortMembersByJoinTree(references.measures); } references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); From f4649fe29f11c70ff4eefe1e7ccd446c03c5eac7 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 20:53:35 +0200 Subject: [PATCH 27/38] revert back obsolete --- .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index c2004a9c15f0f..715792c803728 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -669,10 +669,7 @@ export class PreAggregations { references.dimensions.length === filterDimensionsSingleValueEqual.size && R.all(d => filterDimensionsSingleValueEqual.has(d), backAliasDimensions) || transformedQuery.allFiltersWithinSelectedDimensions && - // references.dimensions might be reordered because of joinTree join order, - // so we need to compare without order here. - backAliasDimensions.length === transformedQuery.sortedDimensions.length && - R.equals(new Set(backAliasDimensions), new Set(transformedQuery.sortedDimensions)) + R.equals(backAliasDimensions, transformedQuery.sortedDimensions) ) && ( R.all(m => backAliasMeasures.includes(m), transformedQuery.measures) || // TODO do we need backAlias here? From 23726b659eb4f7421d3e7a5c78ac3cc65ce7443e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 21:14:28 +0200 Subject: [PATCH 28/38] fix joint hints collection for transitive joins --- .../src/adapter/BaseQuery.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index d1092cdd5a221..6f4a12afc84d4 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -387,16 +387,14 @@ export class BaseQuery { } /** - * Is used by native * This function follows the same logic as in this.collectJoinHints() - * @private + * @public * @param {Array<(Array | string)>} hints * @return {import('../compiler/JoinGraph').FinishedJoinTree} */ joinTreeForHints(hints) { - const explicitJoinHintMembers = new Set(hints.filter(j => Array.isArray(j)).flat()); const queryJoinMaps = this.queryJoinMap(); - const newCollectedHints = []; + let newCollectedHints = []; const constructJH = () => R.uniq(this.enrichHintsWithJoinMap([ ...newCollectedHints, @@ -421,8 +419,12 @@ export class BaseQuery { const iterationCollectedHints = joinMembersJoinHints.filter(j => !allJoinHintsFlatten.has(j)); newJoinHintsCollectedCnt = iterationCollectedHints.length; cnt++; - if (newJoin) { - newCollectedHints.push(...joinMembersJoinHints.filter(j => !explicitJoinHintMembers.has(j))); + if (newJoin && newJoin.joins.length > 0) { + // Even if there is no join tree changes, we still + // push correctly ordered join hints, collected from the resolving of members of join tree + // upfront the all existing query members. This ensures the correct cube join order + // with transitive joins even if they are already presented among query members. + newCollectedHints = this.enrichedJoinHintsFromJoinTree(newJoin, joinMembersJoinHints); } } while (newJoin?.joins.length > 0 && !this.isJoinTreesEqual(prevJoin, newJoin) && cnt < 10000 && newJoinHintsCollectedCnt > 0); @@ -2694,7 +2696,6 @@ export class BaseQuery { */ collectJoinHints(excludeTimeDimensions = false) { const allMembersJoinHints = this.collectJoinHintsFromMembers(this.allMembersConcat(excludeTimeDimensions)); - const explicitJoinHintMembers = new Set(allMembersJoinHints.filter(j => Array.isArray(j)).flat()); const queryJoinMaps = this.queryJoinMap(); const customSubQueryJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery()); let newCollectedHints = []; From 85d238cad85a58bf241200ccf5f5230dd82ec421 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 22:00:00 +0200 Subject: [PATCH 29/38] fix joint hints collection for transitive joins --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 6f4a12afc84d4..d46953cc7d9a6 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -432,7 +432,7 @@ export class BaseQuery { throw new UserError('Can not construct joins for the query, potential loop detected'); } - return newJoin; + return this.joinGraph.buildJoin(constructJH()); } cacheValue(key, fn, { contextPropNames, inputProps, cache } = {}) { From 0db02fcc1def3e2348f7ac1d1bb5cbd1cf46a108 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 23:06:24 +0200 Subject: [PATCH 30/38] update joinTreeForHints() with skipQueryJoinMap flag --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index d46953cc7d9a6..3a5834d455977 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -388,12 +388,14 @@ export class BaseQuery { /** * This function follows the same logic as in this.collectJoinHints() + * skipQueryJoinMap is used by PreAggregations to build join tree without user's query all members map * @public * @param {Array<(Array | string)>} hints + * @param { boolean } skipQueryJoinMap * @return {import('../compiler/JoinGraph').FinishedJoinTree} */ - joinTreeForHints(hints) { - const queryJoinMaps = this.queryJoinMap(); + joinTreeForHints(hints, skipQueryJoinMap = false) { + const queryJoinMaps = skipQueryJoinMap ? {} : this.queryJoinMap(); let newCollectedHints = []; const constructJH = () => R.uniq(this.enrichHintsWithJoinMap([ From 3d7c1e1f7471e49472b0f500349dd11c72fa8a36 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 23:07:00 +0200 Subject: [PATCH 31/38] fix buildRollupJoin() --- .../src/adapter/PreAggregations.ts | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 715792c803728..270b32e835934 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -977,16 +977,9 @@ export class PreAggregations { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { - // It's important to build join graph not only using the pre-agg members, but also - // taking into account all explicit underlying rollup pre-aggregation joins, because - // otherwise the built join tree might differ from the actual pre-aggregation. - const preAggJoinsJoinHints = preAggObj.references.rollupsReferences.map( - this.collectJoinHintsFromRollupReferences - ).flat(); - - const builtJoinTree = this.query.joinGraph.buildJoin( - preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) - ); + // It's not enough to call buildJoin() directly on cubesFromPreAggregation() + // because transitive joins won't be collected in that case. + const builtJoinTree = this.query.joinTreeForHints(this.cubesHintsFromPreAggregation(preAggObj), true); if (!builtJoinTree) { throw new UserError(`Can't build join tree for pre-aggregation ${preAggObj.cube}.${preAggObj.preAggregationName}`); @@ -997,9 +990,7 @@ export class PreAggregations { // TODO join hints? const existingJoins = preAggObjsToJoin .map(p => this.resolveJoinMembers( - this.query.joinGraph.buildJoin( - this.collectJoinHintsFromRollupReferences(p.references).concat(this.cubesFromPreAggregation(p)) - )! + this.query.joinTreeForHints(this.cubesHintsFromPreAggregation(p), true) )) .flat(); @@ -1074,11 +1065,11 @@ export class PreAggregations { }); } - private cubesFromPreAggregation(preAggObj: PreAggregationForQuery): string[] { + private cubesHintsFromPreAggregation(preAggObj: PreAggregationForQuery): string[][] { return R.uniq( - preAggObj.references.measures.map(m => this.query.cubeEvaluator.parsePath('measures', this.query.cubeEvaluator.memberShortNameFromPath(m))).concat( - preAggObj.references.dimensions.map(m => this.query.cubeEvaluator.parsePathAnyType(this.query.cubeEvaluator.memberShortNameFromPath(m))) - ).map(p => p[0]) + preAggObj.references.measures.concat( + preAggObj.references.dimensions + ).map(p => p.split('.').slice(0, -1)) ); } From c2ea506432e5f9759d69660d6c2fc5aed1ae8db2 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 23:07:59 +0200 Subject: [PATCH 32/38] fix datamodel for rollupJoin tests --- .../test/integration/postgres/pre-aggregations.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index ceb594faba42a..f7bbe38ca8a04 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -874,7 +874,7 @@ describe('PreAggregations', () => { dimensions: [ dim_a, cube_b.dim_b, - cube_c.dim_c + cube_b.cube_c.dim_c ], rollups: [ aaa_rollup, From cc6a5aef85c81000d8efd80a1f556c149e7196cd Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 23:28:35 +0200 Subject: [PATCH 33/38] simplify buildRollupJoin() and evaluatedPreAggregationObj() --- .../src/adapter/PreAggregations.ts | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 270b32e835934..65c8175018706 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -92,11 +92,6 @@ export type FullPreAggregationDescription = any; */ export type TransformedQuery = any; -type BuildRollupJoinResult = { - rollupJoin: RollupJoin; - existingJoins: JoinEdgeWithMembers[]; -}; - export class PreAggregations { private readonly query: BaseQuery; @@ -973,7 +968,7 @@ export class PreAggregations { } // TODO check multiplication factor didn't change - private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): BuildRollupJoinResult { + private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): RollupJoin { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { @@ -1013,10 +1008,7 @@ export class PreAggregations { }; }); - return { - rollupJoin, - existingJoins, - }; + return rollupJoin; } ); } @@ -1107,19 +1099,7 @@ export class PreAggregations { preAggregationsToJoin.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); }); - const { rollupJoin, existingJoins } = this.buildRollupJoin(preAggObj, preAggregationsToJoin); - - const joinsMap: Record = {}; - for (const j of rollupJoin) { - joinsMap[j.to] = j.from; - } - for (const j of existingJoins) { - joinsMap[j.to] = j.from; - } - - references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); - references.measures = this.buildMembersFullName(references.measures, joinsMap); - references.timeDimensions = this.buildTimeDimensionsFullName(references.timeDimensions, joinsMap); + const rollupJoin = this.buildRollupJoin(preAggObj, preAggregationsToJoin); return { ...preAggObj, From 1f555b9cd2336da49eaa24318a9373c756fa3ca4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 23:28:39 +0200 Subject: [PATCH 34/38] fix tests --- .../integration/postgres/pre-aggregations.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index f7bbe38ca8a04..d913158ed1b9e 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -675,8 +675,8 @@ describe('PreAggregations', () => { type: 'rollupJoin', dimensions: [ dim_1, - cube_2.dim_1, - cube_2.dim_2 // XXX + CUBE.cube_2.dim_1, + CUBE.cube_2.dim_2 // XXX ], rollups: [ aaa, @@ -873,8 +873,8 @@ describe('PreAggregations', () => { type: 'rollupJoin', dimensions: [ dim_a, - cube_b.dim_b, - cube_b.cube_c.dim_c + CUBE.cube_b.dim_b, + CUBE.cube_b.cube_c.dim_c ], rollups: [ aaa_rollup, @@ -1163,8 +1163,8 @@ describe('PreAggregations', () => { dimensions: [ merchant_sk, product_sk, - merchant_and_product_dims.status, - other_facts.fact + CUBE.merchant_and_product_dims.status, + CUBE.other_facts.fact ], rollups: [ facts_rollup, From c11c2a609bdd2e7dd18bb6f5c9a341e6cedeb634 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 27 Oct 2025 23:35:44 +0200 Subject: [PATCH 35/38] remove obsolete --- .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 65c8175018706..cb5496dc857cd 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -998,7 +998,7 @@ export class PreAggregations { if (!nonExistingJoins.length) { throw new UserError(`Nothing to join in rollup join. Target joins ${JSON.stringify(targetJoins)} are included in existing rollup joins ${JSON.stringify(existingJoins)}`); } - const rollupJoin = nonExistingJoins.map(join => { + return nonExistingJoins.map(join => { const fromPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.fromMembers, join); const toPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.toMembers, join); return { @@ -1007,8 +1007,6 @@ export class PreAggregations { toPreAggObj }; }); - - return rollupJoin; } ); } From 6e0a1f3836da9ca331b42b4fb306df09b3b15c89 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Oct 2025 12:25:29 +0200 Subject: [PATCH 36/38] add test for pre-agg with not-full paths --- .../postgres/pre-aggregations.test.ts | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index d913158ed1b9e..e41433d96417a 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -967,6 +967,60 @@ describe('PreAggregations', () => { ] }); + // Cube with not full paths in rollupJoin pre-aggregation + cube('cube_a_to_fail_pre_agg', { + sql: \`SELECT 1 as id, 'dim_a' as dim_a\`, + + joins: { + cube_b: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_b.dim_a}\` + }, + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_c.dim_a}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + aaa_rollup: { + dimensions: [ + dim_a + ] + }, + rollupJoinAB: { + type: 'rollupJoin', + dimensions: [ + dim_a, + cube_b.dim_b, + cube_c.dim_c + ], + rollups: [ + aaa_rollup, + cube_b.bbb_rollup + ] + } + } + }); + // Models with transitive joins for rollupJoin matching cube('merchant_dims', { sql: \` @@ -3537,4 +3591,23 @@ describe('PreAggregations', () => { ]); }); }); + + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollupJoin pre-aggregation with not-full paths should fail', () => { + // Need to investigate tesseract internals of how pre-aggs members are resolved and how + // rollups are used to construct rollupJoins. + }); + } else { + it('rollupJoin pre-aggregation with not-full paths should fail', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_a_to_fail_pre_agg.dim_a', 'cube_b.dim_b', 'cube_c.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + expect(() => query.buildSqlAndParams()).toThrow('No rollups found that can be used for rollup join'); + }); + } }); From ca14e468839252880bd4a98c2d79971689803582 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Oct 2025 12:48:08 +0200 Subject: [PATCH 37/38] more types --- packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index cb5496dc857cd..9eb4137c954e7 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1011,7 +1011,7 @@ export class PreAggregations { ); } - private preAggObjForJoin(preAggObjsToJoin: PreAggregationForQuery[], joinMembers, join): PreAggregationForQuery { + private preAggObjForJoin(preAggObjsToJoin: PreAggregationForQuery[], joinMembers: string[], join: JoinEdgeWithMembers): PreAggregationForQuery { const fromPreAggObj = preAggObjsToJoin .filter(p => joinMembers.every(m => !!p.references.dimensions.find(d => m === d))); if (!fromPreAggObj.length) { From 6a7d6d31553b494e859347bf038f934d4da04b8c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 28 Oct 2025 13:05:16 +0200 Subject: [PATCH 38/38] improve error message in preAggObjForJoin() --- .../src/adapter/PreAggregations.ts | 17 +++++++++++++---- .../postgres/pre-aggregations.test.ts | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 9eb4137c954e7..d7fd3d3479d3f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -999,8 +999,8 @@ export class PreAggregations { throw new UserError(`Nothing to join in rollup join. Target joins ${JSON.stringify(targetJoins)} are included in existing rollup joins ${JSON.stringify(existingJoins)}`); } return nonExistingJoins.map(join => { - const fromPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.fromMembers, join); - const toPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.toMembers, join); + const fromPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.fromMembers, join, `${preAggObj.cube}.${preAggObj.preAggregationName}`); + const toPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.toMembers, join, `${preAggObj.cube}.${preAggObj.preAggregationName}`); return { ...join, fromPreAggObj, @@ -1011,11 +1011,20 @@ export class PreAggregations { ); } - private preAggObjForJoin(preAggObjsToJoin: PreAggregationForQuery[], joinMembers: string[], join: JoinEdgeWithMembers): PreAggregationForQuery { + private preAggObjForJoin( + preAggObjsToJoin: PreAggregationForQuery[], + joinMembers: string[], + join: JoinEdgeWithMembers, + rollupJoinPreAggName: string, + ): PreAggregationForQuery { const fromPreAggObj = preAggObjsToJoin .filter(p => joinMembers.every(m => !!p.references.dimensions.find(d => m === d))); if (!fromPreAggObj.length) { - throw new UserError(`No rollups found that can be used for rollup join: ${JSON.stringify(join)}`); + const msg = `No rollups found that can be used for a rollup join from "${ + join.from}" (fromMembers: ${JSON.stringify(join.fromMembers)}) to "${join.to}" (toMembers: ${ + JSON.stringify(join.toMembers)}). Check the "${ + rollupJoinPreAggName}" pre-aggregation definition — you may have forgotten to specify the full dimension paths`; + throw new UserError(msg); } if (fromPreAggObj.length > 1) { throw new UserError( diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index e41433d96417a..eab0d3a137e5d 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -3607,7 +3607,7 @@ describe('PreAggregations', () => { preAggregationsSchema: '' }); - expect(() => query.buildSqlAndParams()).toThrow('No rollups found that can be used for rollup join'); + expect(() => query.buildSqlAndParams()).toThrow('No rollups found that can be used for a rollup join'); }); } });