Skip to content

fix(schema-compiler): Fix incorrect pre-aggregation matching for 'rollupJoin' and 'rollupLambda' pre-aggs #9957

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 10, 2025
Merged

fix(schema-compiler): Fix incorrect pre-aggregation matching for 'rollupJoin' and 'rollupLambda' pre-aggs #9957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
697a11e
fix test for rollup join
KSDaemon Sep 10, 2025
86ba037
fix(schema-compiler): Fix incorrect pre-aggregation matching for 'rol…
KSDaemon Sep 10, 2025
078a8e3
fix tests
KSDaemon Sep 10, 2025
2d8e76f
re-enable tests
KSDaemon Sep 10, 2025
85fb61f
add more checks in tests
KSDaemon Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,6 @@ export class PreAggregations {
}),
R.unnest,
R.uniq,
R.map(resolveFullMemberPath),
)(measures);

function allValuesEq1(map) {
Expand Down Expand Up @@ -515,6 +514,7 @@ export class PreAggregations {
measures: measurePaths,
leafMeasureAdditive,
leafMeasures: leafMeasurePaths,
leafMeasuresFullPaths: leafMeasurePaths.map(resolveFullMemberPath),
measureToLeafMeasures,
hasNoTimeDimensionsWithoutGranularity,
allFiltersWithinSelectedDimensions,
Expand Down Expand Up @@ -728,7 +728,7 @@ export class PreAggregations {

// In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are
// no connections in the joinTree between cubes from different datasources
const dimsToMatch = references.fullNameDimensions.length > 0 ? references.fullNameDimensions : references.dimensions;
const dimsToMatch = references.rollups.length > 0 ? references.dimensions : references.fullNameDimensions;

const dimensionsMatch = (dimensions, doBackAlias) => R.all(
d => (
Expand All @@ -741,7 +741,7 @@ export class PreAggregations {

// In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are
// no connections in the joinTree between cubes from different datasources
const timeDimsToMatch = references.fullNameTimeDimensions.length > 0 ? references.fullNameTimeDimensions : references.timeDimensions;
const timeDimsToMatch = references.rollups.length > 0 ? references.timeDimensions : references.fullNameTimeDimensions;

const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass(
timeDimensionsList.map(
Expand Down Expand Up @@ -781,7 +781,7 @@ export class PreAggregations {
) && (
R.all(
(m: string) => references.measures.indexOf(m) !== -1,
transformedQuery.leafMeasures,
references.rollups.length > 0 ? transformedQuery.leafMeasures : transformedQuery.leafMeasuresFullPaths,
) || R.all(
m => backAliasMeasures.indexOf(m) !== -1,
transformedQuery.measures,
Expand Down
221 changes: 106 additions & 115 deletions packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2387,147 +2387,138 @@ describe('PreAggregations', () => {
});
}

if (getEnv('nativeSqlPlanner')) {
it.skip('rollup join: should be fixed in Tesseract', () => {
// This should be fixed in Tesseract.
it('rollup join', async () => {
await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitor_checkins.count',
],
dimensions: ['visitors.source'],
preAggregationsSchema: '',
order: [{
id: 'visitors.source',
}],
timezone: 'UTC',
});
} else {
it('rollup join', async () => {
await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitor_checkins.count',
],
dimensions: ['visitors.source'],
preAggregationsSchema: '',
order: [{
id: 'visitors.source',
}],
timezone: 'UTC',
});
const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
console.log(preAggregationsDescription);

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
console.log(preAggregationsDescription);
expect(queryAndParams[0]).toContain('visitors_for_join');
expect(queryAndParams[0]).toContain('vc_for_join');

console.log(query.preAggregations?.rollupMatchResultDescriptions());
console.log(query.preAggregations?.rollupMatchResultDescriptions());

const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);

console.log(JSON.stringify(queries.concat(queryAndParams)));
console.log(JSON.stringify(queries.concat(queryAndParams)));

return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
console.log(JSON.stringify(res));
expect(res).toEqual(
[
{ visitors__source: 'google', vc__count: '1' },
{ visitors__source: 'some', vc__count: '5' },
{ visitors__source: null, vc__count: '0' },
],
);
});
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
console.log(JSON.stringify(res));
expect(res).toEqual(
[
{ visitors__source: 'google', vc__count: '1' },
{ visitors__source: 'some', vc__count: '5' },
{ visitors__source: null, vc__count: null },
],
);
});
}
});

if (getEnv('nativeSqlPlanner')) {
it.skip('rollup join existing joins: should be fixed in Tesseract', () => {
// This should be fixed in Tesseract.
it('rollup join existing joins', async () => {
await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitor_checkins.count',
],
dimensions: ['visitors.source', 'cards.visitorId'],
preAggregationsSchema: '',
order: [{
id: 'visitors.source',
}, {
id: 'cards.visitorId',
}],
timezone: 'UTC',
});
} else {
it('rollup join existing joins', async () => {
await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitor_checkins.count',
],
dimensions: ['visitors.source', 'cards.visitorId'],
preAggregationsSchema: '',
order: [{
id: 'visitors.source',
}, {
id: 'cards.visitorId',
}],
timezone: 'UTC',
});
const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
console.log(preAggregationsDescription);

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
console.log(preAggregationsDescription);
expect(queryAndParams[0]).toContain('visitors_for_join_inc_cards');
expect(queryAndParams[0]).toContain('vc_for_join');

console.log(query.preAggregations?.rollupMatchResultDescriptions());
console.log(query.preAggregations?.rollupMatchResultDescriptions());

const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);

console.log(JSON.stringify(queries.concat(queryAndParams)));
console.log(JSON.stringify(queries.concat(queryAndParams)));

return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
console.log(JSON.stringify(res));
expect(res).toEqual(
[
{ visitors__source: 'google', cards__visitor_id: 3, vc__count: '1' },
{ visitors__source: 'some', cards__visitor_id: 1, vc__count: '3' },
{ visitors__source: 'some', cards__visitor_id: null, vc__count: '2' },
{ visitors__source: null, cards__visitor_id: null, vc__count: '0' },
],
);
});
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
console.log(JSON.stringify(res));
expect(res).toEqual(
[
{ visitors__source: 'google', cards__visitor_id: 3, vc__count: '1' },
{ visitors__source: 'some', cards__visitor_id: 1, vc__count: '3' },
{ visitors__source: 'some', cards__visitor_id: null, vc__count: '2' },
{ visitors__source: null, cards__visitor_id: null, vc__count: null },
],
);
});
}
});

if (getEnv('nativeSqlPlanner')) {
it.skip('rollup join partitioned: should be fixed in Tesseract', () => {
// This should be fixed in Tesseract.
it('rollup join partitioned', async () => {
await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitor_checkins.count',
],
dimensions: ['visitors.source'],
timezone: 'America/Los_Angeles',
preAggregationsSchema: '',
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'hour',
dateRange: ['2017-01-03', '2017-01-04']
}],
order: [{
id: 'visitors.createdAt'
}],
});
} else {
it('rollup join partitioned', async () => {
await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitor_checkins.count',
],
dimensions: ['visitors.source'],
timezone: 'America/Los_Angeles',
preAggregationsSchema: '',
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'hour',
dateRange: ['2017-01-03', '2017-01-04']
}],
order: [{
id: 'visitors.createdAt'
}],
});
const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
console.log(preAggregationsDescription);

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
console.log(preAggregationsDescription);
expect(queryAndParams[0]).toContain('visitors_partitioned_hourly_for_join');
expect(queryAndParams[0]).toContain('vc_for_join');

console.log(query.preAggregations?.rollupMatchResultDescriptions());
console.log(query.preAggregations?.rollupMatchResultDescriptions());

const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);

console.log(JSON.stringify(queries.concat(queryAndParams)));
console.log(JSON.stringify(queries.concat(queryAndParams)));

return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
console.log(JSON.stringify(res));
expect(res).toEqual(
[
{
visitors__source: 'some',
visitors__created_at_hour: '2017-01-04T16:00:00.000Z',
vc__count: '2'
}
],
);
});
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
console.log(JSON.stringify(res));
expect(res).toEqual(
[
{
visitors__source: 'some',
visitors__created_at_hour: '2017-01-04T16:00:00.000Z',
vc__count: '2'
}
],
);
});
}
});

it('partitioned without time', async () => {
await compiler.compile();
Expand Down
Loading