Skip to content

Commit 502d39e

Browse files
KSDaemonigorlukanin
KSDaemon
authored and
igorlukanin
committed
fix(schema-compiler): Fix incorrect pre-aggregation matching for 'rollupJoin' and 'rollupLambda' pre-aggs (#9957)
* fix test for rollup join * fix(schema-compiler): Fix incorrect pre-aggregation matching for 'rollupJoin' and 'rollupLambda' pre-aggs * fix tests * re-enable tests * add more checks in tests
1 parent 4ba2d9d commit 502d39e

File tree

2 files changed

+110
-119
lines changed

2 files changed

+110
-119
lines changed

‎packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -459,7 +459,6 @@ export class PreAggregations {
459459
}),
460460
R.unnest,
461461
R.uniq,
462-
R.map(resolveFullMemberPath),
463462
)(measures);
464463

465464
function allValuesEq1(map) {
@@ -515,6 +514,7 @@ export class PreAggregations {
515514
measures: measurePaths,
516515
leafMeasureAdditive,
517516
leafMeasures: leafMeasurePaths,
517+
leafMeasuresFullPaths: leafMeasurePaths.map(resolveFullMemberPath),
518518
measureToLeafMeasures,
519519
hasNoTimeDimensionsWithoutGranularity,
520520
allFiltersWithinSelectedDimensions,
@@ -728,7 +728,7 @@ export class PreAggregations {
728728

729729
// In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are
730730
// no connections in the joinTree between cubes from different datasources
731-
const dimsToMatch = references.fullNameDimensions.length > 0 ? references.fullNameDimensions : references.dimensions;
731+
const dimsToMatch = references.rollups.length > 0 ? references.dimensions : references.fullNameDimensions;
732732

733733
const dimensionsMatch = (dimensions, doBackAlias) => R.all(
734734
d => (
@@ -741,7 +741,7 @@ export class PreAggregations {
741741

742742
// In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are
743743
// no connections in the joinTree between cubes from different datasources
744-
const timeDimsToMatch = references.fullNameTimeDimensions.length > 0 ? references.fullNameTimeDimensions : references.timeDimensions;
744+
const timeDimsToMatch = references.rollups.length > 0 ? references.timeDimensions : references.fullNameTimeDimensions;
745745

746746
const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass(
747747
timeDimensionsList.map(
@@ -781,7 +781,7 @@ export class PreAggregations {
781781
) && (
782782
R.all(
783783
(m: string) => references.measures.indexOf(m) !== -1,
784-
transformedQuery.leafMeasures,
784+
references.rollups.length > 0 ? transformedQuery.leafMeasures : transformedQuery.leafMeasuresFullPaths,
785785
) || R.all(
786786
m => backAliasMeasures.indexOf(m) !== -1,
787787
transformedQuery.measures,

‎packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts‎

Lines changed: 106 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -2387,147 +2387,138 @@ describe('PreAggregations', () => {
23872387
});
23882388
}
23892389

2390-
if (getEnv('nativeSqlPlanner')) {
2391-
it.skip('rollup join: should be fixed in Tesseract', () => {
2392-
// This should be fixed in Tesseract.
2390+
it('rollup join', async () => {
2391+
await compiler.compile();
2392+
2393+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
2394+
measures: [
2395+
'visitor_checkins.count',
2396+
],
2397+
dimensions: ['visitors.source'],
2398+
preAggregationsSchema: '',
2399+
order: [{
2400+
id: 'visitors.source',
2401+
}],
2402+
timezone: 'UTC',
23932403
});
2394-
} else {
2395-
it('rollup join', async () => {
2396-
await compiler.compile();
23972404

2398-
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
2399-
measures: [
2400-
'visitor_checkins.count',
2401-
],
2402-
dimensions: ['visitors.source'],
2403-
preAggregationsSchema: '',
2404-
order: [{
2405-
id: 'visitors.source',
2406-
}],
2407-
timezone: 'UTC',
2408-
});
2405+
const queryAndParams = query.buildSqlAndParams();
2406+
console.log(queryAndParams);
2407+
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
2408+
console.log(preAggregationsDescription);
24092409

2410-
const queryAndParams = query.buildSqlAndParams();
2411-
console.log(queryAndParams);
2412-
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
2413-
console.log(preAggregationsDescription);
2410+
expect(queryAndParams[0]).toContain('visitors_for_join');
2411+
expect(queryAndParams[0]).toContain('vc_for_join');
24142412

2415-
console.log(query.preAggregations?.rollupMatchResultDescriptions());
2413+
console.log(query.preAggregations?.rollupMatchResultDescriptions());
24162414

2417-
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
2415+
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
24182416

2419-
console.log(JSON.stringify(queries.concat(queryAndParams)));
2417+
console.log(JSON.stringify(queries.concat(queryAndParams)));
24202418

2421-
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
2422-
console.log(JSON.stringify(res));
2423-
expect(res).toEqual(
2424-
[
2425-
{ visitors__source: 'google', vc__count: '1' },
2426-
{ visitors__source: 'some', vc__count: '5' },
2427-
{ visitors__source: null, vc__count: '0' },
2428-
],
2429-
);
2430-
});
2419+
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
2420+
console.log(JSON.stringify(res));
2421+
expect(res).toEqual(
2422+
[
2423+
{ visitors__source: 'google', vc__count: '1' },
2424+
{ visitors__source: 'some', vc__count: '5' },
2425+
{ visitors__source: null, vc__count: null },
2426+
],
2427+
);
24312428
});
2432-
}
2429+
});
24332430

2434-
if (getEnv('nativeSqlPlanner')) {
2435-
it.skip('rollup join existing joins: should be fixed in Tesseract', () => {
2436-
// This should be fixed in Tesseract.
2431+
it('rollup join existing joins', async () => {
2432+
await compiler.compile();
2433+
2434+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
2435+
measures: [
2436+
'visitor_checkins.count',
2437+
],
2438+
dimensions: ['visitors.source', 'cards.visitorId'],
2439+
preAggregationsSchema: '',
2440+
order: [{
2441+
id: 'visitors.source',
2442+
}, {
2443+
id: 'cards.visitorId',
2444+
}],
2445+
timezone: 'UTC',
24372446
});
2438-
} else {
2439-
it('rollup join existing joins', async () => {
2440-
await compiler.compile();
24412447

2442-
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
2443-
measures: [
2444-
'visitor_checkins.count',
2445-
],
2446-
dimensions: ['visitors.source', 'cards.visitorId'],
2447-
preAggregationsSchema: '',
2448-
order: [{
2449-
id: 'visitors.source',
2450-
}, {
2451-
id: 'cards.visitorId',
2452-
}],
2453-
timezone: 'UTC',
2454-
});
2448+
const queryAndParams = query.buildSqlAndParams();
2449+
console.log(queryAndParams);
2450+
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
2451+
console.log(preAggregationsDescription);
24552452

2456-
const queryAndParams = query.buildSqlAndParams();
2457-
console.log(queryAndParams);
2458-
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
2459-
console.log(preAggregationsDescription);
2453+
expect(queryAndParams[0]).toContain('visitors_for_join_inc_cards');
2454+
expect(queryAndParams[0]).toContain('vc_for_join');
24602455

2461-
console.log(query.preAggregations?.rollupMatchResultDescriptions());
2456+
console.log(query.preAggregations?.rollupMatchResultDescriptions());
24622457

2463-
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
2458+
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
24642459

2465-
console.log(JSON.stringify(queries.concat(queryAndParams)));
2460+
console.log(JSON.stringify(queries.concat(queryAndParams)));
24662461

2467-
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
2468-
console.log(JSON.stringify(res));
2469-
expect(res).toEqual(
2470-
[
2471-
{ visitors__source: 'google', cards__visitor_id: 3, vc__count: '1' },
2472-
{ visitors__source: 'some', cards__visitor_id: 1, vc__count: '3' },
2473-
{ visitors__source: 'some', cards__visitor_id: null, vc__count: '2' },
2474-
{ visitors__source: null, cards__visitor_id: null, vc__count: '0' },
2475-
],
2476-
);
2477-
});
2462+
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
2463+
console.log(JSON.stringify(res));
2464+
expect(res).toEqual(
2465+
[
2466+
{ visitors__source: 'google', cards__visitor_id: 3, vc__count: '1' },
2467+
{ visitors__source: 'some', cards__visitor_id: 1, vc__count: '3' },
2468+
{ visitors__source: 'some', cards__visitor_id: null, vc__count: '2' },
2469+
{ visitors__source: null, cards__visitor_id: null, vc__count: null },
2470+
],
2471+
);
24782472
});
2479-
}
2473+
});
24802474

2481-
if (getEnv('nativeSqlPlanner')) {
2482-
it.skip('rollup join partitioned: should be fixed in Tesseract', () => {
2483-
// This should be fixed in Tesseract.
2475+
it('rollup join partitioned', async () => {
2476+
await compiler.compile();
2477+
2478+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
2479+
measures: [
2480+
'visitor_checkins.count',
2481+
],
2482+
dimensions: ['visitors.source'],
2483+
timezone: 'America/Los_Angeles',
2484+
preAggregationsSchema: '',
2485+
timeDimensions: [{
2486+
dimension: 'visitors.createdAt',
2487+
granularity: 'hour',
2488+
dateRange: ['2017-01-03', '2017-01-04']
2489+
}],
2490+
order: [{
2491+
id: 'visitors.createdAt'
2492+
}],
24842493
});
2485-
} else {
2486-
it('rollup join partitioned', async () => {
2487-
await compiler.compile();
24882494

2489-
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
2490-
measures: [
2491-
'visitor_checkins.count',
2492-
],
2493-
dimensions: ['visitors.source'],
2494-
timezone: 'America/Los_Angeles',
2495-
preAggregationsSchema: '',
2496-
timeDimensions: [{
2497-
dimension: 'visitors.createdAt',
2498-
granularity: 'hour',
2499-
dateRange: ['2017-01-03', '2017-01-04']
2500-
}],
2501-
order: [{
2502-
id: 'visitors.createdAt'
2503-
}],
2504-
});
2495+
const queryAndParams = query.buildSqlAndParams();
2496+
console.log(queryAndParams);
2497+
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
2498+
console.log(preAggregationsDescription);
25052499

2506-
const queryAndParams = query.buildSqlAndParams();
2507-
console.log(queryAndParams);
2508-
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
2509-
console.log(preAggregationsDescription);
2500+
expect(queryAndParams[0]).toContain('visitors_partitioned_hourly_for_join');
2501+
expect(queryAndParams[0]).toContain('vc_for_join');
25102502

2511-
console.log(query.preAggregations?.rollupMatchResultDescriptions());
2503+
console.log(query.preAggregations?.rollupMatchResultDescriptions());
25122504

2513-
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
2505+
const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription);
25142506

2515-
console.log(JSON.stringify(queries.concat(queryAndParams)));
2507+
console.log(JSON.stringify(queries.concat(queryAndParams)));
25162508

2517-
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
2518-
console.log(JSON.stringify(res));
2519-
expect(res).toEqual(
2520-
[
2521-
{
2522-
visitors__source: 'some',
2523-
visitors__created_at_hour: '2017-01-04T16:00:00.000Z',
2524-
vc__count: '2'
2525-
}
2526-
],
2527-
);
2528-
});
2509+
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
2510+
console.log(JSON.stringify(res));
2511+
expect(res).toEqual(
2512+
[
2513+
{
2514+
visitors__source: 'some',
2515+
visitors__created_at_hour: '2017-01-04T16:00:00.000Z',
2516+
vc__count: '2'
2517+
}
2518+
],
2519+
);
25292520
});
2530-
}
2521+
});
25312522

25322523
it('partitioned without time', async () => {
25332524
await compiler.compile();

0 commit comments

Comments
 (0)