Skip to content

Commit b90b3f2

Browse files
paveltiunovpaveltiunov
authored
feat: Pre-aggregations API for rollupLambda support (#6009)
1 parent 1feac4d commit b90b3f2

File tree

4 files changed

+65
-16
lines changed

4 files changed

+65
-16
lines changed

‎packages/cubejs-schema-compiler/src/adapter/PreAggregations.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -660,7 +660,8 @@ export class PreAggregations {
660660
this.preAggregationForQuery =
661661
this
662662
.rollupMatchResults()
663-
.find(p => p.canUsePreAggregation);
663+
// Refresh worker can access specific pre-aggregations even in case those hidden by others
664+
.find(p => p.canUsePreAggregation && (!this.query.options.preAggregationId || p.preAggregationId === this.query.options.preAggregationId));
664665
}
665666
return this.preAggregationForQuery;
666667
}
@@ -827,7 +828,8 @@ export class PreAggregations {
827828
preAggregation,
828829
cube,
829830
canUsePreAggregation: canUsePreAggregation(references),
830-
references
831+
references,
832+
preAggregationId: `${cube}.${preAggregationName}`
831833
};
832834

833835
if (preAggregation.type === 'rollupJoin') {

‎packages/cubejs-server-core/src/core/RefreshScheduler.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ export class RefreshScheduler {
190190
const compilers = await compilerApi.getCompilers();
191191
const query = await compilerApi.createQueryByDataSource(compilers, queryingOptions);
192192
if (preAggregation.preAggregation.partitionGranularity || preAggregation.preAggregation.type === 'rollup') {
193-
return { ...queryingOptions, ...preAggregation.references };
193+
return { ...queryingOptions, ...preAggregation.references, preAggregationId: preAggregation.id };
194194
} else if (preAggregation.preAggregation.type === 'originalSql') {
195195
const cubeFromPath = query.cubeEvaluator.cubeFromPath(preAggregation.cube);
196196
const measuresCount = Object.keys(cubeFromPath.measures || {}).length;

‎packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts‎

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -400,42 +400,47 @@ describe('Refresh Scheduler', () => {
400400
} = setupScheduler({ repository: repositoryWithPreAggregations, useOriginalSqlPreAggregations: true });
401401
const result1 = [
402402
{ tableName: 'stb_pre_aggregations.foo_first20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
403-
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' },
404403
{ tableName: 'stb_pre_aggregations.foo_main', timezone: null, fromTable: 'foo_tenant1' },
404+
{
405+
tableName: 'stb_pre_aggregations.foo_second20201231',
406+
timezone: 'UTC',
407+
fromTable: { preAggTable: 'stb_pre_aggregations.foo_main' },
408+
},
409+
{ tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
405410
{
406411
tableName: 'stb_pre_aggregations.foo_second20201230',
407412
timezone: 'UTC',
408413
fromTable: { preAggTable: 'stb_pre_aggregations.foo_main' },
409414
},
410415
{ tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
411-
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' },
412416
{
413-
tableName: 'stb_pre_aggregations.foo_second20201228',
417+
tableName: 'stb_pre_aggregations.foo_second20201229',
414418
timezone: 'UTC',
415419
fromTable: { preAggTable: 'stb_pre_aggregations.foo_main' },
416420
},
421+
{ tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
417422
{
418-
tableName: 'stb_pre_aggregations.foo_second20201227',
423+
tableName: 'stb_pre_aggregations.foo_second20201228',
419424
timezone: 'UTC',
420425
fromTable: { preAggTable: 'stb_pre_aggregations.foo_main' },
421426
},
427+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
422428
];
423429

424430
const result2 = [
425-
{
426-
tableName: 'stb_pre_aggregations.foo_second20201231',
427-
timezone: 'UTC',
428-
fromTable: { preAggTable: 'stb_pre_aggregations.foo_main' },
429-
},
430-
{ tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
431+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
432+
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' },
433+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
431434
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'UTC', fromTable: 'bar' },
435+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
436+
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' },
437+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
438+
{ tableName: 'stb_pre_aggregations.foo_first20201227', timezone: 'UTC', fromTable: 'foo_tenant1', },
432439
{
433-
tableName: 'stb_pre_aggregations.foo_second20201229',
440+
tableName: 'stb_pre_aggregations.foo_second20201227',
434441
timezone: 'UTC',
435442
fromTable: { preAggTable: 'stb_pre_aggregations.foo_main' },
436443
},
437-
{ tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
438-
{ tableName: 'stb_pre_aggregations.foo_first20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
439444
];
440445

441446
const ctx = { authInfo: { tenantId: 'tenant1' }, securityContext: { tenantId: 'tenant1' }, requestId: 'XXX' };
@@ -669,13 +674,15 @@ describe('Refresh Scheduler', () => {
669674
} = setupScheduler({ repository: repositoryWithPreAggregations });
670675
const result = [
671676
{ tableName: 'stb_pre_aggregations.foo_first20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
677+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
672678
{ tableName: 'stb_pre_aggregations.foo_second20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
673679
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' },
674680
{
675681
tableName: 'stb_pre_aggregations.foo_first20201231',
676682
timezone: 'America/Los_Angeles',
677683
fromTable: 'foo_tenant1',
678684
},
685+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
679686
{
680687
tableName: 'stb_pre_aggregations.foo_second20201231',
681688
timezone: 'America/Los_Angeles',
@@ -684,13 +691,15 @@ describe('Refresh Scheduler', () => {
684691
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'America/Los_Angeles', fromTable: 'bar' },
685692

686693
{ tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
694+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
687695
{ tableName: 'stb_pre_aggregations.foo_second20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
688696
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'UTC', fromTable: 'bar' },
689697
{
690698
tableName: 'stb_pre_aggregations.foo_first20201230',
691699
timezone: 'America/Los_Angeles',
692700
fromTable: 'foo_tenant1',
693701
},
702+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
694703
{
695704
tableName: 'stb_pre_aggregations.foo_second20201230',
696705
timezone: 'America/Los_Angeles',
@@ -699,13 +708,15 @@ describe('Refresh Scheduler', () => {
699708
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' },
700709

701710
{ tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
711+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
702712
{ tableName: 'stb_pre_aggregations.foo_second20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
703713
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' },
704714
{
705715
tableName: 'stb_pre_aggregations.foo_first20201229',
706716
timezone: 'America/Los_Angeles',
707717
fromTable: 'foo_tenant1',
708718
},
719+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
709720
{
710721
tableName: 'stb_pre_aggregations.foo_second20201229',
711722
timezone: 'America/Los_Angeles',
@@ -714,25 +725,29 @@ describe('Refresh Scheduler', () => {
714725
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' },
715726

716727
{ tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
728+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
717729
{ tableName: 'stb_pre_aggregations.foo_second20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
718730
{
719731
tableName: 'stb_pre_aggregations.foo_first20201228',
720732
timezone: 'America/Los_Angeles',
721733
fromTable: 'foo_tenant1',
722734
},
735+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
723736
{
724737
tableName: 'stb_pre_aggregations.foo_second20201228',
725738
timezone: 'America/Los_Angeles',
726739
fromTable: 'foo_tenant1',
727740
},
728741

729742
{ tableName: 'stb_pre_aggregations.foo_first20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
743+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
730744
{ tableName: 'stb_pre_aggregations.foo_second20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
731745
{
732746
tableName: 'stb_pre_aggregations.foo_first20201227',
733747
timezone: 'America/Los_Angeles',
734748
fromTable: 'foo_tenant1',
735749
},
750+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
736751
{
737752
tableName: 'stb_pre_aggregations.foo_second20201227',
738753
timezone: 'America/Los_Angeles',
@@ -792,17 +807,22 @@ describe('Refresh Scheduler', () => {
792807
} = setupScheduler({ repository: repositoryWithPreAggregations });
793808
const result = [
794809
{ tableName: 'stb_pre_aggregations.foo_first20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
810+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
795811
{ tableName: 'stb_pre_aggregations.foo_second20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
796812
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' },
797813
{ tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
814+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
798815
{ tableName: 'stb_pre_aggregations.foo_second20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
799816
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'UTC', fromTable: 'bar' },
800817
{ tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
818+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
801819
{ tableName: 'stb_pre_aggregations.foo_second20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
802820
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' },
803821
{ tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
822+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
804823
{ tableName: 'stb_pre_aggregations.foo_second20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
805824
{ tableName: 'stb_pre_aggregations.foo_first20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
825+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
806826
{ tableName: 'stb_pre_aggregations.foo_second20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
807827
];
808828

‎packages/cubejs-testing/test/smoke-lambda.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,4 +330,31 @@ describe('lambda', () => {
330330
console.log(await preAggs.json());
331331
expect(preAggs.status).toBe(200);
332332
});
333+
334+
test('Pre-aggregations API partitions', async () => {
335+
const partitions = await (await fetch(`${birdbox.configuration.systemUrl}/pre-aggregations/partitions`, {
336+
method: 'POST',
337+
headers: { 'Content-Type': 'application/json' },
338+
body: JSON.stringify({
339+
query: {
340+
preAggregations: [
341+
{
342+
id: 'Orders.ordersByCompletedAtLambda'
343+
},
344+
{
345+
id: 'Orders.ordersByCompletedAt'
346+
},
347+
{
348+
id: 'Orders.ordersByCompletedByDay'
349+
}
350+
]
351+
}
352+
}),
353+
})).json();
354+
console.log(JSON.stringify(partitions, null, 2));
355+
const completedAtPartition = partitions.preAggregationPartitions[1].partitions[0];
356+
expect(completedAtPartition.loadSql[0]).toMatch(/orders_orders_by_completed_at/);
357+
const completedByDayPartition = partitions.preAggregationPartitions[2].partitions[0];
358+
expect(completedByDayPartition.loadSql[0]).toMatch(/orders_orders_by_completed_by_day/);
359+
});
333360
});

0 commit comments

Comments
 (0)