Skip to content

Commit 8911dc8

Browse files
KSDaemonKSDaemon
committed
some tests for PreAggregationPartitionRangeLoader
1 parent e628a11 commit 8911dc8

File tree

1 file changed

+257
-0
lines changed

1 file changed

+257
-0
lines changed

‎packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
/* eslint-disable global-require */
22
/* globals describe, jest, beforeEach, test, expect */
33
import R from 'ramda';
4+
import {
5+
BUILD_RANGE_END_LOCAL,
6+
BUILD_RANGE_START_LOCAL,
7+
FROM_PARTITION_RANGE,
8+
TO_PARTITION_RANGE
9+
} from '@cubejs-backend/shared';
410
import { PreAggregationPartitionRangeLoader, PreAggregations } from '../../src';
511

612
class MockDriver {
@@ -67,6 +73,43 @@ class MockDriver {
6773
}
6874
}
6975

76+
const mockPreAggregation = (overrides = {}) => ({
77+
tableName: 'test_table',
78+
partitionGranularity: 'day',
79+
timezone: 'UTC',
80+
timestampFormat: 'YYYY-MM-DDTHH:mm:ss.SSS',
81+
timestampPrecision: 3,
82+
dataSource: 'default',
83+
preAggregationStartEndQueries: [
84+
['SELECT MIN(ts)', [], {}],
85+
['SELECT MAX(ts)', [], {}]
86+
],
87+
loadSql: ['SELECT * FROM test_table WHERE ts >= ? and ts < ?', [FROM_PARTITION_RANGE, TO_PARTITION_RANGE]],
88+
...overrides,
89+
});
90+
91+
const createLoader = (overrides = {}, options = {}) => {
92+
const loader = new PreAggregationPartitionRangeLoader(
93+
{}, // driverFactory
94+
{}, // logger
95+
{ options: {} }, // queryCache
96+
{}, // preAggregations
97+
mockPreAggregation(overrides),
98+
[], // preAggregationsTablesToTempTables
99+
{}, // loadCache
100+
options,
101+
);
102+
103+
jest.spyOn(loader, 'loadRangeQuery').mockImplementation(async (query, _partitionRange) => {
104+
if (query[0].includes('MIN')) {
105+
return [{ value: '2024-01-01T00:00:00.000' }];
106+
}
107+
return [{ value: '2024-01-03T23:59:59.999' }];
108+
});
109+
110+
return loader;
111+
};
112+
70113
describe('PreAggregations', () => {
71114
let mockDriver = null;
72115
let mockExternalDriver = null;
@@ -434,4 +477,218 @@ describe('PreAggregations', () => {
434477
)).toThrow('Date range expected to be in YYYY-MM-DDTHH:mm:ss.SSS format');
435478
});
436479
});
480+
481+
describe('partitionTableName', () => {
482+
test('should generate correct table names for different granularities', () => {
483+
const testDateRange = ['2024-01-05T12:34:56.789', '2024-01-05T23:59:59.999'];
484+
485+
// Daily granularity
486+
expect(PreAggregationPartitionRangeLoader.partitionTableName(
487+
'test_table',
488+
'day',
489+
testDateRange
490+
)).toBe('test_table20240105');
491+
492+
// Hourly granularity
493+
expect(PreAggregationPartitionRangeLoader.partitionTableName(
494+
'test_table',
495+
'hour',
496+
testDateRange
497+
)).toBe('test_table2024010512');
498+
499+
// Minute granularity
500+
expect(PreAggregationPartitionRangeLoader.partitionTableName(
501+
'test_table',
502+
'minute',
503+
testDateRange
504+
)).toBe('test_table202401051234');
505+
});
506+
});
507+
508+
describe('replaceQueryBuildRangeParams', () => {
509+
test('should replace BUILD_RANGE params with actual dates', async () => {
510+
const loader = createLoader();
511+
jest.spyOn(loader, 'loadBuildRange').mockResolvedValue([
512+
'2023-01-01T00:00:00.000',
513+
'2023-01-31T23:59:59.999',
514+
]);
515+
516+
const result = await loader.replaceQueryBuildRangeParams([
517+
'other_param_that_should_not_be_modified',
518+
BUILD_RANGE_START_LOCAL,
519+
BUILD_RANGE_END_LOCAL,
520+
]);
521+
522+
expect(result).toEqual([
523+
'other_param_that_should_not_be_modified',
524+
'2023-01-01T00:00:00.000',
525+
'2023-01-31T23:59:59.999',
526+
]);
527+
});
528+
529+
test('should return null when no BUILD_RANGE params', async () => {
530+
const loader = createLoader();
531+
const result = await loader.replaceQueryBuildRangeParams(['param1', 'param2']);
532+
expect(result).toBeNull();
533+
});
534+
});
535+
536+
describe('partitionPreAggregations', () => {
537+
test('should construct correct partitionPreAggregations for dateRange in UTC', async () => {
538+
const loader = createLoader({
539+
timezone: 'UTC',
540+
});
541+
542+
const results = await loader.partitionPreAggregations();
543+
expect(results.length).toEqual(3);
544+
545+
let [preAggDesc] = results;
546+
expect(preAggDesc.tableName).toEqual('test_table20240101'); // Partition tables are the same for all time zones
547+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones
548+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999');
549+
expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy();
550+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000');
551+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T23:59:59.999');
552+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240101')).toBeTruthy();
553+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000');
554+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T23:59:59.999');
555+
556+
[, preAggDesc] = results;
557+
expect(preAggDesc.tableName).toEqual('test_table20240102');
558+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-02T00:00:00.000');
559+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-02T23:59:59.999');
560+
expect(preAggDesc.loadSql[0].includes('test_table20240102')).toBeTruthy();
561+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-02T00:00:00.000');
562+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-02T23:59:59.999');
563+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240102')).toBeTruthy();
564+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-02T00:00:00.000');
565+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T23:59:59.999');
566+
567+
[,, preAggDesc] = results;
568+
expect(preAggDesc.tableName).toEqual('test_table20240103');
569+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T00:00:00.000');
570+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999');
571+
expect(preAggDesc.loadSql[0].includes('test_table20240103')).toBeTruthy();
572+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T00:00:00.000');
573+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999');
574+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240103')).toBeTruthy();
575+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T00:00:00.000');
576+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999');
577+
});
578+
579+
test('should construct correct partitionPreAggregations for dateRange in America/New_York', async () => {
580+
const loader = createLoader({
581+
timezone: 'America/New_York', // UTC-5
582+
});
583+
584+
const results = await loader.partitionPreAggregations();
585+
expect(results.length).toEqual(3);
586+
587+
let [preAggDesc] = results;
588+
expect(preAggDesc.tableName).toEqual('test_table20240101'); // Partition tables are the same for all time zones
589+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones
590+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999');
591+
expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy();
592+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T05:00:00.000');
593+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-02T04:59:59.999');
594+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240101')).toBeTruthy();
595+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T05:00:00.000');
596+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T04:59:59.999');
597+
598+
[, preAggDesc] = results;
599+
expect(preAggDesc.tableName).toEqual('test_table20240102');
600+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-02T00:00:00.000');
601+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-02T23:59:59.999');
602+
expect(preAggDesc.loadSql[0].includes('test_table20240102')).toBeTruthy();
603+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-02T05:00:00.000');
604+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T04:59:59.999');
605+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240102')).toBeTruthy();
606+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-02T05:00:00.000');
607+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T04:59:59.999');
608+
609+
[,, preAggDesc] = results;
610+
expect(preAggDesc.tableName).toEqual('test_table20240103');
611+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T00:00:00.000');
612+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999');
613+
expect(preAggDesc.loadSql[0].includes('test_table20240103')).toBeTruthy();
614+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T05:00:00.000');
615+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T04:59:59.999');
616+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240103')).toBeTruthy();
617+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T05:00:00.000');
618+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T04:59:59.999');
619+
});
620+
621+
test('should construct correct partitionPreAggregations for dateRange in Asia/Tokyo', async () => {
622+
const loader = createLoader({
623+
timezone: 'Asia/Tokyo', // UTC+9
624+
});
625+
626+
const results = await loader.partitionPreAggregations();
627+
expect(results.length).toEqual(3);
628+
629+
let [preAggDesc] = results;
630+
expect(preAggDesc.tableName).toEqual('test_table20240101'); // Partition tables are the same for all time zones
631+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones
632+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999');
633+
expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy();
634+
expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T15:00:00.000');
635+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T14:59:59.999');
636+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240101')).toBeTruthy();
637+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T15:00:00.000');
638+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T14:59:59.999');
639+
640+
[, preAggDesc] = results;
641+
expect(preAggDesc.tableName).toEqual('test_table20240102');
642+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-02T00:00:00.000');
643+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-02T23:59:59.999');
644+
expect(preAggDesc.loadSql[0].includes('test_table20240102')).toBeTruthy();
645+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T15:00:00.000');
646+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-02T14:59:59.999');
647+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240102')).toBeTruthy();
648+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T15:00:00.000');
649+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T14:59:59.999');
650+
651+
[,, preAggDesc] = results;
652+
expect(preAggDesc.tableName).toEqual('test_table20240103');
653+
expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T00:00:00.000');
654+
expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999');
655+
expect(preAggDesc.loadSql[0].includes('test_table20240103')).toBeTruthy();
656+
expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-02T15:00:00.000');
657+
expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T14:59:59.999');
658+
expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240103')).toBeTruthy();
659+
expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-02T15:00:00.000');
660+
expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T14:59:59.999');
661+
});
662+
});
663+
664+
describe('partitionPreAggregations', () => {
665+
test('should generate partitioned pre-aggregations', async () => {
666+
const compilerCacheFn = jest.fn((subKey, fn) => fn());
667+
const loader = createLoader(
668+
{
669+
partitionGranularity: 'day',
670+
matchedTimeDimensionDateRange: ['2023-01-01T00:00:00.000', '2023-01-02T23:59:59.999'],
671+
},
672+
{ compilerCacheFn }
673+
);
674+
675+
jest.spyOn(loader, 'partitionRanges').mockResolvedValue({
676+
buildRange: ['2023-01-01T00:00:00.000', '2023-01-02T23:59:59.999'],
677+
partitionRanges: [
678+
['2023-01-01T00:00:00.000', '2023-01-01T23:59:59.999'],
679+
['2023-01-02T00:00:00.000', '2023-01-02T23:59:59.999'],
680+
],
681+
});
682+
683+
const result = await loader.partitionPreAggregations();
684+
685+
expect(result.length).toBe(2);
686+
expect(result[0].tableName).toMatch(/test_table20230101/);
687+
expect(result[1].tableName).toMatch(/test_table20230102/);
688+
expect(compilerCacheFn).toHaveBeenCalledWith(
689+
['partitions', JSON.stringify(['2023-01-01T00:00:00.000', '2023-01-02T23:59:59.999'])],
690+
expect.any(Function)
691+
);
692+
});
693+
});
437694
});

0 commit comments

Comments
 (0)