Skip to content

Commit 685383f

Browse files
KSDaemonKSDaemon
committed
make loadBuildRange() timezone-aware + align tests
1 parent 58fde93 commit 685383f

File tree

3 files changed

+89
-55
lines changed

3 files changed

+89
-55
lines changed

‎packages/cubejs-backend-shared/src/time.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,13 @@ export const inDbTimeZone = (timezone: string, timestampFormat: string, timestam
227227
return moment.tz(timestamp, timezone).utc().format(timestampFormat);
228228
};
229229

230-
export const utcToLocalTimeZone = (timezone: string, timestampFormat: string, timestamp: string): string => {
230+
/**
231+
* Takes timestamp in UTC, treat it as local time in provided timezone and returns the corresponding timestamp in UTC
232+
*/
233+
export const utcToLocalTimeZoneInUtc = (timezone: string, timestampFormat: string, timestamp: string): string | null => {
234+
if (!timestamp) {
235+
return null;
236+
}
231237
if (timestamp.length === 23) {
232238
const zone = moment.tz.zone(timezone);
233239
if (!zone) {

‎packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts‎

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
TO_PARTITION_RANGE,
77
MAX_SOURCE_ROW_LIMIT,
88
reformatInIsoLocal,
9-
utcToLocalTimeZone,
9+
utcToLocalTimeZoneInUtc,
1010
timeSeries,
1111
inDbTimeZone,
1212
extractDate
@@ -131,14 +131,17 @@ export class PreAggregationPartitionRangeLoader {
131131
}
132132

133133
public async replaceQueryBuildRangeParams(queryValues: string[]): Promise<string[] | null> {
134-
if (queryValues?.find(p => p === BUILD_RANGE_START_LOCAL || p === BUILD_RANGE_END_LOCAL)) {
134+
if (queryValues.find(p => p === BUILD_RANGE_START_LOCAL || p === BUILD_RANGE_END_LOCAL)) {
135135
const [buildRangeStart, buildRangeEnd] = await this.loadBuildRange();
136136
return queryValues?.map(
137137
param => {
138138
if (param === BUILD_RANGE_START_LOCAL) {
139-
return utcToLocalTimeZone(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeStart);
139+
// buildRangeStart was already localized in loadBuildRange() but the preAggregation.timestampFormat
140+
// might be different from 'YYYY-MM-DDTHH:mm:ss.SSS' so we need to convert.
141+
// Same applies to buildRangeEnd 2 below.
142+
return utcToLocalTimeZoneInUtc('UTC', this.preAggregation.timestampFormat, buildRangeStart);
140143
} else if (param === BUILD_RANGE_END_LOCAL) {
141-
return utcToLocalTimeZone(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeEnd);
144+
return utcToLocalTimeZoneInUtc('UTC', this.preAggregation.timestampFormat, buildRangeEnd);
142145
} else {
143146
return param;
144147
}
@@ -370,57 +373,79 @@ export class PreAggregationPartitionRangeLoader {
370373
if (!buildRange[0] || !buildRange[1]) {
371374
return { buildRange, partitionRanges: [] };
372375
}
376+
377+
// buildRange is localized in loadBuildRange()
378+
// preAggregation.matchedTimeDimensionDateRange is also localized
379+
// in BaseFilter->formatToDate()/formatFromDate()
373380
let dateRange = PreAggregationPartitionRangeLoader.intersectDateRanges(
374381
buildRange,
375382
ignoreMatchedDateRange ? undefined : this.preAggregation.matchedTimeDimensionDateRange,
376383
);
384+
377385
if (!dateRange) {
378386
// If there's no date range intersection between query data range and pre-aggregation build range
379387
// use last partition so outer query can receive expected table structure.
380388
dateRange = [buildRange[1], buildRange[1]];
381389
}
382-
const partitionRanges = this.compilerCacheFn(['timeSeries', this.preAggregation.partitionGranularity, JSON.stringify(dateRange), `${this.preAggregation.timestampPrecision}`], () => PreAggregationPartitionRangeLoader.timeSeries(
383-
this.preAggregation.partitionGranularity,
384-
dateRange,
385-
this.preAggregation.timestampPrecision
386-
));
390+
391+
const partitionRanges = this.compilerCacheFn(
392+
['timeSeries', this.preAggregation.partitionGranularity, JSON.stringify(dateRange), `${this.preAggregation.timestampPrecision}`],
393+
() => PreAggregationPartitionRangeLoader.timeSeries(
394+
this.preAggregation.partitionGranularity,
395+
dateRange,
396+
this.preAggregation.timestampPrecision
397+
)
398+
);
399+
387400
if (partitionRanges.length > this.options.maxPartitions) {
388401
throw new Error(
389402
`Pre-aggregation '${this.preAggregation.tableName}' requested to build ${partitionRanges.length} partitions which exceeds the maximum number of partitions per pre-aggregation of ${this.options.maxPartitions}`
390403
);
391404
}
405+
392406
return { buildRange: dateRange, partitionRanges };
393407
}
394408

395409
public async loadBuildRange(): Promise<QueryDateRange> {
396410
const { preAggregationStartEndQueries } = this.preAggregation;
397411
const [startDate, endDate] = await Promise.all(
398412
preAggregationStartEndQueries.map(
399-
async rangeQuery => PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery)),
413+
async rangeQuery => utcToLocalTimeZoneInUtc(
414+
this.preAggregation.timezone,
415+
'YYYY-MM-DDTHH:mm:ss.SSS',
416+
PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery)),
417+
)
400418
),
401419
);
420+
402421
if (!this.preAggregation.partitionGranularity) {
403422
return this.orNowIfEmpty([startDate, endDate]);
404423
}
424+
425+
// startDate & endDate are `localized` here
405426
const wholeSeriesRanges = PreAggregationPartitionRangeLoader.timeSeries(
406427
this.preAggregation.partitionGranularity,
407428
this.orNowIfEmpty([startDate, endDate]),
408429
this.preAggregation.timestampPrecision,
409430
);
410431
const [rangeStart, rangeEnd] = await Promise.all(
411432
preAggregationStartEndQueries.map(
412-
async (rangeQuery, i) => PreAggregationPartitionRangeLoader.extractDate(
413-
await this.loadRangeQuery(
414-
rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1],
433+
async (rangeQuery, i) => utcToLocalTimeZoneInUtc(
434+
this.preAggregation.timezone,
435+
'YYYY-MM-DDTHH:mm:ss.SSS',
436+
PreAggregationPartitionRangeLoader.extractDate(
437+
await this.loadRangeQuery(
438+
rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1],
439+
),
415440
),
416-
),
441+
)
417442
),
418443
);
419444
return this.orNowIfEmpty([rangeStart, rangeEnd]);
420445
}
421446

422447
private now() {
423-
return utcToLocalTimeZone(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
448+
return utcToLocalTimeZoneInUtc(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
424449
}
425450

426451
private orNowIfEmpty(dateRange: QueryDateRange): QueryDateRange {
@@ -455,7 +480,7 @@ export class PreAggregationPartitionRangeLoader {
455480
}
456481
}
457482

458-
public static intersectDateRanges(rangeA: QueryDateRange | null, rangeB: QueryDateRange | null): QueryDateRange {
483+
public static intersectDateRanges(rangeA: QueryDateRange | null, rangeB: QueryDateRange | null): QueryDateRange | null {
459484
PreAggregationPartitionRangeLoader.checkDataRangeType(rangeA);
460485
PreAggregationPartitionRangeLoader.checkDataRangeType(rangeB);
461486
if (!rangeB) {
@@ -475,7 +500,10 @@ export class PreAggregationPartitionRangeLoader {
475500
];
476501
}
477502

478-
public static timeSeries(granularity: string, dateRange: QueryDateRange, timestampPrecision: number): QueryDateRange[] {
503+
public static timeSeries(granularity: string, dateRange: QueryDateRange | null, timestampPrecision: number): QueryDateRange[] {
504+
if (!dateRange) {
505+
return [];
506+
}
479507
return timeSeries(granularity, dateRange, {
480508
timestampPrecision
481509
});

‎packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,25 @@ import { CompilerApi } from '../../src/core/CompilerApi';
88
const schemaContent = `
99
cube('Foo', {
1010
sql: \`select * from foo_\${SECURITY_CONTEXT.tenantId.unsafeValue()}\`,
11-
11+
1212
measures: {
1313
count: {
1414
type: 'count'
1515
},
16-
16+
1717
total: {
1818
sql: 'amount',
1919
type: 'sum'
2020
},
2121
},
22-
22+
2323
dimensions: {
2424
time: {
2525
sql: 'timestamp',
2626
type: 'time'
2727
}
2828
},
29-
29+
3030
preAggregations: {
3131
main: {
3232
type: 'originalSql',
@@ -87,20 +87,20 @@ cube('Foo', {
8787
8888
cube('Bar', {
8989
sql: 'select * from bar',
90-
90+
9191
measures: {
9292
count: {
9393
type: 'count'
9494
}
9595
},
96-
96+
9797
dimensions: {
9898
time: {
9999
sql: 'timestamp',
100100
type: 'time'
101101
}
102102
},
103-
103+
104104
preAggregations: {
105105
first: {
106106
type: 'rollup',
@@ -131,63 +131,63 @@ const repositoryWithRollupJoin: SchemaFileRepository = {
131131
{ fileName: 'main.js', content: `
132132
cube(\`Users\`, {
133133
sql: \`SELECT * FROM public.users\`,
134-
134+
135135
preAggregations: {
136136
usersRollup: {
137137
dimensions: [CUBE.id],
138138
},
139139
},
140-
140+
141141
measures: {
142142
count: {
143143
type: \`count\`,
144144
},
145145
},
146-
146+
147147
dimensions: {
148148
id: {
149149
sql: \`id\`,
150150
type: \`string\`,
151151
primaryKey: true,
152152
},
153-
153+
154154
name: {
155155
sql: \`name\`,
156156
type: \`string\`,
157157
},
158158
},
159159
});
160-
160+
161161
cube('Orders', {
162162
sql: \`SELECT * FROM orders\`,
163-
163+
164164
preAggregations: {
165165
ordersRollup: {
166166
measures: [CUBE.count],
167167
dimensions: [CUBE.userId, CUBE.status],
168168
},
169-
169+
170170
ordersRollupJoin: {
171171
type: \`rollupJoin\`,
172172
measures: [CUBE.count],
173173
dimensions: [Users.name],
174174
rollups: [Users.usersRollup, CUBE.ordersRollup],
175175
},
176176
},
177-
177+
178178
joins: {
179179
Users: {
180180
relationship: \`belongsTo\`,
181181
sql: \`\${CUBE.userId} = \${Users.id}\`,
182182
},
183183
},
184-
184+
185185
measures: {
186186
count: {
187187
type: \`count\`,
188188
},
189189
},
190-
190+
191191
dimensions: {
192192
id: {
193193
sql: \`id\`,
@@ -215,13 +215,13 @@ const repositoryWithoutPreAggregations: SchemaFileRepository = {
215215
fileName: 'main.js', content: `
216216
cube('Bar', {
217217
sql: 'select * from bar',
218-
218+
219219
measures: {
220220
count: {
221221
type: 'count'
222222
}
223223
},
224-
224+
225225
dimensions: {
226226
time: {
227227
sql: 'timestamp',
@@ -698,63 +698,63 @@ describe('Refresh Scheduler', () => {
698698
{ tableName: 'stb_pre_aggregations.foo_second20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
699699
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' },
700700
{
701-
tableName: 'stb_pre_aggregations.foo_first20201231',
701+
tableName: 'stb_pre_aggregations.foo_first20201230',
702702
timezone: 'America/Los_Angeles',
703703
fromTable: 'foo_tenant1',
704704
},
705-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
705+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
706706
{
707-
tableName: 'stb_pre_aggregations.foo_second20201231',
707+
tableName: 'stb_pre_aggregations.foo_second20201230',
708708
timezone: 'America/Los_Angeles',
709709
fromTable: 'foo_tenant1',
710710
},
711-
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'America/Los_Angeles', fromTable: 'bar' },
711+
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' },
712712

713713
{ tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
714714
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
715715
{ tableName: 'stb_pre_aggregations.foo_second20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
716716
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'UTC', fromTable: 'bar' },
717717
{
718-
tableName: 'stb_pre_aggregations.foo_first20201230',
718+
tableName: 'stb_pre_aggregations.foo_first20201229',
719719
timezone: 'America/Los_Angeles',
720720
fromTable: 'foo_tenant1',
721721
},
722-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
722+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
723723
{
724-
tableName: 'stb_pre_aggregations.foo_second20201230',
724+
tableName: 'stb_pre_aggregations.foo_second20201229',
725725
timezone: 'America/Los_Angeles',
726726
fromTable: 'foo_tenant1',
727727
},
728-
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' },
728+
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' },
729729

730730
{ tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
731731
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
732732
{ tableName: 'stb_pre_aggregations.foo_second20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
733733
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' },
734734
{
735-
tableName: 'stb_pre_aggregations.foo_first20201229',
735+
tableName: 'stb_pre_aggregations.foo_first20201228',
736736
timezone: 'America/Los_Angeles',
737737
fromTable: 'foo_tenant1',
738738
},
739-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
739+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
740740
{
741-
tableName: 'stb_pre_aggregations.foo_second20201229',
741+
tableName: 'stb_pre_aggregations.foo_second20201228',
742742
timezone: 'America/Los_Angeles',
743743
fromTable: 'foo_tenant1',
744744
},
745-
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' },
745+
{ tableName: 'stb_pre_aggregations.bar_first20201228', timezone: 'America/Los_Angeles', fromTable: 'bar' },
746746

747747
{ tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
748748
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
749749
{ tableName: 'stb_pre_aggregations.foo_second20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
750750
{
751-
tableName: 'stb_pre_aggregations.foo_first20201228',
751+
tableName: 'stb_pre_aggregations.foo_first20201227',
752752
timezone: 'America/Los_Angeles',
753753
fromTable: 'foo_tenant1',
754754
},
755-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
755+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
756756
{
757-
tableName: 'stb_pre_aggregations.foo_second20201228',
757+
tableName: 'stb_pre_aggregations.foo_second20201227',
758758
timezone: 'America/Los_Angeles',
759759
fromTable: 'foo_tenant1',
760760
},
@@ -763,13 +763,13 @@ describe('Refresh Scheduler', () => {
763763
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
764764
{ tableName: 'stb_pre_aggregations.foo_second20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
765765
{
766-
tableName: 'stb_pre_aggregations.foo_first20201227',
766+
tableName: 'stb_pre_aggregations.foo_first20201226',
767767
timezone: 'America/Los_Angeles',
768768
fromTable: 'foo_tenant1',
769769
},
770-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
770+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201226', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
771771
{
772-
tableName: 'stb_pre_aggregations.foo_second20201227',
772+
tableName: 'stb_pre_aggregations.foo_second20201226',
773773
timezone: 'America/Los_Angeles',
774774
fromTable: 'foo_tenant1',
775775
},

0 commit comments

Comments
 (0)