Skip to content

Commit 120d17d

Browse files
KSDaemonKSDaemon
committed
make loadBuildRange() timezone-aware + align tests
1 parent 90f6f27 commit 120d17d

File tree

3 files changed

+64
-47
lines changed

3 files changed

+64
-47
lines changed

‎packages/cubejs-backend-shared/src/time.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,13 @@ export const inDbTimeZone = (timezone: string, timestampFormat: string, timestam
227227
return moment.tz(timestamp, timezone).utc().format(timestampFormat);
228228
};
229229

230-
export const utcToLocalTimeZone = (timezone: string, timestampFormat: string, timestamp: string): string => {
230+
/**
231+
* Takes timestamp in UTC, treat it as local time in provided timezone and returns the corresponding timestamp in UTC
232+
*/
233+
export const utcToLocalTimeZoneInUtc = (timezone: string, timestampFormat: string, timestamp: string): string | null => {
234+
if (!timestamp) {
235+
return null;
236+
}
231237
if (timestamp.length === 23) {
232238
const zone = moment.tz.zone(timezone);
233239
if (!zone) {

‎packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
TO_PARTITION_RANGE,
77
MAX_SOURCE_ROW_LIMIT,
88
reformatInIsoLocal,
9-
utcToLocalTimeZone,
9+
utcToLocalTimeZoneInUtc,
1010
timeSeries,
1111
inDbTimeZone,
1212
extractDate
@@ -136,9 +136,9 @@ export class PreAggregationPartitionRangeLoader {
136136
return queryValues?.map(
137137
param => {
138138
if (param === BUILD_RANGE_START_LOCAL) {
139-
return utcToLocalTimeZone(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeStart);
139+
return utcToLocalTimeZoneInUtc(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeStart);
140140
} else if (param === BUILD_RANGE_END_LOCAL) {
141-
return utcToLocalTimeZone(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeEnd);
141+
return utcToLocalTimeZoneInUtc(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeEnd);
142142
} else {
143143
return param;
144144
}
@@ -396,31 +396,42 @@ export class PreAggregationPartitionRangeLoader {
396396
const { preAggregationStartEndQueries } = this.preAggregation;
397397
const [startDate, endDate] = await Promise.all(
398398
preAggregationStartEndQueries.map(
399-
async rangeQuery => PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery)),
399+
async rangeQuery => utcToLocalTimeZoneInUtc(
400+
this.preAggregation.timezone,
401+
'YYYY-MM-DDTHH:mm:ss.SSS',
402+
PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery)),
403+
)
400404
),
401405
);
406+
402407
if (!this.preAggregation.partitionGranularity) {
403408
return this.orNowIfEmpty([startDate, endDate]);
404409
}
410+
411+
// startDate & endDate are `localized` here
405412
const wholeSeriesRanges = PreAggregationPartitionRangeLoader.timeSeries(
406413
this.preAggregation.partitionGranularity,
407414
this.orNowIfEmpty([startDate, endDate]),
408415
this.preAggregation.timestampPrecision,
409416
);
410417
const [rangeStart, rangeEnd] = await Promise.all(
411418
preAggregationStartEndQueries.map(
412-
async (rangeQuery, i) => PreAggregationPartitionRangeLoader.extractDate(
413-
await this.loadRangeQuery(
414-
rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1],
419+
async (rangeQuery, i) => utcToLocalTimeZoneInUtc(
420+
this.preAggregation.timezone,
421+
'YYYY-MM-DDTHH:mm:ss.SSS',
422+
PreAggregationPartitionRangeLoader.extractDate(
423+
await this.loadRangeQuery(
424+
rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1],
425+
),
415426
),
416-
),
427+
)
417428
),
418429
);
419430
return this.orNowIfEmpty([rangeStart, rangeEnd]);
420431
}
421432

422433
private now() {
423-
return utcToLocalTimeZone(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
434+
return utcToLocalTimeZoneInUtc(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
424435
}
425436

426437
private orNowIfEmpty(dateRange: QueryDateRange): QueryDateRange {

‎packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,25 @@ import { CompilerApi } from '../../src/core/CompilerApi';
88
const schemaContent = `
99
cube('Foo', {
1010
sql: \`select * from foo_\${SECURITY_CONTEXT.tenantId.unsafeValue()}\`,
11-
11+
1212
measures: {
1313
count: {
1414
type: 'count'
1515
},
16-
16+
1717
total: {
1818
sql: 'amount',
1919
type: 'sum'
2020
},
2121
},
22-
22+
2323
dimensions: {
2424
time: {
2525
sql: 'timestamp',
2626
type: 'time'
2727
}
2828
},
29-
29+
3030
preAggregations: {
3131
main: {
3232
type: 'originalSql',
@@ -87,20 +87,20 @@ cube('Foo', {
8787
8888
cube('Bar', {
8989
sql: 'select * from bar',
90-
90+
9191
measures: {
9292
count: {
9393
type: 'count'
9494
}
9595
},
96-
96+
9797
dimensions: {
9898
time: {
9999
sql: 'timestamp',
100100
type: 'time'
101101
}
102102
},
103-
103+
104104
preAggregations: {
105105
first: {
106106
type: 'rollup',
@@ -131,63 +131,63 @@ const repositoryWithRollupJoin: SchemaFileRepository = {
131131
{ fileName: 'main.js', content: `
132132
cube(\`Users\`, {
133133
sql: \`SELECT * FROM public.users\`,
134-
134+
135135
preAggregations: {
136136
usersRollup: {
137137
dimensions: [CUBE.id],
138138
},
139139
},
140-
140+
141141
measures: {
142142
count: {
143143
type: \`count\`,
144144
},
145145
},
146-
146+
147147
dimensions: {
148148
id: {
149149
sql: \`id\`,
150150
type: \`string\`,
151151
primaryKey: true,
152152
},
153-
153+
154154
name: {
155155
sql: \`name\`,
156156
type: \`string\`,
157157
},
158158
},
159159
});
160-
160+
161161
cube('Orders', {
162162
sql: \`SELECT * FROM orders\`,
163-
163+
164164
preAggregations: {
165165
ordersRollup: {
166166
measures: [CUBE.count],
167167
dimensions: [CUBE.userId, CUBE.status],
168168
},
169-
169+
170170
ordersRollupJoin: {
171171
type: \`rollupJoin\`,
172172
measures: [CUBE.count],
173173
dimensions: [Users.name],
174174
rollups: [Users.usersRollup, CUBE.ordersRollup],
175175
},
176176
},
177-
177+
178178
joins: {
179179
Users: {
180180
relationship: \`belongsTo\`,
181181
sql: \`\${CUBE.userId} = \${Users.id}\`,
182182
},
183183
},
184-
184+
185185
measures: {
186186
count: {
187187
type: \`count\`,
188188
},
189189
},
190-
190+
191191
dimensions: {
192192
id: {
193193
sql: \`id\`,
@@ -215,13 +215,13 @@ const repositoryWithoutPreAggregations: SchemaFileRepository = {
215215
fileName: 'main.js', content: `
216216
cube('Bar', {
217217
sql: 'select * from bar',
218-
218+
219219
measures: {
220220
count: {
221221
type: 'count'
222222
}
223223
},
224-
224+
225225
dimensions: {
226226
time: {
227227
sql: 'timestamp',
@@ -698,63 +698,63 @@ describe('Refresh Scheduler', () => {
698698
{ tableName: 'stb_pre_aggregations.foo_second20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
699699
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' },
700700
{
701-
tableName: 'stb_pre_aggregations.foo_first20201231',
701+
tableName: 'stb_pre_aggregations.foo_first20201230',
702702
timezone: 'America/Los_Angeles',
703703
fromTable: 'foo_tenant1',
704704
},
705-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
705+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
706706
{
707-
tableName: 'stb_pre_aggregations.foo_second20201231',
707+
tableName: 'stb_pre_aggregations.foo_second20201230',
708708
timezone: 'America/Los_Angeles',
709709
fromTable: 'foo_tenant1',
710710
},
711-
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'America/Los_Angeles', fromTable: 'bar' },
711+
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' },
712712

713713
{ tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
714714
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
715715
{ tableName: 'stb_pre_aggregations.foo_second20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
716716
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'UTC', fromTable: 'bar' },
717717
{
718-
tableName: 'stb_pre_aggregations.foo_first20201230',
718+
tableName: 'stb_pre_aggregations.foo_first20201229',
719719
timezone: 'America/Los_Angeles',
720720
fromTable: 'foo_tenant1',
721721
},
722-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
722+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
723723
{
724-
tableName: 'stb_pre_aggregations.foo_second20201230',
724+
tableName: 'stb_pre_aggregations.foo_second20201229',
725725
timezone: 'America/Los_Angeles',
726726
fromTable: 'foo_tenant1',
727727
},
728-
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' },
728+
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' },
729729

730730
{ tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
731731
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
732732
{ tableName: 'stb_pre_aggregations.foo_second20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
733733
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' },
734734
{
735-
tableName: 'stb_pre_aggregations.foo_first20201229',
735+
tableName: 'stb_pre_aggregations.foo_first20201228',
736736
timezone: 'America/Los_Angeles',
737737
fromTable: 'foo_tenant1',
738738
},
739-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
739+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
740740
{
741-
tableName: 'stb_pre_aggregations.foo_second20201229',
741+
tableName: 'stb_pre_aggregations.foo_second20201228',
742742
timezone: 'America/Los_Angeles',
743743
fromTable: 'foo_tenant1',
744744
},
745-
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' },
745+
{ tableName: 'stb_pre_aggregations.bar_first20201228', timezone: 'America/Los_Angeles', fromTable: 'bar' },
746746

747747
{ tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
748748
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
749749
{ tableName: 'stb_pre_aggregations.foo_second20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
750750
{
751-
tableName: 'stb_pre_aggregations.foo_first20201228',
751+
tableName: 'stb_pre_aggregations.foo_first20201227',
752752
timezone: 'America/Los_Angeles',
753753
fromTable: 'foo_tenant1',
754754
},
755-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
755+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
756756
{
757-
tableName: 'stb_pre_aggregations.foo_second20201228',
757+
tableName: 'stb_pre_aggregations.foo_second20201227',
758758
timezone: 'America/Los_Angeles',
759759
fromTable: 'foo_tenant1',
760760
},
@@ -763,13 +763,13 @@ describe('Refresh Scheduler', () => {
763763
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
764764
{ tableName: 'stb_pre_aggregations.foo_second20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
765765
{
766-
tableName: 'stb_pre_aggregations.foo_first20201227',
766+
tableName: 'stb_pre_aggregations.foo_first20201226',
767767
timezone: 'America/Los_Angeles',
768768
fromTable: 'foo_tenant1',
769769
},
770-
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
770+
{ tableName: 'stb_pre_aggregations.foo_orphaned20201226', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
771771
{
772-
tableName: 'stb_pre_aggregations.foo_second20201227',
772+
tableName: 'stb_pre_aggregations.foo_second20201226',
773773
timezone: 'America/Los_Angeles',
774774
fromTable: 'foo_tenant1',
775775
},

0 commit comments

Comments
 (0)