Skip to content

Commit 2d1b850

Browse files
ovrovr
authored
fix(clickhouse-driver): Respect timezone for origin in custom granula… (#10110)
## Problem When using custom granularities with an `origin` timestamp, ClickHouse's `dateBin` implementation was not respecting time zones properly: ClickHouse aligns both timestamps internally to the same timezone before computing the difference, causing an unintended offset. As a result, the date difference and the final rounded time are incorrect, and the returned values are effectively in UTC.
1 parent 5a69f7b commit 2d1b850

File tree

11 files changed

+318
-35
lines changed

11 files changed

+318
-35
lines changed

‎packages/cubejs-schema-compiler/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@
8282
"source-map-support": "^0.5.19",
8383
"sqlstring": "^2.3.1",
8484
"testcontainers": "^10.28.0",
85-
"typescript": "~5.2.2"
85+
"typescript": "~5.2.2",
86+
"moment": "^2.30.1"
8687
},
8788
"license": "Apache-2.0",
8889
"eslintConfig": {

‎packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,20 +63,23 @@ export class ClickHouseQuery extends BaseQuery {
6363
}
6464

6565
/**
66-
* Returns sql for source expression floored to timestamps aligned with
67-
* intervals relative to origin timestamp point.
66+
* Returns SQL for source expression floored to timestamps aligned with
67+
* intervals relative to the origin timestamp point.
6868
*/
6969
public dateBin(interval: string, source: string, origin: string): string {
70+
// Pass timezone to dateTimeCast to ensure origin is in the same timezone as a source, because ClickHouse aligns
71+
// both timestamps internally to the same timezone before computing the difference, causing an unintended offset.
72+
const alignedOrigin = this.dateTimeCast(`'${origin}'`, this.timezone);
7073
const intervalFormatted = this.formatInterval(interval);
7174
const timeUnit = this.diffTimeUnitForInterval(interval);
7275
const beginOfTime = 'fromUnixTimestamp(0)';
7376

7477
return `date_add(${timeUnit},
7578
FLOOR(
76-
date_diff(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) /
79+
date_diff(${timeUnit}, ${alignedOrigin}, ${source}) /
7780
date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted})
7881
) * date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}),
79-
${this.dateTimeCast(`'${origin}'`)}
82+
${alignedOrigin}
8083
)`;
8184
}
8285

@@ -105,11 +108,17 @@ export class ClickHouseQuery extends BaseQuery {
105108
return this.dateTimeCast(value);
106109
}
107110

108-
public dateTimeCast(value: string): string {
111+
public dateTimeCast(value: string, timezone?: string): string {
112+
// This is critical for custom granularity, because timezone should be aligned between origin and source column, otherwise
113+
// clickhouse will align the source column to the origin timezone, which will cause an unintended offset.
114+
if (timezone) {
115+
// Use precision 3 for milliseconds to match the format 'YYYY-MM-DDTHH:mm:ss.SSS'
116+
return `toDateTime64(${value}, 3, '${timezone}')`;
117+
}
118+
109119
// value yields a string formatted in ISO8601, so this function returns a expression to parse a string to a DateTime
110120
// ClickHouse provides toDateTime which expects dates in UTC in format YYYY-MM-DD HH:MM:SS
111121
// However parseDateTimeBestEffort works with ISO8601
112-
//
113122
return `parseDateTimeBestEffort(${value})`;
114123
}
115124

‎packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { GenericContainer } from 'testcontainers';
44
import type { StartedTestContainer } from 'testcontainers';
55
import { format as formatSql } from 'sqlstring';
66
import { v4 as uuidv4 } from 'uuid';
7+
import moment from 'moment';
8+
79
import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery';
810
import { BaseDbRunner } from '../utils/BaseDbRunner';
911

@@ -210,10 +212,14 @@ export class ClickHouseDbRunner extends BaseDbRunner {
210212
if (fieldMeta === undefined) {
211213
throw new Error(`Missing meta for field ${field}`);
212214
}
213-
if (fieldMeta.type.includes('DateTime')) {
215+
216+
if (fieldMeta.type.includes('DateTime64')) {
217+
row[field] = moment.utc(value).format(moment.HTML5_FMT.DATETIME_LOCAL_MS);
218+
} else if (fieldMeta.type.includes('DateTime') /** Can be DateTime or DateTime('timezone') */) {
214219
if (typeof value !== 'string') {
215220
throw new Error(`Unexpected value for ${field}`);
216221
}
222+
217223
row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
218224
} else if (fieldMeta.type.includes('Date')) {
219225
row[field] = `${value}T00:00:00.000`;

‎packages/cubejs-schema-compiler/test/integration/clickhouse/custom-granularities.test.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,13 @@ describe('Custom Granularities', () => {
5858
- name: fifteen_days_hours_minutes_seconds
5959
interval: 15 days 3 hours 25 minutes 40 seconds
6060
origin: '2024-01-01 10:15:00'
61+
- name: five_minutes_from_utc_origin
62+
interval: 5 minutes
63+
# 10:15 UTC = 11:15 Paris time (UTC+1)
64+
origin: '2024-01-01T10:15:00Z'
65+
- name: five_minutes_from_local_origin
66+
interval: 5 minutes
67+
origin: '2024-01-01 10:15:00'
6168
- name: fiscal_year_by_1st_feb
6269
interval: 1 year
6370
origin: '2024-02-01'
@@ -564,4 +571,62 @@ describe('Custom Granularities', () => {
564571
],
565572
{ joinGraph, cubeEvaluator, compiler }
566573
));
574+
575+
it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
576+
{
577+
measures: ['orders.count'],
578+
timeDimensions: [{
579+
dimension: 'orders.createdAt',
580+
granularity: 'five_minutes_from_utc_origin',
581+
dateRange: ['2024-01-01', '2024-01-31']
582+
}],
583+
dimensions: [],
584+
filters: [],
585+
timezone: 'Europe/Paris'
586+
},
587+
[
588+
{
589+
orders__count: '1',
590+
orders__created_at_five_minutes_from_utc_origin: '2024-01-01T01:00:00.000',
591+
},
592+
{
593+
orders__count: '1',
594+
orders__created_at_five_minutes_from_utc_origin: '2024-01-15T01:00:00.000',
595+
},
596+
{
597+
orders__count: '1',
598+
orders__created_at_five_minutes_from_utc_origin: '2024-01-29T01:00:00.000',
599+
},
600+
],
601+
{ joinGraph, cubeEvaluator, compiler }
602+
));
603+
604+
it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
605+
{
606+
measures: ['orders.count'],
607+
timeDimensions: [{
608+
dimension: 'orders.createdAt',
609+
granularity: 'five_minutes_from_local_origin',
610+
dateRange: ['2024-01-01', '2024-01-31']
611+
}],
612+
dimensions: [],
613+
filters: [],
614+
timezone: 'Europe/Paris'
615+
},
616+
[
617+
{
618+
orders__count: '1',
619+
orders__created_at_five_minutes_from_local_origin: '2024-01-01T01:00:00.000',
620+
},
621+
{
622+
orders__count: '1',
623+
orders__created_at_five_minutes_from_local_origin: '2024-01-15T01:00:00.000',
624+
},
625+
{
626+
orders__count: '1',
627+
orders__created_at_five_minutes_from_local_origin: '2024-01-29T01:00:00.000',
628+
},
629+
],
630+
{ joinGraph, cubeEvaluator, compiler }
631+
));
567632
});

‎packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@ describe('Custom Granularities', () => {
5151
- name: twenty_five_minutes
5252
interval: 25 minutes
5353
origin: '2024-01-01 10:15:00'
54+
- name: five_minutes_from_utc_origin
55+
interval: 5 minutes
56+
# 10:15 UTC = 11:15 Paris time (UTC+1)
57+
origin: '2024-01-01T10:15:00Z'
58+
- name: five_minutes_from_local_origin
59+
interval: 5 minutes
60+
origin: '2024-01-01 10:15:00'
5461
- name: fifteen_days_hours_minutes_seconds
5562
interval: 15 days 3 hours 25 minutes 40 seconds
5663
origin: '2024-01-01 10:15:00'
@@ -151,6 +158,66 @@ describe('Custom Granularities', () => {
151158
{ joinGraph, cubeEvaluator, compiler }
152159
));
153160

161+
/// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z
162+
xit('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
163+
{
164+
measures: ['orders.count'],
165+
timeDimensions: [{
166+
dimension: 'orders.createdAt',
167+
granularity: 'five_minutes_from_utc_origin',
168+
dateRange: ['2024-01-01', '2024-01-31']
169+
}],
170+
dimensions: [],
171+
filters: [],
172+
timezone: 'Europe/Paris'
173+
},
174+
[
175+
{
176+
orders__count: 1,
177+
orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-01T01:00:00.000Z'),
178+
},
179+
{
180+
orders__count: 1,
181+
orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-15T01:00:00.000Z'),
182+
},
183+
{
184+
orders__count: 1,
185+
orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-29T01:00:00.000Z'),
186+
},
187+
],
188+
{ joinGraph, cubeEvaluator, compiler }
189+
));
190+
191+
/// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z
192+
xit('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
193+
{
194+
measures: ['orders.count'],
195+
timeDimensions: [{
196+
dimension: 'orders.createdAt',
197+
granularity: 'five_minutes_from_local_origin',
198+
dateRange: ['2024-01-01', '2024-01-31']
199+
}],
200+
dimensions: [],
201+
filters: [],
202+
timezone: 'Europe/Paris'
203+
},
204+
[
205+
{
206+
orders__count: 1,
207+
orders__created_at_five_minutes_from_local_origin: new Date('2024-01-01T01:00:00.000Z'),
208+
},
209+
{
210+
orders__count: 1,
211+
orders__created_at_five_minutes_from_local_origin: new Date('2024-01-15T01:00:00.000Z'),
212+
},
213+
{
214+
orders__count: 1,
215+
orders__created_at_five_minutes_from_local_origin: new Date('2024-01-29T01:00:00.000Z'),
216+
},
217+
],
218+
{ joinGraph, cubeEvaluator, compiler }
219+
));
220+
154221
it('works with half_year custom granularity with dimension query', async () => dbRunner.runQueryTest(
155222
{
156223
measures: ['orders.count'],

‎packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ describe('MSSqlPreAggregations', () => {
270270

271271
expect(preAggregationsDescription[0].invalidateKeyQueries[0][0].replace(/(\r\n|\n|\r)/gm, '')
272272
.replace(/\s+/g, ' '))
273-
.toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END');
273+
.toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-28800 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key');
274274

275275
return dbRunner
276276
.evaluateQueryWithPreAggregations(query)

‎packages/cubejs-schema-compiler/test/integration/mysql/custom-granularities.test.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@ describe('Custom Granularities', () => {
5151
- name: twenty_five_minutes
5252
interval: 25 minutes
5353
origin: '2024-01-01 10:15:00'
54+
- name: five_minutes_from_utc_origin
55+
interval: 5 minutes
56+
# 10:15 UTC = 11:15 Paris time (UTC+1)
57+
origin: '2024-01-01T10:15:00Z'
58+
- name: five_minutes_from_local_origin
59+
interval: 5 minutes
60+
origin: '2024-01-01 10:15:00'
5461
- name: fifteen_days_hours_minutes_seconds
5562
interval: 15 days 3 hours 25 minutes 40 seconds
5663
origin: '2024-01-01 10:15:00'
@@ -151,6 +158,64 @@ describe('Custom Granularities', () => {
151158
{ joinGraph, cubeEvaluator, compiler }
152159
));
153160

161+
it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
162+
{
163+
measures: ['orders.count'],
164+
timeDimensions: [{
165+
dimension: 'orders.createdAt',
166+
granularity: 'five_minutes_from_utc_origin',
167+
dateRange: ['2024-01-01', '2024-01-31']
168+
}],
169+
dimensions: [],
170+
filters: [],
171+
timezone: 'Europe/Paris'
172+
},
173+
[
174+
{
175+
orders__count: 1,
176+
orders__created_at_five_minutes_from_utc_origin: '2024-01-01 01:00:00.000',
177+
},
178+
{
179+
orders__count: 1,
180+
orders__created_at_five_minutes_from_utc_origin: '2024-01-15 01:00:00.000',
181+
},
182+
{
183+
orders__count: 1,
184+
orders__created_at_five_minutes_from_utc_origin: '2024-01-29 01:00:00.000',
185+
},
186+
],
187+
{ joinGraph, cubeEvaluator, compiler }
188+
));
189+
190+
it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
191+
{
192+
measures: ['orders.count'],
193+
timeDimensions: [{
194+
dimension: 'orders.createdAt',
195+
granularity: 'five_minutes_from_local_origin',
196+
dateRange: ['2024-01-01', '2024-01-31']
197+
}],
198+
dimensions: [],
199+
filters: [],
200+
timezone: 'Europe/Paris'
201+
},
202+
[
203+
{
204+
orders__count: 1,
205+
orders__created_at_five_minutes_from_local_origin: '2024-01-01 01:00:00.000',
206+
},
207+
{
208+
orders__count: 1,
209+
orders__created_at_five_minutes_from_local_origin: '2024-01-15 01:00:00.000',
210+
},
211+
{
212+
orders__count: 1,
213+
orders__created_at_five_minutes_from_local_origin: '2024-01-29 01:00:00.000',
214+
},
215+
],
216+
{ joinGraph, cubeEvaluator, compiler }
217+
));
218+
154219
it('works with half_year custom granularity with dimension query', async () => dbRunner.runQueryTest(
155220
{
156221
measures: ['orders.count'],

0 commit comments

Comments
 (0)