fix(clickhouse-driver): Respect timezone for origin in custom granula…

packages/cubejs-schema-compiler/package.json

@@ -82,7 +82,8 @@
     "source-map-support": "^0.5.19",
     "sqlstring": "^2.3.1",
     "testcontainers": "^10.28.0",
-    "typescript": "~5.2.2"
+    "typescript": "~5.2.2",
+    "moment": "^2.30.1"
   },
   "license": "Apache-2.0",
   "eslintConfig": {

packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts

@@ -63,20 +63,23 @@ export class ClickHouseQuery extends BaseQuery {
   }
 
   /**
-   * Returns sql for source expression floored to timestamps aligned with
-   * intervals relative to origin timestamp point.
+   * Returns SQL for source expression floored to timestamps aligned with
+   * intervals relative to the origin timestamp point.
    */
   public dateBin(interval: string, source: string, origin: string): string {
+    // Pass timezone to dateTimeCast to ensure origin is in the same timezone as a source, because ClickHouse aligns
+    // both timestamps internally to the same timezone before computing the difference, causing an unintended offset.
+    const alignedOrigin = this.dateTimeCast(`'${origin}'`, this.timezone);
     const intervalFormatted = this.formatInterval(interval);
     const timeUnit = this.diffTimeUnitForInterval(interval);
     const beginOfTime = 'fromUnixTimestamp(0)';
 
     return `date_add(${timeUnit},
         FLOOR(
-          date_diff(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) /
+          date_diff(${timeUnit}, ${alignedOrigin}, ${source}) /
           date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted})
         ) * date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}),
-        ${this.dateTimeCast(`'${origin}'`)}
+        ${alignedOrigin}
     )`;
   }
 
@@ -105,11 +108,17 @@ export class ClickHouseQuery extends BaseQuery {
     return this.dateTimeCast(value);
   }
 
-  public dateTimeCast(value: string): string {
+  public dateTimeCast(value: string, timezone?: string): string {
+    // This is critical for custom granularity, because timezone should be aligned between origin and source column, otherwise
+    // clickhouse will align the source column to the origin timezone, which will cause an unintended offset.
+    if (timezone) {
+      // Use precision 3 for milliseconds to match the format 'YYYY-MM-DDTHH:mm:ss.SSS'
+      return `toDateTime64(${value}, 3, '${timezone}')`;
+    }
+
     // value yields a string formatted in ISO8601, so this function returns a expression to parse a string to a DateTime
     // ClickHouse provides toDateTime which expects dates in UTC in format YYYY-MM-DD HH:MM:SS
     // However parseDateTimeBestEffort works with ISO8601
-    //
     return `parseDateTimeBestEffort(${value})`;
   }
 

packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts

@@ -4,6 +4,8 @@ import { GenericContainer } from 'testcontainers';
 import type { StartedTestContainer } from 'testcontainers';
 import { format as formatSql } from 'sqlstring';
 import { v4 as uuidv4 } from 'uuid';
+import moment from 'moment';
+
 import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery';
 import { BaseDbRunner } from '../utils/BaseDbRunner';
 
@@ -210,10 +212,14 @@ export class ClickHouseDbRunner extends BaseDbRunner {
           if (fieldMeta === undefined) {
             throw new Error(`Missing meta for field ${field}`);
           }
-          if (fieldMeta.type.includes('DateTime')) {
+
+          if (fieldMeta.type.includes('DateTime64')) {
+            row[field] = moment.utc(value).format(moment.HTML5_FMT.DATETIME_LOCAL_MS);
+          } else if (fieldMeta.type.includes('DateTime') /** Can be DateTime or DateTime('timezone') */) {
             if (typeof value !== 'string') {
               throw new Error(`Unexpected value for ${field}`);
             }
+
             row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
           } else if (fieldMeta.type.includes('Date')) {
             row[field] = `${value}T00:00:00.000`;

packages/cubejs-schema-compiler/test/integration/clickhouse/custom-granularities.test.ts

@@ -58,6 +58,13 @@ describe('Custom Granularities', () => {
             - name: fifteen_days_hours_minutes_seconds
               interval: 15 days 3 hours 25 minutes 40 seconds
               origin: '2024-01-01 10:15:00'
+            - name: five_minutes_from_utc_origin
+              interval: 5 minutes
+              # 10:15 UTC = 11:15 Paris time (UTC+1)
+              origin: '2024-01-01T10:15:00Z'
+            - name: five_minutes_from_local_origin
+              interval: 5 minutes
+              origin: '2024-01-01 10:15:00'
             - name: fiscal_year_by_1st_feb
               interval: 1 year
               origin: '2024-02-01'
@@ -564,4 +571,62 @@ describe('Custom Granularities', () => {
     ],
     { joinGraph, cubeEvaluator, compiler }
   ));
+
+  it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
+    {
+      measures: ['orders.count'],
+      timeDimensions: [{
+        dimension: 'orders.createdAt',
+        granularity: 'five_minutes_from_utc_origin',
+        dateRange: ['2024-01-01', '2024-01-31']
+      }],
+      dimensions: [],
+      filters: [],
+      timezone: 'Europe/Paris'
+    },
+    [
+      {
+        orders__count: '1',
+        orders__created_at_five_minutes_from_utc_origin: '2024-01-01T01:00:00.000',
+      },
+      {
+        orders__count: '1',
+        orders__created_at_five_minutes_from_utc_origin: '2024-01-15T01:00:00.000',
+      },
+      {
+        orders__count: '1',
+        orders__created_at_five_minutes_from_utc_origin: '2024-01-29T01:00:00.000',
+      },
+    ],
+    { joinGraph, cubeEvaluator, compiler }
+  ));
+
+  it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
+    {
+      measures: ['orders.count'],
+      timeDimensions: [{
+        dimension: 'orders.createdAt',
+        granularity: 'five_minutes_from_local_origin',
+        dateRange: ['2024-01-01', '2024-01-31']
+      }],
+      dimensions: [],
+      filters: [],
+      timezone: 'Europe/Paris'
+    },
+    [
+      {
+        orders__count: '1',
+        orders__created_at_five_minutes_from_local_origin: '2024-01-01T01:00:00.000',
+      },
+      {
+        orders__count: '1',
+        orders__created_at_five_minutes_from_local_origin: '2024-01-15T01:00:00.000',
+      },
+      {
+        orders__count: '1',
+        orders__created_at_five_minutes_from_local_origin: '2024-01-29T01:00:00.000',
+      },
+    ],
+    { joinGraph, cubeEvaluator, compiler }
+  ));
 });

packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts

@@ -51,6 +51,13 @@ describe('Custom Granularities', () => {
             - name: twenty_five_minutes
               interval: 25 minutes
               origin: '2024-01-01 10:15:00'
+            - name: five_minutes_from_utc_origin
+              interval: 5 minutes
+              # 10:15 UTC = 11:15 Paris time (UTC+1)
+              origin: '2024-01-01T10:15:00Z'
+            - name: five_minutes_from_local_origin
+              interval: 5 minutes
+              origin: '2024-01-01 10:15:00'
             - name: fifteen_days_hours_minutes_seconds
               interval: 15 days 3 hours 25 minutes 40 seconds
               origin: '2024-01-01 10:15:00'
@@ -151,6 +158,66 @@ describe('Custom Granularities', () => {
     { joinGraph, cubeEvaluator, compiler }
   ));
 
+  /// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z
+  xit('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
+    {
+      measures: ['orders.count'],
+      timeDimensions: [{
+        dimension: 'orders.createdAt',
+        granularity: 'five_minutes_from_utc_origin',
+        dateRange: ['2024-01-01', '2024-01-31']
+      }],
+      dimensions: [],
+      filters: [],
+      timezone: 'Europe/Paris'
+    },
+    [
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-01T01:00:00.000Z'),
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-15T01:00:00.000Z'),
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-29T01:00:00.000Z'),
+      },
+    ],
+    { joinGraph, cubeEvaluator, compiler }
+  ));
+
+  /// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z
+  xit('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
+    {
+      measures: ['orders.count'],
+      timeDimensions: [{
+        dimension: 'orders.createdAt',
+        granularity: 'five_minutes_from_local_origin',
+        dateRange: ['2024-01-01', '2024-01-31']
+      }],
+      dimensions: [],
+      filters: [],
+      timezone: 'Europe/Paris'
+    },
+    [
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_local_origin: new Date('2024-01-01T01:00:00.000Z'),
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_local_origin: new Date('2024-01-15T01:00:00.000Z'),
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_local_origin: new Date('2024-01-29T01:00:00.000Z'),
+      },
+    ],
+    { joinGraph, cubeEvaluator, compiler }
+  ));
+
   it('works with half_year custom granularity with dimension query', async () => dbRunner.runQueryTest(
     {
       measures: ['orders.count'],

packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts

@@ -270,7 +270,7 @@ describe('MSSqlPreAggregations', () => {
 
       expect(preAggregationsDescription[0].invalidateKeyQueries[0][0].replace(/(\r\n|\n|\r)/gm, '')
         .replace(/\s+/g, ' '))
-        .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END');
+        .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-28800 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key');
 
       return dbRunner
         .evaluateQueryWithPreAggregations(query)

packages/cubejs-schema-compiler/test/integration/mysql/custom-granularities.test.ts

@@ -51,6 +51,13 @@ describe('Custom Granularities', () => {
             - name: twenty_five_minutes
               interval: 25 minutes
               origin: '2024-01-01 10:15:00'
+            - name: five_minutes_from_utc_origin
+              interval: 5 minutes
+              # 10:15 UTC = 11:15 Paris time (UTC+1)
+              origin: '2024-01-01T10:15:00Z'
+            - name: five_minutes_from_local_origin
+              interval: 5 minutes
+              origin: '2024-01-01 10:15:00'
             - name: fifteen_days_hours_minutes_seconds
               interval: 15 days 3 hours 25 minutes 40 seconds
               origin: '2024-01-01 10:15:00'
@@ -151,6 +158,64 @@ describe('Custom Granularities', () => {
     { joinGraph, cubeEvaluator, compiler }
   ));
 
+  it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
+    {
+      measures: ['orders.count'],
+      timeDimensions: [{
+        dimension: 'orders.createdAt',
+        granularity: 'five_minutes_from_utc_origin',
+        dateRange: ['2024-01-01', '2024-01-31']
+      }],
+      dimensions: [],
+      filters: [],
+      timezone: 'Europe/Paris'
+    },
+    [
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_utc_origin: '2024-01-01 01:00:00.000',
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_utc_origin: '2024-01-15 01:00:00.000',
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_utc_origin: '2024-01-29 01:00:00.000',
+      },
+    ],
+    { joinGraph, cubeEvaluator, compiler }
+  ));
+
+  it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest(
+    {
+      measures: ['orders.count'],
+      timeDimensions: [{
+        dimension: 'orders.createdAt',
+        granularity: 'five_minutes_from_local_origin',
+        dateRange: ['2024-01-01', '2024-01-31']
+      }],
+      dimensions: [],
+      filters: [],
+      timezone: 'Europe/Paris'
+    },
+    [
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_local_origin: '2024-01-01 01:00:00.000',
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_local_origin: '2024-01-15 01:00:00.000',
+      },
+      {
+        orders__count: 1,
+        orders__created_at_five_minutes_from_local_origin: '2024-01-29 01:00:00.000',
+      },
+    ],
+    { joinGraph, cubeEvaluator, compiler }
+  ));
+
   it('works with half_year custom granularity with dimension query', async () => dbRunner.runQueryTest(
     {
       measures: ['orders.count'],