diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index 9e25ab219b635..938f605cfdf85 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -82,7 +82,8 @@ "source-map-support": "^0.5.19", "sqlstring": "^2.3.1", "testcontainers": "^10.28.0", - "typescript": "~5.2.2" + "typescript": "~5.2.2", + "moment": "^2.30.1" }, "license": "Apache-2.0", "eslintConfig": { diff --git a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts index 8c3cdaedea6b1..0c392ea888c00 100644 --- a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts @@ -63,20 +63,23 @@ export class ClickHouseQuery extends BaseQuery { } /** - * Returns sql for source expression floored to timestamps aligned with - * intervals relative to origin timestamp point. + * Returns SQL for source expression floored to timestamps aligned with + * intervals relative to the origin timestamp point. */ public dateBin(interval: string, source: string, origin: string): string { + // Pass timezone to dateTimeCast to ensure origin is in the same timezone as a source, because ClickHouse aligns + // both timestamps internally to the same timezone before computing the difference, causing an unintended offset. + const alignedOrigin = this.dateTimeCast(`'${origin}'`, this.timezone); const intervalFormatted = this.formatInterval(interval); const timeUnit = this.diffTimeUnitForInterval(interval); const beginOfTime = 'fromUnixTimestamp(0)'; return `date_add(${timeUnit}, FLOOR( - date_diff(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) / + date_diff(${timeUnit}, ${alignedOrigin}, ${source}) / date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}) ) * date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}), - ${this.dateTimeCast(`'${origin}'`)} + ${alignedOrigin} )`; } @@ -105,11 +108,17 @@ export class ClickHouseQuery extends BaseQuery { return this.dateTimeCast(value); } - public dateTimeCast(value: string): string { + public dateTimeCast(value: string, timezone?: string): string { + // This is critical for custom granularity, because timezone should be aligned between origin and source column, otherwise + // clickhouse will align the source column to the origin timezone, which will cause an unintended offset. + if (timezone) { + // Use precision 3 for milliseconds to match the format 'YYYY-MM-DDTHH:mm:ss.SSS' + return `toDateTime64(${value}, 3, '${timezone}')`; + } + // value yields a string formatted in ISO8601, so this function returns a expression to parse a string to a DateTime // ClickHouse provides toDateTime which expects dates in UTC in format YYYY-MM-DD HH:MM:SS // However parseDateTimeBestEffort works with ISO8601 - // return `parseDateTimeBestEffort(${value})`; } diff --git a/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts b/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts index 296a224446a57..67138b027802e 100644 --- a/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts +++ b/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts @@ -4,6 +4,8 @@ import { GenericContainer } from 'testcontainers'; import type { StartedTestContainer } from 'testcontainers'; import { format as formatSql } from 'sqlstring'; import { v4 as uuidv4 } from 'uuid'; +import moment from 'moment'; + import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery'; import { BaseDbRunner } from '../utils/BaseDbRunner'; @@ -210,10 +212,14 @@ export class ClickHouseDbRunner extends BaseDbRunner { if (fieldMeta === undefined) { throw new Error(`Missing meta for field ${field}`); } - if (fieldMeta.type.includes('DateTime')) { + + if (fieldMeta.type.includes('DateTime64')) { + row[field] = moment.utc(value).format(moment.HTML5_FMT.DATETIME_LOCAL_MS); + } else if (fieldMeta.type.includes('DateTime') /** Can be DateTime or DateTime('timezone') */) { if (typeof value !== 'string') { throw new Error(`Unexpected value for ${field}`); } + row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`; } else if (fieldMeta.type.includes('Date')) { row[field] = `${value}T00:00:00.000`; diff --git a/packages/cubejs-schema-compiler/test/integration/clickhouse/custom-granularities.test.ts b/packages/cubejs-schema-compiler/test/integration/clickhouse/custom-granularities.test.ts index 2d8a1665d66b6..e25084afe1829 100644 --- a/packages/cubejs-schema-compiler/test/integration/clickhouse/custom-granularities.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/clickhouse/custom-granularities.test.ts @@ -58,6 +58,13 @@ describe('Custom Granularities', () => { - name: fifteen_days_hours_minutes_seconds interval: 15 days 3 hours 25 minutes 40 seconds origin: '2024-01-01 10:15:00' + - name: five_minutes_from_utc_origin + interval: 5 minutes + # 10:15 UTC = 11:15 Paris time (UTC+1) + origin: '2024-01-01T10:15:00Z' + - name: five_minutes_from_local_origin + interval: 5 minutes + origin: '2024-01-01 10:15:00' - name: fiscal_year_by_1st_feb interval: 1 year origin: '2024-02-01' @@ -564,4 +571,62 @@ describe('Custom Granularities', () => { ], { joinGraph, cubeEvaluator, compiler } )); + + it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_utc_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: '1', + orders__created_at_five_minutes_from_utc_origin: '2024-01-01T01:00:00.000', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_utc_origin: '2024-01-15T01:00:00.000', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_utc_origin: '2024-01-29T01:00:00.000', + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); + + it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_local_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: '1', + orders__created_at_five_minutes_from_local_origin: '2024-01-01T01:00:00.000', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_local_origin: '2024-01-15T01:00:00.000', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_local_origin: '2024-01-29T01:00:00.000', + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); }); diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts index 922b37e48b80a..24f2f4e2ccb7d 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts @@ -51,6 +51,13 @@ describe('Custom Granularities', () => { - name: twenty_five_minutes interval: 25 minutes origin: '2024-01-01 10:15:00' + - name: five_minutes_from_utc_origin + interval: 5 minutes + # 10:15 UTC = 11:15 Paris time (UTC+1) + origin: '2024-01-01T10:15:00Z' + - name: five_minutes_from_local_origin + interval: 5 minutes + origin: '2024-01-01 10:15:00' - name: fifteen_days_hours_minutes_seconds interval: 15 days 3 hours 25 minutes 40 seconds origin: '2024-01-01 10:15:00' @@ -151,6 +158,66 @@ describe('Custom Granularities', () => { { joinGraph, cubeEvaluator, compiler } )); + /// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z + xit('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_utc_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: 1, + orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-01T01:00:00.000Z'), + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-15T01:00:00.000Z'), + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_utc_origin: new Date('2024-01-29T01:00:00.000Z'), + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); + + /// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z + xit('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_local_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: 1, + orders__created_at_five_minutes_from_local_origin: new Date('2024-01-01T01:00:00.000Z'), + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_local_origin: new Date('2024-01-15T01:00:00.000Z'), + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_local_origin: new Date('2024-01-29T01:00:00.000Z'), + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); + it('works with half_year custom granularity with dimension query', async () => dbRunner.runQueryTest( { measures: ['orders.count'], diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts index c9aeedda541ad..3dd2680884fc4 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts @@ -270,7 +270,7 @@ describe('MSSqlPreAggregations', () => { expect(preAggregationsDescription[0].invalidateKeyQueries[0][0].replace(/(\r\n|\n|\r)/gm, '') .replace(/\s+/g, ' ')) - .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END'); + .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-28800 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key'); return dbRunner .evaluateQueryWithPreAggregations(query) diff --git a/packages/cubejs-schema-compiler/test/integration/mysql/custom-granularities.test.ts b/packages/cubejs-schema-compiler/test/integration/mysql/custom-granularities.test.ts index 1505442c453b0..676bee6c1884d 100644 --- a/packages/cubejs-schema-compiler/test/integration/mysql/custom-granularities.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mysql/custom-granularities.test.ts @@ -51,6 +51,13 @@ describe('Custom Granularities', () => { - name: twenty_five_minutes interval: 25 minutes origin: '2024-01-01 10:15:00' + - name: five_minutes_from_utc_origin + interval: 5 minutes + # 10:15 UTC = 11:15 Paris time (UTC+1) + origin: '2024-01-01T10:15:00Z' + - name: five_minutes_from_local_origin + interval: 5 minutes + origin: '2024-01-01 10:15:00' - name: fifteen_days_hours_minutes_seconds interval: 15 days 3 hours 25 minutes 40 seconds origin: '2024-01-01 10:15:00' @@ -151,6 +158,64 @@ describe('Custom Granularities', () => { { joinGraph, cubeEvaluator, compiler } )); + it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_utc_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: 1, + orders__created_at_five_minutes_from_utc_origin: '2024-01-01 01:00:00.000', + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_utc_origin: '2024-01-15 01:00:00.000', + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_utc_origin: '2024-01-29 01:00:00.000', + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); + + it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_local_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: 1, + orders__created_at_five_minutes_from_local_origin: '2024-01-01 01:00:00.000', + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_local_origin: '2024-01-15 01:00:00.000', + }, + { + orders__count: 1, + orders__created_at_five_minutes_from_local_origin: '2024-01-29 01:00:00.000', + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); + it('works with half_year custom granularity with dimension query', async () => dbRunner.runQueryTest( { measures: ['orders.count'], diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/custom-granularities.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/custom-granularities.test.ts index 000d8482df0d2..7ab4a61d10a79 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/custom-granularities.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/custom-granularities.test.ts @@ -56,6 +56,13 @@ describe('Custom Granularities', () => { - name: twenty_five_minutes interval: 25 minutes origin: '2024-01-01 10:15:00' + - name: five_minutes_from_utc_origin + interval: 5 minutes + # 10:15 UTC = 11:15 Paris time (UTC+1) + origin: '2024-01-01T10:15:00Z' + - name: five_minutes_from_local_origin + interval: 5 minutes + origin: '2024-01-01 10:15:00' - name: fiscal_year_by_1st_feb interval: 1 year origin: '2024-02-01' @@ -163,6 +170,64 @@ describe('Custom Granularities', () => { { joinGraph, cubeEvaluator, compiler } )); + it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_utc_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: '1', + orders__created_at_five_minutes_from_utc_origin: '2024-01-01T01:00:00.000Z', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_utc_origin: '2024-01-15T01:00:00.000Z', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_utc_origin: '2024-01-29T01:00:00.000Z', + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); + + it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + { + measures: ['orders.count'], + timeDimensions: [{ + dimension: 'orders.createdAt', + granularity: 'five_minutes_from_local_origin', + dateRange: ['2024-01-01', '2024-01-31'] + }], + dimensions: [], + filters: [], + timezone: 'Europe/Paris' + }, + [ + { + orders__count: '1', + orders__created_at_five_minutes_from_local_origin: '2024-01-01T01:00:00.000Z', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_local_origin: '2024-01-15T01:00:00.000Z', + }, + { + orders__count: '1', + orders__created_at_five_minutes_from_local_origin: '2024-01-29T01:00:00.000Z', + }, + ], + { joinGraph, cubeEvaluator, compiler } + )); + it('works with proxied createdAtHalfYear custom granularity as dimension query', async () => dbRunner.runQueryTest( { measures: ['orders.count'], diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs index 4d20e5b414dc4..e3eb2f7032059 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs @@ -1,10 +1,11 @@ use chrono::{DateTime, Duration, LocalResult, NaiveDate, NaiveDateTime, TimeZone}; use chrono_tz::Tz; use cubenativeutils::CubeError; +use lazy_static::lazy_static; use regex::Regex; + pub struct QueryDateTimeHelper {} -use lazy_static::lazy_static; lazy_static! { static ref DATE_TIME_LOCAL_MS_RE: Regex = Regex::new(r"^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\d$").unwrap(); @@ -12,10 +13,10 @@ lazy_static! { Regex::new(r"^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\d\d\d\d$").unwrap(); static ref DATE_RE: Regex = Regex::new(r"^\d\d\d\d-\d\d-\d\d$").unwrap(); } + impl QueryDateTimeHelper { pub fn parse_native_date_time(date: &str) -> Result { let formats = &[ - "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S%.f", @@ -32,6 +33,11 @@ impl QueryDateTimeHelper { return Ok(d.and_hms_opt(0, 0, 0).unwrap()); } + // Fallback as RFC3339/ISO8601 with 'Z' timezone + if let Ok(dt) = DateTime::parse_from_rfc3339(date) { + return Ok(dt.naive_utc()); + } + Err(CubeError::user(format!("Can't parse date: '{}'", date))) } @@ -153,6 +159,8 @@ impl QueryDateTimeHelper { #[cfg(test)] mod tests { use super::*; + use chrono::NaiveDate; + #[test] fn test_parse_native_date_time() { assert_eq!( @@ -190,5 +198,20 @@ mod tests { .and_hms_milli_opt(12, 10, 15, 345) .unwrap() ); + // Test parsing with 'Z' timezone (UTC) + assert_eq!( + QueryDateTimeHelper::parse_native_date_time("2024-01-01T10:15:00Z").unwrap(), + NaiveDate::from_ymd_opt(2024, 1, 1) + .unwrap() + .and_hms_opt(10, 15, 0) + .unwrap() + ); + assert_eq!( + QueryDateTimeHelper::parse_native_date_time("2024-01-01T10:15:00.123Z").unwrap(), + NaiveDate::from_ymd_opt(2024, 1, 1) + .unwrap() + .and_hms_milli_opt(10, 15, 0, 123) + .unwrap() + ); } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs index 10bd055b21741..56ed9420e50a0 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs @@ -1,7 +1,7 @@ use crate::cube_bridge::evaluator::CubeEvaluator; use crate::planner::sql_evaluator::Compiler; use crate::planner::sql_evaluator::TimeDimensionSymbol; -use crate::planner::Granularity; +use crate::planner::{Granularity, QueryDateTimeHelper}; use chrono::prelude::*; use chrono_tz::Tz; use cubenativeutils::CubeError; @@ -200,25 +200,7 @@ impl GranularityHelper { } pub fn parse_date_time(date: &str) -> Result { - let formats = &[ - "%Y-%m-%d", - "%Y-%m-%d %H:%M:%S%.f", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%dT%H:%M:%S%.f", - "%Y-%m-%dT%H:%M:%S", - ]; - - for format in formats { - if let Ok(dt) = NaiveDateTime::parse_from_str(date, format) { - return Ok(dt); - } - } - - if let Ok(d) = NaiveDate::parse_from_str(date, "%Y-%m-%d") { - return Ok(d.and_hms_opt(0, 0, 0).unwrap()); - } - - Err(CubeError::user(format!("Can't parse date: '{}'", date))) + QueryDateTimeHelper::parse_native_date_time(date) } pub fn make_granularity_obj( diff --git a/yarn.lock b/yarn.lock index 64cb91defa159..67c07534d9a60 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19505,10 +19505,10 @@ moment-timezone@^0.5.15, moment-timezone@^0.5.33, moment-timezone@^0.5.46, momen dependencies: moment "^2.29.4" -moment@^2.17.1, moment@^2.24.0, moment@^2.25.3, moment@^2.29.1, moment@^2.29.4: - version "2.29.4" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" - integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== +moment@^2.17.1, moment@^2.24.0, moment@^2.25.3, moment@^2.29.1, moment@^2.29.4, moment@^2.30.1: + version "2.30.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae" + integrity sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how== mrmime@2.0.0: version "2.0.0"