diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts index 64ae2c70f235d..1a37613ba222b 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts @@ -96,9 +96,9 @@ export class DatabricksQuery extends BaseQuery { const [intervalFormatted, timeUnit] = this.formatInterval(interval); const beginOfTime = this.dateTimeCast('\'1970-01-01T00:00:00\''); - return `${this.timeStampCast(`'${origin}'`)} + INTERVAL ${intervalFormatted} * + return `${this.dateTimeCast(`'${origin}'`)} + INTERVAL ${intervalFormatted} * floor( - date_diff(${timeUnit}, ${this.timeStampCast(`'${origin}'`)}, ${source}) / + date_diff(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) / date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + INTERVAL ${intervalFormatted}) )`; } diff --git a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts index 822ff864f6895..96cd50a725128 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts @@ -35,11 +35,11 @@ export class DuckDBQuery extends BaseQuery { */ public dateBin(interval: string, source: string, origin: string): string { const timeUnit = this.diffTimeUnitForInterval(interval); - const beginOfTime = this.timeStampCast('\'1970-01-01 00:00:00.000\''); + const beginOfTime = this.dateTimeCast('\'1970-01-01 00:00:00.000\''); - return `${this.timeStampCast(`'${origin}'`)}' + INTERVAL '${interval}' * + return `${this.dateTimeCast(`'${origin}'`)}' + INTERVAL '${interval}' * floor( - date_diff('${timeUnit}', ${this.timeStampCast(`'${origin}'`)}, ${source}) / + date_diff('${timeUnit}', ${this.dateTimeCast(`'${origin}'`)}, ${source}) / date_diff('${timeUnit}', ${beginOfTime}, ${beginOfTime} + INTERVAL '${interval}') )::int`; } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index bf1f5254efe8a..7d68475bc8873 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -2780,7 +2780,7 @@ export class BaseQuery { * intervals relative to origin timestamp point * @param {string} interval (a value expression of type interval) * @param {string} source (a value expression of type timestamp/date) - * @param {string} origin (a value expression of type timestamp/date) + * @param {string} origin (a value expression of type timestamp/date without timezone) * @returns {string} */ // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -2838,7 +2838,7 @@ export class BaseQuery { return this.timeGroupedColumn(granularity.granularityFromInterval(), dimension); } - return this.dateBin(granularity.granularityInterval, dimension, granularity.originFormatted()); + return this.dateBin(granularity.granularityInterval, dimension, granularity.originLocalFormatted()); } /** diff --git a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts index ed393b971a453..d643f1cdfcd9f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts @@ -73,10 +73,10 @@ export class ClickHouseQuery extends BaseQuery { return `date_add(${timeUnit}, FLOOR( - date_diff(${timeUnit}, ${this.timeStampCast(`'${origin}'`)}, ${source}) / + date_diff(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) / date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}) ) * date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}), - ${this.timeStampCast(`'${origin}'`)} + ${this.dateTimeCast(`'${origin}'`)} )`; } diff --git a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts index 057f112c70229..535baf1eead09 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts @@ -71,7 +71,7 @@ export class CubeStoreQuery extends BaseQuery { * intervals relative to origin timestamp point. */ public dateBin(interval: string, source: string, origin: string): string { - return `DATE_BIN(INTERVAL ${this.formatInterval(interval)}, ${this.timeStampCast(source)}, ${this.timeStampCast(`'${origin}'`)})`; + return `DATE_BIN(INTERVAL ${this.formatInterval(interval)}, ${this.dateTimeCast(source)}, ${this.dateTimeCast(`'${origin}'`)})`; } /** diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index b074a5193741e..c12028a0ea9a1 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -25,7 +25,7 @@ export class Granularity { ) { this.granularity = timeDimension.granularity; this.predefinedGranularity = isPredefinedGranularity(this.granularity); - this.origin = moment.tz('UTC').startOf('year'); // Defaults to current year start + this.origin = moment.tz(query.timezone).startOf('year'); // Defaults to current year start if (this.predefinedGranularity) { this.granularityInterval = `1 ${this.granularity}`; @@ -43,7 +43,7 @@ export class Granularity { this.granularityInterval = customGranularity.interval; if (customGranularity.origin) { - this.origin = moment.tz(customGranularity.origin, 'UTC'); + this.origin = moment.tz(customGranularity.origin, query.timezone); } else if (customGranularity.offset) { this.granularityOffset = customGranularity.offset; this.origin = addInterval(this.origin, parseSqlInterval(customGranularity.offset)); @@ -55,10 +55,20 @@ export class Granularity { return this.predefinedGranularity; } - public originFormatted(): string { + /** + * @returns origin date string in Query timezone + */ + public originLocalFormatted(): string { return this.origin.format('YYYY-MM-DDTHH:mm:ss.SSS'); } + /** + * @returns origin date string in UTC timezone + */ + public originUtcFormatted(): string { + return this.origin.clone().utc().format('YYYY-MM-DDTHH:mm:ss.SSSZ'); + } + public minGranularity(): string { if (this.predefinedGranularity) { return this.granularity; @@ -86,7 +96,10 @@ export class Granularity { return timeSeries(this.granularity, dateRange, options); } - return timeSeriesFromCustomInterval(this.granularityInterval, dateRange, this.origin, options); + // Interval range doesn't take timezone into account and operate in kinda local timezone, + // but origin is treated as a timestamp in query timezone, so we pass it as the naive timestamp + // to be in sync with date range during calculation. + return timeSeriesFromCustomInterval(this.granularityInterval, dateRange, moment(this.originLocalFormatted()), options); } public resolvedGranularity(): string { diff --git a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts index 1428bfd1883f7..663ccfc3a525c 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts @@ -71,7 +71,7 @@ export class MssqlQuery extends BaseQuery { } public timeStampCast(value: string) { - return this.dateTimeCast(value); + return `CAST(${value} AS DATETIMEOFFSET)`; } public dateTimeCast(value: string) { @@ -88,16 +88,16 @@ export class MssqlQuery extends BaseQuery { * The formula operates with seconds diffs so it won't produce human-expected dates aligned with offset date parts. */ public dateBin(interval: string, source: string, origin: string): string { - const beginOfTime = this.timeStampCast('DATEFROMPARTS(1970, 1, 1)'); + const beginOfTime = this.dateTimeCast('DATEFROMPARTS(1970, 1, 1)'); const timeUnit = this.diffTimeUnitForInterval(interval); // Need to explicitly cast one argument of floor to float to trigger correct sign logic return `DATEADD(${timeUnit}, FLOOR( - CAST(DATEDIFF(${timeUnit}, ${this.timeStampCast(`'${origin}'`)}, ${source}) AS FLOAT) / + CAST(DATEDIFF(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) AS FLOAT) / DATEDIFF(${timeUnit}, ${beginOfTime}, ${this.addInterval(beginOfTime, interval)}) ) * DATEDIFF(${timeUnit}, ${beginOfTime}, ${this.addInterval(beginOfTime, interval)}), - ${this.timeStampCast(`'${origin}'`)} + ${this.dateTimeCast(`'${origin}'`)} )`; } diff --git a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts index bac1922efd893..a6a304b7e9f01 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts @@ -72,10 +72,10 @@ export class MysqlQuery extends BaseQuery { return `TIMESTAMPADD(${timeUnit}, FLOOR( - TIMESTAMPDIFF(${timeUnit}, ${this.timeStampCast(`'${origin}'`)}, ${source}) / + TIMESTAMPDIFF(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) / TIMESTAMPDIFF(${timeUnit}, '1970-01-01 00:00:00', '1970-01-01 00:00:00' + INTERVAL ${intervalFormatted}) ) * TIMESTAMPDIFF(${timeUnit}, '1970-01-01 00:00:00', '1970-01-01 00:00:00' + INTERVAL ${intervalFormatted}), - ${this.timeStampCast(`'${origin}'`)} + ${this.dateTimeCast(`'${origin}'`)} )`; } diff --git a/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts b/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts index e005f537a3f23..d5e4871b88497 100644 --- a/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/OracleQuery.ts @@ -19,7 +19,7 @@ class OracleFilter extends BaseFilter { } /** - * "ILIKE" does't support + * "ILIKE" is not supported */ public likeIgnoreCase(column, not, param, type) { const p = (!type || type === 'contains' || type === 'ends') ? '\'%\' || ' : ''; @@ -30,7 +30,7 @@ class OracleFilter extends BaseFilter { export class OracleQuery extends BaseQuery { /** - * "LIMIT" on Oracle it's illegal + * "LIMIT" on Oracle is illegal * TODO replace with limitOffsetClause override */ public groupByDimensionLimit() { diff --git a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts index 0e3a5c50db690..8d3e5a63dc85a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/SnowflakeQuery.ts @@ -53,10 +53,10 @@ export class SnowflakeQuery extends BaseQuery { return `DATEADD(${timeUnit}, FLOOR( - DATEDIFF(${timeUnit}, ${this.timeStampCast(`'${origin}'`)}, ${source}) / + DATEDIFF(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) / DATEDIFF(${timeUnit}, ${beginOfTime}, (${beginOfTime} + interval '${intervalFormatted}')) ) * DATEDIFF(${timeUnit}, ${beginOfTime}, (${beginOfTime} + interval '${intervalFormatted}')), - ${this.timeStampCast(`'${origin}'`)})`; + ${this.dateTimeCast(`'${origin}'`)})`; } /** diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts index e0f8475144dd6..3cdfc8539567d 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts @@ -18,7 +18,7 @@ describe('MSSqlPreAggregations', () => { sql: \` select * from ##visitors \`, - + joins: { visitor_checkins: { relationship: 'hasMany', @@ -30,22 +30,22 @@ describe('MSSqlPreAggregations', () => { count: { type: 'count' }, - + checkinsTotal: { sql: \`\${checkinsCount}\`, type: 'sum' }, - + uniqueSourceCount: { sql: 'source', type: 'countDistinct' }, - + countDistinctApprox: { sql: 'id', type: 'countDistinctApprox' }, - + ratio: { sql: \`1.0 * \${uniqueSourceCount} / nullif(\${checkinsTotal}, 0)\`, type: 'number' @@ -72,13 +72,13 @@ describe('MSSqlPreAggregations', () => { subQuery: true } }, - + segments: { google: { sql: \`source = 'google'\` } }, - + preAggregations: { default: { type: 'originalSql' @@ -125,8 +125,8 @@ describe('MSSqlPreAggregations', () => { } } }) - - + + cube('visitor_checkins', { sql: \` select * from ##visitor_checkins @@ -157,7 +157,7 @@ describe('MSSqlPreAggregations', () => { sql: 'created_at' } }, - + preAggregations: { main: { type: 'originalSql' @@ -168,7 +168,7 @@ describe('MSSqlPreAggregations', () => { } } }) - + cube('GoogleVisitors', { extends: visitors, sql: \`select v.* from \${visitors.sql()} v where v.source = 'google'\` @@ -276,7 +276,7 @@ describe('MSSqlPreAggregations', () => { expect(preAggregationsDescription[0].invalidateKeyQueries[0][0].replace(/(\r\n|\n|\r)/gm, '') .replace(/\s+/g, ' ')) - .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIME2)) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END'); + .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END'); return dbRunner .evaluateQueryWithPreAggregations(query) diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts index bd57b53da0c4e..90338cab25deb 100644 --- a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts @@ -1356,7 +1356,7 @@ describe('SQL Generation', () => { expect(preAggregations.length).toEqual(1); expect(preAggregations[0].invalidateKeyQueries).toEqual([ [ - 'SELECT CASE\n WHEN CURRENT_TIMESTAMP < CAST(@_1 AS DATETIME2) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key', + 'SELECT CASE\n WHEN CURRENT_TIMESTAMP < CAST(@_1 AS DATETIMEOFFSET) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key', [ '__TO_PARTITION_RANGE', ], diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index fe346c685368d..83120f1564950 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -104,7 +104,11 @@ describe('Pre Aggregation by filter match tests', () => { )); it('1 count measure, day, two_weeks_by_1st_feb_00am', () => testPreAggregationMatch( - true, ['count'], 'day', 'two_weeks_by_1st_feb_00am' + true, ['count'], 'day', 'two_weeks_by_1st_feb_00am', 'UTC' + )); + + it('1 count measure, day, two_weeks_by_1st_feb_00am', () => testPreAggregationMatch( + false, ['count'], 'day', 'two_weeks_by_1st_feb_00am', 'Europe/Berlin' )); it('1 count measure, day, two_weeks_by_1st_feb_10am', () => testPreAggregationMatch(