diff --git a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts index 0c392ea888c00..37fa3ce84bcb8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts @@ -69,17 +69,17 @@ export class ClickHouseQuery extends BaseQuery { public dateBin(interval: string, source: string, origin: string): string { // Pass timezone to dateTimeCast to ensure origin is in the same timezone as a source, because ClickHouse aligns // both timestamps internally to the same timezone before computing the difference, causing an unintended offset. - const alignedOrigin = this.dateTimeCast(`'${origin}'`, this.timezone); + const originAligned = this.dateTimeCast(`'${origin}'`, this.timezone); const intervalFormatted = this.formatInterval(interval); const timeUnit = this.diffTimeUnitForInterval(interval); const beginOfTime = 'fromUnixTimestamp(0)'; return `date_add(${timeUnit}, FLOOR( - date_diff(${timeUnit}, ${alignedOrigin}, ${source}) / + date_diff(${timeUnit}, ${originAligned}, ${source}) / date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}) ) * date_diff(${timeUnit}, ${beginOfTime}, ${beginOfTime} + ${intervalFormatted}), - ${alignedOrigin} + ${originAligned} )`; } diff --git a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts index b1b4b16603c85..24798c89228ac 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts @@ -90,7 +90,12 @@ export class MssqlQuery extends BaseQuery { } public convertTz(field) { - return `TODATETIMEOFFSET(${field}, '${moment().tz(this.timezone).format('Z')}')`; + const offset = moment().tz(this.timezone).format('Z'); + + // 1. Treating the field as UTC (add '+00:00' offset) + // 2. Switch to target timezone offset + // 3. Cast to DATETIME2 to get naive timestamp in target timezone + return `CAST(SWITCHOFFSET(TODATETIMEOFFSET(${field}, '+00:00'), '${offset}') AS DATETIME2)`; } public timeStampCast(value: string) { @@ -106,21 +111,24 @@ export class MssqlQuery extends BaseQuery { } /** - * Returns sql for source expression floored to timestamps aligned with - * intervals relative to origin timestamp point. - * The formula operates with seconds diffs so it won't produce human-expected dates aligned with offset date parts. + * Returns SQL for source expression floored to timestamps aligned with + * intervals relative to the origin timestamp point. + * The formula operates with seconds diffs, so it won't produce human-expected dates aligned with offset date parts. */ public dateBin(interval: string, source: string, origin: string): string { + // Both source and origin are now DATETIME2 in the query timezone (naive timestamps), + // ensuring their in the same timezone context for correct date bin calculation. + const originAligned = this.dateTimeCast(`'${origin}'`); const beginOfTime = this.dateTimeCast('DATEFROMPARTS(1970, 1, 1)'); const timeUnit = this.diffTimeUnitForInterval(interval); // Need to explicitly cast one argument of floor to float to trigger correct sign logic return `DATEADD(${timeUnit}, FLOOR( - CAST(DATEDIFF(${timeUnit}, ${this.dateTimeCast(`'${origin}'`)}, ${source}) AS FLOAT) / + CAST(DATEDIFF(${timeUnit}, ${originAligned}, ${source}) AS FLOAT) / DATEDIFF(${timeUnit}, ${beginOfTime}, ${this.addInterval(beginOfTime, interval)}) ) * DATEDIFF(${timeUnit}, ${beginOfTime}, ${this.addInterval(beginOfTime, interval)}), - ${this.dateTimeCast(`'${origin}'`)} + ${originAligned} )`; } diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts index 24f2f4e2ccb7d..53a14a8dace6c 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/custom-granularities.test.ts @@ -158,8 +158,7 @@ describe('Custom Granularities', () => { { joinGraph, cubeEvaluator, compiler } )); - /// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z - xit('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + it('works with five_minutes_from_utc_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( { measures: ['orders.count'], timeDimensions: [{ @@ -188,8 +187,7 @@ describe('Custom Granularities', () => { { joinGraph, cubeEvaluator, compiler } )); - /// TODO: fix date bin calculation... for some reason it goes from 2023-12-31T23:00:00.000Z - xit('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( + it('works with five_minutes_from_local_origin custom granularity in Europe/Paris timezone', async () => dbRunner.runQueryTest( { measures: ['orders.count'], timeDimensions: [{ diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts index e65d7a44084b1..626973e2329e1 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts @@ -79,22 +79,22 @@ describe('MSSqlCumulativeMeasures', () => { expect(await dbRunner.testQuery(query.buildSqlAndParams())).toEqual([ { visitors__count: 1, - visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-02T00:00:00.000Z'), visitors__unbounded_count: 2, }, { visitors__count: 1, - visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitors__unbounded_count: 3, }, { visitors__count: 1, - visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitors__unbounded_count: 4, }, { visitors__count: 2, - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'), visitors__unbounded_count: 6, } ]); @@ -128,25 +128,25 @@ describe('MSSqlCumulativeMeasures', () => { expect(await dbRunner.testQuery(query.buildSqlAndParams())).toEqual([ { visitors__count: 1, - visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-02T00:00:00.000Z'), visitors__source: 'some', visitors__unbounded_count: 1 }, { visitors__count: 1, - visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitors__source: 'some', visitors__unbounded_count: 2, }, { visitors__count: 1, - visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitors__source: 'google', visitors__unbounded_count: 1, }, { visitors__count: 2, - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'), visitors__source: null, visitors__unbounded_count: 3, } diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts index 3dd2680884fc4..7b9a927ae5f87 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts @@ -278,22 +278,22 @@ describe('MSSqlPreAggregations', () => { expect(res) .toEqual([ { - visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-02T00:00:00.000Z'), visitors__checkins_total: 3, visitors__source: 'some', }, { - visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitors__checkins_total: 2, visitors__source: 'some', }, { - visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitors__checkins_total: 1, visitors__source: 'google', }, { - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z'), + visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'), visitors__checkins_total: 0, visitors__source: null } diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts index 7c19c7e8767ba..507a9b04593af 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts @@ -366,12 +366,12 @@ describe('MSSqlUngrouped', () => { ungrouped: true, allowUngroupedWithoutPrimaryKey: true, }, [ + { visitor_checkins__created_at_day: new Date('2017-01-02T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, { visitor_checkins__created_at_day: new Date('2017-01-03T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-06T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: 1 }, + { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, + { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: null }, + { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__google_sourced_checkins: 1 }, ])); it('ungrouped filtered distinct count', () => runQueryTest({ @@ -390,12 +390,12 @@ describe('MSSqlUngrouped', () => { ungrouped: true, allowUngroupedWithoutPrimaryKey: true, }, [ + { visitor_checkins__created_at_day: new Date('2017-01-02T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, { visitor_checkins__created_at_day: new Date('2017-01-03T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, - { visitor_checkins__created_at_day: new Date('2017-01-06T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: 1 }, + { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, + { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: null }, + { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__unique_google_sourced_checkins: 1 }, ])); it('ungrouped ratio measure', () => runQueryTest({ @@ -414,12 +414,12 @@ describe('MSSqlUngrouped', () => { ungrouped: true, allowUngroupedWithoutPrimaryKey: true, }, [ + { visitor_checkins__created_at_day: new Date('2017-01-02T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, { visitor_checkins__created_at_day: new Date('2017-01-03T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, + { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, + { visitor_checkins__created_at_day: new Date('2017-01-04T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, - { visitor_checkins__created_at_day: new Date('2017-01-05T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, - { visitor_checkins__created_at_day: new Date('2017-01-06T00:00:00.000Z'), visitor_checkins__unique_sources_per_checking: 1 }, ])); it('ungrouped', () => runQueryTest({ @@ -439,22 +439,22 @@ describe('MSSqlUngrouped', () => { ungrouped: true }, [{ visitors__id: 6, - visitors__created_at_day: new Date('2016-09-07T00:00:00.000Z') + visitors__created_at_day: new Date('2016-09-06T00:00:00.000Z') }, { visitors__id: 1, - visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-02T00:00:00.000Z') }, { visitors__id: 2, - visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-04T00:00:00.000Z') }, { visitors__id: 3, - visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z') }, { visitors__id: 4, - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z') }, { visitors__id: 5, - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z') }])); it('offset cache', () => runQueryTest({ @@ -475,7 +475,7 @@ describe('MSSqlUngrouped', () => { offset: 5 }, [{ visitors__id: 5, - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z') }])); it('ungrouped without id', () => runQueryTest({ @@ -493,17 +493,17 @@ describe('MSSqlUngrouped', () => { ungrouped: true, allowUngroupedWithoutPrimaryKey: true }, [{ - visitors__created_at_day: new Date('2016-09-07T00:00:00.000Z') + visitors__created_at_day: new Date('2016-09-06T00:00:00.000Z') + }, { + visitors__created_at_day: new Date('2017-01-02T00:00:00.000Z') }, { - visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-04T00:00:00.000Z') }, { visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z') }, { visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z') }, { - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z') - }, { - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z') }])); it('ungrouped false without id', () => runQueryTest({ @@ -520,14 +520,14 @@ describe('MSSqlUngrouped', () => { timezone: 'America/Los_Angeles', ungrouped: false }, [{ - visitors__created_at_day: new Date('2016-09-07T00:00:00.000Z') + visitors__created_at_day: new Date('2016-09-06T00:00:00.000Z') }, { - visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z') + visitors__created_at_day: new Date('2017-01-02T00:00:00.000Z') + }, { + visitors__created_at_day: new Date('2017-01-04T00:00:00.000Z') }, { visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z') }, { visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z') - }, { - visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z') }])); }); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/postgres-cumulative-measures.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/postgres-cumulative-measures.test.ts new file mode 100644 index 0000000000000..a9ecb73116cfd --- /dev/null +++ b/packages/cubejs-schema-compiler/test/integration/postgres/postgres-cumulative-measures.test.ts @@ -0,0 +1,149 @@ +import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; +import { dbRunner } from './PostgresDBRunner'; + +describe('PostgresCumulativeMeasures', () => { + jest.setTimeout(200000); + + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` + cube(\`visitors\`, { + sql: \` + select * from visitors + \`, + + joins: {}, + + measures: { + count: { + type: 'count' + }, + + unboundedCount: { + type: 'count', + rollingWindow: { + trailing: 'unbounded' + } + } + }, + + dimensions: { + id: { + type: 'number', + sql: 'id', + primaryKey: true + }, + source: { + type: 'string', + sql: 'source' + }, + createdAt: { + type: 'time', + sql: 'created_at' + }, + }, + + preAggregations: {} + }) + `); + + it('should group by the created_at field on the calculated granularity for unbounded trailing windows without dimension', () => compiler.compile().then(async () => { + const query = new PostgresQuery( + { joinGraph, cubeEvaluator, compiler }, + { + measures: ['visitors.count', 'visitors.unboundedCount'], + timeDimensions: [ + { + dimension: 'visitors.createdAt', + granularity: 'day', + dateRange: ['2017-01-01', '2017-01-30'], + }, + ], + timezone: 'America/Los_Angeles', + order: [ + { + id: 'visitors.createdAt', + }, + ], + } + ); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + + expect(await dbRunner.testQuery(query.buildSqlAndParams())).toEqual([ + { + visitors__count: '1', + visitors__created_at_day: '2017-01-02T00:00:00.000Z', + visitors__unbounded_count: '2', + }, + { + visitors__count: '1', + visitors__created_at_day: '2017-01-04T00:00:00.000Z', + visitors__unbounded_count: '3', + }, + { + visitors__count: '1', + visitors__created_at_day: '2017-01-05T00:00:00.000Z', + visitors__unbounded_count: '4', + }, + { + visitors__count: '2', + visitors__created_at_day: '2017-01-06T00:00:00.000Z', + visitors__unbounded_count: '6', + } + ]); + })); + + it('should group by the created_at field on the calculated granularity for unbounded trailing windows with dimension', () => compiler.compile().then(async () => { + const query = new PostgresQuery( + { joinGraph, cubeEvaluator, compiler }, + { + measures: ['visitors.count', 'visitors.unboundedCount'], + dimensions: ['visitors.source'], + timeDimensions: [ + { + dimension: 'visitors.createdAt', + granularity: 'day', + dateRange: ['2017-01-01', '2017-01-30'], + }, + ], + timezone: 'America/Los_Angeles', + order: [ + { + id: 'visitors.createdAt', + }, + ], + } + ); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + + expect(await dbRunner.testQuery(query.buildSqlAndParams())).toEqual([ + { + visitors__count: '1', + visitors__created_at_day: '2017-01-02T00:00:00.000Z', + visitors__source: 'some', + visitors__unbounded_count: '1' + }, + { + visitors__count: '1', + visitors__created_at_day: '2017-01-04T00:00:00.000Z', + visitors__source: 'some', + visitors__unbounded_count: '2', + }, + { + visitors__count: '1', + visitors__created_at_day: '2017-01-05T00:00:00.000Z', + visitors__source: 'google', + visitors__unbounded_count: '1', + }, + { + visitors__count: '2', + visitors__created_at_day: '2017-01-06T00:00:00.000Z', + visitors__source: null, + visitors__unbounded_count: '3', + } + ]); + })); +});