diff --git a/docs/pages/reference/data-model/cube.mdx b/docs/pages/reference/data-model/cube.mdx index 2fb5e5dec58ef..e7c4374e6735b 100644 --- a/docs/pages/reference/data-model/cube.mdx +++ b/docs/pages/reference/data-model/cube.mdx @@ -358,7 +358,7 @@ When applicable, it will be displayed in [Playground][ref-playground] and expose to data consumers via [APIs and integrations][ref-apis]. A description can give a hint both to your team and end users, making sure they -interpret the data correctly. +interpret the data correctly. @@ -478,7 +478,8 @@ cubes: `every` - can be set as an interval with granularities `second`, `minute`, `hour`, `day`, and `week` or accept CRON string with some limitations. If you -set `every` as CRON string, you can use the `timezone` parameter. +set `every` as CRON string, you can use the `timezone` parameter. It takes +precedence over the query timezone. For example: @@ -643,4 +644,4 @@ The `access_policy` parameter is used to configure [data access policies][ref-re [ref-ref-joins]: /reference/data-model/joins [ref-ref-pre-aggs]: /reference/data-model/pre-aggregations [ref-ref-dap]: /reference/data-model/data-access-policies -[ref-syntax-cube-sql]: /product/data-modeling/syntax#cubesql-function \ No newline at end of file +[ref-syntax-cube-sql]: /product/data-modeling/syntax#cubesql-function diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/BaseQueueEventsBus.ts b/packages/cubejs-query-orchestrator/src/orchestrator/BaseQueueEventsBus.ts index 744ffedfd2c17..bd1141ae925e2 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/BaseQueueEventsBus.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/BaseQueueEventsBus.ts @@ -1,11 +1,11 @@ export class BaseQueueEventsBus { protected readonly subscribers: Record = {}; - public subscribe(id, callback) { + public subscribe(id: string, callback) { this.subscribers[id] = { id, callback }; } - public unsubscribe(id) { + public unsubscribe(id: string) { delete this.subscribers[id]; } } diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts index bf59b5e3df8b1..bf0f480e01821 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts @@ -350,7 +350,7 @@ export class PreAggregations { } /** - * Determines whether the partition table is already exists or not. + * Determines whether the partition table already exists or not. */ public async isPartitionExist( request: string, @@ -512,7 +512,7 @@ export class PreAggregations { */ public async checkPartitionsBuildRangeCache(queryBody) { const preAggregations = queryBody.preAggregations || []; - const result = await Promise.all( + return Promise.all( preAggregations.map(async (preAggregation) => { const { preAggregationStartEndQueries } = preAggregation; const invalidate = @@ -538,7 +538,6 @@ export class PreAggregations { }; }) ); - return result; } public async expandPartitionsInPreAggregations(queryBody: Query): Promise { diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts index c518224961d96..d24bc065663be 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts @@ -9,7 +9,7 @@ import { InlineTables, CacheDriverInterface, TableStructure, - DriverInterface, QueryKey, + DriverInterface, } from '@cubejs-backend/base-driver'; import { QueryQueue } from './QueryQueue'; @@ -208,9 +208,7 @@ export class QueryCache { .cacheKeyQueriesFrom(queryBody) .map(replacePreAggregationTableNames); - const renewalThreshold = - queryBody.cacheKeyQueries && - queryBody.cacheKeyQueries.renewalThreshold; + const renewalThreshold = queryBody.cacheKeyQueries?.renewalThreshold; const expireSecs = this.getExpireSecs(queryBody); @@ -350,7 +348,7 @@ export class QueryCache { } private cacheKeyQueriesFrom(queryBody: QueryBody): QueryWithParams[] { - return queryBody.cacheKeyQueries && queryBody.cacheKeyQueries.queries || + return queryBody.cacheKeyQueries?.queries || queryBody.cacheKeyQueries || []; } diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts index 20a4a24c5b3ae..46e55da79519f 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts @@ -448,11 +448,11 @@ export class QueryOrchestrator { return this.preAggregations.cancelQueriesFromQueue(queryKeys, dataSource); } - public async subscribeQueueEvents(id, callback) { + public async subscribeQueueEvents(id: string, callback) { return this.getQueueEventsBus().subscribe(id, callback); } - public async unSubscribeQueueEvents(id) { + public async unSubscribeQueueEvents(id: string) { return this.getQueueEventsBus().unsubscribe(id); } diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index f7229a1d3e59c..ca741871d63c8 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -50,7 +50,7 @@ "joi": "^17.13.3", "js-yaml": "^4.1.0", "lru-cache": "^11.1.0", - "moment-timezone": "^0.5.46", + "moment-timezone": "^0.5.48", "node-dijkstra": "^2.5.0", "ramda": "^0.27.2", "syntax-error": "^1.3.0", diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c83c50a50fa38..234ca924bb6f4 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -3714,7 +3714,7 @@ export class BaseQuery { let utcOffset = 0; - if (refreshKey.timezone) { + if (refreshKey.timezone || this.timezone) { utcOffset = moment.tz(refreshKey.timezone).utcOffset() * 60; } @@ -3733,7 +3733,9 @@ export class BaseQuery { const every = refreshKey.every || '1 hour'; if (/^(\d+) (second|minute|hour|day|week)s?$/.test(every)) { - return [this.floorSql(`(${this.unixTimestampSql()}) / ${this.parseSecondDuration(every)}`), external, this]; + const utcOffset = this.timezone ? moment.tz(this.timezone).utcOffset() * 60 : 0; + const utcOffsetPrefix = utcOffset ? `${utcOffset} + ` : ''; + return [this.floorSql(`(${utcOffsetPrefix}${this.unixTimestampSql()}) / ${this.parseSecondDuration(every)}`), external, this]; } const { dayOffset, utcOffset, interval } = this.calcIntervalForCronString(refreshKey); diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts index 949e881692296..7a0c6e3af2cd6 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts @@ -276,7 +276,7 @@ describe('MSSqlPreAggregations', () => { expect(preAggregationsDescription[0].invalidateKeyQueries[0][0].replace(/(\r\n|\n|\r)/gm, '') .replace(/\s+/g, ' ')) - .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END'); + .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END'); return dbRunner .evaluateQueryWithPreAggregations(query) diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts index 13fc1cc6575f2..bfa9a12bf5b9c 100644 --- a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts @@ -1183,7 +1183,7 @@ describe('SQL Generation', () => { const utcOffset = moment.tz('America/Los_Angeles').utcOffset() * 60; expect(query.everyRefreshKeySql({ every: '1 hour' - })).toEqual(['FLOOR((EXTRACT(EPOCH FROM NOW())) / 3600)', false, expect.any(BaseQuery)]); + })).toEqual(['FLOOR((-25200 + EXTRACT(EPOCH FROM NOW())) / 3600)', false, expect.any(BaseQuery)]); // Standard syntax (minutes hours day month dow) expect(query.everyRefreshKeySql({ every: '0 * * * *', timezone })) @@ -1290,7 +1290,7 @@ describe('SQL Generation', () => { expect(query.cacheKeyQueries()).toEqual([ [ // Postgres dialect - 'SELECT FLOOR((EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key', + 'SELECT FLOOR((-25200 + EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key', [], { // false, because there is no externalQueryClass @@ -1311,7 +1311,7 @@ describe('SQL Generation', () => { ], timeDimensions: [], filters: [], - timezone: 'America/Los_Angeles', + timezone: 'Europe/London', externalQueryClass: MssqlQuery }); @@ -1319,7 +1319,7 @@ describe('SQL Generation', () => { expect(query.cacheKeyQueries()).toEqual([ [ // MSSQL dialect, because externalQueryClass - 'SELECT FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key', + 'SELECT FLOOR((3600 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key', [], { // true, because externalQueryClass @@ -1352,7 +1352,7 @@ describe('SQL Generation', () => { expect(preAggregations[0].invalidateKeyQueries).toEqual([ [ // MSSQL dialect - 'SELECT FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) as refresh_key', + 'SELECT FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) as refresh_key', [], { external: true, @@ -1405,7 +1405,7 @@ describe('SQL Generation', () => { dateRange: ['2016-12-30', '2017-01-05'] }], filters: [], - timezone: 'America/Los_Angeles', + timezone: 'Asia/Tokyo', externalQueryClass: MssqlQuery }); @@ -1413,7 +1413,7 @@ describe('SQL Generation', () => { expect(preAggregations.length).toEqual(1); expect(preAggregations[0].invalidateKeyQueries).toEqual([ [ - 'SELECT CASE\n WHEN CURRENT_TIMESTAMP < CAST(@_1 AS DATETIMEOFFSET) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key', + 'SELECT CASE\n WHEN CURRENT_TIMESTAMP < CAST(@_1 AS DATETIMEOFFSET) THEN FLOOR((32400 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key', [ '__TO_PARTITION_RANGE', ], @@ -1496,7 +1496,7 @@ describe('SQL Generation', () => { expect(preAggregations.length).toEqual(1); expect(preAggregations[0].invalidateKeyQueries).toEqual([ [ - 'SELECT FLOOR((EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key', + 'SELECT FLOOR((-25200 + EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key', [], { external: false, @@ -1527,7 +1527,7 @@ describe('SQL Generation', () => { expect(preAggregations.length).toEqual(1); expect(preAggregations[0].invalidateKeyQueries).toEqual([ [ - 'SELECT FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key', + 'SELECT FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key', [], { external: true, diff --git a/packages/cubejs-server-core/src/core/OrchestratorApi.ts b/packages/cubejs-server-core/src/core/OrchestratorApi.ts index 4579923353015..df47cc0d8ecc5 100644 --- a/packages/cubejs-server-core/src/core/OrchestratorApi.ts +++ b/packages/cubejs-server-core/src/core/OrchestratorApi.ts @@ -71,7 +71,7 @@ export class OrchestratorApi { * error otherwise. */ public async executeQuery(query: QueryBody) { - const queryForLog = query.query && query.query.replace(/\s+/g, ' '); + const queryForLog = query.query?.replace(/\s+/g, ' '); const startQueryTime = (new Date()).getTime(); try { @@ -174,7 +174,7 @@ export class OrchestratorApi { } /** - * Tests worker's connections to the Cubstore and, if not in the rollup only + * Tests worker's connections to the Cubestore and, if not in the rollup only * mode, to the datasources. */ public async testConnection() { @@ -297,11 +297,11 @@ export class OrchestratorApi { return this.orchestrator.cancelPreAggregationQueriesFromQueue(queryKeys, dataSource); } - public async subscribeQueueEvents(id, callback) { + public async subscribeQueueEvents(id: string, callback) { return this.orchestrator.subscribeQueueEvents(id, callback); } - public async unSubscribeQueueEvents(id) { + public async unSubscribeQueueEvents(id: string) { return this.orchestrator.unSubscribeQueueEvents(id); } diff --git a/yarn.lock b/yarn.lock index 740f3feb474eb..3aea5a52362b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -21190,10 +21190,10 @@ moment-range@*, moment-range@^4.0.1, moment-range@^4.0.2: dependencies: es6-symbol "^3.1.0" -moment-timezone@^0.5.15, moment-timezone@^0.5.33, moment-timezone@^0.5.46, moment-timezone@^0.5.47: - version "0.5.47" - resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.47.tgz#d4d1a21b78372d914d6d69ae285454732a429749" - integrity sha512-UbNt/JAWS0m/NJOebR0QMRHBk0hu03r5dx9GK8Cs0AS3I81yDcOc9k+DytPItgVvBP7J6Mf6U2n3BPAacAV9oA== +moment-timezone@^0.5.15, moment-timezone@^0.5.33, moment-timezone@^0.5.46, moment-timezone@^0.5.47, moment-timezone@^0.5.48: + version "0.5.48" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.48.tgz#111727bb274734a518ae154b5ca589283f058967" + integrity sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw== dependencies: moment "^2.29.4"