fix(schema-compiler): Make RefreshKeySql timezone-aware

docs/pages/reference/data-model/cube.mdx

@@ -358,7 +358,7 @@ When applicable, it will be displayed in [Playground][ref-playground] and expose
 to data consumers via [APIs and integrations][ref-apis].
 
 A description can give a hint both to your team and end users, making sure they
-interpret the data correctly. 
+interpret the data correctly.
 
 <CodeTabs>
 
@@ -478,7 +478,8 @@ cubes:
 
 `every` - can be set as an interval with granularities `second`, `minute`,
 `hour`, `day`, and `week` or accept CRON string with some limitations. If you
-set `every` as CRON string, you can use the `timezone` parameter.
+set `every` as CRON string, you can use the `timezone` parameter. It takes
+precedence over the query timezone.
 
 For example:
 
@@ -643,4 +644,4 @@ The `access_policy` parameter is used to configure [data access policies][ref-re
 [ref-ref-joins]: /reference/data-model/joins
 [ref-ref-pre-aggs]: /reference/data-model/pre-aggregations
 [ref-ref-dap]: /reference/data-model/data-access-policies
-[ref-syntax-cube-sql]: /product/data-modeling/syntax#cubesql-function
+[ref-syntax-cube-sql]: /product/data-modeling/syntax#cubesql-function

packages/cubejs-query-orchestrator/src/orchestrator/BaseQueueEventsBus.ts

@@ -1,11 +1,11 @@
 export class BaseQueueEventsBus {
   protected readonly subscribers: Record<string, any> = {};
 
-  public subscribe(id, callback) {
+  public subscribe(id: string, callback) {
     this.subscribers[id] = { id, callback };
   }
 
-  public unsubscribe(id) {
+  public unsubscribe(id: string) {
     delete this.subscribers[id];
   }
 }

packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts

@@ -350,7 +350,7 @@
   }
 
   /**
-   * Determines whether the partition table is already exists or not.
+   * Determines whether the partition table already exists or not.
    */
   public async isPartitionExist(
     request: string,
@@ -512,7 +512,7 @@
    */
   public async checkPartitionsBuildRangeCache(queryBody) {
     const preAggregations = queryBody.preAggregations || [];
-    const result = await Promise.all(
+    return Promise.all(
       preAggregations.map(async (preAggregation) => {
         const { preAggregationStartEndQueries } = preAggregation;
         const invalidate =
@@ -538,7 +538,6 @@
         };
       })
     );
-    return result;
   }
 
   public async expandPartitionsInPreAggregations(queryBody: Query): Promise<Query> {

packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts

@@ -9,7 +9,7 @@ import {
   InlineTables,
   CacheDriverInterface,
   TableStructure,
-  DriverInterface, QueryKey,
+  DriverInterface,
 } from '@cubejs-backend/base-driver';
 
 import { QueryQueue } from './QueryQueue';
@@ -208,9 +208,7 @@ export class QueryCache {
       .cacheKeyQueriesFrom(queryBody)
       .map(replacePreAggregationTableNames);
 
-    const renewalThreshold =
-      queryBody.cacheKeyQueries &&
-      queryBody.cacheKeyQueries.renewalThreshold;
+    const renewalThreshold = queryBody.cacheKeyQueries?.renewalThreshold;
 
     const expireSecs = this.getExpireSecs(queryBody);
 
@@ -350,7 +348,7 @@ export class QueryCache {
   }
 
   private cacheKeyQueriesFrom(queryBody: QueryBody): QueryWithParams[] {
-    return queryBody.cacheKeyQueries && queryBody.cacheKeyQueries.queries ||
+    return queryBody.cacheKeyQueries?.queries ||
       queryBody.cacheKeyQueries ||
       [];
   }

packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts

@@ -448,11 +448,11 @@
     return this.preAggregations.cancelQueriesFromQueue(queryKeys, dataSource);
   }
 
-  public async subscribeQueueEvents(id, callback) {
+  public async subscribeQueueEvents(id: string, callback) {
     return this.getQueueEventsBus().subscribe(id, callback);
   }
 
-  public async unSubscribeQueueEvents(id) {
+  public async unSubscribeQueueEvents(id: string) {
     return this.getQueueEventsBus().unsubscribe(id);
   }
 

packages/cubejs-schema-compiler/package.json

@@ -50,7 +50,7 @@
     "joi": "^17.13.3",
     "js-yaml": "^4.1.0",
     "lru-cache": "^11.1.0",
-    "moment-timezone": "^0.5.46",
+    "moment-timezone": "^0.5.48",
     "node-dijkstra": "^2.5.0",
     "ramda": "^0.27.2",
     "syntax-error": "^1.3.0",

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -3714,7 +3714,7 @@ export class BaseQuery {
 
     let utcOffset = 0;
 
-    if (refreshKey.timezone) {
+    if (refreshKey.timezone || this.timezone) {
       utcOffset = moment.tz(refreshKey.timezone).utcOffset() * 60;
     }
 
@@ -3733,7 +3733,9 @@ export class BaseQuery {
     const every = refreshKey.every || '1 hour';
 
     if (/^(\d+) (second|minute|hour|day|week)s?$/.test(every)) {
-      return [this.floorSql(`(${this.unixTimestampSql()}) / ${this.parseSecondDuration(every)}`), external, this];
+      const utcOffset = this.timezone ? moment.tz(this.timezone).utcOffset() * 60 : 0;
+      const utcOffsetPrefix = utcOffset ? `${utcOffset} + ` : '';
+      return [this.floorSql(`(${utcOffsetPrefix}${this.unixTimestampSql()}) / ${this.parseSecondDuration(every)}`), external, this];
     }
 
     const { dayOffset, utcOffset, interval } = this.calcIntervalForCronString(refreshKey);

packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts

@@ -276,7 +276,7 @@ describe('MSSqlPreAggregations', () => {
 
       expect(preAggregationsDescription[0].invalidateKeyQueries[0][0].replace(/(\r\n|\n|\r)/gm, '')
         .replace(/\s+/g, ' '))
-        .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END');
+        .toMatch('SELECT CASE WHEN CURRENT_TIMESTAMP < DATEADD(day, 7, CAST(@_1 AS DATETIMEOFFSET)) THEN FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END');
 
       return dbRunner
         .evaluateQueryWithPreAggregations(query)

packages/cubejs-schema-compiler/test/unit/base-query.test.ts

@@ -1183,7 +1183,7 @@ describe('SQL Generation', () => {
       const utcOffset = moment.tz('America/Los_Angeles').utcOffset() * 60;
       expect(query.everyRefreshKeySql({
         every: '1 hour'
-      })).toEqual(['FLOOR((EXTRACT(EPOCH FROM NOW())) / 3600)', false, expect.any(BaseQuery)]);
+      })).toEqual(['FLOOR((-25200 + EXTRACT(EPOCH FROM NOW())) / 3600)', false, expect.any(BaseQuery)]);
 
       // Standard syntax (minutes hours day month dow)
       expect(query.everyRefreshKeySql({ every: '0 * * * *', timezone }))
@@ -1290,7 +1290,7 @@ describe('SQL Generation', () => {
       expect(query.cacheKeyQueries()).toEqual([
         [
           // Postgres dialect
-          'SELECT FLOOR((EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key',
+          'SELECT FLOOR((-25200 + EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key',
           [],
           {
             // false, because there is no externalQueryClass
@@ -1311,15 +1311,15 @@ describe('SQL Generation', () => {
         ],
         timeDimensions: [],
         filters: [],
-        timezone: 'America/Los_Angeles',
+        timezone: 'Europe/London',
         externalQueryClass: MssqlQuery
       });
 
       // Query should not match any pre-aggregation!
       expect(query.cacheKeyQueries()).toEqual([
         [
           // MSSQL dialect, because externalQueryClass
-          'SELECT FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key',
+          'SELECT FLOOR((3600 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key',
           [],
           {
             // true, because externalQueryClass
@@ -1352,7 +1352,7 @@ describe('SQL Generation', () => {
       expect(preAggregations[0].invalidateKeyQueries).toEqual([
         [
           // MSSQL dialect
-          'SELECT FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) as refresh_key',
+          'SELECT FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) as refresh_key',
           [],
           {
             external: true,
@@ -1405,15 +1405,15 @@ describe('SQL Generation', () => {
           dateRange: ['2016-12-30', '2017-01-05']
         }],
         filters: [],
-        timezone: 'America/Los_Angeles',
+        timezone: 'Asia/Tokyo',
         externalQueryClass: MssqlQuery
       });
 
       const preAggregations: any = query.newPreAggregations().preAggregationsDescription();
       expect(preAggregations.length).toEqual(1);
       expect(preAggregations[0].invalidateKeyQueries).toEqual([
         [
-          'SELECT CASE\n    WHEN CURRENT_TIMESTAMP < CAST(@_1 AS DATETIMEOFFSET) THEN FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key',
+          'SELECT CASE\n    WHEN CURRENT_TIMESTAMP < CAST(@_1 AS DATETIMEOFFSET) THEN FLOOR((32400 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 3600) END as refresh_key',
           [
             '__TO_PARTITION_RANGE',
           ],
@@ -1496,7 +1496,7 @@ describe('SQL Generation', () => {
       expect(preAggregations.length).toEqual(1);
       expect(preAggregations[0].invalidateKeyQueries).toEqual([
         [
-          'SELECT FLOOR((EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key',
+          'SELECT FLOOR((-25200 + EXTRACT(EPOCH FROM NOW())) / 600) as refresh_key',
           [],
           {
             external: false,
@@ -1527,7 +1527,7 @@ describe('SQL Generation', () => {
       expect(preAggregations.length).toEqual(1);
       expect(preAggregations[0].invalidateKeyQueries).toEqual([
         [
-          'SELECT FLOOR((DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key',
+          'SELECT FLOOR((-25200 + DATEDIFF(SECOND,\'1970-01-01\', GETUTCDATE())) / 600) as refresh_key',
           [],
           {
             external: true,

packages/cubejs-server-core/src/core/OrchestratorApi.ts

@@ -71,7 +71,7 @@
    * error otherwise.
    */
   public async executeQuery(query: QueryBody) {
-    const queryForLog = query.query && query.query.replace(/\s+/g, ' ');
+    const queryForLog = query.query?.replace(/\s+/g, ' ');
     const startQueryTime = (new Date()).getTime();
 
     try {
@@ -174,7 +174,7 @@
   }
 
   /**
-   * Tests worker's connections to the Cubstore and, if not in the rollup only
+   * Tests worker's connections to the Cubestore and, if not in the rollup only
    * mode, to the datasources.
    */
   public async testConnection() {
@@ -297,11 +297,11 @@
     return this.orchestrator.cancelPreAggregationQueriesFromQueue(queryKeys, dataSource);
   }
 
-  public async subscribeQueueEvents(id, callback) {
+  public async subscribeQueueEvents(id: string, callback) {
     return this.orchestrator.subscribeQueueEvents(id, callback);
   }
 
-  public async unSubscribeQueueEvents(id) {
+  public async unSubscribeQueueEvents(id: string) {
     return this.orchestrator.unSubscribeQueueEvents(id);
   }
 

yarn.lock

@@ -21190,10 +21190,10 @@ moment-range@*, moment-range@^4.0.1, moment-range@^4.0.2:
   dependencies:
     es6-symbol "^3.1.0"
 
-moment-timezone@^0.5.15, moment-timezone@^0.5.33, moment-timezone@^0.5.46, moment-timezone@^0.5.47:
-  version "0.5.47"
-  resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.47.tgz#d4d1a21b78372d914d6d69ae285454732a429749"
-  integrity sha512-UbNt/JAWS0m/NJOebR0QMRHBk0hu03r5dx9GK8Cs0AS3I81yDcOc9k+DytPItgVvBP7J6Mf6U2n3BPAacAV9oA==
+moment-timezone@^0.5.15, moment-timezone@^0.5.33, moment-timezone@^0.5.46, moment-timezone@^0.5.47, moment-timezone@^0.5.48:
+  version "0.5.48"
+  resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.48.tgz#111727bb274734a518ae154b5ca589283f058967"
+  integrity sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw==
   dependencies:
     moment "^2.29.4"