fix(query-orchestrator): Fix improper pre-aggregation buildRange construction for non UTC timezones (#9284)

* improve partitionTableName() in PreAggregationPartitionRangeLoader

* make loadBuildRange() timezone-aware + align tests

* fix loadBuildRange() to use inDbTimeZone()

* return back utcToLocalTimeZone

* return back utcToLocalTimeZone in replaceQueryBuildRangeParams

* rename inDbTimeZone to localTimestampToUtc

* small improvement in BaseFilter format*Date

* remove unneeded

* another round in loadBuildRange()

* optimize to avoid doble ts convertion, add ts format to loadBuildRange()

* fix/specify some types

* fix invalidateKeyQueries not to be in the future

* fix correct extractDate parsing

* fix localTimestampToUtc()

* tests for all time functions

* revert change for sealAt

* fix tests

* fix extractDate and rename it to parseLocalDate

* fix tests for parseLocalDate()

* align PreAggregationPartitionRangeLoader with changes

* fix BaseDbRunner

* fix date formatting in replacePartitionSqlAndParams

* fix tests

* spelling

* add more tests for timeSeries()

* add tests for reformatUtcTimestamp()

* refactor alignToOrigin()

* convert to utc only in replacePartitionSqlAndParams()

* remove reformatUtcTimestamp() as oblsolete

* code polish

* remove unused redis-related utils

* move widely used ts format literal to const

* more tests for PreAggregationPartitionRangeLoader.intersectDateRanges()

* some improvements in tests

* some tests for PreAggregationPartitionRangeLoader

* some test polishment

Author: KSDaemon

packages/cubejs-backend-shared/src/time.ts

@@ -78,31 +78,17 @@ export function subtractInterval(date: moment.Moment, interval: ParsedInterval):
  */
 export const alignToOrigin = (startDate: moment.Moment, interval: ParsedInterval, origin: moment.Moment): moment.Moment => {
   let alignedDate = startDate.clone();
-  let intervalOp;
-  let isIntervalNegative = false;
-
-  let offsetDate = addInterval(origin, interval);
-
-  // The easiest way to check the interval sign
-  if (offsetDate.isBefore(origin)) {
-    isIntervalNegative = true;
-  }
-
-  offsetDate = origin.clone();
+  let offsetDate = origin.clone();
 
   if (startDate.isBefore(origin)) {
-    intervalOp = isIntervalNegative ? addInterval : subtractInterval;
-
     while (offsetDate.isAfter(startDate)) {
-      offsetDate = intervalOp(offsetDate, interval);
+      offsetDate = subtractInterval(offsetDate, interval);
     }
     alignedDate = offsetDate;
   } else {
-    intervalOp = isIntervalNegative ? subtractInterval : addInterval;
-
     while (offsetDate.isBefore(startDate)) {
       alignedDate = offsetDate.clone();
-      offsetDate = intervalOp(offsetDate, interval);
+      offsetDate = addInterval(offsetDate, interval);
     }
 
     if (offsetDate.isSame(startDate)) {
@@ -192,7 +178,13 @@ export const BUILD_RANGE_START_LOCAL = '__BUILD_RANGE_START_LOCAL';
 
 export const BUILD_RANGE_END_LOCAL = '__BUILD_RANGE_END_LOCAL';
 
-export const inDbTimeZone = (timezone: string, timestampFormat: string, timestamp: string): string => {
+/**
+ * Takes timestamp, treat it as time in provided timezone and returns the corresponding timestamp in UTC
+ */
+export const localTimestampToUtc = (timezone: string, timestampFormat: string, timestamp?: string): string | null => {
+  if (!timestamp) {
+    return null;
+  }
   if (timestamp.length === 23 || timestamp.length === 26) {
     const zone = moment.tz.zone(timezone);
     if (!zone) {
@@ -217,8 +209,14 @@ export const inDbTimeZone = (timezone: string, timestampFormat: string, timestam
     } else if (timestampFormat === 'YYYY-MM-DDTHH:mm:ss.SSS') {
       return inDbTimeZoneDate.toJSON().replace('Z', '');
     } else if (timestampFormat === 'YYYY-MM-DDTHH:mm:ss.SSSSSS') {
+      const value = inDbTimeZoneDate.toJSON();
+      if (value.endsWith('999Z')) {
+        // emulate microseconds
+        return value.replace('Z', '999');
+      }
+
       // emulate microseconds
-      return inDbTimeZoneDate.toJSON().replace('Z', '000');
+      return value.replace('Z', '000');
     }
   }
 
@@ -227,7 +225,13 @@ export const inDbTimeZone = (timezone: string, timestampFormat: string, timestam
   return moment.tz(timestamp, timezone).utc().format(timestampFormat);
 };
 
-export const utcToLocalTimeZone = (timezone: string, timestampFormat: string, timestamp: string): string => {
+/**
+ * Takes timestamp in UTC, shift it into provided timezone and returns the corresponding timestamp in UTC
+ */
+export const utcToLocalTimeZone = (timezone: string, timestampFormat: string, timestamp?: string): string | null => {
+  if (!timestamp) {
+    return null;
+  }
   if (timestamp.length === 23) {
     const zone = moment.tz.zone(timezone);
     if (!zone) {
@@ -247,16 +251,45 @@ export const utcToLocalTimeZone = (timezone: string, timestampFormat: string, ti
   return moment.tz(timestamp, 'UTC').tz(timezone).format(timestampFormat);
 };
 
-export const extractDate = (data: any): string | null => {
+export const parseLocalDate = (data: any, timezone: string, timestampFormat: string = 'YYYY-MM-DDTHH:mm:ss.SSS'): string | null => {
   if (!data) {
     return null;
   }
   data = JSON.parse(JSON.stringify(data));
   const value = data[0] && data[0][Object.keys(data[0])[0]];
   if (!value) {
-    return value;
+    return null;
+  }
+
+  const zone = moment.tz.zone(timezone);
+  if (!zone) {
+    throw new Error(`Unknown timezone: ${timezone}`);
+  }
+
+  // Most common formats
+  const formats = [
+    moment.ISO_8601,
+    'YYYY-MM-DD HH:mm:ss',
+    'YYYY-MM-DD HH:mm:ss.SSS',
+    'YYYY-MM-DDTHH:mm:ss.SSS',
+    'YYYY-MM-DDTHH:mm:ss'
+  ];
+
+  let parsedMoment;
+
+  if (value.includes('Z') || /([+-]\d{2}:?\d{2})$/.test(value.trim())) {
+    // We have timezone info
+    parsedMoment = moment(value, formats, true);
+  } else {
+    // If no tz info - use provided timezone
+    parsedMoment = moment.tz(value, formats, true, timezone);
   }
-  return moment.tz(value, 'UTC').utc().format(moment.HTML5_FMT.DATETIME_LOCAL_MS);
+
+  if (!parsedMoment.isValid()) {
+    return null;
+  }
+
+  return parsedMoment.tz(timezone).format(timestampFormat);
 };
 
 export const addSecondsToLocalTimestamp = (timestamp: string, timezone: string, seconds: number): Date => {