feat(tesseract): Custom granularities support (cube-js#9400)

Author: waralexrom

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -20,7 +20,8 @@ import {
   QueryAlias,
   getEnv,
   localTimestampToUtc,
-  timeSeries as timeSeriesBase
+  timeSeries as timeSeriesBase,
+  timeSeriesFromCustomInterval
 } from '@cubejs-backend/shared';
 
 import { UserError } from '../compiler/UserError';
@@ -720,11 +721,16 @@ export class BaseQuery {
     return this.paramAllocator.getParams();
   }
 
-  // FIXME helper for native generator, maybe should be moved entire to rust
+  // FIXME helper for native generator, maybe should be moved entirely to rust
   generateTimeSeries(granularity, dateRange) {
     return timeSeriesBase(granularity, dateRange);
   }
 
+  // FIXME helper for native generator, maybe should be moved entirely to rust
+  generateCustomTimeSeries(granularityInterval, dateRange, origin) {
+    return timeSeriesFromCustomInterval(granularityInterval, dateRange, moment(origin), { timestampPrecision: 3 });
+  }
+
   get shouldReuseParams() {
     return false;
   }

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -640,37 +640,32 @@ export class CubeSymbols {
 
   protected resolveSymbolsCallDeps(cubeName, sql) {
     try {
-      return this.resolveSymbolsCallDeps2(cubeName, sql);
+      const deps: any[] = [];
+      this.resolveSymbolsCall(sql, (name) => {
+        deps.push({ name });
+        const resolvedSymbol = this.resolveSymbol(
+          cubeName,
+          name
+        );
+        if (resolvedSymbol._objectWithResolvedProperties) {
+          return resolvedSymbol;
+        }
+        return '';
+      }, {
+        depsResolveFn: (name, parent) => {
+          deps.push({ name, parent });
+          return deps.length - 1;
+        },
+        currResolveIndexFn: () => deps.length - 1,
+        contextSymbols: this.depsContextSymbols(),
+
+      });
+      return deps;
     } catch (e) {
-      console.log(e);
       return [];
     }
   }
 
-  protected resolveSymbolsCallDeps2(cubeName, sql) {
-    const deps: any[] = [];
-    this.resolveSymbolsCall(sql, (name) => {
-      deps.push({ name, undefined });
-      const resolvedSymbol = this.resolveSymbol(
-        cubeName,
-        name
-      );
-      if (resolvedSymbol._objectWithResolvedProperties) {
-        return resolvedSymbol;
-      }
-      return '';
-    }, {
-      depsResolveFn: (name, parent) => {
-        deps.push({ name, parent });
-        return deps.length - 1;
-      },
-      currResolveIndexFn: () => deps.length - 1,
-      contextSymbols: this.depsContextSymbols(),
-
-    });
-    return deps;
-  }
-
   protected depsContextSymbols() {
     return Object.assign({
       filterParams: this.filtersProxyDep(),
@@ -719,7 +714,6 @@ export class CubeSymbols {
 
   public resolveSymbol(cubeName, name) {
     const { sqlResolveFn, contextSymbols, collectJoinHints, depsResolveFn, currResolveIndexFn } = this.resolveSymbolsCallContext || {};
-
     if (name === 'USER_CONTEXT') {
       throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.');
     }
@@ -758,6 +752,9 @@ export class CubeSymbols {
         const parentIndex = currResolveIndexFn();
         cube = this.cubeDependenciesProxy(parentIndex, newCubeName);
         return cube;
+      } else if (this.symbols[cubeName] && this.symbols[cubeName][name] && this.symbols[cubeName][name].type === 'time') {
+        const parentIndex = currResolveIndexFn();
+        return this.timeDimDependenciesProxy(parentIndex);
       }
     }
     return cube || (this.symbols[cubeName] && this.symbols[cubeName][name]);
@@ -877,6 +874,10 @@ export class CubeSymbols {
         }
         if (cube[propertyName]) {
           depsResolveFn(propertyName, parentIndex);
+          if (cube[propertyName].type === 'time') {
+            return this.timeDimDependenciesProxy(parentIndex);
+          }
+
           return '';
         }
         if (self.symbols[propertyName]) {
@@ -891,6 +892,25 @@ export class CubeSymbols {
     });
   }
 
+  protected timeDimDependenciesProxy(parentIndex) {
+    const self = this;
+    const { depsResolveFn } = self.resolveSymbolsCallContext || {};
+    return new Proxy({}, {
+      get: (v, propertyName) => {
+        if (propertyName === '_objectWithResolvedProperties') {
+          return true;
+        }
+        if (propertyName === 'toString') {
+          return () => '';
+        }
+        if (typeof propertyName === 'string') {
+          depsResolveFn(propertyName, parentIndex);
+        }
+        return undefined;
+      }
+    });
+  }
+
   public isCurrentCube(name) {
     return CURRENT_CUBE_CONSTANTS.indexOf(name) >= 0;
   }

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -968,6 +968,80 @@ SELECT 1 AS revenue,  cast('2024-01-01' AS timestamp) as time UNION ALL
     }
   ]));
 
+  it('rolling window with two time dimension granularities one custom one regular', async () => runQueryTest({
+
+    measures: [
+      'visitors.countRollingWeekToDate'
+    ],
+    timeDimensions: [
+      {
+        dimension: 'visitors.created_at',
+        granularity: 'three_days',
+        dateRange: ['2017-01-01', '2017-01-10']
+      },
+      {
+        dimension: 'visitors.created_at',
+        granularity: 'day',
+        dateRange: ['2017-01-01', '2017-01-10']
+      }
+    ],
+    order: [{
+      id: 'visitors.created_at'
+    }],
+    timezone: 'America/Los_Angeles'
+  }, [
+    {
+      visitors__count_rolling_week_to_date: null,
+      visitors__created_at_day: '2017-01-01T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: '1',
+      visitors__created_at_day: '2017-01-02T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: '1',
+      visitors__created_at_day: '2017-01-03T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: '2',
+      visitors__created_at_day: '2017-01-04T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: '3',
+      visitors__created_at_day: '2017-01-05T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: '5',
+      visitors__created_at_day: '2017-01-06T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: '5',
+      visitors__created_at_day: '2017-01-07T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: '5',
+      visitors__created_at_day: '2017-01-08T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: null,
+      visitors__created_at_day: '2017-01-09T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
+    },
+    {
+      visitors__count_rolling_week_to_date: null,
+      visitors__created_at_day: '2017-01-10T00:00:00.000Z',
+      visitors__created_at_three_days: '2017-01-10T00:00:00.000Z',
+    }
+  ]));
+
   it('two rolling windows with two time dimension granularities', async () => runQueryTest({
     measures: [
       'visitors.countRollingUnbounded',

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

@@ -1,7 +1,6 @@
 use super::base_query_options::FilterItem;
 use super::filter_group::{FilterGroup, NativeFilterGroup};
 use super::filter_params::{FilterParams, NativeFilterParams};
-use super::member_sql::{MemberSql, NativeMemberSql};
 use super::security_context::{NativeSecurityContext, SecurityContext};
 use super::sql_templates_render::{NativeSqlTemplatesRender, SqlTemplatesRender};
 use super::sql_utils::{NativeSqlUtils, SqlUtils};
@@ -11,16 +10,9 @@ use cubenativeutils::wrappers::serializer::{
 use cubenativeutils::wrappers::NativeContextHolder;
 use cubenativeutils::wrappers::NativeObjectHandle;
 use cubenativeutils::CubeError;
-use serde::Deserialize;
 use std::any::Any;
 use std::rc::Rc;
 
-#[derive(Deserialize, Debug)]
-pub struct CallDep {
-    pub name: String,
-    pub parent: Option<usize>,
-}
-
 #[nativebridge::native_bridge]
 pub trait BaseTools {
     fn convert_tz(&self, field: String) -> Result<String, CubeError>;
@@ -30,11 +22,6 @@ pub trait BaseTools {
         dimension: String,
     ) -> Result<String, CubeError>;
     fn sql_templates(&self) -> Result<Rc<dyn SqlTemplatesRender>, CubeError>;
-    fn resolve_symbols_call_deps(
-        &self,
-        cube_name: String,
-        sql: Rc<dyn MemberSql>,
-    ) -> Result<Vec<CallDep>, CubeError>;
     fn security_context_for_rust(&self) -> Result<Rc<dyn SecurityContext>, CubeError>;
     fn sql_utils_for_rust(&self) -> Result<Rc<dyn SqlUtils>, CubeError>;
     fn filters_proxy_for_rust(
@@ -52,11 +39,23 @@ pub trait BaseTools {
         granularity: String,
         date_range: Vec<String>,
     ) -> Result<Vec<Vec<String>>, CubeError>;
+    fn generate_custom_time_series(
+        &self,
+        granularity: String,
+        date_range: Vec<String>,
+        origin: String,
+    ) -> Result<Vec<Vec<String>>, CubeError>;
     fn get_allocated_params(&self) -> Result<Vec<String>, CubeError>;
     fn all_cube_members(&self, path: String) -> Result<Vec<String>, CubeError>;
     //===== TODO Move to templates
     fn hll_init(&self, sql: String) -> Result<String, CubeError>;
     fn hll_merge(&self, sql: String) -> Result<String, CubeError>;
     fn hll_cardinality_merge(&self, sql: String) -> Result<String, CubeError>;
     fn count_distinct_approx(&self, sql: String) -> Result<String, CubeError>;
+    fn date_bin(
+        &self,
+        interval: String,
+        source: String,
+        origin: String,
+    ) -> Result<String, CubeError>;
 }

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/dimension_definition.rs

@@ -11,6 +11,12 @@ use serde::{Deserialize, Serialize};
 use std::any::Any;
 use std::rc::Rc;
 
+#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct GranularityDefinition {
+    pub interval: String,
+    pub origin: Option<String>,
+    pub offset: Option<String>,
+}
 #[derive(Serialize, Deserialize, Debug)]
 pub struct DimenstionDefinitionStatic {
     #[serde(rename = "type")]

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs

@@ -1,5 +1,7 @@
 use super::cube_definition::{CubeDefinition, NativeCubeDefinition};
-use super::dimension_definition::{DimensionDefinition, NativeDimensionDefinition};
+use super::dimension_definition::{
+    DimensionDefinition, GranularityDefinition, NativeDimensionDefinition,
+};
 use super::measure_definition::{MeasureDefinition, NativeMeasureDefinition};
 use super::member_sql::{MemberSql, NativeMemberSql};
 use super::segment_definition::{NativeSegmentDefinition, SegmentDefinition};
@@ -48,4 +50,5 @@ pub trait CubeEvaluator {
         cube_name: String,
         sql: Rc<dyn MemberSql>,
     ) -> Result<Vec<CallDep>, CubeError>;
+    fn resolve_granularity(&self, path: Vec<String>) -> Result<GranularityDefinition, CubeError>;
 }

rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs

@@ -2,6 +2,7 @@ use super::{Schema, SchemaColumn};
 use crate::planner::{
     query_tools::QueryTools,
     sql_templates::{PlanSqlTemplates, TemplateProjectionColumn},
+    Granularity,
 };
 use cubenativeutils::CubeError;
 use std::rc::Rc;
@@ -11,7 +12,7 @@ pub struct TimeSeries {
     #[allow(dead_code)]
     time_dimension_name: String,
     date_range: TimeSeriesDateRange,
-    granularity: String,
+    granularity: Granularity,
     schema: Rc<Schema>,
 }
 
@@ -25,7 +26,7 @@ impl TimeSeries {
         query_tools: Rc<QueryTools>,
         time_dimension_name: String,
         date_range: TimeSeriesDateRange,
-        granularity: String,
+        granularity: Granularity,
     ) -> Self {
         let column = SchemaColumn::new(format!("date_from"), Some(time_dimension_name.clone()));
         let schema = Rc::new(Schema::new(vec![column]));
@@ -88,7 +89,11 @@ impl TimeSeries {
                     (format!("({})", from), format!("({})", to))
                 }
             };
-            templates.generated_time_series_select(&from_date, &to_date, &self.granularity)
+            templates.generated_time_series_select(
+                &from_date,
+                &to_date,
+                &self.granularity.granularity_interval(),
+            )
         } else {
             let (from_date, to_date) = match &self.date_range {
                 TimeSeriesDateRange::Filter(from_date, to_date) => {
@@ -100,10 +105,18 @@ impl TimeSeries {
                     ));
                 }
             };
-            let series = self.query_tools.base_tools().generate_time_series(
-                self.granularity.clone(),
-                vec![from_date.clone(), to_date.clone()],
-            )?;
+            let series = if self.granularity.is_predefined_granularity() {
+                self.query_tools.base_tools().generate_time_series(
+                    self.granularity.granularity().clone(),
+                    vec![from_date.clone(), to_date.clone()],
+                )?
+            } else {
+                self.query_tools.base_tools().generate_custom_time_series(
+                    self.granularity.granularity_interval().clone(),
+                    vec![from_date.clone(), to_date.clone()],
+                    self.granularity.origin_local_formatted(),
+                )?
+            };
             templates.time_series_select(from_date.clone(), to_date.clone(), series)
         }
     }

rust/cubesqlplanner/cubesqlplanner/src/planner/base_dimension.rs

@@ -132,6 +132,10 @@ impl BaseDimension {
         self.member_evaluator.clone()
     }
 
+    pub fn definition(&self) -> Option<Rc<dyn DimensionDefinition>> {
+        self.definition.clone()
+    }
+
     pub fn sql_call(&self) -> Result<Rc<SqlCall>, CubeError> {
         match self.member_evaluator.as_ref() {
             MemberSymbol::Dimension(d) => {