feat(tesseract): Pre-aggregation planning fallback

Author: waralexrom

packages/cubejs-backend-native/Cargo.lock

@@ -286,11 +286,9 @@ export class BaseQuery {
       return dimension;
     }).filter(R.identity).map(this.newTimeDimension.bind(this));
     this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters);
+    this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner');
+    this.prebuildJoin();
 
-    if (!getEnv('nativeSqlPlanner')) {
-      // Tesseract doesn't require join to be prebuilt and there's a case where single join can't be built for multi-fact query
-      this.join = this.joinGraph.buildJoin(this.allJoinHints);
-    }
     this.cubeAliasPrefix = this.options.cubeAliasPrefix;
     this.preAggregationsSchemaOption = this.options.preAggregationsSchema ?? DEFAULT_PREAGGREGATIONS_SCHEMA;
     this.externalQueryClass = this.options.externalQueryClass;
@@ -307,6 +305,15 @@ export class BaseQuery {
     this.initUngrouped();
   }
 
+  prebuildJoin() {
+    /* if (!this.useNativeSqlPlanner) { We still need this join for the follback to preaggregation to work properly. This condition should be returned after the tesseract starts working with pre-aggregations
+      // Tesseract doesn't require join to be prebuilt and there's a case where single join can't be built for multi-fact query
+      this.join = this.joinGraph.buildJoin(this.allJoinHints);
+    } */
+
+    this.join = this.joinGraph.buildJoin(this.allJoinHints);
+  }
+
   cacheValue(key, fn, { contextPropNames, inputProps, cache } = {}) {
     const currentContext = this.safeEvaluateSymbolContext();
     if (contextPropNames) {
@@ -383,8 +390,7 @@ export class BaseQuery {
   initUngrouped() {
     this.ungrouped = this.options.ungrouped;
     if (this.ungrouped) {
-      // this.join is not defined for Tesseract
-      if (!this.options.allowUngroupedWithoutPrimaryKey && !getEnv('nativeSqlPlanner')) {
+      if (!this.options.allowUngroupedWithoutPrimaryKey) {
         const cubes = R.uniq([this.join.root].concat(this.join.joins.map(j => j.originalTo)));
         const primaryKeyNames = cubes.flatMap(c => this.primaryKeyNames(c));
         const missingPrimaryKeys = primaryKeyNames.filter(key => !this.dimensions.find(d => d.dimension === key));
@@ -611,33 +617,61 @@ export class BaseQuery {
     return false;
   }
 
+  newQueryNotUseNative() {
+    const QueryClass = this.constructor;
+    return new QueryClass(this.compilers, { ...this.options, useNativeSqlPlanner: false });
+  }
+
   /**
    * Returns a pair of SQL query string and parameter values for the query.
    * @param {boolean} [exportAnnotatedSql] - returns annotated sql with not rendered params if true
    * @returns {[string, Array<unknown>]}
    */
   buildSqlAndParams(exportAnnotatedSql) {
-    if (getEnv('nativeSqlPlanner')) {
-      return this.buildSqlAndParamsRust(exportAnnotatedSql);
-    } else {
-      if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
-        if (this.externalPreAggregationQuery()) { // TODO performance
-          return this.externalQuery().buildSqlAndParams(exportAnnotatedSql);
+    if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
+      if (this.externalPreAggregationQuery()) { // TODO performance
+        return this.externalQuery().buildSqlAndParams(exportAnnotatedSql);
+      }
+    }
+
+    if (this.useNativeSqlPlanner) {
+      let isRelatedToPreAggregation = false;
+      if (this.options.preAggregationQuery) {
+        isRelatedToPreAggregation = true;
+      } else if (!this.options.disableExternalPreAggregations && this.externalQueryClass) {
+        if (this.externalPreAggregationQuery()) {
+          isRelatedToPreAggregation = true;
+        }
+      } else {
+        let preAggForQuery =
+          this.preAggregations.findPreAggregationForQuery();
+        if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) {
+          preAggForQuery = undefined;
+        }
+        if (preAggForQuery) {
+          isRelatedToPreAggregation = true;
         }
       }
-      return this.compilers.compiler.withQuery(
-        this,
-        () => this.cacheValue(
-          ['buildSqlAndParams', exportAnnotatedSql],
-          () => this.paramAllocator.buildSqlAndParams(
-            this.buildParamAnnotatedSql(),
-            exportAnnotatedSql,
-            this.shouldReuseParams
-          ),
-          { cache: this.queryCache }
-        )
-      );
+
+      if (isRelatedToPreAggregation) {
+        return this.newQueryNotUseNative().buildSqlAndParams(exportAnnotatedSql);
+      }
+
+      return this.buildSqlAndParamsRust(exportAnnotatedSql);
     }
+
+    return this.compilers.compiler.withQuery(
+      this,
+      () => this.cacheValue(
+        ['buildSqlAndParams', exportAnnotatedSql],
+        () => this.paramAllocator.buildSqlAndParams(
+          this.buildParamAnnotatedSql(),
+          exportAnnotatedSql,
+          this.shouldReuseParams
+        ),
+        { cache: this.queryCache }
+      )
+    );
   }
 
   buildSqlAndParamsRust(exportAnnotatedSql) {
@@ -3052,6 +3086,7 @@ export class BaseQuery {
   }
 
   newSubQueryForCube(cube, options) {
+    options = { ...options, useNativeSqlPlanner: false }; // We not use tesseract for pre-aggregations generation yet
     if (this.options.queryFactory) {
       // When dealing with rollup joins, it's crucial to use the correct parameter allocator for the specific cube in use.
       // By default, we'll use BaseQuery, but it's important to note that different databases (Oracle, PostgreSQL, MySQL, Druid, etc.)
@@ -3075,6 +3110,7 @@ export class BaseQuery {
       historyQueries: this.options.historyQueries,
       externalQueryClass: this.options.externalQueryClass,
       queryFactory: this.options.queryFactory,
+      useNativeSqlPlanner: this.options.useNativeSqlPlanner,
       ...options,
     };
   }

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -57,10 +57,6 @@ export class PreAggregations {
   }
 
   preAggregationCubes() {
-    if (getEnv('nativeSqlPlanner')) {
-      // No join defined in Tesseract
-      return [];
-    }
     const { join } = this.query;
     return join.joins.map(j => j.originalTo).concat([join.root]);
   }

packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

@@ -9,7 +9,6 @@ pub trait NativeContext<IT: InnerTypes>: Clone {
     fn null(&self) -> Result<NativeObjectHandle<IT>, CubeError>;
     fn empty_array(&self) -> Result<IT::Array, CubeError>;
     fn empty_struct(&self) -> Result<IT::Struct, CubeError>;
-    //fn boxed<T: 'static>(&self, value: T) -> impl NativeBox<IT, T>;
     fn to_string_fn(&self, result: String) -> Result<IT::Function, CubeError>;
 }
 

rust/cubenativeutils/src/wrappers/context.rs

@@ -16,7 +16,7 @@ serde = "1.0.115"
 serde_json = "1.0.56"
 cubenativeutils = { path = "../../cubenativeutils/" }
 minijinja = { version = "1", features = ["json", "loader"] }
-convert_case = "0.6"
+convert_case = "0.7.1"
 chrono = "0.4.15"
 chrono-tz = "0.8.2"
 lazy_static = "1.4.0"

rust/cubesqlplanner/Cargo.lock

@@ -9,6 +9,10 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use std::rc::Rc;
 
+const FROM_PARTITION_RANGE: &'static str = "__FROM_PARTITION_RANGE";
+
+const TO_PARTITION_RANGE: &'static str = "__TO_PARTITION_RANGE";
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum FilterType {
     Dimension,
@@ -393,7 +397,7 @@ impl BaseFilter {
             let from = if let Some(from_str) = &self.values[0] {
                 let from = self.format_from_date(&from_str)?;
 
-                if use_db_time_zone {
+                if use_db_time_zone && &from != FROM_PARTITION_RANGE {
                     self.query_tools.base_tools().in_db_time_zone(from)?
                 } else {
                     from
@@ -407,7 +411,7 @@ impl BaseFilter {
             let to = if let Some(to_str) = &self.values[1] {
                 let to = self.format_to_date(&to_str)?;
 
-                if use_db_time_zone {
+                if use_db_time_zone && &to != TO_PARTITION_RANGE {
                     self.query_tools.base_tools().in_db_time_zone(to)?
                 } else {
                     to

rust/cubesqlplanner/cubesqlplanner/Cargo.toml

@@ -65,6 +65,12 @@ impl SimpleQueryPlanner {
         }
         let render_references = dimension_subquery_planner.dimensions_refs().clone();
         context_factory.set_render_references(render_references);
+        context_factory.set_rendered_as_multiplied_measures(
+            self.query_properties
+                .full_key_aggregate_measures()?
+                .rendered_as_multiplied_measures
+                .clone(),
+        );
         select_builder.set_filter(filter);
         select_builder.set_group_by(self.query_properties.group_by());
         select_builder.set_order_by(self.order_planner.default_order());

rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs

@@ -11,7 +11,7 @@ use crate::plan::FilterItem;
 use crate::planner::sql_evaluator::collectors::collect_join_hints;
 use crate::planner::sql_templates::PlanSqlTemplates;
 use chrono_tz::Tz;
-use convert_case::{Case, Casing};
+use convert_case::{Boundary, Case, Casing};
 use cubenativeutils::CubeError;
 use itertools::Itertools;
 use lazy_static::lazy_static;
@@ -187,7 +187,9 @@ impl QueryTools {
     }
 
     pub fn alias_name(&self, name: &str) -> String {
-        name.to_case(Case::Snake).replace(".", "__")
+        name.without_boundaries(&[Boundary::LOWER_DIGIT, Boundary::UPPER_DIGIT])
+            .to_case(Case::Snake)
+            .replace(".", "__")
     }
 
     pub fn escaped_alias_name(&self, name: &str) -> String {

rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs

@@ -1,7 +1,7 @@
 use super::{TemplateGroupByColumn, TemplateOrderByColumn, TemplateProjectionColumn};
 use crate::cube_bridge::sql_templates_render::SqlTemplatesRender;
 use crate::plan::join::JoinType;
-use convert_case::{Case, Casing};
+use convert_case::{Boundary, Case, Casing};
 use cubenativeutils::CubeError;
 use minijinja::context;
 use std::rc::Rc;
@@ -17,7 +17,12 @@ impl PlanSqlTemplates {
     }
 
     pub fn alias_name(name: &str) -> String {
-        name.to_case(Case::Snake).replace(".", "__")
+        let res = name
+            .from_case(Case::Camel)
+            .without_boundaries(&[Boundary::LOWER_DIGIT, Boundary::UPPER_DIGIT])
+            .to_case(Case::Snake)
+            .replace(".", "__");
+        res
     }
 
     pub fn memeber_alias_name(cube_name: &str, name: &str, suffix: &Option<String>) -> String {