feat(tesseract): Initial BigQuery support

Author: waralexrom

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -326,7 +326,7 @@ export class BaseQuery {
     this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters);
     this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner');
     this.canUseNativeSqlPlannerPreAggregation = false;
-    if (this.useNativeSqlPlanner) {
+    if (this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation()) {
       const hasMultiStageMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }).multiStageMembers.length > 0;
       this.canUseNativeSqlPlannerPreAggregation = hasMultiStageMeasures;
     }
@@ -349,6 +349,11 @@ export class BaseQuery {
     this.initUngrouped();
   }
 
+  // Temporary workaround to avoid checking for multistage in CubeStoreQuery, since that could lead to errors when HLL functions are present in the query.
+  neverUseSqlPlannerPreaggregation() {
+    return false;
+  }
+
   prebuildJoin() {
     try {
       // TODO allJoinHints should contain join hints form pre-agg
@@ -747,7 +752,8 @@ export class BaseQuery {
         }
       }
 
-      return this.buildSqlAndParamsRust(exportAnnotatedSql);
+      const res = this.buildSqlAndParamsRust(exportAnnotatedSql);
+      return res;
     }
 
     if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
@@ -756,7 +762,7 @@ export class BaseQuery {
       }
     }
 
-    return this.compilers.compiler.withQuery(
+    const res = this.compilers.compiler.withQuery(
       this,
       () => this.cacheValue(
         ['buildSqlAndParams', exportAnnotatedSql],
@@ -768,14 +774,14 @@ export class BaseQuery {
         { cache: this.queryCache }
       )
     );
+    return res;
   }
 
   buildSqlAndParamsRust(exportAnnotatedSql) {
     const order = this.options.order && R.pipe(
       R.map((hash) => ((!hash || !hash.id) ? null : hash)),
       R.reject(R.isNil),
     )(this.options.order);
-
     const queryParams = {
       measures: this.options.measures,
       dimensions: this.options.dimensions,
@@ -792,7 +798,8 @@ export class BaseQuery {
       baseTools: this,
       ungrouped: this.options.ungrouped,
       exportAnnotatedSql: exportAnnotatedSql === true,
-      preAggregationQuery: this.options.preAggregationQuery
+      preAggregationQuery: this.options.preAggregationQuery,
+      totalQuery: this.options.totalQuery,
     };
 
     const buildResult = nativeBuildSqlAndParams(queryParams);
@@ -871,12 +878,12 @@ export class BaseQuery {
 
   // FIXME helper for native generator, maybe should be moved entirely to rust
   generateTimeSeries(granularity, dateRange) {
-    return timeSeriesBase(granularity, dateRange);
+    return timeSeriesBase(granularity, dateRange, { timestampPrecision: this.timestampPrecision() });
   }
 
   // FIXME helper for native generator, maybe should be moved entirely to rust
   generateCustomTimeSeries(granularityInterval, dateRange, origin) {
-    return timeSeriesFromCustomInterval(granularityInterval, dateRange, moment(origin), { timestampPrecision: 3 });
+    return timeSeriesFromCustomInterval(granularityInterval, dateRange, moment(origin), { timestampPrecision: this.timestampPrecision() });
   }
 
   getPreAggregationByName(cube, preAggregationName) {
@@ -3827,6 +3834,9 @@ export class BaseQuery {
         like_escape: '{{ like_expr }} ESCAPE {{ escape_char }}',
         concat_strings: '{{ strings | join(\' || \' ) }}',
       },
+      tesseract: {
+        ilike: '{{ expr }} {% if negated %}NOT {% endif %}ILIKE {{ pattern }}', // May require different overloads in Tesseract than the ilike from expressions used in SQLAPI.
+      },
       filters: {
         equals: '{{ column }} = {{ value }}{{ is_null_check }}',
         not_equals: '{{ column }} <> {{ value }}{{ is_null_check }}',

packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts

@@ -261,13 +261,24 @@ export class BigqueryQuery extends BaseQuery {
     templates.expressions.timestamp_literal = 'TIMESTAMP(\'{{ value }}\')';
     delete templates.expressions.ilike;
     delete templates.expressions.like_escape;
+    templates.filters.like_pattern = 'CONCAT({% if start_wild %}\'%\'{% else %}\'\'{% endif %}, LOWER({{ value }}), {% if end_wild %}\'%\'{% else %}\'\'{% endif %})';
+    templates.tesseract.ilike = 'LOWER({{ expr }}) {% if negated %}NOT {% endif %} LIKE {{ pattern }}';
     templates.types.boolean = 'BOOL';
     templates.types.float = 'FLOAT64';
     templates.types.double = 'FLOAT64';
     templates.types.decimal = 'BIGDECIMAL({{ precision }},{{ scale }})';
     templates.types.binary = 'BYTES';
+    templates.expressions.cast_to_string = 'CAST({{ expr }} AS STRING)';
     templates.operators.is_not_distinct_from = 'IS NOT DISTINCT FROM';
     templates.join_types.full = 'FULL';
+    templates.statements.time_series_select = 'SELECT DATETIME(TIMESTAMP(f)) date_from, DATETIME(TIMESTAMP(t)) date_to \n' +
+    'FROM (\n' +
+    '{% for time_item in seria  %}' +
+    '    select \'{{ time_item[0] }}\' f, \'{{ time_item[1] }}\' t \n' +
+    '{% if not loop.last %} UNION ALL\n{% endif %}' +
+    '{% endfor %}' +
+    ') AS dates';
+      
     return templates;
   }
 }

packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts

@@ -68,6 +68,11 @@ export class CubeStoreQuery extends BaseQuery {
     return `date_trunc('${GRANULARITY_TO_INTERVAL[granularity]}', ${dimension})`;
   }
 
+  // Temporary workaround to avoid checking for multistage in CubeStoreQuery, since that could lead to errors when HLL functions are present in the query.
+  public neverUseSqlPlannerPreaggregation() {
+    return true;
+  }
+
   /**
    * Returns sql for source expression floored to timestamps aligned with
    * intervals relative to origin timestamp point.

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -17,7 +17,8 @@
       "CUBESQL_SQL_PUSH_DOWN": "true",
 
       "CUBEJS_DB_EXPORT_BUCKET": "cube-open-source-export-bucket",
-      "CUBEJS_DB_EXPORT_BUCKET_TYPE": "gcp"
+      "CUBEJS_DB_EXPORT_BUCKET_TYPE": "gcp",
+      "_CUBEJS_TESSERACT_SQL_PLANNER": "true"
     },
     "ports" : ["4000", "5656"]
   },

packages/cubejs-testing-drivers/fixtures/postgres.json

@@ -12,7 +12,8 @@
       "CUBEJS_PG_SQL_PORT": "5656",
       "CUBEJS_SQL_USER": "admin",
       "CUBEJS_SQL_PASSWORD": "admin_password",
-      "CUBESQL_SQL_PUSH_DOWN": "true"
+      "CUBESQL_SQL_PUSH_DOWN": "true",
+      "_CUBEJS_TESSERACT_SQL_PLANNER": "true"
     },
     "depends_on": ["data"],
     "links": ["data"],

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs

@@ -63,6 +63,8 @@ pub struct BaseQueryOptionsStatic {
     pub export_annotated_sql: bool,
     #[serde(rename = "preAggregationQuery")]
     pub pre_aggregation_query: Option<bool>,
+    #[serde(rename = "totalQuery")]
+    pub total_query: Option<bool>,
 }
 
 #[nativebridge::native_bridge(BaseQueryOptionsStatic)]

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

@@ -34,6 +34,8 @@ pub trait BaseTools {
         used_filters: Option<Vec<FilterItem>>,
     ) -> Result<Rc<dyn FilterGroup>, CubeError>;
     fn timestamp_precision(&self) -> Result<u32, CubeError>;
+    fn time_stamp_cast(&self, field: String) -> Result<String, CubeError>; //TODO move to templates
+    fn date_time_cast(&self, field: String) -> Result<String, CubeError>; //TODO move to templates
     fn in_db_time_zone(&self, date: String) -> Result<String, CubeError>;
     fn generate_time_series(
         &self,
@@ -47,6 +49,8 @@ pub trait BaseTools {
         origin: String,
     ) -> Result<Vec<Vec<String>>, CubeError>;
     fn get_allocated_params(&self) -> Result<Vec<String>, CubeError>;
+    fn subtract_interval(&self, date: String, interval: String) -> Result<String, CubeError>;
+    fn add_interval(&self, date: String, interval: String) -> Result<String, CubeError>;
     fn all_cube_members(&self, path: String) -> Result<Vec<String>, CubeError>;
     //===== TODO Move to templates
     fn hll_init(&self, sql: String) -> Result<String, CubeError>;

rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs

@@ -16,6 +16,8 @@ use itertools::Itertools;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::rc::Rc;
+const TOTAL_COUNT: &'static str = "total_count";
+const ORIGINAL_QUERY: &'static str = "original_query";
 
 #[derive(Clone, Debug)]
 struct PhysicalPlanBuilderContext {
@@ -56,7 +58,7 @@ pub struct PhysicalPlanBuilder {
 
 impl PhysicalPlanBuilder {
     pub fn new(query_tools: Rc<QueryTools>) -> Self {
-        let plan_sql_templates = PlanSqlTemplates::new(query_tools.templates_render());
+        let plan_sql_templates = query_tools.plan_sql_templates();
         Self {
             query_tools,
             plan_sql_templates,
@@ -67,10 +69,25 @@ impl PhysicalPlanBuilder {
         &self,
         logical_plan: Rc<Query>,
         original_sql_pre_aggregations: HashMap<String, String>,
+        total_query: bool,
     ) -> Result<Rc<Select>, CubeError> {
         let mut context = PhysicalPlanBuilderContext::default();
         context.original_sql_pre_aggregations = original_sql_pre_aggregations;
-        self.build_impl(logical_plan, &context)
+        let query = self.build_impl(logical_plan, &context)?;
+        let query = if total_query {
+            self.build_total_count(query, &context)?
+        } else {
+            query
+        };
+        Ok(query)
+    }
+
+    fn build_total_count(&self, source: Rc<Select>, context: &PhysicalPlanBuilderContext) -> Result<Rc<Select>, CubeError> {
+        let from = From::new_from_subselect(source.clone(), ORIGINAL_QUERY.to_string());
+        let mut select_builder = SelectBuilder::new(from);
+        select_builder.add_count_all(TOTAL_COUNT.to_string());
+        let context_factory = context.make_sql_nodes_factory();
+        Ok(Rc::new(select_builder.build(context_factory)))
     }
 
     fn build_impl(
@@ -957,7 +974,7 @@ impl PhysicalPlanBuilder {
             ));
         };
 
-        let templates = PlanSqlTemplates::new(self.query_tools.templates_render());
+        let templates = self.query_tools.plan_sql_templates();
 
         let ts_date_range = if templates.supports_generated_time_series() {
             if let Some(date_range) = time_dimension_symbol

rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs

@@ -75,6 +75,18 @@ impl SelectBuilder {
             .add_column(SchemaColumn::new(alias.clone(), Some(member.full_name())));
     }
 
+    pub fn add_count_all(&mut self, alias: String) {
+        let func = Expr::Function(FunctionExpression {
+            function: "COUNT".to_string(),
+            arguments: vec![Expr::Asterisk],
+        });
+        let aliased_expr = AliasedExpr {
+            expr: func,
+            alias: alias.clone(),
+        };
+        self.projection_columns.push(aliased_expr);
+        self.result_schema.add_column(SchemaColumn::new(alias.clone(), None));
+    }
     pub fn add_projection_function_expression(
         &mut self,
         function: &str,

rust/cubesqlplanner/cubesqlplanner/src/plan/expression.rs

@@ -34,6 +34,7 @@ pub enum Expr {
     Member(MemberExpression),
     Reference(QualifiedColumnName),
     Function(FunctionExpression),
+    Asterisk,
 }
 
 impl Expr {
@@ -65,6 +66,7 @@ impl Expr {
                 None,
                 None,
             ),
+            Self::Asterisk => Ok("*".to_string()),
         }
     }
 }