feat(tesseract): Initial BigQuery support (#9577)

Author: waralexrom

.github/workflows/drivers-tests.yml

@@ -57,9 +57,19 @@ on:
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'
       - 'rust/cubesql/**'
+  workflow_dispatch:
+    inputs:
+      use_tesseract_sql_planner:
+        description: 'Enable TESSERACT_SQL_PLANNER?'
+        required: true
+        default: 'false'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
 
 env:
-  CUBEJS_TESSERACT_ORCHESTRATOR: true
+  USE_TESSERACT_SQL_PLANNER: false
 
 jobs:
   latest-tag-sha:
@@ -316,6 +326,8 @@ jobs:
           (contains(env.CLOUD_DATABASES, matrix.database) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
           (!contains(env.CLOUD_DATABASES, matrix.database))
         env:
+          DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.use_tesseract_sql_planner || env.USE_TESSERACT_SQL_PLANNER }}
+
           # Athena
           DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
           DRIVERS_TESTS_ATHENA_CUBEJS_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_SECRET }}

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -330,7 +330,7 @@ export class BaseQuery {
     this.customSubQueryJoins = this.options.subqueryJoins ?? [];
     this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner');
     this.canUseNativeSqlPlannerPreAggregation = false;
-    if (this.useNativeSqlPlanner) {
+    if (this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation()) {
       const hasMultiStageMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }).multiStageMembers.length > 0;
       this.canUseNativeSqlPlannerPreAggregation = hasMultiStageMeasures;
     }
@@ -348,6 +348,11 @@ export class BaseQuery {
     this.initUngrouped();
   }
 
+  // Temporary workaround to avoid checking for multistage in CubeStoreQuery, since that could lead to errors when HLL functions are present in the query.
+  neverUseSqlPlannerPreaggregation() {
+    return false;
+  }
+
   prebuildJoin() {
     try {
       // TODO allJoinHints should contain join hints form pre-agg
@@ -774,7 +779,6 @@ export class BaseQuery {
       R.map((hash) => ((!hash || !hash.id) ? null : hash)),
       R.reject(R.isNil),
     )(this.options.order);
-
     const queryParams = {
       measures: this.options.measures,
       dimensions: this.options.dimensions,
@@ -791,7 +795,8 @@ export class BaseQuery {
       baseTools: this,
       ungrouped: this.options.ungrouped,
       exportAnnotatedSql: exportAnnotatedSql === true,
-      preAggregationQuery: this.options.preAggregationQuery
+      preAggregationQuery: this.options.preAggregationQuery,
+      totalQuery: this.options.totalQuery,
     };
 
     const buildResult = nativeBuildSqlAndParams(queryParams);
@@ -870,12 +875,12 @@ export class BaseQuery {
 
   // FIXME helper for native generator, maybe should be moved entirely to rust
   generateTimeSeries(granularity, dateRange) {
-    return timeSeriesBase(granularity, dateRange);
+    return timeSeriesBase(granularity, dateRange, { timestampPrecision: this.timestampPrecision() });
   }
 
   // FIXME helper for native generator, maybe should be moved entirely to rust
   generateCustomTimeSeries(granularityInterval, dateRange, origin) {
-    return timeSeriesFromCustomInterval(granularityInterval, dateRange, moment(origin), { timestampPrecision: 3 });
+    return timeSeriesFromCustomInterval(granularityInterval, dateRange, moment(origin), { timestampPrecision: this.timestampPrecision() });
   }
 
   getPreAggregationByName(cube, preAggregationName) {
@@ -3869,6 +3874,9 @@ export class BaseQuery {
         like_escape: '{{ like_expr }} ESCAPE {{ escape_char }}',
         concat_strings: '{{ strings | join(\' || \' ) }}',
       },
+      tesseract: {
+        ilike: '{{ expr }} {% if negated %}NOT {% endif %}ILIKE {{ pattern }}', // May require different overloads in Tesseract than the ilike from expressions used in SQLAPI.
+      },
       filters: {
         equals: '{{ column }} = {{ value }}{{ is_null_check }}',
         not_equals: '{{ column }} <> {{ value }}{{ is_null_check }}',

packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts

@@ -261,13 +261,24 @@ export class BigqueryQuery extends BaseQuery {
     templates.expressions.timestamp_literal = 'TIMESTAMP(\'{{ value }}\')';
     delete templates.expressions.ilike;
     delete templates.expressions.like_escape;
+    templates.filters.like_pattern = 'CONCAT({% if start_wild %}\'%\'{% else %}\'\'{% endif %}, LOWER({{ value }}), {% if end_wild %}\'%\'{% else %}\'\'{% endif %})';
+    templates.tesseract.ilike = 'LOWER({{ expr }}) {% if negated %}NOT {% endif %} LIKE {{ pattern }}';
     templates.types.boolean = 'BOOL';
     templates.types.float = 'FLOAT64';
     templates.types.double = 'FLOAT64';
     templates.types.decimal = 'BIGDECIMAL({{ precision }},{{ scale }})';
     templates.types.binary = 'BYTES';
+    templates.expressions.cast_to_string = 'CAST({{ expr }} AS STRING)';
     templates.operators.is_not_distinct_from = 'IS NOT DISTINCT FROM';
     templates.join_types.full = 'FULL';
+    templates.statements.time_series_select = 'SELECT DATETIME(TIMESTAMP(f)) date_from, DATETIME(TIMESTAMP(t)) date_to \n' +
+    'FROM (\n' +
+    '{% for time_item in seria  %}' +
+    '    select \'{{ time_item[0] }}\' f, \'{{ time_item[1] }}\' t \n' +
+    '{% if not loop.last %} UNION ALL\n{% endif %}' +
+    '{% endfor %}' +
+    ') AS dates';
+      
     return templates;
   }
 }

packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts

@@ -68,6 +68,11 @@ export class CubeStoreQuery extends BaseQuery {
     return `date_trunc('${GRANULARITY_TO_INTERVAL[granularity]}', ${dimension})`;
   }
 
+  // Temporary workaround to avoid checking for multistage in CubeStoreQuery, since that could lead to errors when HLL functions are present in the query.
+  public neverUseSqlPlannerPreaggregation() {
+    return true;
+  }
+
   /**
    * Returns sql for source expression floored to timestamps aligned with
    * intervals relative to origin timestamp point.

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -15,7 +15,8 @@
       "CUBEJS_PG_SQL_PORT": "5656",
       "CUBEJS_SQL_USER": "admin",
       "CUBEJS_SQL_PASSWORD": "admin_password",
-      "CUBESQL_SQL_PUSH_DOWN": "true"
+      "CUBESQL_SQL_PUSH_DOWN": "true",
+      "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}"
     },
     "ports" : ["4000", "5656"]
   },

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -17,7 +17,8 @@
       "CUBESQL_SQL_PUSH_DOWN": "true",
 
       "CUBEJS_DB_EXPORT_BUCKET": "cube-open-source-export-bucket",
-      "CUBEJS_DB_EXPORT_BUCKET_TYPE": "gcp"
+      "CUBEJS_DB_EXPORT_BUCKET_TYPE": "gcp",
+      "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}"
     },
     "ports" : ["4000", "5656"]
   },

packages/cubejs-testing-drivers/fixtures/clickhouse.json

@@ -25,7 +25,8 @@
       "CUBEJS_PG_SQL_PORT": "5656",
       "CUBEJS_SQL_USER": "admin",
       "CUBEJS_SQL_PASSWORD": "admin_password",
-      "CUBESQL_SQL_PUSH_DOWN": "true"
+      "CUBESQL_SQL_PUSH_DOWN": "true",
+      "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}"
     },
     "depends_on": ["data"],
     "links": ["data"],

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -73,7 +73,8 @@
       "CUBEJS_PG_SQL_PORT": "5656",
       "CUBEJS_SQL_USER": "admin",
       "CUBEJS_SQL_PASSWORD": "admin_password",
-      "CUBESQL_SQL_PUSH_DOWN": "true"
+      "CUBESQL_SQL_PUSH_DOWN": "true",
+      "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}"
     },
     "ports" : ["4000", "5656"]
   },

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -11,7 +11,8 @@
       "CUBEJS_PG_SQL_PORT": "5656",
       "CUBEJS_SQL_USER": "admin",
       "CUBEJS_SQL_PASSWORD": "admin_password",
-      "CUBESQL_SQL_PUSH_DOWN": "true"
+      "CUBESQL_SQL_PUSH_DOWN": "true",
+      "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}"
     },
     "depends_on": ["data"],
     "links": ["data"],

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -12,7 +12,8 @@
       "CUBEJS_PG_SQL_PORT": "5656",
       "CUBEJS_SQL_USER": "admin",
       "CUBEJS_SQL_PASSWORD": "admin_password",
-      "CUBESQL_SQL_PUSH_DOWN": "true"
+      "CUBESQL_SQL_PUSH_DOWN": "true",
+      "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}"
     },
     "depends_on": ["data"],
     "links": ["data"],