feat(tesseract): Support time series queries in Databricks (#9871)

* feat(tesseract): Support time series queries in Databricks

* run databricks drivers-tests with tesseract

* fix templates.statements.generated_time_series_select

* fix templates.statements.generated_time_series_with_cte_range_source

* enable supportGeneratedSeriesForCustomTd

* add snapshot: querying BigECommerce: rolling window by 2 day without date range

* add snapshot: querying BigECommerce: rolling window by 2 month without date range

* update snapshot: querying BigECommerce: rolling window YTD without date range

* add snapshot: querying BigECommerce with Retail Calendar

* add snapshot: querying custom granularities ECommerce: count by two_mo_by_feb + no dimension + rollingCountByLeading without date range

* add snapshot: SQL API: Extended nested Rollup over asterisk

Author: KSDaemon

.github/workflows/drivers-tests.yml

@@ -273,6 +273,8 @@ jobs:
             use_tesseract_sql_planner: true
           - database: athena-export-bucket-s3
             use_tesseract_sql_planner: true
+          - database: databricks-jdbc
+            use_tesseract_sql_planner: true
       fail-fast: false
 
     steps:

packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts

@@ -161,6 +161,10 @@ export class DatabricksQuery extends BaseQuery {
     return 120;
   }
 
+  public supportGeneratedSeriesForCustomTd() {
+    return true;
+  }
+
   public sqlTemplates() {
     const templates = super.sqlTemplates();
     templates.functions.CURRENTDATE = 'CURRENT_DATE';
@@ -178,6 +182,32 @@ export class DatabricksQuery extends BaseQuery {
     templates.expressions.interval_single_date_part = 'INTERVAL \'{{ num }}\' {{ date_part }}';
     templates.quotes.identifiers = '`';
     templates.quotes.escape = '``';
+    templates.statements.time_series_select = 'SELECT date_from::timestamp AS `date_from`,\n' +
+      'date_to::timestamp AS `date_to` \n' +
+      'FROM(\n' +
+      '    VALUES ' +
+      '{% for time_item in seria  %}' +
+      '(\'{{ time_item | join(\'\\\', \\\'\') }}\')' +
+      '{% if not loop.last %}, {% endif %}' +
+      '{% endfor %}' +
+      ') AS dates (date_from, date_to)';
+    templates.statements.generated_time_series_select = 'SELECT d AS date_from,\n' +
+      '(d + INTERVAL {{ granularity }}) - INTERVAL 1 MILLISECOND AS date_to\n' +
+      '  FROM (SELECT explode(sequence(\n' +
+      '    from_utc_timestamp({{ start }}, \'UTC\'), from_utc_timestamp({{ end }}, \'UTC\'), INTERVAL {{ granularity }}\n' +
+      '  )) AS d)';
+    templates.statements.generated_time_series_with_cte_range_source =
+    'SELECT d AS date_from,\n' +
+    '(d + INTERVAL {{ granularity }}) - INTERVAL 1 MILLISECOND AS date_to\n' +
+    'FROM {{ range_source }}\n' +
+    'LATERAL VIEW explode(\n' +
+    '    sequence(\n' +
+    '        CAST({{ min_name }} AS TIMESTAMP),\n' +
+    '        CAST({{ max_name }} AS TIMESTAMP),\n' +
+    '        INTERVAL {{ granularity }}\n' +
+    '    )\n' +
+    ') dates AS d';
+
     // TODO: Databricks has `TIMESTAMP_NTZ` with logic similar to Pg's `TIMESTAMP`
     // but that requires Runtime 13.3+. Should this be enabled?
     // templates.types.timestamp = 'TIMESTAMP_NTZ';

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -235,5 +235,48 @@
     "SQL API: Nested Rollup with aliases",
     "SQL API: Nested Rollup over asteriks",
     "SQL API: Extended nested Rollup over asterisk"
+  ],
+  "tesseractSkip": [
+    "must download query from the data source via memory",
+    "must download query from the data source via stream",
+    "for the Customers.RollingExternal",
+    "for the Customers.RollingInternal",
+    "for the ECommerce.SimpleAnalysisExternal",
+    "for the ECommerce.SimpleAnalysisInternal",
+    "for the ECommerce.TimeAnalysisInternal",
+    "for the ECommerce.TimeAnalysisExternal",
+    "querying Products: dimensions -- doesn't work wo ordering",
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
+    "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying custom granularities ECommerce: count by three_months_by_march + no dimension",
+    "querying custom granularities ECommerce: count by three_months_by_march + dimension",
+    "SQL API: SQL push down push to cube quoted alias",
+    "SQL API: Complex Rollup",
+    "SQL API: Simple Rollup",
+    "SQL API: Rollup over exprs",
+    "SQL API: Rollup with aliases",
+    "SQL API: Nested Rollup",
+    "SQL API: Nested Rollup with aliases",
+    "SQL API: Nested Rollup over asterisk",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)",
+
+    "---- Different results comparing to baseQuery version. Need to investigate ----",
+    "querying ECommerce: dimensions",
+    "querying ECommerce: dimensions + order",
+    "querying ECommerce: dimensions + limit",
+    "querying ECommerce: dimensions + total",
+    "querying ECommerce: dimensions + order + limit + total",
+    "querying ECommerce: dimensions + order + total + offset",
+    "querying ECommerce: dimensions + order + limit + total + offset",
+    "filtering ECommerce: contains dimensions, first",
+    "filtering ECommerce: contains dimensions, second",
+    "filtering ECommerce: startsWith + dimensions, first",
+    "filtering ECommerce: startsWith + dimensions, second",
+    "filtering ECommerce: endsWith + dimensions, first",
+    "filtering ECommerce: endsWith + dimensions, second",
+    "querying BigECommerce: rolling window YTD (month + week)",
+    "querying BigECommerce: rolling window YTD (month + week + no gran)"
   ]
 }