fix(tesseract): Dimnesions with granularity in group_by + drivers support

.github/workflows/drivers-tests.yml

@@ -269,6 +269,10 @@ jobs:
         include:
           - database: postgres
             use_tesseract_sql_planner: true
+          - database: snowflake
+            use_tesseract_sql_planner: true
+          - database: redshift
+            use_tesseract_sql_planner: true
           - database: bigquery-export-bucket-gcs
             use_tesseract_sql_planner: true
           - database: athena-export-bucket-s3

packages/cubejs-schema-compiler/src/adapter/RedshiftQuery.ts

@@ -85,6 +85,15 @@ export class RedshiftQuery extends PostgresQuery {
     const templates = super.sqlTemplates();
     templates.functions.DLOG10 = 'LOG(10, {{ args_concat }})';
     templates.functions.DATEDIFF = 'DATEDIFF({{ date_part }}, {{ args[1] }}, {{ args[2] }})';
+    templates.statements.time_series_select = 'SELECT dates.f::timestamp date_from, dates.t::timestamp date_to \n' +
+    'FROM (\n' +
+    '{% for time_item in seria  %}' +
+    '    select \'{{ time_item[0] }}\' f, \'{{ time_item[1] }}\' t \n' +
+    '{% if not loop.last %} UNION ALL\n{% endif %}' +
+    '{% endfor %}' +
+    ') AS dates';
+    delete templates.statements.generated_time_series_select;
+    delete templates.operators.is_not_distinct_from;
     delete templates.functions.COVAR_POP;
     delete templates.functions.COVAR_SAMP;
     delete templates.window_frame_types.range;

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -302,6 +302,18 @@ describe('SQL Generation', () => {
           type: 'sum',
           group_by: [visitors.source]
         },
+        visitors_revenue_per_year: {
+          multi_stage: true,
+          sql: \`\${revenue}\`,
+          type: 'sum',
+          group_by: [visitors.created_at.year]
+        },
+        visitors_revenue_reduce_day: {
+          multi_stage: true,
+          sql: \`\${revenue}\`,
+          type: 'sum',
+          reduce_by: [visitors.created_at.day]
+        },
         visitors_revenue_without_date: {
           multi_stage: true,
           sql: \`\${revenue}\`,
@@ -4434,6 +4446,184 @@ SELECT 1 AS revenue,  cast('2024-01-01' AS timestamp) as time UNION ALL
     }]
   ));
 
+  if (getEnv('nativeSqlPlanner')) {
+    it('multi stage sum with group by time dim with granularity', async () => runQueryTest(
+      {
+        measures: ['visitors.visitors_revenue_per_year', 'visitors.revenue'],
+        dimensions: ['visitors.source'],
+        timeDimensions: [{
+          dimension: 'visitors.created_at',
+          granularity: 'year',
+          dateRange: ['2016-01-01', '2017-01-30']
+        }],
+        timezone: 'America/Los_Angeles',
+        order: [{
+          id: 'visitors.source'
+        }, {
+          id: 'visitors.created_at'
+        }],
+      },
+
+      [{
+        visitors__source: 'google',
+        visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+        visitors__visitors_revenue_per_year: '1500',
+        visitors__revenue: '300'
+      },
+      {
+        visitors__source: 'some',
+        visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+        visitors__visitors_revenue_per_year: '1500',
+        visitors__revenue: '300'
+      },
+      {
+        visitors__source: null,
+        visitors__created_at_year: '2016-01-01T00:00:00.000Z',
+        visitors__visitors_revenue_per_year: '500',
+        visitors__revenue: '500'
+      },
+      {
+        visitors__source: null,
+        visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+        visitors__visitors_revenue_per_year: '1500',
+        visitors__revenue: '900'
+      }]
+    ));
+  } else {
+    it.skip('multi stage sum with group by time dim with granularity', async () => {
+    // Works only in Tesseract
+    });
+  }
+
+  if (getEnv('nativeSqlPlanner')) {
+    it('multi stage sum multiple time dims group by time dim with granularity', async () => runQueryTest(
+      {
+        measures: ['visitors.visitors_revenue_per_year', 'visitors.revenue'],
+        dimensions: ['visitors.source'],
+        timeDimensions: [{
+          dimension: 'visitors.created_at',
+          granularity: 'year',
+          dateRange: ['2014-01-01', '2018-01-30']
+        },
+        {
+          dimension: 'visitors.created_at',
+          granularity: 'day',
+          dateRange: ['2014-01-01', '2018-01-30']
+        }],
+        timezone: 'America/Los_Angeles',
+        order: [{
+          id: 'visitors.source'
+        }, {
+          id: 'visitors.created_at'
+        }],
+      },
+
+      [
+        {
+          visitors__source: 'google',
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-05T00:00:00.000Z',
+          visitors__visitors_revenue_per_year: '1500',
+          visitors__revenue: '300'
+        },
+        {
+          visitors__source: 'some',
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-02T00:00:00.000Z',
+          visitors__visitors_revenue_per_year: '1500',
+          visitors__revenue: '100'
+        },
+        {
+          visitors__source: 'some',
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-04T00:00:00.000Z',
+          visitors__visitors_revenue_per_year: '1500',
+          visitors__revenue: '200'
+        },
+        {
+          visitors__source: null,
+          visitors__created_at_year: '2016-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2016-09-06T00:00:00.000Z',
+          visitors__visitors_revenue_per_year: '500',
+          visitors__revenue: '500'
+        },
+        {
+          visitors__source: null,
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-06T00:00:00.000Z',
+          visitors__visitors_revenue_per_year: '1500',
+          visitors__revenue: '900'
+        }
+      ]
+    ));
+  } else {
+    it.skip('multi stage sum multiple time dims group by time dim with granularity', async () => {
+    // Works only in Tesseract
+    });
+  }
+
+  if (getEnv('nativeSqlPlanner')) {
+    it('multi stage sum multiple time dims reduce by time dim with granularity', async () => runQueryTest(
+      {
+        measures: ['visitors.visitors_revenue_reduce_day', 'visitors.revenue'],
+        timeDimensions: [{
+          dimension: 'visitors.created_at',
+          granularity: 'year',
+          dateRange: ['2014-01-01', '2018-01-30']
+        },
+        {
+          dimension: 'visitors.created_at',
+          granularity: 'day',
+          dateRange: ['2014-01-01', '2018-01-30']
+        }],
+        timezone: 'America/Los_Angeles',
+        order: [{
+          id: 'visitors.source'
+        }, {
+          id: 'visitors.created_at'
+        }],
+      },
+
+      [
+
+        {
+          visitors__created_at_year: '2016-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2016-09-06T00:00:00.000Z',
+          visitors__visitors_revenue_reduce_day: '500',
+          visitors__revenue: '500'
+        },
+        {
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-02T00:00:00.000Z',
+          visitors__visitors_revenue_reduce_day: '1500',
+          visitors__revenue: '100'
+        },
+        {
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-04T00:00:00.000Z',
+          visitors__visitors_revenue_reduce_day: '1500',
+          visitors__revenue: '200'
+        },
+        {
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-05T00:00:00.000Z',
+          visitors__visitors_revenue_reduce_day: '1500',
+          visitors__revenue: '300'
+        },
+        {
+          visitors__created_at_year: '2017-01-01T00:00:00.000Z',
+          visitors__created_at_day: '2017-01-06T00:00:00.000Z',
+          visitors__visitors_revenue_reduce_day: '1500',
+          visitors__revenue: '900'
+        }
+      ]
+    ));
+  } else {
+    it.skip('multi stage sum multiple time dims reduce by time dim with granularity', async () => {
+    // Works only in Tesseract
+    });
+  }
+
   if (getEnv('nativeSqlPlanner')) {
     it('multi stage sum with group by over view', async () => runQueryTest(
       {

packages/cubejs-testing-drivers/fixtures/_schemas.json

@@ -424,6 +424,13 @@
             "type": "prior"
           }]
         },
+        {
+          "name": "totalProfitForQuarter",
+          "type": "sum",
+          "sql": "{totalProfit}",
+          "multi_stage": true,
+          "group_by": ["orderDate.quarter"]
+        },
         {
           "name": "totalProfitForStatus",
           "type": "sum",

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -157,6 +157,7 @@
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -171,6 +171,7 @@
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension",
 
     "---------------------------------------",
     "SKIPPED SQL API (Need work)",

packages/cubejs-testing-drivers/fixtures/clickhouse.json

@@ -208,6 +208,7 @@
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -224,6 +224,7 @@
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -150,6 +150,7 @@
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension",
 
     "---------------------------------------",
     "SKIPPED SQL API (Need work)",

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -146,6 +146,7 @@
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/postgres.json

@@ -170,7 +170,8 @@
     "querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
-    "Tesseract: SQL API: Timeshift measure from cube"
+    "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension"
   ],
   "tesseractSkip": [
     "querying Products: dimensions -- doesn't work wo ordering",

packages/cubejs-testing-drivers/fixtures/redshift.json

@@ -182,10 +182,45 @@
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
     "Tesseract: querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
     "Tesseract: SQL API: Timeshift measure from cube",
+    "querying BigECommerce: multi-stage group by time dimension",
 
     "---------------------------------------",
     "SKIPPED SQL API (Need work)            ",
     "---------------------------------------",
     "SQL API: SQL push down push to cube quoted alias"
+  ],
+  "tesseractSkip": [
+    "querying custom granularities ECommerce: count by three_months_by_march + no dimension",
+    "querying custom granularities ECommerce: count by three_months_by_march + dimension",
+    "querying Products: dimensions -- doesn't work wo ordering",
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + no dimension",
+    "querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + dimension",
+    "querying BigECommerce with Retail Calendar: totalCountRetailMonthAgo",
+    "querying BigECommerce with Retail Calendar: totalCountRetailWeekAgo",
+    "SQL API: Timeshift measure from cube",
+
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD without date range",
+    "querying custom granularities ECommerce: count by two_mo_by_feb + no dimension + rollingCountByLeading without date range",
+
+    "SQL API: Simple Rollup",
+    "SQL API: Complex Rollup",
+    "SQL API: Rollup with aliases",
+    "SQL API: Rollup over exprs",
+    "SQL API: Nested Rollup",
+    "SQL API: Nested Rollup with aliases",
+    "SQL API: Nested Rollup over asterisk",
+    "SQL API: Extended nested Rollup over asterisk",
+    "SQL API: SQL push down push to cube quoted alias",
+
+    "---- Different results comparing to baseQuery version. Need to investigate ----",
+    "querying BigECommerce: rolling window YTD (month + week)",
+    "querying BigECommerce: rolling window YTD (month + week + no gran)",
+    "querying BigECommerce: rolling window YTD without granularity",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
   ]
 }