chore(tesseract): Tesseract improvements

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -21,8 +21,8 @@ import {
   getEnv,
   localTimestampToUtc,
   timeSeries as timeSeriesBase,
-  timeSeriesFromCustomInterval
-} from '@cubejs-backend/shared';
+  timeSeriesFromCustomInterval,
+  parseSqlInterval } from '@cubejs-backend/shared';
 
 import { CubeSymbols } from '../compiler/CubeSymbols';
 import { UserError } from '../compiler/UserError';
@@ -1890,6 +1890,64 @@ export class BaseQuery {
     return `${value}::timestamp`;
   }
 
+  /**
+   * Converts the input interval (e.g. "2 years", "3 months", "5 days")
+   * into a format compatible with the target SQL dialect.
+   * Also returns the minimal time unit required (e.g. for use in DATEDIFF).
+   *
+   * Returns a tuple: (formatted interval, minimal time unit)
+   */
+  intervalAndMinimalTimeUnit(interval) {
+    const intervalParsed = parseSqlInterval(interval);
+    const intKeys = Object.keys(intervalParsed).length;
+
+    if (intervalParsed.year && intKeys === 1) {
+      return [interval, 'year'];
+    } else if (intervalParsed.year && intervalParsed.month && intKeys === 2) {
+      return [interval, 'month'];
+    } else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intKeys === 3) {
+      return [interval, 'day'];
+    } else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intervalParsed.hour && intKeys === 4) {
+      return [interval, 'hour'];
+    } else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 5) {
+      return [interval, 'minute'];
+    } else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 6) {
+      return [interval, 'second'];
+    } else if (intervalParsed.quarter && intKeys === 1) {
+      return [interval, 'quarter'];
+    } else if (intervalParsed.month && intKeys === 1) {
+      return [interval, 'month'];
+    } else if (intervalParsed.month && intervalParsed.day && intKeys === 2) {
+      return [interval, 'day'];
+    } else if (intervalParsed.month && intervalParsed.day && intervalParsed.hour && intKeys === 3) {
+      return [interval, 'hour'];
+    } else if (intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 4) {
+      return [interval, 'minute'];
+    } else if (intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 5) {
+      return [interval, 'second'];
+    } else if (intervalParsed.week && intKeys === 1) {
+      return [interval, 'week'];
+    } else if (intervalParsed.day && intKeys === 1) {
+      return [interval, 'day'];
+    } else if (intervalParsed.day && intervalParsed.hour && intKeys === 2) {
+      return [interval, 'hour'];
+    } else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 3) {
+      return [interval, 'minute'];
+    } else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 4) {
+      return [interval, 'second'];
+    } else if (intervalParsed.hour && intervalParsed.minute && intKeys === 2) {
+      return [interval, 'minute'];
+    } else if (intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 3) {
+      return [interval, 'second'];
+    } else if (intervalParsed.minute && intervalParsed.second && intKeys === 2) {
+      return [interval, 'second'];
+    }
+
+    // No need to support microseconds.
+
+    throw new Error(`Cannot transform interval expression "${interval}"`);
+  }
+
   commonQuery() {
     return `SELECT${this.topLimit()}
       ${this.baseSelect()}

packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts

@@ -83,7 +83,7 @@ export class BigqueryQuery extends BaseQuery {
    * @see https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#interval_type
    * It returns a tuple of (formatted interval, timeUnit to use in datediff functions)
    */
-  private formatInterval(interval: string): [string, string] {
+  public formatInterval(interval: string): [string, string] {
     const intervalParsed = parseSqlInterval(interval);
     const intKeys = Object.keys(intervalParsed).length;
 
@@ -134,6 +134,10 @@ export class BigqueryQuery extends BaseQuery {
     throw new Error(`Cannot transform interval expression "${interval}" to BigQuery dialect`);
   }
 
+  public intervalAndMinimalTimeUnit(interval: string): [string, string] {
+    return this.formatInterval(interval);
+  }
+
   public newFilter(filter) {
     return new BigqueryFilter(this, filter);
   }
@@ -278,7 +282,25 @@ export class BigqueryQuery extends BaseQuery {
     '{% if not loop.last %} UNION ALL\n{% endif %}' +
     '{% endfor %}' +
     ') AS dates';
-
+    templates.statements.generated_time_series_select = 'SELECT DATETIME(d) AS date_from,\n' +
+    'DATETIME_SUB(DATETIME_ADD(DATETIME(d),  INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
+    'FROM UNNEST(\n' +
+    '{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
+    'GENERATE_DATE_ARRAY(DATE({{ start }}), DATE({{ end }}), INTERVAL {{ granularity }})\n' +
+    '{% else %}' +
+    'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ start }}), TIMESTAMP({{ end }}), INTERVAL {{ granularity }})\n' +
+    '{% endif %}' +
+    ') AS d';
+
+    templates.statements.generated_time_series_with_cte_range_source = 'SELECT DATETIME(d) AS date_from,\n' +
+    'DATETIME_SUB(DATETIME_ADD(DATETIME(d),  INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
+    'FROM {{ range_source }}, UNNEST(\n' +
+    '{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
+    'GENERATE_DATE_ARRAY(DATE({{ range_source }}.{{ min_name }}), DATE({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
+    '{% else %}' +
+    'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ range_source }}.{{ min_name }}), TIMESTAMP({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
+    '{% endif %}' +
+    ') AS d';
     return templates;
   }
 }

packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts

@@ -82,9 +82,12 @@ export class PostgresQuery extends BaseQuery {
     templates.types.double = 'DOUBLE PRECISION';
     templates.types.binary = 'BYTEA';
     templates.operators.is_not_distinct_from = 'IS NOT DISTINCT FROM';
-    templates.statements.generated_time_series_select = 'SELECT date_from AS "date_from",\n' +
-      'date_from + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
-      'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) "date_from" ';
+    templates.statements.generated_time_series_select = 'SELECT d AS "date_from",\n' +
+      'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
+      'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) d ';
+    templates.statements.generated_time_series_with_cte_range_source = 'SELECT d AS "date_from",\n' +
+      'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
+      'FROM {{ range_source }}, LATERAL generate_series({{ range_source }}.{{ min_name }}, {{ range_source }}.{{ max_name }}, \'{{ granularity }}\'::interval) d ';
     return templates;
   }
 

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -1010,9 +1010,9 @@ export class CubeSymbols {
           return true;
         }
         if (cube[propertyName]) {
-          depsResolveFn(propertyName, parentIndex);
+          const index = depsResolveFn(propertyName, parentIndex);
           if (cube[propertyName].type === 'time') {
-            return this.timeDimDependenciesProxy(parentIndex);
+            return this.timeDimDependenciesProxy(index);
           }
 
           return '';

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -316,6 +316,10 @@ describe('SQL Generation', () => {
             }
           }
         },
+        created_month: {
+          type: 'time',
+          sql: \`\${created_at.month}\`
+        },
         updated_at: {
           type: 'time',
           sql: 'updated_at'
@@ -1343,6 +1347,33 @@ SELECT 1 AS revenue,  cast('2024-01-01' AS timestamp) as time UNION ALL
     });
   }
 
+  if (getEnv('nativeSqlPlanner')) {
+    it('rolling count proxy time dimension', async () => {
+      await runQueryTest({
+        measures: [
+          'visitors.countRollingThreeMonth'
+        ],
+        dimensions: [
+          'visitors.created_month'
+        ],
+        order: [{
+          id: 'visitors.created_month'
+        }],
+        timezone: 'America/Los_Angeles'
+      }, [
+        { visitors__created_month: '2016-09-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
+        { visitors__created_month: '2016-10-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
+        { visitors__created_month: '2016-11-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
+        { visitors__created_month: '2016-12-01T00:00:00.000Z', visitors__count_rolling_three_month: null },
+        { visitors__created_month: '2017-01-01T00:00:00.000Z', visitors__count_rolling_three_month: '5' },
+      ]);
+    });
+  } else {
+    it.skip('rolling count without date range', () => {
+      // Skipping because it works only in Tesseract
+    });
+  }
+
   it('rolling qtd', async () => runQueryTest({
     measures: [
       'visitors.revenue_qtd'

packages/cubejs-testing-drivers/fixtures/_schemas.json

@@ -421,6 +421,14 @@
             "trailing": "2 month"
           }
         },
+        {
+          "name": "rollingCountYTD",
+          "type": "count",
+          "rollingWindow": {
+            "type": "to_date",
+            "granularity": "year"
+          }
+        },
         {
           "name": "rollingCountApproxBy2Day",
           "type": "count_distinct_approx",

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -140,6 +140,9 @@
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract",
     "querying BigECommerce: partitioned pre-agg",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -154,6 +154,9 @@
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
 

packages/cubejs-testing-drivers/fixtures/clickhouse.json

@@ -163,6 +163,9 @@
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract",
     "querying ECommerce: partitioned pre-agg",
     "querying ECommerce: partitioned pre-agg higher granularity",
     "querying BigECommerce: partitioned pre-agg",

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -205,6 +205,9 @@
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -135,6 +135,9 @@
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract",
 
     "---------------------------------------",
     "SKIPPED SQL API (Need work)",

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -131,6 +131,9 @@
     "querying BigECommerce: partitioned pre-agg",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/postgres.json

@@ -155,6 +155,9 @@
     "---------------------------------------",
     "querying Products: dimensions -- doesn't work wo ordering",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
-    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract"
   ]
 }

packages/cubejs-testing-drivers/fixtures/redshift.json

@@ -166,6 +166,9 @@
     "---------------------------------------",
     "querying Products: dimensions -- doesn't work wo ordering",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
-    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract"
   ]
 }

packages/cubejs-testing-drivers/fixtures/snowflake.json

@@ -200,6 +200,9 @@
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
     "querying BigECommerce: null sum",
-    "querying BigECommerce: null boolean"
+    "querying BigECommerce: null boolean",
+    "querying BigECommerce: rolling window by 2 day without date range -- requires tesseract",
+    "querying BigECommerce: rolling window by 2 month without date range -- requires tesseract",
+    "querying BigECommerce: rolling window YTD without date range -- requires tesseract"
   ]
 }

packages/cubejs-testing-drivers/src/tests/testQueries.ts

@@ -1525,6 +1525,19 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
       expect(response.rawData()).toMatchSnapshot();
     });
 
+    execute('querying BigECommerce: rolling window by 2 day without date range', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountBy2Day',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
     execute('querying BigECommerce: rolling window by 2 week', async () => {
       const response = await client.load({
         measures: [
@@ -1553,6 +1566,46 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
       expect(response.rawData()).toMatchSnapshot();
     });
 
+    execute('querying BigECommerce: rolling window by 2 month without date range', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountBy2Month',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
+    execute('querying BigECommerce: rolling window YTD', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountYTD',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+          dateRange: ['2020-01-01', '2020-12-31'],
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
+    execute('querying BigECommerce: rolling window YTD without date range', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountYTD',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
     if (includeHLLSuite) {
       execute('querying BigECommerce: rolling count_distinct_approx window by 2 day', async () => {
         const response = await client.load({