chore(tesseract): Tesseract improvements

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -21,7 +21,8 @@ import {
   getEnv,
   localTimestampToUtc,
   timeSeries as timeSeriesBase,
-  timeSeriesFromCustomInterval
+  timeSeriesFromCustomInterval,
+  parseSqlInterval
 } from '@cubejs-backend/shared';
 
 import { CubeSymbols } from '../compiler/CubeSymbols';
@@ -1890,6 +1891,19 @@ export class BaseQuery {
     return `${value}::timestamp`;
   }
 
+  /**
+   * Converts the input interval (e.g. "2 years", "3 months", "5 days")
+   * into a format compatible with the target SQL dialect.
+   * Also returns the minimal time unit required (e.g. for use in DATEDIFF).
+   *
+   * Returns a tuple: (formatted interval, minimal time unit)
+   */
+  intervalAndMinimalTimeUnit(interval) {
+    const intervalParsed = parseSqlInterval(interval);
+    const minGranularity = this.diffTimeUnitForInterval(interval);
+    return [interval, minGranularity];
+  }
+
   commonQuery() {
     return `SELECT${this.topLimit()}
       ${this.baseSelect()}

packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts

@@ -42,7 +42,7 @@ export class BigqueryQuery extends BaseQuery {
   }
 
   public convertTz(field) {
-    return `DATETIME(${field}, '${this.timezone}')`;
+    return `DATETIME(${this.timeStampCast(field)}, '${this.timezone}')`;
   }
 
   public timeStampCast(value) {
@@ -134,6 +134,10 @@ export class BigqueryQuery extends BaseQuery {
     throw new Error(`Cannot transform interval expression "${interval}" to BigQuery dialect`);
   }
 
+  public override intervalAndMinimalTimeUnit(interval: string): [string, string] {
+    return this.formatInterval(interval);
+  }
+
   public newFilter(filter) {
     return new BigqueryFilter(this, filter);
   }
@@ -258,7 +262,7 @@ export class BigqueryQuery extends BaseQuery {
     templates.expressions.binary = '{% if op == \'%\' %}MOD({{ left }}, {{ right }}){% else %}({{ left }} {{ op }} {{ right }}){% endif %}';
     templates.expressions.interval = 'INTERVAL {{ interval }}';
     templates.expressions.extract = 'EXTRACT({% if date_part == \'DOW\' %}DAYOFWEEK{% elif date_part == \'DOY\' %}DAYOFYEAR{% else %}{{ date_part }}{% endif %} FROM {{ expr }})';
-    templates.expressions.timestamp_literal = 'TIMESTAMP(\'{{ value }}\')';
+    templates.expressions.timestamp_literal = 'DATETIME(TIMESTAMP(\'{{ value }}\'))';
     delete templates.expressions.ilike;
     delete templates.expressions.like_escape;
     templates.filters.like_pattern = 'CONCAT({% if start_wild %}\'%\'{% else %}\'\'{% endif %}, LOWER({{ value }}), {% if end_wild %}\'%\'{% else %}\'\'{% endif %})';
@@ -278,7 +282,25 @@ export class BigqueryQuery extends BaseQuery {
     '{% if not loop.last %} UNION ALL\n{% endif %}' +
     '{% endfor %}' +
     ') AS dates';
-
+    templates.statements.generated_time_series_select = 'SELECT DATETIME(d) AS date_from,\n' +
+    'DATETIME_SUB(DATETIME_ADD(DATETIME(d),  INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
+    'FROM UNNEST(\n' +
+    '{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
+    'GENERATE_DATE_ARRAY(DATE({{ start }}), DATE({{ end }}), INTERVAL {{ granularity }})\n' +
+    '{% else %}' +
+    'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ start }}), TIMESTAMP({{ end }}), INTERVAL {{ granularity }})\n' +
+    '{% endif %}' +
+    ') AS d';
+
+    templates.statements.generated_time_series_with_cte_range_source = 'SELECT DATETIME(d) AS date_from,\n' +
+    'DATETIME_SUB(DATETIME_ADD(DATETIME(d),  INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
+    'FROM {{ range_source }}, UNNEST(\n' +
+    '{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
+    'GENERATE_DATE_ARRAY(DATE({{ range_source }}.{{ min_name }}), DATE({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
+    '{% else %}' +
+    'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ range_source }}.{{ min_name }}), TIMESTAMP({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
+    '{% endif %}' +
+    ') AS d';
     return templates;
   }
 }

packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts

@@ -82,9 +82,12 @@ export class PostgresQuery extends BaseQuery {
     templates.types.double = 'DOUBLE PRECISION';
     templates.types.binary = 'BYTEA';
     templates.operators.is_not_distinct_from = 'IS NOT DISTINCT FROM';
-    templates.statements.generated_time_series_select = 'SELECT date_from AS "date_from",\n' +
-      'date_from + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
-      'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) "date_from" ';
+    templates.statements.generated_time_series_select = 'SELECT d AS "date_from",\n' +
+      'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
+      'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) d ';
+    templates.statements.generated_time_series_with_cte_range_source = 'SELECT d AS "date_from",\n' +
+      'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
+      'FROM {{ range_source }}, LATERAL generate_series({{ range_source }}.{{ min_name }}, {{ range_source }}.{{ max_name }}, \'{{ granularity }}\'::interval) d ';
     return templates;
   }
 

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -1010,9 +1010,9 @@ export class CubeSymbols {
           return true;
         }
         if (cube[propertyName]) {
-          depsResolveFn(propertyName, parentIndex);
+          const index = depsResolveFn(propertyName, parentIndex);
           if (cube[propertyName].type === 'time') {
-            return this.timeDimDependenciesProxy(parentIndex);
+            return this.timeDimDependenciesProxy(index);
           }
 
           return '';

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -316,6 +316,10 @@ describe('SQL Generation', () => {
             }
           }
         },
+        created_month: {
+          type: 'time',
+          sql: \`\${created_at.month}\`
+        },
         updated_at: {
           type: 'time',
           sql: 'updated_at'
@@ -1343,6 +1347,33 @@ SELECT 1 AS revenue,  cast('2024-01-01' AS timestamp) as time UNION ALL
     });
   }
 
+  if (getEnv('nativeSqlPlanner')) {
+    it('rolling count proxy time dimension', async () => {
+      await runQueryTest({
+        measures: [
+          'visitors.countRollingThreeMonth'
+        ],
+        dimensions: [
+          'visitors.created_month'
+        ],
+        order: [{
+          id: 'visitors.created_month'
+        }],
+        timezone: 'America/Los_Angeles'
+      }, [
+        { visitors__created_month: '2016-09-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
+        { visitors__created_month: '2016-10-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
+        { visitors__created_month: '2016-11-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
+        { visitors__created_month: '2016-12-01T00:00:00.000Z', visitors__count_rolling_three_month: null },
+        { visitors__created_month: '2017-01-01T00:00:00.000Z', visitors__count_rolling_three_month: '5' },
+      ]);
+    });
+  } else {
+    it.skip('rolling count without date range', () => {
+      // Skipping because it works only in Tesseract
+    });
+  }
+
   it('rolling qtd', async () => runQueryTest({
     measures: [
       'visitors.revenue_qtd'

packages/cubejs-testing-drivers/fixtures/_schemas.json

@@ -421,6 +421,14 @@
             "trailing": "2 month"
           }
         },
+        {
+          "name": "rollingCountYTD",
+          "type": "count",
+          "rollingWindow": {
+            "type": "to_date",
+            "granularity": "year"
+          }
+        },
         {
           "name": "rollingCountApproxBy2Day",
           "type": "count_distinct_approx",

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -145,6 +145,15 @@
     "querying BigECommerce: null boolean",
     "--------------------",
 
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD",
+    "querying BigECommerce: rolling window YTD without date range",
+    "--------------------",
+
     "week granularity is not supported for intervals",
     "--------------------",
     "querying BigECommerce: rolling window by 2 week",

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -157,6 +157,14 @@
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
 
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD without date range",
+    "--------------------",
+
     "SKIPPED SQL API (Need work)",
     "---------------------------------------",
     "SQL API: reuse params",

packages/cubejs-testing-drivers/fixtures/clickhouse.json

@@ -174,6 +174,15 @@
     "querying BigECommerce: rolling window by 2 week",
     "querying BigECommerce: rolling window by 2 month",
 
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD",
+    "querying BigECommerce: rolling window YTD without date range",
+    "--------------------",
+
     "---------------------------------------",
     "Custom Granularities                   ",
     "---------------------------------------",

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -206,6 +206,15 @@
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
 
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD",
+    "querying BigECommerce: rolling window YTD without date range",
+    "--------------------",
+
     "---------------------------------------",
     "Custom Granularities                   ",
     "---------------------------------------",

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -136,6 +136,14 @@
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
 
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD",
+    "querying BigECommerce: rolling window YTD without date range",
+    "--------------------",
     "---------------------------------------",
     "SKIPPED SQL API (Need work)",
     "---------------------------------------",

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -131,6 +131,13 @@
     "querying BigECommerce: partitioned pre-agg",
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD",
+    "querying BigECommerce: rolling window YTD without date range",
 
     "---------------------------------------",
     "Custom Granularities                   ",

packages/cubejs-testing-drivers/fixtures/postgres.json

@@ -155,6 +155,12 @@
     "---------------------------------------",
     "querying Products: dimensions -- doesn't work wo ordering",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
-    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD without date range"
   ]
 }

packages/cubejs-testing-drivers/fixtures/redshift.json

@@ -166,6 +166,13 @@
     "---------------------------------------",
     "querying Products: dimensions -- doesn't work wo ordering",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
-    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD",
+    "querying BigECommerce: rolling window YTD without date range"
   ]
 }

packages/cubejs-testing-drivers/fixtures/snowflake.json

@@ -200,6 +200,13 @@
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
     "querying BigECommerce: null sum",
-    "querying BigECommerce: null boolean"
+    "querying BigECommerce: null boolean",
+    "---------------------------------------",
+    "Requires Tesseract.                    ",
+    "---------------------------------------",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window YTD",
+    "querying BigECommerce: rolling window YTD without date range"
   ]
 }

packages/cubejs-testing-drivers/src/tests/testQueries.ts

@@ -1525,6 +1525,19 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
       expect(response.rawData()).toMatchSnapshot();
     });
 
+    execute('querying BigECommerce: rolling window by 2 day without date range', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountBy2Day',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
     execute('querying BigECommerce: rolling window by 2 week', async () => {
       const response = await client.load({
         measures: [
@@ -1553,6 +1566,46 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
       expect(response.rawData()).toMatchSnapshot();
     });
 
+    execute('querying BigECommerce: rolling window by 2 month without date range', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountBy2Month',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
+    execute('querying BigECommerce: rolling window YTD', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountYTD',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+          dateRange: ['2020-01-01', '2020-12-31'],
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
+    execute('querying BigECommerce: rolling window YTD without date range', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.rollingCountYTD',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
     if (includeHLLSuite) {
       execute('querying BigECommerce: rolling count_distinct_approx window by 2 day', async () => {
         const response = await client.load({