Skip to content

chore(tesseract): Tesseract improvements #9582

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
May 19, 2025
Merged

chore(tesseract): Tesseract improvements #9582

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
53f5bcd
chore(tesseract): Various improvements
waralexrom May 17, 2025
3e9f6db
in work
waralexrom May 18, 2025
ed9dfb4
in work
waralexrom May 18, 2025
0c3a255
in work
waralexrom May 18, 2025
2a60452
in work
waralexrom May 18, 2025
42434d8
in work
waralexrom May 18, 2025
ed5c593
in work
waralexrom May 18, 2025
c2d7d2e
in work
waralexrom May 18, 2025
acce9a8
in work
waralexrom May 19, 2025
1511c76
in work
waralexrom May 19, 2025
a88f8e7
in work
waralexrom May 19, 2025
abc8c6d
in work
waralexrom May 19, 2025
3674ef8
in work
waralexrom May 19, 2025
3f528e1
should work
waralexrom May 19, 2025
f1aa39f
fix
waralexrom May 19, 2025
545af17
final fix
waralexrom May 19, 2025
733136d
final fix
waralexrom May 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 60 additions & 2 deletions packages/cubejs-schema-compiler/src/adapter/BaseQuery.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ import {
getEnv,
localTimestampToUtc,
timeSeries as timeSeriesBase,
timeSeriesFromCustomInterval
} from '@cubejs-backend/shared';
timeSeriesFromCustomInterval,
parseSqlInterval } from '@cubejs-backend/shared';

import { CubeSymbols } from '../compiler/CubeSymbols';
import { UserError } from '../compiler/UserError';
Expand Down Expand Up @@ -1890,6 +1890,64 @@ export class BaseQuery {
return `${value}::timestamp`;
}

/**
* Converts the input interval (e.g. "2 years", "3 months", "5 days")
* into a format compatible with the target SQL dialect.
* Also returns the minimal time unit required (e.g. for use in DATEDIFF).
*
* Returns a tuple: (formatted interval, minimal time unit)
*/
intervalAndMinimalTimeUnit(interval) {
const intervalParsed = parseSqlInterval(interval);
const intKeys = Object.keys(intervalParsed).length;

if (intervalParsed.year && intKeys === 1) {
return [interval, 'year'];
} else if (intervalParsed.year && intervalParsed.month && intKeys === 2) {
return [interval, 'month'];
} else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intKeys === 3) {
return [interval, 'day'];
} else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intervalParsed.hour && intKeys === 4) {
return [interval, 'hour'];
} else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 5) {
return [interval, 'minute'];
} else if (intervalParsed.year && intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 6) {
return [interval, 'second'];
} else if (intervalParsed.quarter && intKeys === 1) {
return [interval, 'quarter'];
} else if (intervalParsed.month && intKeys === 1) {
return [interval, 'month'];
} else if (intervalParsed.month && intervalParsed.day && intKeys === 2) {
return [interval, 'day'];
} else if (intervalParsed.month && intervalParsed.day && intervalParsed.hour && intKeys === 3) {
return [interval, 'hour'];
} else if (intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 4) {
return [interval, 'minute'];
} else if (intervalParsed.month && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 5) {
return [interval, 'second'];
} else if (intervalParsed.week && intKeys === 1) {
return [interval, 'week'];
} else if (intervalParsed.day && intKeys === 1) {
return [interval, 'day'];
} else if (intervalParsed.day && intervalParsed.hour && intKeys === 2) {
return [interval, 'hour'];
} else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 3) {
return [interval, 'minute'];
} else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 4) {
return [interval, 'second'];
} else if (intervalParsed.hour && intervalParsed.minute && intKeys === 2) {
return [interval, 'minute'];
} else if (intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 3) {
return [interval, 'second'];
} else if (intervalParsed.minute && intervalParsed.second && intKeys === 2) {
return [interval, 'second'];
}

// No need to support microseconds.

throw new Error(`Cannot transform interval expression "${interval}"`);
}

commonQuery() {
return `SELECT${this.topLimit()}
${this.baseSelect()}
Expand Down
26 changes: 24 additions & 2 deletions packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ export class BigqueryQuery extends BaseQuery {
* @see https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#interval_type
* It returns a tuple of (formatted interval, timeUnit to use in datediff functions)
*/
private formatInterval(interval: string): [string, string] {
public formatInterval(interval: string): [string, string] {
const intervalParsed = parseSqlInterval(interval);
const intKeys = Object.keys(intervalParsed).length;

Expand Down Expand Up @@ -134,6 +134,10 @@ export class BigqueryQuery extends BaseQuery {
throw new Error(`Cannot transform interval expression "${interval}" to BigQuery dialect`);
}

public intervalAndMinimalTimeUnit(interval: string): [string, string] {
return this.formatInterval(interval);
}

public newFilter(filter) {
return new BigqueryFilter(this, filter);
}
Expand Down Expand Up @@ -278,7 +282,25 @@ export class BigqueryQuery extends BaseQuery {
'{% if not loop.last %} UNION ALL\n{% endif %}' +
'{% endfor %}' +
') AS dates';

templates.statements.generated_time_series_select = 'SELECT DATETIME(d) AS date_from,\n' +
'DATETIME_SUB(DATETIME_ADD(DATETIME(d), INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
'FROM UNNEST(\n' +
'{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
'GENERATE_DATE_ARRAY(DATE({{ start }}), DATE({{ end }}), INTERVAL {{ granularity }})\n' +
'{% else %}' +
'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ start }}), TIMESTAMP({{ end }}), INTERVAL {{ granularity }})\n' +
'{% endif %}' +
') AS d';

templates.statements.generated_time_series_with_cte_range_source = 'SELECT DATETIME(d) AS date_from,\n' +
'DATETIME_SUB(DATETIME_ADD(DATETIME(d), INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
'FROM {{ range_source }}, UNNEST(\n' +
'{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
'GENERATE_DATE_ARRAY(DATE({{ range_source }}.{{ min_name }}), DATE({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
'{% else %}' +
'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ range_source }}.{{ min_name }}), TIMESTAMP({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
'{% endif %}' +
') AS d';
return templates;
}
}
9 changes: 6 additions & 3 deletions packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,12 @@ export class PostgresQuery extends BaseQuery {
templates.types.double = 'DOUBLE PRECISION';
templates.types.binary = 'BYTEA';
templates.operators.is_not_distinct_from = 'IS NOT DISTINCT FROM';
templates.statements.generated_time_series_select = 'SELECT date_from AS "date_from",\n' +
'date_from + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) "date_from" ';
templates.statements.generated_time_series_select = 'SELECT d AS "date_from",\n' +
'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) d ';
templates.statements.generated_time_series_with_cte_range_source = 'SELECT d AS "date_from",\n' +
'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
'FROM {{ range_source }}, LATERAL generate_series({{ range_source }}.{{ min_name }}, {{ range_source }}.{{ max_name }}, \'{{ granularity }}\'::interval) d ';
return templates;
}

Expand Down
4 changes: 2 additions & 2 deletions packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1010,9 +1010,9 @@ export class CubeSymbols {
return true;
}
if (cube[propertyName]) {
depsResolveFn(propertyName, parentIndex);
const index = depsResolveFn(propertyName, parentIndex);
if (cube[propertyName].type === 'time') {
return this.timeDimDependenciesProxy(parentIndex);
return this.timeDimDependenciesProxy(index);
}

return '';
Expand Down
31 changes: 31 additions & 0 deletions packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,10 @@ describe('SQL Generation', () => {
}
}
},
created_month: {
type: 'time',
sql: \`\${created_at.month}\`
},
updated_at: {
type: 'time',
sql: 'updated_at'
Expand Down Expand Up @@ -1343,6 +1347,33 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
});
}

if (getEnv('nativeSqlPlanner')) {
it('rolling count proxy time dimension', async () => {
await runQueryTest({
measures: [
'visitors.countRollingThreeMonth'
],
dimensions: [
'visitors.created_month'
],
order: [{
id: 'visitors.created_month'
}],
timezone: 'America/Los_Angeles'
}, [
{ visitors__created_month: '2016-09-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
{ visitors__created_month: '2016-10-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
{ visitors__created_month: '2016-11-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
{ visitors__created_month: '2016-12-01T00:00:00.000Z', visitors__count_rolling_three_month: null },
{ visitors__created_month: '2017-01-01T00:00:00.000Z', visitors__count_rolling_three_month: '5' },
]);
});
} else {
it.skip('rolling count without date range', () => {
// Skipping because it works only in Tesseract
});
}

it('rolling qtd', async () => runQueryTest({
measures: [
'visitors.revenue_qtd'
Expand Down
8 changes: 8 additions & 0 deletions packages/cubejs-testing-drivers/fixtures/_schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,14 @@
"trailing": "2 month"
}
},
{
"name": "rollingCountYTD",
"type": "count",
"rollingWindow": {
"type": "to_date",
"granularity": "year"
}
},
{
"name": "rollingCountApproxBy2Day",
"type": "count_distinct_approx",
Expand Down
9 changes: 9 additions & 0 deletions packages/cubejs-testing-drivers/fixtures/athena.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,15 @@
"querying BigECommerce: null boolean",
"--------------------",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"week granularity is not supported for intervals",
"--------------------",
"querying BigECommerce: rolling window by 2 week",
Expand Down
8 changes: 8 additions & 0 deletions packages/cubejs-testing-drivers/fixtures/bigquery.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,14 @@
"querying BigECommerce: null sum",
"querying BigECommerce: null boolean",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"SKIPPED SQL API (Need work)",
"---------------------------------------",
"SQL API: reuse params",
Expand Down
9 changes: 9 additions & 0 deletions packages/cubejs-testing-drivers/fixtures/clickhouse.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,15 @@
"querying BigECommerce: rolling window by 2 week",
"querying BigECommerce: rolling window by 2 month",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"---------------------------------------",
"Custom Granularities ",
"---------------------------------------",
Expand Down
9 changes: 9 additions & 0 deletions packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,15 @@
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"---------------------------------------",
"Custom Granularities ",
"---------------------------------------",
Expand Down
8 changes: 8 additions & 0 deletions packages/cubejs-testing-drivers/fixtures/mssql.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,14 @@
"querying BigECommerce: null sum",
"querying BigECommerce: null boolean",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",
"---------------------------------------",
"SKIPPED SQL API (Need work)",
"---------------------------------------",
Expand Down
7 changes: 7 additions & 0 deletions packages/cubejs-testing-drivers/fixtures/mysql.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,13 @@
"querying BigECommerce: partitioned pre-agg",
"querying BigECommerce: null sum",
"querying BigECommerce: null boolean",
"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",

"---------------------------------------",
"Custom Granularities ",
Expand Down
8 changes: 7 additions & 1 deletion packages/cubejs-testing-drivers/fixtures/postgres.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,12 @@
"---------------------------------------",
"querying Products: dimensions -- doesn't work wo ordering",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD without date range"
]
}
9 changes: 8 additions & 1 deletion packages/cubejs-testing-drivers/fixtures/redshift.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,13 @@
"---------------------------------------",
"querying Products: dimensions -- doesn't work wo ordering",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range"
]
}
9 changes: 8 additions & 1 deletion packages/cubejs-testing-drivers/fixtures/snowflake.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,13 @@
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
"querying BigECommerce: null sum",
"querying BigECommerce: null boolean"
"querying BigECommerce: null boolean",
"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range"
]
}
Loading
Loading