Skip to content

Commit 017b1f8

Browse files
waralexrommarianore-muttdata
waralexrom
authored and
marianore-muttdata
committed
chore(tesseract): Tesseract improvements (cube-js#9582)
1 parent 9b1b909 commit 017b1f8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1050
-142
lines changed

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ import {
2121
getEnv,
2222
localTimestampToUtc,
2323
timeSeries as timeSeriesBase,
24-
timeSeriesFromCustomInterval
24+
timeSeriesFromCustomInterval,
25+
parseSqlInterval
2526
} from '@cubejs-backend/shared';
2627

2728
import { CubeSymbols } from '../compiler/CubeSymbols';
@@ -1890,6 +1891,19 @@ export class BaseQuery {
18901891
return `${value}::timestamp`;
18911892
}
18921893

1894+
/**
1895+
* Converts the input interval (e.g. "2 years", "3 months", "5 days")
1896+
* into a format compatible with the target SQL dialect.
1897+
* Also returns the minimal time unit required (e.g. for use in DATEDIFF).
1898+
*
1899+
* Returns a tuple: (formatted interval, minimal time unit)
1900+
*/
1901+
intervalAndMinimalTimeUnit(interval) {
1902+
const intervalParsed = parseSqlInterval(interval);
1903+
const minGranularity = this.diffTimeUnitForInterval(interval);
1904+
return [interval, minGranularity];
1905+
}
1906+
18931907
commonQuery() {
18941908
return `SELECT${this.topLimit()}
18951909
${this.baseSelect()}

‎packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ export class BigqueryQuery extends BaseQuery {
4242
}
4343

4444
public convertTz(field) {
45-
return `DATETIME(${field}, '${this.timezone}')`;
45+
return `DATETIME(${this.timeStampCast(field)}, '${this.timezone}')`;
4646
}
4747

4848
public timeStampCast(value) {
@@ -134,6 +134,10 @@ export class BigqueryQuery extends BaseQuery {
134134
throw new Error(`Cannot transform interval expression "${interval}" to BigQuery dialect`);
135135
}
136136

137+
public override intervalAndMinimalTimeUnit(interval: string): [string, string] {
138+
return this.formatInterval(interval);
139+
}
140+
137141
public newFilter(filter) {
138142
return new BigqueryFilter(this, filter);
139143
}
@@ -252,13 +256,13 @@ export class BigqueryQuery extends BaseQuery {
252256
templates.functions.STRPOS = 'STRPOS({{ args_concat }})';
253257
templates.functions.DATEDIFF = 'DATETIME_DIFF(CAST({{ args[2] }} AS DATETIME), CAST({{ args[1] }} AS DATETIME), {{ date_part }})';
254258
// DATEADD is being rewritten to DATE_ADD
255-
templates.functions.DATE_ADD = '{% if date_part|upper in [\'YEAR\', \'MONTH\', \'QUARTER\'] %}TIMESTAMP(DATETIME_ADD(DATETIME({{ args[0] }}), INTERVAL {{ interval }} {{ date_part }})){% else %}TIMESTAMP_ADD({{ args[0] }}, INTERVAL {{ interval }} {{ date_part }}){% endif %}';
259+
templates.functions.DATE_ADD = 'DATETIME_ADD(DATETIME({{ args[0] }}), INTERVAL {{ interval }} {{ date_part }})';
256260
templates.functions.CURRENTDATE = 'CURRENT_DATE';
257261
delete templates.functions.TO_CHAR;
258262
templates.expressions.binary = '{% if op == \'%\' %}MOD({{ left }}, {{ right }}){% else %}({{ left }} {{ op }} {{ right }}){% endif %}';
259263
templates.expressions.interval = 'INTERVAL {{ interval }}';
260264
templates.expressions.extract = 'EXTRACT({% if date_part == \'DOW\' %}DAYOFWEEK{% elif date_part == \'DOY\' %}DAYOFYEAR{% else %}{{ date_part }}{% endif %} FROM {{ expr }})';
261-
templates.expressions.timestamp_literal = 'TIMESTAMP(\'{{ value }}\')';
265+
templates.expressions.timestamp_literal = 'DATETIME(TIMESTAMP(\'{{ value }}\'))';
262266
delete templates.expressions.ilike;
263267
delete templates.expressions.like_escape;
264268
templates.filters.like_pattern = 'CONCAT({% if start_wild %}\'%\'{% else %}\'\'{% endif %}, LOWER({{ value }}), {% if end_wild %}\'%\'{% else %}\'\'{% endif %})';
@@ -278,7 +282,25 @@ export class BigqueryQuery extends BaseQuery {
278282
'{% if not loop.last %} UNION ALL\n{% endif %}' +
279283
'{% endfor %}' +
280284
') AS dates';
281-
285+
templates.statements.generated_time_series_select = 'SELECT DATETIME(d) AS date_from,\n' +
286+
'DATETIME_SUB(DATETIME_ADD(DATETIME(d), INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
287+
'FROM UNNEST(\n' +
288+
'{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
289+
'GENERATE_DATE_ARRAY(DATE({{ start }}), DATE({{ end }}), INTERVAL {{ granularity }})\n' +
290+
'{% else %}' +
291+
'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ start }}), TIMESTAMP({{ end }}), INTERVAL {{ granularity }})\n' +
292+
'{% endif %}' +
293+
') AS d';
294+
295+
templates.statements.generated_time_series_with_cte_range_source = 'SELECT DATETIME(d) AS date_from,\n' +
296+
'DATETIME_SUB(DATETIME_ADD(DATETIME(d), INTERVAL {{ granularity }}), INTERVAL 1 MILLISECOND) AS date_to \n' +
297+
'FROM {{ range_source }}, UNNEST(\n' +
298+
'{% if minimal_time_unit|upper in ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"] %}' +
299+
'GENERATE_DATE_ARRAY(DATE({{ range_source }}.{{ min_name }}), DATE({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
300+
'{% else %}' +
301+
'GENERATE_TIMESTAMP_ARRAY(TIMESTAMP({{ range_source }}.{{ min_name }}), TIMESTAMP({{ range_source }}.{{ max_name }}), INTERVAL {{ granularity }})\n' +
302+
'{% endif %}' +
303+
') AS d';
282304
return templates;
283305
}
284306
}

‎packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,12 @@ export class PostgresQuery extends BaseQuery {
8282
templates.types.double = 'DOUBLE PRECISION';
8383
templates.types.binary = 'BYTEA';
8484
templates.operators.is_not_distinct_from = 'IS NOT DISTINCT FROM';
85-
templates.statements.generated_time_series_select = 'SELECT date_from AS "date_from",\n' +
86-
'date_from + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
87-
'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) "date_from" ';
85+
templates.statements.generated_time_series_select = 'SELECT d AS "date_from",\n' +
86+
'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
87+
'FROM generate_series({{ start }}::timestamp, {{ end }}:: timestamp, \'{{ granularity }}\'::interval) d ';
88+
templates.statements.generated_time_series_with_cte_range_source = 'SELECT d AS "date_from",\n' +
89+
'd + interval \'{{ granularity }}\' - interval \'1 millisecond\' AS "date_to" \n' +
90+
'FROM {{ range_source }}, LATERAL generate_series({{ range_source }}.{{ min_name }}, {{ range_source }}.{{ max_name }}, \'{{ granularity }}\'::interval) d ';
8891
return templates;
8992
}
9093

‎packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1010,9 +1010,9 @@ export class CubeSymbols {
10101010
return true;
10111011
}
10121012
if (cube[propertyName]) {
1013-
depsResolveFn(propertyName, parentIndex);
1013+
const index = depsResolveFn(propertyName, parentIndex);
10141014
if (cube[propertyName].type === 'time') {
1015-
return this.timeDimDependenciesProxy(parentIndex);
1015+
return this.timeDimDependenciesProxy(index);
10161016
}
10171017

10181018
return '';

‎packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,10 @@ describe('SQL Generation', () => {
316316
}
317317
}
318318
},
319+
created_month: {
320+
type: 'time',
321+
sql: \`\${created_at.month}\`
322+
},
319323
updated_at: {
320324
type: 'time',
321325
sql: 'updated_at'
@@ -1343,6 +1347,33 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
13431347
});
13441348
}
13451349

1350+
if (getEnv('nativeSqlPlanner')) {
1351+
it('rolling count proxy time dimension', async () => {
1352+
await runQueryTest({
1353+
measures: [
1354+
'visitors.countRollingThreeMonth'
1355+
],
1356+
dimensions: [
1357+
'visitors.created_month'
1358+
],
1359+
order: [{
1360+
id: 'visitors.created_month'
1361+
}],
1362+
timezone: 'America/Los_Angeles'
1363+
}, [
1364+
{ visitors__created_month: '2016-09-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
1365+
{ visitors__created_month: '2016-10-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
1366+
{ visitors__created_month: '2016-11-01T00:00:00.000Z', visitors__count_rolling_three_month: '1' },
1367+
{ visitors__created_month: '2016-12-01T00:00:00.000Z', visitors__count_rolling_three_month: null },
1368+
{ visitors__created_month: '2017-01-01T00:00:00.000Z', visitors__count_rolling_three_month: '5' },
1369+
]);
1370+
});
1371+
} else {
1372+
it.skip('rolling count without date range', () => {
1373+
// Skipping because it works only in Tesseract
1374+
});
1375+
}
1376+
13461377
it('rolling qtd', async () => runQueryTest({
13471378
measures: [
13481379
'visitors.revenue_qtd'

‎packages/cubejs-testing-drivers/fixtures/_schemas.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,14 @@
421421
"trailing": "2 month"
422422
}
423423
},
424+
{
425+
"name": "rollingCountYTD",
426+
"type": "count",
427+
"rollingWindow": {
428+
"type": "to_date",
429+
"granularity": "year"
430+
}
431+
},
424432
{
425433
"name": "rollingCountApproxBy2Day",
426434
"type": "count_distinct_approx",

‎packages/cubejs-testing-drivers/fixtures/athena.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,15 @@
145145
"querying BigECommerce: null boolean",
146146
"--------------------",
147147

148+
"---------------------------------------",
149+
"Requires Tesseract. ",
150+
"---------------------------------------",
151+
"querying BigECommerce: rolling window by 2 day without date range",
152+
"querying BigECommerce: rolling window by 2 month without date range",
153+
"querying BigECommerce: rolling window YTD",
154+
"querying BigECommerce: rolling window YTD without date range",
155+
"--------------------",
156+
148157
"week granularity is not supported for intervals",
149158
"--------------------",
150159
"querying BigECommerce: rolling window by 2 week",

‎packages/cubejs-testing-drivers/fixtures/bigquery.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,14 @@
157157
"querying BigECommerce: null sum",
158158
"querying BigECommerce: null boolean",
159159

160+
"---------------------------------------",
161+
"Requires Tesseract. ",
162+
"---------------------------------------",
163+
"querying BigECommerce: rolling window by 2 day without date range",
164+
"querying BigECommerce: rolling window by 2 month without date range",
165+
"querying BigECommerce: rolling window YTD without date range",
166+
"--------------------",
167+
160168
"SKIPPED SQL API (Need work)",
161169
"---------------------------------------",
162170
"SQL API: reuse params",

‎packages/cubejs-testing-drivers/fixtures/clickhouse.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,15 @@
185185
"querying BigECommerce: rolling window by 2 week",
186186
"querying BigECommerce: rolling window by 2 month",
187187

188+
"---------------------------------------",
189+
"Requires Tesseract. ",
190+
"---------------------------------------",
191+
"querying BigECommerce: rolling window by 2 day without date range",
192+
"querying BigECommerce: rolling window by 2 month without date range",
193+
"querying BigECommerce: rolling window YTD",
194+
"querying BigECommerce: rolling window YTD without date range",
195+
"--------------------",
196+
188197
"---------------------------------------",
189198
"Custom Granularities ",
190199
"---------------------------------------",

‎packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,15 @@
206206
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
207207
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
208208

209+
"---------------------------------------",
210+
"Requires Tesseract. ",
211+
"---------------------------------------",
212+
"querying BigECommerce: rolling window by 2 day without date range",
213+
"querying BigECommerce: rolling window by 2 month without date range",
214+
"querying BigECommerce: rolling window YTD",
215+
"querying BigECommerce: rolling window YTD without date range",
216+
"--------------------",
217+
209218
"---------------------------------------",
210219
"Custom Granularities ",
211220
"---------------------------------------",

0 commit comments

Comments
 (0)