From fb189001a2565804ac4de23fae12e99e7b1c2da5 Mon Sep 17 00:00:00 2001 From: wardsi Date: Mon, 27 Jan 2025 23:19:03 +0200 Subject: [PATCH 1/4] Correct dremio date interval functions --- .../driver/DremioQuery.js | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-dremio-driver/driver/DremioQuery.js b/packages/cubejs-dremio-driver/driver/DremioQuery.js index c46ade9c517df..da356af27ce11 100644 --- a/packages/cubejs-dremio-driver/driver/DremioQuery.js +++ b/packages/cubejs-dremio-driver/driver/DremioQuery.js @@ -59,11 +59,27 @@ class DremioQuery extends BaseQuery { } subtractInterval(date, interval) { - return `DATE_SUB(${date}, INTERVAL ${interval})`; + const intervalParts = interval.trim().split(' '); + const intervalNumber = parseInt(intervalParts[0]); + const intervalName = intervalParts[1].toLowerCase(); + if (intervalName == 'quarter') { + intervalParts[0] = intervalNumber * 3; + intervalParts[1] = 'month'; + } + + return `DATE_SUB(${date}, CAST(${intervalParts[0]} as INTERVAL ${intervalParts[1]}))`; } addInterval(date, interval) { - return `DATE_ADD(${date}, INTERVAL ${interval})`; + const intervalParts = interval.trim().split(' '); + const intervalNumber = parseInt(intervalParts[0]); + const intervalName = intervalParts[1].toLowerCase(); + if (intervalName == 'quarter') { + intervalParts[0] = intervalNumber * 3; + intervalParts[1] = 'month'; + } + + return `DATE_ADD(${date}, CAST(${intervalParts[0]} as INTERVAL ${intervalParts[1]}))`; } timeGroupedColumn(granularity, dimension) { From b861adb9d56c3cb4a8412a12bdf8ea075a4ee41a Mon Sep 17 00:00:00 2001 From: wardsi Date: Tue, 28 Jan 2025 09:24:22 +0200 Subject: [PATCH 2/4] Dremio date interval functions - support for year, month, qtr and day only --- .../driver/DremioQuery.js | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/packages/cubejs-dremio-driver/driver/DremioQuery.js b/packages/cubejs-dremio-driver/driver/DremioQuery.js index da356af27ce11..db3884f588a07 100644 --- a/packages/cubejs-dremio-driver/driver/DremioQuery.js +++ b/packages/cubejs-dremio-driver/driver/DremioQuery.js @@ -1,4 +1,5 @@ const { BaseFilter, BaseQuery } = require('@cubejs-backend/schema-compiler'); +const { parseSqlInterval } = require('@cubejs-backend/shared'); const GRANULARITY_TO_INTERVAL = { week: (date) => `DATE_TRUNC('week', ${date})`, @@ -59,27 +60,17 @@ class DremioQuery extends BaseQuery { } subtractInterval(date, interval) { - const intervalParts = interval.trim().split(' '); - const intervalNumber = parseInt(intervalParts[0]); - const intervalName = intervalParts[1].toLowerCase(); - if (intervalName == 'quarter') { - intervalParts[0] = intervalNumber * 3; - intervalParts[1] = 'month'; - } - - return `DATE_SUB(${date}, CAST(${intervalParts[0]} as INTERVAL ${intervalParts[1]}))`; + const formattedTimeIntervals = this.formatInterval(interval); + const intervalFormatted = formattedTimeIntervals[0]; + const timeUnit = formattedTimeIntervals[1]; + return `DATE_SUB(${date}, CAST(${intervalFormatted} as INTERVAL ${timeUnit}))`; } addInterval(date, interval) { - const intervalParts = interval.trim().split(' '); - const intervalNumber = parseInt(intervalParts[0]); - const intervalName = intervalParts[1].toLowerCase(); - if (intervalName == 'quarter') { - intervalParts[0] = intervalNumber * 3; - intervalParts[1] = 'month'; - } - - return `DATE_ADD(${date}, CAST(${intervalParts[0]} as INTERVAL ${intervalParts[1]}))`; + const formattedTimeIntervals = this.formatInterval(interval); + const intervalFormatted = formattedTimeIntervals[0]; + const timeUnit = formattedTimeIntervals[1]; + return `DATE_ADD(${date}, CAST(${intervalFormatted} as INTERVAL ${timeUnit}))`; } timeGroupedColumn(granularity, dimension) { @@ -108,6 +99,34 @@ class DremioQuery extends BaseQuery { wrapSegmentForDimensionSelect(sql) { return `IF(${sql}, 1, 0)`; } + + /** + * The input interval with (possible) plural units, like "1 hour 2 minutes", "2 year", "3 months", "4 weeks", "5 days", "3 months 24 days 15 minutes", ... + * will be converted to Dremio dialect. + * @see https://docs.dremio.com/24.3.x/reference/sql/sql-functions/functions/DATE_ADD/ + * @see https://docs.dremio.com/24.3.x/reference/sql/sql-functions/functions/DATE_SUB/ + * It returns a tuple of (formatted interval, timeUnit to use in date functions) + * This function only supports the following scenarios for now: + * ie. n year[s] or n month[3] or n day[s] + */ + formatInterval(interval) { + const intervalParsed = parseSqlInterval(interval); + const intKeys = Object.keys(intervalParsed).length; + + if (intervalParsed.year && intKeys === 1) { + return [`${intervalParsed.year}`, 'YEAR']; + } else if (intervalParsed.quarter && intKeys === 1) { + // dremio interval does not support quarter. Convert to month + return [`${intervalParsed.quarter * 3}`, 'MONTH']; + } else if (intervalParsed.month && intKeys === 1) { + return [`${intervalParsed.month}`, 'MONTH']; + } else if (intervalParsed.month && intKeys === 1) { + return [`${intervalParsed.day}`, 'DAY']; + } + + throw new Error(`Cannot transform interval expression "${interval}" to Dremio dialect`); + } + } module.exports = DremioQuery; From 0afdfd681058649f144b287a65a136bfc5e3a811 Mon Sep 17 00:00:00 2001 From: wardsi Date: Wed, 29 Jan 2025 07:47:35 +0200 Subject: [PATCH 3/4] Removing space and block row padding - lint failing --- packages/cubejs-dremio-driver/driver/DremioQuery.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-dremio-driver/driver/DremioQuery.js b/packages/cubejs-dremio-driver/driver/DremioQuery.js index db3884f588a07..f76362e6da022 100644 --- a/packages/cubejs-dremio-driver/driver/DremioQuery.js +++ b/packages/cubejs-dremio-driver/driver/DremioQuery.js @@ -99,16 +99,16 @@ class DremioQuery extends BaseQuery { wrapSegmentForDimensionSelect(sql) { return `IF(${sql}, 1, 0)`; } - + /** * The input interval with (possible) plural units, like "1 hour 2 minutes", "2 year", "3 months", "4 weeks", "5 days", "3 months 24 days 15 minutes", ... * will be converted to Dremio dialect. * @see https://docs.dremio.com/24.3.x/reference/sql/sql-functions/functions/DATE_ADD/ * @see https://docs.dremio.com/24.3.x/reference/sql/sql-functions/functions/DATE_SUB/ * It returns a tuple of (formatted interval, timeUnit to use in date functions) - * This function only supports the following scenarios for now: - * ie. n year[s] or n month[3] or n day[s] - */ + * This function only supports the following scenarios for now: + * ie. n year[s] or n month[3] or n day[s] + */ formatInterval(interval) { const intervalParsed = parseSqlInterval(interval); const intKeys = Object.keys(intervalParsed).length; @@ -122,11 +122,10 @@ class DremioQuery extends BaseQuery { return [`${intervalParsed.month}`, 'MONTH']; } else if (intervalParsed.month && intKeys === 1) { return [`${intervalParsed.day}`, 'DAY']; - } + } throw new Error(`Cannot transform interval expression "${interval}" to Dremio dialect`); } - } module.exports = DremioQuery; From be5bdf99d0f66c7db67ed239085579878657c111 Mon Sep 17 00:00:00 2001 From: wardsi Date: Sun, 2 Feb 2025 11:15:45 +0200 Subject: [PATCH 4/4] Adding support for hrs, mins, seconds. Additionally, timestamp formatting corrections --- .../driver/DremioQuery.js | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-dremio-driver/driver/DremioQuery.js b/packages/cubejs-dremio-driver/driver/DremioQuery.js index f76362e6da022..8e231f3d25caa 100644 --- a/packages/cubejs-dremio-driver/driver/DremioQuery.js +++ b/packages/cubejs-dremio-driver/driver/DremioQuery.js @@ -56,7 +56,7 @@ class DremioQuery extends BaseQuery { } dateTimeCast(value) { - return `TO_TIMESTAMP(${value})`; + return `TO_TIMESTAMP(${value}, 'YYYY-MM-DD"T"HH24:MI:SS.FFF')`; } subtractInterval(date, interval) { @@ -73,6 +73,24 @@ class DremioQuery extends BaseQuery { return `DATE_ADD(${date}, CAST(${intervalFormatted} as INTERVAL ${timeUnit}))`; } + /** + * @param {string} timestamp + * @param {string} interval + * @returns {string} + */ + addTimestampInterval(timestamp, interval) { + return this.addInterval(timestamp, interval); + } + + /** + * @param {string} timestamp + * @param {string} interval + * @returns {string} + */ + subtractTimestampInterval(timestamp, interval) { + return this.subtractInterval(timestamp, interval); + } + timeGroupedColumn(granularity, dimension) { return GRANULARITY_TO_INTERVAL[granularity](dimension); } @@ -85,7 +103,8 @@ class DremioQuery extends BaseQuery { const values = timeDimension.timeSeries().map( ([from, to]) => `select '${from}' f, '${to}' t` ).join(' UNION ALL '); - return `SELECT TO_TIMESTAMP(dates.f, 'YYYY-MM-DDTHH:MI:SS.FFF') date_from, TO_TIMESTAMP(dates.t, 'YYYY-MM-DDTHH:MI:SS.FFF') date_to FROM (${values}) AS dates`; + + return `SELECT TO_TIMESTAMP(dates.f, 'YYYY-MM-DD"T"HH24:MI:SS.FFF') date_from, TO_TIMESTAMP(dates.t, 'YYYY-MM-DD"T"HH24:MI:SS.FFF') date_to FROM (${values}) AS dates`; } concatStringsSql(strings) { @@ -107,7 +126,7 @@ class DremioQuery extends BaseQuery { * @see https://docs.dremio.com/24.3.x/reference/sql/sql-functions/functions/DATE_SUB/ * It returns a tuple of (formatted interval, timeUnit to use in date functions) * This function only supports the following scenarios for now: - * ie. n year[s] or n month[3] or n day[s] + * ie. n year[s] or n quarter[s] or n month[s] or n week[s] or n day[s] */ formatInterval(interval) { const intervalParsed = parseSqlInterval(interval); @@ -118,14 +137,36 @@ class DremioQuery extends BaseQuery { } else if (intervalParsed.quarter && intKeys === 1) { // dremio interval does not support quarter. Convert to month return [`${intervalParsed.quarter * 3}`, 'MONTH']; + } else if (intervalParsed.week && intKeys === 1) { + // dremio interval does not support week. Convert to days + return [`${intervalParsed.week * 7}`, 'DAY']; } else if (intervalParsed.month && intKeys === 1) { return [`${intervalParsed.month}`, 'MONTH']; } else if (intervalParsed.month && intKeys === 1) { return [`${intervalParsed.day}`, 'DAY']; + } else if (intervalParsed.hour && intKeys === 1) { + return [`${intervalParsed.hour}`, 'HOUR']; + } else if (intervalParsed.minute && intKeys === 1) { + return [`${intervalParsed.minute}`, 'MINUTE']; + } else if (intervalParsed.second && intKeys === 1) { + return [`${intervalParsed.second}`, 'SECOND']; } throw new Error(`Cannot transform interval expression "${interval}" to Dremio dialect`); } + + sqlTemplates() { + const templates = super.sqlTemplates(); + templates.functions.CURRENTDATE = 'CURRENT_DATE'; + templates.functions.DATETRUNC = 'DATE_TRUNC(\'{{ date_part }}\', {{ args_concat }})'; + templates.functions.DATEPART = 'DATE_PART(\'{{ date_part }}\', {{ args_concat }})'; + // really need the date locale formatting here... + templates.functions.DATE = 'TO_DATE({{ args_concat }},\'YYYY-MM-DD\', 1)'; + templates.functions.DATEDIFF = 'DATE_DIFF(DATE, DATE_TRUNC(\'{{ date_part }}\', {{ args[1] }}), DATE_TRUNC(\'{{ date_part }}\', {{ args[2] }}))'; + templates.expressions.interval_single_date_part = 'CAST({{ num }} as INTERVAL {{ date_part }})'; + templates.quotes.identifiers = '"'; + return templates; + } } module.exports = DremioQuery;