Skip to content

Commit e40f1c4

Browse files
waralexromwaralexrom
committed
fix(tesseract): Support rolling window with multiple granularities
1 parent fd94b02 commit e40f1c4

File tree

18 files changed

+313
-109
lines changed

18 files changed

+313
-109
lines changed

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -572,9 +572,9 @@ export class BaseQuery {
572572
*/
573573
countAllQuery(sql) {
574574
return `select count(*) ${this.escapeColumnName(QueryAlias.TOTAL_COUNT)
575-
} from (\n${sql
576-
}\n) ${this.escapeColumnName(QueryAlias.ORIGINAL_QUERY)
577-
}`;
575+
} from (\n${sql
576+
}\n) ${this.escapeColumnName(QueryAlias.ORIGINAL_QUERY)
577+
}`;
578578
}
579579

580580
regularAndTimeSeriesRollupQuery(regularMeasures, multipliedMeasures, cumulativeMeasures, preAggregationForQuery) {
@@ -1953,7 +1953,7 @@ export class BaseQuery {
19531953
), inlineWhereConditions);
19541954

19551955
return `SELECT ${this.selectAllDimensionsAndMeasures(measures)} FROM ${query
1956-
} ${this.baseWhere(filters.concat(inlineWhereConditions))}` +
1956+
} ${this.baseWhere(filters.concat(inlineWhereConditions))}` +
19571957
(!this.safeEvaluateSymbolContext().ungrouped && this.groupByClause() || '');
19581958
}
19591959

@@ -2103,7 +2103,7 @@ export class BaseQuery {
21032103
)
21042104
), inlineWhereConditions);
21052105
return `SELECT DISTINCT ${this.keysSelect(primaryKeyDimensions)} FROM ${query
2106-
} ${this.baseWhere(filters.concat(inlineWhereConditions))}`;
2106+
} ${this.baseWhere(filters.concat(inlineWhereConditions))}`;
21072107
}
21082108

21092109
keysSelect(primaryKeyDimensions) {
@@ -3825,7 +3825,7 @@ export class BaseQuery {
38253825
sql: `${refreshKeyQuery.nowTimestampSql()} < ${updateWindow ?
38263826
refreshKeyQuery.addTimestampInterval(dateTo, updateWindow) :
38273827
dateTo
3828-
}`,
3828+
}`,
38293829
label: originalRefreshKey
38303830
}]);
38313831
}

‎packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts‎

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -919,52 +919,52 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
919919
{
920920
visitors__count_rolling_week_to_date: null,
921921
visitors__created_at_day: '2017-01-01T00:00:00.000Z',
922-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
922+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
923923
},
924924
{
925925
visitors__count_rolling_week_to_date: '1',
926926
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
927-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
927+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
928928
},
929929
{
930930
visitors__count_rolling_week_to_date: '1',
931931
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
932-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
932+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
933933
},
934934
{
935935
visitors__count_rolling_week_to_date: '2',
936936
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
937-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
937+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
938938
},
939939
{
940940
visitors__count_rolling_week_to_date: '3',
941941
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
942-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
942+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
943943
},
944944
{
945945
visitors__count_rolling_week_to_date: '5',
946946
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
947-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
947+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
948948
},
949949
{
950950
visitors__count_rolling_week_to_date: '5',
951951
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
952-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
952+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
953953
},
954954
{
955955
visitors__count_rolling_week_to_date: '5',
956956
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
957-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
957+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
958958
},
959959
{
960960
visitors__count_rolling_week_to_date: null,
961961
visitors__created_at_day: '2017-01-09T00:00:00.000Z',
962-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
962+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
963963
},
964964
{
965965
visitors__count_rolling_week_to_date: null,
966966
visitors__created_at_day: '2017-01-10T00:00:00.000Z',
967-
visitors__created_at_three_days: '2017-01-10T00:00:00.000Z',
967+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
968968
}
969969
]));
970970

@@ -976,12 +976,12 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
976976
timeDimensions: [
977977
{
978978
dimension: 'visitors.created_at',
979-
granularity: 'three_days',
979+
granularity: 'day',
980980
dateRange: ['2017-01-01', '2017-01-10']
981981
},
982982
{
983983
dimension: 'visitors.created_at',
984-
granularity: 'day',
984+
granularity: 'week',
985985
dateRange: ['2017-01-01', '2017-01-10']
986986
}
987987
],
@@ -994,61 +994,61 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
994994
visitors__count_rolling_unbounded: '1',
995995
visitors__count_rolling_week_to_date: null,
996996
visitors__created_at_day: '2017-01-01T00:00:00.000Z',
997-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
997+
visitors__created_at_week: '2016-12-26T00:00:00.000Z',
998998
},
999999
{
10001000
visitors__count_rolling_unbounded: '2',
10011001
visitors__count_rolling_week_to_date: '1',
1002-
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
1003-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
1002+
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
1003+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10041004
},
10051005
{
10061006
visitors__count_rolling_unbounded: '2',
10071007
visitors__count_rolling_week_to_date: '1',
1008-
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
1009-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
1008+
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
1009+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10101010
},
10111011
{
10121012
visitors__count_rolling_unbounded: '3',
10131013
visitors__count_rolling_week_to_date: '2',
10141014
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
1015-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1015+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10161016
},
10171017
{
10181018
visitors__count_rolling_unbounded: '4',
10191019
visitors__count_rolling_week_to_date: '3',
10201020
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
1021-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1021+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10221022
},
10231023
{
10241024
visitors__count_rolling_unbounded: '6',
10251025
visitors__count_rolling_week_to_date: '5',
10261026
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
1027-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1027+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10281028
},
10291029
{
10301030
visitors__count_rolling_unbounded: '6',
10311031
visitors__count_rolling_week_to_date: '5',
1032-
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
1033-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1032+
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
1033+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10341034
},
10351035
{
10361036
visitors__count_rolling_unbounded: '6',
10371037
visitors__count_rolling_week_to_date: '5',
1038-
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
1039-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1038+
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
1039+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10401040
},
10411041
{
10421042
visitors__count_rolling_unbounded: '6',
10431043
visitors__count_rolling_week_to_date: null,
10441044
visitors__created_at_day: '2017-01-09T00:00:00.000Z',
1045-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1045+
visitors__created_at_week: '2017-01-09T00:00:00.000Z',
10461046
},
10471047
{
10481048
visitors__count_rolling_unbounded: '6',
10491049
visitors__count_rolling_week_to_date: null,
10501050
visitors__created_at_day: '2017-01-10T00:00:00.000Z',
1051-
visitors__created_at_three_days: '2017-01-10T00:00:00.000Z',
1051+
visitors__created_at_week: '2017-01-09T00:00:00.000Z',
10521052
}
10531053
]));
10541054

@@ -3443,18 +3443,18 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
34433443
cubeEvaluator: joinedSchemaCompilers.cubeEvaluator,
34443444
compiler: joinedSchemaCompilers.compiler,
34453445
},
3446-
{
3447-
measures: ['B.bval_sum', 'B.count'],
3448-
dimensions: ['B.aid'],
3449-
filters: [{
3450-
member: 'C.did',
3451-
operator: 'lt',
3452-
values: ['10']
3453-
}],
3454-
order: [{
3455-
'B.bval_sum': 'desc'
3456-
}]
3457-
});
3446+
{
3447+
measures: ['B.bval_sum', 'B.count'],
3448+
dimensions: ['B.aid'],
3449+
filters: [{
3450+
member: 'C.did',
3451+
operator: 'lt',
3452+
values: ['10']
3453+
}],
3454+
order: [{
3455+
'B.bval_sum': 'desc'
3456+
}]
3457+
});
34583458
const sql = query.buildSqlAndParams();
34593459
return dbRunner
34603460
.testQuery(sql)

‎rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,31 @@ impl RegularRollingWindowJoinCondition {
8888
}
8989
}
9090

91+
pub struct RollingTotalJoinCondition {
92+
time_series_source: String,
93+
time_dimension: Expr,
94+
}
95+
96+
impl RollingTotalJoinCondition {
97+
pub fn new(time_series_source: String, time_dimension: Expr) -> Self {
98+
Self {
99+
time_series_source,
100+
time_dimension,
101+
}
102+
}
103+
104+
pub fn to_sql(
105+
&self,
106+
templates: &PlanSqlTemplates,
107+
context: Rc<VisitorContext>,
108+
) -> Result<String, CubeError> {
109+
let date_column = self.time_dimension.to_sql(templates, context)?;
110+
let date_to =
111+
templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?;
112+
let result = format!("{date_column} <= {date_to}");
113+
Ok(result)
114+
}
115+
}
91116
pub struct ToDateRollingWindowJoinCondition {
92117
time_series_source: String,
93118
granularity: String,
@@ -117,8 +142,6 @@ impl ToDateRollingWindowJoinCondition {
117142
) -> Result<String, CubeError> {
118143
let date_column = self.time_dimension.to_sql(templates, context)?;
119144

120-
//(dateFrom, dateTo, dateField, dimensionDateFrom, dimensionDateTo, isFromStartToEnd) => `${dateField} >= ${this.timeGroupedColumn(granularity, dateFrom)} AND ${dateField} <= ${dateTo}`
121-
122145
let date_from =
123146
templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?;
124147
let date_to =
@@ -192,6 +215,7 @@ pub enum JoinCondition {
192215
BaseJoinCondition(Rc<dyn BaseJoinCondition>),
193216
RegularRollingWindowJoinCondition(RegularRollingWindowJoinCondition),
194217
ToDateRollingWindowJoinCondition(ToDateRollingWindowJoinCondition),
218+
RollingTotalJoinCondition(RollingTotalJoinCondition),
195219
}
196220

197221
impl JoinCondition {
@@ -229,6 +253,13 @@ impl JoinCondition {
229253
))
230254
}
231255

256+
pub fn new_rolling_total_join(time_series_source: String, time_dimension: Expr) -> Self {
257+
Self::RollingTotalJoinCondition(RollingTotalJoinCondition::new(
258+
time_series_source,
259+
time_dimension,
260+
))
261+
}
262+
232263
pub fn new_base_join(base: Rc<dyn BaseJoinCondition>) -> Self {
233264
Self::BaseJoinCondition(base)
234265
}
@@ -247,6 +278,7 @@ impl JoinCondition {
247278
JoinCondition::ToDateRollingWindowJoinCondition(cond) => {
248279
cond.to_sql(templates, context)
249280
}
281+
JoinCondition::RollingTotalJoinCondition(cond) => cond.to_sql(templates, context),
250282
}
251283
}
252284
}

‎rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ pub use cte::Cte;
1616
pub use expression::{Expr, MemberExpression};
1717
pub use filter::{Filter, FilterGroup, FilterItem};
1818
pub use from::{From, FromSource, SingleAliasedSource, SingleSource};
19-
pub use join::{Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition};
19+
pub use join::{
20+
Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition, RollingTotalJoinCondition,
21+
};
2022
pub use order::OrderBy;
2123
pub use query_plan::QueryPlan;
2224
pub use schema::{QualifiedColumnName, Schema, SchemaColumn};

‎rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ impl BaseTimeDimension {
126126
self.dimension.clone()
127127
}
128128

129-
pub fn member_evaluator(&self) -> Rc<MemberSymbol> {
129+
pub fn base_member_evaluator(&self) -> Rc<MemberSymbol> {
130130
self.dimension.member_evaluator()
131131
}
132132

‎rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -273,19 +273,19 @@ impl BaseFilter {
273273
date: String,
274274
interval: &Option<String>,
275275
is_sub: bool,
276-
) -> Result<String, CubeError> {
276+
) -> Result<Option<String>, CubeError> {
277277
if let Some(interval) = interval {
278278
if interval != "unbounded" {
279279
if is_sub {
280-
self.templates.sub_interval(date, interval.clone())
280+
Ok(Some(self.templates.sub_interval(date, interval.clone())?))
281281
} else {
282-
self.templates.add_interval(date, interval.clone())
282+
Ok(Some(self.templates.add_interval(date, interval.clone())?))
283283
}
284284
} else {
285-
Ok(date.to_string())
285+
Ok(None)
286286
}
287287
} else {
288-
Ok(date.to_string())
288+
Ok(Some(date.to_string()))
289289
}
290290
}
291291

@@ -295,20 +295,29 @@ impl BaseFilter {
295295
let from = if self.values.len() >= 3 {
296296
self.extend_date_range_bound(from, &self.values[2], true)?
297297
} else {
298-
from
298+
Some(from)
299299
};
300300

301301
let to = if self.values.len() >= 4 {
302302
self.extend_date_range_bound(to, &self.values[3], false)?
303303
} else {
304-
to
304+
Some(to)
305305
};
306306

307307
let date_field = self
308308
.query_tools
309309
.base_tools()
310310
.convert_tz(member_sql.to_string())?;
311-
self.templates.time_range_filter(date_field, from, to)
311+
if let (Some(from), Some(to)) = (&from, &to) {
312+
self.templates
313+
.time_range_filter(date_field, from.clone(), to.clone())
314+
} else if let Some(from) = &from {
315+
self.templates.gte(date_field, from.clone())
316+
} else if let Some(to) = &to {
317+
self.templates.lte(date_field, to.clone())
318+
} else {
319+
self.templates.always_true()
320+
}
312321
}
313322

314323
fn to_date_rolling_window_date_range(&self, member_sql: &str) -> Result<String, CubeError> {

‎rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ impl<'a> FilterCompiler<'a> {
4343
if let Some(date_range) = item.get_date_range() {
4444
let filter = BaseFilter::try_new(
4545
self.query_tools.clone(),
46-
item.member_evaluator(),
46+
item.base_member_evaluator(),
4747
FilterType::Dimension,
4848
FilterOperator::InDateRange,
4949
Some(date_range.into_iter().map(|v| Some(v)).collect()),

0 commit comments

Comments
 (0)