Skip to content

Commit aea0d1e

Browse files
waralexrommarianore-muttdata
waralexrom
authored and
marianore-muttdata
committed
fix(tesseract): Support rolling window with multiple granularities (cube-js#9382)
1 parent 1975b84 commit aea0d1e

File tree

17 files changed

+296
-92
lines changed

17 files changed

+296
-92
lines changed

‎packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -902,7 +902,7 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
902902
timeDimensions: [
903903
{
904904
dimension: 'visitors.created_at',
905-
granularity: 'three_days',
905+
granularity: 'month',
906906
dateRange: ['2017-01-01', '2017-01-10']
907907
},
908908
{
@@ -919,52 +919,52 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
919919
{
920920
visitors__count_rolling_week_to_date: null,
921921
visitors__created_at_day: '2017-01-01T00:00:00.000Z',
922-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
922+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
923923
},
924924
{
925925
visitors__count_rolling_week_to_date: '1',
926926
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
927-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
927+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
928928
},
929929
{
930930
visitors__count_rolling_week_to_date: '1',
931931
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
932-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
932+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
933933
},
934934
{
935935
visitors__count_rolling_week_to_date: '2',
936936
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
937-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
937+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
938938
},
939939
{
940940
visitors__count_rolling_week_to_date: '3',
941941
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
942-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
942+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
943943
},
944944
{
945945
visitors__count_rolling_week_to_date: '5',
946946
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
947-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
947+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
948948
},
949949
{
950950
visitors__count_rolling_week_to_date: '5',
951951
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
952-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
952+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
953953
},
954954
{
955955
visitors__count_rolling_week_to_date: '5',
956956
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
957-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
957+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
958958
},
959959
{
960960
visitors__count_rolling_week_to_date: null,
961961
visitors__created_at_day: '2017-01-09T00:00:00.000Z',
962-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
962+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
963963
},
964964
{
965965
visitors__count_rolling_week_to_date: null,
966966
visitors__created_at_day: '2017-01-10T00:00:00.000Z',
967-
visitors__created_at_three_days: '2017-01-10T00:00:00.000Z',
967+
visitors__created_at_month: '2017-01-01T00:00:00.000Z',
968968
}
969969
]));
970970

@@ -976,12 +976,12 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
976976
timeDimensions: [
977977
{
978978
dimension: 'visitors.created_at',
979-
granularity: 'three_days',
979+
granularity: 'day',
980980
dateRange: ['2017-01-01', '2017-01-10']
981981
},
982982
{
983983
dimension: 'visitors.created_at',
984-
granularity: 'day',
984+
granularity: 'week',
985985
dateRange: ['2017-01-01', '2017-01-10']
986986
}
987987
],
@@ -994,61 +994,61 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
994994
visitors__count_rolling_unbounded: '1',
995995
visitors__count_rolling_week_to_date: null,
996996
visitors__created_at_day: '2017-01-01T00:00:00.000Z',
997-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
997+
visitors__created_at_week: '2016-12-26T00:00:00.000Z',
998998
},
999999
{
10001000
visitors__count_rolling_unbounded: '2',
10011001
visitors__count_rolling_week_to_date: '1',
1002-
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
1003-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
1002+
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
1003+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10041004
},
10051005
{
10061006
visitors__count_rolling_unbounded: '2',
10071007
visitors__count_rolling_week_to_date: '1',
1008-
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
1009-
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
1008+
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
1009+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10101010
},
10111011
{
10121012
visitors__count_rolling_unbounded: '3',
10131013
visitors__count_rolling_week_to_date: '2',
10141014
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
1015-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1015+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10161016
},
10171017
{
10181018
visitors__count_rolling_unbounded: '4',
10191019
visitors__count_rolling_week_to_date: '3',
10201020
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
1021-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1021+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10221022
},
10231023
{
10241024
visitors__count_rolling_unbounded: '6',
10251025
visitors__count_rolling_week_to_date: '5',
10261026
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
1027-
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1027+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10281028
},
10291029
{
10301030
visitors__count_rolling_unbounded: '6',
10311031
visitors__count_rolling_week_to_date: '5',
1032-
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
1033-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1032+
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
1033+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10341034
},
10351035
{
10361036
visitors__count_rolling_unbounded: '6',
10371037
visitors__count_rolling_week_to_date: '5',
1038-
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
1039-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1038+
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
1039+
visitors__created_at_week: '2017-01-02T00:00:00.000Z',
10401040
},
10411041
{
10421042
visitors__count_rolling_unbounded: '6',
10431043
visitors__count_rolling_week_to_date: null,
10441044
visitors__created_at_day: '2017-01-09T00:00:00.000Z',
1045-
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1045+
visitors__created_at_week: '2017-01-09T00:00:00.000Z',
10461046
},
10471047
{
10481048
visitors__count_rolling_unbounded: '6',
10491049
visitors__count_rolling_week_to_date: null,
10501050
visitors__created_at_day: '2017-01-10T00:00:00.000Z',
1051-
visitors__created_at_three_days: '2017-01-10T00:00:00.000Z',
1051+
visitors__created_at_week: '2017-01-09T00:00:00.000Z',
10521052
}
10531053
]));
10541054

‎rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,31 @@ impl RegularRollingWindowJoinCondition {
8888
}
8989
}
9090

91+
pub struct RollingTotalJoinCondition {
92+
time_series_source: String,
93+
time_dimension: Expr,
94+
}
95+
96+
impl RollingTotalJoinCondition {
97+
pub fn new(time_series_source: String, time_dimension: Expr) -> Self {
98+
Self {
99+
time_series_source,
100+
time_dimension,
101+
}
102+
}
103+
104+
pub fn to_sql(
105+
&self,
106+
templates: &PlanSqlTemplates,
107+
context: Rc<VisitorContext>,
108+
) -> Result<String, CubeError> {
109+
let date_column = self.time_dimension.to_sql(templates, context)?;
110+
let date_to =
111+
templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?;
112+
let result = format!("{date_column} <= {date_to}");
113+
Ok(result)
114+
}
115+
}
91116
pub struct ToDateRollingWindowJoinCondition {
92117
time_series_source: String,
93118
granularity: String,
@@ -117,8 +142,6 @@ impl ToDateRollingWindowJoinCondition {
117142
) -> Result<String, CubeError> {
118143
let date_column = self.time_dimension.to_sql(templates, context)?;
119144

120-
//(dateFrom, dateTo, dateField, dimensionDateFrom, dimensionDateTo, isFromStartToEnd) => `${dateField} >= ${this.timeGroupedColumn(granularity, dateFrom)} AND ${dateField} <= ${dateTo}`
121-
122145
let date_from =
123146
templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?;
124147
let date_to =
@@ -192,6 +215,7 @@ pub enum JoinCondition {
192215
BaseJoinCondition(Rc<dyn BaseJoinCondition>),
193216
RegularRollingWindowJoinCondition(RegularRollingWindowJoinCondition),
194217
ToDateRollingWindowJoinCondition(ToDateRollingWindowJoinCondition),
218+
RollingTotalJoinCondition(RollingTotalJoinCondition),
195219
}
196220

197221
impl JoinCondition {
@@ -229,6 +253,13 @@ impl JoinCondition {
229253
))
230254
}
231255

256+
pub fn new_rolling_total_join(time_series_source: String, time_dimension: Expr) -> Self {
257+
Self::RollingTotalJoinCondition(RollingTotalJoinCondition::new(
258+
time_series_source,
259+
time_dimension,
260+
))
261+
}
262+
232263
pub fn new_base_join(base: Rc<dyn BaseJoinCondition>) -> Self {
233264
Self::BaseJoinCondition(base)
234265
}
@@ -247,6 +278,7 @@ impl JoinCondition {
247278
JoinCondition::ToDateRollingWindowJoinCondition(cond) => {
248279
cond.to_sql(templates, context)
249280
}
281+
JoinCondition::RollingTotalJoinCondition(cond) => cond.to_sql(templates, context),
250282
}
251283
}
252284
}

‎rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ pub use cte::Cte;
1616
pub use expression::{Expr, MemberExpression};
1717
pub use filter::{Filter, FilterGroup, FilterItem};
1818
pub use from::{From, FromSource, SingleAliasedSource, SingleSource};
19-
pub use join::{Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition};
19+
pub use join::{
20+
Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition, RollingTotalJoinCondition,
21+
};
2022
pub use order::OrderBy;
2123
pub use query_plan::QueryPlan;
2224
pub use schema::{QualifiedColumnName, Schema, SchemaColumn};

‎rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ impl BaseTimeDimension {
126126
self.dimension.clone()
127127
}
128128

129-
pub fn member_evaluator(&self) -> Rc<MemberSymbol> {
129+
pub fn base_member_evaluator(&self) -> Rc<MemberSymbol> {
130130
self.dimension.member_evaluator()
131131
}
132132

‎rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -273,19 +273,19 @@ impl BaseFilter {
273273
date: String,
274274
interval: &Option<String>,
275275
is_sub: bool,
276-
) -> Result<String, CubeError> {
276+
) -> Result<Option<String>, CubeError> {
277277
if let Some(interval) = interval {
278278
if interval != "unbounded" {
279279
if is_sub {
280-
self.templates.sub_interval(date, interval.clone())
280+
Ok(Some(self.templates.sub_interval(date, interval.clone())?))
281281
} else {
282-
self.templates.add_interval(date, interval.clone())
282+
Ok(Some(self.templates.add_interval(date, interval.clone())?))
283283
}
284284
} else {
285-
Ok(date.to_string())
285+
Ok(None)
286286
}
287287
} else {
288-
Ok(date.to_string())
288+
Ok(Some(date.to_string()))
289289
}
290290
}
291291

@@ -295,20 +295,29 @@ impl BaseFilter {
295295
let from = if self.values.len() >= 3 {
296296
self.extend_date_range_bound(from, &self.values[2], true)?
297297
} else {
298-
from
298+
Some(from)
299299
};
300300

301301
let to = if self.values.len() >= 4 {
302302
self.extend_date_range_bound(to, &self.values[3], false)?
303303
} else {
304-
to
304+
Some(to)
305305
};
306306

307307
let date_field = self
308308
.query_tools
309309
.base_tools()
310310
.convert_tz(member_sql.to_string())?;
311-
self.templates.time_range_filter(date_field, from, to)
311+
if let (Some(from), Some(to)) = (&from, &to) {
312+
self.templates
313+
.time_range_filter(date_field, from.clone(), to.clone())
314+
} else if let Some(from) = &from {
315+
self.templates.gte(date_field, from.clone())
316+
} else if let Some(to) = &to {
317+
self.templates.lte(date_field, to.clone())
318+
} else {
319+
self.templates.always_true()
320+
}
312321
}
313322

314323
fn to_date_rolling_window_date_range(&self, member_sql: &str) -> Result<String, CubeError> {

‎rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ impl<'a> FilterCompiler<'a> {
4343
if let Some(date_range) = item.get_date_range() {
4444
let filter = BaseFilter::try_new(
4545
self.query_tools.clone(),
46-
item.member_evaluator(),
46+
item.base_member_evaluator(),
4747
FilterType::Dimension,
4848
FilterOperator::InDateRange,
4949
Some(date_range.into_iter().map(|v| Some(v)).collect()),

‎rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ use cubenativeutils::CubeError;
22
use itertools::Itertools;
33
use lazy_static::lazy_static;
44
use std::collections::HashMap;
5+
use std::rc::Rc;
6+
7+
use super::BaseTimeDimension;
58

69
pub struct GranularityHelper {}
710

@@ -41,6 +44,29 @@ impl GranularityHelper {
4144
}
4245
}
4346

47+
pub fn find_dimension_with_min_granularity(
48+
dimensions: &Vec<Rc<BaseTimeDimension>>,
49+
) -> Result<Rc<BaseTimeDimension>, CubeError> {
50+
if dimensions.is_empty() {
51+
return Err(CubeError::internal(
52+
"No dimensions provided for find_dimension_with_min_granularity".to_string(),
53+
));
54+
}
55+
let first = Ok(dimensions[0].clone());
56+
dimensions.iter().skip(1).fold(first, |acc, d| match acc {
57+
Ok(min_dim) => {
58+
let min_granularity =
59+
Self::min_granularity(&min_dim.get_granularity(), &d.get_granularity())?;
60+
if min_granularity == min_dim.get_granularity() {
61+
Ok(min_dim)
62+
} else {
63+
Ok(d.clone())
64+
}
65+
}
66+
Err(e) => Err(e),
67+
})
68+
}
69+
4470
pub fn granularity_from_interval(interval: &Option<String>) -> Option<String> {
4571
if let Some(interval) = interval {
4672
if interval.find("day").is_some() {

‎rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::plan::{FilterGroup, FilterItem};
22
use crate::planner::filter::FilterOperator;
33
use crate::planner::planners::multi_stage::MultiStageTimeShift;
4-
use crate::planner::{BaseDimension, BaseTimeDimension};
4+
use crate::planner::{BaseDimension, BaseMember, BaseTimeDimension};
55
use itertools::Itertools;
66
use std::cmp::PartialEq;
77
use std::collections::HashMap;
@@ -88,15 +88,19 @@ impl MultiStageAppliedState {
8888
&self.time_dimensions
8989
}
9090

91+
pub fn set_time_dimensions(&mut self, time_dimensions: Vec<Rc<BaseTimeDimension>>) {
92+
self.time_dimensions = time_dimensions;
93+
}
94+
9195
pub fn change_time_dimension_granularity(
9296
&mut self,
93-
dimension_name: &str,
97+
time_dimension: &Rc<BaseTimeDimension>,
9498
new_granularity: Option<String>,
9599
) {
96100
if let Some(time_dimension) = self
97101
.time_dimensions
98102
.iter_mut()
99-
.find(|dim| dim.member_evaluator().full_name() == dimension_name)
103+
.find(|dim| dim.full_name() == time_dimension.full_name())
100104
{
101105
*time_dimension = time_dimension.change_granularity(new_granularity);
102106
}

0 commit comments

Comments
 (0)