Skip to content

Commit 133899c

Browse files
KSDaemonFrank-TXS
KSDaemon
authored and
Frank-TXS
committed
feat(tesseract): Support custom granularities in to_date rolling windows (cube-js#9739)
* remove useless comment * implement baseQuery's dimensionTimeGroupedColumn() in Granularity object * Use Granularity obj in ToDateRollingWindowJoinCondition * add missing quarter to SqlInterval * cargo fmt * add support for custom granularities to base filter * add tests * cargo fmt * skip tests for basequery
1 parent 3c5d5fc commit 133899c

File tree

11 files changed

+290
-72
lines changed

11 files changed

+290
-72
lines changed

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3665,7 +3665,6 @@ export class BaseQuery {
36653665
* @param {import('./Granularity').Granularity} granularity
36663666
* @return {string}
36673667
*/
3668-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
36693668
dimensionTimeGroupedColumn(dimension, granularity) {
36703669
let dtDate;
36713670

‎packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,13 @@ describe('SQL Generation', () => {
144144
granularity: 'week'
145145
}
146146
},
147+
countRollingThreeDaysToDate: {
148+
type: 'count',
149+
rollingWindow: {
150+
type: 'to_date',
151+
granularity: 'three_days'
152+
}
153+
},
147154
revenue_qtd: {
148155
type: 'sum',
149156
sql: 'amount',
@@ -1210,6 +1217,149 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL
12101217
}
12111218
]));
12121219

1220+
if (getEnv('nativeSqlPlanner')) {
1221+
it('custom granularity rolling window to_date with one time dimension with regular granularity', async () => runQueryTest({
1222+
measures: [
1223+
'visitors.countRollingThreeDaysToDate'
1224+
],
1225+
timeDimensions: [
1226+
{
1227+
dimension: 'visitors.created_at',
1228+
granularity: 'day',
1229+
dateRange: ['2017-01-01', '2017-01-10']
1230+
}
1231+
],
1232+
order: [{
1233+
id: 'visitors.created_at'
1234+
}],
1235+
timezone: 'America/Los_Angeles'
1236+
}, [
1237+
{
1238+
visitors__count_rolling_three_days_to_date: null,
1239+
visitors__created_at_day: '2017-01-01T00:00:00.000Z',
1240+
},
1241+
{
1242+
visitors__count_rolling_three_days_to_date: '1',
1243+
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
1244+
},
1245+
{
1246+
visitors__count_rolling_three_days_to_date: '1',
1247+
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
1248+
},
1249+
{
1250+
visitors__count_rolling_three_days_to_date: '1',
1251+
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
1252+
},
1253+
{
1254+
visitors__count_rolling_three_days_to_date: '2',
1255+
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
1256+
},
1257+
{
1258+
visitors__count_rolling_three_days_to_date: '4',
1259+
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
1260+
},
1261+
{
1262+
visitors__count_rolling_three_days_to_date: null,
1263+
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
1264+
},
1265+
{
1266+
visitors__count_rolling_three_days_to_date: null,
1267+
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
1268+
},
1269+
{
1270+
visitors__count_rolling_three_days_to_date: null,
1271+
visitors__created_at_day: '2017-01-09T00:00:00.000Z',
1272+
},
1273+
{
1274+
visitors__count_rolling_three_days_to_date: null,
1275+
visitors__created_at_day: '2017-01-10T00:00:00.000Z',
1276+
},
1277+
]));
1278+
} else {
1279+
it.skip('NO_BASE_QUERY_SUPPORT: custom granularity rolling window to_date with one time dimension with regular granularity', () => {
1280+
// Skipping because it works only in Tesseract
1281+
});
1282+
}
1283+
1284+
if (getEnv('nativeSqlPlanner')) {
1285+
it('custom granularity rolling window to_date with two time dimension granularities one custom one regular', async () => runQueryTest({
1286+
measures: [
1287+
'visitors.countRollingThreeDaysToDate'
1288+
],
1289+
timeDimensions: [
1290+
{
1291+
dimension: 'visitors.created_at',
1292+
granularity: 'three_days',
1293+
dateRange: ['2017-01-01', '2017-01-10']
1294+
},
1295+
{
1296+
dimension: 'visitors.created_at',
1297+
granularity: 'day',
1298+
dateRange: ['2017-01-01', '2017-01-10']
1299+
}
1300+
],
1301+
order: [{
1302+
id: 'visitors.created_at'
1303+
}],
1304+
timezone: 'America/Los_Angeles'
1305+
}, [
1306+
{
1307+
visitors__count_rolling_three_days_to_date: null,
1308+
visitors__created_at_day: '2017-01-01T00:00:00.000Z',
1309+
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
1310+
},
1311+
{
1312+
visitors__count_rolling_three_days_to_date: '1',
1313+
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
1314+
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
1315+
},
1316+
{
1317+
visitors__count_rolling_three_days_to_date: '1',
1318+
visitors__created_at_day: '2017-01-03T00:00:00.000Z',
1319+
visitors__created_at_three_days: '2017-01-01T00:00:00.000Z',
1320+
},
1321+
{
1322+
visitors__count_rolling_three_days_to_date: '1',
1323+
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
1324+
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1325+
},
1326+
{
1327+
visitors__count_rolling_three_days_to_date: '2',
1328+
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
1329+
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1330+
},
1331+
{
1332+
visitors__count_rolling_three_days_to_date: '4',
1333+
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
1334+
visitors__created_at_three_days: '2017-01-04T00:00:00.000Z',
1335+
},
1336+
{
1337+
visitors__count_rolling_three_days_to_date: null,
1338+
visitors__created_at_day: '2017-01-07T00:00:00.000Z',
1339+
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1340+
},
1341+
{
1342+
visitors__count_rolling_three_days_to_date: null,
1343+
visitors__created_at_day: '2017-01-08T00:00:00.000Z',
1344+
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1345+
},
1346+
{
1347+
visitors__count_rolling_three_days_to_date: null,
1348+
visitors__created_at_day: '2017-01-09T00:00:00.000Z',
1349+
visitors__created_at_three_days: '2017-01-07T00:00:00.000Z',
1350+
},
1351+
{
1352+
visitors__count_rolling_three_days_to_date: null,
1353+
visitors__created_at_day: '2017-01-10T00:00:00.000Z',
1354+
visitors__created_at_three_days: '2017-01-10T00:00:00.000Z',
1355+
},
1356+
]));
1357+
} else {
1358+
it.skip('NO_BASE_QUERY_SUPPORT: custom granularity rolling window to_date with two time dimension granularities one custom one regular', () => {
1359+
// Skipping because it works only in Tesseract
1360+
});
1361+
}
1362+
12131363
it('rolling window with same td with and without granularity', async () => runQueryTest({
12141364
measures: [
12151365
'visitors.countRollingWeekToDate'

‎rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/rolling_window.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use crate::logical_plan::*;
22
use crate::planner::query_properties::OrderByItem;
33
use crate::planner::sql_evaluator::MemberSymbol;
4+
use crate::planner::Granularity;
45
use std::rc::Rc;
56

67
pub struct MultiStageRegularRollingWindow {
@@ -24,14 +25,17 @@ impl PrettyPrint for MultiStageRegularRollingWindow {
2425
}
2526

2627
pub struct MultiStageToDateRollingWindow {
27-
pub granularity: String,
28+
pub granularity_obj: Rc<Granularity>,
2829
}
2930

3031
impl PrettyPrint for MultiStageToDateRollingWindow {
3132
fn pretty_print(&self, result: &mut PrettyPrintResult, state: &PrettyPrintState) {
3233
result.println("ToDate Rolling Window", state);
3334
let state = state.new_level();
34-
result.println(&format!("granularity: {}", self.granularity), &state);
35+
result.println(
36+
&format!("granularity: {}", self.granularity_obj.granularity()),
37+
&state,
38+
);
3539
}
3640
}
3741

‎rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1061,7 +1061,7 @@ impl PhysicalPlanBuilder {
10611061
MultiStageRollingWindowType::ToDate(to_date_rolling_window) => {
10621062
JoinCondition::new_to_date_rolling_join(
10631063
root_alias.clone(),
1064-
to_date_rolling_window.granularity.clone(),
1064+
to_date_rolling_window.granularity_obj.clone(),
10651065
Expr::Reference(QualifiedColumnName::new(
10661066
Some(measure_input_alias.clone()),
10671067
base_time_dimension_alias,
@@ -1092,7 +1092,7 @@ impl PhysicalPlanBuilder {
10921092
let mut render_references = HashMap::new();
10931093
let mut select_builder = SelectBuilder::new(from.clone());
10941094

1095-
//We insert render reference for main time dimension (with the some granularity as in time series to avoid unnecessary date_tranc)
1095+
//We insert render reference for main time dimension (with some granularity as in time series to avoid unnecessary date_tranc)
10961096
render_references.insert(
10971097
time_dimension.full_name(),
10981098
QualifiedColumnName::new(Some(root_alias.clone()), format!("date_from")),

‎rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use super::{Expr, SingleAliasedSource};
22
use crate::planner::query_tools::QueryTools;
33
use crate::planner::sql_templates::PlanSqlTemplates;
4-
use crate::planner::{BaseJoinCondition, VisitorContext};
4+
use crate::planner::{BaseJoinCondition, Granularity, VisitorContext};
55
use cubenativeutils::CubeError;
66
use lazy_static::lazy_static;
77

@@ -118,15 +118,15 @@ impl RollingTotalJoinCondition {
118118
}
119119
pub struct ToDateRollingWindowJoinCondition {
120120
time_series_source: String,
121-
granularity: String,
121+
granularity: Rc<Granularity>,
122122
time_dimension: Expr,
123123
_query_tools: Rc<QueryTools>,
124124
}
125125

126126
impl ToDateRollingWindowJoinCondition {
127127
pub fn new(
128128
time_series_source: String,
129-
granularity: String,
129+
granularity: Rc<Granularity>,
130130
time_dimension: Expr,
131131
query_tools: Rc<QueryTools>,
132132
) -> Self {
@@ -151,7 +151,7 @@ impl ToDateRollingWindowJoinCondition {
151151
templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?;
152152
let date_from = templates.rolling_window_expr_timestamp_cast(&date_from)?;
153153
let date_to = templates.rolling_window_expr_timestamp_cast(&date_to)?;
154-
let grouped_from = templates.time_grouped_column(self.granularity.clone(), date_from)?;
154+
let grouped_from = self.granularity.apply_to_input_sql(templates, date_from)?;
155155
let result = format!("{date_column} >= {grouped_from} and {date_column} <= {date_to}");
156156
Ok(result)
157157
}
@@ -243,7 +243,7 @@ impl JoinCondition {
243243

244244
pub fn new_to_date_rolling_join(
245245
time_series_source: String,
246-
granularity: String,
246+
granularity: Rc<Granularity>,
247247
time_dimension: Expr,
248248
query_tools: Rc<QueryTools>,
249249
) -> Self {

‎rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs‎

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ use crate::planner::query_tools::QueryTools;
33
use crate::planner::sql_evaluator::MemberSymbol;
44
use crate::planner::sql_templates::PlanSqlTemplates;
55
use crate::planner::sql_templates::TemplateProjectionColumn;
6-
use crate::planner::QueryDateTimeHelper;
76
use crate::planner::{evaluate_with_context, FiltersContext, VisitorContext};
7+
use crate::planner::{Granularity, GranularityHelper, QueryDateTimeHelper};
88
use cubenativeutils::CubeError;
99
use std::rc::Rc;
1010

@@ -188,13 +188,47 @@ impl BaseFilter {
188188
filters_context,
189189
&member_type,
190190
)?,
191-
FilterOperator::ToDateRollingWindowDateRange => self
192-
.to_date_rolling_window_date_range(
191+
FilterOperator::ToDateRollingWindowDateRange => {
192+
let query_granularity = if self.values.len() >= 3 {
193+
if let Some(granularity) = &self.values[2] {
194+
granularity
195+
} else {
196+
return Err(CubeError::user(
197+
"Granularity required for to_date rolling window".to_string(),
198+
));
199+
}
200+
} else {
201+
return Err(CubeError::user(
202+
"Granularity required for to_date rolling window".to_string(),
203+
));
204+
};
205+
let evaluator_compiler_cell = self.query_tools.evaluator_compiler().clone();
206+
let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut();
207+
208+
let Some(granularity_obj) = GranularityHelper::make_granularity_obj(
209+
self.query_tools.cube_evaluator().clone(),
210+
&mut evaluator_compiler,
211+
self.query_tools.timezone().clone(),
212+
&symbol.cube_name(),
213+
&symbol.name(),
214+
Some(query_granularity.clone()),
215+
)?
216+
else {
217+
return Err(CubeError::internal(format!(
218+
"Rolling window granularity '{}' is not found in time dimension '{}'",
219+
query_granularity,
220+
symbol.name()
221+
)));
222+
};
223+
224+
self.to_date_rolling_window_date_range(
193225
&member_sql,
194226
plan_templates,
195227
filters_context,
196228
&member_type,
197-
)?,
229+
granularity_obj,
230+
)?
231+
}
198232
FilterOperator::In => {
199233
self.in_where(&member_sql, plan_templates, filters_context, &member_type)?
200234
}
@@ -539,22 +573,11 @@ impl BaseFilter {
539573
plan_templates: &PlanSqlTemplates,
540574
_filters_context: &FiltersContext,
541575
_member_type: &Option<String>,
576+
granularity_obj: Granularity,
542577
) -> Result<String, CubeError> {
543578
let (from, to) = self.date_range_from_time_series(plan_templates)?;
544579

545-
let from = if self.values.len() >= 3 {
546-
if let Some(granularity) = &self.values[2] {
547-
plan_templates.time_grouped_column(granularity.clone(), from)?
548-
} else {
549-
return Err(CubeError::user(format!(
550-
"Granularity required for to_date rolling window"
551-
)));
552-
}
553-
} else {
554-
return Err(CubeError::user(format!(
555-
"Granularity required for to_date rolling window"
556-
)));
557-
};
580+
let from = granularity_obj.apply_to_input_sql(plan_templates, from.clone())?;
558581

559582
let date_field = plan_templates.convert_tz(member_sql.to_string())?;
560583
plan_templates.time_range_filter(date_field, from, to)

‎rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ use crate::logical_plan::*;
66
use crate::planner::planners::{multi_stage::RollingWindowType, QueryPlanner, SimpleQueryPlanner};
77
use crate::planner::query_tools::QueryTools;
88
use crate::planner::sql_evaluator::MemberSymbol;
9-
use crate::planner::{BaseDimension, BaseMeasure, BaseMember, BaseMemberHelper, BaseTimeDimension};
9+
use crate::planner::{
10+
BaseDimension, BaseMeasure, BaseMember, BaseMemberHelper, BaseTimeDimension, GranularityHelper,
11+
};
1012
use crate::planner::{OrderByItem, QueryProperties};
1113

1214
use cubenativeutils::CubeError;
@@ -126,8 +128,30 @@ impl MultiStageMemberQueryPlanner {
126128
})
127129
}
128130
RollingWindowType::ToDate(to_date_rolling_window) => {
131+
let time_dimension = &rolling_window_desc.time_dimension;
132+
let query_granularity = to_date_rolling_window.granularity.clone();
133+
134+
let evaluator_compiler_cell = self.query_tools.evaluator_compiler().clone();
135+
let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut();
136+
137+
let Some(granularity_obj) = GranularityHelper::make_granularity_obj(
138+
self.query_tools.cube_evaluator().clone(),
139+
&mut evaluator_compiler,
140+
self.query_tools.timezone().clone(),
141+
time_dimension.cube_name(),
142+
time_dimension.name(),
143+
Some(query_granularity.clone()),
144+
)?
145+
else {
146+
return Err(CubeError::internal(format!(
147+
"Rolling window granularity '{}' is not found in time dimension '{}'",
148+
query_granularity,
149+
time_dimension.name()
150+
)));
151+
};
152+
129153
MultiStageRollingWindowType::ToDate(MultiStageToDateRollingWindow {
130-
granularity: to_date_rolling_window.granularity.clone(),
154+
granularity_obj: Rc::new(granularity_obj),
131155
})
132156
}
133157
RollingWindowType::RunningTotal => MultiStageRollingWindowType::RunningTotal,

0 commit comments

Comments
 (0)