From e40f1c4a770721023a7348d6e12c71161de292eb Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Mon, 24 Mar 2025 14:38:03 +0100 Subject: [PATCH 1/3] fix(tesseract): Support rolling window with multiple granularities --- .../src/adapter/BaseQuery.js | 12 +-- .../postgres/sql-generation.test.ts | 76 +++++++++---------- .../cubesqlplanner/src/plan/join.rs | 36 ++++++++- .../cubesqlplanner/src/plan/mod.rs | 4 +- .../src/planner/base_time_dimension.rs | 2 +- .../src/planner/filter/base_filter.rs | 25 ++++-- .../src/planner/filter/compiler.rs | 2 +- .../src/planner/granularity_helper.rs | 26 +++++++ .../planners/multi_stage/applied_state.rs | 10 ++- .../planner/planners/multi_stage/member.rs | 19 ++--- .../multi_stage/member_query_planner.rs | 41 ++++++++-- .../multi_stage/rolling_window_planner.rs | 44 ++++++----- .../src/planner/planners/order_planner.rs | 8 +- .../sql_evaluator/sql_nodes/evaluate_sql.rs | 11 +-- .../sql_evaluator/sql_nodes/factory.rs | 20 ++++- .../planner/sql_evaluator/sql_nodes/mod.rs | 2 + .../sql_evaluator/sql_nodes/root_processor.rs | 10 +++ .../sql_evaluator/sql_nodes/time_dimension.rs | 74 ++++++++++++++++++ 18 files changed, 313 insertions(+), 109 deletions(-) create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index ce8cfd60371fe..9dab4d8bfb64e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -572,9 +572,9 @@ export class BaseQuery { */ countAllQuery(sql) { return `select count(*) ${this.escapeColumnName(QueryAlias.TOTAL_COUNT) - } from (\n${sql - }\n) ${this.escapeColumnName(QueryAlias.ORIGINAL_QUERY) - }`; + } from (\n${sql + }\n) ${this.escapeColumnName(QueryAlias.ORIGINAL_QUERY) + }`; } regularAndTimeSeriesRollupQuery(regularMeasures, multipliedMeasures, cumulativeMeasures, preAggregationForQuery) { @@ -1953,7 +1953,7 @@ export class BaseQuery { ), inlineWhereConditions); return `SELECT ${this.selectAllDimensionsAndMeasures(measures)} FROM ${query - } ${this.baseWhere(filters.concat(inlineWhereConditions))}` + + } ${this.baseWhere(filters.concat(inlineWhereConditions))}` + (!this.safeEvaluateSymbolContext().ungrouped && this.groupByClause() || ''); } @@ -2103,7 +2103,7 @@ export class BaseQuery { ) ), inlineWhereConditions); return `SELECT DISTINCT ${this.keysSelect(primaryKeyDimensions)} FROM ${query - } ${this.baseWhere(filters.concat(inlineWhereConditions))}`; + } ${this.baseWhere(filters.concat(inlineWhereConditions))}`; } keysSelect(primaryKeyDimensions) { @@ -3825,7 +3825,7 @@ export class BaseQuery { sql: `${refreshKeyQuery.nowTimestampSql()} < ${updateWindow ? refreshKeyQuery.addTimestampInterval(dateTo, updateWindow) : dateTo - }`, + }`, label: originalRefreshKey }]); } diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index b9d0088fe7252..47bef39bc5f77 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -919,52 +919,52 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL { visitors__count_rolling_week_to_date: null, visitors__created_at_day: '2017-01-01T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-01T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: '1', visitors__created_at_day: '2017-01-02T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-01T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: '1', visitors__created_at_day: '2017-01-03T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-01T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: '2', visitors__created_at_day: '2017-01-04T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-04T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: '3', visitors__created_at_day: '2017-01-05T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-04T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: '5', visitors__created_at_day: '2017-01-06T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-04T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: '5', visitors__created_at_day: '2017-01-07T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-07T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: '5', visitors__created_at_day: '2017-01-08T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-07T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: null, visitors__created_at_day: '2017-01-09T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-07T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', }, { visitors__count_rolling_week_to_date: null, visitors__created_at_day: '2017-01-10T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-10T00:00:00.000Z', + visitors__created_at_month: '2017-01-01T00:00:00.000Z', } ])); @@ -976,12 +976,12 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL timeDimensions: [ { dimension: 'visitors.created_at', - granularity: 'three_days', + granularity: 'day', dateRange: ['2017-01-01', '2017-01-10'] }, { dimension: 'visitors.created_at', - granularity: 'day', + granularity: 'week', dateRange: ['2017-01-01', '2017-01-10'] } ], @@ -994,61 +994,61 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL visitors__count_rolling_unbounded: '1', visitors__count_rolling_week_to_date: null, visitors__created_at_day: '2017-01-01T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-01T00:00:00.000Z', + visitors__created_at_week: '2016-12-26T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '2', visitors__count_rolling_week_to_date: '1', - visitors__created_at_day: '2017-01-03T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-01T00:00:00.000Z', + visitors__created_at_day: '2017-01-02T00:00:00.000Z', + visitors__created_at_week: '2017-01-02T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '2', visitors__count_rolling_week_to_date: '1', - visitors__created_at_day: '2017-01-02T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-01T00:00:00.000Z', + visitors__created_at_day: '2017-01-03T00:00:00.000Z', + visitors__created_at_week: '2017-01-02T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '3', visitors__count_rolling_week_to_date: '2', visitors__created_at_day: '2017-01-04T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-04T00:00:00.000Z', + visitors__created_at_week: '2017-01-02T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '4', visitors__count_rolling_week_to_date: '3', visitors__created_at_day: '2017-01-05T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-04T00:00:00.000Z', + visitors__created_at_week: '2017-01-02T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '6', visitors__count_rolling_week_to_date: '5', visitors__created_at_day: '2017-01-06T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-04T00:00:00.000Z', + visitors__created_at_week: '2017-01-02T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '6', visitors__count_rolling_week_to_date: '5', - visitors__created_at_day: '2017-01-08T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-07T00:00:00.000Z', + visitors__created_at_day: '2017-01-07T00:00:00.000Z', + visitors__created_at_week: '2017-01-02T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '6', visitors__count_rolling_week_to_date: '5', - visitors__created_at_day: '2017-01-07T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-07T00:00:00.000Z', + visitors__created_at_day: '2017-01-08T00:00:00.000Z', + visitors__created_at_week: '2017-01-02T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '6', visitors__count_rolling_week_to_date: null, visitors__created_at_day: '2017-01-09T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-07T00:00:00.000Z', + visitors__created_at_week: '2017-01-09T00:00:00.000Z', }, { visitors__count_rolling_unbounded: '6', visitors__count_rolling_week_to_date: null, visitors__created_at_day: '2017-01-10T00:00:00.000Z', - visitors__created_at_three_days: '2017-01-10T00:00:00.000Z', + visitors__created_at_week: '2017-01-09T00:00:00.000Z', } ])); @@ -3443,18 +3443,18 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL cubeEvaluator: joinedSchemaCompilers.cubeEvaluator, compiler: joinedSchemaCompilers.compiler, }, - { - measures: ['B.bval_sum', 'B.count'], - dimensions: ['B.aid'], - filters: [{ - member: 'C.did', - operator: 'lt', - values: ['10'] - }], - order: [{ - 'B.bval_sum': 'desc' - }] - }); + { + measures: ['B.bval_sum', 'B.count'], + dimensions: ['B.aid'], + filters: [{ + member: 'C.did', + operator: 'lt', + values: ['10'] + }], + order: [{ + 'B.bval_sum': 'desc' + }] + }); const sql = query.buildSqlAndParams(); return dbRunner .testQuery(sql) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index 818cbfed95865..5ce70d718f376 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -88,6 +88,31 @@ impl RegularRollingWindowJoinCondition { } } +pub struct RollingTotalJoinCondition { + time_series_source: String, + time_dimension: Expr, +} + +impl RollingTotalJoinCondition { + pub fn new(time_series_source: String, time_dimension: Expr) -> Self { + Self { + time_series_source, + time_dimension, + } + } + + pub fn to_sql( + &self, + templates: &PlanSqlTemplates, + context: Rc, + ) -> Result { + let date_column = self.time_dimension.to_sql(templates, context)?; + let date_to = + templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?; + let result = format!("{date_column} <= {date_to}"); + Ok(result) + } +} pub struct ToDateRollingWindowJoinCondition { time_series_source: String, granularity: String, @@ -117,8 +142,6 @@ impl ToDateRollingWindowJoinCondition { ) -> Result { let date_column = self.time_dimension.to_sql(templates, context)?; - //(dateFrom, dateTo, dateField, dimensionDateFrom, dimensionDateTo, isFromStartToEnd) => `${dateField} >= ${this.timeGroupedColumn(granularity, dateFrom)} AND ${dateField} <= ${dateTo}` - let date_from = templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?; let date_to = @@ -192,6 +215,7 @@ pub enum JoinCondition { BaseJoinCondition(Rc), RegularRollingWindowJoinCondition(RegularRollingWindowJoinCondition), ToDateRollingWindowJoinCondition(ToDateRollingWindowJoinCondition), + RollingTotalJoinCondition(RollingTotalJoinCondition), } impl JoinCondition { @@ -229,6 +253,13 @@ impl JoinCondition { )) } + pub fn new_rolling_total_join(time_series_source: String, time_dimension: Expr) -> Self { + Self::RollingTotalJoinCondition(RollingTotalJoinCondition::new( + time_series_source, + time_dimension, + )) + } + pub fn new_base_join(base: Rc) -> Self { Self::BaseJoinCondition(base) } @@ -247,6 +278,7 @@ impl JoinCondition { JoinCondition::ToDateRollingWindowJoinCondition(cond) => { cond.to_sql(templates, context) } + JoinCondition::RollingTotalJoinCondition(cond) => cond.to_sql(templates, context), } } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs index 97d6d46cd27f7..534e32c0a5d02 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs @@ -16,7 +16,9 @@ pub use cte::Cte; pub use expression::{Expr, MemberExpression}; pub use filter::{Filter, FilterGroup, FilterItem}; pub use from::{From, FromSource, SingleAliasedSource, SingleSource}; -pub use join::{Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition}; +pub use join::{ + Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition, RollingTotalJoinCondition, +}; pub use order::OrderBy; pub use query_plan::QueryPlan; pub use schema::{QualifiedColumnName, Schema, SchemaColumn}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs index 61e5e606c7514..9eda676d80f25 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs @@ -126,7 +126,7 @@ impl BaseTimeDimension { self.dimension.clone() } - pub fn member_evaluator(&self) -> Rc { + pub fn base_member_evaluator(&self) -> Rc { self.dimension.member_evaluator() } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index 4ee4af6536d67..8c0a43a624a34 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -273,19 +273,19 @@ impl BaseFilter { date: String, interval: &Option, is_sub: bool, - ) -> Result { + ) -> Result, CubeError> { if let Some(interval) = interval { if interval != "unbounded" { if is_sub { - self.templates.sub_interval(date, interval.clone()) + Ok(Some(self.templates.sub_interval(date, interval.clone())?)) } else { - self.templates.add_interval(date, interval.clone()) + Ok(Some(self.templates.add_interval(date, interval.clone())?)) } } else { - Ok(date.to_string()) + Ok(None) } } else { - Ok(date.to_string()) + Ok(Some(date.to_string())) } } @@ -295,20 +295,29 @@ impl BaseFilter { let from = if self.values.len() >= 3 { self.extend_date_range_bound(from, &self.values[2], true)? } else { - from + Some(from) }; let to = if self.values.len() >= 4 { self.extend_date_range_bound(to, &self.values[3], false)? } else { - to + Some(to) }; let date_field = self .query_tools .base_tools() .convert_tz(member_sql.to_string())?; - self.templates.time_range_filter(date_field, from, to) + if let (Some(from), Some(to)) = (&from, &to) { + self.templates + .time_range_filter(date_field, from.clone(), to.clone()) + } else if let Some(from) = &from { + self.templates.gte(date_field, from.clone()) + } else if let Some(to) = &to { + self.templates.lte(date_field, to.clone()) + } else { + self.templates.always_true() + } } fn to_date_rolling_window_date_range(&self, member_sql: &str) -> Result { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs index 43109bd5ffb6e..3751492d97344 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs @@ -43,7 +43,7 @@ impl<'a> FilterCompiler<'a> { if let Some(date_range) = item.get_date_range() { let filter = BaseFilter::try_new( self.query_tools.clone(), - item.member_evaluator(), + item.base_member_evaluator(), FilterType::Dimension, FilterOperator::InDateRange, Some(date_range.into_iter().map(|v| Some(v)).collect()), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs index b1e4bf1863b80..9bede1a918470 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/granularity_helper.rs @@ -2,6 +2,9 @@ use cubenativeutils::CubeError; use itertools::Itertools; use lazy_static::lazy_static; use std::collections::HashMap; +use std::rc::Rc; + +use super::BaseTimeDimension; pub struct GranularityHelper {} @@ -41,6 +44,29 @@ impl GranularityHelper { } } + pub fn find_dimension_with_min_granularity( + dimensions: &Vec>, + ) -> Result, CubeError> { + if dimensions.is_empty() { + return Err(CubeError::internal( + "No dimensions provided for find_dimension_with_min_granularity".to_string(), + )); + } + let first = Ok(dimensions[0].clone()); + dimensions.iter().skip(1).fold(first, |acc, d| match acc { + Ok(min_dim) => { + let min_granularity = + Self::min_granularity(&min_dim.get_granularity(), &d.get_granularity())?; + if min_granularity == min_dim.get_granularity() { + Ok(min_dim) + } else { + Ok(d.clone()) + } + } + Err(e) => Err(e), + }) + } + pub fn granularity_from_interval(interval: &Option) -> Option { if let Some(interval) = interval { if interval.find("day").is_some() { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index 0846d4c6bdbf0..0378b090b7106 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -1,7 +1,7 @@ use crate::plan::{FilterGroup, FilterItem}; use crate::planner::filter::FilterOperator; use crate::planner::planners::multi_stage::MultiStageTimeShift; -use crate::planner::{BaseDimension, BaseTimeDimension}; +use crate::planner::{BaseDimension, BaseMember, BaseTimeDimension}; use itertools::Itertools; use std::cmp::PartialEq; use std::collections::HashMap; @@ -88,15 +88,19 @@ impl MultiStageAppliedState { &self.time_dimensions } + pub fn set_time_dimensions(&mut self, time_dimensions: Vec>) { + self.time_dimensions = time_dimensions; + } + pub fn change_time_dimension_granularity( &mut self, - dimension_name: &str, + time_dimension: &Rc, new_granularity: Option, ) { if let Some(time_dimension) = self .time_dimensions .iter_mut() - .find(|dim| dim.member_evaluator().full_name() == dimension_name) + .find(|dim| dim.full_name() == time_dimension.full_name()) { *time_dimension = time_dimension.change_granularity(new_granularity); } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs index 982318a1cf5d2..2731bdde32618 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs @@ -1,6 +1,5 @@ use crate::cube_bridge::measure_definition::TimeShiftReference; use crate::planner::sql_evaluator::MemberSymbol; -use crate::planner::BaseMember; use crate::planner::BaseTimeDimension; use cubenativeutils::CubeError; use lazy_static::lazy_static; @@ -88,17 +87,18 @@ pub struct ToDateRollingWindow { pub enum RollingWindowType { Regular(RegularRollingWindow), ToDate(ToDateRollingWindow), + RunningTotal, } #[derive(Clone)] pub struct RollingWindowDescription { - pub time_dimension: Rc, + pub time_dimension: Rc, pub rolling_window: RollingWindowType, } impl RollingWindowDescription { pub fn new_regular( - time_dimension: Rc, + time_dimension: Rc, trailing: Option, leading: Option, offset: String, @@ -114,17 +114,19 @@ impl RollingWindowDescription { } } - pub fn new_to_date(time_dimension: Rc, granularity: String) -> Self { + pub fn new_to_date(time_dimension: Rc, granularity: String) -> Self { Self { time_dimension, rolling_window: RollingWindowType::ToDate(ToDateRollingWindow { granularity }), } } -} -#[derive(Clone)] -pub struct RunningTotalDescription { - pub time_dimension: Rc, + pub fn new_running_total(time_dimension: Rc) -> Self { + Self { + time_dimension, + rolling_window: RollingWindowType::RunningTotal, + } + } } #[derive(Clone)] @@ -133,7 +135,6 @@ pub enum MultiStageInodeMemberType { Aggregate, Calculate, RollingWindow(RollingWindowDescription), - RunningTotal(RunningTotalDescription), } #[derive(Clone)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 0a7752bd3a21a..1b0395df745e6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -150,7 +150,7 @@ impl MultiStageMemberQueryPlanner { ) -> Result, CubeError> { let inputs = self.input_cte_aliases(); assert!(inputs.len() == 2); - let dimensions = self.all_dimensions(); + let all_dimensions = self.all_dimensions(); let root_alias = format!("time_series"); let cte_schema = cte_schemas.get(&inputs[0]).unwrap().clone(); @@ -164,8 +164,8 @@ impl MultiStageMemberQueryPlanner { let input = &inputs[1]; let alias = format!("rolling_source"); let rolling_base_cte_schema = cte_schemas.get(input).unwrap().clone(); - let time_dimension_alias = - rolling_base_cte_schema.resolve_member_alias(&rolling_window_desc.time_dimension); + let time_dimension_alias = rolling_base_cte_schema + .resolve_member_alias(&rolling_window_desc.time_dimension.clone().as_base_member()); let on = match &rolling_window_desc.rolling_window { RollingWindowType::Regular(regular_rolling_window) => { JoinCondition::new_regular_rolling_join( @@ -190,6 +190,13 @@ impl MultiStageMemberQueryPlanner { self.query_tools.clone(), ) } + RollingWindowType::RunningTotal => JoinCondition::new_rolling_total_join( + root_alias.clone(), + Expr::Reference(QualifiedColumnName::new( + Some(alias.clone()), + time_dimension_alias, + )), + ), }; join_builder.left_join_table_reference( input.clone(), @@ -203,7 +210,7 @@ impl MultiStageMemberQueryPlanner { let group_by = if self.description.member().is_ungrupped() { vec![] } else { - dimensions + all_dimensions .iter() .map(|dim| Expr::Member(MemberExpression::new(dim.clone()))) .collect_vec() @@ -219,7 +226,29 @@ impl MultiStageMemberQueryPlanner { let references_builder = ReferencesBuilder::new(from.clone()); let mut render_references = HashMap::new(); let mut select_builder = SelectBuilder::new(from.clone()); - for dim in dimensions.iter() { + + //We insert render reference for main time dimension (with the some granularity as in time series to avoid unnessesary date_tranc) + render_references.insert( + rolling_window_desc.time_dimension.full_name(), + QualifiedColumnName::new(Some(root_alias.clone()), format!("date_from")), + ); + + //We also insert render reference for the base dimension of time dimension (i.e. without `_granularit` prefix to let other time dimensions make date_tranc) + render_references.insert( + rolling_window_desc + .time_dimension + .base_dimension() + .full_name(), + QualifiedColumnName::new(Some(root_alias.clone()), format!("date_from")), + ); + + for dim in self.description.state().time_dimensions().iter() { + let alias = + references_builder.resolve_alias_for_member(&dim.full_name(), &Some(alias.clone())); + context_factory.add_dimensions_with_ignored_timezone(dim.full_name()); + select_builder.add_projection_member(&dim.clone().as_base_member(), alias); + } + for dim in self.description.state().dimensions().iter() { if dim.full_name() == rolling_window_desc.time_dimension.full_name() { render_references.insert( dim.full_name(), @@ -234,7 +263,7 @@ impl MultiStageMemberQueryPlanner { } let alias = references_builder.resolve_alias_for_member(&dim.full_name(), &Some(alias.clone())); - select_builder.add_projection_member(&dim, alias); + select_builder.add_projection_member(&dim.clone().as_base_member(), alias); } let query_member = self.query_member_as_base_member()?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs index 418648c4541c0..946a7ec8ea4b9 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs @@ -8,8 +8,9 @@ use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_templates::{PlanSqlTemplates, TemplateProjectionColumn}; use crate::planner::BaseMeasure; -use crate::planner::{BaseTimeDimension, GranularityHelper, QueryProperties}; +use crate::planner::{BaseMember, BaseTimeDimension, GranularityHelper, QueryProperties}; use cubenativeutils::CubeError; +use itertools::Itertools; use std::rc::Rc; pub struct RollingWindowPlanner { @@ -37,7 +38,7 @@ impl RollingWindowPlanner { rolling_window.clone() } else { RollingWindow { - trailing: Some("unbounded".to_string()), + trailing: None, leading: None, offset: None, rolling_type: None, @@ -60,12 +61,19 @@ impl RollingWindowPlanner { )?; return Ok(Some(rolling_base)); } - if time_dimensions.len() != 1 { + let uniq_time_dimensions = time_dimensions + .iter() + .unique_by(|a| (a.cube_name(), a.name(), a.get_date_range())) + .collect_vec(); + if uniq_time_dimensions.len() != 1 { return Err(CubeError::internal( - "Rolling window requires one time dimension".to_string(), + "Rolling window requires one time dimension and equal date ranges" + .to_string(), )); } - let time_dimension = time_dimensions[0].clone(); + + let time_dimension = + GranularityHelper::find_dimension_with_min_granularity(&time_dimensions)?; let input = vec![ self.add_time_series(time_dimension.clone(), state.clone(), descriptions)?, @@ -81,11 +89,11 @@ impl RollingWindowPlanner { )?, ]; - let time_dimension = time_dimensions[0].clone(); - let alias = format!("cte_{}", descriptions.len()); - let rolling_window_descr = if let Some(granularity) = + let rolling_window_descr = if measure.is_running_total() { + RollingWindowDescription::new_running_total(time_dimension) + } else if let Some(granularity) = self.get_to_date_rolling_granularity(&rolling_window)? { RollingWindowDescription::new_to_date(time_dimension, granularity) @@ -297,7 +305,7 @@ impl RollingWindowPlanner { rolling_window: &RollingWindow, state: Rc, ) -> Result, CubeError> { - let time_dimension_name = time_dimension.member_evaluator().full_name(); + let time_dimension_base_name = time_dimension.base_dimension().full_name(); let mut new_state = state.clone_state(); let trailing_granularity = GranularityHelper::granularity_from_interval(&rolling_window.trailing); @@ -315,10 +323,10 @@ impl RollingWindowPlanner { if templates.supports_generated_time_series() { let (from, to) = self.make_time_seires_from_to_dates_suqueries_conditions("time_series")?; - new_state.replace_range_to_subquery_in_date_filter(&time_dimension_name, from, to); + new_state.replace_range_to_subquery_in_date_filter(&time_dimension_base_name, from, to); } else if time_dimension.get_date_range().is_some() && result_granularity.is_some() { - let granularity = time_dimension.get_granularity().unwrap(); //FIXME remove this unwrap - let date_range = time_dimension.get_date_range().unwrap(); //FIXME remove this unwrap + let granularity = time_dimension.get_granularity().unwrap(); + let date_range = time_dimension.get_date_range().unwrap(); let series = self .query_tools .base_tools() @@ -327,21 +335,21 @@ impl RollingWindowPlanner { let new_from_date = series.first().unwrap()[0].clone(); let new_to_date = series.last().unwrap()[1].clone(); new_state.replace_range_in_date_filter( - &time_dimension_name, + &time_dimension_base_name, new_from_date, new_to_date, ); } } - - new_state - .change_time_dimension_granularity(&time_dimension_name, result_granularity.clone()); + let new_time_dimension = time_dimension.change_granularity(result_granularity.clone()); + //We keep only one time_dimension in the leaf query because, even if time_dimension values have different granularity, in the leaf query we need to group by the lowest granularity. + new_state.set_time_dimensions(vec![new_time_dimension]); if let Some(granularity) = self.get_to_date_rolling_granularity(rolling_window)? { - new_state.replace_to_date_date_range_filter(&time_dimension_name, &granularity); + new_state.replace_to_date_date_range_filter(&time_dimension_base_name, &granularity); } else { new_state.replace_regular_date_range_filter( - &time_dimension_name, + &time_dimension_base_name, rolling_window.trailing.clone(), rolling_window.leading.clone(), ); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs index b5e9d6e2aa856..263db7b5437b9 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs @@ -24,14 +24,14 @@ impl OrderPlanner { ) -> Vec { let mut result = Vec::new(); for itm in order_by.iter() { - if let Some((pos, member)) = members + for found_item in members .iter() .enumerate() - .find(|(_, m)| m.full_name().to_lowercase() == itm.name().to_lowercase()) + .filter(|(_, m)| m.full_name().to_lowercase() == itm.name().to_lowercase()) { result.push(OrderBy::new( - Expr::Member(MemberExpression::new(member.clone())), - pos + 1, + Expr::Member(MemberExpression::new(found_item.1.clone())), + found_item.0 + 1, itm.desc(), )); } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs index 6eeda7310d040..9e14eddda76ea 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs @@ -32,16 +32,7 @@ impl SqlNode for EvaluateSqlNode { templates, ), MemberSymbol::TimeDimension(ev) => { - let input_sql = - visitor.apply(&ev.base_symbol(), node_processor.clone(), templates)?; - let res = if let Some(granularity) = ev.granularity() { - let converted_tz = query_tools.base_tools().convert_tz(input_sql)?; - query_tools - .base_tools() - .time_grouped_column(granularity.clone(), converted_tz)? - } else { - input_sql - }; + let res = visitor.apply(&ev.base_symbol(), node_processor.clone(), templates)?; Ok(res) } MemberSymbol::Measure(ev) => { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index eb1ab1f7fe6f1..3ffcc61e0ed5d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -1,8 +1,8 @@ use super::{ AutoPrefixSqlNode, CaseDimensionSqlNode, EvaluateSqlNode, FinalMeasureSqlNode, GeoDimensionSqlNode, MeasureFilterSqlNode, MultiStageRankNode, MultiStageWindowNode, - RenderReferencesSqlNode, RollingWindowNode, RootSqlNode, SqlNode, TimeShiftSqlNode, - UngroupedMeasureSqlNode, UngroupedQueryFinalMeasureSqlNode, + RenderReferencesSqlNode, RollingWindowNode, RootSqlNode, SqlNode, TimeDimensionNode, + TimeShiftSqlNode, UngroupedMeasureSqlNode, UngroupedQueryFinalMeasureSqlNode, }; use crate::plan::schema::QualifiedColumnName; use std::collections::{HashMap, HashSet}; @@ -21,6 +21,7 @@ pub struct SqlNodesFactory { multi_stage_rank: Option>, //partition_by multi_stage_window: Option>, //partition_by rolling_window: bool, + dimensions_with_ignored_timezone: HashSet, } impl SqlNodesFactory { @@ -37,6 +38,7 @@ impl SqlNodesFactory { multi_stage_rank: None, multi_stage_window: None, rolling_window: false, + dimensions_with_ignored_timezone: HashSet::new(), } } @@ -68,6 +70,10 @@ impl SqlNodesFactory { self.render_references.insert(key, value); } + pub fn add_dimensions_with_ignored_timezone(&mut self, value: String) { + self.dimensions_with_ignored_timezone.insert(value); + } + pub fn set_multi_stage_rank(&mut self, partition_by: Vec) { self.multi_stage_rank = Some(partition_by); } @@ -121,6 +127,7 @@ impl SqlNodesFactory { let root_node = RootSqlNode::new( self.dimension_processor(evaluate_sql_processor.clone()), + self.time_dimension_processor(evaluate_sql_processor.clone()), measure_processor.clone(), auto_prefix_processor.clone(), evaluate_sql_processor.clone(), @@ -179,6 +186,8 @@ impl SqlNodesFactory { } fn dimension_processor(&self, input: Rc) -> Rc { + let input: Rc = + TimeDimensionNode::new(self.dimensions_with_ignored_timezone.clone(), input); let input: Rc = GeoDimensionSqlNode::new(input); let input: Rc = CaseDimensionSqlNode::new(input); @@ -193,4 +202,11 @@ impl SqlNodesFactory { input } + + fn time_dimension_processor(&self, input: Rc) -> Rc { + let input: Rc = + TimeDimensionNode::new(self.dimensions_with_ignored_timezone.clone(), input); + + input + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs index c1a791c631adf..3aa68e34f88c9 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs @@ -11,6 +11,7 @@ pub mod render_references; pub mod rolling_window; pub mod root_processor; pub mod sql_node; +pub mod time_dimension; pub mod time_shift; pub mod ungroupped_measure; pub mod ungroupped_query_final_measure; @@ -28,6 +29,7 @@ pub use render_references::RenderReferencesSqlNode; pub use rolling_window::RollingWindowNode; pub use root_processor::RootSqlNode; pub use sql_node::SqlNode; +pub use time_dimension::TimeDimensionNode; pub use time_shift::TimeShiftSqlNode; pub use ungroupped_measure::UngroupedMeasureSqlNode; pub use ungroupped_query_final_measure::UngroupedQueryFinalMeasureSqlNode; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs index 5aae2dbb2b090..d4196772a08a3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs @@ -9,6 +9,7 @@ use std::rc::Rc; pub struct RootSqlNode { dimension_processor: Rc, + time_dimesions_processor: Rc, measure_processor: Rc, cube_name_processor: Rc, default_processor: Rc, @@ -17,12 +18,14 @@ pub struct RootSqlNode { impl RootSqlNode { pub fn new( dimension_processor: Rc, + time_dimesions_processor: Rc, measure_processor: Rc, cube_name_processor: Rc, default_processor: Rc, ) -> Rc { Rc::new(Self { dimension_processor, + time_dimesions_processor, measure_processor, cube_name_processor, default_processor, @@ -63,6 +66,13 @@ impl SqlNode for RootSqlNode { node_processor.clone(), templates, )?, + MemberSymbol::TimeDimension(_) => self.time_dimesions_processor.to_sql( + visitor, + node, + query_tools.clone(), + node_processor.clone(), + templates, + )?, MemberSymbol::Measure(_) => self.measure_processor.to_sql( visitor, node, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs new file mode 100644 index 0000000000000..3d878c8400e66 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs @@ -0,0 +1,74 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; +use crate::planner::sql_evaluator::SqlEvaluatorVisitor; +use crate::planner::sql_templates::PlanSqlTemplates; +use cubenativeutils::CubeError; +use std::any::Any; +use std::collections::HashSet; +use std::rc::Rc; + +pub struct TimeDimensionNode { + dimensions_with_ignored_timezone: HashSet, + input: Rc, +} + +impl TimeDimensionNode { + pub fn new( + dimensions_with_ignored_timezone: HashSet, + input: Rc, + ) -> Rc { + Rc::new(Self { + dimensions_with_ignored_timezone, + input, + }) + } +} + +impl SqlNode for TimeDimensionNode { + fn to_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + templates: &PlanSqlTemplates, + ) -> Result { + let input_sql = self.input.to_sql( + visitor, + node, + query_tools.clone(), + node_processor.clone(), + templates, + )?; + match node.as_ref() { + MemberSymbol::TimeDimension(ev) => { + let res = if let Some(granularity) = ev.granularity() { + let converted_tz = if self + .dimensions_with_ignored_timezone + .contains(&ev.full_name()) + { + input_sql + } else { + query_tools.base_tools().convert_tz(input_sql)? + }; + query_tools + .base_tools() + .time_grouped_column(granularity.clone(), converted_tz)? + } else { + input_sql + }; + Ok(res) + } + _ => Ok(input_sql), + } + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![] + } +} From 424bdbd91212fd116c5eeeb8cb26e6b5c18b70c7 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Fri, 28 Mar 2025 18:08:00 +0100 Subject: [PATCH 2/3] fix --- .../test/integration/postgres/sql-generation.test.ts | 2 +- .../src/planner/planners/multi_stage/member_query_planner.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index 47bef39bc5f77..ddc88b49a70e3 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -902,7 +902,7 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL timeDimensions: [ { dimension: 'visitors.created_at', - granularity: 'three_days', + granularity: 'month', dateRange: ['2017-01-01', '2017-01-10'] }, { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 1b0395df745e6..7bc51ca8c348f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -233,7 +233,7 @@ impl MultiStageMemberQueryPlanner { QualifiedColumnName::new(Some(root_alias.clone()), format!("date_from")), ); - //We also insert render reference for the base dimension of time dimension (i.e. without `_granularit` prefix to let other time dimensions make date_tranc) + //We also insert render reference for the base dimension of time dimension (i.e. without `_granularity` prefix to let other time dimensions make date_tranc) render_references.insert( rolling_window_desc .time_dimension From 2d63c58bbbba29b558f510eeda1e333a8157da38 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Fri, 28 Mar 2025 18:37:17 +0100 Subject: [PATCH 3/3] fix --- .../src/adapter/BaseQuery.js | 12 +++++----- .../postgres/sql-generation.test.ts | 24 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 9dab4d8bfb64e..ce8cfd60371fe 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -572,9 +572,9 @@ export class BaseQuery { */ countAllQuery(sql) { return `select count(*) ${this.escapeColumnName(QueryAlias.TOTAL_COUNT) - } from (\n${sql - }\n) ${this.escapeColumnName(QueryAlias.ORIGINAL_QUERY) - }`; + } from (\n${sql + }\n) ${this.escapeColumnName(QueryAlias.ORIGINAL_QUERY) + }`; } regularAndTimeSeriesRollupQuery(regularMeasures, multipliedMeasures, cumulativeMeasures, preAggregationForQuery) { @@ -1953,7 +1953,7 @@ export class BaseQuery { ), inlineWhereConditions); return `SELECT ${this.selectAllDimensionsAndMeasures(measures)} FROM ${query - } ${this.baseWhere(filters.concat(inlineWhereConditions))}` + + } ${this.baseWhere(filters.concat(inlineWhereConditions))}` + (!this.safeEvaluateSymbolContext().ungrouped && this.groupByClause() || ''); } @@ -2103,7 +2103,7 @@ export class BaseQuery { ) ), inlineWhereConditions); return `SELECT DISTINCT ${this.keysSelect(primaryKeyDimensions)} FROM ${query - } ${this.baseWhere(filters.concat(inlineWhereConditions))}`; + } ${this.baseWhere(filters.concat(inlineWhereConditions))}`; } keysSelect(primaryKeyDimensions) { @@ -3825,7 +3825,7 @@ export class BaseQuery { sql: `${refreshKeyQuery.nowTimestampSql()} < ${updateWindow ? refreshKeyQuery.addTimestampInterval(dateTo, updateWindow) : dateTo - }`, + }`, label: originalRefreshKey }]); } diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index ddc88b49a70e3..ebbcef962ba4d 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -3443,18 +3443,18 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL cubeEvaluator: joinedSchemaCompilers.cubeEvaluator, compiler: joinedSchemaCompilers.compiler, }, - { - measures: ['B.bval_sum', 'B.count'], - dimensions: ['B.aid'], - filters: [{ - member: 'C.did', - operator: 'lt', - values: ['10'] - }], - order: [{ - 'B.bval_sum': 'desc' - }] - }); + { + measures: ['B.bval_sum', 'B.count'], + dimensions: ['B.aid'], + filters: [{ + member: 'C.did', + operator: 'lt', + values: ['10'] + }], + order: [{ + 'B.bval_sum': 'desc' + }] + }); const sql = query.buildSqlAndParams(); return dbRunner .testQuery(sql)