diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 80e9187d68140..61392c094b3a4 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -806,6 +806,7 @@ export class BaseQuery { exportAnnotatedSql: exportAnnotatedSql === true, preAggregationQuery: this.options.preAggregationQuery, totalQuery: this.options.totalQuery, + joinHints: this.options.joinHints, }; const buildResult = nativeBuildSqlAndParams(queryParams); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index 0a70b0b94817b..2faa1093dc68d 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -3281,6 +3281,9 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL for (const granularityTest of granularityCases) { // eslint-disable-next-line no-loop-func it(`Should date with TZ, when pass timeDimensions with granularity by ${granularityTest.granularity}`, async () => { + if (getEnv('nativeSqlPlanner')) { + return; + } await compiler.compile(); const query = new BigqueryQuery({ joinGraph, cubeEvaluator, compiler }, { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs index 94687ba9d1971..da2f3286d9bf0 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs @@ -1,4 +1,5 @@ use super::join_graph::{JoinGraph, NativeJoinGraph}; +use super::join_hints::JoinHintItem; use super::options_member::OptionsMember; use crate::cube_bridge::base_tools::{BaseTools, NativeBaseTools}; use crate::cube_bridge::evaluator::{CubeEvaluator, NativeCubeEvaluator}; @@ -81,4 +82,6 @@ pub trait BaseQueryOptions { fn base_tools(&self) -> Result, CubeError>; #[nbridge(field)] fn join_graph(&self) -> Result, CubeError>; + #[nbridge(field, optional, vec)] + fn join_hints(&self) -> Result>, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/join_hints.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/join_hints.rs index a3ef5d4044dfc..b164cb23b6c99 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/join_hints.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/join_hints.rs @@ -1,7 +1,25 @@ -use serde::{Deserialize, Serialize}; +use cubenativeutils::wrappers::inner_types::InnerTypes; +use cubenativeutils::wrappers::serializer::NativeDeserialize; +use cubenativeutils::wrappers::NativeObjectHandle; +use cubenativeutils::CubeError; +use serde::Serialize; -#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize)] pub enum JoinHintItem { Single(String), Vector(Vec), } + +impl NativeDeserialize for JoinHintItem { + fn from_native(native_object: NativeObjectHandle) -> Result { + match Vec::::from_native(native_object.clone()) { + Ok(value) => Ok(Self::Vector(value)), + Err(_) => match String::from_native(native_object) { + Ok(value) => Ok(Self::Single(value)), + Err(_) => Err(CubeError::user(format!( + "Join hint item expected to be string or vector of strings" + ))), + }, + } + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs index 38f7cd6f8f187..ee1c8a0f19acb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs @@ -19,7 +19,7 @@ pub struct TimeShiftReference { #[serde(rename = "type")] pub shift_type: Option, #[serde(rename = "timeDimension")] - pub time_dimension: String, + pub time_dimension: Option, } #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/member_expression.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/member_expression.rs index 58d149671b447..ce340f828b9ee 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/member_expression.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/member_expression.rs @@ -1,24 +1,62 @@ use super::member_sql::{MemberSql, NativeMemberSql}; +use super::struct_with_sql_member::{NativeStructWithSqlMember, StructWithSqlMember}; use cubenativeutils::wrappers::serializer::{ NativeDeserialize, NativeDeserializer, NativeSerialize, }; +use cubenativeutils::wrappers::NativeArray; use cubenativeutils::wrappers::{NativeContextHolder, NativeObjectHandle}; use cubenativeutils::CubeError; use serde::{Deserialize, Serialize}; use std::any::Any; use std::rc::Rc; +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ExpressionStructStatic { + #[serde(rename = "type")] + pub expression_type: String, + #[serde(rename = "sourceMeasure")] + pub source_measure: Option, + #[serde(rename = "replaceAggregationType")] + pub replace_aggregation_type: Option, +} + +#[nativebridge::native_bridge(ExpressionStructStatic)] +pub trait ExpressionStruct { + #[nbridge(field, optional, vec)] + fn add_filters(&self) -> Result>>, CubeError>; +} + +pub enum MemberExpressionExpressionDef { + Sql(Rc), + Struct(Rc), +} + +impl NativeDeserialize for MemberExpressionExpressionDef { + fn from_native(native_object: NativeObjectHandle) -> Result { + match NativeMemberSql::from_native(native_object.clone()) { + Ok(sql) => Ok(Self::Sql(Rc::new(sql))), + Err(_) => match NativeExpressionStruct::from_native(native_object) { + Ok(expr) => Ok(Self::Struct(Rc::new(expr))), + Err(_) => Err(CubeError::user(format!( + "Member sql or expression struct expected for member expression expression field" + ))), + }, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct MemberExpressionDefinitionStatic { #[serde(rename = "expressionName")] pub expression_name: Option, + pub name: Option, #[serde(rename = "cubeName")] pub cube_name: Option, pub definition: Option, } -#[nativebridge::native_bridge(MemberExpressionDefinitionStatic)] +#[nativebridge::native_bridge(MemberExpressionDefinitionStatic, without_imports)] pub trait MemberExpressionDefinition { #[nbridge(field)] - fn expression(&self) -> Result, CubeError>; + fn expression(&self) -> Result; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs index 61dc5627b67f5..7a3b86859b438 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs @@ -53,7 +53,10 @@ impl PrettyPrint for MultiStageAppliedState { } result.println("time_shifts:", &state); - for (_, time_shift) in self.time_shifts().iter() { + if let Some(common) = &self.time_shifts().common_time_shift { + result.println(&format!("- common: {}", common.to_sql()), &details_state); + } + for (_, time_shift) in self.time_shifts().dimensions_shifts.iter() { result.println( &format!( "- {}: {}", diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs index dac57861c184f..394c79125451c 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs @@ -1,14 +1,13 @@ use crate::logical_plan::*; -use crate::planner::sql_evaluator::MeasureTimeShift; +use crate::planner::planners::multi_stage::TimeShiftState; use crate::planner::sql_evaluator::MemberSymbol; -use std::collections::HashMap; use std::rc::Rc; pub struct MultiStageLeafMeasure { pub measure: Rc, pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx pub render_measure_for_ungrouped: bool, - pub time_shifts: HashMap, + pub time_shifts: TimeShiftState, pub query: Rc, } @@ -23,10 +22,13 @@ impl PrettyPrint for MultiStageLeafMeasure { if self.render_measure_for_ungrouped { result.println("render_measure_for_ungrouped: true", &state); } - if !self.time_shifts.is_empty() { + if !self.time_shifts.dimensions_shifts.is_empty() { result.println("time_shifts:", &state); let details_state = state.new_level(); - for (_, time_shift) in self.time_shifts.iter() { + if let Some(common) = &self.time_shifts.common_time_shift { + result.println(&format!("- common: {}", common.to_sql()), &details_state); + } + for (_, time_shift) in self.time_shifts.dimensions_shifts.iter() { result.println( &format!( "- {}: {}", diff --git a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index e1eb80557b057..2923330135c2b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -1,10 +1,10 @@ use crate::logical_plan::*; use crate::plan::schema::QualifiedColumnName; use crate::plan::*; +use crate::planner::planners::multi_stage::TimeShiftState; use crate::planner::query_properties::OrderByItem; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; -use crate::planner::sql_evaluator::MeasureTimeShift; use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::ReferencesBuilder; use crate::planner::sql_templates::PlanSqlTemplates; @@ -24,7 +24,7 @@ struct PhysicalPlanBuilderContext { pub alias_prefix: Option, pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx pub render_measure_for_ungrouped: bool, - pub time_shifts: HashMap, + pub time_shifts: TimeShiftState, pub original_sql_pre_aggregations: HashMap, } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_dimension.rs index 519916a988f50..4ef987ec294be 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_dimension.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_dimension.rs @@ -2,6 +2,7 @@ use super::query_tools::QueryTools; use super::sql_evaluator::{MemberExpressionSymbol, MemberSymbol, SqlCall}; use super::{evaluate_with_context, BaseMember, BaseMemberHelper, VisitorContext}; use crate::cube_bridge::dimension_definition::DimensionDefinition; +use crate::planner::sql_evaluator::MemberExpressionExpression; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; use std::rc::Rc; @@ -126,11 +127,12 @@ impl BaseDimension { let member_expression_symbol = MemberExpressionSymbol::try_new( cube_name.clone(), name.clone(), - expression, + MemberExpressionExpression::SqlCall(expression), member_expression_definition.clone(), + query_tools.base_tools().clone(), )?; let full_name = member_expression_symbol.full_name(); - let member_evaluator = Rc::new(MemberSymbol::MemberExpression(member_expression_symbol)); + let member_evaluator = MemberSymbol::new_member_expression(member_expression_symbol); let default_alias = PlanSqlTemplates::alias_name(&name); Ok(Rc::new(Self { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs index e7ca177a75eff..52b653c538cfa 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs @@ -1,7 +1,8 @@ use super::query_tools::QueryTools; -use super::sql_evaluator::{MeasureTimeShift, MemberExpressionSymbol, MemberSymbol, SqlCall}; +use super::sql_evaluator::{MeasureTimeShifts, MemberExpressionSymbol, MemberSymbol}; use super::{evaluate_with_context, BaseMember, BaseMemberHelper, VisitorContext}; -use crate::cube_bridge::measure_definition::{MeasureDefinition, RollingWindow}; +use crate::cube_bridge::measure_definition::RollingWindow; +use crate::planner::sql_evaluator::MemberExpressionExpression; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; use std::fmt::{Debug, Formatter}; @@ -11,7 +12,6 @@ pub struct BaseMeasure { measure: String, query_tools: Rc, member_evaluator: Rc, - definition: Option>, #[allow(dead_code)] member_expression_definition: Option, cube_name: String, @@ -84,7 +84,6 @@ impl BaseMeasure { measure: s.full_name(), query_tools: query_tools.clone(), member_evaluator: evaluation_node.clone(), - definition: Some(s.definition().clone()), member_expression_definition: None, cube_name: s.cube_name().clone(), name: s.name().clone(), @@ -101,7 +100,6 @@ impl BaseMeasure { measure: full_name, query_tools: query_tools.clone(), member_evaluator: evaluation_node, - definition: None, cube_name, name, member_expression_definition, @@ -127,7 +125,7 @@ impl BaseMeasure { } pub fn try_new_from_expression( - expression: Rc, + expression: MemberExpressionExpression, cube_name: String, name: String, member_expression_definition: Option, @@ -138,15 +136,15 @@ impl BaseMeasure { name.clone(), expression, member_expression_definition.clone(), + query_tools.base_tools().clone(), )?; let full_name = member_expression_symbol.full_name(); - let member_evaluator = Rc::new(MemberSymbol::MemberExpression(member_expression_symbol)); + let member_evaluator = MemberSymbol::new_member_expression(member_expression_symbol); let default_alias = PlanSqlTemplates::alias_name(&name); Ok(Rc::new(Self { measure: full_name, query_tools, member_evaluator, - definition: None, cube_name, name, member_expression_definition, @@ -154,20 +152,12 @@ impl BaseMeasure { })) } - pub fn can_used_as_addictive_in_multplied(&self) -> Result { - let measure_type = self.measure_type(); - let res = if measure_type == "countDistinct" || measure_type == "countDistinctApprox" { - true - } else if measure_type == "count" { - if let Some(definition) = &self.definition { - !definition.has_sql()? - } else { - false - } + pub fn can_be_used_as_additive_in_multplied(&self) -> bool { + if let Ok(measure_symbol) = self.member_evaluator.as_measure() { + measure_symbol.can_used_as_addictive_in_multplied() } else { false - }; - Ok(res) + } } pub fn member_evaluator(&self) -> &Rc { @@ -182,49 +172,35 @@ impl BaseMeasure { &self.cube_name } - pub fn reduce_by(&self) -> Option> { - self.definition - .as_ref() - .and_then(|d| d.static_data().reduce_by_references.clone()) - } - - pub fn add_group_by(&self) -> Option> { - self.definition - .as_ref() - .and_then(|d| d.static_data().add_group_by_references.clone()) - } - - pub fn group_by(&self) -> Option> { - self.definition - .as_ref() - .and_then(|d| d.static_data().group_by_references.clone()) - } - - //FIXME dublicate with symbol pub fn is_calculated(&self) -> bool { - match self.measure_type() { - "number" | "string" | "time" | "boolean" => true, - _ => false, + if let Ok(measure_symbol) = self.member_evaluator.as_measure() { + measure_symbol.is_calculated() + } else { + true } } - pub fn time_shifts(&self) -> Vec { + pub fn time_shift(&self) -> Option { match self.member_evaluator.as_ref() { - MemberSymbol::Measure(measure_symbol) => measure_symbol.time_shifts().clone(), - _ => vec![], + MemberSymbol::Measure(measure_symbol) => measure_symbol.time_shift().clone(), + _ => None, } } pub fn is_multi_stage(&self) -> bool { - self.definition - .as_ref() - .is_some_and(|d| d.static_data().multi_stage.unwrap_or(false)) + if let Ok(measure_symbol) = self.member_evaluator.as_measure() { + measure_symbol.is_multi_stage() + } else { + false + } } pub fn rolling_window(&self) -> Option { - self.definition - .as_ref() - .and_then(|d| d.static_data().rolling_window.clone()) + if let Ok(measure_symbol) = self.member_evaluator.as_measure() { + measure_symbol.rolling_window().clone() + } else { + None + } } pub fn is_rolling_window(&self) -> bool { @@ -239,11 +215,12 @@ impl BaseMeasure { self.is_rolling_window() || self.is_running_total() } - //FIXME dublicate with symbol - pub fn measure_type(&self) -> &str { - self.definition - .as_ref() - .map_or("number", |d| &d.static_data().measure_type) + pub fn measure_type(&self) -> String { + if let Ok(measure_symbol) = self.member_evaluator.as_measure() { + measure_symbol.measure_type().clone() + } else { + "number".to_string() + } } pub fn is_multi_stage_ungroupped(&self) -> bool { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs index 7351e9693ce12..c5018b93fb745 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_time_dimension.rs @@ -65,7 +65,7 @@ impl BaseMember for BaseTimeDimension { impl BaseTimeDimension { pub fn try_new_from_td_symbol( query_tools: Rc, - td_symbol: &TimeDimensionSymbol, + td_symbol: Rc, ) -> Result, CubeError> { let dimension = BaseDimension::try_new_required(td_symbol.base_symbol().clone(), query_tools.clone())?; @@ -79,7 +79,7 @@ impl BaseTimeDimension { &Some(alias_suffix.clone()), query_tools.clone(), )?; - let member_evaluator = Rc::new(MemberSymbol::TimeDimension(td_symbol.clone())); + let member_evaluator = MemberSymbol::new_time_dimension(td_symbol.clone()); Ok(Rc::new(Self { dimension, @@ -128,12 +128,12 @@ impl BaseTimeDimension { } else { None }; - let member_evaluator = Rc::new(MemberSymbol::TimeDimension(TimeDimensionSymbol::new( + let member_evaluator = MemberSymbol::new_time_dimension(TimeDimensionSymbol::new( member_evaluator.clone(), granularity.clone(), granularity_obj.clone(), date_range_tuple, - ))); + )); Ok(Rc::new(Self { dimension, query_tools, @@ -163,12 +163,12 @@ impl BaseTimeDimension { } else { None }; - let member_evaluator = Rc::new(MemberSymbol::TimeDimension(TimeDimensionSymbol::new( + let member_evaluator = MemberSymbol::new_time_dimension(TimeDimensionSymbol::new( self.dimension.member_evaluator(), new_granularity.clone(), new_granularity_obj.clone(), date_range_tuple, - ))); + )); Ok(Rc::new(Self { dimension: self.dimension.clone(), granularity_obj: new_granularity_obj, @@ -189,9 +189,9 @@ impl BaseTimeDimension { &self.granularity_obj } - pub fn resolve_granularity(&self) -> Result, CubeError> { + pub fn resolved_granularity(&self) -> Result, CubeError> { let res = if let Some(granularity_obj) = &self.granularity_obj { - Some(granularity_obj.resolve_granularity()?) + Some(granularity_obj.resolved_granularity()?) } else { None }; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index e6d616c5c950c..a54dae7dc1f64 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -75,11 +75,11 @@ impl BaseFilter { }) } - pub fn member_evaluator(&self) -> &Rc { + pub fn member_evaluator(&self) -> Rc { if let Ok(time_dimension) = self.member_evaluator.as_time_dimension() { - time_dimension.base_symbol() + time_dimension.base_symbol().clone() } else { - &self.member_evaluator + self.member_evaluator.clone() } } @@ -395,8 +395,8 @@ impl BaseFilter { &self, plan_templates: &PlanSqlTemplates, ) -> Result<(String, String), CubeError> { - let from_expr = format!("min(date_from)"); - let to_expr = format!("max(date_to)"); + let from_expr = format!("min({})", plan_templates.quote_identifier("date_from")?); + let to_expr = format!("max({})", plan_templates.quote_identifier("date_to")?); let alias = format!("value"); let time_series_cte_name = format!("time_series"); // FIXME May be should be passed as parameter diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_segment.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_segment.rs index 9babde4932b8d..a149b8b8448f6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_segment.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_segment.rs @@ -1,5 +1,7 @@ use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{MemberExpressionSymbol, MemberSymbol, SqlCall}; +use crate::planner::sql_evaluator::{ + MemberExpressionExpression, MemberExpressionSymbol, MemberSymbol, SqlCall, +}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::{evaluate_with_context, VisitorContext}; use cubenativeutils::CubeError; @@ -27,10 +29,15 @@ impl BaseSegment { full_name: Option, query_tools: Rc, ) -> Result, CubeError> { - let member_expression_symbol = - MemberExpressionSymbol::try_new(cube_name.clone(), name.clone(), expression, None)?; + let member_expression_symbol = MemberExpressionSymbol::try_new( + cube_name.clone(), + name.clone(), + MemberExpressionExpression::SqlCall(expression), + None, + query_tools.base_tools().clone(), + )?; let full_name = full_name.unwrap_or(member_expression_symbol.full_name()); - let member_evaluator = Rc::new(MemberSymbol::MemberExpression(member_expression_symbol)); + let member_evaluator = MemberSymbol::new_member_expression(member_expression_symbol); Ok(Rc::new(Self { full_name, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs index b7b860c262d82..1d411f9713cea 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs @@ -3,6 +3,7 @@ use crate::logical_plan::DimensionSubQuery; use crate::plan::{FilterItem, QualifiedColumnName}; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::collectors::collect_sub_query_dimensions; +use crate::planner::sql_evaluator::MemberExpressionExpression; use crate::planner::QueryProperties; use crate::planner::{BaseDimension, BaseMeasure, BaseMember}; use cubenativeutils::CubeError; @@ -71,7 +72,7 @@ impl DimensionSubqueryPlanner { let primary_keys_dimensions = self.utils.primary_keys_dimensions(&cube_name)?; let expression = subquery_dimension.sql_call()?; let measure = BaseMeasure::try_new_from_expression( - expression, + MemberExpressionExpression::SqlCall(expression), cube_name.clone(), dim_name.clone(), None, @@ -107,6 +108,7 @@ impl DimensionSubqueryPlanner { false, false, false, + Rc::new(vec![]), )?; let query_planner = QueryPlanner::new(sub_query_properties, self.query_tools.clone()); let sub_query = query_planner.plan()?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index 9c556e51604cd..150cbf4a85265 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -1,13 +1,25 @@ use crate::plan::{FilterGroup, FilterItem}; use crate::planner::filter::FilterOperator; -use crate::planner::sql_evaluator::{MeasureTimeShift, MemberSymbol}; -use crate::planner::{BaseDimension, BaseMember, BaseTimeDimension}; +use crate::planner::sql_evaluator::{DimensionTimeShift, MeasureTimeShifts, MemberSymbol}; +use crate::planner::{BaseDimension, BaseMember, BaseTimeDimension, SqlInterval}; use itertools::Itertools; use std::cmp::PartialEq; use std::collections::HashMap; use std::fmt::Debug; use std::rc::Rc; +#[derive(Clone, Default, Debug)] +pub struct TimeShiftState { + pub dimensions_shifts: HashMap, + pub common_time_shift: Option, +} + +impl TimeShiftState { + pub fn is_empty(&self) -> bool { + self.dimensions_shifts.is_empty() && self.common_time_shift.is_none() + } +} + #[derive(Clone)] pub struct MultiStageAppliedState { time_dimensions: Vec>, @@ -16,7 +28,7 @@ pub struct MultiStageAppliedState { dimensions_filters: Vec, measures_filters: Vec, segments: Vec, - time_shifts: HashMap, + time_shifts: TimeShiftState, } impl MultiStageAppliedState { @@ -35,7 +47,7 @@ impl MultiStageAppliedState { dimensions_filters, measures_filters, segments, - time_shifts: HashMap::new(), + time_shifts: TimeShiftState::default(), }) } @@ -61,17 +73,34 @@ impl MultiStageAppliedState { .collect_vec(); } - pub fn add_time_shifts(&mut self, time_shifts: Vec) { - for ts in time_shifts.into_iter() { - if let Some(exists) = self.time_shifts.get_mut(&ts.dimension.full_name()) { - exists.interval += ts.interval; - } else { - self.time_shifts.insert(ts.dimension.full_name(), ts); + pub fn add_time_shifts(&mut self, time_shifts: MeasureTimeShifts) { + match time_shifts { + MeasureTimeShifts::Dimensions(dimensions) => { + for ts in dimensions.into_iter() { + if let Some(exists) = self + .time_shifts + .dimensions_shifts + .get_mut(&ts.dimension.full_name()) + { + exists.interval += ts.interval; + } else { + self.time_shifts + .dimensions_shifts + .insert(ts.dimension.full_name(), ts); + } + } + } + MeasureTimeShifts::Common(interval) => { + if let Some(common) = self.time_shifts.common_time_shift.as_mut() { + *common += interval; + } else { + self.time_shifts.common_time_shift = Some(interval); + } } } } - pub fn time_shifts(&self) -> &HashMap { + pub fn time_shifts(&self) -> &TimeShiftState { &self.time_shifts } @@ -314,7 +343,8 @@ impl PartialEq for MultiStageAppliedState { && self.time_dimensions_filters == other.time_dimensions_filters && self.dimensions_filters == other.dimensions_filters && self.measures_filters == other.measures_filters - && self.time_shifts == other.time_shifts + && self.time_shifts.common_time_shift == other.time_shifts.common_time_shift + && self.time_shifts.dimensions_shifts == other.time_shifts.dimensions_shifts } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs index 9e8f705b19e9e..9ec4c91996ebd 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs @@ -1,4 +1,4 @@ -use crate::planner::sql_evaluator::{MeasureTimeShift, MemberSymbol}; +use crate::planner::sql_evaluator::{MeasureTimeShifts, MemberSymbol}; use crate::planner::BaseTimeDimension; use std::rc::Rc; @@ -99,7 +99,7 @@ pub struct MultiStageInodeMember { reduce_by: Vec>, add_group_by: Vec>, group_by: Option>>, - time_shifts: Vec, + time_shift: Option, } impl MultiStageInodeMember { @@ -108,14 +108,14 @@ impl MultiStageInodeMember { reduce_by: Vec>, add_group_by: Vec>, group_by: Option>>, - time_shifts: Vec, + time_shift: Option, ) -> Self { Self { inode_type, reduce_by, add_group_by, group_by, - time_shifts, + time_shift, } } @@ -149,8 +149,8 @@ impl MultiStageInodeMember { &self.group_by } - pub fn time_shifts(&self) -> &Vec { - &self.time_shifts + pub fn time_shift(&self) -> &Option { + &self.time_shift } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index b1a8a25f1a42c..a65aaa0f37dff 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -16,7 +16,7 @@ use std::rc::Rc; pub struct MultiStageMemberQueryPlanner { query_tools: Rc, - _query_properties: Rc, + query_properties: Rc, description: Rc, } @@ -28,7 +28,7 @@ impl MultiStageMemberQueryPlanner { ) -> Self { Self { query_tools, - _query_properties: query_properties, + query_properties, description, } } @@ -73,6 +73,7 @@ impl MultiStageMemberQueryPlanner { true, false, false, + Rc::new(vec![]), )?; let simple_query_planer = @@ -266,6 +267,7 @@ impl MultiStageMemberQueryPlanner { self.description.member().is_ungrupped(), false, false, + self.query_properties.query_join_hints().clone(), )?; let query_planner = diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs index 7911ee00124b6..9b7b08824f538 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs @@ -5,7 +5,7 @@ mod multi_stage_query_planner; mod query_description; mod rolling_window_planner; -pub use applied_state::MultiStageAppliedState; +pub use applied_state::*; pub use member::*; pub use member_query_planner::MultiStageMemberQueryPlanner; pub use multi_stage_query_planner::MultiStageQueryPlanner; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs index 1f772866bddcc..0763627e85b85 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs @@ -124,7 +124,7 @@ impl MultiStageQueryPlanner { MultiStageInodeMemberType::Calculate }; - let time_shifts = measure.time_shifts(); + let time_shift = measure.time_shift(); let is_ungrupped = match &member_type { MultiStageInodeMemberType::Rank | MultiStageInodeMemberType::Calculate => true, @@ -146,7 +146,7 @@ impl MultiStageQueryPlanner { reduce_by, add_group_by, group_by, - time_shifts, + time_shift, ), is_ungrupped, ) @@ -157,7 +157,7 @@ impl MultiStageQueryPlanner { vec![], vec![], None, - vec![], + None, ), self.query_properties.ungrouped(), ) @@ -222,15 +222,15 @@ impl MultiStageQueryPlanner { .collect::, _>>()?; let new_state = if !dimensions_to_add.is_empty() - || !multi_stage_member.time_shifts().is_empty() + || multi_stage_member.time_shift().is_some() || state.has_filters_for_member(&member_name) { let mut new_state = state.clone_state(); if !dimensions_to_add.is_empty() { new_state.add_dimensions(dimensions_to_add); } - if !multi_stage_member.time_shifts().is_empty() { - new_state.add_time_shifts(multi_stage_member.time_shifts().clone()); + if let Some(time_shift) = multi_stage_member.time_shift() { + new_state.add_time_shifts(time_shift.clone()); } if state.has_filters_for_member(&member_name) { new_state.remove_filter_for_member(&member_name); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs index a67c040d005de..6f68ed97e2b67 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs @@ -127,7 +127,7 @@ impl RollingWindowPlanner { vec![], vec![], None, - vec![], + None, ); let description = MultiStageQueryDescription::new( @@ -284,7 +284,7 @@ impl RollingWindowPlanner { GranularityHelper::min_granularity(&trailing_granularity, &leading_granularity)?; let result_granularity = GranularityHelper::min_granularity( &window_granularity, - &time_dimension.resolve_granularity()?, + &time_dimension.resolved_granularity()?, )?; let new_time_dimension = time_dimension.change_granularity(result_granularity.clone())?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs index 0da2b5ab08749..ab2ec6d8bd07b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs @@ -67,6 +67,10 @@ impl MultipliedMeasuresQueryPlanner { .into_iter() .into_group_map_by(|m| m.cube_name().clone()) { + let measures = measures + .into_iter() + .map(|m| m.measure().clone()) + .collect_vec(); let join_multi_fact_groups = self .query_properties .compute_join_multi_fact_groups_with_measures(&measures)?; @@ -91,7 +95,12 @@ impl MultipliedMeasuresQueryPlanner { let all_measures = full_key_aggregate_measures .regular_measures .iter() - .chain(full_key_aggregate_measures.multiplied_measures.iter()) + .chain( + full_key_aggregate_measures + .multiplied_measures + .iter() + .map(|m| m.measure()), + ) .map(|m| m.member_evaluator().clone()) .collect_vec(); let schema = Rc::new(LogicalSchema { @@ -191,7 +200,17 @@ impl MultipliedMeasuresQueryPlanner { key_cube_name: &String, ) -> Result { for measure in measures.iter() { - let cubes = collect_cube_names(measure.member_evaluator())?; + let member_expression_over_dimensions_cubes = + if let Ok(member_expression) = measure.member_evaluator().as_member_expression() { + member_expression.cube_names_if_dimension_only_expression()? + } else { + None + }; + let cubes = if let Some(cubes) = member_expression_over_dimensions_cubes { + cubes + } else { + collect_cube_names(measure.member_evaluator())? + }; let join_hints = collect_join_hints(measure.member_evaluator())?; if cubes.iter().any(|cube| cube != key_cube_name) { let measures_join = self.query_tools.join_graph().build_join(join_hints)?; @@ -298,8 +317,8 @@ impl MultipliedMeasuresQueryPlanner { let query = SimpleQuery { schema, filter: logical_filter, - offset: self.query_properties.offset(), - limit: self.query_properties.row_limit(), + offset: None, + limit: None, ungrouped: self.query_properties.ungrouped(), dimension_subqueries: subquery_dimension_queries, source: SimpleQuerySource::LogicalJoin(source), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index cdc6e055cec98..e2b3bc5ee33c6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -1,6 +1,9 @@ use super::filter::compiler::FilterCompiler; use super::filter::BaseSegment; use super::query_tools::QueryTools; +use crate::cube_bridge::join_hints::JoinHintItem; +use crate::cube_bridge::member_expression::MemberExpressionExpressionDef; +use crate::planner::sql_evaluator::MemberExpressionExpression; use super::sql_evaluator::MemberSymbol; use super::{BaseDimension, BaseMeasure, BaseMember, BaseMemberHelper, BaseTimeDimension}; @@ -43,9 +46,29 @@ impl OrderByItem { } } +#[derive(Debug, Clone)] +pub struct MultipliedMeasure { + measure: Rc, + cube_name: String, //May differ from cube_name of the measure for a member_expression that refers to a dimension. +} + +impl MultipliedMeasure { + pub fn new(measure: Rc, cube_name: String) -> Rc { + Rc::new(Self { measure, cube_name }) + } + + pub fn measure(&self) -> &Rc { + &self.measure + } + + pub fn cube_name(&self) -> &String { + &self.cube_name + } +} + #[derive(Default, Clone, Debug)] pub struct FullKeyAggregateMeasures { - pub multiplied_measures: Vec>, + pub multiplied_measures: Vec>, pub regular_measures: Vec>, pub multi_stage_measures: Vec>, pub rendered_as_multiplied_measures: HashSet, @@ -79,6 +102,7 @@ pub struct QueryProperties { multi_fact_join_groups: Vec<(Rc, Vec>)>, pre_aggregation_query: bool, total_query: bool, + query_join_hints: Rc>, } impl QueryProperties { @@ -111,8 +135,16 @@ impl QueryProperties { } else { "".to_string() }; - let expression_evaluator = evaluator_compiler - .compile_sql_call(&cube_name, member_expression.expression()?)?; + let expression_evaluator = match member_expression.expression()? { + MemberExpressionExpressionDef::Sql(sql) => { + evaluator_compiler.compile_sql_call(&cube_name, sql)? + } + MemberExpressionExpressionDef::Struct(_) => { + return Err(CubeError::user(format!( + "Expression struct not supported for dimension" + ))); + } + }; BaseDimension::try_new_from_expression( expression_evaluator, cube_name, @@ -165,13 +197,45 @@ impl QueryProperties { let name = if let Some(name) = &member_expression.static_data().expression_name { name.clone() + } else if let Some(name) = &member_expression.static_data().name { + format!("{}.{}", cube_name, name) } else { "".to_string() }; - let expression_evaluator = evaluator_compiler - .compile_sql_call(&cube_name, member_expression.expression()?)?; + let expression = match member_expression.expression()? { + MemberExpressionExpressionDef::Sql(sql) => { + MemberExpressionExpression::SqlCall( + evaluator_compiler.compile_sql_call(&cube_name, sql)?, + ) + } + MemberExpressionExpressionDef::Struct(expr) => { + if expr.static_data().expression_type != "PatchMeasure" { + return Err(CubeError::user(format!("Only `PatchMeasure` type of memeber expression is supported"))); + } + + if let Some(source_measure) = &expr.static_data().source_measure { + + let new_measure_type = expr.static_data().replace_aggregation_type.clone(); + let mut filters_to_add = vec![]; + if let Some(add_filters) = expr.add_filters()? { + for filter in add_filters.iter() { + let node = evaluator_compiler.compile_sql_call(&cube_name, filter.sql()?)?; + filters_to_add.push(node); + } + } + let source_measure_compiled = evaluator_compiler.add_measure_evaluator(source_measure.clone())?; + let patched_measure = source_measure_compiled.as_measure()?.new_patched(new_measure_type, filters_to_add)?; + let patched_symbol = MemberSymbol::new_measure(patched_measure); + MemberExpressionExpression::PatchedSymbol(patched_symbol) + + } else { + return Err(CubeError::user(format!("Source measure is required for `PatchMeasure` type of memeber expression"))); + } + + } + }; BaseMeasure::try_new_from_expression( - expression_evaluator, + expression, cube_name, name, member_expression.static_data().definition.clone(), @@ -230,8 +294,16 @@ impl QueryProperties { } else { "".to_string() }; - let expression_evaluator = evaluator_compiler - .compile_sql_call(&cube_name, member_expression.expression()?)?; + let expression_evaluator = match member_expression.expression()? { + MemberExpressionExpressionDef::Sql(sql) => { + evaluator_compiler.compile_sql_call(&cube_name, sql)? + } + MemberExpressionExpressionDef::Struct(_) => { + return Err(CubeError::user(format!( + "Expression struct not supported for dimension" + ))); + } + }; BaseSegment::try_new( expression_evaluator, cube_name, @@ -300,7 +372,10 @@ impl QueryProperties { }; let ungrouped = options.static_data().ungrouped.unwrap_or(false); + let query_join_hints = Rc::new(options.join_hints()?.unwrap_or_default()); + let multi_fact_join_groups = Self::compute_join_multi_fact_groups( + query_join_hints.clone(), query_tools.clone(), &measures, &dimensions, @@ -331,6 +406,7 @@ impl QueryProperties { multi_fact_join_groups, pre_aggregation_query, total_query, + query_join_hints, })) } @@ -350,6 +426,7 @@ impl QueryProperties { ungrouped: bool, pre_aggregation_query: bool, total_query: bool, + query_join_hints: Rc>, ) -> Result, CubeError> { let order_by = if order_by.is_empty() { Self::default_order(&dimensions, &time_dimensions, &measures) @@ -358,6 +435,7 @@ impl QueryProperties { }; let multi_fact_join_groups = Self::compute_join_multi_fact_groups( + query_join_hints.clone(), query_tools.clone(), &measures, &dimensions, @@ -385,6 +463,7 @@ impl QueryProperties { multi_fact_join_groups, pre_aggregation_query, total_query, + query_join_hints, })) } @@ -393,6 +472,7 @@ impl QueryProperties { measures: &Vec>, ) -> Result, Vec>)>, CubeError> { Self::compute_join_multi_fact_groups( + self.query_join_hints.clone(), self.query_tools.clone(), measures, &self.dimensions, @@ -409,6 +489,7 @@ impl QueryProperties { } pub fn compute_join_multi_fact_groups( + query_join_hints: Rc>, query_tools: Rc, measures: &Vec>, dimensions: &Vec>, @@ -437,7 +518,7 @@ impl QueryProperties { .cached_data_mut() .join_hints_for_filter_item_vec(&measures_filters)?; - let mut dimension_and_filter_join_hints_concat = Vec::new(); + let mut dimension_and_filter_join_hints_concat = vec![query_join_hints]; dimension_and_filter_join_hints_concat.extend(dimensions_join_hints.into_iter()); dimension_and_filter_join_hints_concat.extend(time_dimensions_join_hints.into_iter()); @@ -558,6 +639,10 @@ impl QueryProperties { self.row_limit } + pub fn query_join_hints(&self) -> &Rc> { + &self.query_join_hints + } + pub fn offset(&self) -> Option { self.offset } @@ -811,8 +896,10 @@ impl QueryProperties { .rendered_as_multiplied_measures .insert(item.measure.full_name()); } - if item.multiplied && !item.measure.can_used_as_addictive_in_multplied()? { - result.multiplied_measures.push(item.measure.clone()); + if item.multiplied && !item.measure.can_be_used_as_additive_in_multplied() { + result + .multiplied_measures + .push(MultipliedMeasure::new(item.measure.clone(), item.cube_name)); } else { result.regular_measures.push(item.measure.clone()); } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs index 2e1e7f1a2b128..826bbfee2ddaa 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs @@ -142,6 +142,7 @@ impl QueryTools { }; let evaluator_compiler = Rc::new(RefCell::new(Compiler::new( cube_evaluator.clone(), + base_tools.clone(), timezone.clone(), ))); Ok(Rc::new(Self { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs index 191d0b0c479a0..a808ebf327858 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs @@ -29,7 +29,7 @@ impl TraversalVisitor for CubeNamesCollector { ) -> Result, CubeError> { match node.as_ref() { MemberSymbol::Dimension(e) => { - if e.owned_by_cube() { + if !e.is_view() { if !path.is_empty() { for p in path { self.names.insert(p.clone()); @@ -46,7 +46,7 @@ impl TraversalVisitor for CubeNamesCollector { return self.on_node_traverse(e.base_symbol(), path, &()) } MemberSymbol::Measure(e) => { - if e.owned_by_cube() { + if !e.is_view() { if !path.is_empty() { for p in path { self.names.insert(p.clone()); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs index 086b610afd169..e98b27a23753b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs @@ -29,7 +29,7 @@ impl TraversalVisitor for JoinHintsCollector { ) -> Result, CubeError> { match node.as_ref() { MemberSymbol::Dimension(e) => { - if e.owned_by_cube() { + if !e.is_view() { if !path.is_empty() { if path.len() == 1 { self.hints.push(JoinHintItem::Single(path[0].clone())) @@ -48,7 +48,7 @@ impl TraversalVisitor for JoinHintsCollector { return self.on_node_traverse(e.base_symbol(), path, &()) } MemberSymbol::Measure(e) => { - if e.owned_by_cube() { + if !e.is_view() { if !path.is_empty() { if path.len() == 1 { self.hints.push(JoinHintItem::Single(path[0].clone())) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs index 4678927fdc31e..f824efaa7f709 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs @@ -10,6 +10,7 @@ struct CompositeMeasuresCollector { composite_measures: HashSet, } +#[derive(Clone)] struct CompositeMeasureCollectorState { pub parent_measure: Option>, } @@ -52,15 +53,18 @@ impl TraversalVisitor for CompositeMeasuresCollector { Some(new_state) } MemberSymbol::Dimension(_) => None, + MemberSymbol::MemberExpression(_) => Some(state.clone()), _ => None, }; Ok(res) } } +#[derive(Debug)] pub struct MeasureResult { pub multiplied: bool, pub measure: Rc, + pub cube_name: String, } pub struct MultipliedMeasuresCollector { @@ -113,6 +117,7 @@ impl TraversalVisitor for MultipliedMeasuresCollector { multiplied, measure: BaseMeasure::try_new(node.clone(), self.query_tools.clone())? .unwrap(), + cube_name: node.cube_name(), }) } @@ -135,10 +140,47 @@ pub fn collect_multiplied_measures( node: &Rc, join: Rc, ) -> Result, CubeError> { + if let Ok(member_expression) = node.as_member_expression() { + if let Some(cube_names) = member_expression.cube_names_if_dimension_only_expression()? { + let result = if cube_names.is_empty() { + let measure = BaseMeasure::try_new(node.clone(), query_tools.clone())?.unwrap(); + vec![MeasureResult { + cube_name: measure.cube_name().clone(), + measure, + multiplied: false, + }] + } else if cube_names.len() == 1 { + let cube_name = cube_names[0].clone(); + let multiplied = join + .static_data() + .multiplication_factor + .get(&cube_name) + .unwrap_or(&false) + .clone(); + let measure = BaseMeasure::try_new(node.clone(), query_tools.clone())?.unwrap(); + + vec![MeasureResult { + measure, + cube_name, + multiplied, + }] + } else { + return Err(CubeError::user(format!( + "Expected single cube for dimension-only measure {}, got {:?}", + node.full_name(), + cube_names + ))); + }; + return Ok(result); + } + } + let mut composite_collector = CompositeMeasuresCollector::new(); composite_collector.apply(node, &CompositeMeasureCollectorState::new(None))?; let composite_measures = composite_collector.extract_result(); - let mut visitor = MultipliedMeasuresCollector::new(query_tools, composite_measures, join); + let mut visitor = + MultipliedMeasuresCollector::new(query_tools.clone(), composite_measures, join.clone()); visitor.apply(node, &())?; - Ok(visitor.extract_result()) + let result = visitor.extract_result(); + Ok(result) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs index f68389acde48a..6d1bbd594624e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs @@ -5,6 +5,7 @@ use super::{ CubeNameSymbolFactory, CubeTableSymbolFactory, DimensionSymbolFactory, MeasureSymbolFactory, SqlCall, SymbolFactory, TraversalVisitor, }; +use crate::cube_bridge::base_tools::BaseTools; use crate::cube_bridge::evaluator::CubeEvaluator; use crate::cube_bridge::join_hints::JoinHintItem; use crate::cube_bridge::member_sql::MemberSql; @@ -14,15 +15,21 @@ use std::collections::HashMap; use std::rc::Rc; pub struct Compiler { cube_evaluator: Rc, + base_tools: Rc, timezone: Tz, /* (type, name) */ members: HashMap<(String, String), Rc>, } impl Compiler { - pub fn new(cube_evaluator: Rc, timezone: Tz) -> Self { + pub fn new( + cube_evaluator: Rc, + base_tools: Rc, + timezone: Tz, + ) -> Self { Self { cube_evaluator, + base_tools, timezone, members: HashMap::new(), } @@ -45,6 +52,10 @@ impl Compiler { } } + pub fn base_tools(&self) -> Rc { + self.base_tools.clone() + } + pub fn add_measure_evaluator( &mut self, measure: String, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs index 95ec50002acba..9a00ffe4767c9 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs @@ -88,7 +88,7 @@ impl<'a> DependenciesBuilder<'a> { ) -> Result, CubeError> { let call_deps = if member_sql.need_deps_resolve() { self.cube_evaluator - .resolve_symbols_call_deps(cube_name.clone(), member_sql)? + .resolve_symbols_call_deps(cube_name.clone(), member_sql.clone())? } else { vec![] }; @@ -104,13 +104,13 @@ impl<'a> DependenciesBuilder<'a> { result.push(context_dep); continue; } - if childs[i].is_empty() { + if self.check_cube_exists(&dep.name)? { + let dep = self.build_cube_dependency(&cube_name, i, &call_deps, &childs)?; + result.push(Dependency::CubeDependency(dep)); + } else if childs[i].is_empty() { result.push(Dependency::SymbolDependency( self.build_evaluator(&cube_name, &dep.name)?, )); - } else if self.check_cube_exists(&dep.name)? { - let dep = self.build_cube_dependency(&cube_name, i, &call_deps, &childs)?; - result.push(Dependency::CubeDependency(dep)); } else { //Assuming this is a time dimension with an explicit granularity let dep = @@ -177,13 +177,12 @@ impl<'a> DependenciesBuilder<'a> { &dep.name, Some(granularity.clone()), )? { - let member_evaluator = - Rc::new(MemberSymbol::TimeDimension(TimeDimensionSymbol::new( - base_evaluator.clone(), - Some(granularity.clone()), - Some(granularity_obj), - None, - ))); + let member_evaluator = MemberSymbol::new_time_dimension(TimeDimensionSymbol::new( + base_evaluator.clone(), + Some(granularity.clone()), + Some(granularity_obj), + None, + )); granularities.insert(granularity.clone(), member_evaluator); } else { return Err(CubeError::user(format!( @@ -288,7 +287,9 @@ impl<'a> DependenciesBuilder<'a> { name: &String, ) -> Result, CubeError> { let dep_full_name = format!("{}.{}", cube_name, name); - self.compiler - .add_auto_resolved_member_evaluator(dep_full_name) + let res = self + .compiler + .add_auto_resolved_member_evaluator(dep_full_name.clone())?; + Ok(res) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs index 0a9e167c0746c..570e6ae2db716 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs @@ -16,7 +16,8 @@ pub use sql_visitor::SqlEvaluatorVisitor; pub use symbols::{ CubeNameSymbol, CubeNameSymbolFactory, CubeTableSymbol, CubeTableSymbolFactory, DimensionCaseDefinition, DimensionCaseWhenItem, DimensionSymbol, DimensionSymbolFactory, - DimenstionCaseLabel, MeasureSymbol, MeasureSymbolFactory, MeasureTimeShift, - MemberExpressionSymbol, MemberSymbol, SymbolFactory, TimeDimensionSymbol, + DimensionTimeShift, DimenstionCaseLabel, MeasureSymbol, MeasureSymbolFactory, + MeasureTimeShifts, MemberExpressionExpression, MemberExpressionSymbol, MemberSymbol, + SymbolFactory, TimeDimensionSymbol, }; pub use visitor::TraversalVisitor; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs index 803faad127bf1..a64f5345416ba 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs @@ -4,6 +4,7 @@ use super::dependecy::{ use super::sql_nodes::SqlNode; use super::{symbols::MemberSymbol, SqlEvaluatorVisitor}; use crate::cube_bridge::base_query_options::FilterItem as NativeFilterItem; +use crate::cube_bridge::base_tools::BaseTools; use crate::cube_bridge::member_sql::{ContextSymbolArg, MemberSql, MemberSqlArg, MemberSqlStruct}; use crate::plan::{Filter, FilterItem}; use crate::planner::query_tools::QueryTools; @@ -45,7 +46,7 @@ impl SqlCall { self.member_sql.call(args) } - pub fn is_direct_reference(&self) -> Result { + pub fn is_direct_reference(&self, base_tools: Rc) -> Result { let dependencies = self.get_dependencies(); if dependencies.len() != 1 { return Ok(false); @@ -56,7 +57,7 @@ impl SqlCall { let args = self .deps .iter() - .map(|d| self.evaluate_single_dep_for_ref_check(&d)) + .map(|d| self.evaluate_single_dep_for_ref_check(&d, base_tools.clone())) .collect::, _>>()?; let eval_result = self.member_sql.call(args)?; @@ -190,6 +191,7 @@ impl SqlCall { fn evaluate_single_dep_for_ref_check( &self, dep: &Dependency, + base_tools: Rc, ) -> Result { match dep { Dependency::SymbolDependency(dep) => Ok(MemberSqlArg::String(dep.full_name())), @@ -197,7 +199,26 @@ impl SqlCall { self.evaluate_time_dimesion_dep_for_ref_check(dep) } Dependency::CubeDependency(dep) => self.evaluate_cube_dep_for_ref_check(dep), - Dependency::ContextDependency(_) => Ok(MemberSqlArg::String(format!("Context Symbol"))), + Dependency::ContextDependency(dep) => match dep { + ContextSymbolDep::SecurityContext => Ok(MemberSqlArg::ContextSymbol( + ContextSymbolArg::SecurityContext(base_tools.security_context_for_rust()?), + )), + ContextSymbolDep::FilterParams => { + let r = base_tools.filters_proxy_for_rust(None)?; + Ok(MemberSqlArg::ContextSymbol(ContextSymbolArg::FilterParams( + r, + ))) + } + ContextSymbolDep::FilterGroup => { + let r = base_tools.filter_group_function_for_rust(None)?; + Ok(MemberSqlArg::ContextSymbol(ContextSymbolArg::FilterGroup( + r, + ))) + } + ContextSymbolDep::SqlUtils => Ok(MemberSqlArg::ContextSymbol( + ContextSymbolArg::SqlUtils(base_tools.sql_utils_for_rust()?), + )), + }, } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index 6f49f3b471fb5..e14749ce2007d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -6,13 +6,13 @@ use super::{ TimeShiftSqlNode, UngroupedMeasureSqlNode, UngroupedQueryFinalMeasureSqlNode, }; use crate::plan::schema::QualifiedColumnName; -use crate::planner::sql_evaluator::MeasureTimeShift; +use crate::planner::planners::multi_stage::TimeShiftState; use std::collections::{HashMap, HashSet}; use std::rc::Rc; #[derive(Clone)] pub struct SqlNodesFactory { - time_shifts: HashMap, + time_shifts: TimeShiftState, ungrouped: bool, ungrouped_measure: bool, count_approx_as_state: bool, @@ -33,7 +33,7 @@ pub struct SqlNodesFactory { impl SqlNodesFactory { pub fn new() -> Self { Self { - time_shifts: HashMap::new(), + time_shifts: TimeShiftState::default(), ungrouped: false, ungrouped_measure: false, count_approx_as_state: false, @@ -52,7 +52,7 @@ impl SqlNodesFactory { } } - pub fn set_time_shifts(&mut self, time_shifts: HashMap) { + pub fn set_time_shifts(&mut self, time_shifts: TimeShiftState) { self.time_shifts = time_shifts; } @@ -262,7 +262,7 @@ impl SqlNodesFactory { let input: Rc = AutoPrefixSqlNode::new(input, self.cube_name_references.clone()); - let input = if !&self.time_shifts.is_empty() { + let input = if !self.time_shifts.is_empty() { TimeShiftSqlNode::new(self.time_shifts.clone(), input) } else { input diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs index 4cc97efb89791..d658fad328906 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs @@ -1,27 +1,24 @@ use super::SqlNode; +use crate::planner::planners::multi_stage::TimeShiftState; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{MeasureTimeShift, MemberSymbol}; use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::SqlInterval; use cubenativeutils::CubeError; use std::any::Any; -use std::collections::HashMap; use std::rc::Rc; pub struct TimeShiftSqlNode { - shifts: HashMap, + shifts: TimeShiftState, input: Rc, } impl TimeShiftSqlNode { - pub fn new(shifts: HashMap, input: Rc) -> Rc { + pub fn new(shifts: TimeShiftState, input: Rc) -> Rc { Rc::new(Self { shifts, input }) } - pub fn shifts(&self) -> &HashMap { - &self.shifts - } - pub fn input(&self) -> &Rc { &self.input } @@ -45,10 +42,18 @@ impl SqlNode for TimeShiftSqlNode { )?; let res = match node.as_ref() { MemberSymbol::Dimension(ev) => { - if let Some(shift) = self.shifts.get(&ev.full_name()) { - let shift = shift.interval.to_sql(); - let res = templates.add_timestamp_interval(input, shift)?; - format!("({})", res) + if !ev.is_reference() && ev.dimension_type() == "time" { + let mut interval = self.shifts.common_time_shift.clone().unwrap_or_default(); + if let Some(shift) = self.shifts.dimensions_shifts.get(&ev.full_name()) { + interval += &shift.interval; + } + if interval == SqlInterval::default() { + input + } else { + let shift = interval.to_sql(); + let res = templates.add_timestamp_interval(input, shift)?; + format!("({})", res) + } } else { input } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs index bfa3be5cfc2d5..9bcb1979f1c46 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs @@ -15,8 +15,8 @@ pub struct CubeNameSymbol { } impl CubeNameSymbol { - pub fn new(cube_name: String) -> Self { - Self { cube_name } + pub fn new(cube_name: String) -> Rc { + Rc::new(Self { cube_name }) } pub fn evaluate_sql(&self) -> Result { @@ -80,13 +80,13 @@ impl CubeTableSymbol { member_sql: Option>, definition: Rc, is_table_sql: bool, - ) -> Self { - Self { + ) -> Rc { + Rc::new(Self { cube_name, member_sql, definition, is_table_sql, - } + }) } pub fn evaluate_sql( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs index f3af88c124914..b2f12cdba339e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs @@ -33,6 +33,7 @@ pub struct DimensionSymbol { case: Option, definition: Rc, is_reference: bool, // Symbol is a direct reference to another symbol without any calculations + is_view: bool, } impl DimensionSymbol { @@ -41,12 +42,13 @@ impl DimensionSymbol { name: String, member_sql: Option>, is_reference: bool, + is_view: bool, latitude: Option>, longitude: Option>, case: Option, definition: Rc, - ) -> Self { - Self { + ) -> Rc { + Rc::new(Self { cube_name, name, member_sql, @@ -55,7 +57,8 @@ impl DimensionSymbol { longitude, definition, case, - } + is_view, + }) } pub fn evaluate_sql( @@ -116,6 +119,10 @@ impl DimensionSymbol { self.is_reference } + pub fn is_view(&self) -> bool { + self.is_view + } + pub fn reference_member(&self) -> Option> { if !self.is_reference() { return None; @@ -263,7 +270,7 @@ impl SymbolFactory for DimensionSymbolFactory { }; let is_sql_direct_ref = if let Some(sql) = &sql { - sql.is_direct_reference()? + sql.is_direct_reference(compiler.base_tools())? } else { false }; @@ -331,6 +338,7 @@ impl SymbolFactory for DimensionSymbolFactory { name, sql, is_reference, + is_view, latitude, longitude, case, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs index e61d7baaf7259..00e32afb884a7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs @@ -1,8 +1,6 @@ use super::{MemberSymbol, SymbolFactory}; use crate::cube_bridge::evaluator::CubeEvaluator; -use crate::cube_bridge::measure_definition::{ - MeasureDefinition, RollingWindow, TimeShiftReference, -}; +use crate::cube_bridge::measure_definition::{MeasureDefinition, RollingWindow}; use crate::cube_bridge::member_sql::MemberSql; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::collectors::find_owned_by_cube_child; @@ -39,28 +37,38 @@ impl MeasureOrderBy { } #[derive(Clone, Debug)] -pub struct MeasureTimeShift { +pub struct DimensionTimeShift { pub interval: SqlInterval, pub dimension: Rc, } -impl PartialEq for MeasureTimeShift { +impl PartialEq for DimensionTimeShift { fn eq(&self, other: &Self) -> bool { self.interval == other.interval && self.dimension.full_name() == other.dimension.full_name() } } -impl Eq for MeasureTimeShift {} +impl Eq for DimensionTimeShift {} + +#[derive(Clone, Debug)] +pub enum MeasureTimeShifts { + Dimensions(Vec), + Common(SqlInterval), +} #[derive(Clone)] pub struct MeasureSymbol { cube_name: String, name: String, - definition: Rc, + owned_by_cube: bool, + measure_type: String, + rolling_window: Option, + is_multi_stage: bool, is_reference: bool, + is_view: bool, measure_filters: Vec>, measure_drill_filters: Vec>, - time_shifts: Vec, + time_shift: Option, measure_order_by: Vec, reduce_by: Option>>, add_group_by: Option>>, @@ -76,32 +84,120 @@ impl MeasureSymbol { name: String, member_sql: Option>, is_reference: bool, + is_view: bool, pk_sqls: Vec>, definition: Rc, measure_filters: Vec>, measure_drill_filters: Vec>, - time_shifts: Vec, + time_shift: Option, measure_order_by: Vec, reduce_by: Option>>, add_group_by: Option>>, group_by: Option>>, - ) -> Self { - Self { + ) -> Rc { + let owned_by_cube = definition.static_data().owned_by_cube.unwrap_or(true); + let measure_type = definition.static_data().measure_type.clone(); + let rolling_window = definition.static_data().rolling_window.clone(); + let is_multi_stage = definition.static_data().multi_stage.unwrap_or(false); + Rc::new(Self { cube_name, name, member_sql, is_reference, + is_view, pk_sqls, - definition, + owned_by_cube, + measure_type, + rolling_window, measure_filters, measure_drill_filters, measure_order_by, - time_shifts, + is_multi_stage, + time_shift, is_splitted_source: false, reduce_by, add_group_by, group_by, + }) + } + + pub fn new_patched( + &self, + new_measure_type: Option, + add_filters: Vec>, + ) -> Result, CubeError> { + let result_measure_type = if let Some(new_measure_type) = new_measure_type { + match self.measure_type.as_str() { + "sum" | "avg" | "min" | "max" => match new_measure_type.as_str() { + "sum" | "avg" | "min" | "max" | "count_distinct" | "count_distinct_approx" => {} + _ => { + return Err(CubeError::user(format!( + "Unsupported measure type replacement for {}: {} => {}", + self.name, self.measure_type, new_measure_type + ))) + } + }, + "count_distinct" | "count_distinct_approx" => match new_measure_type.as_str() { + "count_distinct" | "count_distinct_approx" => {} + _ => { + return Err(CubeError::user(format!( + "Unsupported measure type replacement for {}: {} => {}", + self.name, self.measure_type, new_measure_type + ))) + } + }, + + _ => { + return Err(CubeError::user(format!( + "Unsupported measure type replacement for {}: {} => {}", + self.name, self.measure_type, new_measure_type + ))) + } + } + new_measure_type + } else { + self.measure_type.clone() + }; + + let mut measure_filters = self.measure_filters.clone(); + if !add_filters.is_empty() { + match result_measure_type.as_str() { + "sum" + | "avg" + | "min" + | "max" + | "count" + | "count_distinct" + | "count_distinct_approx" => {} + _ => { + return Err(CubeError::user(format!( + "Unsupported additional filters for measure {} type {}", + self.name, result_measure_type + ))) + } + } + measure_filters.extend(add_filters.into_iter()); } + Ok(Rc::new(Self { + cube_name: self.cube_name.clone(), + name: self.name.clone(), + owned_by_cube: self.owned_by_cube, + measure_type: result_measure_type, + rolling_window: self.rolling_window.clone(), + is_multi_stage: self.is_multi_stage, + is_reference: self.is_reference, + is_view: self.is_view, + measure_filters, + measure_drill_filters: self.measure_drill_filters.clone(), + time_shift: self.time_shift.clone(), + measure_order_by: self.measure_order_by.clone(), + reduce_by: self.reduce_by.clone(), + add_group_by: self.add_group_by.clone(), + group_by: self.group_by.clone(), + member_sql: self.member_sql.clone(), + pk_sqls: self.pk_sqls.clone(), + is_splitted_source: self.is_splitted_source, + })) } pub fn full_name(&self) -> String { @@ -116,12 +212,12 @@ impl MeasureSymbol { &self.pk_sqls } - pub fn time_shifts(&self) -> &Vec { - &self.time_shifts + pub fn time_shift(&self) -> &Option { + &self.time_shift } pub fn is_calculated(&self) -> bool { - Self::is_calculated_type(&self.definition.static_data().measure_type) + Self::is_calculated_type(&self.measure_type) } pub fn is_calculated_type(measure_type: &str) -> bool { @@ -224,17 +320,28 @@ impl MeasureSymbol { cubes } + pub fn can_used_as_addictive_in_multplied(&self) -> bool { + if &self.measure_type == "countDistinct" || &self.measure_type == "countDistinctApprox" { + true + } else if &self.measure_type == "count" && self.member_sql.is_none() { + true + } else { + false + } + } + pub fn owned_by_cube(&self) -> bool { - self.definition() - .static_data() - .owned_by_cube - .unwrap_or(true) + self.owned_by_cube } pub fn is_reference(&self) -> bool { self.is_reference } + pub fn is_view(&self) -> bool { + self.is_view + } + pub fn reference_member(&self) -> Option> { if !self.is_reference() { return None; @@ -247,11 +354,11 @@ impl MeasureSymbol { } pub fn measure_type(&self) -> &String { - &self.definition.static_data().measure_type + &self.measure_type } pub fn rolling_window(&self) -> &Option { - &self.definition.static_data().rolling_window + &self.rolling_window } pub fn is_rolling_window(&self) -> bool { @@ -278,10 +385,6 @@ impl MeasureSymbol { &self.measure_order_by } - pub fn definition(&self) -> Rc { - self.definition.clone() - } - pub fn reduce_by(&self) -> &Option>> { &self.reduce_by } @@ -294,12 +397,8 @@ impl MeasureSymbol { &self.group_by } - pub fn time_shift_references(&self) -> &Option> { - &self.definition.static_data().time_shift_references - } - pub fn is_multi_stage(&self) -> bool { - self.definition.static_data().multi_stage.unwrap_or(false) + self.is_multi_stage } pub fn cube_name(&self) -> &String { @@ -425,24 +524,26 @@ impl SymbolFactory for MeasureSymbolFactory { }; let is_sql_is_direct_ref = if let Some(sql) = &sql { - sql.is_direct_reference()? + sql.is_direct_reference(compiler.base_tools())? } else { false }; - let time_shifts = - if let Some(time_shift_references) = &definition.static_data().time_shift_references { - let mut shifts: HashMap = HashMap::new(); - for shift_ref in time_shift_references.iter() { - let interval = shift_ref.interval.parse::()?; - let interval = - if shift_ref.shift_type.as_ref().unwrap_or(&format!("prior")) == "next" { - -interval - } else { - interval - }; - let dimension = - compiler.add_dimension_evaluator(shift_ref.time_dimension.clone())?; + let time_shifts = if let Some(time_shift_references) = + &definition.static_data().time_shift_references + { + let mut shifts: HashMap = HashMap::new(); + let mut common_shift = None; + for shift_ref in time_shift_references.iter() { + let interval = shift_ref.interval.parse::()?; + let interval = + if shift_ref.shift_type.as_ref().unwrap_or(&format!("prior")) == "next" { + -interval + } else { + interval + }; + if let Some(time_dimension) = &shift_ref.time_dimension { + let dimension = compiler.add_dimension_evaluator(time_dimension.clone())?; let dimension = find_owned_by_cube_child(&dimension)?; let dimension_name = dimension.full_name(); if let Some(exists) = shifts.get(&dimension_name) { @@ -455,17 +556,38 @@ impl SymbolFactory for MeasureSymbolFactory { } else { shifts.insert( dimension_name, - MeasureTimeShift { + DimensionTimeShift { interval: interval.clone(), dimension: dimension.clone(), }, ); }; + } else { + if common_shift.is_none() { + common_shift = Some(interval); + } else { + if common_shift != Some(interval) { + return Err(CubeError::user(format!( + "Measure can contain only one common time_shift (without time_dimension).", + ))); + } + } } - shifts.into_values().collect_vec() + } + if common_shift.is_some() && !shifts.is_empty() { + return Err(CubeError::user(format!( + "Measure cannot mix common time_shifts (without time_dimension) with dimension-specific ones.", + ))); + } else if common_shift.is_some() { + Some(MeasureTimeShifts::Common(common_shift.unwrap())) } else { - vec![] - }; + Some(MeasureTimeShifts::Dimensions( + shifts.into_values().collect_vec(), + )) + } + } else { + None + }; let reduce_by = if let Some(reduce_by) = &definition.static_data().reduce_by_references { let symbols = reduce_by @@ -514,7 +636,7 @@ impl SymbolFactory for MeasureSymbolFactory { && !is_multi_stage && measure_filters.is_empty() && measure_drill_filters.is_empty() - && time_shifts.is_empty() + && time_shifts.is_none() && measure_order_by.is_empty() && reduce_by.is_none() && add_group_by.is_none() @@ -525,6 +647,7 @@ impl SymbolFactory for MeasureSymbolFactory { name, sql, is_reference, + is_view, pk_sqls, definition, measure_filters, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs index 90029b39f4d75..76e10f82543c7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs @@ -1,14 +1,22 @@ use super::MemberSymbol; +use crate::cube_bridge::base_tools::BaseTools; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::collectors::member_childs; use crate::planner::sql_evaluator::{sql_nodes::SqlNode, SqlCall, SqlEvaluatorVisitor}; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; +use itertools::Itertools; use std::rc::Rc; +pub enum MemberExpressionExpression { + SqlCall(Rc), + PatchedSymbol(Rc), +} + pub struct MemberExpressionSymbol { cube_name: String, name: String, - expression: Rc, + expression: MemberExpressionExpression, #[allow(dead_code)] definition: Option, is_reference: bool, @@ -18,17 +26,23 @@ impl MemberExpressionSymbol { pub fn try_new( cube_name: String, name: String, - expression: Rc, + expression: MemberExpressionExpression, definition: Option, - ) -> Result { - let is_reference = expression.is_direct_reference()?; - Ok(Self { + base_tools: Rc, + ) -> Result, CubeError> { + let is_reference = match &expression { + MemberExpressionExpression::SqlCall(sql_call) => { + sql_call.is_direct_reference(base_tools.clone())? + } + MemberExpressionExpression::PatchedSymbol(_symbol) => false, + }; + Ok(Rc::new(Self { cube_name, name, expression, definition, is_reference, - }) + })) } pub fn evaluate_sql( @@ -38,16 +52,17 @@ impl MemberExpressionSymbol { query_tools: Rc, templates: &PlanSqlTemplates, ) -> Result { - let sql = self - .expression - .eval(visitor, node_processor, query_tools, templates)?; + let sql = match &self.expression { + MemberExpressionExpression::SqlCall(sql_call) => { + sql_call.eval(visitor, node_processor, query_tools, templates)? + } + MemberExpressionExpression::PatchedSymbol(symbol) => { + visitor.apply(symbol, node_processor, templates)? + } + }; Ok(sql) } - pub fn expression(&self) -> &Rc { - &self.expression - } - pub fn full_name(&self) -> String { format!("expr:{}.{}", self.cube_name, self.name) } @@ -69,16 +84,45 @@ impl MemberExpressionSymbol { pub fn get_dependencies(&self) -> Vec> { let mut deps = vec![]; - self.expression.extract_symbol_deps(&mut deps); + match &self.expression { + MemberExpressionExpression::SqlCall(sql_call) => { + sql_call.extract_symbol_deps(&mut deps) + } + MemberExpressionExpression::PatchedSymbol(member_symbol) => { + deps.push(member_symbol.clone()) + } + } deps } pub fn get_dependencies_with_path(&self) -> Vec<(Rc, Vec)> { let mut deps = vec![]; - self.expression.extract_symbol_deps_with_path(&mut deps); + match &self.expression { + MemberExpressionExpression::SqlCall(sql_call) => { + sql_call.extract_symbol_deps_with_path(&mut deps) + } + MemberExpressionExpression::PatchedSymbol(member_symbol) => { + deps.push((member_symbol.clone(), vec![])) + } + } deps } + pub fn cube_names_if_dimension_only_expression( + self: Rc, + ) -> Result>, CubeError> { + let childs = member_childs(&MemberSymbol::new_member_expression(self), true)?; + if childs.iter().any(|s| !s.is_dimension()) { + Ok(None) + } else { + let cube_names = childs + .into_iter() + .map(|child| child.cube_name()) + .collect_vec(); + Ok(Some(cube_names)) + } + } + pub fn cube_name(&self) -> &String { &self.cube_name } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs index 3f94fd68e65f9..5d01eca2d1fa0 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs @@ -10,12 +10,12 @@ use std::fmt::Debug; use std::rc::Rc; pub enum MemberSymbol { - Dimension(DimensionSymbol), - TimeDimension(TimeDimensionSymbol), - Measure(MeasureSymbol), - CubeName(CubeNameSymbol), - CubeTable(CubeTableSymbol), - MemberExpression(MemberExpressionSymbol), + Dimension(Rc), + TimeDimension(Rc), + Measure(Rc), + CubeName(Rc), + CubeTable(Rc), + MemberExpression(Rc), } impl Debug for MemberSymbol { @@ -38,22 +38,30 @@ impl Debug for MemberSymbol { } impl MemberSymbol { - pub fn new_measure(symbol: MeasureSymbol) -> Rc { + pub fn new_measure(symbol: Rc) -> Rc { Rc::new(Self::Measure(symbol)) } - pub fn new_dimension(symbol: DimensionSymbol) -> Rc { + pub fn new_dimension(symbol: Rc) -> Rc { Rc::new(Self::Dimension(symbol)) } - pub fn new_cube_name(symbol: CubeNameSymbol) -> Rc { + pub fn new_cube_name(symbol: Rc) -> Rc { Rc::new(Self::CubeName(symbol)) } - pub fn new_cube_table(symbol: CubeTableSymbol) -> Rc { + pub fn new_cube_table(symbol: Rc) -> Rc { Rc::new(Self::CubeTable(symbol)) } + pub fn new_member_expression(symbol: Rc) -> Rc { + Rc::new(Self::MemberExpression(symbol)) + } + + pub fn new_time_dimension(symbol: Rc) -> Rc { + Rc::new(Self::TimeDimension(symbol)) + } + pub fn full_name(&self) -> String { match self { Self::Dimension(d) => d.full_name(), @@ -166,9 +174,9 @@ impl MemberSymbol { } } - pub fn as_time_dimension(&self) -> Result<&TimeDimensionSymbol, CubeError> { + pub fn as_time_dimension(&self) -> Result, CubeError> { match self { - Self::TimeDimension(d) => Ok(d), + Self::TimeDimension(d) => Ok(d.clone()), _ => Err(CubeError::internal(format!( "{} is not a time dimension", self.full_name() @@ -176,9 +184,9 @@ impl MemberSymbol { } } - pub fn as_dimension(&self) -> Result<&DimensionSymbol, CubeError> { + pub fn as_dimension(&self) -> Result, CubeError> { match self { - Self::Dimension(d) => Ok(d), + Self::Dimension(d) => Ok(d.clone()), _ => Err(CubeError::internal(format!( "{} is not a dimension", self.full_name() @@ -186,9 +194,9 @@ impl MemberSymbol { } } - pub fn as_measure(&self) -> Result<&MeasureSymbol, CubeError> { + pub fn as_measure(&self) -> Result, CubeError> { match self { - Self::Measure(m) => Ok(m), + Self::Measure(m) => Ok(m.clone()), _ => Err(CubeError::internal(format!( "{} is not a measure", self.full_name() @@ -196,6 +204,16 @@ impl MemberSymbol { } } + pub fn as_member_expression(&self) -> Result, CubeError> { + match self { + Self::MemberExpression(m) => Ok(m.clone()), + _ => Err(CubeError::internal(format!( + "{} is not a member expression", + self.full_name() + ))), + } + } + pub fn alias_suffix(&self) -> Option { match self { Self::TimeDimension(d) => Some(d.alias_suffix()), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs index c29d900d2ed6c..ebd15bbac33f4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs @@ -13,8 +13,10 @@ pub use dimension_symbol::{ DimensionCaseDefinition, DimensionCaseWhenItem, DimensionSymbol, DimensionSymbolFactory, DimenstionCaseLabel, }; -pub use measure_symbol::{MeasureSymbol, MeasureSymbolFactory, MeasureTimeShift}; -pub use member_expression_symbol::MemberExpressionSymbol; +pub use measure_symbol::{ + DimensionTimeShift, MeasureSymbol, MeasureSymbolFactory, MeasureTimeShifts, +}; +pub use member_expression_symbol::{MemberExpressionExpression, MemberExpressionSymbol}; pub use member_symbol::MemberSymbol; pub use symbol_factory::SymbolFactory; pub use time_dimension_symbol::TimeDimensionSymbol; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs index 69eba92875e85..b658cb67dda33 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs @@ -23,21 +23,21 @@ impl TimeDimensionSymbol { granularity: Option, granularity_obj: Option, date_range: Option<(String, String)>, - ) -> Self { + ) -> Rc { let name_suffix = if let Some(granularity) = &granularity { granularity.clone() } else { "day".to_string() }; let full_name = format!("{}_{}", base_symbol.full_name(), name_suffix); - Self { + Rc::new(Self { base_symbol, granularity, granularity_obj, full_name, date_range, alias_suffix: name_suffix, - } + }) } pub fn base_symbol(&self) -> &Rc { @@ -84,7 +84,7 @@ impl TimeDimensionSymbol { self.granularity_obj.clone(), self.date_range.clone(), ); - Rc::new(MemberSymbol::TimeDimension(result)) + MemberSymbol::new_time_dimension(result) } else { s.clone() } @@ -118,7 +118,7 @@ impl TimeDimensionSymbol { self.granularity_obj.clone(), self.date_range.clone(), ); - Some(Rc::new(MemberSymbol::TimeDimension(new_time_dim))) + Some(MemberSymbol::new_time_dimension(new_time_dim)) } else { None } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity.rs index b3d120632acef..cdaee8d8dfcb9 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity.rs @@ -127,7 +127,7 @@ impl Granularity { ) } - pub fn resolve_granularity(&self) -> Result { + pub fn resolved_granularity(&self) -> Result { if self.is_predefined_granularity { Ok(self.granularity.clone()) } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs index 89876ed317b54..bc04336bd358d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/granularity_helper.rs @@ -58,8 +58,10 @@ impl GranularityHelper { let first = Ok(dimensions[0].clone()); dimensions.iter().skip(1).fold(first, |acc, d| match acc { Ok(min_dim) => { - let min_granularity = - Self::min_granularity(&min_dim.get_granularity(), &d.get_granularity())?; + let min_granularity = Self::min_granularity( + &min_dim.resolved_granularity()?, + &d.resolved_granularity()?, + )?; if min_granularity == min_dim.get_granularity() { Ok(min_dim) } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/sql_interval.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/sql_interval.rs index f544dc27a9bda..852277f6d0dcb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/sql_interval.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/sql_interval.rs @@ -3,7 +3,7 @@ use itertools::Itertools; use std::ops::{Add, AddAssign, Neg, Sub}; use std::str::FromStr; -#[derive(Debug, PartialEq, Clone, Hash, Eq, Default)] +#[derive(Default, Debug, PartialEq, Clone, Hash, Eq)] pub struct SqlInterval { pub year: i32, pub month: i32, @@ -112,8 +112,8 @@ impl Add for SqlInterval { } } -impl AddAssign for SqlInterval { - fn add_assign(&mut self, other: SqlInterval) { +impl AddAssign<&SqlInterval> for SqlInterval { + fn add_assign(&mut self, other: &SqlInterval) { self.year += other.year; self.month += other.month; self.week += other.week; @@ -124,6 +124,12 @@ impl AddAssign for SqlInterval { } } +impl AddAssign for SqlInterval { + fn add_assign(&mut self, other: SqlInterval) { + *self += &other; + } +} + impl Sub for SqlInterval { type Output = SqlInterval; fn sub(self, other: SqlInterval) -> SqlInterval { diff --git a/rust/cubesqlplanner/nativebridge/src/lib.rs b/rust/cubesqlplanner/nativebridge/src/lib.rs index cdba883071094..3d3e65fccbac1 100644 --- a/rust/cubesqlplanner/nativebridge/src/lib.rs +++ b/rust/cubesqlplanner/nativebridge/src/lib.rs @@ -14,6 +14,18 @@ use syn::{ pub fn native_bridge(args: TokenStream, input: TokenStream) -> proc_macro::TokenStream { let mut svc = parse_macro_input!(input as NativeService); let args = parse_macro_input!(args with Punctuated::::parse_terminated); + for arg in args.iter() { + match arg { + Meta::Path(p) => { + if p.is_ident("without_imports") { + svc.without_imports = true; + } else { + svc.static_data_type = Some(p.clone()) + } + } + _ => {} + } + } if args.len() > 0 { let arg = args.first().unwrap(); match arg { @@ -29,6 +41,7 @@ struct NativeService { ident: Ident, methods: Vec, pub static_data_type: Option, + pub without_imports: bool, } enum NativeMethodType { @@ -119,6 +132,7 @@ impl Parse for NativeService { ident: trait_item.ident.clone(), methods, static_data_type: None, + without_imports: false, } } x => { @@ -373,10 +387,14 @@ impl NativeService { } fn imports(&self) -> proc_macro2::TokenStream { - quote! { - use cubenativeutils::wrappers::inner_types::InnerTypes; - use cubenativeutils::wrappers::object::NativeStruct; + let mut imports = quote! {}; + if !self.without_imports { + imports.extend(quote! { + use cubenativeutils::wrappers::inner_types::InnerTypes; + use cubenativeutils::wrappers::object::NativeStruct; + }); } + imports } fn original_trait(&self) -> proc_macro2::TokenStream {