fix(tesseract): Fix rolling window with few time dimensions, filter_group in segments and member expressions

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -806,6 +806,7 @@ export class BaseQuery {
       exportAnnotatedSql: exportAnnotatedSql === true,
       preAggregationQuery: this.options.preAggregationQuery,
       totalQuery: this.options.totalQuery,
+      joinHints: this.options.joinHints,
     };
 
     const buildResult = nativeBuildSqlAndParams(queryParams);

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -3281,6 +3281,9 @@ SELECT 1 AS revenue,  cast('2024-01-01' AS timestamp) as time UNION ALL
   for (const granularityTest of granularityCases) {
     // eslint-disable-next-line no-loop-func
     it(`Should date with TZ, when pass timeDimensions with granularity by ${granularityTest.granularity}`, async () => {
+      if (getEnv('nativeSqlPlanner')) {
+        return;
+      }
       await compiler.compile();
 
       const query = new BigqueryQuery({ joinGraph, cubeEvaluator, compiler }, {

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs

@@ -1,4 +1,5 @@
 use super::join_graph::{JoinGraph, NativeJoinGraph};
+use super::join_hints::JoinHintItem;
 use super::options_member::OptionsMember;
 use crate::cube_bridge::base_tools::{BaseTools, NativeBaseTools};
 use crate::cube_bridge::evaluator::{CubeEvaluator, NativeCubeEvaluator};
@@ -81,4 +82,6 @@ pub trait BaseQueryOptions {
     fn base_tools(&self) -> Result<Rc<dyn BaseTools>, CubeError>;
     #[nbridge(field)]
     fn join_graph(&self) -> Result<Rc<dyn JoinGraph>, CubeError>;
+    #[nbridge(field, optional, vec)]
+    fn join_hints(&self) -> Result<Option<Vec<JoinHintItem>>, CubeError>;
 }

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/join_hints.rs

@@ -1,7 +1,25 @@
-use serde::{Deserialize, Serialize};
+use cubenativeutils::wrappers::inner_types::InnerTypes;
+use cubenativeutils::wrappers::serializer::NativeDeserialize;
+use cubenativeutils::wrappers::NativeObjectHandle;
+use cubenativeutils::CubeError;
+use serde::Serialize;
 
-#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize)]
 pub enum JoinHintItem {
     Single(String),
     Vector(Vec<String>),
 }
+
+impl<IT: InnerTypes> NativeDeserialize<IT> for JoinHintItem {
+    fn from_native(native_object: NativeObjectHandle<IT>) -> Result<Self, CubeError> {
+        match Vec::<String>::from_native(native_object.clone()) {
+            Ok(value) => Ok(Self::Vector(value)),
+            Err(_) => match String::from_native(native_object) {
+                Ok(value) => Ok(Self::Single(value)),
+                Err(_) => Err(CubeError::user(format!(
+                    "Join hint item expected to be string or vector of strings"
+                ))),
+            },
+        }
+    }
+}

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs

@@ -19,7 +19,7 @@ pub struct TimeShiftReference {
     #[serde(rename = "type")]
     pub shift_type: Option<String>,
     #[serde(rename = "timeDimension")]
-    pub time_dimension: String,
+    pub time_dimension: Option<String>,
 }
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/member_expression.rs

@@ -1,24 +1,62 @@
 use super::member_sql::{MemberSql, NativeMemberSql};
+use super::struct_with_sql_member::{NativeStructWithSqlMember, StructWithSqlMember};
 use cubenativeutils::wrappers::serializer::{
     NativeDeserialize, NativeDeserializer, NativeSerialize,
 };
+use cubenativeutils::wrappers::NativeArray;
 use cubenativeutils::wrappers::{NativeContextHolder, NativeObjectHandle};
 use cubenativeutils::CubeError;
 use serde::{Deserialize, Serialize};
 use std::any::Any;
 use std::rc::Rc;
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ExpressionStructStatic {
+    #[serde(rename = "type")]
+    pub expression_type: String,
+    #[serde(rename = "sourceMeasure")]
+    pub source_measure: Option<String>,
+    #[serde(rename = "replaceAggregationType")]
+    pub replace_aggregation_type: Option<String>,
+}
+
+#[nativebridge::native_bridge(ExpressionStructStatic)]
+pub trait ExpressionStruct {
+    #[nbridge(field, optional, vec)]
+    fn add_filters(&self) -> Result<Option<Vec<Rc<dyn StructWithSqlMember>>>, CubeError>;
+}
+
+pub enum MemberExpressionExpressionDef {
+    Sql(Rc<dyn MemberSql>),
+    Struct(Rc<dyn ExpressionStruct>),
+}
+
+impl<IT: InnerTypes> NativeDeserialize<IT> for MemberExpressionExpressionDef {
+    fn from_native(native_object: NativeObjectHandle<IT>) -> Result<Self, CubeError> {
+        match NativeMemberSql::from_native(native_object.clone()) {
+            Ok(sql) => Ok(Self::Sql(Rc::new(sql))),
+            Err(_) => match NativeExpressionStruct::from_native(native_object) {
+                Ok(expr) => Ok(Self::Struct(Rc::new(expr))),
+                Err(_) => Err(CubeError::user(format!(
+                    "Member sql or expression struct expected for member expression expression field"
+                ))),
+            },
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct MemberExpressionDefinitionStatic {
     #[serde(rename = "expressionName")]
     pub expression_name: Option<String>,
+    pub name: Option<String>,
     #[serde(rename = "cubeName")]
     pub cube_name: Option<String>,
     pub definition: Option<String>,
 }
 
-#[nativebridge::native_bridge(MemberExpressionDefinitionStatic)]
+#[nativebridge::native_bridge(MemberExpressionDefinitionStatic, without_imports)]
 pub trait MemberExpressionDefinition {
     #[nbridge(field)]
-    fn expression(&self) -> Result<Rc<dyn MemberSql>, CubeError>;
+    fn expression(&self) -> Result<MemberExpressionExpressionDef, CubeError>;
 }

rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs

@@ -53,7 +53,10 @@ impl PrettyPrint for MultiStageAppliedState {
         }
 
         result.println("time_shifts:", &state);
-        for (_, time_shift) in self.time_shifts().iter() {
+        if let Some(common) = &self.time_shifts().common_time_shift {
+            result.println(&format!("- common: {}", common.to_sql()), &details_state);
+        }
+        for (_, time_shift) in self.time_shifts().dimensions_shifts.iter() {
             result.println(
                 &format!(
                     "- {}: {}",

rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/leaf_measure.rs

@@ -1,14 +1,13 @@
 use crate::logical_plan::*;
-use crate::planner::sql_evaluator::MeasureTimeShift;
+use crate::planner::planners::multi_stage::TimeShiftState;
 use crate::planner::sql_evaluator::MemberSymbol;
-use std::collections::HashMap;
 use std::rc::Rc;
 
 pub struct MultiStageLeafMeasure {
     pub measure: Rc<MemberSymbol>,
     pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx
     pub render_measure_for_ungrouped: bool,
-    pub time_shifts: HashMap<String, MeasureTimeShift>,
+    pub time_shifts: TimeShiftState,
     pub query: Rc<Query>,
 }
 
@@ -23,10 +22,13 @@ impl PrettyPrint for MultiStageLeafMeasure {
         if self.render_measure_for_ungrouped {
             result.println("render_measure_for_ungrouped: true", &state);
         }
-        if !self.time_shifts.is_empty() {
+        if !self.time_shifts.dimensions_shifts.is_empty() {
             result.println("time_shifts:", &state);
             let details_state = state.new_level();
-            for (_, time_shift) in self.time_shifts.iter() {
+            if let Some(common) = &self.time_shifts.common_time_shift {
+                result.println(&format!("- common: {}", common.to_sql()), &details_state);
+            }
+            for (_, time_shift) in self.time_shifts.dimensions_shifts.iter() {
                 result.println(
                     &format!(
                         "- {}: {}",

rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs

@@ -1,10 +1,10 @@
 use crate::logical_plan::*;
 use crate::plan::schema::QualifiedColumnName;
 use crate::plan::*;
+use crate::planner::planners::multi_stage::TimeShiftState;
 use crate::planner::query_properties::OrderByItem;
 use crate::planner::query_tools::QueryTools;
 use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory;
-use crate::planner::sql_evaluator::MeasureTimeShift;
 use crate::planner::sql_evaluator::MemberSymbol;
 use crate::planner::sql_evaluator::ReferencesBuilder;
 use crate::planner::sql_templates::PlanSqlTemplates;
@@ -24,7 +24,7 @@ struct PhysicalPlanBuilderContext {
     pub alias_prefix: Option<String>,
     pub render_measure_as_state: bool, //Render measure as state, for example hll state for count_approx
     pub render_measure_for_ungrouped: bool,
-    pub time_shifts: HashMap<String, MeasureTimeShift>,
+    pub time_shifts: TimeShiftState,
     pub original_sql_pre_aggregations: HashMap<String, String>,
 }
 

rust/cubesqlplanner/cubesqlplanner/src/planner/base_dimension.rs

@@ -2,6 +2,7 @@ use super::query_tools::QueryTools;
 use super::sql_evaluator::{MemberExpressionSymbol, MemberSymbol, SqlCall};
 use super::{evaluate_with_context, BaseMember, BaseMemberHelper, VisitorContext};
 use crate::cube_bridge::dimension_definition::DimensionDefinition;
+use crate::planner::sql_evaluator::MemberExpressionExpression;
 use crate::planner::sql_templates::PlanSqlTemplates;
 use cubenativeutils::CubeError;
 use std::rc::Rc;
@@ -126,11 +127,12 @@ impl BaseDimension {
         let member_expression_symbol = MemberExpressionSymbol::try_new(
             cube_name.clone(),
             name.clone(),
-            expression,
+            MemberExpressionExpression::SqlCall(expression),
             member_expression_definition.clone(),
+            query_tools.base_tools().clone(),
         )?;
         let full_name = member_expression_symbol.full_name();
-        let member_evaluator = Rc::new(MemberSymbol::MemberExpression(member_expression_symbol));
+        let member_evaluator = MemberSymbol::new_member_expression(member_expression_symbol);
         let default_alias = PlanSqlTemplates::alias_name(&name);
 
         Ok(Rc::new(Self {

rust/cubesqlplanner/cubesqlplanner/src/planner/base_measure.rs

@@ -1,7 +1,8 @@
 use super::query_tools::QueryTools;
-use super::sql_evaluator::{MeasureTimeShift, MemberExpressionSymbol, MemberSymbol, SqlCall};
+use super::sql_evaluator::{MeasureTimeShifts, MemberExpressionSymbol, MemberSymbol};
 use super::{evaluate_with_context, BaseMember, BaseMemberHelper, VisitorContext};
-use crate::cube_bridge::measure_definition::{MeasureDefinition, RollingWindow};
+use crate::cube_bridge::measure_definition::RollingWindow;
+use crate::planner::sql_evaluator::MemberExpressionExpression;
 use crate::planner::sql_templates::PlanSqlTemplates;
 use cubenativeutils::CubeError;
 use std::fmt::{Debug, Formatter};
@@ -11,7 +12,6 @@ pub struct BaseMeasure {
     measure: String,
     query_tools: Rc<QueryTools>,
     member_evaluator: Rc<MemberSymbol>,
-    definition: Option<Rc<dyn MeasureDefinition>>,
     #[allow(dead_code)]
     member_expression_definition: Option<String>,
     cube_name: String,
@@ -84,7 +84,6 @@ impl BaseMeasure {
                     measure: s.full_name(),
                     query_tools: query_tools.clone(),
                     member_evaluator: evaluation_node.clone(),
-                    definition: Some(s.definition().clone()),
                     member_expression_definition: None,
                     cube_name: s.cube_name().clone(),
                     name: s.name().clone(),
@@ -101,7 +100,6 @@ impl BaseMeasure {
                     measure: full_name,
                     query_tools: query_tools.clone(),
                     member_evaluator: evaluation_node,
-                    definition: None,
                     cube_name,
                     name,
                     member_expression_definition,
@@ -127,7 +125,7 @@ impl BaseMeasure {
     }
 
     pub fn try_new_from_expression(
-        expression: Rc<SqlCall>,
+        expression: MemberExpressionExpression,
         cube_name: String,
         name: String,
         member_expression_definition: Option<String>,
@@ -138,36 +136,28 @@ impl BaseMeasure {
             name.clone(),
             expression,
             member_expression_definition.clone(),
+            query_tools.base_tools().clone(),
         )?;
         let full_name = member_expression_symbol.full_name();
-        let member_evaluator = Rc::new(MemberSymbol::MemberExpression(member_expression_symbol));
+        let member_evaluator = MemberSymbol::new_member_expression(member_expression_symbol);
         let default_alias = PlanSqlTemplates::alias_name(&name);
         Ok(Rc::new(Self {
             measure: full_name,
             query_tools,
             member_evaluator,
-            definition: None,
             cube_name,
             name,
             member_expression_definition,
             default_alias,
         }))
     }
 
-    pub fn can_used_as_addictive_in_multplied(&self) -> Result<bool, CubeError> {
-        let measure_type = self.measure_type();
-        let res = if measure_type == "countDistinct" || measure_type == "countDistinctApprox" {
-            true
-        } else if measure_type == "count" {
-            if let Some(definition) = &self.definition {
-                !definition.has_sql()?
-            } else {
-                false
-            }
+    pub fn can_used_as_addictive_in_multplied(&self) -> bool {
+        if let Ok(measure_symbol) = self.member_evaluator.as_measure() {
+            measure_symbol.can_used_as_addictive_in_multplied()
         } else {
             false
-        };
-        Ok(res)
+        }
     }
 
     pub fn member_evaluator(&self) -> &Rc<MemberSymbol> {
@@ -182,49 +172,35 @@ impl BaseMeasure {
         &self.cube_name
     }
 
-    pub fn reduce_by(&self) -> Option<Vec<String>> {
-        self.definition
-            .as_ref()
-            .and_then(|d| d.static_data().reduce_by_references.clone())
-    }
-
-    pub fn add_group_by(&self) -> Option<Vec<String>> {
-        self.definition
-            .as_ref()
-            .and_then(|d| d.static_data().add_group_by_references.clone())
-    }
-
-    pub fn group_by(&self) -> Option<Vec<String>> {
-        self.definition
-            .as_ref()
-            .and_then(|d| d.static_data().group_by_references.clone())
-    }
-
-    //FIXME dublicate with symbol
     pub fn is_calculated(&self) -> bool {
-        match self.measure_type() {
-            "number" | "string" | "time" | "boolean" => true,
-            _ => false,
+        if let Ok(measure_symbol) = self.member_evaluator.as_measure() {
+            measure_symbol.is_calculated()
+        } else {
+            true
         }
     }
 
-    pub fn time_shifts(&self) -> Vec<MeasureTimeShift> {
+    pub fn time_shift(&self) -> Option<MeasureTimeShifts> {
         match self.member_evaluator.as_ref() {
-            MemberSymbol::Measure(measure_symbol) => measure_symbol.time_shifts().clone(),
-            _ => vec![],
+            MemberSymbol::Measure(measure_symbol) => measure_symbol.time_shift().clone(),
+            _ => None,
         }
     }
 
     pub fn is_multi_stage(&self) -> bool {
-        self.definition
-            .as_ref()
-            .is_some_and(|d| d.static_data().multi_stage.unwrap_or(false))
+        if let Ok(measure_symbol) = self.member_evaluator.as_measure() {
+            measure_symbol.is_multi_stage()
+        } else {
+            false
+        }
     }
 
     pub fn rolling_window(&self) -> Option<RollingWindow> {
-        self.definition
-            .as_ref()
-            .and_then(|d| d.static_data().rolling_window.clone())
+        if let Ok(measure_symbol) = self.member_evaluator.as_measure() {
+            measure_symbol.rolling_window().clone()
+        } else {
+            None
+        }
     }
 
     pub fn is_rolling_window(&self) -> bool {
@@ -239,11 +215,12 @@ impl BaseMeasure {
         self.is_rolling_window() || self.is_running_total()
     }
 
-    //FIXME dublicate with symbol
-    pub fn measure_type(&self) -> &str {
-        self.definition
-            .as_ref()
-            .map_or("number", |d| &d.static_data().measure_type)
+    pub fn measure_type(&self) -> String {
+        if let Ok(measure_symbol) = self.member_evaluator.as_measure() {
+            measure_symbol.measure_type().clone()
+        } else {
+            "number".to_string()
+        }
     }
 
     pub fn is_multi_stage_ungroupped(&self) -> bool {