recursive static filter

Author: waralexrom

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -556,9 +556,6 @@ export class CubeEvaluator extends CubeSymbols {
         if (funcArgs.length > 0 && cubeReferencesUsed.length === 0) {
           ownedByCube = false;
         }
-        if (member.type && member.type === 'switch') {
-          ownedByCube = false;
-        }
         // Aliases one to one some another member as in case of views
         // Note: Segments do not have type set
         if (!ownedByCube && !member.filters && (!member.type || CubeSymbols.isCalculatedMeasureType(member.type)) && pathReferencesUsed.length === 1 && this.pathFromArray(pathReferencesUsed[0]) === evaluatedSql) {

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -699,7 +699,8 @@ const DimensionsSchema = Joi.object().pattern(identifierRegex, Joi.alternatives(
   then: SwitchDimension,
   otherwise: Joi.alternatives().try(
     inherit(BaseDimensionWithoutSubQuery, {
-      case: CaseVariants
+      case: CaseVariants,
+      multiStage: Joi.boolean().strict(),
     }),
     inherit(BaseDimensionWithoutSubQuery, {
       latitude: Joi.object().keys({

packages/cubejs-schema-compiler/test/integration/postgres/calc-groups.test.ts

@@ -247,19 +247,11 @@ cubes:
       - name: count
         type: count
 
-  - name: source_root
+  - name: source
     sql: >
       SELECT 1
     public: false
 
-    joins:
-      - name: source_a
-        sql: "1 = 1"
-        relationship: one_to_many
-      - name: source_b
-        sql: "1 = 1"
-        relationship: one_to_many
-
     dimensions:
       - name: pk
         type: number
@@ -276,6 +268,7 @@ cubes:
 
       - name: product_category
         type: string
+        multi_stage: true
         case:
           switch: "{CUBE.source}"
           when:
@@ -411,15 +404,9 @@ cubes:
         sql: PRICE_EUR
 
 views:
-  - name: source
-    cubes:
-      - join_path: source_root
-        includes: "*"
-
 
 
   - name: orders_view
-
     cubes:
       - join_path: orders
         includes:
@@ -1209,6 +1196,29 @@ views:
       { joinGraph, cubeEvaluator, compiler });
     });
 
+    it('source switch - source_a + usd + filter by category', async () => {
+      await dbRunner.runQueryTest({
+        dimensions: ['source.currency', 'source.product_category'],
+        measures: ['source.price'],
+        order: [{
+          id: 'source.product_category'
+        },
+        ],
+        filters: [
+          { dimension: 'source.currency', operator: 'equals', values: ['USD'] },
+          { dimension: 'source.source', operator: 'equals', values: ['A'] },
+          { dimension: 'source.product_category', operator: 'equals', values: ['some category'] },
+        ],
+      }, [
+        {
+          source__currency: 'USD',
+          source__product_category: 'some category',
+          source__price: '600'
+        },
+      ],
+      { joinGraph, cubeEvaluator, compiler });
+    });
+
     it('source switch - source_b + eur', async () => {
       await dbRunner.runQueryTest({
         dimensions: ['source.currency', 'source.product_category'],

rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/multi_stage_measure_calculation.rs

@@ -50,6 +50,7 @@ impl<'a> LogicalNodeProcessor<'a, MultiStageMeasureCalculation>
         }
 
         for measure in measure_calculation.schema().measures.iter() {
+            println!("!!!! ooooo: {}", measure.full_name());
             references_builder.resolve_references_for_member(
                 measure.clone(),
                 &None,

rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs

@@ -18,6 +18,7 @@ pub enum FilterType {
     Measure,
 }
 
+#[derive(Clone)]
 pub struct BaseFilter {
     query_tools: Rc<QueryTools>,
     member_evaluator: Rc<MemberSymbol>,
@@ -83,6 +84,12 @@ impl BaseFilter {
         }
     }
 
+    pub fn with_member_evaluator(&self, member_evaluator: Rc<MemberSymbol>) -> Rc<Self> {
+        let mut result = self.clone();
+        result.member_evaluator = member_evaluator;
+        Rc::new(result)
+    }
+
     //FIXME Not very good solution, but suitable for check time dimension filters in pre-aggregations
     pub fn time_dimension_symbol(&self) -> Option<Rc<MemberSymbol>> {
         if self.member_evaluator.as_time_dimension().is_ok() {

rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_segment.rs

@@ -7,6 +7,7 @@ use crate::planner::{evaluate_with_context, VisitorContext};
 use cubenativeutils::CubeError;
 use std::rc::Rc;
 
+#[derive(Clone)]
 pub struct BaseSegment {
     full_name: String,
     member_evaluator: Rc<MemberSymbol>,
@@ -61,6 +62,12 @@ impl BaseSegment {
         self.member_evaluator.clone()
     }
 
+    pub fn with_member_evaluator(&self, member_evaluator: Rc<MemberSymbol>) -> Rc<Self> {
+        let mut result = self.clone();
+        result.member_evaluator = member_evaluator;
+        Rc::new(result)
+    }
+
     pub fn cube_name(&self) -> &String {
         &self.cube_name
     }

rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs

@@ -200,10 +200,15 @@ impl MultiStageMemberQueryPlanner {
             _ => MultiStageCalculationWindowFunction::None,
         };
 
+        let measures = if self.description.member().evaluation_node().is_measure() {
+            vec![self.description.member().evaluation_node().clone()]
+        } else {
+            vec![]
+        };
         let schema = LogicalSchema::default()
             .set_dimensions(self.description.state().dimensions_symbols())
             .set_time_dimensions(self.description.state().time_dimensions_symbols())
-            .set_measures(vec![self.description.member().evaluation_node().clone()])
+            .set_measures(measures)
             .into_rc();
 
         let calculation_type = match multi_stage_member.inode_type() {
@@ -221,10 +226,12 @@ impl MultiStageMemberQueryPlanner {
             .input_cte_aliases()
             .into_iter()
             .map(|(name, symbols)| {
-                Rc::new(MultiStageSubqueryRef::builder()
-                    .name(name.clone())
-                    .symbols(symbols.clone())
-                    .build())
+                Rc::new(
+                    MultiStageSubqueryRef::builder()
+                        .name(name.clone())
+                        .symbols(symbols.clone())
+                        .build(),
+                )
             })
             .collect_vec();
 
@@ -236,11 +243,13 @@ impl MultiStageMemberQueryPlanner {
             .partition_by(partition_by)
             .window_function_to_use(window_function_to_use)
             .order_by(self.query_order_by()?)
-            .source(Rc::new(FullKeyAggregate::builder()
-                .schema(full_key_aggregate_schema)
-                .use_full_join_and_coalesce(true)
-                .multi_stage_subquery_refs(input_sources)
-                .build()))
+            .source(Rc::new(
+                FullKeyAggregate::builder()
+                    .schema(full_key_aggregate_schema)
+                    .use_full_join_and_coalesce(true)
+                    .multi_stage_subquery_refs(input_sources)
+                    .build(),
+            ))
             .build();
 
         let result = LogicalMultiStageMember {

rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs

@@ -92,10 +92,12 @@ impl MultiStageQueryPlanner {
         let top_level_ctes = top_level_ctes
             .iter()
             .map(|(alias, symbols)| {
-                Rc::new(MultiStageSubqueryRef::builder()
-                    .name(alias.clone())
-                    .symbols(symbols.clone())
-                    .build())
+                Rc::new(
+                    MultiStageSubqueryRef::builder()
+                        .name(alias.clone())
+                        .symbols(symbols.clone())
+                        .build(),
+                )
             })
             .collect_vec();
 
@@ -136,6 +138,7 @@ impl MultiStageQueryPlanner {
                 is_ungrupped,
             )
         } else {
+            println!("!!!! jjjjj");
             (
                 MultiStageInodeMember::new(
                     MultiStageInodeMemberType::Calculate,
@@ -157,7 +160,7 @@ impl MultiStageQueryPlanner {
         descriptions: &mut Vec<Rc<MultiStageQueryDescription>>,
     ) -> Result<Rc<MultiStageQueryDescription>, CubeError> {
         let member = member.resolve_reference_chain();
-        let member = apply_static_filter_to_symbol(&member, state.dimensions_filters());
+        let member = apply_static_filter_to_symbol(&member, state.dimensions_filters())?;
 
         let member_name = member.full_name();
         if let Some(exists) = descriptions

rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs

@@ -4,7 +4,8 @@ use super::query_tools::QueryTools;
 use crate::cube_bridge::join_hints::JoinHintItem;
 use crate::cube_bridge::member_expression::MemberExpressionExpressionDef;
 use crate::planner::sql_evaluator::{
-    MemberExpressionExpression, MemberExpressionSymbol, TimeDimensionSymbol,
+    apply_static_filter_to_filter_item, apply_static_filter_to_symbol, MemberExpressionExpression,
+    MemberExpressionSymbol, TimeDimensionSymbol,
 };
 use crate::planner::GranularityHelper;
 
@@ -401,22 +402,10 @@ impl QueryProperties {
 
         let query_join_hints = Rc::new(options.join_hints()?.unwrap_or_default());
 
-        let multi_fact_join_groups = Self::compute_join_multi_fact_groups(
-            query_join_hints.clone(),
-            query_tools.clone(),
-            &measures,
-            &dimensions,
-            &time_dimensions,
-            &time_dimensions_filters,
-            &dimensions_filters,
-            &measures_filters,
-            &segments,
-        )?;
-
         let pre_aggregation_query = options.static_data().pre_aggregation_query.unwrap_or(false);
         let total_query = options.static_data().total_query.unwrap_or(false);
 
-        let res = Self {
+        let mut res = Self {
             measures,
             dimensions,
             segments,
@@ -430,11 +419,12 @@ impl QueryProperties {
             query_tools,
             ignore_cumulative: false,
             ungrouped,
-            multi_fact_join_groups,
+            multi_fact_join_groups: vec![],
             pre_aggregation_query,
             total_query,
             query_join_hints,
         };
+        res.apply_static_filters()?;
         Ok(Rc::new(res))
     }
 
@@ -462,19 +452,7 @@ impl QueryProperties {
             order_by
         };
 
-        let multi_fact_join_groups = Self::compute_join_multi_fact_groups(
-            query_join_hints.clone(),
-            query_tools.clone(),
-            &measures,
-            &dimensions,
-            &time_dimensions,
-            &time_dimensions_filters,
-            &dimensions_filters,
-            &measures_filters,
-            &segments,
-        )?;
-
-        let res = Self {
+        let mut res = Self {
             measures,
             dimensions,
             time_dimensions,
@@ -488,14 +466,55 @@ impl QueryProperties {
             query_tools,
             ignore_cumulative,
             ungrouped,
-            multi_fact_join_groups,
+            multi_fact_join_groups: vec![],
             pre_aggregation_query,
             total_query,
             query_join_hints,
         };
+        res.apply_static_filters()?;
+
         Ok(Rc::new(res))
     }
 
+    fn apply_static_filters(&mut self) -> Result<(), CubeError> {
+        let dimensions_filters = self.dimensions_filters.clone();
+        for dim in self.dimensions.iter_mut() {
+            *dim = apply_static_filter_to_symbol(dim, &dimensions_filters)?;
+        }
+        for meas in self.measures.iter_mut() {
+            *meas = apply_static_filter_to_symbol(meas, &dimensions_filters)?;
+        }
+        for filter_item in self.dimensions_filters.iter_mut() {
+            *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?;
+        }
+        for filter_item in self.measures_filters.iter_mut() {
+            *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?;
+        }
+        for filter_item in self.time_dimensions_filters.iter_mut() {
+            *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?;
+        }
+        for filter_item in self.segments.iter_mut() {
+            *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?;
+        }
+        for order_item in self.order_by.iter_mut() {
+            order_item.member_evaluator =
+                apply_static_filter_to_symbol(&order_item.member_evaluator, &dimensions_filters)?;
+        }
+
+        self.multi_fact_join_groups = Self::compute_join_multi_fact_groups(
+            self.query_join_hints.clone(),
+            self.query_tools.clone(),
+            &self.measures,
+            &self.dimensions,
+            &self.time_dimensions,
+            &self.time_dimensions_filters,
+            &self.dimensions_filters,
+            &self.measures_filters,
+            &self.segments,
+        )?;
+        Ok(())
+    }
+
     pub fn compute_join_multi_fact_groups_with_measures(
         &self,
         measures: &Vec<Rc<MemberSymbol>>,

rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs

@@ -26,6 +26,10 @@ impl TraversalVisitor for JoinHintsCollector {
         path: &Vec<String>,
         _: &Self::State,
     ) -> Result<Option<Self::State>, CubeError> {
+        if !node.owned_by_cube() {
+            return Ok(Some(()));
+        }
+        println!("!!! node: {}", node.full_name());
         match node.as_ref() {
             MemberSymbol::Dimension(e) => {
                 if !e.is_view() {
@@ -79,6 +83,7 @@ pub fn collect_join_hints(node: &Rc<MemberSymbol>) -> Result<Vec<JoinHintItem>,
     let mut visitor = JoinHintsCollector::new();
     visitor.apply(node, &())?;
     let res = visitor.extract_result();
+    println!("!!! join hints 0: {:#?}", res);
     Ok(res)
 }
 
@@ -90,5 +95,6 @@ pub fn collect_join_hints_for_measures(
         visitor.apply(&meas, &())?;
     }
     let res = visitor.extract_result();
+    println!("!!! join hints: {:#?}", res);
     Ok(res)
 }