in work

Author: waralexrom

packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-multi-stage.test.ts

@@ -161,8 +161,8 @@ describe('PreAggregationsMultiStage', () => {
     const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
     const sqlAndParams = query.buildSqlAndParams();
       console.log("!!!! sqlAndParamsl", sqlAndParams);
-/*     expect(preAggregationsDescription[0].tableName).toEqual('rvis_rollupalias');
-    expect(sqlAndParams[0]).toContain('rvis_rollupalias'); */
+    expect(preAggregationsDescription[0].tableName).toEqual('rvis_rollupalias');
+    expect(sqlAndParams[0]).toContain('rvis_rollupalias');
 
     return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
         console.log("!!!! res", res);

packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts

@@ -3,6 +3,9 @@ import { PostgresQuery } from '../../../src/adapter/PostgresQuery';
 import { BigqueryQuery } from '../../../src/adapter/BigqueryQuery';
 import { prepareJsCompiler } from '../../unit/PrepareCompiler';
 import { dbRunner } from './PostgresDBRunner';
+import {
+  getEnv,
+} from '@cubejs-backend/shared';
 
 describe('PreAggregations', () => {
   jest.setTimeout(200000);
@@ -568,6 +571,9 @@ describe('PreAggregations', () => {
 
 
   it('simple pre-aggregation proxy time dimension', () => compiler.compile().then(() => {
+    if (!getEnv('nativeSqlPlanner')) {
+      return;
+    }
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
       measures: [
         'visitors.count'

rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/dimension_matcher.rs

@@ -1,142 +1,256 @@
 use super::CompiledPreAggregation;
 use super::MatchState;
-use crate::plan::{Filter, FilterItem};
 use crate::plan::filter::FilterGroupOperator;
+use crate::plan::{Filter, FilterItem};
+use crate::planner::filter::BaseFilter;
+use crate::planner::query_tools::QueryTools;
+use crate::planner::sql_evaluator::DimensionSymbol;
 use crate::planner::sql_evaluator::MemberSymbol;
+use crate::planner::sql_evaluator::TimeDimensionSymbol;
+use crate::planner::GranularityHelper;
 use cubenativeutils::CubeError;
 use std::collections::HashMap;
-use crate::planner::filter::BaseFilter;
 use std::rc::Rc;
 
-pub struct DimensionMatcher {
+pub struct DimensionMatcher<'a> {
+    query_tools: Rc<QueryTools>,
+    pre_aggregation: &'a CompiledPreAggregation,
+    pre_aggregation_dimensions: HashMap<String, bool>,
+    pre_aggregation_time_dimensions: HashMap<String, (Option<String>, bool)>,
+    result: MatchState,
 }
 
-impl DimensionMatcher {
-    pub fn new() -> Self {
-        Self {}
+impl<'a> DimensionMatcher<'a> {
+    pub fn new(query_tools: Rc<QueryTools>, pre_aggregation: &'a CompiledPreAggregation) -> Self {
+        let pre_aggregation_dimensions = pre_aggregation
+            .dimensions
+            .iter()
+            .map(|d| (d.full_name(), false))
+            .collect();
+        let pre_aggregation_time_dimensions = pre_aggregation
+            .time_dimensions
+            .iter()
+            .map(|(dim, granularity)| (dim.full_name(), (granularity.clone(), false)))
+            .collect::<HashMap<_, _>>();
+        Self {
+            query_tools,
+            pre_aggregation,
+            pre_aggregation_dimensions,
+            pre_aggregation_time_dimensions,
+            result: MatchState::Full
+        }
     }
 
     pub fn try_match(
-        &self,
-        symbols: &Vec<Rc<MemberSymbol>>,
+        &mut self,
+        dimensions: &Vec<Rc<MemberSymbol>>,
+        time_dimensions: &Vec<Rc<MemberSymbol>>,
         filters: &Vec<FilterItem>,
+        time_dimension_filters: &Vec<FilterItem>,
         segments: &Vec<FilterItem>,
-        pre_aggregation: &CompiledPreAggregation,
-    ) -> Result<MatchState, CubeError> {
-        let mut pre_aggregation_dimensions = pre_aggregation
-            .dimensions
-            .iter()
-            .map(|d| (d.full_name(), false))
-            .collect();
-        let mut result = MatchState::Full;
-        for symbol in symbols.iter() {
-            let symbol_match = self.try_match_symbol(symbol, true, &mut pre_aggregation_dimensions)?;
-            result = result.combine(&symbol_match);
-            if result == MatchState::NotMatched {
-                return Ok(result);
+    ) -> Result<(), CubeError> {
+        for dimension in dimensions.iter() {
+            let dimension_match = self.try_match_symbol(dimension, true)?;
+            self.result = self.result.combine(&dimension_match);
+            if self.result == MatchState::NotMatched {
+                return Ok(());
+            }
+        }
+        for time_dimension in time_dimensions.iter() {
+            let time_dimension_match = self.try_match_symbol(time_dimension, true)?;
+            self.result = self.result.combine(&time_dimension_match);
+            if self.result == MatchState::NotMatched {
+                return Ok(());
             }
         }
 
         for filter in filters.iter() {
-            let filter_match = self.try_match_filter_item(filter, true, &mut pre_aggregation_dimensions)?;
-            result = result.combine(&filter_match);
-            if result == MatchState::NotMatched {
-                return Ok(result);
+            let filter_match = self.try_match_filter_item(filter, true)?;
+            self.result = self.result.combine(&filter_match);
+            if self.result == MatchState::NotMatched {
+                return Ok(());
+            }
+        }
+
+        for filter in time_dimension_filters.iter() {
+            let filter_match = self.try_match_filter_item(filter, true)?;
+            self.result = self.result.combine(&filter_match);
+            if self.result == MatchState::NotMatched {
+                return Ok(());
             }
         }
 
         for segment in segments.iter() {
-            let segment_match = self.try_match_filter_item(segment, true, &mut pre_aggregation_dimensions)?;
-            result = result.combine(&segment_match);
-            if result == MatchState::NotMatched {
-                return Ok(result);
+            let segment_match = self.try_match_filter_item(segment, true)?;
+            self.result = self.result.combine(&segment_match);
+            if self.result == MatchState::NotMatched {
+                return Ok(());
             }
         }
+        Ok(())
+    }
 
-        let coverage_result = if pre_aggregation_dimensions.values().all(|v| *v) {
+    pub fn result(mut self) -> MatchState {
+        let dimension_coverage_result = if self.pre_aggregation_dimensions.values().all(|v| *v) {
             MatchState::Full
         } else {
             MatchState::Partial
         };
-
-        Ok(result.combine(&coverage_result))
+        self.result = self.result.combine(&dimension_coverage_result);
+        let time_dimension_coverage_result = if self.pre_aggregation_time_dimensions.values().all(|v| v.1) {
+            MatchState::Full
+        } else {
+            MatchState::Partial
+        };
+        self.result = self.result.combine(&time_dimension_coverage_result);
+        self.result
     }
 
     fn try_match_symbol(
-        &self,
+        &mut self,
         symbol: &Rc<MemberSymbol>,
         add_to_matched_dimension: bool,
-        pre_aggregation_dimensions: &mut HashMap<String, bool>,
     ) -> Result<MatchState, CubeError> {
-        let mut result = match symbol.as_ref() {
+        match symbol.as_ref() {
             MemberSymbol::Dimension(dimension) => {
-                if let Some(found) = pre_aggregation_dimensions.get_mut(&dimension.full_name()) {
-                    if add_to_matched_dimension {
-                        *found = true;
-                    }
-                    return Ok(MatchState::Full);
-                } else if dimension.owned_by_cube() {
-                    return Ok(MatchState::NotMatched);
-                }
-                MatchState::Full
+                self.try_match_dimension(dimension, add_to_matched_dimension)
             }
-            MemberSymbol::MemberExpression(_member_expression) => MatchState::NotMatched, //TODO We not allow to use pre-aggregations with member expressions before sqlapi ready for it
-            _ => return Ok(MatchState::NotMatched),
-        };
-
-        if symbol.get_dependencies().is_empty() {
-            return Ok(MatchState::NotMatched);
+            MemberSymbol::TimeDimension(time_dimension) => {
+                self.try_match_time_dimension(time_dimension, add_to_matched_dimension)
+            }
+            MemberSymbol::MemberExpression(_member_expression) => Ok(MatchState::NotMatched), //TODO We not allow to use pre-aggregations with member expressions before sqlapi ready for it
+            _ => Ok(MatchState::NotMatched),
         }
+    }
 
-        if !symbol.is_reference() {
-            result = result.combine(&MatchState::Partial);
+    fn try_match_dimension(
+        &mut self,
+        dimension: &DimensionSymbol,
+        add_to_matched_dimension: bool,
+    ) -> Result<MatchState, CubeError> {
+        if let Some(found) = self
+            .pre_aggregation_dimensions
+            .get_mut(&dimension.full_name())
+        {
+            if add_to_matched_dimension {
+                *found = true;
+            }
+            Ok(MatchState::Full)
+        } else if dimension.owned_by_cube() {
+            Ok(MatchState::NotMatched)
+        } else {
+            let dependencies = dimension.get_dependencies();
+            if dependencies.is_empty() {
+                Ok(MatchState::NotMatched)
+            } else {
+                let mut result = if dimension.is_reference() {
+                    MatchState::Full
+                } else {
+                    MatchState::Partial
+                };
+                for dep in dimension.get_dependencies() {
+                    let dep_match = self.try_match_symbol(&dep, add_to_matched_dimension)?;
+                    if dep_match == MatchState::NotMatched {
+                        return Ok(MatchState::NotMatched);
+                    }
+                    result = result.combine(&dep_match);
+                }
+                Ok(result)
+            }
         }
+    }
 
-        for dep in symbol.get_dependencies() {
-            let dep_match = self.try_match_symbol(&dep, add_to_matched_dimension, pre_aggregation_dimensions)?;
-            if dep_match == MatchState::NotMatched {
-                return Ok(MatchState::NotMatched);
+    fn try_match_time_dimension(
+        &mut self,
+        time_dimension: &TimeDimensionSymbol,
+        add_to_matched_dimension: bool,
+    ) -> Result<MatchState, CubeError> {
+        let granularity = if self.pre_aggregation.allow_non_strict_date_range_match {
+            time_dimension.granularity().clone()
+        } else {
+            time_dimension.rollup_granularity(self.query_tools.clone())?
+        };
+        let base_symbol_name = time_dimension.base_symbol().full_name();
+        if let Some(found) = self
+            .pre_aggregation_time_dimensions
+            .get_mut(&base_symbol_name)
+        {
+            if add_to_matched_dimension {
+                found.1 = true;
+            }
+            let pre_aggr_granularity = &found.0;
+            if granularity.is_none() || pre_aggr_granularity == &granularity {
+                Ok(MatchState::Full)
+            } else if pre_aggr_granularity.is_none()
+                || GranularityHelper::is_predefined_granularity(
+                    pre_aggr_granularity.as_ref().unwrap(),
+                )
+            {
+                let min_granularity =
+                    GranularityHelper::min_granularity(&granularity, &pre_aggr_granularity)?;
+                if &min_granularity == pre_aggr_granularity {
+                    Ok(MatchState::Partial)
+                } else {
+                    Ok(MatchState::NotMatched)
+                }
+            } else {
+                Ok(MatchState::NotMatched) //TODO Custom granularities!!!
+            }
+        } else {
+            if time_dimension.owned_by_cube() {
+                Ok(MatchState::NotMatched)
+            } else {
+                let mut result = if time_dimension.is_reference() {
+                    MatchState::Full
+                } else {
+                    MatchState::Partial
+                };
+                for dep in time_dimension.get_dependencies_as_time_dimensions() {
+                    let dep_match = self.try_match_symbol(&dep, add_to_matched_dimension)?;
+                    if dep_match == MatchState::NotMatched {
+                        return Ok(MatchState::NotMatched);
+                    }
+                    result = result.combine(&dep_match);
+                }
+                Ok(result)
             }
-            result = result.combine(&dep_match);
         }
-        Ok(result)
     }
 
     fn try_match_filter_item(
-        &self,
+        &mut self,
         filter_item: &FilterItem,
         add_to_matched_dimension: bool,
-        pre_aggregation_dimensions: &mut HashMap<String, bool>,
     ) -> Result<MatchState, CubeError> {
         match filter_item {
-            FilterItem::Item(filter) => {
-                self.try_match_filter(filter, add_to_matched_dimension, pre_aggregation_dimensions)
-            }
+            FilterItem::Item(filter) => self.try_match_filter(filter, add_to_matched_dimension),
             FilterItem::Group(group) => {
-                let add_to_matched_dimension = add_to_matched_dimension && group.operator == FilterGroupOperator::And;
+                let add_to_matched_dimension =
+                    add_to_matched_dimension && group.operator == FilterGroupOperator::And;
                 let mut result = MatchState::Full;
                 for item in group.items.iter() {
-                    result = result.combine(&self.try_match_filter_item(item, add_to_matched_dimension, pre_aggregation_dimensions)?);
+                    result = result
+                        .combine(&self.try_match_filter_item(item, add_to_matched_dimension)?);
                 }
                 Ok(result)
-            },
+            }
             FilterItem::Segment(segment) => {
-                self.try_match_symbol(&segment.member_evaluator(), add_to_matched_dimension, pre_aggregation_dimensions)
-            },
+                self.try_match_symbol(&segment.member_evaluator(), add_to_matched_dimension)
+            }
         }
     }
 
     fn try_match_filter(
-        &self,
+        &mut self,
         filter: &Rc<BaseFilter>,
         add_to_matched_dimension: bool,
-        pre_aggregation_dimensions: &mut HashMap<String, bool>,
     ) -> Result<MatchState, CubeError> {
-        let symbol = filter.member_evaluator().clone();
+        let symbol = if let Some(time_dimension) = filter.time_dimension_symbol() {
+            time_dimension
+        } else {
+            filter.member_evaluator().clone()
+        };
         let add_to_matched_dimension = add_to_matched_dimension && filter.is_single_value_equal();
-        self.try_match_symbol(&symbol, add_to_matched_dimension, pre_aggregation_dimensions)
+        self.try_match_symbol(&symbol, add_to_matched_dimension)
     }
-
-
-
 }