diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 3ac1a2de9e600..1701ad115f3af 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -11,7 +11,7 @@ import cronParser from 'cron-parser'; import moment from 'moment-timezone'; import inflection from 'inflection'; -import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv } from '@cubejs-backend/shared'; +import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv, timeSeries as timeSeriesBase } from '@cubejs-backend/shared'; import { buildSqlAndParams as nativeBuildSqlAndParams, @@ -606,6 +606,11 @@ export class BaseQuery { } buildSqlAndParamsRust(exportAnnotatedSql) { + const order = this.options.order && R.pipe( + R.map((hash) => ((!hash || !hash.id) ? null : hash)), + R.reject(R.isNil), + )(this.options.order); + const queryParams = { measures: this.options.measures, dimensions: this.options.dimensions, @@ -614,12 +619,13 @@ export class BaseQuery { joinRoot: this.join.root, joinGraph: this.joinGraph, cubeEvaluator: this.cubeEvaluator, - order: this.options.order, + order, filters: this.options.filters, limit: this.options.limit ? this.options.limit.toString() : null, rowLimit: this.options.rowLimit ? this.options.rowLimit.toString() : null, offset: this.options.offset ? this.options.offset.toString() : null, baseTools: this, + ungrouped: this.options.ungrouped }; const res = nativeBuildSqlAndParams(queryParams); @@ -628,6 +634,21 @@ export class BaseQuery { return res; } + allCubeMembers(path) { + const fromPath = this.cubeEvaluator.cubeFromPath(path); + + return Object.keys(fromPath.measures).concat(Object.keys(fromPath.dimensions)); + } + + getAllocatedParams() { + return this.paramAllocator.getParams(); + } + + // FIXME helper for native generator, maybe should be moved entire to rust + generateTimeSeries(granularity, dateRange) { + return timeSeriesBase(granularity, dateRange); + } + get shouldReuseParams() { return false; } @@ -3232,7 +3253,16 @@ export class BaseQuery { '{% if offset is not none %}\nOFFSET {{ offset }}{% endif %}', group_by_exprs: '{{ group_by | map(attribute=\'index\') | join(\', \') }}', join: '{{ join_type }} JOIN {{ source }} ON {{ condition }}', - cte: '{{ alias }} AS ({{ query | indent(2, true) }})' + cte: '{{ alias }} AS ({{ query | indent(2, true) }})', + time_series_select: 'SELECT date_from::timestamp AS "date_from",\n' + + 'date_to::timestamp AS "date_to" \n' + + 'FROM(\n' + + ' VALUES ' + + '{% for time_item in seria %}' + + '(\'{{ time_item | join(\'\\\', \\\'\') }}\')' + + '{% if not loop.last %}, {% endif %}' + + '{% endfor %}' + + ') AS dates (date_from, date_to)' }, expressions: { column_reference: '{% if table_name %}{{ table_name }}.{% endif %}{{ name }}', @@ -3253,6 +3283,8 @@ export class BaseQuery { cube: 'CUBE({{ exprs_concat }})', negative: '-({{ expr }})', not: 'NOT ({{ expr }})', + add_interval: '{{ date }} + interval \'{{ interval }}\'', + sub_interval: '{{ date }} - interval \'{{ interval }}\'', true: 'TRUE', false: 'FALSE', like: '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}', @@ -3272,7 +3304,8 @@ export class BaseQuery { gte: '{{ column }} >= {{ param }}', lt: '{{ column }} < {{ param }}', lte: '{{ column }} <= {{ param }}', - always_true: '1 == 1' + like_pattern: '{% if start_wild %}\'%\' || {% endif %}{{ value }}{% if end_wild %}|| \'%\'{% endif %}', + always_true: '1 = 1' }, quotes: { diff --git a/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts b/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts index 5dd8ae43d2c87..360dd90eecb35 100644 --- a/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts +++ b/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts @@ -51,6 +51,10 @@ export class ParamAllocator { return `$${paramIndex}$`; } + public getParams() { + return this.params; + } + // eslint-disable-next-line no-unused-vars protected paramPlaceHolder(paramIndex) { return '?'; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js index ee8b0ab6e333f..ef5edc931a01a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js @@ -106,7 +106,7 @@ export class CubeSymbols { }, set segments(v) { // Dont allow to modify - } + }, }, cubeDefinition); if (cubeDefinition.extends) { diff --git a/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts b/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts index f399bcfa72a7b..296a224446a57 100644 --- a/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts +++ b/packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts @@ -5,7 +5,7 @@ import type { StartedTestContainer } from 'testcontainers'; import { format as formatSql } from 'sqlstring'; import { v4 as uuidv4 } from 'uuid'; import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery'; -import { BaseDbRunner } from "../utils/BaseDbRunner"; +import { BaseDbRunner } from '../utils/BaseDbRunner'; process.env.TZ = 'GMT'; diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts index e8d6d6e4b2aac..c902b7d1faf37 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts @@ -453,7 +453,7 @@ describe('SQL Generation', () => { } }); - it('having filter with operator OR', async () => { + it('having filter with operator OR 1', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -648,7 +648,7 @@ describe('SQL Generation', () => { }); }); - it('where filter with operators OR & AND', async () => { + it('where filter with operators OR & AND 1', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index f9493faba34a0..20105aeaf1435 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -64,7 +64,7 @@ describe('SQL Generation', () => { offset: 'start' } }, - revenueRolling3day: { + revenueRollingThreeDay: { type: 'sum', sql: 'amount', rollingWindow: { @@ -769,7 +769,7 @@ describe('SQL Generation', () => { it('rolling month', async () => runQueryTest({ measures: [ - 'visitors.revenueRolling3day' + 'visitors.revenueRollingThreeDay' ], timeDimensions: [{ dimension: 'visitors.created_at', @@ -781,7 +781,7 @@ describe('SQL Generation', () => { }], timezone: 'America/Los_Angeles' }, [ - { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling3day: '900' } + { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling_three_day: '900' } ])); it('rolling count', async () => runQueryTest({ @@ -1721,7 +1721,7 @@ describe('SQL Generation', () => { ])); it( - 'contains filter', + 'contains filter 1', () => runQueryTest({ measures: [], dimensions: [ @@ -2348,7 +2348,7 @@ describe('SQL Generation', () => { ] )); - it('rank measure', async () => runQueryTest( + it('rank measure 1', async () => runQueryTest( { measures: ['visitors.revenue_rank'], }, diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts index 3159fab098c41..1b1f28b9d1817 100644 --- a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts @@ -1596,7 +1596,7 @@ describe('SQL Generation', () => { sql: 'product_id', type: 'avg', filters: [ - { sql: `{FILTER_PARAMS.Order.category.filter('category')}` } + { sql: '{FILTER_PARAMS.Order.category.filter(\'category\')}' } ] } ], @@ -1613,7 +1613,7 @@ describe('SQL Generation', () => { }, { name: 'proxied', - sql: `{FILTER_PARAMS.Order.type.filter("x => type = 'online'")}`, + sql: '{FILTER_PARAMS.Order.type.filter("x => type = \'online\'")}', type: 'boolean', } ] diff --git a/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts index cb6d653b4b23c..641f2aea699d5 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts @@ -217,8 +217,8 @@ describe('pre-aggregations', () => { console.log(JSON.stringify(preAggregationsDescription, null, 2)); expect(preAggregationsDescription.length).toEqual(2); - expect(preAggregationsDescription[0].preAggregationId).toEqual("Orders.simple1"); - expect(preAggregationsDescription[1].preAggregationId).toEqual("Orders.simple2"); + expect(preAggregationsDescription[0].preAggregationId).toEqual('Orders.simple1'); + expect(preAggregationsDescription[1].preAggregationId).toEqual('Orders.simple2'); }); // @link https://github.com/cube-js/cube/issues/6623 diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs index f08447d48c81f..2e0571cfa0fb5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs @@ -60,6 +60,7 @@ pub struct BaseQueryOptionsStatic { #[serde(rename = "rowLimit")] pub row_limit: Option, pub offset: Option, + pub ungrouped: Option, } #[nativebridge::native_bridge(BaseQueryOptionsStatic)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs index d5aef3c6d812d..ded10dda2e5d4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs @@ -38,4 +38,11 @@ pub trait BaseTools { fn filter_group_function(&self) -> Result, CubeError>; fn timestamp_precision(&self) -> Result; fn in_db_time_zone(&self, date: String) -> Result; + fn generate_time_series( + &self, + granularity: String, + date_range: Vec, + ) -> Result>, CubeError>; + fn get_allocated_params(&self) -> Result, CubeError>; + fn all_cube_members(&self, path: String) -> Result, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs index 5b23f4e971e4e..4c1cdb21f2596 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs @@ -21,6 +21,13 @@ pub struct TimeShiftReference { pub time_dimension: String, } +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +pub struct RollingWindow { + pub trailing: Option, + pub leading: Option, + pub offset: Option, +} + #[derive(Serialize, Deserialize, Debug)] pub struct MeasureDefinitionStatic { #[serde(rename = "type")] @@ -37,6 +44,8 @@ pub struct MeasureDefinitionStatic { pub group_by_references: Option>, #[serde(rename = "timeShiftReferences")] pub time_shift_references: Option>, + #[serde(rename = "rollingWindow")] + pub rolling_window: Option, } #[nativebridge::native_bridge(MeasureDefinitionStatic)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/memeber_sql.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/memeber_sql.rs index bb18007581a10..3dbf3a967f713 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/memeber_sql.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/memeber_sql.rs @@ -19,7 +19,7 @@ use std::rc::Rc; pub struct MemberSqlStruct { pub sql_fn: Option, pub to_string_fn: Option, - pub properties: HashMap, + pub properties: HashMap, } pub enum ContextSymbolArg { @@ -64,6 +64,39 @@ impl NativeSerialize for MemberSqlStruct { } } +impl NativeSerialize for MemberSqlArg { + fn to_native( + &self, + context_holder: NativeContextHolder, + ) -> Result, CubeError> { + let res = match self { + MemberSqlArg::String(s) => s.to_native(context_holder.clone()), + MemberSqlArg::Struct(s) => s.to_native(context_holder.clone()), + MemberSqlArg::ContextSymbol(symbol) => match symbol { + ContextSymbolArg::SecurityContext(context) => context + .clone() + .as_any() + .downcast::>() + .unwrap() + .to_native(context_holder.clone()), + ContextSymbolArg::FilterParams(params) => params + .clone() + .as_any() + .downcast::>() + .unwrap() + .to_native(context_holder.clone()), + ContextSymbolArg::FilterGroup(group) => group + .clone() + .as_any() + .downcast::>() + .unwrap() + .to_native(context_holder.clone()), + }, + }?; + Ok(NativeObjectHandle::new(res.into_object())) + } +} + impl NativeMemberSql { pub fn try_new(native_object: NativeObjectHandle) -> Result { let args_names = native_object.to_function()?.args_names()?; @@ -89,27 +122,7 @@ impl MemberSql for NativeMemberSql { let context_holder = NativeContextHolder::::new(self.native_object.get_context()); let native_args = args .into_iter() - .map(|a| match a { - MemberSqlArg::String(s) => s.to_native(context_holder.clone()), - MemberSqlArg::Struct(s) => s.to_native(context_holder.clone()), - MemberSqlArg::ContextSymbol(symbol) => match symbol { - ContextSymbolArg::SecurityContext(context) => context - .as_any() - .downcast::>() - .unwrap() - .to_native(context_holder.clone()), - ContextSymbolArg::FilterParams(params) => params - .as_any() - .downcast::>() - .unwrap() - .to_native(context_holder.clone()), - ContextSymbolArg::FilterGroup(group) => group - .as_any() - .downcast::>() - .unwrap() - .to_native(context_holder.clone()), - }, - }) + .map(|a| a.to_native(context_holder.clone())) .collect::, _>>()?; let res = self.native_object.to_function()?.call(native_args)?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs index cf5c00864b6ff..4e143cb86f703 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs @@ -1,27 +1,29 @@ use crate::plan::{ - AliasedExpr, Cte, Expr, Filter, From, MemberExpression, OrderBy, Schema, Select, + AliasedExpr, Cte, Expr, Filter, From, MemberExpression, OrderBy, Schema, SchemaColumn, Select, + SingleAliasedSource, SingleSource, }; + +use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; use crate::planner::{BaseMember, VisitorContext}; +use std::collections::HashMap; use std::rc::Rc; pub struct SelectBuilder { projection_columns: Vec, - from: From, + from: Rc, filter: Option, group_by: Vec, having: Option, order_by: Vec, - context: Rc, ctes: Vec>, is_distinct: bool, limit: Option, offset: Option, - input_schema: Rc, + result_schema: Schema, } impl SelectBuilder { - pub fn new(from: From, context: VisitorContext) -> Self { - let input_schema = from.schema.clone(); + pub fn new(from: Rc) -> Self { Self { projection_columns: vec![], from, @@ -29,33 +31,30 @@ impl SelectBuilder { group_by: vec![], having: None, order_by: vec![], - context: Rc::new(context), ctes: vec![], is_distinct: false, limit: None, offset: None, - input_schema, + result_schema: Schema::empty(), } } - pub fn add_projection_member( - &mut self, - member: &Rc, - source: Option, - alias: Option, - ) { + pub fn add_projection_member(&mut self, member: &Rc, alias: Option) { let alias = if let Some(alias) = alias { alias } else { - self.input_schema.resolve_member_alias(&member, &source) + member.alias_name() }; - let expr = Expr::Member(MemberExpression::new(member.clone(), source)); + + let expr = Expr::Member(MemberExpression::new(member.clone())); let aliased_expr = AliasedExpr { expr, alias: alias.clone(), }; self.projection_columns.push(aliased_expr); + self.result_schema + .add_column(SchemaColumn::new(alias.clone(), Some(member.full_name()))); } pub fn set_filter(&mut self, filter: Option) { @@ -85,11 +84,65 @@ impl SelectBuilder { pub fn set_offset(&mut self, offset: Option) { self.offset = offset; } + pub fn set_ctes(&mut self, ctes: Vec>) { self.ctes = ctes; } - pub fn build(self) -> Select { + fn make_cube_references(&self) -> HashMap { + let mut refs = HashMap::new(); + match &self.from.source { + crate::plan::FromSource::Single(source) => { + self.add_cube_reference_if_needed(source, &mut refs) + } + crate::plan::FromSource::Join(join) => { + self.add_cube_reference_if_needed(&join.root, &mut refs); + for join_item in join.joins.iter() { + self.add_cube_reference_if_needed(&join_item.from, &mut refs); + } + } + crate::plan::FromSource::Empty => {} + } + refs + } + + fn add_cube_reference_if_needed( + &self, + source: &SingleAliasedSource, + refs: &mut HashMap, + ) { + match &source.source { + SingleSource::Cube(cube) => { + refs.insert(cube.name().clone(), source.alias.clone()); + } + _ => {} + } + } + + fn make_asteriks_schema(&self) -> Rc { + let schema = match &self.from.source { + crate::plan::FromSource::Empty => Rc::new(Schema::empty()), + crate::plan::FromSource::Single(source) => source.source.schema(), + crate::plan::FromSource::Join(join) => { + let mut schema = Schema::empty(); + schema.merge(join.root.source.schema().as_ref()); + for itm in join.joins.iter() { + schema.merge(itm.from.source.schema().as_ref()) + } + Rc::new(schema) + } + }; + schema + } + + pub fn build(self, mut nodes_factory: SqlNodesFactory) -> Select { + let cube_references = self.make_cube_references(); + nodes_factory.set_cube_name_references(cube_references); + let schema = if self.projection_columns.is_empty() { + self.make_asteriks_schema() + } else { + Rc::new(self.result_schema) + }; Select { projection_columns: self.projection_columns, from: self.from, @@ -97,11 +150,12 @@ impl SelectBuilder { group_by: self.group_by, having: self.having, order_by: self.order_by, - context: self.context.clone(), + context: Rc::new(VisitorContext::new(&nodes_factory)), ctes: self.ctes, is_distinct: self.is_distinct, limit: self.limit, offset: self.offset, + schema, } } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/cte.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/cte.rs index dba7dd3a9c7a3..4d3ee55a6fceb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/cte.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/cte.rs @@ -1,4 +1,4 @@ -use super::{QueryPlan, Schema, Select}; +use super::{QueryPlan, Select}; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; @@ -22,10 +22,6 @@ impl Cte { } } - pub fn make_schema(&self) -> Schema { - self.query.make_schema(Some(self.name().clone())) - } - pub fn query(&self) -> &Rc { &self.query } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/expression.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/expression.rs index 69e99cbcd2a30..be13ad59816e7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/expression.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/expression.rs @@ -1,4 +1,4 @@ -use super::Schema; +use super::QualifiedColumnName; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::{BaseMember, VisitorContext}; use cubenativeutils::CubeError; @@ -7,44 +7,45 @@ use std::rc::Rc; #[derive(Clone)] pub struct MemberExpression { pub member: Rc, - pub source: Option, } impl MemberExpression { - pub fn new(member: Rc, source: Option) -> Self { - Self { member, source } + pub fn new(member: Rc) -> Self { + Self { member } } pub fn to_sql( &self, - templates: &PlanSqlTemplates, + _templates: &PlanSqlTemplates, context: Rc, - schema: Rc, ) -> Result { - if let Some(reference_column) = - schema.find_column_for_member(&self.member.full_name(), &self.source) - { - templates.column_reference(&reference_column.table_name, &reference_column.alias) - } else { - self.member.to_sql(context, schema) - } + self.member.to_sql(context) } } #[derive(Clone)] pub enum Expr { Member(MemberExpression), + Reference(QualifiedColumnName), } impl Expr { + pub fn new_member(member: Rc) -> Self { + Self::Member(MemberExpression::new(member)) + } + pub fn new_reference(source: Option, reference: String) -> Self { + Self::Reference(QualifiedColumnName::new(source, reference)) + } pub fn to_sql( &self, templates: &PlanSqlTemplates, context: Rc, - schema: Rc, ) -> Result { match self { - Expr::Member(member) => member.to_sql(templates, context, schema), + Self::Member(member) => member.to_sql(templates, context), + Self::Reference(reference) => { + templates.column_reference(reference.source(), &reference.name()) + } } } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs index f118aefe5d8e5..b7e59049cb373 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs @@ -1,4 +1,3 @@ -use super::Schema; use crate::planner::filter::BaseFilter; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::VisitorContext; @@ -6,7 +5,7 @@ use cubenativeutils::CubeError; use std::fmt; use std::rc::Rc; -#[derive(Clone)] +#[derive(Clone, PartialEq)] pub enum FilterGroupOperator { Or, And, @@ -18,13 +17,19 @@ pub struct FilterGroup { pub items: Vec, } +impl PartialEq for FilterGroup { + fn eq(&self, other: &Self) -> bool { + self.operator == other.operator && self.items == other.items + } +} + impl FilterGroup { pub fn new(operator: FilterGroupOperator, items: Vec) -> Self { Self { operator, items } } } -#[derive(Clone)] +#[derive(Clone, PartialEq)] pub enum FilterItem { Group(Rc), Item(Rc), @@ -48,7 +53,6 @@ impl FilterItem { &self, templates: &PlanSqlTemplates, context: Rc, - schema: Rc, ) -> Result { let res = match self { FilterItem::Group(group) => { @@ -56,17 +60,20 @@ impl FilterItem { let items_sql = group .items .iter() - .map(|itm| itm.to_sql(templates, context.clone(), schema.clone())) - .collect::, _>>()?; - let result = if items_sql.is_empty() { - templates.always_true()? + .map(|itm| itm.to_sql(templates, context.clone())) + .collect::, _>>()? + .into_iter() + .filter(|itm| !itm.is_empty()) + .collect::>(); + if items_sql.is_empty() { + "".to_string() } else { - items_sql.join(&operator) - }; - format!("({})", result) + let result = items_sql.join(&operator); + format!("({})", result) + } } FilterItem::Item(item) => { - let sql = item.to_sql(context.clone(), schema)?; + let sql = item.to_sql(context.clone())?; format!("({})", sql) } }; @@ -79,12 +86,11 @@ impl Filter { &self, templates: &PlanSqlTemplates, context: Rc, - schema: Rc, ) -> Result { let res = self .items .iter() - .map(|itm| itm.to_sql(templates, context.clone(), schema.clone())) + .map(|itm| itm.to_sql(templates, context.clone())) .collect::, _>>()? .join(" AND "); Ok(res) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs index 7da4fb0dbcf39..175fb662ca89b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/from.rs @@ -1,4 +1,4 @@ -use super::{Join, QueryPlan, Schema, SchemaCube, Select}; +use super::{Join, QueryPlan, Schema, Select}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::{BaseCube, VisitorContext}; use cubenativeutils::CubeError; @@ -27,6 +27,14 @@ impl SingleSource { }; Ok(sql) } + + pub fn schema(&self) -> Rc { + match self { + SingleSource::Subquery(subquery) => subquery.schema(), + SingleSource::Cube(_) => Rc::new(Schema::empty()), + SingleSource::TableReference(_, schema) => schema.clone(), + } + } } #[derive(Clone)] @@ -72,18 +80,6 @@ impl SingleAliasedSource { templates.query_aliased(&sql, &self.alias) } - - pub fn make_schema(&self) -> Schema { - match &self.source { - SingleSource::Subquery(query) => query.make_schema(Some(self.alias.clone())), - SingleSource::Cube(cube) => { - let mut schema = Schema::empty(); - schema.add_cube(SchemaCube::new(cube.name().clone(), self.alias.clone())); - schema - } - SingleSource::TableReference(_, schema) => schema.move_to_source(&self.alias), - } - } } #[derive(Clone)] @@ -93,30 +89,17 @@ pub enum FromSource { Join(Rc), } -impl FromSource { - pub fn get_schema(&self) -> Rc { - let schema = match self { - FromSource::Empty => Schema::empty(), - FromSource::Single(source) => source.make_schema(), - FromSource::Join(join) => join.make_schema(), - }; - Rc::new(schema) - } -} - #[derive(Clone)] pub struct From { pub source: FromSource, - pub schema: Rc, } impl From { - pub fn new(source: FromSource) -> Self { - let schema = source.get_schema(); - Self { source, schema } + pub fn new(source: FromSource) -> Rc { + Rc::new(Self { source }) } - pub fn new_from_cube(cube: Rc, alias: Option) -> Self { + pub fn new_from_cube(cube: Rc, alias: Option) -> Rc { Self::new(FromSource::Single(SingleAliasedSource::new_from_cube( cube, alias, ))) @@ -126,23 +109,23 @@ impl From { reference: String, schema: Rc, alias: Option, - ) -> Self { + ) -> Rc { Self::new(FromSource::Single( SingleAliasedSource::new_from_table_reference(reference, schema, alias), )) } - pub fn new_from_join(join: Rc) -> Self { + pub fn new_from_join(join: Rc) -> Rc { Self::new(FromSource::Join(join)) } - pub fn new_from_subquery(plan: Rc, alias: String) -> Self { + pub fn new_from_subquery(plan: Rc, alias: String) -> Rc { Self::new(FromSource::Single(SingleAliasedSource::new_from_subquery( plan, alias, ))) } - pub fn new_from_subselect(plan: Rc, alias: String) -> Rc { Self::new(FromSource::Single(SingleAliasedSource::new_from_subquery( Rc::new(QueryPlan::Select(plan)), alias, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index 19e8283bc5e68..3a19170443d80 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -1,28 +1,101 @@ -use super::{Schema, SingleAliasedSource}; +use super::{Expr, SingleAliasedSource}; use crate::planner::sql_templates::PlanSqlTemplates; -use crate::planner::{BaseJoinCondition, BaseMember, VisitorContext}; +use crate::planner::{BaseJoinCondition, VisitorContext}; use cubenativeutils::CubeError; +use lazy_static::lazy_static; use std::rc::Rc; +pub struct RollingWindowJoinCondition { + time_series_source: String, + trailing_interval: Option, + leading_interval: Option, + offset: String, + time_dimension: Expr, +} + +impl RollingWindowJoinCondition { + pub fn new( + time_series_source: String, + trailing_interval: Option, + leading_interval: Option, + offset: String, + time_dimension: Expr, + ) -> Self { + Self { + time_series_source, + trailing_interval, + leading_interval, + offset, + time_dimension, + } + } + + pub fn to_sql( + &self, + templates: &PlanSqlTemplates, + context: Rc, + ) -> Result { + let mut conditions = vec![]; + let date_column = self.time_dimension.to_sql(templates, context)?; + + lazy_static! { + static ref UNBOUNDED: Option = Some("unbounded".to_string()); + } + + if self.trailing_interval != *UNBOUNDED { + let start_date = if self.offset == "start" { + templates.column_reference(&Some(self.time_series_source.clone()), "date_from")? + } else { + templates.column_reference(&Some(self.time_series_source.clone()), "date_to")? + }; + + let trailing_start = if let Some(trailing_interval) = &self.trailing_interval { + format!("{start_date} - interval '{trailing_interval}'") + } else { + start_date + }; + + let sign = if self.offset == "start" { ">=" } else { ">" }; + + conditions.push(format!("{date_column} {sign} {trailing_start}")); + } + + if self.leading_interval != *UNBOUNDED { + let end_date = if self.offset == "end" { + templates.column_reference(&Some(self.time_series_source.clone()), "date_to")? + } else { + templates.column_reference(&Some(self.time_series_source.clone()), "date_from")? + }; + + let leading_end = if let Some(leading_interval) = &self.leading_interval { + format!("{end_date} + interval '{leading_interval}'") + } else { + end_date + }; + + let sign = if self.offset == "end" { "<=" } else { "<" }; + + conditions.push(format!("{date_column} {sign} {leading_end}")); + } + let result = if conditions.is_empty() { + templates.always_true()? + } else { + conditions.join(" AND ") + }; + Ok(result) + } +} + pub struct DimensionJoinCondition { - left_source: String, - right_source: String, - dimensions: Vec>, + conditions: Vec<(Expr, Expr)>, null_check: bool, } impl DimensionJoinCondition { - pub fn new( - left_source: String, - right_source: String, - dimensions: Vec>, - null_check: bool, - ) -> Self { + pub fn new(conditions: Vec<(Expr, Expr)>, null_check: bool) -> Self { Self { - left_source, - right_source, - dimensions, + conditions, null_check, } } @@ -31,15 +104,14 @@ impl DimensionJoinCondition { &self, templates: &PlanSqlTemplates, context: Rc, - schema: Rc, ) -> Result { - let result = if self.dimensions.is_empty() { + let result = if self.conditions.is_empty() { format!("1 = 1") } else { - self.dimensions + self.conditions .iter() - .map(|dim| -> Result { - self.dimension_condition(templates, context.clone(), dim, schema.clone()) + .map(|(left, right)| -> Result { + self.dimension_condition(templates, context.clone(), left, right) }) .collect::, _>>()? .join(" AND ") @@ -51,61 +123,39 @@ impl DimensionJoinCondition { &self, templates: &PlanSqlTemplates, context: Rc, - dimension: &Rc, - schema: Rc, - ) -> Result { - let left_column = self.resolve_member_alias( - templates, - context.clone(), - &self.left_source, - dimension, - schema.clone(), - )?; - let right_column = self.resolve_member_alias( - templates, - context.clone(), - &self.right_source, - dimension, - schema.clone(), - )?; - templates.join_by_dimension_conditions(&left_column, &right_column, self.null_check) - } - - fn resolve_member_alias( - &self, - templates: &PlanSqlTemplates, - context: Rc, - source: &String, - dimension: &Rc, - schema: Rc, + left_expr: &Expr, + right_expr: &Expr, ) -> Result { - let schema = schema.extract_source_schema(source); - let source = Some(source.clone()); - if let Some(column) = schema.find_column_for_member(&dimension.full_name(), &source) { - templates.column_reference(&source, &column.alias.clone()) - } else { - dimension.to_sql(context.clone(), schema.clone()) - } + let left_sql = left_expr.to_sql(templates, context.clone())?; + let right_sql = right_expr.to_sql(templates, context.clone())?; + templates.join_by_dimension_conditions(&left_sql, &right_sql, self.null_check) } } pub enum JoinCondition { DimensionJoinCondition(DimensionJoinCondition), BaseJoinCondition(Rc), + RollingWindowJoinCondition(RollingWindowJoinCondition), } impl JoinCondition { - pub fn new_dimension_join( - left_source: String, - right_source: String, - dimensions: Vec>, - null_check: bool, + pub fn new_dimension_join(conditions: Vec<(Expr, Expr)>, null_check: bool) -> Self { + Self::DimensionJoinCondition(DimensionJoinCondition::new(conditions, null_check)) + } + + pub fn new_rolling_join( + time_series_source: String, + trailing_interval: Option, + leading_interval: Option, + offset: String, + time_dimension: Expr, ) -> Self { - Self::DimensionJoinCondition(DimensionJoinCondition::new( - left_source, - right_source, - dimensions, - null_check, + Self::RollingWindowJoinCondition(RollingWindowJoinCondition::new( + time_series_source, + trailing_interval, + leading_interval, + offset, + time_dimension, )) } @@ -117,11 +167,11 @@ impl JoinCondition { &self, templates: &PlanSqlTemplates, context: Rc, - schema: Rc, ) -> Result { match &self { - JoinCondition::DimensionJoinCondition(cond) => cond.to_sql(templates, context, schema), - JoinCondition::BaseJoinCondition(cond) => cond.to_sql(context, schema), + JoinCondition::DimensionJoinCondition(cond) => cond.to_sql(templates, context), + JoinCondition::BaseJoinCondition(cond) => cond.to_sql(context), + JoinCondition::RollingWindowJoinCondition(cond) => cond.to_sql(templates, context), } } } @@ -142,9 +192,8 @@ impl JoinItem { &self, templates: &PlanSqlTemplates, context: Rc, - schema: Rc, ) -> Result { - let on_sql = self.on.to_sql(templates, context.clone(), schema)?; + let on_sql = self.on.to_sql(templates, context.clone())?; let result = templates.join( &self.from.to_sql(templates, context)?, &on_sql, @@ -155,24 +204,15 @@ impl JoinItem { } impl Join { - pub fn make_schema(&self) -> Schema { - let mut schema = self.root.make_schema(); - for itm in self.joins.iter() { - schema.merge(itm.from.make_schema()); - } - schema - } - pub fn to_sql( &self, templates: &PlanSqlTemplates, context: Rc, ) -> Result { - let schema = Rc::new(self.make_schema()); let joins_sql = self .joins .iter() - .map(|j| j.to_sql(templates, context.clone(), schema.clone())) + .map(|j| j.to_sql(templates, context.clone())) .collect::, _>>()?; let res = format!( "{}\n{}", diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs index 5d98909183278..3799935638bd2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs @@ -8,6 +8,7 @@ pub mod order; pub mod query_plan; pub mod schema; pub mod select; +pub mod time_series; pub mod union; pub use builder::{JoinBuilder, SelectBuilder}; @@ -15,9 +16,10 @@ pub use cte::Cte; pub use expression::{Expr, MemberExpression}; pub use filter::{Filter, FilterGroup, FilterItem}; pub use from::{From, FromSource, SingleAliasedSource, SingleSource}; -pub use join::{Join, JoinCondition, JoinItem}; +pub use join::{Join, JoinCondition, JoinItem, RollingWindowJoinCondition}; pub use order::OrderBy; pub use query_plan::QueryPlan; -pub use schema::{Schema, SchemaColumn, SchemaCube}; +pub use schema::{QualifiedColumnName, Schema, SchemaColumn}; pub use select::{AliasedExpr, Select}; +pub use time_series::TimeSeries; pub use union::Union; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs index e8224fb74aebf..8957321274111 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/query_plan.rs @@ -1,4 +1,4 @@ -use super::{Schema, Select, Union}; +use super::{Schema, Select, TimeSeries, Union}; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; use std::rc::Rc; @@ -6,19 +6,22 @@ use std::rc::Rc; pub enum QueryPlan { Select(Rc { let schema = cte_schemas.get(alias).unwrap().clone(); - let select_builder = SelectBuilder::new( - From::new_from_table_reference(alias.clone(), schema, None), - VisitorContext::default(SqlNodesFactory::new()), + let select_builder = + SelectBuilder::new(From::new_from_table_reference(alias.clone(), schema, None)); + + Rc::new(select_builder.build(SqlNodesFactory::new())) + } + + fn create_multi_stage_inode_member( + &self, + base_member: Rc, + ) -> Result { + let inode = if let Some(measure) = + BaseMeasure::try_new(base_member.clone(), self.query_tools.clone())? + { + let member_type = if measure.measure_type() == "rank" { + MultiStageInodeMemberType::Rank + } else if !measure.is_calculated() { + MultiStageInodeMemberType::Aggregate + } else { + MultiStageInodeMemberType::Calculate + }; + + let time_shifts = if let Some(refs) = measure.time_shift_references() { + let time_shifts = refs + .iter() + .map(|r| MultiStageTimeShift::try_from_reference(r)) + .collect::, _>>()?; + time_shifts + } else { + vec![] + }; + let is_ungrupped = match &member_type { + MultiStageInodeMemberType::Rank | MultiStageInodeMemberType::Calculate => true, + _ => false, + }; + MultiStageInodeMember::new( + member_type, + measure.reduce_by().clone().unwrap_or_default(), + measure.add_group_by().clone().unwrap_or_default(), + measure.group_by().clone(), + time_shifts, + is_ungrupped, + ) + } else { + MultiStageInodeMember::new( + MultiStageInodeMemberType::Calculate, + vec![], + vec![], + None, + vec![], + false, + ) + }; + Ok(inode) + } + + fn add_time_series( + &self, + time_dimension: Rc, + state: Rc, + descriptions: &mut Vec>, + ) -> Result, CubeError> { + let description = + if let Some(description) = descriptions.iter().find(|d| d.alias() == "time_series") { + description.clone() + } else { + let time_series_node = MultiStageQueryDescription::new( + MultiStageMember::new( + MultiStageMemberType::Leaf(MultiStageLeafMemberType::TimeSeries( + time_dimension.clone(), + )), + time_dimension.member_evaluator(), + ), + state.clone(), + vec![], + "time_series".to_string(), + ); + descriptions.push(time_series_node.clone()); + time_series_node + }; + Ok(description) + } + + fn add_rolling_window_base( + &self, + member: Rc, + state: Rc, + descriptions: &mut Vec>, + ) -> Result, CubeError> { + let alias = format!("cte_{}", descriptions.len()); + let description = MultiStageQueryDescription::new( + MultiStageMember::new( + MultiStageMemberType::Leaf(MultiStageLeafMemberType::Measure), + member, + ), + state, + vec![], + alias.clone(), ); + descriptions.push(description.clone()); + Ok(description) + } + + fn make_rolling_base_state( + &self, + time_dimension: Rc, + rolling_window: &RollingWindow, + state: Rc, + ) -> Result, CubeError> { + let time_dimension_name = time_dimension.member_evaluator().full_name(); + let mut new_state = state.clone_state(); + let trailing_granularity = + GranularityHelper::granularity_from_interval(&rolling_window.trailing); + let leading_granularity = + GranularityHelper::granularity_from_interval(&rolling_window.leading); + let window_granularity = + GranularityHelper::min_granularity(&trailing_granularity, &leading_granularity)?; + let result_granularity = GranularityHelper::min_granularity( + &window_granularity, + &time_dimension.get_granularity(), + )?; + + new_state.change_time_dimension_granularity(&time_dimension_name, result_granularity); + + new_state.expand_date_range_filter( + &time_dimension_name, + rolling_window.trailing.clone(), + rolling_window.leading.clone(), + ); + + Ok(Rc::new(new_state)) + } - Rc::new(select_builder.build()) + fn try_make_rolling_window( + &self, + member: Rc, + state: Rc, + descriptions: &mut Vec>, + ) -> Result>, CubeError> { + if let Some(measure) = BaseMeasure::try_new(member.clone(), self.query_tools.clone())? { + if measure.is_cumulative() { + let rolling_window = if let Some(rolling_window) = measure.rolling_window() { + rolling_window.clone() + } else { + RollingWindow { + trailing: Some("unbounded".to_string()), + leading: None, + offset: None, + } + }; + let time_dimensions = self.query_properties.time_dimensions(); + if time_dimensions.len() == 0 { + let rolling_base = + self.add_rolling_window_base(member.clone(), state.clone(), descriptions)?; + return Ok(Some(rolling_base)); + } + if time_dimensions.len() != 1 { + return Err(CubeError::internal( + "Rolling window requires one time dimension".to_string(), + )); + } + let time_dimension = time_dimensions[0].clone(); + + let input = vec![ + self.add_time_series(time_dimension.clone(), state.clone(), descriptions)?, + self.add_rolling_window_base( + member.clone(), + self.make_rolling_base_state( + time_dimension.clone(), + &rolling_window, + state.clone(), + )?, + descriptions, + )?, + ]; + + let time_dimension = time_dimensions[0].clone(); + + let alias = format!("cte_{}", descriptions.len()); + + let rolling_window_descr = RollingWindowDescription { + time_dimension: time_dimension.clone(), + trailing: rolling_window.trailing.clone(), + leading: rolling_window.leading.clone(), + offset: rolling_window.offset.clone().unwrap_or("end".to_string()), + }; + + let inode_member = MultiStageInodeMember::new( + MultiStageInodeMemberType::RollingWindow(rolling_window_descr), + vec![], + vec![], + None, + vec![], + false, + ); + + let description = MultiStageQueryDescription::new( + MultiStageMember::new(MultiStageMemberType::Inode(inode_member), member), + state.clone(), + input, + alias.clone(), + ); + descriptions.push(description.clone()); + Ok(Some(description)) + } else { + Ok(None) + } + } else { + Ok(None) + } } fn make_queries_descriptions( &self, - member: Rc, + member: Rc, state: Rc, descriptions: &mut Vec>, ) -> Result, CubeError> { @@ -113,54 +322,77 @@ impl MultiStageQueryPlanner { return Ok(exists.clone()); }; - let (dimensions_to_add, time_shifts) = if let Some(measure) = - BaseMeasure::try_new(member.clone(), self.query_tools.clone())? + if let Some(rolling_window_query) = + self.try_make_rolling_window(member.clone(), state.clone(), descriptions)? { - let dimensions_to_add = if let Some(add_group_by) = measure.add_group_by() { - add_group_by - .iter() - .map(|name| self.compile_dimension(name)) - .collect::, _>>()? - } else { - vec![] - }; + return Ok(rolling_window_query); + } - (dimensions_to_add, measure.time_shifts().clone()) - } else { - (vec![], vec![]) - }; + let childs = member_childs(&member)?; - let new_state = if !dimensions_to_add.is_empty() - || !time_shifts.is_empty() - || state.is_filter_allowed(&member_name) - { - let mut new_state = state.clone_state(); - if !dimensions_to_add.is_empty() { - new_state.add_dimensions(dimensions_to_add); - } - if !time_shifts.is_empty() { - new_state.add_time_shifts(time_shifts); - } - if state.is_filter_allowed(&member_name) { - new_state.disallow_filter(&member_name); + let description = if childs.is_empty() { + if has_multi_stage_members(&member, false)? { + return Err(CubeError::internal(format!( + "Leaf multi stage query cannot contain multi stage member" + ))); } - Rc::new(new_state) + + let alias = format!("cte_{}", descriptions.len()); + MultiStageQueryDescription::new( + MultiStageMember::new( + MultiStageMemberType::Leaf(MultiStageLeafMemberType::Measure), + member.clone(), + ), + state.clone(), + vec![], + alias.clone(), + ) } else { - state.clone() - }; + let multi_stage_member = self.create_multi_stage_inode_member(member.clone())?; - let childs = member_childs(&member)?; - let input = childs - .into_iter() - .map( - |child| -> Result, CubeError> { - self.make_queries_descriptions(child, new_state.clone(), descriptions) - }, + let dimensions_to_add = multi_stage_member + .add_group_by() + .iter() + .map(|name| self.compile_dimension(name)) + .collect::, _>>()?; + + let new_state = if !dimensions_to_add.is_empty() + || !multi_stage_member.time_shifts().is_empty() + || state.has_filters_for_member(&member_name) + { + let mut new_state = state.clone_state(); + if !dimensions_to_add.is_empty() { + new_state.add_dimensions(dimensions_to_add); + } + if !multi_stage_member.time_shifts().is_empty() { + new_state.add_time_shifts(multi_stage_member.time_shifts().clone()); + } + if state.has_filters_for_member(&member_name) { + new_state.remove_filter_for_member(&member_name); + } + Rc::new(new_state) + } else { + state.clone() + }; + + let input = childs + .into_iter() + .map( + |child| -> Result, CubeError> { + self.make_queries_descriptions(child, new_state.clone(), descriptions) + }, + ) + .collect::, _>>()?; + + let alias = format!("cte_{}", descriptions.len()); + MultiStageQueryDescription::new( + MultiStageMember::new(MultiStageMemberType::Inode(multi_stage_member), member), + state.clone(), + input, + alias.clone(), ) - .collect::, _>>()?; - let alias = format!("cte_{}", descriptions.len()); - let description = - MultiStageQueryDescription::new(member, state.clone(), input, alias.clone()); + }; + descriptions.push(description.clone()); Ok(description) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs index 6ac49c628cea6..1d1152d276e91 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs @@ -1,28 +1,36 @@ use super::{CommonUtils, JoinPlanner}; -use crate::plan::{From, JoinBuilder, JoinCondition, Select, SelectBuilder}; +use crate::plan::{ + Expr, From, JoinBuilder, JoinCondition, MemberExpression, QualifiedColumnName, Select, + SelectBuilder, +}; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::collectors::{ + collect_cube_names, collect_join_hints, collect_join_hints_for_measures, +}; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; -use crate::planner::BaseMember; -use crate::planner::QueryProperties; -use crate::planner::{BaseMeasure, VisitorContext}; +use crate::planner::sql_evaluator::ReferencesBuilder; +use crate::planner::{BaseMeasure, BaseMember, BaseMemberHelper, QueryProperties}; use cubenativeutils::CubeError; use itertools::Itertools; +use std::collections::HashMap; use std::rc::Rc; pub struct MultipliedMeasuresQueryPlanner { + query_tools: Rc, query_properties: Rc, join_planner: JoinPlanner, common_utils: CommonUtils, - context_factory: Rc, + context_factory: SqlNodesFactory, } impl MultipliedMeasuresQueryPlanner { pub fn new( query_tools: Rc, query_properties: Rc, - context_factory: Rc, + context_factory: SqlNodesFactory, ) -> Self { Self { + query_tools: query_tools.clone(), join_planner: JoinPlanner::new(query_tools.clone()), common_utils: CommonUtils::new(query_tools.clone()), query_properties, @@ -66,40 +74,157 @@ impl MultipliedMeasuresQueryPlanner { let primary_keys_dimensions = self.common_utils.primary_keys_dimensions(key_cube_name)?; let keys_query = self.key_query(&primary_keys_dimensions, key_cube_name)?; let keys_query_alias = format!("keys"); + let should_build_join_for_measure_select = + self.check_should_build_join_for_measure_select(measures, key_cube_name)?; let mut join_builder = - JoinBuilder::new_from_subselect(keys_query, keys_query_alias.clone()); + JoinBuilder::new_from_subselect(keys_query.clone(), keys_query_alias.clone()); let pk_cube = self.common_utils.cube_from_path(key_cube_name.clone())?; let pk_cube_alias = pk_cube.default_alias_with_prefix(&Some(format!("{key_cube_name}_key"))); - join_builder.left_join_cube( - pk_cube.clone(), - Some(pk_cube_alias.clone()), - JoinCondition::new_dimension_join( - keys_query_alias, - pk_cube_alias, - primary_keys_dimensions, - false, - ), - ); - - let mut select_builder = SelectBuilder::new( - From::new_from_join(join_builder.build()), - VisitorContext::new_with_cube_alias_prefix( - self.context_factory.clone(), - format!("{}_key", key_cube_name), - ), - ); + let mut ungrouped_measure_references = HashMap::new(); + if should_build_join_for_measure_select { + let subquery = self.aggregate_subquery_measure_join( + key_cube_name, + &measures, + &primary_keys_dimensions, + )?; + + let conditions = primary_keys_dimensions + .iter() + .map(|dim| { + let alias_in_keys_query = keys_query.schema().resolve_member_alias(dim); + let keys_query_ref = Expr::Reference(QualifiedColumnName::new( + Some(keys_query_alias.clone()), + alias_in_keys_query, + )); + let alias_in_subquery = subquery.schema().resolve_member_alias(dim); + let subquery_ref = Expr::Reference(QualifiedColumnName::new( + Some(pk_cube_alias.clone()), + alias_in_subquery, + )); + (keys_query_ref, subquery_ref) + }) + .collect_vec(); + + for meas in measures.iter() { + ungrouped_measure_references.insert( + meas.full_name(), + QualifiedColumnName::new( + Some(pk_cube_alias.clone()), + subquery + .schema() + .resolve_member_alias(&meas.clone().as_base_member()), + ), + ); + } + + join_builder.left_join_subselect( + subquery, + pk_cube_alias.clone(), + JoinCondition::new_dimension_join(conditions, false), + ); + } else { + let conditions = primary_keys_dimensions + .iter() + .map(|dim| { + let alias_in_keys_query = keys_query.schema().resolve_member_alias(dim); + let keys_query_ref = Expr::Reference(QualifiedColumnName::new( + Some(keys_query_alias.clone()), + alias_in_keys_query, + )); + let pk_cube_expr = Expr::Member(MemberExpression::new(dim.clone())); + (keys_query_ref, pk_cube_expr) + }) + .collect_vec(); + join_builder.left_join_cube( + pk_cube.clone(), + Some(pk_cube_alias.clone()), + JoinCondition::new_dimension_join(conditions, false), + ); + }; + + let from = From::new_from_join(join_builder.build()); + let references_builder = ReferencesBuilder::new(from.clone()); + let mut select_builder = SelectBuilder::new(from.clone()); + let mut render_references = HashMap::new(); for member in self .query_properties - .all_dimensions_and_measures(&measures)? + .all_dimensions_and_measures(&vec![])? .iter() { - select_builder.add_projection_member(member, None, None); + references_builder.resolve_references_for_member( + member.member_evaluator(), + &None, + &mut render_references, + )?; + let alias = references_builder.resolve_alias_for_member(&member.full_name(), &None); + select_builder.add_projection_member(member, alias); + } + for member in BaseMemberHelper::iter_as_base_member(&measures) { + let alias = if !should_build_join_for_measure_select { + references_builder.resolve_references_for_member( + member.member_evaluator(), + &None, + &mut render_references, + )?; + references_builder.resolve_alias_for_member(&member.full_name(), &None) + } else { + None + }; + select_builder.add_projection_member(&member, alias); } select_builder.set_group_by(self.query_properties.group_by()); - Ok(Rc::new(select_builder.build())) + let mut context_factory = self.context_factory.clone(); + context_factory.set_render_references(render_references); + context_factory.set_ungrouped_measure_references(ungrouped_measure_references); + Ok(Rc::new(select_builder.build(context_factory))) + } + + fn check_should_build_join_for_measure_select( + &self, + measures: &Vec>, + key_cube_name: &String, + ) -> Result { + for measure in measures.iter() { + let cubes = collect_cube_names(measure.member_evaluator())?; + let join_hints = collect_join_hints(measure.member_evaluator())?; + if cubes.iter().any(|cube| cube != key_cube_name) { + let measures_join = self.query_tools.join_graph().build_join(join_hints)?; + if *measures_join + .static_data() + .multiplication_factor + .get(key_cube_name) + .unwrap_or(&false) + { + return Err(CubeError::user(format!("{}' references cubes that lead to row multiplication. Please rewrite it using sub query.", measure.full_name()))); + } + return Ok(true); + } + } + Ok(false) + } + fn aggregate_subquery_measure_join( + &self, + _key_cube_name: &String, + measures: &Vec>, + primary_keys_dimensions: &Vec>, + ) -> Result, CubeError> { + let join_hints = collect_join_hints_for_measures(measures)?; + let from = self + .join_planner + .make_join_node_with_prefix_and_join_hints(&None, join_hints)?; + let mut context_factory = self.context_factory.clone(); + context_factory.set_ungrouped_measure(true); + let mut select_builder = SelectBuilder::new(from); + for dim in primary_keys_dimensions.iter() { + select_builder.add_projection_member(dim, None); + } + for meas in measures.iter() { + select_builder.add_projection_member(&meas.clone().as_base_member(), None); + } + Ok(Rc::new(select_builder.build(context_factory))) } fn regular_measures_subquery( @@ -109,23 +234,26 @@ impl MultipliedMeasuresQueryPlanner { let source = self .join_planner .make_join_node_with_prefix(&Some(format!("main")))?; - let mut select_builder = SelectBuilder::new( - source, - VisitorContext::new_with_cube_alias_prefix( - self.context_factory.clone(), - "main".to_string(), - ), - ); + + let mut select_builder = SelectBuilder::new(source.clone()); + let mut context_factory = self.context_factory.clone(); + for time_dim in self.query_properties.time_dimensions() { + if let Some(granularity) = time_dim.get_granularity() { + context_factory.add_leaf_time_dimension(&time_dim.full_name(), &granularity); + } + } + for member in self .query_properties .all_dimensions_and_measures(&measures)? .iter() { - select_builder.add_projection_member(member, None, None); + select_builder.add_projection_member(member, None); } - select_builder.set_filter(self.query_properties.all_filters()); + let filter = self.query_properties.all_filters(); + select_builder.set_filter(filter); select_builder.set_group_by(self.query_properties.group_by()); - Ok(Rc::new(select_builder.build())) + Ok(Rc::new(select_builder.build(context_factory))) } fn key_query( @@ -140,19 +268,19 @@ impl MultipliedMeasuresQueryPlanner { .query_properties .dimensions_for_select_append(dimensions); - let mut select_builder = SelectBuilder::new( - source, - VisitorContext::new_with_cube_alias_prefix( - self.context_factory.clone(), - format!("{}_key", key_cube_name), - ), - ); + let mut select_builder = SelectBuilder::new(source); + let mut context_factory = self.context_factory.clone(); + for time_dim in self.query_properties.time_dimensions() { + if let Some(granularity) = time_dim.get_granularity() { + context_factory.add_leaf_time_dimension(&time_dim.full_name(), &granularity); + } + } for member in dimensions.iter() { - select_builder.add_projection_member(&member, None, None); + select_builder.add_projection_member(&member, None); } select_builder.set_distinct(); select_builder.set_filter(self.query_properties.all_filters()); - Ok(Rc::new(select_builder.build())) + Ok(Rc::new(select_builder.build(context_factory))) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs index ada3ec3df38f9..b5e9d6e2aa856 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs @@ -30,7 +30,7 @@ impl OrderPlanner { .find(|(_, m)| m.full_name().to_lowercase() == itm.name().to_lowercase()) { result.push(OrderBy::new( - Expr::Member(MemberExpression::new(member.clone(), None)), + Expr::Member(MemberExpression::new(member.clone())), pos + 1, itm.desc(), )); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs index 981bd6ad4dcfa..9229314a0cc51 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs @@ -2,8 +2,7 @@ use super::{JoinPlanner, OrderPlanner}; use crate::plan::{Filter, Select, SelectBuilder}; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; -use crate::planner::QueryProperties; -use crate::planner::VisitorContext; +use crate::planner::{BaseMember, QueryProperties}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -11,13 +10,13 @@ pub struct SimpleQueryPlanner { query_properties: Rc, join_planner: JoinPlanner, order_planner: OrderPlanner, - context_factory: Rc, + context_factory: SqlNodesFactory, } impl SimpleQueryPlanner { pub fn new( query_tools: Rc, query_properties: Rc, - context_factory: Rc, + context_factory: SqlNodesFactory, ) -> Self { Self { join_planner: JoinPlanner::new(query_tools.clone()), @@ -36,16 +35,20 @@ impl SimpleQueryPlanner { items: self.query_properties.measures_filters().clone(), }) }; - let mut select_builder = SelectBuilder::new( - self.join_planner.make_join_node()?, - VisitorContext::default(self.context_factory.clone()), - ); + let mut context_factory = self.context_factory.clone(); + let from = self.join_planner.make_join_node()?; + let mut select_builder = SelectBuilder::new(from.clone()); + for time_dim in self.query_properties.time_dimensions() { + if let Some(granularity) = time_dim.get_granularity() { + context_factory.add_leaf_time_dimension(&time_dim.full_name(), &granularity); + } + } for member in self .query_properties .all_dimensions_and_measures(self.query_properties.measures())? .iter() { - select_builder.add_projection_member(member, None, None); + select_builder.add_projection_member(member, None); } select_builder.set_filter(filter); select_builder.set_group_by(self.query_properties.group_by()); @@ -53,7 +56,7 @@ impl SimpleQueryPlanner { select_builder.set_having(having); select_builder.set_limit(self.query_properties.row_limit()); select_builder.set_offset(self.query_properties.offset()); - let res = select_builder.build(); + let res = select_builder.build(context_factory); Ok(res) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 384ff238c12f2..54b31fa40ed20 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -4,9 +4,8 @@ use super::{BaseDimension, BaseMeasure, BaseMember, BaseMemberHelper, BaseTimeDi use crate::cube_bridge::base_query_options::BaseQueryOptions; use crate::plan::{Expr, Filter, FilterItem, MemberExpression}; use crate::planner::sql_evaluator::collectors::{ - collect_multiplied_measures, has_multi_stage_members, + collect_multiplied_measures, has_cumulative_members, has_multi_stage_members, }; -use crate::planner::sql_evaluator::EvaluationNode; use cubenativeutils::CubeError; use itertools::Itertools; use std::collections::HashSet; @@ -32,12 +31,6 @@ impl OrderByItem { } } -enum SymbolAggregateType { - Regular, - Multiplied, - MultiStage, -} - #[derive(Default, Clone)] pub struct FullKeyAggregateMeasures { pub multiplied_measures: Vec>, @@ -67,6 +60,8 @@ pub struct QueryProperties { row_limit: Option, offset: Option, query_tools: Rc, + ignore_cumulative: bool, + ungrouped: bool, } impl QueryProperties { @@ -160,6 +155,7 @@ impl QueryProperties { } else { None }; + let ungrouped = options.static_data().ungrouped.unwrap_or(false); Ok(Rc::new(Self { measures, @@ -172,6 +168,8 @@ impl QueryProperties { row_limit, offset, query_tools, + ignore_cumulative: false, + ungrouped, })) } @@ -186,6 +184,8 @@ impl QueryProperties { order_by: Vec, row_limit: Option, offset: Option, + ignore_cumulative: bool, + ungrouped: bool, ) -> Result, CubeError> { let order_by = if order_by.is_empty() { Self::default_order(&dimensions, &time_dimensions, &measures) @@ -204,6 +204,8 @@ impl QueryProperties { row_limit, offset, query_tools, + ignore_cumulative, + ungrouped, })) } @@ -248,6 +250,10 @@ impl QueryProperties { Self::default_order(&self.dimensions, &self.time_dimensions, &self.measures); } + pub fn ungrouped(&self) -> bool { + self.ungrouped + } + pub fn all_filters(&self) -> Option { let items = self .time_dimensions_filters @@ -322,15 +328,19 @@ impl QueryProperties { } pub fn group_by(&self) -> Vec { - self.dimensions - .iter() - .map(|f| Expr::Member(MemberExpression::new(f.clone(), None))) - .chain( - self.time_dimensions - .iter() - .map(|f| Expr::Member(MemberExpression::new(f.clone(), None))), - ) - .collect() + if self.ungrouped { + vec![] + } else { + self.dimensions + .iter() + .map(|f| Expr::Member(MemberExpression::new(f.clone()))) + .chain( + self.time_dimensions + .iter() + .map(|f| Expr::Member(MemberExpression::new(f.clone()))), + ) + .collect() + } } pub fn default_order( @@ -377,46 +387,44 @@ impl QueryProperties { } pub fn is_simple_query(&self) -> Result { + let full_aggregate_measure = self.full_key_aggregate_measures()?; + if full_aggregate_measure.multiplied_measures.is_empty() + && full_aggregate_measure.multi_stage_measures.is_empty() + { + Ok(true) + } else { + Ok(false) + } + } + + pub fn should_use_time_series(&self) -> Result { for member in self.all_members(false) { - match self.get_symbol_aggregate_type(&member.member_evaluator())? { - SymbolAggregateType::Regular => {} - _ => return Ok(false), + if has_cumulative_members(&member.member_evaluator())? { + return Ok(true); } } - Ok(true) + Ok(false) } pub fn full_key_aggregate_measures(&self) -> Result { let mut result = FullKeyAggregateMeasures::default(); let measures = self.measures(); for m in measures.iter() { - match self.get_symbol_aggregate_type(m.member_evaluator())? { - SymbolAggregateType::Regular => result.regular_measures.push(m.clone()), - SymbolAggregateType::Multiplied => result.multiplied_measures.push(m.clone()), - SymbolAggregateType::MultiStage => result.multi_stage_measures.push(m.clone()), + if has_multi_stage_members(m.member_evaluator(), self.ignore_cumulative)? { + result.multi_stage_measures.push(m.clone()) + } else { + for item in + collect_multiplied_measures(self.query_tools.clone(), m.member_evaluator())? + { + if item.multiplied { + result.multiplied_measures.push(item.measure.clone()); + } else { + result.regular_measures.push(item.measure.clone()); + } + } } } Ok(result) } - - fn get_symbol_aggregate_type( - &self, - symbol: &Rc, - ) -> Result { - let symbol_type = if has_multi_stage_members(symbol)? { - SymbolAggregateType::MultiStage - } else if let Some(multiple) = - collect_multiplied_measures(self.query_tools.clone(), symbol)? - { - if multiple.multiplied { - SymbolAggregateType::Multiplied - } else { - SymbolAggregateType::Regular - } - } else { - SymbolAggregateType::Regular - }; - Ok(symbol_type) - } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs index d20896c4bec23..b2cb0497606c8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs @@ -8,6 +8,7 @@ use crate::cube_bridge::sql_templates_render::SqlTemplatesRender; use chrono_tz::Tz; use convert_case::{Case, Casing}; use cubenativeutils::CubeError; +use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; use std::cell::{Ref, RefCell, RefMut}; @@ -118,6 +119,18 @@ impl QueryTools { } } + pub fn parse_member_path(&self, name: &str) -> Result<(String, String), CubeError> { + let path = name.split('.').collect_vec(); + if path.len() == 2 { + Ok((path[0].to_string(), path[1].to_string())) + } else { + Err(CubeError::internal(format!( + "Invalid member name: '{}'", + name + ))) + } + } + pub fn auto_prefix_with_cube_name(&self, cube_name: &str, sql: &str) -> String { lazy_static! { static ref SINGLE_MEMBER_RE: Regex = Regex::new(r"^[_a-zA-Z][_a-zA-Z0-9]*$").unwrap(); @@ -141,7 +154,7 @@ impl QueryTools { self.templates_render.clone() } - pub fn allocaate_param(&self, name: &str) -> usize { + pub fn allocate_param(&self, name: &str) -> String { self.params_allocator.borrow_mut().allocate_param(name) } pub fn get_allocated_params(&self) -> Vec { @@ -152,8 +165,11 @@ impl QueryTools { sql: &str, should_reuse_params: bool, ) -> Result<(String, Vec), CubeError> { - self.params_allocator - .borrow() - .build_sql_and_params(sql, should_reuse_params) + let native_allocated_params = self.base_tools.get_allocated_params()?; + self.params_allocator.borrow().build_sql_and_params( + sql, + native_allocated_params, + should_reuse_params, + ) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs new file mode 100644 index 0000000000000..73f875a07adc1 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs @@ -0,0 +1,61 @@ +use crate::planner::sql_evaluator::{MemberSymbol, TraversalVisitor}; +use cubenativeutils::CubeError; +use std::collections::HashSet; +use std::rc::Rc; + +pub struct CubeNamesCollector { + names: HashSet, +} + +impl CubeNamesCollector { + pub fn new() -> Self { + Self { + names: HashSet::new(), + } + } + + pub fn extract_result(self) -> Vec { + self.names.into_iter().collect() + } +} + +impl TraversalVisitor for CubeNamesCollector { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + _: &Self::State, + ) -> Result, CubeError> { + match node.as_ref() { + MemberSymbol::Dimension(e) => { + if e.owned_by_cube() { + self.names.insert(e.cube_name().clone()); + } + for name in e.get_dependent_cubes().into_iter() { + self.names.insert(name); + } + } + MemberSymbol::Measure(e) => { + if e.owned_by_cube() { + self.names.insert(e.cube_name().clone()); + } + for name in e.get_dependent_cubes().into_iter() { + self.names.insert(name); + } + } + MemberSymbol::CubeName(e) => { + self.names.insert(e.cube_name().clone()); + } + MemberSymbol::CubeTable(e) => { + self.names.insert(e.cube_name().clone()); + } + }; + Ok(Some(())) + } +} + +pub fn collect_cube_names(node: &Rc) -> Result, CubeError> { + let mut visitor = CubeNamesCollector::new(); + visitor.apply(node, &())?; + Ok(visitor.extract_result()) +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs new file mode 100644 index 0000000000000..deec7e9cbda2c --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs @@ -0,0 +1,48 @@ +use crate::planner::sql_evaluator::{MemberSymbol, TraversalVisitor}; +use cubenativeutils::CubeError; +use std::rc::Rc; + +pub struct HasCumulativeMembersCollector { + pub has_cumulative_members: bool, +} + +impl HasCumulativeMembersCollector { + pub fn new() -> Self { + Self { + has_cumulative_members: false, + } + } + + pub fn extract_result(self) -> bool { + self.has_cumulative_members + } +} + +impl TraversalVisitor for HasCumulativeMembersCollector { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + _: &Self::State, + ) -> Result, CubeError> { + match node.as_ref() { + MemberSymbol::Measure(s) => { + if s.is_rolling_window() { + self.has_cumulative_members = true; + } + } + _ => {} + }; + if self.has_cumulative_members { + Ok(None) + } else { + Ok(Some(())) + } + } +} + +pub fn has_cumulative_members(node: &Rc) -> Result { + let mut visitor = HasCumulativeMembersCollector::new(); + visitor.apply(node, &())?; + Ok(visitor.extract_result()) +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs index aedeaf794b7a3..7190d39c469d7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs @@ -1,14 +1,16 @@ -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType, TraversalVisitor}; +use crate::planner::sql_evaluator::{MemberSymbol, TraversalVisitor}; use cubenativeutils::CubeError; use std::rc::Rc; pub struct HasMultiStageMembersCollector { + pub ignore_cumulative: bool, pub has_multi_stage: bool, } impl HasMultiStageMembersCollector { - pub fn new() -> Self { + pub fn new(ignore_cumulative: bool) -> Self { Self { + ignore_cumulative, has_multi_stage: false, } } @@ -19,33 +21,42 @@ impl HasMultiStageMembersCollector { } impl TraversalVisitor for HasMultiStageMembersCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { - match node.symbol() { - MemberSymbolType::Measure(s) => { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + _: &Self::State, + ) -> Result, CubeError> { + match node.as_ref() { + MemberSymbol::Measure(s) => { if s.is_multi_stage() { self.has_multi_stage = true; - } else { - for filter_node in s.measure_filters() { - self.apply(filter_node)? - } - for order_by in s.measure_order_by() { - self.apply(order_by.evaluation_node())? - } + } else if !self.ignore_cumulative + && (s.is_rolling_window() || s.measure_type() == "runningTotal") + { + self.has_multi_stage = true; } } - MemberSymbolType::Dimension(s) => { + MemberSymbol::Dimension(s) => { if s.is_multi_stage() { self.has_multi_stage = true; } } _ => {} }; - Ok(!self.has_multi_stage) + if self.has_multi_stage { + Ok(None) + } else { + Ok(Some(())) + } } } -pub fn has_multi_stage_members(node: &Rc) -> Result { - let mut visitor = HasMultiStageMembersCollector::new(); - visitor.apply(node)?; +pub fn has_multi_stage_members( + node: &Rc, + ignore_cumulative: bool, +) -> Result { + let mut visitor = HasMultiStageMembersCollector::new(ignore_cumulative); + visitor.apply(node, &())?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs index 09e332098364e..b33ea186c6083 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs @@ -1,51 +1,69 @@ -use crate::planner::sql_evaluator::{ - EvaluationNode, MemberSymbol, MemberSymbolType, TraversalVisitor, -}; +use crate::planner::sql_evaluator::{MemberSymbol, TraversalVisitor}; +use crate::planner::BaseMeasure; use cubenativeutils::CubeError; -use std::collections::HashSet; use std::rc::Rc; pub struct JoinHintsCollector { - hints: HashSet, + hints: Vec, } impl JoinHintsCollector { pub fn new() -> Self { - Self { - hints: HashSet::new(), - } + Self { hints: Vec::new() } } pub fn extract_result(self) -> Vec { - self.hints.into_iter().collect() + self.hints } } impl TraversalVisitor for JoinHintsCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { - let res = match node.symbol() { - MemberSymbolType::Dimension(e) => { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + _: &Self::State, + ) -> Result, CubeError> { + match node.as_ref() { + MemberSymbol::Dimension(e) => { if e.owned_by_cube() { - self.hints.insert(e.cube_name().clone()); + self.hints.push(e.cube_name().clone()); + } + for name in e.get_dependent_cubes().into_iter() { + self.hints.push(name); } - true } - MemberSymbolType::Measure(e) => { + MemberSymbol::Measure(e) => { if e.owned_by_cube() { - self.hints.insert(e.cube_name().clone()); + self.hints.push(e.cube_name().clone()); + } + for name in e.get_dependent_cubes().into_iter() { + self.hints.push(name); } - true } - MemberSymbolType::CubeName(e) => { - self.hints.insert(e.cube_name().clone()); - true + MemberSymbol::CubeName(e) => { + self.hints.push(e.cube_name().clone()); } - MemberSymbolType::CubeTable(e) => { - self.hints.insert(e.cube_name().clone()); - true + MemberSymbol::CubeTable(e) => { + self.hints.push(e.cube_name().clone()); } - _ => false, }; - Ok(res) + Ok(Some(())) + } +} + +pub fn collect_join_hints(node: &Rc) -> Result, CubeError> { + let mut visitor = JoinHintsCollector::new(); + visitor.apply(node, &())?; + Ok(visitor.extract_result()) +} + +pub fn collect_join_hints_for_measures( + measures: &Vec>, +) -> Result, CubeError> { + let mut visitor = JoinHintsCollector::new(); + for meas in measures.iter() { + visitor.apply(&meas.member_evaluator(), &())?; } + Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs index a2f4de39ddd23..8bb049ee48b6a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs @@ -1,56 +1,60 @@ -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType, TraversalVisitor}; +use crate::planner::sql_evaluator::{MemberSymbol, TraversalVisitor}; use cubenativeutils::CubeError; use std::rc::Rc; pub struct MemberChildsCollector { + pub childs: Vec>, +} + +#[derive(Clone)] +pub struct MemberChildsCollectorState { pub is_root: bool, - pub childs: Vec>, +} + +impl MemberChildsCollectorState { + pub fn new(is_root: bool) -> Self { + Self { is_root } + } } impl MemberChildsCollector { pub fn new() -> Self { - Self { - is_root: true, - childs: vec![], - } + Self { childs: vec![] } } - pub fn extract_result(self) -> Vec> { + pub fn extract_result(self) -> Vec> { self.childs } } impl TraversalVisitor for MemberChildsCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { - if self.is_root { - self.is_root = false; - match node.symbol() { - MemberSymbolType::Measure(s) => { - for filter_node in s.measure_filters() { - self.apply(filter_node)? - } - for order_by in s.measure_order_by() { - self.apply(order_by.evaluation_node())? - } - Ok(true) - } - MemberSymbolType::Dimension(_) => Ok(true), - _ => Ok(false), + type State = MemberChildsCollectorState; + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { + if state.is_root { + let new_state = MemberChildsCollectorState::new(false); + match node.as_ref() { + MemberSymbol::Measure(_) => Ok(Some(new_state)), + MemberSymbol::Dimension(_) => Ok(Some(new_state)), + _ => Ok(None), } } else { - match node.symbol() { - MemberSymbolType::Measure(_) | MemberSymbolType::Dimension(_) => { + match node.as_ref() { + MemberSymbol::Measure(_) | MemberSymbol::Dimension(_) => { self.childs.push(node.clone()); - Ok(false) + Ok(None) } - _ => Ok(true), + _ => Ok(Some(state.clone())), } } } } -pub fn member_childs(node: &Rc) -> Result>, CubeError> { +pub fn member_childs(node: &Rc) -> Result>, CubeError> { let mut visitor = MemberChildsCollector::new(); - visitor.apply(node)?; + visitor.apply(node, &MemberChildsCollectorState::new(true))?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs index d276385a25241..1a3241bf740c4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs @@ -1,9 +1,15 @@ +mod cube_names_collector; +mod has_cumulative_members; mod has_multi_stage_members; mod join_hints_collector; mod member_childs_collector; mod multiplied_measures_collector; +pub use cube_names_collector::collect_cube_names; +pub use has_cumulative_members::{has_cumulative_members, HasCumulativeMembersCollector}; pub use has_multi_stage_members::{has_multi_stage_members, HasMultiStageMembersCollector}; -pub use join_hints_collector::JoinHintsCollector; +pub use join_hints_collector::{ + collect_join_hints, collect_join_hints_for_measures, JoinHintsCollector, +}; pub use member_childs_collector::{member_childs, MemberChildsCollector}; pub use multiplied_measures_collector::{collect_multiplied_measures, MultipliedMeasuresCollector}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs index 8ee83c026ec8e..c285bff19379f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs @@ -1,39 +1,95 @@ use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{ - EvaluationNode, MemberSymbol, MemberSymbolType, TraversalVisitor, -}; +use crate::planner::sql_evaluator::{MemberSymbol, TraversalVisitor}; +use crate::planner::BaseMeasure; use cubenativeutils::CubeError; +use std::collections::HashSet; use std::rc::Rc; -pub struct RootMeasureResult { +struct CompositeMeasuresCollector { + composite_measures: HashSet, +} + +struct CompositeMeasureCollectorState { + pub parent_measure: Option>, +} + +impl CompositeMeasureCollectorState { + pub fn new(parent_measure: Option>) -> Self { + Self { parent_measure } + } +} + +impl CompositeMeasuresCollector { + pub fn new() -> Self { + Self { + composite_measures: HashSet::new(), + } + } + + pub fn extract_result(self) -> HashSet { + self.composite_measures + } +} + +impl TraversalVisitor for CompositeMeasuresCollector { + type State = CompositeMeasureCollectorState; + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { + let res = match node.as_ref() { + MemberSymbol::Measure(_) => { + if let Some(parent) = &state.parent_measure { + if parent.cube_name() != node.cube_name() { + self.composite_measures.insert(parent.full_name()); + } + } + + let new_state = CompositeMeasureCollectorState::new(Some(node.clone())); + Some(new_state) + } + MemberSymbol::Dimension(_) => None, + _ => None, + }; + Ok(res) + } +} + +pub struct MeasureResult { pub multiplied: bool, - pub measure: String, + pub measure: Rc, } pub struct MultipliedMeasuresCollector { query_tools: Rc, - parent_measure: Option, - root_measure: Option, + composite_measures: HashSet, + colllected_measures: Vec, } impl MultipliedMeasuresCollector { - pub fn new(query_tools: Rc) -> Self { + pub fn new(query_tools: Rc, composite_measures: HashSet) -> Self { Self { query_tools, - parent_measure: None, - root_measure: None, + composite_measures, + colllected_measures: vec![], } } - pub fn extract_result(self) -> Option { - self.root_measure + pub fn extract_result(self) -> Vec { + self.colllected_measures } } impl TraversalVisitor for MultipliedMeasuresCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { - let res = match node.symbol() { - MemberSymbolType::Measure(e) => { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + _: &Self::State, + ) -> Result, CubeError> { + let res = match node.as_ref() { + MemberSymbol::Measure(e) => { let full_name = e.full_name(); let join = self.query_tools.cached_data().join()?; let multiplied = join @@ -43,17 +99,22 @@ impl TraversalVisitor for MultipliedMeasuresCollector { .unwrap_or(&false) .clone(); - if self.parent_measure.is_none() { - self.root_measure = Some(RootMeasureResult { + if !self.composite_measures.contains(&full_name) { + self.colllected_measures.push(MeasureResult { multiplied, - measure: full_name.clone(), + measure: BaseMeasure::try_new(node.clone(), self.query_tools.clone())? + .unwrap(), }) } - self.parent_measure = Some(full_name); - true + + if self.composite_measures.contains(&full_name) { + Some(()) + } else { + None + } } - MemberSymbolType::Dimension(_) => true, - _ => false, + MemberSymbol::Dimension(_) => None, + _ => None, }; Ok(res) } @@ -61,9 +122,12 @@ impl TraversalVisitor for MultipliedMeasuresCollector { pub fn collect_multiplied_measures( query_tools: Rc, - node: &Rc, -) -> Result, CubeError> { - let mut visitor = MultipliedMeasuresCollector::new(query_tools); - visitor.apply(node)?; + node: &Rc, +) -> Result, CubeError> { + let mut composite_collector = CompositeMeasuresCollector::new(); + composite_collector.apply(node, &CompositeMeasureCollectorState::new(None))?; + let composite_measures = composite_collector.extract_result(); + let mut visitor = MultipliedMeasuresCollector::new(query_tools, composite_measures); + visitor.apply(node, &())?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs index 97c9eac59b4e3..c23f085fbee99 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs @@ -1,8 +1,9 @@ use super::collectors::JoinHintsCollector; use super::dependecy::DependenciesBuilder; +use super::symbols::MemberSymbol; use super::{ - CubeNameSymbolFactory, CubeTableSymbolFactory, DimensionSymbolFactory, EvaluationNode, - MeasureSymbolFactory, SimpleSqlSymbolFactory, SymbolFactory, TraversalVisitor, + CubeNameSymbolFactory, CubeTableSymbolFactory, DimensionSymbolFactory, MeasureSymbolFactory, + SqlCall, SymbolFactory, TraversalVisitor, }; use crate::cube_bridge::evaluator::CubeEvaluator; use crate::cube_bridge::memeber_sql::MemberSql; @@ -11,7 +12,7 @@ use std::collections::HashMap; use std::rc::Rc; pub struct Compiler { cube_evaluator: Rc, - members: HashMap<(String, String), Rc>, + members: HashMap<(String, String), Rc>, } impl Compiler { @@ -22,22 +23,10 @@ impl Compiler { } } - pub fn add_evaluator( - &mut self, - full_name: &String, - factory: T, - ) -> Result, CubeError> { - if let Some(exists) = self.exists_member::(full_name) { - Ok(exists.clone()) - } else { - self.add_evaluator_impl(full_name, factory) - } - } - pub fn add_measure_evaluator( &mut self, measure: String, - ) -> Result, CubeError> { + ) -> Result, CubeError> { if let Some(exists) = self.exists_member::(&measure) { Ok(exists.clone()) } else { @@ -51,7 +40,7 @@ impl Compiler { pub fn add_dimension_evaluator( &mut self, dimension: String, - ) -> Result, CubeError> { + ) -> Result, CubeError> { if let Some(exists) = self.exists_member::(&dimension) { Ok(exists.clone()) } else { @@ -65,7 +54,7 @@ impl Compiler { pub fn add_cube_name_evaluator( &mut self, cube_name: String, - ) -> Result, CubeError> { + ) -> Result, CubeError> { if let Some(exists) = self.exists_member::(&cube_name) { Ok(exists.clone()) } else { @@ -79,7 +68,7 @@ impl Compiler { pub fn add_cube_table_evaluator( &mut self, cube_name: String, - ) -> Result, CubeError> { + ) -> Result, CubeError> { if let Some(exists) = self.exists_member::(&cube_name) { Ok(exists.clone()) } else { @@ -90,37 +79,26 @@ impl Compiler { } } - pub fn add_join_condition_evaluator( - &mut self, - cube_name: String, - sql: Rc, - ) -> Result, CubeError> { - self.add_evaluator_impl( - &cube_name, - SimpleSqlSymbolFactory::try_new(&cube_name, sql)?, - ) - } - - pub fn add_simple_sql_evaluator( - &mut self, - cube_name: String, - sql: Rc, - ) -> Result, CubeError> { - self.add_evaluator_impl( - &cube_name, - SimpleSqlSymbolFactory::try_new(&cube_name, sql)?, - ) - } - pub fn join_hints(&self) -> Result, CubeError> { let mut collector = JoinHintsCollector::new(); for member in self.members.values() { - collector.apply(member)?; + collector.apply(member, &())?; } Ok(collector.extract_result()) } - fn exists_member(&self, full_name: &String) -> Option> { + pub fn compile_sql_call( + &mut self, + cube_name: &String, + member_sql: Rc, + ) -> Result, CubeError> { + let dep_builder = DependenciesBuilder::new(self, self.cube_evaluator.clone()); + let deps = dep_builder.build(cube_name.clone(), member_sql.clone())?; + let sql_call = SqlCall::new(member_sql, deps); + Ok(Rc::new(sql_call)) + } + + fn exists_member(&self, full_name: &String) -> Option> { if T::is_cachable() { let key = (T::symbol_name(), full_name.clone()); self.members.get(&key).cloned() @@ -133,12 +111,8 @@ impl Compiler { &mut self, full_name: &String, factory: T, - ) -> Result, CubeError> { - let cube_name = factory.cube_name(); - let dep_builder = DependenciesBuilder::new(self, self.cube_evaluator.clone()); - let deps = dep_builder.build(cube_name.clone(), factory.member_sql())?; - - let node = factory.build(deps, self)?; + ) -> Result, CubeError> { + let node = factory.build(self)?; let key = (T::symbol_name().to_string(), full_name.clone()); if T::is_cachable() { self.members.insert(key, node.clone()); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs index a9ac62c42aba6..ce284b26387b3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/dependecy.rs @@ -1,23 +1,34 @@ -use super::{Compiler, EvaluationNode}; -use crate::cube_bridge::evaluator::CubeEvaluator; +use super::symbols::MemberSymbol; +use super::Compiler; +use crate::cube_bridge::evaluator::{CallDep, CubeEvaluator}; use crate::cube_bridge::memeber_sql::MemberSql; use cubenativeutils::CubeError; use std::collections::HashMap; use std::rc::Rc; -pub struct StructDependency { - pub sql_fn: Option>, - pub to_string_fn: Option>, - pub properties: HashMap, +#[derive(Clone)] +pub enum CubeDepProperty { + CubeDependency(CubeDependency), + SymbolDependency(Rc), } -impl StructDependency { +#[derive(Clone)] +pub struct CubeDependency { + pub cube_symbol: Rc, + pub sql_fn: Option>, + pub to_string_fn: Option>, + pub properties: HashMap, +} + +impl CubeDependency { pub fn new( - sql_fn: Option>, - to_string_fn: Option>, - properties: HashMap, + cube_symbol: Rc, + sql_fn: Option>, + to_string_fn: Option>, + properties: HashMap, ) -> Self { - StructDependency { + CubeDependency { + cube_symbol, sql_fn, to_string_fn, properties, @@ -25,15 +36,17 @@ impl StructDependency { } } +#[derive(Clone)] pub enum ContextSymbolDep { SecurityContext, FilterParams, FilterGroup, } +#[derive(Clone)] pub enum Dependency { - SingleDependency(Rc), - StructDependency(StructDependency), + SymbolDependency(Rc), + CubeDependency(CubeDependency), ContextDependency(ContextSymbolDep), } @@ -53,14 +66,11 @@ impl<'a> DependenciesBuilder<'a> { pub fn build( mut self, cube_name: String, - member_sql: Option>, + member_sql: Rc, ) -> Result, CubeError> { - let call_deps = if let Some(member_sql) = member_sql { - self.cube_evaluator - .resolve_symbols_call_deps(cube_name.clone(), member_sql)? - } else { - vec![] - }; + let call_deps = self + .cube_evaluator + .resolve_symbols_call_deps(cube_name.clone(), member_sql)?; let mut childs = Vec::new(); for (i, dep) in call_deps.iter().enumerate() { @@ -69,6 +79,7 @@ impl<'a> DependenciesBuilder<'a> { childs[parent].push(i); } } + let mut result = Vec::new(); for (i, dep) in call_deps.iter().enumerate() { @@ -80,50 +91,71 @@ impl<'a> DependenciesBuilder<'a> { continue; } if childs[i].is_empty() { - result.push(Dependency::SingleDependency( + result.push(Dependency::SymbolDependency( self.build_evaluator(&cube_name, &dep.name)?, )); } else { - let new_cube_name = if self.is_current_cube(&dep.name) { - cube_name.clone() - } else { - dep.name.clone() - }; - let mut sql_fn = None; - let mut to_string_fn: Option> = None; - let mut properties = HashMap::new(); - for child_ind in childs[i].iter() { - let name = &call_deps[*child_ind].name; - if name.as_str() == "sql" { - sql_fn = Some( - self.compiler - .add_cube_table_evaluator(new_cube_name.clone())?, - ); - } else if name.as_str() == "toString" { - to_string_fn = Some( - self.compiler - .add_cube_name_evaluator(new_cube_name.clone())?, - ); - } else { - properties.insert( - name.clone(), - Dependency::SingleDependency( - self.build_evaluator(&new_cube_name, &name)?, - ), - ); - } - } - result.push(Dependency::StructDependency(StructDependency::new( - sql_fn, - to_string_fn, - properties, - ))); + let dep = self.build_cube_dependency(&cube_name, i, &call_deps, &childs)?; + result.push(Dependency::CubeDependency(dep)); } } Ok(result) } + fn build_cube_dependency( + &mut self, + cube_name: &String, + dep_index: usize, + call_deps: &Vec, + call_childs: &Vec>, + ) -> Result { + let dep = &call_deps[dep_index]; + let new_cube_name = if self.is_current_cube(&dep.name) { + cube_name.clone() + } else { + dep.name.clone() + }; + let mut sql_fn = None; + let mut to_string_fn: Option> = None; + let mut properties = HashMap::new(); + let cube_symbol = self + .compiler + .add_cube_table_evaluator(new_cube_name.clone())?; + for child_ind in call_childs[dep_index].iter() { + let name = &call_deps[*child_ind].name; + if name.as_str() == "sql" { + sql_fn = Some( + self.compiler + .add_cube_table_evaluator(new_cube_name.clone())?, + ); + } else if name.as_str() == "toString" { + to_string_fn = Some( + self.compiler + .add_cube_name_evaluator(new_cube_name.clone())?, + ); + } else { + let child_dep = if call_childs[*child_ind].is_empty() { + CubeDepProperty::SymbolDependency(self.build_evaluator(&new_cube_name, &name)?) + } else { + CubeDepProperty::CubeDependency(self.build_cube_dependency( + &new_cube_name, + *child_ind, + call_deps, + call_childs, + )?) + }; + properties.insert(name.clone(), child_dep); + } + } + Ok(CubeDependency::new( + cube_symbol, + sql_fn, + to_string_fn, + properties, + )) + } + fn build_context_dep(&self, name: &str) -> Option { match name { "USER_CONTEXT" | "SECURITY_CONTEXT" => Some(Dependency::ContextDependency( @@ -149,7 +181,7 @@ impl<'a> DependenciesBuilder<'a> { &mut self, cube_name: &String, name: &String, - ) -> Result, CubeError> { + ) -> Result, CubeError> { let dep_full_name = format!("{}.{}", cube_name, name); //FIXME avoid cloning let dep_path = vec![cube_name.clone(), name.clone()]; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs deleted file mode 100644 index 02355626a2c28..0000000000000 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs +++ /dev/null @@ -1,71 +0,0 @@ -use super::dependecy::Dependency; -use super::{ - CubeNameSymbol, CubeTableSymbol, DimensionSymbol, MeasureSymbol, MemberSymbolType, - SimpleSqlSymbol, -}; -use std::rc::Rc; - -pub struct EvaluationNode { - symbol: MemberSymbolType, - deps: Vec, -} - -impl EvaluationNode { - pub fn new(symbol: MemberSymbolType, deps: Vec) -> Rc { - Rc::new(Self { symbol, deps }) - } - - pub fn new_measure(symbol: MeasureSymbol, deps: Vec) -> Rc { - Rc::new(Self { - symbol: MemberSymbolType::Measure(symbol), - deps, - }) - } - - pub fn new_dimension(symbol: DimensionSymbol, deps: Vec) -> Rc { - Rc::new(Self { - symbol: MemberSymbolType::Dimension(symbol), - deps, - }) - } - - pub fn new_cube_name(symbol: CubeNameSymbol) -> Rc { - Rc::new(Self { - symbol: MemberSymbolType::CubeName(symbol), - deps: vec![], - }) - } - - pub fn new_cube_table(symbol: CubeTableSymbol, deps: Vec) -> Rc { - Rc::new(Self { - symbol: MemberSymbolType::CubeTable(symbol), - deps, - }) - } - - pub fn new_simple_sql(symbol: SimpleSqlSymbol, deps: Vec) -> Rc { - Rc::new(Self { - symbol: MemberSymbolType::SimpleSql(symbol), - deps, - }) - } - - pub fn deps(&self) -> &Vec { - &self.deps - } - - pub fn symbol(&self) -> &MemberSymbolType { - &self.symbol - } - - pub fn full_name(&self) -> String { - self.symbol.full_name() - } - - pub fn is_measure(&self) -> bool { - self.symbol.is_measure() - } - pub fn is_dimension(&self) -> bool { - self.symbol.is_dimension() - } -} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs index 40a62074f3962..df22f8bc00b64 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/mod.rs @@ -1,20 +1,21 @@ pub mod collectors; pub mod compiler; mod dependecy; -pub mod evaluation_node; -pub mod sql_node_transformers; +pub mod references_builder; +pub mod sql_call; pub mod sql_nodes; pub mod sql_visitor; pub mod symbols; pub mod visitor; pub use compiler::Compiler; -pub use dependecy::Dependency; -pub use evaluation_node::EvaluationNode; +pub use dependecy::{CubeDepProperty, Dependency}; +pub use references_builder::ReferencesBuilder; +pub use sql_call::SqlCall; pub use sql_visitor::SqlEvaluatorVisitor; pub use symbols::{ CubeNameSymbol, CubeNameSymbolFactory, CubeTableSymbol, CubeTableSymbolFactory, DimensionSymbol, DimensionSymbolFactory, MeasureSymbol, MeasureSymbolFactory, MemberSymbol, - MemberSymbolType, SimpleSqlSymbol, SimpleSqlSymbolFactory, SymbolFactory, + SymbolFactory, }; pub use visitor::TraversalVisitor; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/references_builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/references_builder.rs new file mode 100644 index 0000000000000..f14f6f15e500d --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/references_builder.rs @@ -0,0 +1,310 @@ +use crate::plan::{ + Filter, FilterItem, From, Join, QualifiedColumnName, SingleAliasedSource, SingleSource, +}; +use cubenativeutils::CubeError; +use std::collections::HashMap; +use std::rc::Rc; + +use super::MemberSymbol; + +pub struct ReferencesBuilder { + source: Rc, +} + +impl ReferencesBuilder { + pub fn new(source: Rc) -> Self { + Self { source } + } + + pub fn validate_member( + &self, + member: Rc, + strict_source: &Option, + ) -> Result<(), CubeError> { + let member_name = member.full_name(); + if self + .find_reference_for_member(&member_name, strict_source) + .is_some() + { + return Ok(()); + } + + let dependencies = member.get_dependencies(); + if !dependencies.is_empty() { + for dep in dependencies.iter() { + self.validate_member(dep.clone(), strict_source)?; + } + } else { + if !self.has_source_for_leaf_memeber(&member, strict_source) { + return Err(CubeError::internal(format!( + "Planning error: member {} has no source", + member_name + ))); + } + } + Ok(()) + } + + pub fn resolve_references_for_member( + &self, + member: Rc, + strict_source: &Option, + references: &mut HashMap, + ) -> Result<(), CubeError> { + let member_name = member.full_name(); + if let Some(reference) = self.find_reference_for_member(&member_name, strict_source) { + references.insert(member_name.clone(), reference); + return Ok(()); + } + + let dependencies = member.get_dependencies(); + if !dependencies.is_empty() { + for dep in dependencies.iter() { + self.resolve_references_for_member(dep.clone(), strict_source, references)? + } + } else { + if !self.has_source_for_leaf_memeber(&member, strict_source) { + return Err(CubeError::internal(format!( + "Planning error: member {} has no source", + member_name + ))); + } + } + + Ok(()) + } + + pub fn validete_member_for_leaf_query( + &self, + member: Rc, + strict_source: &Option, + ) -> Result<(), CubeError> { + let dependencies = member.get_dependencies(); + if !dependencies.is_empty() { + for dep in dependencies.iter() { + self.validete_member_for_leaf_query(dep.clone(), strict_source)? + } + } else { + if !self.has_source_for_leaf_memeber(&member, strict_source) { + return Err(CubeError::internal(format!( + "Planning error: member {} has no source", + member.full_name() + ))); + } + } + Ok(()) + } + + pub fn validate_filter(&self, filter: &Filter) -> Result<(), CubeError> { + for itm in filter.items.iter() { + self.validate_filter_item(itm)?; + } + Ok(()) + } + + pub fn resolve_references_for_filter( + &self, + filter: &Filter, + references: &mut HashMap, + ) -> Result<(), CubeError> { + for itm in filter.items.iter() { + self.resolve_references_for_filter_item(itm, references)?; + } + Ok(()) + } + + fn validate_filter_item(&self, item: &FilterItem) -> Result<(), CubeError> { + match item { + FilterItem::Item(item) => { + self.validate_member(item.member_evaluator().clone(), &None)? + } + FilterItem::Group(group) => { + for itm in group.items.iter() { + self.validate_filter_item(itm)? + } + } + } + Ok(()) + } + + fn resolve_references_for_filter_item( + &self, + item: &FilterItem, + references: &mut HashMap, + ) -> Result<(), CubeError> { + match item { + FilterItem::Item(item) => self.resolve_references_for_member( + item.member_evaluator().clone(), + &None, + references, + )?, + FilterItem::Group(group) => { + for itm in group.items.iter() { + self.resolve_references_for_filter_item(itm, references)? + } + } + } + Ok(()) + } + + fn has_source_for_leaf_memeber( + &self, + member: &Rc, + strict_source: &Option, + ) -> bool { + match &self.source.source { + crate::plan::FromSource::Empty => false, + crate::plan::FromSource::Single(source) => { + self.is_single_source_has_leaf_member(&source, member, strict_source) + } + crate::plan::FromSource::Join(join) => { + self.is_single_source_has_leaf_member(&join.root, member, strict_source) + || join.joins.iter().any(|itm| { + self.is_single_source_has_leaf_member(&itm.from, member, strict_source) + }) + } + } + } + + fn is_single_source_has_leaf_member( + &self, + source: &SingleAliasedSource, + member: &Rc, + strict_source: &Option, + ) -> bool { + if let Some(strict_source) = strict_source { + if strict_source != &source.alias { + return false; + } + } + + match &source.source { + SingleSource::Cube(cube) => { + cube.name() == &member.cube_name() && cube.has_member(&member.name()) + } + _ => false, + } + } + + pub fn resolve_alias_for_member( + &self, + member_name: &String, + strict_source: &Option, + ) -> Option { + if let Some(reference) = self.find_reference_for_member(member_name, strict_source) { + Some(reference.name().clone()) + } else { + None + } + } + + pub fn find_reference_for_member( + &self, + member_name: &String, + strict_source: &Option, + ) -> Option { + match &self.source.source { + crate::plan::FromSource::Empty => None, + crate::plan::FromSource::Single(source) => self + .find_reference_column_for_member_in_single_source( + &source, + member_name, + strict_source, + ), + crate::plan::FromSource::Join(join) => { + self.find_reference_column_for_member_in_join(&join, member_name, strict_source) + } + } + } + + fn find_reference_column_for_member_in_single_source( + &self, + source: &SingleAliasedSource, + member_name: &String, + strict_source: &Option, + ) -> Option { + if let Some(strict_source) = strict_source { + if strict_source != &source.alias { + return None; + } + } + let column_name = match &source.source { + SingleSource::Subquery(query_plan) => { + query_plan.schema().resolve_member_reference(member_name) + } + SingleSource::Cube(_) => None, + SingleSource::TableReference(_, schema) => schema.resolve_member_reference(member_name), + }; + column_name.map(|col| QualifiedColumnName::new(Some(source.alias.clone()), col)) + } + + fn find_reference_column_for_member_in_join( + &self, + join: &Rc, + member_name: &String, + strict_source: &Option, + ) -> Option { + if let Some(root_ref) = self.find_reference_column_for_member_in_single_source( + &join.root, + member_name, + strict_source, + ) { + return Some(root_ref); + } + join.joins.iter().find_map(|item| { + self.find_reference_column_for_member_in_single_source( + &item.from, + member_name, + strict_source, + ) + }) + } + + /* fn validate_reference(&self, reference: &QualifiedColumnName) -> Result<(), CubeError> { + if self.is_source_has_reference(reference) { + Ok(()) + } else { + Err(CubeError::internal(format!( + "Error while planning: schema does not have reference {}", + reference + ))) + } + } + + fn is_source_has_reference(&self, reference: &QualifiedColumnName) -> bool { + match &self.source.source { + crate::plan::FromSource::Empty => false, + crate::plan::FromSource::Single(source) => { + self.is_single_source_has_reference(&source, reference) + } + crate::plan::FromSource::Join(join) => self.is_join_has_reference(&join, reference), + } + } + + fn is_join_has_reference(&self, join: &Rc, reference: &QualifiedColumnName) -> bool { + if self.is_single_source_has_reference(&join.root, reference) { + return true; + } + join.joins + .iter() + .any(|item| self.is_single_source_has_reference(&item.from, reference)) + } + + fn is_single_source_has_reference( + &self, + source: &SingleAliasedSource, + reference: &QualifiedColumnName, + ) -> bool { + if let Some(reference_source) = &reference.source() { + if reference_source != &source.alias { + return false; + } + } + + match &source.source { + SingleSource::Subquery(query_plan) => query_plan.schema().has_column(reference.name()), + SingleSource::Cube(_) => false, + SingleSource::TableReference(_, schema) => schema.has_column(reference.name()), + } + } */ +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs new file mode 100644 index 0000000000000..35fed31484c9a --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs @@ -0,0 +1,158 @@ +use super::dependecy::{ContextSymbolDep, CubeDepProperty, CubeDependency, Dependency}; +use super::sql_nodes::SqlNode; +use super::{symbols::MemberSymbol, SqlEvaluatorVisitor}; +use crate::cube_bridge::memeber_sql::{ContextSymbolArg, MemberSql, MemberSqlArg, MemberSqlStruct}; +use crate::planner::query_tools::QueryTools; +use cubenativeutils::CubeError; +use std::rc::Rc; + +pub struct SqlCall { + member_sql: Rc, + deps: Vec, +} + +impl SqlCall { + pub fn new(member_sql: Rc, deps: Vec) -> Self { + Self { member_sql, deps } + } + + pub fn eval( + &self, + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + query_tools: Rc, + ) -> Result { + let args = self + .deps + .iter() + .map(|d| { + self.evaluate_single_dep(&d, visitor, node_processor.clone(), query_tools.clone()) + }) + .collect::, _>>()?; + self.member_sql.call(args) + } + + pub fn extract_symbol_deps(&self, result: &mut Vec>) { + for dep in self.deps.iter() { + match dep { + Dependency::SymbolDependency(dep) => result.push(dep.clone()), + Dependency::CubeDependency(cube_dep) => { + self.extract_symbol_deps_from_cube_dep(cube_dep, result) + } + Dependency::ContextDependency(_) => {} + } + } + } + + pub fn extract_cube_deps(&self, result: &mut Vec) { + for dep in self.deps.iter() { + match dep { + Dependency::SymbolDependency(_) => {} + Dependency::CubeDependency(cube_dep) => { + self.extract_cube_deps_from_cube_dep(cube_dep, result) + } + Dependency::ContextDependency(_) => {} + } + } + } + + fn extract_symbol_deps_from_cube_dep( + &self, + cube_dep: &CubeDependency, + result: &mut Vec>, + ) { + for (_, v) in cube_dep.properties.iter() { + match v { + CubeDepProperty::SymbolDependency(dep) => result.push(dep.clone()), + CubeDepProperty::CubeDependency(cube_dep) => { + self.extract_symbol_deps_from_cube_dep(cube_dep, result) + } + }; + } + } + + fn extract_cube_deps_from_cube_dep(&self, cube_dep: &CubeDependency, result: &mut Vec) { + result.push(cube_dep.cube_symbol.name()); + + for (_, v) in cube_dep.properties.iter() { + match v { + CubeDepProperty::CubeDependency(cube_dep) => { + self.extract_cube_deps_from_cube_dep(cube_dep, result) + } + _ => {} + }; + } + } + + fn evaluate_single_dep( + &self, + dep: &Dependency, + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + query_tools: Rc, + ) -> Result { + match dep { + Dependency::SymbolDependency(dep) => Ok(MemberSqlArg::String( + visitor.apply(dep, node_processor.clone())?, + )), + Dependency::CubeDependency(dep) => { + self.evaluate_cube_dep(dep, visitor, node_processor.clone(), query_tools.clone()) + } + Dependency::ContextDependency(contex_symbol) => { + self.apply_context_symbol(contex_symbol, query_tools.clone()) + } + } + } + + fn evaluate_cube_dep( + &self, + dep: &CubeDependency, + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + query_tools: Rc, + ) -> Result { + let mut res = MemberSqlStruct::default(); + if let Some(sql_fn) = &dep.sql_fn { + res.sql_fn = Some(visitor.apply(sql_fn, node_processor.clone())?); + } + if let Some(to_string_fn) = &dep.to_string_fn { + res.to_string_fn = Some(visitor.apply(to_string_fn, node_processor.clone())?); + } + for (k, v) in dep.properties.iter() { + let prop_res = match v { + CubeDepProperty::SymbolDependency(dep) => { + MemberSqlArg::String(visitor.apply(&dep, node_processor.clone())?) + } + CubeDepProperty::CubeDependency(dep) => self.evaluate_cube_dep( + &dep, + visitor, + node_processor.clone(), + query_tools.clone(), + )?, + }; + res.properties.insert(k.clone(), prop_res); + } + Ok(MemberSqlArg::Struct(res)) + } + + pub fn apply_context_symbol( + &self, + context_symbol: &ContextSymbolDep, + query_tools: Rc, + ) -> Result { + let res = match context_symbol { + ContextSymbolDep::SecurityContext => { + MemberSqlArg::ContextSymbol(ContextSymbolArg::SecurityContext( + query_tools.base_tools().security_context_for_rust()?, + )) + } + ContextSymbolDep::FilterParams => MemberSqlArg::ContextSymbol( + ContextSymbolArg::FilterParams(query_tools.base_tools().filters_proxy()?), + ), + ContextSymbolDep::FilterGroup => MemberSqlArg::ContextSymbol( + ContextSymbolArg::FilterGroup(query_tools.base_tools().filter_group_function()?), + ), + }; + Ok(res) + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/mod.rs deleted file mode 100644 index 1c0aeef099360..0000000000000 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod set_schema; - -pub use set_schema::set_schema; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs deleted file mode 100644 index d7e580edcac19..0000000000000 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs +++ /dev/null @@ -1,86 +0,0 @@ -use crate::plan::schema::Schema; -use crate::planner::sql_evaluator::sql_nodes::final_measure::FinalMeasureSqlNode; -use crate::planner::sql_evaluator::sql_nodes::{ - AutoPrefixSqlNode, EvaluateSqlNode, MeasureFilterSqlNode, MultiStageRankNode, - MultiStageWindowNode, RenderReferencesSqlNode, RootSqlNode, SqlNode, TimeShiftSqlNode, -}; -use std::rc::Rc; - -pub fn set_schema(node_processors: Rc, schema: Rc) -> Rc { - set_schema_impl(node_processors, schema) -} - -pub fn set_schema_impl(sql_node: Rc, schema: Rc) -> Rc { - if let Some(auto_prefix) = sql_node - .clone() - .as_any() - .downcast_ref::() - { - let input = set_schema_impl(auto_prefix.input().clone(), schema.clone()); - AutoPrefixSqlNode::new_with_schema(input, schema) - } else if let Some(_) = sql_node.clone().as_any().downcast_ref::() { - sql_node - } else if let Some(final_measure) = sql_node - .clone() - .as_any() - .downcast_ref::() - { - let input = set_schema_impl(final_measure.input().clone(), schema.clone()); - FinalMeasureSqlNode::new(input) - } else if let Some(measure_filter) = sql_node - .clone() - .as_any() - .downcast_ref::() - { - let input = set_schema_impl(measure_filter.input().clone(), schema.clone()); - MeasureFilterSqlNode::new(input) - } else if let Some(multi_stage_rank) = sql_node - .clone() - .as_any() - .downcast_ref::() - { - let else_processor = - set_schema_impl(multi_stage_rank.else_processor().clone(), schema.clone()); - MultiStageRankNode::new(else_processor, multi_stage_rank.partition().clone()) - } else if let Some(multi_stage_window) = sql_node - .clone() - .as_any() - .downcast_ref::() - { - let input = set_schema_impl(multi_stage_window.input().clone(), schema.clone()); - let else_processor = - set_schema_impl(multi_stage_window.else_processor().clone(), schema.clone()); - MultiStageWindowNode::new( - input, - else_processor, - multi_stage_window.partition().clone(), - ) - } else if let Some(render_references) = sql_node - .clone() - .as_any() - .downcast_ref::() - { - let input = set_schema_impl(render_references.input().clone(), schema.clone()); - RenderReferencesSqlNode::new_with_schema(input, schema) - } else if let Some(root_node) = sql_node.clone().as_any().downcast_ref::() { - let dimension_processor = - set_schema_impl(root_node.dimension_processor().clone(), schema.clone()); - let measure_processor = - set_schema_impl(root_node.measure_processor().clone(), schema.clone()); - let cube_name_processor = - set_schema_impl(root_node.cube_name_processor().clone(), schema.clone()); - let default_processor = - set_schema_impl(root_node.default_processor().clone(), schema.clone()); - RootSqlNode::new( - dimension_processor, - measure_processor, - cube_name_processor, - default_processor, - ) - } else if let Some(time_shift) = sql_node.clone().as_any().downcast_ref::() { - let input = set_schema_impl(time_shift.input().clone(), schema.clone()); - TimeShiftSqlNode::new(time_shift.shifts().clone(), input) - } else { - unreachable!("Not all nodes are implemented"); - } -} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/auto_prefix.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/auto_prefix.rs index e0ce9d3093ab9..708242bb1bcdc 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/auto_prefix.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/auto_prefix.rs @@ -1,27 +1,38 @@ use super::SqlNode; use crate::plan::Schema; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbol, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; +use std::collections::HashMap; use std::rc::Rc; pub struct AutoPrefixSqlNode { input: Rc, + cube_references: HashMap, schema: Rc, } impl AutoPrefixSqlNode { - pub fn new(input: Rc) -> Rc { + pub fn new(input: Rc, cube_references: HashMap) -> Rc { Rc::new(Self { input, + cube_references, schema: Rc::new(Schema::empty()), }) } - pub fn new_with_schema(input: Rc, schema: Rc) -> Rc { - Rc::new(Self { input, schema }) + pub fn new_with_schema( + input: Rc, + cube_references: HashMap, + schema: Rc, + ) -> Rc { + Rc::new(Self { + input, + schema, + cube_references, + }) } pub fn input(&self) -> &Rc { @@ -31,30 +42,42 @@ impl AutoPrefixSqlNode { pub fn schema(&self) -> &Rc { &self.schema } + + pub fn cube_references(&self) -> &HashMap { + &self.cube_references + } + + fn resolve_cube_alias(&self, name: &String) -> String { + if let Some(alias) = self.cube_references.get(name) { + alias.clone() + } else { + name.clone() + } + } } impl SqlNode for AutoPrefixSqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { let input = self.input .to_sql(visitor, node, query_tools.clone(), node_processor.clone())?; - let res = match node.symbol() { - MemberSymbolType::Dimension(ev) => { - let cube_alias = self.schema.resolve_cube_alias(&ev.cube_name()); + let res = match node.as_ref() { + MemberSymbol::Dimension(ev) => { + let cube_alias = self.resolve_cube_alias(&ev.cube_name()); query_tools.auto_prefix_with_cube_name(&cube_alias, &input) } - MemberSymbolType::Measure(ev) => { - let cube_alias = self.schema.resolve_cube_alias(&ev.cube_name()); + MemberSymbol::Measure(ev) => { + let cube_alias = self.resolve_cube_alias(&ev.cube_name()); query_tools.auto_prefix_with_cube_name(&cube_alias, &input) } - MemberSymbolType::CubeName(_) => { - let cube_alias = self.schema.resolve_cube_alias(&input); + MemberSymbol::CubeName(_) => { + let cube_alias = self.resolve_cube_alias(&input); query_tools.escape_column_name(&cube_alias) } _ => input, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs index 0acd90c5f868d..63587cb9289ed 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs @@ -1,7 +1,7 @@ use super::SqlNode; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; @@ -17,18 +17,22 @@ impl EvaluateSqlNode { impl SqlNode for EvaluateSqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, - _query_tools: Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, node_processor: Rc, ) -> Result { - let args = visitor.evaluate_deps(node, node_processor.clone())?; - match node.symbol() { - MemberSymbolType::Dimension(ev) => ev.evaluate_sql(args), - MemberSymbolType::Measure(ev) => ev.evaluate_sql(args), - MemberSymbolType::CubeTable(ev) => ev.evaluate_sql(args), - MemberSymbolType::CubeName(ev) => ev.evaluate_sql(args), - MemberSymbolType::SimpleSql(ev) => ev.evaluate_sql(args), + match node.as_ref() { + MemberSymbol::Dimension(ev) => { + ev.evaluate_sql(visitor, node_processor.clone(), query_tools.clone()) + } + MemberSymbol::Measure(ev) => { + ev.evaluate_sql(visitor, node_processor.clone(), query_tools.clone()) + } + MemberSymbol::CubeTable(ev) => { + ev.evaluate_sql(visitor, node_processor.clone(), query_tools.clone()) + } + MemberSymbol::CubeName(ev) => ev.evaluate_sql(), } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index 9d8729896e84d..a288558b292e8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -1,83 +1,184 @@ +use super::leaf_time_dimension::LeafTimeDimensionNode; use super::{ AutoPrefixSqlNode, EvaluateSqlNode, FinalMeasureSqlNode, MeasureFilterSqlNode, - MultiStageRankNode, MultiStageWindowNode, RenderReferencesSqlNode, RootSqlNode, SqlNode, - TimeShiftSqlNode, + MultiStageRankNode, MultiStageWindowNode, RenderReferencesSqlNode, RollingWindowNode, + RootSqlNode, SqlNode, TimeShiftSqlNode, UngroupedMeasureSqlNode, + UngroupedQueryFinalMeasureSqlNode, }; +use crate::plan::schema::QualifiedColumnName; use std::collections::HashMap; use std::rc::Rc; +#[derive(Clone)] pub struct SqlNodesFactory { - time_shifts: Option>, + time_shifts: HashMap, + ungrouped: bool, + ungrouped_measure: bool, + render_references: HashMap, + ungrouped_measure_references: HashMap, + leaf_time_dimensions: HashMap, + cube_name_references: HashMap, + multi_stage_rank: Option>, //partition_by + multi_stage_window: Option>, //partition_by + rolling_window: bool, } impl SqlNodesFactory { - pub fn new() -> Rc { - Rc::new(Self { time_shifts: None }) + pub fn new() -> Self { + Self { + time_shifts: HashMap::new(), + ungrouped: false, + ungrouped_measure: false, + render_references: HashMap::new(), + ungrouped_measure_references: HashMap::new(), + cube_name_references: HashMap::new(), + leaf_time_dimensions: HashMap::new(), + multi_stage_rank: None, + multi_stage_window: None, + rolling_window: false, + } + } + + pub fn set_time_shifts(&mut self, time_shifts: HashMap) { + self.time_shifts = time_shifts; + } + + pub fn set_ungrouped(&mut self, value: bool) { + self.ungrouped = value; + } + + pub fn set_ungrouped_measure(&mut self, value: bool) { + self.ungrouped_measure = value; + } + + pub fn set_render_references(&mut self, value: HashMap) { + self.render_references = value; + } + + pub fn render_references(&self) -> &HashMap { + &self.render_references + } + + pub fn add_render_reference(&mut self, key: String, value: QualifiedColumnName) { + self.render_references.insert(key, value); + } + + pub fn set_multi_stage_rank(&mut self, partition_by: Vec) { + self.multi_stage_rank = Some(partition_by); } - pub fn new_with_time_shifts(time_shifts: HashMap) -> Rc { - Rc::new(Self { - time_shifts: Some(time_shifts), - }) + + pub fn add_leaf_time_dimension(&mut self, dimension_name: &String, granularity: &String) { + self.leaf_time_dimensions + .insert(dimension_name.clone(), granularity.clone()); + } + + pub fn set_multi_stage_window(&mut self, partition_by: Vec) { + self.multi_stage_window = Some(partition_by); + } + + pub fn set_rolling_window(&mut self, value: bool) { + self.rolling_window = value; + } + + pub fn set_ungrouped_measure_references( + &mut self, + value: HashMap, + ) { + self.ungrouped_measure_references = value; } + + pub fn add_ungrouped_measure_reference(&mut self, key: String, value: QualifiedColumnName) { + self.ungrouped_measure_references.insert(key, value); + } + + pub fn set_cube_name_references(&mut self, value: HashMap) { + self.cube_name_references = value; + } + + pub fn add_cube_name_reference(&mut self, key: String, value: String) { + self.cube_name_references.insert(key, value); + } + pub fn default_node_processor(&self) -> Rc { let evaluate_sql_processor = EvaluateSqlNode::new(); - let auto_prefix_processor = AutoPrefixSqlNode::new(evaluate_sql_processor.clone()); - let measure_filter_processor = MeasureFilterSqlNode::new(auto_prefix_processor.clone()); - let final_measure_processor = FinalMeasureSqlNode::new(measure_filter_processor.clone()); - let root_node = RootSqlNode::new( - self.dimension_processor(auto_prefix_processor.clone()), - final_measure_processor.clone(), - auto_prefix_processor.clone(), + let auto_prefix_processor = AutoPrefixSqlNode::new( evaluate_sql_processor.clone(), + self.cube_name_references.clone(), ); - RenderReferencesSqlNode::new(root_node) - } - pub fn multi_stage_rank_node_processor(&self, partition: Vec) -> Rc { - let evaluate_sql_processor = EvaluateSqlNode::new(); - let auto_prefix_processor = AutoPrefixSqlNode::new(evaluate_sql_processor.clone()); let measure_filter_processor = MeasureFilterSqlNode::new(auto_prefix_processor.clone()); - let final_measure_processor = FinalMeasureSqlNode::new(measure_filter_processor.clone()); - let rank_processor = MultiStageRankNode::new(final_measure_processor.clone(), partition); + let measure_processor = + self.add_ungrouped_measure_reference_if_needed(measure_filter_processor.clone()); + let measure_processor = self.final_measure_node_processor(measure_processor); + let measure_processor = self + .add_multi_stage_window_if_needed(measure_processor, measure_filter_processor.clone()); + let measure_processor = self.add_multi_stage_rank_if_needed(measure_processor); - let root_processor = RootSqlNode::new( + let root_node = RootSqlNode::new( self.dimension_processor(auto_prefix_processor.clone()), - rank_processor.clone(), + measure_processor.clone(), auto_prefix_processor.clone(), evaluate_sql_processor.clone(), ); - let references_processor = RenderReferencesSqlNode::new(root_processor); - references_processor + RenderReferencesSqlNode::new(root_node, self.render_references.clone()) } - pub fn multi_stage_window_node_processor(&self, partition: Vec) -> Rc { - let evaluate_sql_processor = EvaluateSqlNode::new(); - let auto_prefix_processor = AutoPrefixSqlNode::new(evaluate_sql_processor.clone()); - let measure_filter_processor = MeasureFilterSqlNode::new(auto_prefix_processor.clone()); - let final_measure_processor = FinalMeasureSqlNode::new(measure_filter_processor.clone()); + fn add_ungrouped_measure_reference_if_needed( + &self, + default: Rc, + ) -> Rc { + if !self.ungrouped_measure_references.is_empty() { + RenderReferencesSqlNode::new(default, self.ungrouped_measure_references.clone()) + } else { + default + } + } - let window_processor = MultiStageWindowNode::new( - evaluate_sql_processor.clone(), - final_measure_processor.clone(), - partition, - ); + fn add_multi_stage_rank_if_needed(&self, default: Rc) -> Rc { + if let Some(partition_by) = &self.multi_stage_rank { + MultiStageRankNode::new(default, partition_by.clone()) + } else { + default + } + } - let root_processor = RootSqlNode::new( - self.dimension_processor(auto_prefix_processor.clone()), - window_processor.clone(), - auto_prefix_processor.clone(), - evaluate_sql_processor.clone(), - ); - let references_processor = RenderReferencesSqlNode::new(root_processor); - references_processor + fn add_multi_stage_window_if_needed( + &self, + default: Rc, + multi_stage_input: Rc, + ) -> Rc { + if let Some(partition_by) = &self.multi_stage_window { + MultiStageWindowNode::new(multi_stage_input, default, partition_by.clone()) + } else { + default + } + } + + fn final_measure_node_processor(&self, input: Rc) -> Rc { + if self.ungrouped_measure { + UngroupedMeasureSqlNode::new(input) + } else if self.ungrouped { + UngroupedQueryFinalMeasureSqlNode::new(input) + } else if self.rolling_window { + RollingWindowNode::new(input) + } else { + FinalMeasureSqlNode::new(input) + } } fn dimension_processor(&self, input: Rc) -> Rc { - if let Some(time_shifts) = &self.time_shifts { - TimeShiftSqlNode::new(time_shifts.clone(), input) + let input = if !&self.time_shifts.is_empty() { + TimeShiftSqlNode::new(self.time_shifts.clone(), input) } else { input - } + }; + + let input = if !&self.leaf_time_dimensions.is_empty() { + LeafTimeDimensionNode::new(input, self.leaf_time_dimensions.clone()) + } else { + input + }; + input } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs index 2fd786a986f9e..c844c2a786de4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs @@ -1,7 +1,7 @@ use super::SqlNode; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; @@ -23,24 +23,30 @@ impl FinalMeasureSqlNode { impl SqlNode for FinalMeasureSqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { - let res = match node.symbol() { - MemberSymbolType::Measure(ev) => { + let res = match node.as_ref() { + MemberSymbol::Measure(ev) => { let input = self.input.to_sql( visitor, node, query_tools.clone(), node_processor.clone(), )?; + //}; if ev.is_calculated() { input } else { - let measure_type = ev.measure_type(); + let measure_type = if ev.measure_type() == "runningTotal" { + "sum" + } else { + &ev.measure_type() + }; + format!("{}({})", measure_type, input) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/leaf_time_dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/leaf_time_dimension.rs new file mode 100644 index 0000000000000..92c06ec1e6ea1 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/leaf_time_dimension.rs @@ -0,0 +1,55 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; +use crate::planner::sql_evaluator::SqlEvaluatorVisitor; +use cubenativeutils::CubeError; +use std::any::Any; +use std::collections::HashMap; +use std::rc::Rc; + +pub struct LeafTimeDimensionNode { + input: Rc, + leaf_time_dimensions: HashMap, +} + +impl LeafTimeDimensionNode { + pub fn new(input: Rc, leaf_time_dimensions: HashMap) -> Rc { + Rc::new(Self { + input, + leaf_time_dimensions, + }) + } +} + +impl SqlNode for LeafTimeDimensionNode { + fn to_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + ) -> Result { + let full_name = node.full_name(); + let input_sql = self + .input + .to_sql(visitor, node, query_tools.clone(), node_processor)?; + + let res = if let Some(granularity) = self.leaf_time_dimensions.get(&full_name) { + let converted_tz = query_tools.base_tools().convert_tz(input_sql)?; + query_tools + .base_tools() + .time_grouped_column(granularity.clone(), converted_tz)? + } else { + input_sql + }; + Ok(res) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/measure_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/measure_filter.rs index 52a6683190fea..e9f7c771c77e6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/measure_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/measure_filter.rs @@ -1,7 +1,7 @@ use super::SqlNode; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; @@ -23,16 +23,16 @@ impl MeasureFilterSqlNode { impl SqlNode for MeasureFilterSqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { let input = self.input .to_sql(visitor, node, query_tools.clone(), node_processor.clone())?; - let res = match node.symbol() { - MemberSymbolType::Measure(ev) => { + let res = match node.as_ref() { + MemberSymbol::Measure(ev) => { let measure_filters = ev.measure_filters(); if !measure_filters.is_empty() { let filters = measure_filters @@ -40,7 +40,11 @@ impl SqlNode for MeasureFilterSqlNode { .map(|filter| -> Result { Ok(format!( "({})", - visitor.apply(filter, node_processor.clone())? + filter.eval( + &visitor, + node_processor.clone(), + query_tools.clone() + )? )) }) .collect::, _>>()? diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs index 9cb6190581a9d..346fb326cc689 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs @@ -2,13 +2,17 @@ pub mod auto_prefix; pub mod evaluate_sql; pub mod factory; pub mod final_measure; +pub mod leaf_time_dimension; pub mod measure_filter; pub mod multi_stage_rank; pub mod multi_stage_window; pub mod render_references; +pub mod rolling_window; pub mod root_processor; pub mod sql_node; pub mod time_shift; +pub mod ungroupped_measure; +pub mod ungroupped_query_final_measure; pub use auto_prefix::AutoPrefixSqlNode; pub use evaluate_sql::EvaluateSqlNode; @@ -18,6 +22,9 @@ pub use measure_filter::MeasureFilterSqlNode; pub use multi_stage_rank::MultiStageRankNode; pub use multi_stage_window::MultiStageWindowNode; pub use render_references::RenderReferencesSqlNode; +pub use rolling_window::RollingWindowNode; pub use root_processor::RootSqlNode; pub use sql_node::SqlNode; pub use time_shift::TimeShiftSqlNode; +pub use ungroupped_measure::UngroupedMeasureSqlNode; +pub use ungroupped_query_final_measure::UngroupedQueryFinalMeasureSqlNode; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_rank.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_rank.rs index 79780132b46a5..493de8228d6c9 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_rank.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_rank.rs @@ -1,7 +1,7 @@ use super::SqlNode; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; @@ -31,21 +31,24 @@ impl MultiStageRankNode { impl SqlNode for MultiStageRankNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { - let res = match node.symbol() { - MemberSymbolType::Measure(m) => { + let res = match node.as_ref() { + MemberSymbol::Measure(m) => { if m.is_multi_stage() && m.measure_type() == "rank" { let order_by = if !m.measure_order_by().is_empty() { let sql = m .measure_order_by() .iter() .map(|item| -> Result { - let sql = visitor - .apply(item.evaluation_node(), node_processor.clone())?; + let sql = item.sql_call().eval( + visitor, + node_processor.clone(), + query_tools.clone(), + )?; Ok(format!("{} {}", sql, item.direction())) }) .collect::, _>>()? diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_window.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_window.rs index 5007ccc7a0b95..4d1fc74c6b2b0 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_window.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/multi_stage_window.rs @@ -1,6 +1,6 @@ use super::SqlNode; use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType, SqlEvaluatorVisitor}; +use crate::planner::sql_evaluator::{MemberSymbol, SqlEvaluatorVisitor}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; @@ -40,13 +40,13 @@ impl MultiStageWindowNode { impl SqlNode for MultiStageWindowNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { - let res = match node.symbol() { - MemberSymbolType::Measure(m) => { + let res = match node.as_ref() { + MemberSymbol::Measure(m) => { if m.is_multi_stage() && !m.is_calculated() { let input_sql = self.input.to_sql( visitor, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/render_references.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/render_references.rs index 6f6a94ec01070..9d4e17a8687ce 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/render_references.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/render_references.rs @@ -1,27 +1,24 @@ use super::SqlNode; -use crate::plan::Schema; +use crate::plan::QualifiedColumnName; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; +use std::collections::HashMap; use std::rc::Rc; pub struct RenderReferencesSqlNode { input: Rc, - schema: Rc, + references: HashMap, } impl RenderReferencesSqlNode { - pub fn new(input: Rc) -> Rc { - Rc::new(Self { - input, - schema: Rc::new(Schema::empty()), - }) - } - - pub fn new_with_schema(input: Rc, schema: Rc) -> Rc { - Rc::new(Self { input, schema }) + pub fn new( + input: Rc, + references: HashMap, + ) -> Rc { + Rc::new(Self { input, references }) } pub fn input(&self) -> &Rc { @@ -32,30 +29,21 @@ impl RenderReferencesSqlNode { impl SqlNode for RenderReferencesSqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { - let reference_column = match node.symbol() { - MemberSymbolType::Dimension(ev) => { - self.schema.find_column_for_member(&ev.full_name(), &None) - } - MemberSymbolType::Measure(ev) => { - self.schema.find_column_for_member(&ev.full_name(), &None) - } - _ => None, - }; - - if let Some(reference_column) = reference_column { - let table_ref = reference_column.table_name.as_ref().map_or_else( + let full_name = node.full_name(); + if let Some(reference) = self.references.get(&full_name) { + let table_ref = reference.source().as_ref().map_or_else( || format!(""), |table_name| format!("{}.", query_tools.escape_column_name(table_name)), ); Ok(format!( "{}{}", table_ref, - query_tools.escape_column_name(&reference_column.alias) + query_tools.escape_column_name(&reference.name()) )) } else { self.input diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs new file mode 100644 index 0000000000000..b6bcc87431855 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs @@ -0,0 +1,66 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::{MemberSymbol, SqlEvaluatorVisitor}; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +pub struct RollingWindowNode { + input: Rc, +} + +impl RollingWindowNode { + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) + } + + pub fn input(&self) -> &Rc { + &self.input + } +} + +impl SqlNode for RollingWindowNode { + fn to_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + ) -> Result { + let res = match node.as_ref() { + MemberSymbol::Measure(m) => { + let input = + self.input + .to_sql(visitor, node, query_tools.clone(), node_processor)?; + if m.is_cumulative() { + let aggregate_function = if m.measure_type() == "sum" + || m.measure_type() == "count" + || m.measure_type() == "runningTotal" + { + "sum" + } else { + m.measure_type() + }; + + format!("{}({})", aggregate_function, input) + } else { + input + } + } + _ => { + return Err(CubeError::internal(format!( + "Unexpected evaluation node type for RollingWindowNode" + ))); + } + }; + Ok(res) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs index 6788f7115000c..16696687864aa 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/root_processor.rs @@ -1,7 +1,7 @@ use super::SqlNode; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; @@ -48,25 +48,25 @@ impl RootSqlNode { impl SqlNode for RootSqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { - let res = match node.symbol() { - MemberSymbolType::Dimension(_) => self.dimension_processor.to_sql( + let res = match node.as_ref() { + MemberSymbol::Dimension(_) => self.dimension_processor.to_sql( visitor, node, query_tools.clone(), node_processor.clone(), )?, - MemberSymbolType::Measure(_) => self.measure_processor.to_sql( + MemberSymbol::Measure(_) => self.measure_processor.to_sql( visitor, node, query_tools.clone(), node_processor.clone(), )?, - MemberSymbolType::CubeName(_) => self.cube_name_processor.to_sql( + MemberSymbol::CubeName(_) => self.cube_name_processor.to_sql( visitor, node, query_tools.clone(), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/sql_node.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/sql_node.rs index 1415f1d24bfc8..1755c36e2817d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/sql_node.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/sql_node.rs @@ -1,5 +1,5 @@ use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{EvaluationNode, SqlEvaluatorVisitor}; +use crate::planner::sql_evaluator::{MemberSymbol, SqlEvaluatorVisitor}; use cubenativeutils::CubeError; use std::any::Any; use std::rc::Rc; @@ -7,8 +7,8 @@ use std::rc::Rc; pub trait SqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result; @@ -17,3 +17,7 @@ pub trait SqlNode { fn childs(&self) -> Vec>; } + +pub trait CubeNameNode { + fn to_sql(&self, cube_name: &String) -> Result; +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs index 677095ded21ef..f3b30dd7f1819 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs @@ -1,7 +1,7 @@ use super::SqlNode; use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; -use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; use cubenativeutils::CubeError; use std::any::Any; use std::collections::HashMap; @@ -29,16 +29,16 @@ impl TimeShiftSqlNode { impl SqlNode for TimeShiftSqlNode { fn to_sql( &self, - visitor: &mut SqlEvaluatorVisitor, - node: &Rc, + visitor: &SqlEvaluatorVisitor, + node: &Rc, query_tools: Rc, node_processor: Rc, ) -> Result { let input = self.input .to_sql(visitor, node, query_tools.clone(), node_processor.clone())?; - let res = match node.symbol() { - MemberSymbolType::Dimension(ev) => { + let res = match node.as_ref() { + MemberSymbol::Dimension(ev) => { if let Some(shift) = self.shifts.get(&ev.full_name()) { format!("({input} + interval '{shift}')") } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_measure.rs new file mode 100644 index 0000000000000..1b2f9675a7492 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_measure.rs @@ -0,0 +1,62 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; +use crate::planner::sql_evaluator::SqlEvaluatorVisitor; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +pub struct UngroupedMeasureSqlNode { + input: Rc, +} + +impl UngroupedMeasureSqlNode { + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) + } + + pub fn input(&self) -> &Rc { + &self.input + } +} + +impl SqlNode for UngroupedMeasureSqlNode { + fn to_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + ) -> Result { + let res = match node.as_ref() { + MemberSymbol::Measure(_) => { + let input = self.input.to_sql( + visitor, + node, + query_tools.clone(), + node_processor.clone(), + )?; + + if input == "*" { + "1".to_string() + } else { + input + } + } + _ => { + return Err(CubeError::internal(format!( + "Measure filter node processor called for wrong node", + ))); + } + }; + Ok(res) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_query_final_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_query_final_measure.rs new file mode 100644 index 0000000000000..c77afa191b65c --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_query_final_measure.rs @@ -0,0 +1,69 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::MemberSymbol; +use crate::planner::sql_evaluator::SqlEvaluatorVisitor; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +pub struct UngroupedQueryFinalMeasureSqlNode { + input: Rc, +} + +impl UngroupedQueryFinalMeasureSqlNode { + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) + } + + pub fn input(&self) -> &Rc { + &self.input + } +} + +impl SqlNode for UngroupedQueryFinalMeasureSqlNode { + fn to_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + ) -> Result { + let res = match node.as_ref() { + MemberSymbol::Measure(ev) => { + let input = self.input.to_sql( + visitor, + node, + query_tools.clone(), + node_processor.clone(), + )?; + + if input == "*" { + "1".to_string() + } else { + if ev.measure_type() == "count" + || ev.measure_type() == "countDistinct" + || ev.measure_type() == "countDistinctApprox" + { + format!("CASE WHEN ({}) IS NOT NULL THEN 1 END", input) //TODO templates!! + } else { + input + } + } + } + _ => { + return Err(CubeError::internal(format!( + "Measure filter node processor called for wrong node", + ))); + } + }; + Ok(res) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs index 9c1a83574ea91..6a583bd86a135 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs @@ -1,7 +1,5 @@ -use super::dependecy::{ContextSymbolDep, Dependency}; use super::sql_nodes::SqlNode; -use super::EvaluationNode; -use crate::cube_bridge::memeber_sql::{ContextSymbolArg, MemberSqlArg, MemberSqlStruct}; +use super::MemberSymbol; use crate::planner::query_tools::QueryTools; use cubenativeutils::CubeError; use std::rc::Rc; @@ -17,80 +15,12 @@ impl SqlEvaluatorVisitor { } pub fn apply( - &mut self, - node: &Rc, + &self, + node: &Rc, node_processor: Rc, ) -> Result { let result = node_processor.to_sql(self, node, self.query_tools.clone(), node_processor.clone())?; Ok(result) } - - pub fn apply_context_symbol( - &mut self, - context_symbol: &ContextSymbolDep, - ) -> Result { - let res = match context_symbol { - ContextSymbolDep::SecurityContext => { - MemberSqlArg::ContextSymbol(ContextSymbolArg::SecurityContext( - self.query_tools.base_tools().security_context_for_rust()?, - )) - } - ContextSymbolDep::FilterParams => MemberSqlArg::ContextSymbol( - ContextSymbolArg::FilterParams(self.query_tools.base_tools().filters_proxy()?), - ), - ContextSymbolDep::FilterGroup => { - MemberSqlArg::ContextSymbol(ContextSymbolArg::FilterGroup( - self.query_tools.base_tools().filter_group_function()?, - )) - } - }; - Ok(res) - } - - pub fn evaluate_deps( - &mut self, - node: &Rc, - node_processor: Rc, - ) -> Result, CubeError> { - node.deps() - .iter() - .map(|d| self.evaluate_single_dep(&d, node_processor.clone())) - .collect() - } - - fn evaluate_single_dep( - &mut self, - dep: &Dependency, - node_processor: Rc, - ) -> Result { - match dep { - Dependency::SingleDependency(dep) => Ok(MemberSqlArg::String( - self.apply(dep, node_processor.clone())?, - )), - Dependency::StructDependency(dep) => { - let mut res = MemberSqlStruct::default(); - if let Some(sql_fn) = &dep.sql_fn { - res.sql_fn = Some(self.apply(sql_fn, node_processor.clone())?); - } - if let Some(to_string_fn) = &dep.to_string_fn { - res.to_string_fn = Some(self.apply(to_string_fn, node_processor.clone())?); - } - for (k, v) in dep.properties.iter() { - match v { - Dependency::SingleDependency(dep) => { - res.properties - .insert(k.clone(), self.apply(dep, node_processor.clone())?); - } - Dependency::StructDependency(_) => unimplemented!(), - Dependency::ContextDependency(_) => unimplemented!(), - } - } - Ok(MemberSqlArg::Struct(res)) - } - Dependency::ContextDependency(contex_symbol) => { - self.apply_context_symbol(contex_symbol) - } - } - } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs index d6a24c0cc56c2..263d28e88c7b6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/cube_symbol.rs @@ -1,8 +1,9 @@ -use super::SymbolFactory; +use super::{MemberSymbol, SymbolFactory}; use crate::cube_bridge::cube_definition::CubeDefinition; use crate::cube_bridge::evaluator::CubeEvaluator; -use crate::cube_bridge::memeber_sql::{MemberSql, MemberSqlArg}; -use crate::planner::sql_evaluator::{dependecy::Dependency, Compiler, EvaluationNode}; +use crate::cube_bridge::memeber_sql::MemberSql; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::{sql_nodes::SqlNode, Compiler, SqlCall, SqlEvaluatorVisitor}; use cubenativeutils::CubeError; use lazy_static::lazy_static; use regex::Regex; @@ -17,7 +18,7 @@ impl CubeNameSymbol { Self { cube_name } } - pub fn evaluate_sql(&self, _args: Vec) -> Result { + pub fn evaluate_sql(&self) -> Result { Ok(self.cube_name.clone()) } pub fn cube_name(&self) -> &String { @@ -58,21 +59,15 @@ impl SymbolFactory for CubeNameSymbolFactory { Ok(vec![]) } - fn build( - self, - _deps: Vec, - _compiler: &mut Compiler, - ) -> Result, CubeError> { + fn build(self, _compiler: &mut Compiler) -> Result, CubeError> { let Self { cube_name } = self; - Ok(EvaluationNode::new_cube_name(CubeNameSymbol::new( - cube_name, - ))) + Ok(MemberSymbol::new_cube_name(CubeNameSymbol::new(cube_name))) } } pub struct CubeTableSymbol { cube_name: String, - member_sql: Rc, + member_sql: Rc, #[allow(dead_code)] definition: Rc, is_table_sql: bool, @@ -81,7 +76,7 @@ pub struct CubeTableSymbol { impl CubeTableSymbol { pub fn new( cube_name: String, - member_sql: Rc, + member_sql: Rc, definition: Rc, is_table_sql: bool, ) -> Self { @@ -92,12 +87,18 @@ impl CubeTableSymbol { is_table_sql, } } - pub fn evaluate_sql(&self, args: Vec) -> Result { + + pub fn evaluate_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + query_tools: Rc, + ) -> Result { lazy_static! { static ref SIMPLE_ASTERIX_RE: Regex = Regex::new(r#"(?i)^\s*select\s+\*\s+from\s+([a-zA-Z0-9_\-`".*]+)\s*$"#).unwrap(); } - let sql = self.member_sql.call(args)?; + let sql = self.member_sql.eval(visitor, node_processor, query_tools)?; let res = if self.is_table_sql { sql } else { @@ -169,20 +170,19 @@ impl SymbolFactory for CubeTableSymbolFactory { Some(self.sql.clone()) } - fn build( - self, - deps: Vec, - _compiler: &mut Compiler, - ) -> Result, CubeError> { + fn build(self, compiler: &mut Compiler) -> Result, CubeError> { let Self { cube_name, sql, definition, is_table_sql, } = self; - Ok(EvaluationNode::new_cube_table( - CubeTableSymbol::new(cube_name, sql, definition, is_table_sql), - deps, - )) + let sql = compiler.compile_sql_call(&cube_name, sql)?; + Ok(MemberSymbol::new_cube_table(CubeTableSymbol::new( + cube_name, + sql, + definition, + is_table_sql, + ))) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs index 8286888a6a7f6..40ec863b6e8ff 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs @@ -1,15 +1,16 @@ use super::{MemberSymbol, SymbolFactory}; use crate::cube_bridge::dimension_definition::DimensionDefinition; use crate::cube_bridge::evaluator::CubeEvaluator; -use crate::cube_bridge::memeber_sql::{MemberSql, MemberSqlArg}; -use crate::planner::sql_evaluator::{Compiler, Dependency, EvaluationNode}; +use crate::cube_bridge::memeber_sql::MemberSql; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::{sql_nodes::SqlNode, Compiler, SqlCall, SqlEvaluatorVisitor}; use cubenativeutils::CubeError; use std::rc::Rc; pub struct DimensionSymbol { cube_name: String, name: String, - member_sql: Rc, + member_sql: Rc, #[allow(dead_code)] definition: Rc, } @@ -18,7 +19,7 @@ impl DimensionSymbol { pub fn new( cube_name: String, name: String, - member_sql: Rc, + member_sql: Rc, definition: Rc, ) -> Self { Self { @@ -29,8 +30,13 @@ impl DimensionSymbol { } } - pub fn evaluate_sql(&self, args: Vec) -> Result { - let sql = self.member_sql.call(args)?; + pub fn evaluate_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + query_tools: Rc, + ) -> Result { + let sql = self.member_sql.eval(visitor, node_processor, query_tools)?; Ok(sql) } @@ -45,14 +51,23 @@ impl DimensionSymbol { pub fn is_multi_stage(&self) -> bool { self.definition.static_data().multi_stage.unwrap_or(false) } -} + pub fn get_dependencies(&self) -> Vec> { + let mut deps = vec![]; + self.member_sql.extract_symbol_deps(&mut deps); + deps + } -impl MemberSymbol for DimensionSymbol { - fn cube_name(&self) -> &String { + pub fn get_dependent_cubes(&self) -> Vec { + let mut cubes = vec![]; + self.member_sql.extract_cube_deps(&mut cubes); + cubes + } + + pub fn cube_name(&self) -> &String { &self.cube_name } - fn name(&self) -> &String { + pub fn name(&self) -> &String { &self.name } } @@ -101,20 +116,16 @@ impl SymbolFactory for DimensionSymbolFactory { Some(self.sql.clone()) } - fn build( - self, - deps: Vec, - _compiler: &mut Compiler, - ) -> Result, CubeError> { + fn build(self, compiler: &mut Compiler) -> Result, CubeError> { let Self { cube_name, name, sql, definition, } = self; - Ok(EvaluationNode::new_dimension( - DimensionSymbol::new(cube_name, name, sql, definition), - deps, - )) + let sql = compiler.compile_sql_call(&cube_name, sql)?; + Ok(MemberSymbol::new_dimension(DimensionSymbol::new( + cube_name, name, sql, definition, + ))) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs index e6ecf3db0e0c5..7ef7024e700c6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs @@ -1,26 +1,30 @@ use super::{MemberSymbol, SymbolFactory}; use crate::cube_bridge::evaluator::CubeEvaluator; -use crate::cube_bridge::measure_definition::{MeasureDefinition, TimeShiftReference}; -use crate::cube_bridge::memeber_sql::{MemberSql, MemberSqlArg}; -use crate::planner::sql_evaluator::{Compiler, Dependency, EvaluationNode}; +use crate::cube_bridge::measure_definition::{ + MeasureDefinition, RollingWindow, TimeShiftReference, +}; +use crate::cube_bridge::memeber_sql::MemberSql; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::{sql_nodes::SqlNode, Compiler, SqlCall, SqlEvaluatorVisitor}; use cubenativeutils::CubeError; use std::rc::Rc; +#[derive(Clone)] pub struct MeasureOrderBy { - evaluation_node: Rc, + sql_call: Rc, direction: String, } impl MeasureOrderBy { - pub fn new(evaluation_node: Rc, direction: String) -> Self { + pub fn new(sql_call: Rc, direction: String) -> Self { Self { - evaluation_node, + sql_call, direction, } } - pub fn evaluation_node(&self) -> &Rc { - &self.evaluation_node + pub fn sql_call(&self) -> &Rc { + &self.sql_call } pub fn direction(&self) -> &String { @@ -28,22 +32,24 @@ impl MeasureOrderBy { } } +#[derive(Clone)] pub struct MeasureSymbol { cube_name: String, name: String, definition: Rc, - measure_filters: Vec>, + measure_filters: Vec>, measure_order_by: Vec, - member_sql: Rc, + member_sql: Rc, + is_splitted_source: bool, } impl MeasureSymbol { pub fn new( cube_name: String, name: String, - member_sql: Rc, + member_sql: Rc, definition: Rc, - measure_filters: Vec>, + measure_filters: Vec>, measure_order_by: Vec, ) -> Self { Self { @@ -53,6 +59,7 @@ impl MeasureSymbol { definition, measure_filters, measure_order_by, + is_splitted_source: false, } } @@ -60,6 +67,24 @@ impl MeasureSymbol { format!("{}.{}", self.cube_name, self.name) } + pub fn is_splitted_source(&self) -> bool { + self.is_splitted_source + } + + pub fn split_with_source(&self, source_name: String) -> (Self, Self) { + let mut measure_with_source = self.clone(); + measure_with_source.is_splitted_source = true; + let source = Self::new( + self.cube_name.clone(), + source_name.clone(), + self.member_sql.clone(), + self.definition().clone(), + self.measure_filters.clone(), + self.measure_order_by.clone(), + ); + (measure_with_source, source) + } + pub fn is_calculated(&self) -> bool { match self.definition.static_data().measure_type.as_str() { "number" | "string" | "time" | "boolean" => true, @@ -67,11 +92,40 @@ impl MeasureSymbol { } } - pub fn evaluate_sql(&self, args: Vec) -> Result { - let sql = self.member_sql.call(args)?; + pub fn evaluate_sql( + &self, + visitor: &SqlEvaluatorVisitor, + node_processor: Rc, + query_tools: Rc, + ) -> Result { + let sql = self.member_sql.eval(visitor, node_processor, query_tools)?; Ok(sql) } + pub fn get_dependencies(&self) -> Vec> { + let mut deps = vec![]; + self.member_sql.extract_symbol_deps(&mut deps); + for filter in self.measure_filters.iter() { + filter.extract_symbol_deps(&mut deps); + } + for order in self.measure_order_by.iter() { + order.sql_call().extract_symbol_deps(&mut deps); + } + deps + } + + pub fn get_dependent_cubes(&self) -> Vec { + let mut cubes = vec![]; + self.member_sql.extract_cube_deps(&mut cubes); + for filter in self.measure_filters.iter() { + filter.extract_cube_deps(&mut cubes); + } + for order in self.measure_order_by.iter() { + order.sql_call().extract_cube_deps(&mut cubes); + } + cubes + } + pub fn owned_by_cube(&self) -> bool { self.definition() .static_data() @@ -83,7 +137,23 @@ impl MeasureSymbol { &self.definition.static_data().measure_type } - pub fn measure_filters(&self) -> &Vec> { + pub fn rolling_window(&self) -> &Option { + &self.definition.static_data().rolling_window + } + + pub fn is_rolling_window(&self) -> bool { + self.rolling_window().is_some() + } + + pub fn is_running_total(&self) -> bool { + self.measure_type() == "runningTotal" + } + + pub fn is_cumulative(&self) -> bool { + self.is_rolling_window() || self.is_running_total() + } + + pub fn measure_filters(&self) -> &Vec> { &self.measure_filters } @@ -102,13 +172,11 @@ impl MeasureSymbol { pub fn is_multi_stage(&self) -> bool { self.definition.static_data().multi_stage.unwrap_or(false) } -} -impl MemberSymbol for MeasureSymbol { - fn cube_name(&self) -> &String { + pub fn cube_name(&self) -> &String { &self.cube_name } - fn name(&self) -> &String { + pub fn name(&self) -> &String { &self.name } } @@ -173,11 +241,7 @@ impl SymbolFactory for MeasureSymbolFactory { } } - fn build( - self, - deps: Vec, - compiler: &mut Compiler, - ) -> Result, CubeError> { + fn build(self, compiler: &mut Compiler) -> Result, CubeError> { let Self { cube_name, name, @@ -187,7 +251,7 @@ impl SymbolFactory for MeasureSymbolFactory { let mut measure_filters = vec![]; if let Some(filters) = definition.filters()? { for filter in filters.items().iter() { - let node = compiler.add_simple_sql_evaluator(cube_name.clone(), filter.sql()?)?; + let node = compiler.compile_sql_call(&cube_name, filter.sql()?)?; measure_filters.push(node); } } @@ -195,21 +259,19 @@ impl SymbolFactory for MeasureSymbolFactory { let mut measure_order_by = vec![]; if let Some(group_by) = definition.order_by()? { for item in group_by.items().iter() { - let node = compiler.add_simple_sql_evaluator(cube_name.clone(), item.sql()?)?; + let node = compiler.compile_sql_call(&cube_name, item.sql()?)?; measure_order_by.push(MeasureOrderBy::new(node, item.dir()?)); } } + let sql = compiler.compile_sql_call(&cube_name, sql)?; - Ok(EvaluationNode::new_measure( - MeasureSymbol::new( - cube_name, - name, - sql, - definition, - measure_filters, - measure_order_by, - ), - deps, - )) + Ok(MemberSymbol::new_measure(MeasureSymbol::new( + cube_name, + name, + sql, + definition, + measure_filters, + measure_order_by, + ))) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs index b8137abda10cb..ac62b640e580e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs @@ -1,4 +1,80 @@ -pub trait MemberSymbol { - fn cube_name(&self) -> &String; - fn name(&self) -> &String; +use super::{CubeNameSymbol, CubeTableSymbol, DimensionSymbol, MeasureSymbol}; +use std::rc::Rc; + +pub enum MemberSymbol { + Dimension(DimensionSymbol), + Measure(MeasureSymbol), + CubeName(CubeNameSymbol), + CubeTable(CubeTableSymbol), +} + +impl MemberSymbol { + pub fn new_measure(symbol: MeasureSymbol) -> Rc { + Rc::new(Self::Measure(symbol)) + } + + pub fn new_dimension(symbol: DimensionSymbol) -> Rc { + Rc::new(Self::Dimension(symbol)) + } + + pub fn new_cube_name(symbol: CubeNameSymbol) -> Rc { + Rc::new(Self::CubeName(symbol)) + } + + pub fn new_cube_table(symbol: CubeTableSymbol) -> Rc { + Rc::new(Self::CubeTable(symbol)) + } + + pub fn full_name(&self) -> String { + match self { + Self::Dimension(d) => d.full_name(), + Self::Measure(m) => m.full_name(), + Self::CubeName(c) => c.cube_name().clone(), + Self::CubeTable(c) => c.cube_name().clone(), + } + } + pub fn name(&self) -> String { + match self { + Self::Dimension(d) => d.name().clone(), + Self::Measure(m) => m.name().clone(), + Self::CubeName(c) => c.cube_name().clone(), + Self::CubeTable(c) => c.cube_name().clone(), + } + } + + pub fn cube_name(&self) -> String { + match self { + Self::Dimension(d) => d.cube_name().clone(), + Self::Measure(m) => m.cube_name().clone(), + Self::CubeName(c) => c.cube_name().clone(), + Self::CubeTable(c) => c.cube_name().clone(), + } + } + pub fn is_measure(&self) -> bool { + matches!(self, Self::Measure(_)) + } + pub fn is_dimension(&self) -> bool { + matches!(self, Self::Dimension(_)) + } + pub fn get_dependencies(&self) -> Vec> { + match self { + Self::Dimension(d) => d.get_dependencies(), + Self::Measure(m) => m.get_dependencies(), + Self::CubeName(_) => vec![], + Self::CubeTable(_) => vec![], + } + } + + pub fn get_dependent_cubes(&self) -> Vec { + match self { + Self::Dimension(d) => d.get_dependent_cubes(), + Self::Measure(m) => m.get_dependent_cubes(), + Self::CubeName(_) => vec![], + Self::CubeTable(_) => vec![], + } + } + + pub fn is_leaf(&self) -> bool { + self.get_dependencies().is_empty() + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs deleted file mode 100644 index 830c8e7f4e367..0000000000000 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs +++ /dev/null @@ -1,26 +0,0 @@ -use super::{CubeNameSymbol, CubeTableSymbol, DimensionSymbol, MeasureSymbol, SimpleSqlSymbol}; -pub enum MemberSymbolType { - Dimension(DimensionSymbol), - Measure(MeasureSymbol), - CubeName(CubeNameSymbol), - CubeTable(CubeTableSymbol), - SimpleSql(SimpleSqlSymbol), -} - -impl MemberSymbolType { - pub fn full_name(&self) -> String { - match self { - MemberSymbolType::Dimension(d) => d.full_name(), - MemberSymbolType::Measure(m) => m.full_name(), - MemberSymbolType::CubeName(c) => c.cube_name().clone(), - MemberSymbolType::CubeTable(c) => c.cube_name().clone(), - MemberSymbolType::SimpleSql(_) => "".to_string(), - } - } - pub fn is_measure(&self) -> bool { - matches!(self, Self::Measure(_)) - } - pub fn is_dimension(&self) -> bool { - matches!(self, Self::Dimension(_)) - } -} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs index 7b8ba1e7e42f2..ddb631648d6f1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/mod.rs @@ -2,8 +2,6 @@ mod cube_symbol; mod dimension_symbol; mod measure_symbol; mod member_symbol; -mod member_symbol_type; -mod simple_sql; mod symbol_factory; pub use cube_symbol::{ @@ -12,6 +10,4 @@ pub use cube_symbol::{ pub use dimension_symbol::{DimensionSymbol, DimensionSymbolFactory}; pub use measure_symbol::{MeasureSymbol, MeasureSymbolFactory}; pub use member_symbol::MemberSymbol; -pub use member_symbol_type::MemberSymbolType; -pub use simple_sql::{SimpleSqlSymbol, SimpleSqlSymbolFactory}; pub use symbol_factory::SymbolFactory; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/simple_sql.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/simple_sql.rs deleted file mode 100644 index 52d21e298ad5c..0000000000000 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/simple_sql.rs +++ /dev/null @@ -1,74 +0,0 @@ -use super::SymbolFactory; -use crate::cube_bridge::memeber_sql::{MemberSql, MemberSqlArg}; -use crate::planner::sql_evaluator::{Compiler, Dependency, EvaluationNode}; -use cubenativeutils::CubeError; -use std::rc::Rc; - -pub struct SimpleSqlSymbol { - cube_name: String, - member_sql: Rc, -} - -impl SimpleSqlSymbol { - pub fn new(cube_name: String, member_sql: Rc) -> Self { - Self { - cube_name, - member_sql, - } - } - - pub fn full_name(&self) -> String { - format!("{}.simple_sql", self.cube_name) - } - - pub fn evaluate_sql(&self, args: Vec) -> Result { - let sql = self.member_sql.call(args)?; - Ok(sql) - } -} - -pub struct SimpleSqlSymbolFactory { - cube_name: String, - sql: Rc, -} - -impl SimpleSqlSymbolFactory { - pub fn try_new(cube_name: &String, sql: Rc) -> Result { - Ok(Self { - cube_name: cube_name.clone(), - sql, - }) - } -} - -impl SymbolFactory for SimpleSqlSymbolFactory { - fn is_cachable() -> bool { - false - } - fn symbol_name() -> String { - "simple_sql".to_string() - } - fn cube_name(&self) -> &String { - &self.cube_name - } - - fn member_sql(&self) -> Option> { - Some(self.sql.clone()) - } - - fn deps_names(&self) -> Result, CubeError> { - Ok(self.sql.args_names().clone()) - } - - fn build( - self, - deps: Vec, - _compiler: &mut Compiler, - ) -> Result, CubeError> { - let Self { cube_name, sql } = self; - Ok(EvaluationNode::new_simple_sql( - SimpleSqlSymbol::new(cube_name, sql), - deps, - )) - } -} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/symbol_factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/symbol_factory.rs index 198bcd85c6710..c0e681412ccfb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/symbol_factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/symbol_factory.rs @@ -1,6 +1,6 @@ +use super::MemberSymbol; use crate::cube_bridge::memeber_sql::MemberSql; -use crate::planner::sql_evaluator::dependecy::Dependency; -use crate::planner::sql_evaluator::{Compiler, EvaluationNode}; +use crate::planner::sql_evaluator::Compiler; use cubenativeutils::CubeError; use std::rc::Rc; @@ -12,9 +12,5 @@ pub trait SymbolFactory: Sized { fn cube_name(&self) -> &String; fn deps_names(&self) -> Result, CubeError>; fn member_sql(&self) -> Option>; - fn build( - self, - deps: Vec, - compiler: &mut Compiler, - ) -> Result, CubeError>; + fn build(self, compiler: &mut Compiler) -> Result, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs index 0a9a49b14f681..abef52ceeaa02 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs @@ -1,50 +1,21 @@ -use super::{dependecy::Dependency, EvaluationNode}; +use super::MemberSymbol; use cubenativeutils::CubeError; use std::rc::Rc; pub trait TraversalVisitor { - fn on_node_traverse(&mut self, node: &Rc) -> Result; - - fn apply(&mut self, node: &Rc) -> Result<(), CubeError> { - if self.on_node_traverse(node)? { - self.travese_deps(node)?; - } - Ok(()) - } - - fn travese_deps(&mut self, node: &Rc) -> Result<(), CubeError> { - for dep in node.deps() { - self.traverse_single_dep(dep, node)?; - } - Ok(()) - } - - fn traverse_single_dep( + type State; + fn on_node_traverse( &mut self, - dep: &Dependency, - node: &Rc, - ) -> Result<(), CubeError> { - match dep { - Dependency::SingleDependency(dep) => self.apply(dep), - Dependency::StructDependency(dep) => { - if dep.sql_fn.is_some() { - self.apply(node)?; - } - if let Some(to_string_fn) = &dep.to_string_fn { - self.apply(to_string_fn)?; - } - for (_, v) in dep.properties.iter() { - match v { - Dependency::SingleDependency(dep) => { - self.apply(dep)?; - } - Dependency::StructDependency(_) => unimplemented!(), - Dependency::ContextDependency(_) => {} - } - } - Ok(()) + node: &Rc, + state: &Self::State, + ) -> Result, CubeError>; + + fn apply(&mut self, node: &Rc, state: &Self::State) -> Result<(), CubeError> { + if let Some(state) = self.on_node_traverse(node, state)? { + for dep in node.get_dependencies() { + self.apply(&dep, &state)? } - Dependency::ContextDependency(_) => Ok(()), } + Ok(()) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs index f0d23de97e232..6ee5d9a3f66d6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs @@ -3,6 +3,7 @@ use cubenativeutils::CubeError; use minijinja::context; use std::rc::Rc; +#[derive(Clone)] pub struct FilterTemplates { render: Rc, } @@ -103,6 +104,26 @@ impl FilterTemplates { ) } + pub fn add_interval(&self, date: String, interval: String) -> Result { + self.render.render_template( + &"expressions/add_interval", + context! { + date => date, + interval => interval + }, + ) + } + + pub fn sub_interval(&self, date: String, interval: String) -> Result { + self.render.render_template( + &"expressions/sub_interval", + context! { + date => date, + interval => interval + }, + ) + } + pub fn set_where(&self, column: String) -> Result { self.render.render_template( &"filters/set_where", @@ -161,11 +182,37 @@ impl FilterTemplates { ) } - fn additional_null_check(&self, need: bool, column: &String) -> Result { + pub fn additional_null_check(&self, need: bool, column: &String) -> Result { if need { self.or_is_null_check(column.clone()) } else { Ok(String::default()) } } + + pub fn ilike( + &self, + column: &str, + value: &str, + start_wild: bool, + end_wild: bool, + not: bool, + ) -> Result { + let pattern = self.render.render_template( + &"filters/like_pattern", + context! { + start_wild => start_wild, + value => value, + end_wild => end_wild + }, + )?; + self.render.render_template( + &"expressions/ilike", + context! { + expr => column, + negated => not, + pattern => pattern + }, + ) + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs index aca5695ef8544..26b12acacd25a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs @@ -121,6 +121,22 @@ impl PlanSqlTemplates { ) } + pub fn time_series_select( + &self, + from_date: Option, + to_date: Option, + seria: Vec>, + ) -> Result { + self.render.render_template( + "statements/time_series_select", + context! { + from_date => from_date, + to_date => to_date, + seria => seria + }, + ) + } + pub fn select( &self, ctes: Vec, @@ -152,59 +168,6 @@ impl PlanSqlTemplates { ) } - /* pub fn select( - &self, - from: String, - projection: Vec, - group_by: Vec, - group_descs: Vec>, - aggregate: Vec, - alias: String, - filter: Option, - _having: Option, - order_by: Vec, - limit: Option, - offset: Option, - distinct: bool, - ) -> Result { - let group_by = self.to_template_columns(group_by)?; - let aggregate = self.to_template_columns(aggregate)?; - let projection = self.to_template_columns(projection)?; - let order_by = self.to_template_columns(order_by)?; - let select_concat = group_by - .iter() - .chain(aggregate.iter()) - .chain(projection.iter()) - .map(|c| c.clone()) - .collect::>(); - let quoted_from_alias = self.quote_identifier(&alias)?; - let has_grouping_sets = group_descs.iter().any(|d| d.is_some()); - let group_by_expr = if has_grouping_sets { - self.group_by_with_grouping_sets(&group_by, &group_descs)? - } else { - self.render_template( - "statements/group_by_exprs", - context! { group_by => group_by }, - )? - }; - self.render.render_template( - "statements/select", - context! { - from => from, - select_concat => select_concat, - group_by => group_by_expr, - aggregate => aggregate, - projection => projection, - order_by => order_by, - filter => filter, - from_alias => quoted_from_alias, - limit => limit, - offset => offset, - distinct => distinct, - }, - ) - } */ - pub fn join(&self, source: &str, condition: &str, is_inner: bool) -> Result { let join_type = if is_inner { self.render.get_template("join_types/inner")? diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs index 378e869c14be3..7e4cf9137d9f8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs @@ -1,39 +1,21 @@ use super::query_tools::QueryTools; use super::sql_evaluator::sql_nodes::{SqlNode, SqlNodesFactory}; -use super::sql_evaluator::EvaluationNode; -use crate::plan::Schema; -use crate::planner::sql_evaluator::sql_node_transformers::set_schema; +use super::sql_evaluator::{MemberSymbol, SqlCall}; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use cubenativeutils::CubeError; use std::rc::Rc; pub struct VisitorContext { - cube_alias_prefix: Option, node_processor: Rc, } impl VisitorContext { - pub fn new(cube_alias_prefix: Option, node_processor: Rc) -> Self { + pub fn new(nodes_factory: &SqlNodesFactory) -> Self { Self { - cube_alias_prefix, - node_processor, + node_processor: nodes_factory.default_node_processor(), } } - pub fn new_with_cube_alias_prefix( - nodes_factory: Rc, - cube_alias_prefix: String, - ) -> Self { - Self::new( - Some(cube_alias_prefix), - nodes_factory.default_node_processor(), - ) - } - - pub fn default(nodes_factory: Rc) -> Self { - Self::new(Default::default(), nodes_factory.default_node_processor()) - } - pub fn make_visitor(&self, query_tools: Rc) -> SqlEvaluatorVisitor { SqlEvaluatorVisitor::new(query_tools) } @@ -41,20 +23,25 @@ impl VisitorContext { pub fn node_processor(&self) -> Rc { self.node_processor.clone() } - - pub fn cube_alias_prefix(&self) -> &Option { - &self.cube_alias_prefix - } } pub fn evaluate_with_context( - node: &Rc, + node: &Rc, query_tools: Rc, context: Rc, - source_schema: Rc, ) -> Result { - let mut visitor = context.make_visitor(query_tools); - let node_processor = set_schema(context.node_processor(), source_schema); + let visitor = context.make_visitor(query_tools); + let node_processor = context.node_processor(); visitor.apply(node, node_processor) } + +pub fn evaluate_sql_call_with_context( + sql_call: &Rc, + query_tools: Rc, + context: Rc, +) -> Result { + let visitor = context.make_visitor(query_tools.clone()); + let node_processor = context.node_processor(); + sql_call.eval(&visitor, node_processor, query_tools) +}