Skip to content

Commit 31d7533

Browse files
waralexromwaralexrom
committed
in work
1 parent 7042ec4 commit 31d7533

File tree

12 files changed

+334
-11
lines changed

12 files changed

+334
-11
lines changed

‎packages/cubejs-schema-compiler/src/adapter/PreAggregations.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,8 @@ export class PreAggregations {
311311
const dimensionsList = query.dimensions.map(dim => dim.expressionPath());
312312
const segmentsList = query.segments.map(s => s.expressionPath());
313313
const ownedDimensions = PreAggregations.ownedMembers(query, flattenDimensionMembers);
314+
console.log("!!! dimensions", dimensionsList);
315+
console.log("!!! ownedDimensions", ownedDimensions);
314316
const ownedTimeDimensions = query.timeDimensions.map(td => {
315317
const owned = PreAggregations.ownedMembers(query, [td]);
316318
let { dimension } = td;

‎packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts‎

Lines changed: 141 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ describe('PreAggregations', () => {
9393
type: 'string',
9494
sql: 'source'
9595
},
96+
shortSource: {
97+
type: 'string',
98+
sql: \`SUBSTRING(\${source}, 0, 2)\`
99+
},
96100
sourceAndId: {
97101
type: 'string',
98102
sql: \`\${source} || '_' || \${id}\`,
@@ -115,6 +119,10 @@ describe('PreAggregations', () => {
115119
type: 'time',
116120
sql: \`\${createdAt}\`
117121
},
122+
createdAtDay: {
123+
type: 'time',
124+
sql: \`\${createdAt.day}\`
125+
},
118126
checkinsCount: {
119127
type: 'number',
120128
sql: \`\${visitor_checkins.count}\`,
@@ -558,9 +566,57 @@ describe('PreAggregations', () => {
558566
});
559567
});
560568

569+
570+
it('simple pre-aggregation proxy time dimension', () => compiler.compile().then(() => {
571+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
572+
measures: [
573+
'visitors.count'
574+
],
575+
dimensions: [
576+
'visitors.createdAtDay',
577+
],
578+
timezone: 'America/Los_Angeles',
579+
order: [{
580+
id: 'visitors.createdAtDay'
581+
}],
582+
preAggregationsSchema: ''
583+
});
584+
585+
const queryAndParams = query.buildSqlAndParams();
586+
console.log(queryAndParams);
587+
console.log(query.preAggregations?.preAggregationsDescription());
588+
expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true);
589+
590+
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
591+
expect(res).toEqual(
592+
[
593+
{
594+
visitors__created_at_day: '2016-09-06T00:00:00.000Z',
595+
visitors__count: '1'
596+
},
597+
{
598+
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
599+
visitors__count: '1'
600+
},
601+
{
602+
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
603+
visitors__count: '1'
604+
},
605+
{
606+
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
607+
visitors__count: '1'
608+
},
609+
{
610+
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
611+
visitors__count: '2'
612+
}
613+
]
614+
);
615+
});
616+
}));
617+
561618
it('simple pre-aggregation (allowNonStrictDateRangeMatch: true)', async () => {
562619
await compiler.compile();
563-
564620
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
565621
measures: [
566622
'visitors.count'
@@ -894,9 +950,92 @@ describe('PreAggregations', () => {
894950
});
895951
});
896952

897-
it('non-additive single value view filtered measure', async () => {
953+
it('non-additive view dimension', () => compiler.compile().then(() => {
954+
await compiler.compile();
955+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
956+
measures: [
957+
'visitors_view.uniqueSourceCount'
958+
],
959+
dimensions: [
960+
'visitors_view.source'
961+
],
962+
timeDimensions: [{
963+
dimension: 'visitors_view.signedUpAt',
964+
granularity: 'day',
965+
dateRange: ['2017-01-01', '2017-01-30']
966+
}],
967+
timezone: 'America/Los_Angeles',
968+
order: [{
969+
id: 'visitors_view.createdAt'
970+
}],
971+
preAggregationsSchema: ''
972+
});
973+
974+
const queryAndParams = query.buildSqlAndParams();
975+
console.log(queryAndParams);
976+
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
977+
console.log(preAggregationsDescription);
978+
expect((<any>preAggregationsDescription)[0].loadSql[0]).toMatch(/visitors_unique_source_count/);
979+
980+
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
981+
expect(res).toEqual(
982+
[
983+
{
984+
visitors_view__source: 'google',
985+
visitors_view__signed_up_at_day: '2017-01-05T00:00:00.000Z',
986+
visitors_view__unique_source_count: '1'
987+
},
988+
{
989+
visitors_view__source: 'some',
990+
visitors_view__signed_up_at_day: '2017-01-02T00:00:00.000Z',
991+
visitors_view__unique_source_count: '1'
992+
},
993+
{
994+
visitors_view__source: 'some',
995+
visitors_view__signed_up_at_day: '2017-01-04T00:00:00.000Z',
996+
visitors_view__unique_source_count: '1'
997+
},
998+
{
999+
visitors_view__source: null,
1000+
visitors_view__signed_up_at_day: '2017-01-06T00:00:00.000Z',
1001+
visitors_view__unique_source_count: '0'
1002+
}
1003+
]
1004+
1005+
);
1006+
});
1007+
}));
1008+
it('non-additive proxy but not direct alias dimension', () => compiler.compile().then(() => {
8981009
await compiler.compile();
1010+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
1011+
measures: [
1012+
'visitors_view.uniqueSourceCount'
1013+
],
1014+
dimensions: [
1015+
'visitors.shortSource'
1016+
],
1017+
timeDimensions: [{
1018+
dimension: 'visitors_view.signedUpAt',
1019+
granularity: 'day',
1020+
dateRange: ['2017-01-01', '2017-01-30']
1021+
}],
1022+
timezone: 'America/Los_Angeles',
1023+
order: [{
1024+
id: 'visitors_view.createdAt'
1025+
}],
1026+
preAggregationsSchema: ''
1027+
});
8991028

1029+
const queryAndParams = query.buildSqlAndParams();
1030+
console.log(queryAndParams);
1031+
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
1032+
console.log(preAggregationsDescription);
1033+
expect((<any>preAggregationsDescription)[0].type).toEqual('originalSql');
1034+
1035+
}));
1036+
1037+
it('non-additive single value view filtered measure', () => compiler.compile().then(() => {
1038+
await compiler.compile();
9001039
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
9011040
measures: [
9021041
'visitors_view.googleUniqueSourceCount'

‎rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/dimension_matcher.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ use std::collections::HashMap;
88
use crate::planner::filter::BaseFilter;
99
use std::rc::Rc;
1010

11-
pub struct DimensionMatcher {}
11+
pub struct DimensionMatcher {
12+
}
1213

1314
impl DimensionMatcher {
1415
pub fn new() -> Self {
@@ -87,6 +88,10 @@ impl DimensionMatcher {
8788
return Ok(MatchState::NotMatched);
8889
}
8990

91+
if !symbol.is_reference() {
92+
result = result.combine(&MatchState::Partial);
93+
}
94+
9095
for dep in symbol.get_dependencies() {
9196
let dep_match = self.try_match_symbol(&dep, add_to_matched_dimension, pre_aggregation_dimensions)?;
9297
if dep_match == MatchState::NotMatched {

‎rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ impl PreAggregationOptimizer {
157157
)?;
158158
}
159159
let all_multi_stage_rewrited = rewrited_multistage.values().all(|v| *v);
160-
println!("!!! pre-aggr: {}, all_multi_stage_rewrited: {}, rewritten: {:#?}", pre_aggregation.name, all_multi_stage_rewrited, rewrited_multistage);
160+
//println!("!!! pre-aggr: {}, all_multi_stage_rewrited: {}, rewritten: {:#?}", pre_aggregation.name, all_multi_stage_rewrited, rewrited_multistage);
161161
if !all_multi_stage_rewrited {
162162
return Ok(None);
163163
}
@@ -439,14 +439,14 @@ impl PreAggregationOptimizer {
439439
let time_dimensions = &schema.time_dimensions;
440440
let time_dimension_filters = &filters.time_dimensions_filters;
441441

442-
println!("!!! ========");
442+
//println!("!!! ========");
443443
let mut match_state = self.match_dimensions(
444444
&schema.dimensions,
445445
&filters.dimensions_filters,
446446
&filters.segments,
447447
pre_aggregation,
448448
)?;
449-
println!("!!!! pre-agg-name: {}, match_state: {:?}", pre_aggregation.name, match_state);
449+
//println!("!!!! pre-agg-name: {}, match_state: {:?}", pre_aggregation.name, match_state);
450450
match_state = match_state.combine(&self.match_time_dimensions(
451451
&time_dimensions,
452452
&time_dimension_filters,
@@ -465,7 +465,7 @@ impl PreAggregationOptimizer {
465465
pre_aggregation,
466466
match_state == MatchState::Partial,
467467
)?;
468-
println!("!!!! pre-agg-name: {}, match_state: {:?}, measures_match: {}", pre_aggregation.name, match_state, measures_match);
468+
//println!("!!!! pre-agg-name: {}, match_state: {:?}, measures_match: {}", pre_aggregation.name, match_state, measures_match);
469469
if !schema.multiplied_measures.is_empty() && match_state == MatchState::Partial {
470470
return Ok(false);
471471
}

‎rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/time_dimension_matcher.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,10 @@ impl TimeDimensionMatcher {
184184
return Ok(MatchState::NotMatched);
185185
}
186186

187+
if !symbol.is_reference() {
188+
result = result.combine(&MatchState::Partial);
189+
}
190+
187191
for dep in symbol.get_dependencies() {
188192
let dep_match = self.try_match_symbol(
189193
&dep,

‎rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,29 @@ impl SqlCall {
4545
self.member_sql.call(args)
4646
}
4747

48+
pub fn is_direct_reference(&self) -> Result<bool, CubeError> {
49+
50+
let dependencies = self.get_dependencies();
51+
if dependencies.len() != 1 {
52+
return Ok(false);
53+
}
54+
55+
let reference_candidate = dependencies[0].clone();
56+
57+
let args = self
58+
.deps
59+
.iter()
60+
.map(|d| {
61+
self.evaluate_single_dep_for_ref_check(
62+
&d,
63+
)
64+
})
65+
.collect::<Result<Vec<_>, _>>()?;
66+
let eval_result = self.member_sql.call(args)?;
67+
68+
Ok(eval_result.trim() == &reference_candidate.full_name())
69+
}
70+
4871
pub fn get_dependencies(&self) -> Vec<Rc<MemberSymbol>> {
4972
let mut deps = Vec::new();
5073
self.extract_symbol_deps(&mut deps);
@@ -160,6 +183,70 @@ impl SqlCall {
160183
};
161184
}
162185
}
186+
187+
188+
//TODO temporary solution, should be removed after refactoring
189+
fn evaluate_single_dep_for_ref_check(
190+
&self,
191+
dep: &Dependency,
192+
) -> Result<MemberSqlArg, CubeError> {
193+
match dep {
194+
Dependency::SymbolDependency(dep) => Ok(MemberSqlArg::String(dep.full_name())),
195+
Dependency::TimeDimensionDependency(dep) => {
196+
self.evaluate_time_dimesion_dep_for_ref_check(dep)
197+
}
198+
Dependency::CubeDependency(dep) => self.evaluate_cube_dep_for_ref_check(
199+
dep,
200+
),
201+
Dependency::ContextDependency(contex_symbol) => {
202+
Ok(MemberSqlArg::String(format!("Context Symbol")))
203+
}
204+
}
205+
}
206+
207+
//TODO temporary solution, should be removed after refactoring
208+
fn evaluate_cube_dep_for_ref_check(
209+
&self,
210+
dep: &CubeDependency,
211+
) -> Result<MemberSqlArg, CubeError> {
212+
let mut res = MemberSqlStruct::default();
213+
if let Some(sql_fn) = &dep.sql_fn {
214+
res.sql_fn = Some(sql_fn.full_name());
215+
}
216+
if let Some(to_string_fn) = &dep.to_string_fn {
217+
res.to_string_fn =
218+
Some(to_string_fn.full_name());
219+
}
220+
for (k, v) in dep.properties.iter() {
221+
let prop_res = match v {
222+
CubeDepProperty::SymbolDependency(dep) => {
223+
MemberSqlArg::String(dep.full_name())
224+
}
225+
226+
CubeDepProperty::TimeDimensionDependency(dep) => self.evaluate_time_dimesion_dep_for_ref_check(
227+
dep,
228+
)?,
229+
230+
CubeDepProperty::CubeDependency(dep) => self.evaluate_cube_dep_for_ref_check(
231+
&dep,
232+
)?,
233+
};
234+
res.properties.insert(k.clone(), prop_res);
235+
}
236+
Ok(MemberSqlArg::Struct(res))
237+
}
238+
239+
fn evaluate_time_dimesion_dep_for_ref_check(
240+
&self,
241+
dep: &TimeDimensionDependency,
242+
) -> Result<MemberSqlArg, CubeError> {
243+
let mut res = MemberSqlStruct::default();
244+
for (k, v) in dep.granularities.iter() {
245+
let arg = MemberSqlArg::String(v.full_name());
246+
res.properties.insert(k.clone(), arg);
247+
}
248+
Ok(MemberSqlArg::Struct(res))
249+
}
163250

164251
fn evaluate_single_dep(
165252
&self,

‎rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,4 @@ pub use time_shift::TimeShiftSqlNode;
3636
pub use ungroupped_measure::UngroupedMeasureSqlNode;
3737
pub use ungroupped_query_final_measure::UngroupedQueryFinalMeasureSqlNode;
3838
pub use final_pre_aggregation_measure::FinalPreAggregationMeasureSqlNode;
39-
pub use original_sql_pre_aggregation::OriginalSqlPreAggregationSqlNode;
39+
pub use original_sql_pre_aggregation::OriginalSqlPreAggregationSqlNode;

0 commit comments

Comments
 (0)