Skip to content

Commit 7042ec4

Browse files
waralexromwaralexrom
committed
in work
1 parent 60131c2 commit 7042ec4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1929
-303
lines changed

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -636,15 +636,9 @@ export class BaseQuery {
636636
* @returns {[string, Array<unknown>]}
637637
*/
638638
buildSqlAndParams(exportAnnotatedSql) {
639-
if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
640-
if (this.externalPreAggregationQuery()) { // TODO performance
641-
return this.externalQuery().buildSqlAndParams(exportAnnotatedSql);
642-
}
643-
}
644-
645639
if (this.useNativeSqlPlanner) {
646-
/* let isRelatedToPreAggregation = false;
647-
if (this.options.preAggregationQuery) {
640+
let isRelatedToPreAggregation = false;
641+
/* if (this.options.preAggregationQuery) {
648642
isRelatedToPreAggregation = true;
649643
} else if (!this.options.disableExternalPreAggregations && this.externalQueryClass) {
650644
if (this.externalPreAggregationQuery()) {
@@ -668,6 +662,12 @@ export class BaseQuery {
668662
return this.buildSqlAndParamsRust(exportAnnotatedSql);
669663
}
670664

665+
if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
666+
if (this.externalPreAggregationQuery()) { // TODO performance
667+
return this.externalQuery().buildSqlAndParams(exportAnnotatedSql);
668+
}
669+
}
670+
671671
return this.compilers.compiler.withQuery(
672672
this,
673673
() => this.cacheValue(
@@ -726,6 +726,57 @@ export class BaseQuery {
726726
return res;
727727
}
728728

729+
//FIXME Temporary solution
730+
findPreAggregationForQueryRust() {
731+
if (!this.preAggregations.preAggregationForQuery) {
732+
let optionsOrder = this.options.order;
733+
if (optionsOrder && !Array.isArray(optionsOrder)) {
734+
optionsOrder = [optionsOrder];
735+
}
736+
const order = optionsOrder ? R.pipe(
737+
R.map((hash) => {
738+
return ((!hash || !hash.id) ? null : hash);
739+
}),
740+
R.reject(R.isNil),
741+
)(optionsOrder) : undefined;
742+
743+
const queryParams = {
744+
measures: this.options.measures,
745+
dimensions: this.options.dimensions,
746+
segments: this.options.segments,
747+
timeDimensions: this.options.timeDimensions,
748+
timezone: this.options.timezone,
749+
joinGraph: this.joinGraph,
750+
cubeEvaluator: this.cubeEvaluator,
751+
order,
752+
filters: this.options.filters,
753+
limit: this.options.limit ? this.options.limit.toString() : null,
754+
rowLimit: this.options.rowLimit ? this.options.rowLimit.toString() : null,
755+
offset: this.options.offset ? this.options.offset.toString() : null,
756+
baseTools: this,
757+
ungrouped: this.options.ungrouped,
758+
exportAnnotatedSql: false,
759+
preAggregationQuery: this.options.preAggregationQuery
760+
};
761+
762+
const buildResult = nativeBuildSqlAndParams(queryParams);
763+
764+
if (buildResult.error) {
765+
if (buildResult.error.cause && buildResult.error.cause === 'User') {
766+
throw new UserError(buildResult.error.message);
767+
} else {
768+
throw new Error(buildResult.error.message);
769+
}
770+
}
771+
772+
const res = buildResult.result;
773+
if (res[2]) {
774+
this.preAggregations.preAggregationForQuery = res[2];
775+
}
776+
}
777+
return this.preAggregations.preAggregationForQuery;
778+
}
779+
729780
allCubeMembers(path) {
730781
const fromPath = this.cubeEvaluator.cubeFromPath(path);
731782

@@ -922,14 +973,14 @@ export class BaseQuery {
922973
multiStageMembers,
923974
} = this.fullKeyQueryAggregateMeasures();
924975

976+
925977
if (!multipliedMeasures.length && !cumulativeMeasures.length && !multiStageMembers.length) {
926978
return this.simpleQuery();
927979
}
928980

929981
const renderedWithQueries = withQueries.map(q => this.renderWithQuery(q));
930982

931983
let toJoin;
932-
933984
if (this.options.preAggregationQuery) {
934985
const allRegular = regularMeasures.concat(
935986
cumulativeMeasures
@@ -3235,7 +3286,7 @@ export class BaseQuery {
32353286
}
32363287

32373288
newSubQueryForCube(cube, options) {
3238-
options = { ...options, useNativeSqlPlanner: false }; // We don't use tesseract for pre-aggregations generation yet
3289+
options = { ...options };
32393290
if (this.options.queryFactory) {
32403291
// When dealing with rollup joins, it's crucial to use the correct parameter allocator for the specific cube in use.
32413292
// By default, we'll use BaseQuery, but it's important to note that different databases (Oracle, PostgreSQL, MySQL, Druid, etc.)
@@ -3420,7 +3471,8 @@ export class BaseQuery {
34203471
});
34213472
} else if (preAggregation.type === 'rollup') {
34223473
const query = this.preAggregations.rollupPreAggregationQuery(cube, preAggregation);
3423-
return query.evaluateSymbolSqlWithContext(() => query.buildSqlAndParams(), {
3474+
const params = query.buildSqlAndParams();
3475+
return query.evaluateSymbolSqlWithContext(() => params, {
34243476
collectOriginalSqlPreAggregations
34253477
});
34263478
} else if (preAggregation.type === 'originalSql') {

‎packages/cubejs-schema-compiler/src/adapter/PreAggregations.js‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -782,11 +782,15 @@ export class PreAggregations {
782782
*/
783783
findPreAggregationForQuery() {
784784
if (!this.preAggregationForQuery) {
785-
this.preAggregationForQuery =
786-
this
787-
.rollupMatchResults()
788-
// Refresh worker can access specific pre-aggregations even in case those hidden by others
789-
.find(p => p.canUsePreAggregation && (!this.query.options.preAggregationId || p.preAggregationId === this.query.options.preAggregationId));
785+
if (this.query.useNativeSqlPlanner) {
786+
this.query.findPreAggregationForQueryRust();
787+
} else {
788+
this.preAggregationForQuery =
789+
this
790+
.rollupMatchResults()
791+
// Refresh worker can access specific pre-aggregations even in case those hidden by others
792+
.find(p => p.canUsePreAggregation && (!this.query.options.preAggregationId || p.preAggregationId === this.query.options.preAggregationId));
793+
}
790794
}
791795
return this.preAggregationForQuery;
792796
}

‎packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,7 @@ describe(
392392
expect(query.indexOf('cube__hourly_data')).toBeGreaterThanOrEqual(0);
393393
});
394394

395-
it('minute query with the `week` granularity match `HourlyData`', async () => {
395+
it('minute query with the `week` granularity match `HourlyData` 11', async () => {
396396
await compiler.compile();
397397
const [,,,, request] = getQueries(compiler, joinGraph, cubeEvaluator);
398398
const [query] = request.buildSqlAndParams();
@@ -410,7 +410,7 @@ describe(
410410
expect(query.indexOf('cube__hourly_data')).toBeGreaterThanOrEqual(0);
411411
});
412412

413-
it('query with no granularity match HourlyData', async () => {
413+
it('query with no granularity match HourlyData 2222', async () => {
414414
await compiler.compile();
415415
const [,,,,,, request] = getQueries(compiler, joinGraph, cubeEvaluator);
416416
const [query] = request.buildSqlAndParams();

‎packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-alias.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,8 +368,9 @@ describe('PreAggregationsAlias', () => {
368368
});
369369

370370
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
371+
const sqlAndParams = query.buildSqlAndParams();
371372
expect(preAggregationsDescription[0].tableName).toEqual('rvis_rollupalias');
372-
expect(query.buildSqlAndParams()[0]).toContain('rvis_rollupalias');
373+
expect(sqlAndParams[0]).toContain('rvis_rollupalias');
373374

374375
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
375376
expect(res).toEqual(

‎packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-multi-stage.test.ts‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
import R from 'ramda';
2+
import { UserError } from '../../../src/compiler/UserError';
3+
import { PostgresQuery } from '../../../src/adapter/PostgresQuery';
4+
import { prepareJsCompiler } from '../../unit/PrepareCompiler';
5+
import { dbRunner } from './PostgresDBRunner';
6+
7+
describe('PreAggregationsMultiStage', () => {
8+
jest.setTimeout(200000);
9+
10+
const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(`
11+
cube(\`visitors\`, {
12+
sql: \`
13+
select * from visitors WHERE \${FILTER_PARAMS.visitors.createdAt.filter('created_at')}
14+
\`,
15+
sqlAlias: 'vis',
16+
17+
joins: {
18+
visitor_checkins: {
19+
relationship: 'hasMany',
20+
sql: \`\${CUBE}.id = \${visitor_checkins}.visitor_id\`
21+
}
22+
},
23+
24+
measures: {
25+
count: {
26+
type: 'count'
27+
},
28+
revenue: {
29+
sql: 'amount',
30+
type: 'sum'
31+
},
32+
33+
34+
checkinsTotal: {
35+
sql: \`\${checkinsCount}\`,
36+
type: 'sum'
37+
},
38+
39+
uniqueSourceCount: {
40+
sql: 'source',
41+
type: 'countDistinct'
42+
},
43+
44+
countDistinctApprox: {
45+
sql: 'id',
46+
type: 'countDistinctApprox'
47+
},
48+
revenue_per_id: {
49+
multi_stage: true,
50+
sql: \`\${revenue} / \${id}\`,
51+
type: 'sum',
52+
add_group_by: [visitors.id],
53+
},
54+
55+
ratio: {
56+
sql: \`\${uniqueSourceCount} / nullif(\${checkinsTotal}, 0)\`,
57+
type: 'number'
58+
}
59+
},
60+
61+
dimensions: {
62+
id: {
63+
type: 'number',
64+
sql: 'id',
65+
primaryKey: true
66+
},
67+
source: {
68+
type: 'string',
69+
sql: 'source'
70+
},
71+
createdAt: {
72+
type: 'time',
73+
sql: 'created_at'
74+
},
75+
checkinsCount: {
76+
type: 'number',
77+
sql: \`\${visitor_checkins.count}\`,
78+
subQuery: true,
79+
propagateFiltersToSubQuery: true
80+
},
81+
82+
83+
},
84+
85+
segments: {
86+
google: {
87+
sql: \`source = 'google'\`
88+
}
89+
},
90+
91+
preAggregations: {
92+
revenuePerIdRollup: {
93+
type: 'rollup',
94+
measureReferences: [revenue],
95+
dimensionReferences: [id],
96+
timeDimensionReference: createdAt,
97+
granularity: 'day',
98+
partitionGranularity: 'month',
99+
},
100+
}
101+
})
102+
103+
104+
105+
cube('visitor_checkins', {
106+
sql: \`
107+
select * from visitor_checkins
108+
\`,
109+
110+
sqlAlias: 'vc',
111+
112+
measures: {
113+
count: {
114+
type: 'count'
115+
}
116+
},
117+
118+
dimensions: {
119+
id: {
120+
type: 'number',
121+
sql: 'id',
122+
primaryKey: true
123+
},
124+
visitor_id: {
125+
type: 'number',
126+
sql: 'visitor_id'
127+
},
128+
source: {
129+
type: 'string',
130+
sql: 'source'
131+
},
132+
created_at: {
133+
type: 'time',
134+
sql: 'created_at'
135+
}
136+
},
137+
138+
})
139+
140+
141+
`);
142+
143+
144+
it('simple multi stage with add_group_by', () => compiler.compile().then(() => {
145+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
146+
measures: [
147+
'visitors.revenue_per_id'
148+
],
149+
timeDimensions: [{
150+
dimension: 'visitors.createdAt',
151+
granularity: 'day',
152+
dateRange: ['2017-01-01', '2017-01-30']
153+
}],
154+
timezone: 'America/Los_Angeles',
155+
order: [{
156+
id: 'visitors.createdAt'
157+
}],
158+
preAggregationsSchema: ''
159+
});
160+
161+
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
162+
const sqlAndParams = query.buildSqlAndParams();
163+
console.log("!!!! sqlAndParamsl", sqlAndParams);
164+
/* expect(preAggregationsDescription[0].tableName).toEqual('rvis_rollupalias');
165+
expect(sqlAndParams[0]).toContain('rvis_rollupalias'); */
166+
167+
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
168+
console.log("!!!! res", res);
169+
expect(res).toEqual(
170+
[
171+
{
172+
vis__created_at_day: '2017-01-02T00:00:00.000Z',
173+
vis__revenue_per_id: '100'
174+
},
175+
{
176+
vis__created_at_day: '2017-01-04T00:00:00.000Z',
177+
vis__revenue_per_id: '100'
178+
},
179+
{
180+
vis__created_at_day: '2017-01-05T00:00:00.000Z',
181+
vis__revenue_per_id: '100'
182+
},
183+
{
184+
vis__created_at_day: '2017-01-06T00:00:00.000Z',
185+
vis__revenue_per_id: '200'
186+
}
187+
]
188+
189+
);
190+
});
191+
}));
192+
});

0 commit comments

Comments
 (0)