Skip to content

Commit 8b0a056

Browse files
ovrovr
authored
feat(schema-compiler): Support multi time dimensions for rollup pre-aggregations (#8291)
1 parent caefd35 commit 8b0a056

File tree

19 files changed

+1439
-106
lines changed

19 files changed

+1439
-106
lines changed

‎packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,11 @@ export class CubeEvaluator extends CubeSymbols {
225225
delete preAggregation.timeDimension;
226226
}
227227

228+
if (preAggregation.timeDimensions) {
229+
preAggregation.timeDimensionReferences = preAggregation.timeDimensions;
230+
delete preAggregation.timeDimensions;
231+
}
232+
228233
if (preAggregation.dimensions) {
229234
preAggregation.dimensionReferences = preAggregation.dimensions;
230235
delete preAggregation.dimensions;
@@ -543,10 +548,23 @@ export class CubeEvaluator extends CubeSymbols {
543548
}
544549

545550
protected evaluatePreAggregationReferences(cube, aggregation) {
546-
const timeDimensions = aggregation.timeDimensionReference ? [{
547-
dimension: this.evaluateReferences(cube, aggregation.timeDimensionReference),
548-
granularity: aggregation.granularity
549-
}] : [];
551+
const timeDimensions: any = [];
552+
553+
if (aggregation.timeDimensionReference) {
554+
timeDimensions.push({
555+
dimension: this.evaluateReferences(cube, aggregation.timeDimensionReference),
556+
granularity: aggregation.granularity
557+
});
558+
} else if (aggregation.timeDimensionReferences) {
559+
// eslint-disable-next-line guard-for-in
560+
for (const timeDimensionReference of aggregation.timeDimensionReferences) {
561+
timeDimensions.push({
562+
dimension: this.evaluateReferences(cube, timeDimensionReference.dimension),
563+
granularity: timeDimensionReference.granularity
564+
});
565+
}
566+
}
567+
550568
return {
551569
allowNonStrictDateRangeMatch: aggregation.allowNonStrictDateRangeMatch,
552570
dimensions:

‎packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ const OriginalSqlSchema = condition(
280280
const GranularitySchema = Joi.string().valid('second', 'minute', 'hour', 'day', 'week', 'month', 'quarter', 'year').required();
281281

282282
const ReferencesFields = ['timeDimensionReference', 'rollupReferences', 'measureReferences', 'dimensionReferences', 'segmentReferences'];
283-
const NonReferencesFields = ['timeDimension', 'rollups', 'measures', 'dimensions', 'segments'];
283+
const NonReferencesFields = ['timeDimension', 'timeDimensions', 'rollups', 'measures', 'dimensions', 'segments'];
284284

285285
function hasAnyField(fields, s) {
286286
return !fields.every((f) => !defined(s[f]));
@@ -373,7 +373,7 @@ const RollupLambdaSchema = condition(
373373
);
374374

375375
const RollUpSchema = condition(
376-
(s) => defined(s.granularity) || defined(s.timeDimension) || defined(s.timeDimensionReference),
376+
(s) => defined(s.granularity) || defined(s.timeDimension) || defined(s.timeDimensions) || defined(s.timeDimensionReference),
377377
condition(
378378
(s) => defined(s.timeDimensionReference),
379379
inherit(BasePreAggregation, {
@@ -385,16 +385,31 @@ const RollUpSchema = condition(
385385
dimensionReferences: Joi.func(),
386386
segmentReferences: Joi.func(),
387387
}),
388-
// Rollup without References postfix
389-
inherit(BasePreAggregation, {
390-
type: Joi.any().valid('rollup').required(),
391-
timeDimension: Joi.func().required(),
392-
allowNonStrictDateRangeMatch: Joi.bool(),
393-
granularity: GranularitySchema,
394-
measures: Joi.func(),
395-
dimensions: Joi.func(),
396-
segments: Joi.func(),
397-
})
388+
condition(
389+
(s) => defined(s.timeDimension),
390+
// Rollup without References postfix
391+
inherit(BasePreAggregation, {
392+
type: Joi.any().valid('rollup').required(),
393+
timeDimension: Joi.func().required(),
394+
allowNonStrictDateRangeMatch: Joi.bool(),
395+
granularity: GranularitySchema,
396+
measures: Joi.func(),
397+
dimensions: Joi.func(),
398+
segments: Joi.func(),
399+
}),
400+
// Rollup with multiple time dimensions
401+
inherit(BasePreAggregation, {
402+
type: Joi.any().valid('rollup').required(),
403+
timeDimensions: Joi.array().items(Joi.object().keys({
404+
dimension: Joi.func(),
405+
granularity: GranularitySchema,
406+
})),
407+
allowNonStrictDateRangeMatch: Joi.bool(),
408+
measures: Joi.func(),
409+
dimensions: Joi.func(),
410+
segments: Joi.func(),
411+
})
412+
)
398413
),
399414
Joi.alternatives().try(
400415
inherit(BasePreAggregation, {

‎packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ export const transpiledFieldsPatterns: Array<RegExp> = [
1818
/^dimensions\.[_a-zA-Z][_a-zA-Z0-9]*\.(reduceBy|reduce_by|groupBy|group_by|addGroupBy|add_group_by)$/,
1919
/^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.indexes\.[_a-zA-Z][_a-zA-Z0-9]*\.columns$/,
2020
/^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.(timeDimensionReference|timeDimension|time_dimension|segments|dimensions|measures|rollups|segmentReferences|dimensionReferences|measureReferences|rollupReferences)$/,
21+
/^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.(timeDimensions|time_dimensions)\.\d+\.dimension$/,
2122
/^contextMembers$/,
2223
/^includes$/,
2324
/^excludes$/,

‎packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,41 @@ describe('Cube Validation', () => {
444444
expect(validationResult.error).toBeFalsy();
445445
});
446446

447+
it('Partition with multi time dimensions', async () => {
448+
const cubeValidator = new CubeValidator(new CubeSymbols());
449+
const cube = {
450+
name: 'name',
451+
sql: () => '',
452+
fileName: 'fileName',
453+
preAggregations: {
454+
eventsByType: {
455+
type: 'rollup',
456+
timeDimensions: [
457+
{
458+
dimension: () => 'field1',
459+
granularity: 'day'
460+
},
461+
{
462+
dimension: () => 'field2',
463+
granularity: 'day'
464+
}
465+
],
466+
partitionGranularity: 'day',
467+
}
468+
}
469+
};
470+
471+
const validationResult = cubeValidator.validate(cube, {
472+
error: (message, e) => {
473+
console.log(message);
474+
// this callback should not be invoked
475+
expect(true).toBeFalsy();
476+
}
477+
} as any);
478+
479+
expect(validationResult.error).toBeFalsy();
480+
});
481+
447482
test('cube - aliases test', async () => {
448483
const cubeA = {
449484
name: 'CubeA',

‎packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,44 @@ describe('Yaml Schema Testing', () => {
2323
}
2424
});
2525

26+
it('pre-aggregations - success', async () => {
27+
const { compiler } = prepareYamlCompiler(
28+
`
29+
cubes:
30+
- name: Orders
31+
sql: "select * from tbl"
32+
dimensions:
33+
- name: created_at
34+
sql: created_at
35+
type: time
36+
- name: completed_at
37+
sql: completed_at
38+
type: time
39+
measures:
40+
- name: count
41+
type: count
42+
pre_aggregations:
43+
- name: multiple_time_dimensions
44+
measures:
45+
- count
46+
time_dimensions:
47+
- dimension: created_at
48+
granularity: day
49+
- dimension: completed_at
50+
granularity: day
51+
partition_granularity: day
52+
build_range_start:
53+
sql: SELECT NOW() - INTERVAL '600 day'
54+
build_range_end:
55+
sql: SELECT NOW()
56+
refresh_key:
57+
every: '1 day'
58+
`
59+
);
60+
61+
await compiler.compile();
62+
});
63+
2664
it('commented file crash', async () => {
2765
const { compiler } = prepareYamlCompiler(
2866
`

‎packages/cubejs-testing-drivers/fixtures/_schemas.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,11 @@
9090
"sql": "order_date",
9191
"type": "time"
9292
},
93+
{
94+
"name": "completedDate",
95+
"sql": "completed_date",
96+
"type": "time"
97+
},
9398
{
9499
"name": "customerId",
95100
"sql": "customer_id",
@@ -207,6 +212,11 @@
207212
"sql": "order_date",
208213
"type": "time"
209214
},
215+
{
216+
"name": "completedDate",
217+
"sql": "completed_date",
218+
"type": "time"
219+
},
210220
{
211221
"name": "customerId",
212222
"sql": "customer_id",

‎packages/cubejs-testing-drivers/fixtures/athena.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,23 @@
9292
"CUBE.totalProfit"
9393
]
9494
},
95+
{
96+
"name": "MultiTimeDimForCount",
97+
"time_dimensions": [
98+
{
99+
"dimension": "CUBE.completedDate",
100+
"granularity": "day"
101+
},
102+
{
103+
"dimension": "CUBE.orderDate",
104+
"granularity": "day"
105+
}
106+
],
107+
"partition_granularity": "month",
108+
"measures": [
109+
"CUBE.count"
110+
]
111+
},
95112
{
96113
"name": "CountByProduct",
97114
"time_dimension": "CUBE.orderDate",

‎packages/cubejs-testing-drivers/fixtures/bigquery.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,23 @@
9494
"CUBE.totalProfit"
9595
]
9696
},
97+
{
98+
"name": "MultiTimeDimForCount",
99+
"time_dimensions": [
100+
{
101+
"dimension": "CUBE.completedDate",
102+
"granularity": "day"
103+
},
104+
{
105+
"dimension": "CUBE.orderDate",
106+
"granularity": "day"
107+
}
108+
],
109+
"partition_granularity": "month",
110+
"measures": [
111+
"CUBE.count"
112+
]
113+
},
97114
{
98115
"name": "CountByProduct",
99116
"time_dimension": "CUBE.orderDate",

‎packages/cubejs-testing-drivers/fixtures/clickhouse.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,28 @@
116116
"name": "category_index",
117117
"columns": ["CUBE.productName"]
118118
}]
119+
},
120+
{
121+
"name": "MultiTimeDimForCount",
122+
"time_dimensions": [
123+
{
124+
"dimension": "CUBE.completedDate",
125+
"granularity": "day"
126+
},
127+
{
128+
"dimension": "CUBE.orderDate",
129+
"granularity": "day"
130+
}
131+
],
132+
"dimensions": ["CUBE.productName"],
133+
"partition_granularity": "month",
134+
"measures": [
135+
"CUBE.count"
136+
],
137+
"indexes": [{
138+
"name": "category_index",
139+
"columns": ["CUBE.productName"]
140+
}]
119141
}
120142
]
121143
},

‎packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,24 @@
103103
"CUBE.totalProfit"
104104
]
105105
},
106+
{
107+
"name": "MultiTimeDimForCount",
108+
"time_dimensions": [
109+
{
110+
"dimension": "CUBE.completedDate",
111+
"granularity": "day"
112+
},
113+
{
114+
"dimension": "CUBE.orderDate",
115+
"granularity": "day"
116+
}
117+
],
118+
"partition_granularity": "month",
119+
"dimensions": ["CUBE.productName"],
120+
"measures": [
121+
"CUBE.count"
122+
]
123+
},
106124
{
107125
"name": "CountByProduct",
108126
"time_dimension": "CUBE.orderDate",

0 commit comments

Comments
 (0)