Skip to content

Commit d94afeb

Browse files
KSDaemonKSDaemon
authored
fix(schema-compiler): Fix timeshift measure queries coming from SQL API (#9570)
* wip * fix expression path for dimension * add comment * add timeshift support for member expressions coming from SQL API * fix timeshifts for views * remove old timeshift flow * Add tests
1 parent c67f57e commit d94afeb

26 files changed

+1078
-41
lines changed

‎packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ export class BaseDimension {
140140

141141
public expressionPath(): string {
142142
if (this.expression) {
143-
return `expr:${this.expression.expressionName}`;
143+
return `expr:${this.expressionName}`;
144144
}
145145
return this.query.cubeEvaluator.pathFromArray(this.path() as string[]);
146146
}

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 63 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1301,8 +1301,27 @@ export class BaseQuery {
13011301
const multiStageMembers = R.uniq(
13021302
this.allMembersConcat(false)
13031303
// TODO boolean logic filter support
1304-
.filter(m => m.expressionPath && hasMultiStageMembers(m.expressionPath()))
1305-
.map(m => m.expressionPath())
1304+
.reduce((acc, m) => {
1305+
if (m.isMemberExpression) {
1306+
let refMemberPath;
1307+
this.evaluateSql(m.cube().name, m.definition().sql, {
1308+
sqlResolveFn: (_symbol, cube, prop) => {
1309+
const path = this.cubeEvaluator.pathFromArray([cube, prop]);
1310+
refMemberPath = path;
1311+
// We don't need real SQL here, so just returning something.
1312+
return path;
1313+
}
1314+
});
1315+
1316+
if (hasMultiStageMembers(refMemberPath)) {
1317+
acc.push(refMemberPath);
1318+
}
1319+
} else if (m.expressionPath && hasMultiStageMembers(m.expressionPath())) {
1320+
acc.push(m.expressionPath());
1321+
}
1322+
1323+
return acc;
1324+
}, [])
13061325
).map(m => this.multiStageWithQueries(
13071326
m,
13081327
{
@@ -1312,7 +1331,8 @@ export class BaseQuery {
13121331
timeDimensions: this.options.timeDimensions || [],
13131332
multiStageTimeDimensions: (this.options.timeDimensions || []).filter(td => !!td.granularity),
13141333
// TODO accessing filters directly from options might miss some processing logic
1315-
filters: this.options.filters || []
1334+
filters: this.options.filters || [],
1335+
segments: this.options.segments || [],
13161336
},
13171337
allMemberChildren,
13181338
withQueries
@@ -1426,46 +1446,27 @@ export class BaseQuery {
14261446
queryContext = { ...queryContext, dimensions: R.uniq(queryContext.dimensions.concat(memberDef.addGroupByReferences)) };
14271447
}
14281448
if (memberDef.timeShiftReferences?.length) {
1429-
let mapFn;
1430-
1431-
const allBackAliasMembers = this.allBackAliasTimeDimensions();
1449+
let { commonTimeShift } = queryContext;
1450+
const timeShifts = queryContext.timeShifts || {};
1451+
const memberOfCube = !this.cubeEvaluator.cubeFromPath(memberPath).isView;
14321452

14331453
if (memberDef.timeShiftReferences.length === 1 && !memberDef.timeShiftReferences[0].timeDimension) {
14341454
const timeShift = memberDef.timeShiftReferences[0];
1435-
mapFn = (td) => {
1436-
// We need to ignore aliased td, because it will match and insert shiftInterval on first
1437-
// occurrence, but later during recursion it will hit the original td but shiftInterval will be
1438-
// present and simple check for td.shiftInterval will always result in error.
1439-
if (td.shiftInterval && !td.dimension === allBackAliasMembers[timeShift.timeDimension]) {
1440-
throw new UserError(`Hierarchical time shift is not supported but was provided for '${td.dimension}'. Parent time shift is '${td.shiftInterval}' and current is '${timeShift.interval}'`);
1441-
}
1442-
return {
1443-
...td,
1444-
shiftInterval: timeShift.type === 'next' ? this.negateInterval(timeShift.interval) : timeShift.interval
1445-
};
1446-
};
1447-
} else {
1448-
mapFn = (td) => {
1449-
const timeShift = memberDef.timeShiftReferences.find(r => r.timeDimension === td.dimension || td.dimension === allBackAliasMembers[r.timeDimension]);
1450-
if (timeShift) {
1451-
// We need to ignore aliased td, because it will match and insert shiftInterval on first
1452-
// occurrence, but later during recursion it will hit the original td but shiftInterval will be
1453-
// present and simple check for td.shiftInterval will always result in error.
1454-
if (td.shiftInterval && !td.dimension === allBackAliasMembers[timeShift.timeDimension]) {
1455-
throw new UserError(`Hierarchical time shift is not supported but was provided for '${td.dimension}'. Parent time shift is '${td.shiftInterval}' and current is '${timeShift.interval}'`);
1456-
}
1457-
return {
1458-
...td,
1459-
shiftInterval: timeShift.type === 'next' ? this.negateInterval(timeShift.interval) : timeShift.interval
1460-
};
1461-
}
1462-
return td;
1463-
};
1455+
// We avoid view's timeshift evaluation as there will be another round of underlying cube's member evaluation
1456+
if (memberOfCube) {
1457+
commonTimeShift = timeShift.type === 'next' ? this.negateInterval(timeShift.interval) : timeShift.interval;
1458+
}
1459+
} else if (memberOfCube) {
1460+
// We avoid view's timeshift evaluation as there will be another round of underlying cube's member evaluation
1461+
memberDef.timeShiftReferences.forEach((r) => {
1462+
timeShifts[r.timeDimension] = r.type === 'next' ? this.negateInterval(r.interval) : r.interval;
1463+
});
14641464
}
14651465

14661466
queryContext = {
14671467
...queryContext,
1468-
timeDimensions: queryContext.timeDimensions.map(mapFn)
1468+
commonTimeShift,
1469+
timeShifts,
14691470
};
14701471
}
14711472
queryContext = {
@@ -1508,11 +1509,15 @@ export class BaseQuery {
15081509
...queryContext,
15091510
// TODO make it same way as keepFilters
15101511
timeDimensions: queryContext.timeDimensions.map(td => ({ ...td, dateRange: undefined })),
1512+
// TODO keep segments related to this multistage (if applicable)
1513+
segments: [],
15111514
filters: this.keepFilters(queryContext.filters, filterMember => filterMember === memberPath),
15121515
};
15131516
} else {
15141517
queryContext = {
15151518
...queryContext,
1519+
// TODO remove not related segments
1520+
// segments: queryContext.segments,
15161521
filters: this.keepFilters(queryContext.filters, filterMember => !this.memberInstanceByPath(filterMember).isMultiStage()),
15171522
};
15181523
}
@@ -1531,12 +1536,16 @@ export class BaseQuery {
15311536
return [m, measure.aliasName()];
15321537
}).concat(from.dimensions.map(m => {
15331538
const member = this.newDimension(m);
1534-
return [m, member.aliasName()];
1539+
// In case of request coming from the SQL API, member could be expression-based
1540+
const mPath = typeof m === 'string' ? m : this.cubeEvaluator.pathFromArray([m.cubeName, m.name]);
1541+
return [mPath, member.aliasName()];
15351542
})).concat(from.timeDimensions.map(m => {
15361543
const member = this.newTimeDimension(m);
15371544
return member.granularity ? [`${member.dimension}.${member.granularity}`, member.aliasName()] : [];
15381545
}))))
1539-
)
1546+
),
1547+
commonTimeShift: withQuery.commonTimeShift,
1548+
timeShifts: withQuery.timeShifts,
15401549
};
15411550

15421551
const fromSubQuery = fromMeasures && this.newSubQuery({
@@ -1568,6 +1577,7 @@ export class BaseQuery {
15681577
multiStageDimensions: withQuery.multiStageDimensions,
15691578
multiStageTimeDimensions: withQuery.multiStageTimeDimensions,
15701579
filters: withQuery.filters,
1580+
segments: withQuery.segments,
15711581
from: fromSql && {
15721582
sql: fromSql,
15731583
alias: `${withQuery.alias}_join`,
@@ -2874,10 +2884,23 @@ export class BaseQuery {
28742884
return td.dimensionSql();
28752885
} else {
28762886
let res = this.autoPrefixAndEvaluateSql(cubeName, symbol.sql, isMemberExpr);
2887+
const memPath = this.cubeEvaluator.pathFromArray([cubeName, name]);
28772888

2878-
if (symbol.shiftInterval) {
2879-
res = `(${this.addTimestampInterval(res, symbol.shiftInterval)})`;
2889+
// Skip view's member evaluation as there will be underlying cube's same member evaluation
2890+
if (symbol.type === 'time' && !this.cubeEvaluator.cubeFromPath(memPath).isView) {
2891+
if (this.safeEvaluateSymbolContext().timeShifts?.[memPath]) {
2892+
if (symbol.shiftInterval) {
2893+
throw new UserError(`Hierarchical time shift is not supported but was provided for '${memPath}'. Parent time shift is '${symbol.shiftInterval}' and current is '${this.safeEvaluateSymbolContext().timeShifts?.[memPath]}'`);
2894+
}
2895+
res = `(${this.addTimestampInterval(res, this.safeEvaluateSymbolContext().timeShifts?.[memPath])})`;
2896+
} else if (this.safeEvaluateSymbolContext().commonTimeShift) {
2897+
if (symbol.shiftInterval) {
2898+
throw new UserError(`Hierarchical time shift is not supported but was provided for '${memPath}'. Parent time shift is '${symbol.shiftInterval}' and current is '${this.safeEvaluateSymbolContext().commonTimeShift}'`);
2899+
}
2900+
res = `(${this.addTimestampInterval(res, this.safeEvaluateSymbolContext().commonTimeShift)})`;
2901+
}
28802902
}
2903+
28812904
if (this.safeEvaluateSymbolContext().convertTzForRawTimeDimension &&
28822905
!this.safeEvaluateSymbolContext().ignoreConvertTzForTimeDimension &&
28832906
!memberExpressionType &&

‎packages/cubejs-testing-drivers/fixtures/_schemas.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,16 @@
197197
"sql": "quantity",
198198
"type": "sum"
199199
},
200+
{
201+
"name": "totalQuantityPriorMonth",
202+
"multiStage": true,
203+
"sql": "{totalQuantity}",
204+
"type": "number",
205+
"timeShift": [{
206+
"interval": "1 month",
207+
"type": "prior"
208+
}]
209+
},
200210
{
201211
"name": "avgDiscount",
202212
"sql": "discount",

‎packages/cubejs-testing-drivers/src/tests/testQueries.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2025,5 +2025,19 @@ from
20252025
`);
20262026
expect(res.rows).toMatchSnapshot('nulls_first_last_sql_push_down');
20272027
});
2028+
2029+
executePg('SQL API: Timeshift measure from cube', async (connection) => {
2030+
const res = await connection.query(`
2031+
SELECT
2032+
DATE_TRUNC('month', orderDate) AS "orderDate",
2033+
MEASURE(totalQuantity) AS "totalQuantity",
2034+
MEASURE(totalQuantityPriorMonth) AS "totalQuantityPriorMonth"
2035+
FROM "ECommerce"
2036+
WHERE orderDate >= CAST('2020-01-01' AS DATE) AND orderDate < CAST('2021-01-01' AS DATE)
2037+
GROUP BY 1
2038+
ORDER BY 1 ASC NULLS FIRST;
2039+
`);
2040+
expect(res.rows).toMatchSnapshot();
2041+
});
20282042
});
20292043
}

‎packages/cubejs-testing-drivers/test/__snapshots__/athena-export-bucket-s3-full.test.ts.snap‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,51 @@ Array [
1414
]
1515
`;
1616

17+
exports[`Queries with the @cubejs-backend/athena-driver SQL API: Timeshift measure from cube 1`] = `
18+
Array [
19+
Object {
20+
"orderDate": 2020-02-01T00:00:00.000Z,
21+
"totalQuantity": 2,
22+
"totalQuantityPriorMonth": 6,
23+
},
24+
Object {
25+
"orderDate": 2020-03-01T00:00:00.000Z,
26+
"totalQuantity": 13,
27+
"totalQuantityPriorMonth": 2,
28+
},
29+
Object {
30+
"orderDate": 2020-04-01T00:00:00.000Z,
31+
"totalQuantity": 3,
32+
"totalQuantityPriorMonth": 13,
33+
},
34+
Object {
35+
"orderDate": 2020-05-01T00:00:00.000Z,
36+
"totalQuantity": 15,
37+
"totalQuantityPriorMonth": 3,
38+
},
39+
Object {
40+
"orderDate": 2020-06-01T00:00:00.000Z,
41+
"totalQuantity": 18,
42+
"totalQuantityPriorMonth": 15,
43+
},
44+
Object {
45+
"orderDate": 2020-10-01T00:00:00.000Z,
46+
"totalQuantity": 11,
47+
"totalQuantityPriorMonth": 27,
48+
},
49+
Object {
50+
"orderDate": 2020-11-01T00:00:00.000Z,
51+
"totalQuantity": 43,
52+
"totalQuantityPriorMonth": 11,
53+
},
54+
Object {
55+
"orderDate": 2020-12-01T00:00:00.000Z,
56+
"totalQuantity": 22,
57+
"totalQuantityPriorMonth": 43,
58+
},
59+
]
60+
`;
61+
1762
exports[`Queries with the @cubejs-backend/athena-driver SQL API: metabase count cast to float32 from push down: metabase_count_cast_to_float32_from_push_down 1`] = `
1863
Array [
1964
Object {

‎packages/cubejs-testing-drivers/test/__snapshots__/bigquery-export-bucket-gcs-full.test.ts.snap‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3226,6 +3226,51 @@ Array [
32263226
]
32273227
`;
32283228

3229+
exports[`Queries with the @cubejs-backend/bigquery-driver SQL API: Timeshift measure from cube 1`] = `
3230+
Array [
3231+
Object {
3232+
"orderDate": 2020-02-01T00:00:00.000Z,
3233+
"totalQuantity": 2,
3234+
"totalQuantityPriorMonth": 6,
3235+
},
3236+
Object {
3237+
"orderDate": 2020-03-01T00:00:00.000Z,
3238+
"totalQuantity": 13,
3239+
"totalQuantityPriorMonth": 2,
3240+
},
3241+
Object {
3242+
"orderDate": 2020-04-01T00:00:00.000Z,
3243+
"totalQuantity": 3,
3244+
"totalQuantityPriorMonth": 13,
3245+
},
3246+
Object {
3247+
"orderDate": 2020-05-01T00:00:00.000Z,
3248+
"totalQuantity": 15,
3249+
"totalQuantityPriorMonth": 3,
3250+
},
3251+
Object {
3252+
"orderDate": 2020-06-01T00:00:00.000Z,
3253+
"totalQuantity": 18,
3254+
"totalQuantityPriorMonth": 15,
3255+
},
3256+
Object {
3257+
"orderDate": 2020-10-01T00:00:00.000Z,
3258+
"totalQuantity": 11,
3259+
"totalQuantityPriorMonth": 27,
3260+
},
3261+
Object {
3262+
"orderDate": 2020-11-01T00:00:00.000Z,
3263+
"totalQuantity": 43,
3264+
"totalQuantityPriorMonth": 11,
3265+
},
3266+
Object {
3267+
"orderDate": 2020-12-01T00:00:00.000Z,
3268+
"totalQuantity": 22,
3269+
"totalQuantityPriorMonth": 43,
3270+
},
3271+
]
3272+
`;
3273+
32293274
exports[`Queries with the @cubejs-backend/bigquery-driver SQL API: metabase count cast to float32 from push down: metabase_count_cast_to_float32_from_push_down 1`] = `
32303275
Array [
32313276
Object {

‎packages/cubejs-testing-drivers/test/__snapshots__/clickhouse-export-bucket-s3-full.test.ts.snap‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1620,6 +1620,51 @@ Array [
16201620
]
16211621
`;
16221622

1623+
exports[`Queries with the @cubejs-backend/clickhouse-driver export-bucket-s3 SQL API: Timeshift measure from cube 1`] = `
1624+
Array [
1625+
Object {
1626+
"orderDate": 2020-02-01T00:00:00.000Z,
1627+
"totalQuantity": 2,
1628+
"totalQuantityPriorMonth": 6,
1629+
},
1630+
Object {
1631+
"orderDate": 2020-03-01T00:00:00.000Z,
1632+
"totalQuantity": 13,
1633+
"totalQuantityPriorMonth": 2,
1634+
},
1635+
Object {
1636+
"orderDate": 2020-04-01T00:00:00.000Z,
1637+
"totalQuantity": 3,
1638+
"totalQuantityPriorMonth": 13,
1639+
},
1640+
Object {
1641+
"orderDate": 2020-05-01T00:00:00.000Z,
1642+
"totalQuantity": 15,
1643+
"totalQuantityPriorMonth": 3,
1644+
},
1645+
Object {
1646+
"orderDate": 2020-06-01T00:00:00.000Z,
1647+
"totalQuantity": 18,
1648+
"totalQuantityPriorMonth": 15,
1649+
},
1650+
Object {
1651+
"orderDate": 2020-10-01T00:00:00.000Z,
1652+
"totalQuantity": 11,
1653+
"totalQuantityPriorMonth": 27,
1654+
},
1655+
Object {
1656+
"orderDate": 2020-11-01T00:00:00.000Z,
1657+
"totalQuantity": 43,
1658+
"totalQuantityPriorMonth": 11,
1659+
},
1660+
Object {
1661+
"orderDate": 2020-12-01T00:00:00.000Z,
1662+
"totalQuantity": 22,
1663+
"totalQuantityPriorMonth": 43,
1664+
},
1665+
]
1666+
`;
1667+
16231668
exports[`Queries with the @cubejs-backend/clickhouse-driver export-bucket-s3 SQL API: metabase count cast to float32 from push down: metabase_count_cast_to_float32_from_push_down 1`] = `
16241669
Array [
16251670
Object {

0 commit comments

Comments
 (0)