Skip to content

Commit ba13bd3

Browse files
KSDaemonKSDaemon
authored
feat(schema-compiler): Allow to specify td with granularity in REST API query order section (#9630)
* feat(schema-compiler): Allow to specify td with granularity in REST API query order section This also fix ordering sql building to use the lowest granularity if there are multiple granularities for the same td provided in the query * add tests * implement getFieldAlias() in baseQuery * update orderHashToString() in QuestQuery * update orderHashToString() in ClickHouseQuery * update getFieldIndex in HiveQuery * refactor AWSElasticSearchQuery & ElasticSearchQuery * update getFieldAlias() in ElasticSearchQuery * update getFieldIndex() in DatabricksQuery * fix defaultOder() issue * lint fix * refactor findMinGranularityDimension() * chore(schema-compiler): move QueryBuilder and QueryCache to ts
1 parent 40dc666 commit ba13bd3

File tree

12 files changed

+439
-193
lines changed

12 files changed

+439
-193
lines changed

‎packages/cubejs-api-gateway/src/query.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ const evaluatedPatchMeasureExpression = parsedPatchMeasureExpression.keys({
5656
});
5757

5858
const id = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$/);
59-
const idOrMemberExpressionName = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+$/);
59+
// It might be member name, td+granularity or member expression
60+
const idOrMemberExpressionName = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$/);
6061
const dimensionWithTime = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+(\.[a-zA-Z0-9_]+)?$/);
6162
const parsedMemberExpression = Joi.object().keys({
6263
expression: Joi.alternatives(

‎packages/cubejs-backend-shared/src/time.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,67 @@ export type TimeSeriesOptions = {
1212
};
1313
type ParsedInterval = Partial<Record<unitOfTime.DurationConstructor, number>>;
1414

15+
const GRANULARITY_LEVELS: Record<string, number> = {
16+
second: 1,
17+
minute: 2,
18+
hour: 3,
19+
day: 4,
20+
week: 5,
21+
month: 6,
22+
quarter: 7,
23+
year: 8,
24+
MAX: 1000,
25+
};
26+
27+
export type DimensionToCompareGranularity = {
28+
dimension: string;
29+
expressionName?: string;
30+
granularityObj?: {
31+
minGranularity(): string;
32+
}
33+
};
34+
35+
/**
36+
* Actually dimensions type is (BaseDimension|BaseTimeDimension)[], but can not ref due to cyclic dependencies refs.
37+
*/
38+
export function findMinGranularityDimension(id: string, dimensions: DimensionToCompareGranularity[]): { index: number, dimension: DimensionToCompareGranularity | undefined } | null {
39+
const equalIgnoreCase = (a: any, b: any): boolean => (
40+
typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
41+
);
42+
43+
let minGranularity = GRANULARITY_LEVELS.MAX;
44+
let index = -1;
45+
let minGranularityIndex = -1;
46+
let field;
47+
let minGranularityField;
48+
49+
dimensions.forEach((d, i) => {
50+
if (equalIgnoreCase(d.dimension, id) || equalIgnoreCase(d.expressionName, id)) {
51+
field = d;
52+
index = i;
53+
54+
if ('granularityObj' in d && d.granularityObj) {
55+
const gr = GRANULARITY_LEVELS[d.granularityObj?.minGranularity()];
56+
if (gr < minGranularity) {
57+
minGranularityIndex = i;
58+
minGranularityField = d;
59+
minGranularity = gr;
60+
}
61+
}
62+
}
63+
});
64+
65+
if (minGranularityIndex > -1) {
66+
return { index: minGranularityIndex, dimension: minGranularityField };
67+
}
68+
69+
if (index > -1) {
70+
return { index, dimension: field };
71+
}
72+
73+
return null;
74+
}
75+
1576
export const TIME_SERIES: Record<string, (range: DateRange, timestampPrecision: number) => QueryDateRange[]> = {
1677
day: (range: DateRange, digits) => Array.from(range.snapTo('day').by('day'))
1778
.map(d => [d.format(`YYYY-MM-DDT00:00:00.${'0'.repeat(digits)}`), d.format(`YYYY-MM-DDT23:59:59.${'9'.repeat(digits)}`)]),

‎packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts‎

Lines changed: 5 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -144,42 +144,19 @@ export class DatabricksQuery extends BaseQuery {
144144
return `\`${name}\``;
145145
}
146146

147-
public getFieldIndex(id: string) {
148-
const dimension = this.dimensionsForSelect().find((d: any) => d.dimension === id);
149-
if (dimension) {
150-
return super.getFieldIndex(id);
147+
public override getFieldIndex(id: string): string | number | null {
148+
const idx = super.getFieldIndex(id);
149+
if (idx !== null) {
150+
return idx;
151151
}
152+
152153
return this.escapeColumnName(this.aliasName(id, false));
153154
}
154155

155156
public unixTimestampSql() {
156157
return 'unix_timestamp()';
157158
}
158159

159-
public orderHashToString(hash: any) {
160-
if (!hash || !hash.id) {
161-
return null;
162-
}
163-
164-
const fieldIndex = this.getFieldIndex(hash.id);
165-
if (fieldIndex === null) {
166-
return null;
167-
}
168-
169-
const dimensionsForSelect = this.dimensionsForSelect();
170-
const dimensionColumns = R.flatten(
171-
dimensionsForSelect.map((s: any) => s.selectColumns() && s.aliasName())
172-
)
173-
.filter(s => !!s);
174-
175-
if (dimensionColumns.length) {
176-
const direction = hash.desc ? 'DESC' : 'ASC';
177-
return `${fieldIndex} ${direction}`;
178-
}
179-
180-
return null;
181-
}
182-
183160
public defaultRefreshKeyRenewalThreshold() {
184161
return 120;
185162
}

‎packages/cubejs-questdb-driver/src/QuestQuery.ts‎

Lines changed: 6 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import { BaseFilter, BaseQuery, ParamAllocator } from '@cubejs-backend/schema-compiler';
1+
import {
2+
BaseFilter,
3+
BaseQuery,
4+
ParamAllocator
5+
} from '@cubejs-backend/schema-compiler';
26

37
const GRANULARITY_TO_INTERVAL: Record<string, string> = {
48
second: 's',
@@ -109,7 +113,7 @@ export class QuestQuery extends BaseQuery {
109113
}
110114
}
111115

112-
public orderHashToString(hash: any): string | null {
116+
public orderHashToString(hash: { id: string, desc: boolean }): string | null {
113117
// QuestDB has partial support for order by index column, so map these to the alias names.
114118
// So, instead of:
115119
// SELECT col_a as "a", col_b as "b" FROM tab ORDER BY 2 ASC
@@ -131,32 +135,6 @@ export class QuestQuery extends BaseQuery {
131135
return `${fieldAlias} ${direction}`;
132136
}
133137

134-
private getFieldAlias(id: string): string | null {
135-
const equalIgnoreCase = (a: any, b: any) => (
136-
typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
137-
);
138-
139-
let field;
140-
141-
field = this.dimensionsForSelect().find(
142-
(d: any) => equalIgnoreCase(d.dimension, id),
143-
);
144-
145-
if (field) {
146-
return field.aliasName();
147-
}
148-
149-
field = this.measures.find(
150-
(d: any) => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id),
151-
);
152-
153-
if (field) {
154-
return field.aliasName();
155-
}
156-
157-
return null;
158-
}
159-
160138
public groupByClause(): string {
161139
// QuestDB doesn't support group by index column, so map these to the alias names.
162140
// So, instead of:

‎packages/cubejs-schema-compiler/src/adapter/AWSElasticSearchQuery.ts‎

Lines changed: 8 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
import R from 'ramda';
21
import { BaseFilter } from './BaseFilter';
3-
import { BaseQuery } from './BaseQuery';
2+
import { ElasticSearchQuery } from './ElasticSearchQuery';
43

54
const GRANULARITY_TO_INTERVAL = {
65
day: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd 00:00:00.000')`,
@@ -21,83 +20,24 @@ class AWSElasticSearchQueryFilter extends BaseFilter {
2120
}
2221
}
2322

24-
export class AWSElasticSearchQuery extends BaseQuery {
25-
public newFilter(filter) {
23+
export class AWSElasticSearchQuery extends ElasticSearchQuery {
24+
public override newFilter(filter) {
2625
return new AWSElasticSearchQueryFilter(this, filter);
2726
}
2827

29-
public convertTz(field) {
30-
return `${field}`; // TODO
31-
}
32-
33-
public timeStampCast(value) {
34-
return `${value}`;
35-
}
36-
37-
public dateTimeCast(value) {
38-
return `${value}`; // TODO
39-
}
40-
41-
public subtractInterval(date, interval) {
28+
public override subtractInterval(date, interval) {
4229
return `DATE_SUB(${date}, INTERVAL ${interval})`;
4330
}
4431

45-
public addInterval(date, interval) {
32+
public override addInterval(date, interval) {
4633
return `DATE_ADD(${date}, INTERVAL ${interval})`;
4734
}
4835

49-
public timeGroupedColumn(granularity, dimension) {
36+
public override timeGroupedColumn(granularity, dimension) {
5037
return GRANULARITY_TO_INTERVAL[granularity](dimension);
5138
}
5239

53-
public groupByClause() {
54-
if (this.ungrouped) {
55-
return '';
56-
}
57-
const dimensionsForSelect = this.dimensionsForSelect();
58-
const dimensionColumns = R.flatten(dimensionsForSelect.map(s => s.selectColumns() && s.dimensionSql()))
59-
.filter(s => !!s);
60-
return dimensionColumns.length ? ` GROUP BY ${dimensionColumns.join(', ')}` : '';
61-
}
62-
63-
public orderHashToString(hash) {
64-
if (!hash || !hash.id) {
65-
return null;
66-
}
67-
68-
const fieldAlias = this.getFieldAlias(hash.id);
69-
70-
if (fieldAlias === null) {
71-
return null;
72-
}
73-
74-
const direction = hash.desc ? 'DESC' : 'ASC';
75-
return `${fieldAlias} ${direction}`;
76-
}
77-
78-
public getFieldAlias(id) {
79-
const equalIgnoreCase = (a, b) => (
80-
typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
81-
);
82-
83-
let field;
84-
85-
field = this.dimensionsForSelect().find(d => equalIgnoreCase(d.dimension, id));
86-
87-
if (field) {
88-
return field.dimensionSql();
89-
}
90-
91-
field = this.measures.find(d => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id));
92-
93-
if (field) {
94-
return field.aliasName(); // TODO isn't supported
95-
}
96-
97-
return null;
98-
}
99-
100-
public escapeColumnName(name) {
101-
return `${name}`; // TODO
40+
public override unixTimestampSql() {
41+
return 'EXTRACT(EPOCH FROM NOW())';
10242
}
10343
}

0 commit comments

Comments
 (0)