diff --git a/packages/cubejs-ksql-driver/src/KsqlQuery.ts b/packages/cubejs-ksql-driver/src/KsqlQuery.ts index 3af3a71f4f16e..91cc854155abc 100644 --- a/packages/cubejs-ksql-driver/src/KsqlQuery.ts +++ b/packages/cubejs-ksql-driver/src/KsqlQuery.ts @@ -83,10 +83,6 @@ export class KsqlQuery extends BaseQuery { return dimensionColumns.length ? ` GROUP BY ${dimensionColumns.join(', ')}` : ''; } - public partitionInvalidateKeyQueries(cube: string, preAggregation: any) { - return []; - } - public preAggregationStartEndQueries(cube: string, preAggregation: any) { if (preAggregation.partitionGranularity) { if (!preAggregation.refreshRangeStart) { @@ -97,7 +93,7 @@ export class KsqlQuery extends BaseQuery { } } const res = this.evaluateSymbolSqlWithContext(() => [ - + preAggregation.refreshRangeStart && [this.evaluateSql(cube, preAggregation.refreshRangeStart.sql, {}), [], { external: true }], preAggregation.refreshRangeEnd && [this.evaluateSql(cube, preAggregation.refreshRangeEnd.sql, {}), [], { external: true }] ], { preAggregationQuery: true }); diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts index 52a24f6df862b..c0f2dc0a38d08 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts @@ -126,7 +126,7 @@ export class BaseDimension { return this.dimensionDefinition().fieldType; } - public path() { + public path(): string[] | null { if (this.expression) { return null; } @@ -138,10 +138,10 @@ export class BaseDimension { return this.query.cubeEvaluator.parsePath('dimensions', this.dimension); } - public expressionPath() { + public expressionPath(): string { if (this.expression) { return `expr:${this.expression.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path()); + return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts index 5b532076b6467..71a59d3ae4f08 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts @@ -47,11 +47,10 @@ export class BaseFilter extends BaseDimension { } // Evaluates filters on measures to whole where statement in query - // It used in drill downs + // It used in drill-downs public measureFilterToWhere() { const measureDefinition = this.measureDefinition(); - if (measureDefinition.filters && measureDefinition.filters.length || - measureDefinition.drillFilters && measureDefinition.drillFilters.length) { + if (measureDefinition.filters?.length || measureDefinition.drillFilters?.length) { return this.query.evaluateFiltersArray( (measureDefinition.filters || []).concat(measureDefinition.drillFilters || []), this.query.cubeEvaluator.cubeNameFromPath(this.measure) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts b/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts index 49f17519739aa..484f9e3aefa23 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts @@ -3,9 +3,9 @@ export class BaseGroupFilter { protected readonly operator: any; - protected readonly measure: any; + public readonly measure: any; - protected readonly dimension: any; + public readonly dimension: any; public constructor(filter: any) { this.values = filter.values; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts index 81d7a8c436814..dff05deb9ff5f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts @@ -331,6 +331,6 @@ export class BaseMeasure { if (this.expression) { return `expr:${this.expression.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path()); + return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index a1e6e2ff73dd8..60236542389df 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -101,6 +101,33 @@ const SecondsDurations = { * and {@code CompilerApi} configuration. */ export class BaseQuery { + /** @type {import('./PreAggregations').PreAggregations} */ + preAggregations; + + /** @type {import('./BaseMeasure').BaseMeasure[]} */ + measures; + + /** @type {import('./BaseDimension').BaseDimension[]} */ + dimensions; + + /** @type {import('./BaseDimension').BaseDimension[]} */ + multiStageDimensions; + + /** @type {import('./BaseTimeDimension').BaseTimeDimension[]} */ + multiStageTimeDimensions; + + /** @type {import('./BaseSegment').BaseSegment[]} */ + segments; + + /** @type {(BaseFilter|BaseGroupFilter)[]} */ + filters; + + /** @type {(BaseFilter|BaseGroupFilter)[]} */ + measureFilters; + + /** @type {import('./BaseTimeDimension').BaseTimeDimension[]} */ + timeDimensions; + /** * BaseQuery class constructor. * @param {Compilers|*} compilers @@ -237,7 +264,7 @@ export class BaseQuery { rowLimit: this.options.rowLimit, preAggregationsSchema: this.options.preAggregationsSchema, className: this.constructor.name, - externalClassName: this.options.externalQueryClass && this.options.externalQueryClass.name, + externalClassName: this.options.externalQueryClass?.name, preAggregationQuery: this.options.preAggregationQuery, disableExternalPreAggregations: this.options.disableExternalPreAggregations, useOriginalSqlPreAggregationsInPreAggregation: this.options.useOriginalSqlPreAggregationsInPreAggregation, @@ -258,20 +285,28 @@ export class BaseQuery { this.timezone = this.options.timezone; this.rowLimit = this.options.rowLimit; this.offset = this.options.offset; + /** @type {import('./PreAggregations').PreAggregations} */ this.preAggregations = this.newPreAggregations(); + /** @type {import('./BaseMeasure').BaseMeasure[]} */ this.measures = (this.options.measures || []).map(this.newMeasure.bind(this)); + /** @type {import('./BaseDimension').BaseDimension[]} */ this.dimensions = (this.options.dimensions || []).map(this.newDimension.bind(this)); + /** @type {import('./BaseDimension').BaseDimension[]} */ this.multiStageDimensions = (this.options.multiStageDimensions || []).map(this.newDimension.bind(this)); + /** @type {import('./BaseTimeDimension').BaseTimeDimension[]} */ this.multiStageTimeDimensions = (this.options.multiStageTimeDimensions || []).map(this.newTimeDimension.bind(this)); + /** @type {import('./BaseSegment').BaseSegment[]} */ this.segments = (this.options.segments || []).map(this.newSegment.bind(this)); const filters = this.extractFiltersAsTree(this.options.filters || []); // measure_filter (the one extracted from filters parameter on measure and - // used in drill downs) should go to WHERE instead of HAVING + // used in drill-downs) should go to WHERE instead of HAVING /** @type {(BaseFilter|BaseGroupFilter)[]} */ this.filters = filters.filter(f => f.dimensionGroup || f.dimension || f.operator === 'measure_filter' || f.operator === 'measureFilter').map(this.initFilter.bind(this)); + /** @type {(BaseFilter|BaseGroupFilter)[]} */ this.measureFilters = filters.filter(f => (f.measureGroup || f.measure) && f.operator !== 'measure_filter' && f.operator !== 'measureFilter').map(this.initFilter.bind(this)); + /** @type {import('./BaseTimeDimension').BaseTimeDimension[]} */ this.timeDimensions = (this.options.timeDimensions || []).map(dimension => { if (!dimension.dimension) { const join = this.joinGraph.buildJoin(this.collectJoinHints(true)); @@ -471,10 +506,20 @@ export class BaseQuery { return res; } + /** + * + * @param measurePath + * @returns {BaseMeasure} + */ newMeasure(measurePath) { return new BaseMeasure(this, measurePath); } + /** + * + * @param dimensionPath + * @returns {BaseDimension} + */ newDimension(dimensionPath) { if (typeof dimensionPath === 'string') { const memberArr = dimensionPath.split('.'); @@ -492,6 +537,11 @@ export class BaseQuery { return new BaseDimension(this, dimensionPath); } + /** + * + * @param segmentPath + * @returns {BaseSegment} + */ newSegment(segmentPath) { return new BaseSegment(this, segmentPath); } @@ -515,6 +565,11 @@ export class BaseQuery { return new BaseFilter(this, filter); } + /** + * + * @param filter + * @returns {BaseGroupFilter} + */ newGroupFilter(filter) { return new BaseGroupFilter(filter); } @@ -527,10 +582,19 @@ export class BaseQuery { return new BaseTimeDimension(this, timeDimension); } + /** + * + * @param expressionParams + * @returns {ParamAllocator} + */ newParamAllocator(expressionParams) { return new ParamAllocator(expressionParams); } + /** + * + * @returns {PreAggregations} + */ newPreAggregations() { return new PreAggregations(this, this.options.historyQueries || [], this.options.cubeLatticeCache); } @@ -558,7 +622,7 @@ export class BaseQuery { if (!this.options.preAggregationQuery) { preAggForQuery = this.preAggregations.findPreAggregationForQuery(); - if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) { + if (this.options.disableExternalPreAggregations && preAggForQuery?.preAggregation.external) { preAggForQuery = undefined; } } @@ -1199,7 +1263,7 @@ export class BaseQuery { } fullKeyQueryAggregateMeasures(context) { - const measureToHierarchy = this.collectRootMeasureToHieararchy(context); + const measureToHierarchy = this.collectRootMeasureToHierarchy(context); const allMemberChildren = this.collectAllMemberChildren(context); const memberToIsMultiStage = this.collectAllMultiStageMembers(allMemberChildren); @@ -1859,7 +1923,7 @@ export class BaseQuery { }]]; } - collectRootMeasureToHieararchy(context) { + collectRootMeasureToHierarchy(context) { const notAddedMeasureFilters = R.flatten(this.measureFilters.map(f => f.getMembers())) .filter(f => R.none(m => m.measure === f.measure, this.measures)); @@ -2291,9 +2355,13 @@ export class BaseQuery { } } - collectCubeNames(excludeTimeDimensions) { + /** + * + * @returns {Array} + */ + collectCubeNames() { return this.collectFromMembers( - excludeTimeDimensions, + [], this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor' ); @@ -2413,6 +2481,11 @@ export class BaseQuery { return this.rollupGroupByClause(dimensionNames); } + /** + * XXX: String as return value is added because of HiveQuery.getFieldIndex() + * @param id + * @returns {number|string|null} + */ getFieldIndex(id) { const equalIgnoreCase = (a, b) => ( typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase() @@ -2884,6 +2957,11 @@ export class BaseQuery { ); } + /** + * + * @param fn + * @returns {Array} + */ collectCubeNamesFor(fn) { const context = { cubeNames: [] }; this.evaluateSymbolSqlWithContext( @@ -2903,6 +2981,11 @@ export class BaseQuery { return context.joinHints; } + /** + * + * @param fn + * @returns {Array} + */ collectMemberNamesFor(fn) { const context = { memberNames: [] }; this.evaluateSymbolSqlWithContext( @@ -3182,6 +3265,10 @@ export class BaseQuery { ); } + /** + * @param cubeName + * @returns Boolean + */ multipliedJoinRowResult(cubeName) { // this.join not initialized on collectCubeNamesForSql return this.join && this.join.multiplicationFactor[cubeName]; @@ -3312,6 +3399,11 @@ export class BaseQuery { return inflection.underscore(name).replace(/\./g, isPreAggregationName ? '_' : '__'); } + /** + * + * @param options + * @returns {BaseQuery} + */ newSubQuery(options) { const QueryClass = this.constructor; return new QueryClass(this.compilers, this.subQueryOptions(options)); @@ -3784,6 +3876,12 @@ export class BaseQuery { }; } + /** + * + * @param cube + * @param preAggregation + * @returns {BaseQuery} + */ // eslint-disable-next-line consistent-return preAggregationQueryForSqlEvaluation(cube, preAggregation) { if (preAggregation.type === 'autoRollup') { diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts index 2c15c80569d1d..854eb8da1c8f2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts @@ -92,6 +92,6 @@ export class BaseSegment { if (this.expression) { return `expr:${this.expression.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path()); + return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts index e4c677c3d1074..2bffbb0177652 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts @@ -3,6 +3,8 @@ import { parseSqlInterval } from '@cubejs-backend/shared'; import { BaseQuery } from './BaseQuery'; import { BaseFilter } from './BaseFilter'; import { BaseMeasure } from './BaseMeasure'; +import { BaseSegment } from './BaseSegment'; +import { BaseGroupFilter } from './BaseGroupFilter'; const GRANULARITY_TO_INTERVAL: Record = { day: 'day', @@ -191,9 +193,9 @@ export class CubeStoreQuery extends BaseQuery { const maxRollingWindow = cumulativeMeasuresWithoutMultiplied.reduce((a, b) => this.maxRollingWindow(a, b.rollingWindowDefinition()), null); const commonDateCondition = this.rollingWindowDateJoinCondition(maxRollingWindow.trailing, maxRollingWindow.leading, maxRollingWindow.offset); - const filters = this.segments.concat(this.filters).concat( + const filters: (BaseSegment | BaseFilter | BaseGroupFilter)[] = [...this.segments, ...this.filters, ...( timeDimension?.dateRange && this.dateFromStartToEndConditionSql(commonDateCondition, true, true) || [] - ); + )]; const rollupGranularity = this.preAggregations?.castGranularity(preAggregationForQuery.preAggregation.granularity) || 'day'; const granularityOverride = timeDimensionWithGranularity && cumulativeMeasuresWithoutMultiplied.reduce((a, b) => this.minGranularity(a, b.windowGranularity()), timeDimensionWithGranularity.granularity) || rollupGranularity; diff --git a/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts b/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts index de5da226f84a2..a60fa33feddd3 100644 --- a/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts @@ -23,7 +23,7 @@ class HiveFilter extends BaseFilter { export class HiveQuery extends BaseQuery { public newFilter(filter) { - return new HiveFilter(this, filter); + return new HiveFilter(this as BaseQuery, filter); } public convertTz(field) { @@ -67,11 +67,11 @@ export class HiveQuery extends BaseQuery { const select = this.evaluateSymbolSqlWithContext( () => this.dimensionsForSelect().map( d => d.aliasName() - ).concat(this.measures.map(m => m.selectColumns())).filter(s => !!s), { + ).concat(this.measures.flatMap(m => m.selectColumns())).filter(s => !!s), { ungroupedAliases: R.fromPairs(this.forSelect().map((m: any) => [m.measure || m.dimension, m.aliasName()])) } ); - return `SELECT ${select} FROM (${ungrouped}) AS ${this.escapeColumnName('hive_wrapper')} + return `SELECT ${select} FROM (${ungrouped}) AS ${this.escapeColumnName('hive_wrapper')} ${this.groupByClause()}${this.baseHaving(this.measureFilters)}${this.orderBy()}${this.groupByDimensionLimit()}`; } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts similarity index 76% rename from packages/cubejs-schema-compiler/src/adapter/PreAggregations.js rename to packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index bf3e041823813..4dfb1db0f080a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1,16 +1,84 @@ import R from 'ramda'; -import { FROM_PARTITION_RANGE, getEnv, TO_PARTITION_RANGE } from '@cubejs-backend/shared'; import { CubeSymbols } from '../compiler/CubeSymbols'; import { UserError } from '../compiler/UserError'; +import { BaseQuery } from './BaseQuery'; +import { + PreAggregationDefinition, PreAggregationDefinitions, + PreAggregationReferences, + PreAggregationTimeDimensionReference +} from '../compiler/CubeEvaluator'; +import { BaseTimeDimension } from './BaseTimeDimension'; +import { BaseMeasure } from './BaseMeasure'; +import { BaseFilter } from './BaseFilter'; +import { BaseGroupFilter } from './BaseGroupFilter'; +import { BaseDimension } from './BaseDimension'; +import { BaseSegment } from './BaseSegment'; + +export type RollupJoin = any; + +export type PreAggregationDefinitionExtended = PreAggregationDefinition & { + unionWithSourceData: boolean; + rollupLambdaId: string; + lastRollupLambda: boolean; + rollupLambdaTimeDimensionsReference: PreAggregationTimeDimensionReference[]; + readOnly: boolean; + partitionGranularity: string; + streamOffset: 'earliest' | 'latest'; + uniqueKeyColumns: string; + sqlAlias?: string; +}; + +export type PreAggregationForQuery = { + preAggregationName: string; + cube: string; + canUsePreAggregation: boolean; + preAggregationId: string; + preAggregation: PreAggregationDefinitionExtended; + references: PreAggregationReferences; + preAggregationsToJoin?: PreAggregationForQuery[]; + referencedPreAggregations?: PreAggregationForQuery[]; + rollupJoin?: RollupJoin; + sqlAlias?: string; +}; + +export type PreAggregationForCube = { + preAggregationName: string; + cube: string; + preAggregation: PreAggregationDefinitionExtended; + references: PreAggregationReferences; +}; + +export type BaseMember = BaseDimension | BaseMeasure | BaseFilter | BaseGroupFilter | BaseSegment; + +export type CanUsePreAggregationFn = (references: PreAggregationReferences) => boolean; + +/** + * TODO: Write a real type. + * @see return value of PreAggregations.preAggregationDescriptionFor() + */ +export type FullPreAggregationDescription = any; + +/** + * TODO: Write a real type. + * @see return value of static PreAggregations.transformQueryToCanUseForm() + */ +export type TransformedQuery = any; export class PreAggregations { - /** - * @param {import('../adapter/BaseQuery').BaseQuery} query - * @param historyQueries - * @param cubeLatticeCache - */ - constructor(query, historyQueries, cubeLatticeCache) { + private readonly query: BaseQuery; + + private readonly historyQueries: any; + + private readonly cubeLatticeCache: any; + + private readonly cubeLattices: {}; + + private hasCumulativeMeasuresValue: boolean = false; + + public preAggregationForQuery: PreAggregationForQuery | undefined = undefined; + + public constructor(query: BaseQuery, historyQueries, cubeLatticeCache) { this.query = query; this.historyQueries = historyQueries; this.cubeLatticeCache = cubeLatticeCache; @@ -18,23 +86,23 @@ export class PreAggregations { } /** - * @return {unknown[]} + * It returns full pre-aggregation object (with keyQueries, previewSql, loadSql, and so on. */ - preAggregationsDescription() { + public preAggregationsDescription(): FullPreAggregationDescription[] { const preAggregations = [this.preAggregationsDescriptionLocal()].concat( this.query.subQueryDimensions.map(d => this.query.subQueryDescription(d).subQuery) .map(q => q.preAggregations.preAggregationsDescription()) ); return R.pipe( - R.unnest, + R.unnest as (list: any[][]) => any[], R.uniqBy(desc => desc.tableName) )( preAggregations ); } - preAggregationsDescriptionLocal() { + private preAggregationsDescriptionLocal(): FullPreAggregationDescription[] { const isInPreAggregationQuery = this.query.options.preAggregationQuery; if (!isInPreAggregationQuery) { const preAggregationForQuery = this.findPreAggregationForQuery(); @@ -50,25 +118,25 @@ export class PreAggregations { const { preAggregations } = this.collectOriginalSqlPreAggregations(() => this.query.cubeSql(cube)); return R.unnest(preAggregations.map(p => this.preAggregationDescriptionsFor(p))); }), - R.filter(R.identity), + R.filter((x): boolean => Boolean(x)), R.unnest )(this.preAggregationCubes()); } return []; } - preAggregationCubes() { + private preAggregationCubes(): string[] { const { join } = this.query; return join.joins.map(j => j.originalTo).concat([join.root]); } - preAggregationDescriptionsFor(foundPreAggregation) { - let preAggregations = [foundPreAggregation]; + private preAggregationDescriptionsFor(foundPreAggregation: PreAggregationForQuery): FullPreAggregationDescription[] { + let preAggregations: PreAggregationForQuery[] = [foundPreAggregation]; if (foundPreAggregation.preAggregation.type === 'rollupJoin') { - preAggregations = foundPreAggregation.preAggregationsToJoin; + preAggregations = foundPreAggregation.preAggregationsToJoin || []; } if (foundPreAggregation.preAggregation.type === 'rollupLambda') { - preAggregations = foundPreAggregation.referencedPreAggregations; + preAggregations = foundPreAggregation.referencedPreAggregations || []; } return preAggregations.map(preAggregation => { @@ -87,42 +155,43 @@ export class PreAggregations { }).reduce((a, b) => a.concat(b), []); } - canPartitionsBeUsed(foundPreAggregation) { - return foundPreAggregation.preAggregation.partitionGranularity && - foundPreAggregation.references.timeDimensions && - foundPreAggregation.references.timeDimensions.length; + private canPartitionsBeUsed(foundPreAggregation: PreAggregationForQuery): boolean { + return !!foundPreAggregation.preAggregation.partitionGranularity && + !!foundPreAggregation.references.timeDimensions?.length; } - addPartitionRangeTo(foundPreAggregation, dimension, range, boundaryDateRange) { - return Object.assign({}, foundPreAggregation, { - preAggregation: Object.assign({}, foundPreAggregation.preAggregation, { + private addPartitionRangeTo(foundPreAggregation: PreAggregationForQuery, dimension, range, boundaryDateRange) { + return { + ...foundPreAggregation, + preAggregation: { + ...foundPreAggregation.preAggregation, partitionTimeDimensions: [{ dimension, dateRange: range, boundaryDateRange }], - }) - }); + } + }; } - partitionDimension(foundPreAggregation) { + private partitionDimension(foundPreAggregation: PreAggregationForQuery): { dimension: string, partitionDimension: BaseTimeDimension } { const { dimension } = foundPreAggregation.references.timeDimensions[0]; const partitionDimension = this.query.newTimeDimension({ dimension, granularity: this.castGranularity(foundPreAggregation.preAggregation.partitionGranularity), - dateRange: this.query.timeDimensions[0] && this.query.timeDimensions[0].dateRange, - boundaryDateRange: this.query.timeDimensions[0] && this.query.timeDimensions[0].boundaryDateRange + dateRange: this.query.timeDimensions[0]?.dateRange, + boundaryDateRange: this.query.timeDimensions[0]?.boundaryDateRange }); return { dimension, partitionDimension }; } - preAggregationDescriptionsForRecursive(cube, foundPreAggregation) { + private preAggregationDescriptionsForRecursive(cube: string, foundPreAggregation: PreAggregationForQuery): FullPreAggregationDescription[] { const query = this.query.preAggregationQueryForSqlEvaluation(cube, foundPreAggregation.preAggregation); const descriptions = query !== this.query ? query.preAggregations.preAggregationsDescription() : []; return descriptions.concat(this.preAggregationDescriptionFor(cube, foundPreAggregation)); } - get hasCumulativeMeasures() { + private hasCumulativeMeasures(): boolean { if (!this.hasCumulativeMeasuresValue) { this.hasCumulativeMeasuresValue = PreAggregations.hasCumulativeMeasures(this.query); } @@ -131,7 +200,7 @@ export class PreAggregations { // Return array of `aggregations` columns descriptions in form `()` // Aggregations used in CubeStore create table for describe measures in CubeStore side - aggregationsColumns(cube, preAggregation) { + public aggregationsColumns(cube: string, preAggregation: PreAggregationDefinition): string[] { if (preAggregation.type === 'rollup') { return this.query .preAggregationQueryForSqlEvaluation(cube, preAggregation) @@ -151,18 +220,19 @@ export class PreAggregations { return []; } - preAggregationDescriptionFor(cube, foundPreAggregation) { + private preAggregationDescriptionFor(cube: string, foundPreAggregation: PreAggregationForQuery): FullPreAggregationDescription { const { preAggregationName, preAggregation, references } = foundPreAggregation; const tableName = this.preAggregationTableName(cube, preAggregationName, preAggregation); const invalidateKeyQueries = this.query.preAggregationInvalidateKeyQueries(cube, preAggregation, preAggregationName); const queryForSqlEvaluation = this.query.preAggregationQueryForSqlEvaluation(cube, preAggregation); - const partitionInvalidateKeyQueries = queryForSqlEvaluation.partitionInvalidateKeyQueries?.(cube, preAggregation); const allBackAliasMembers = this.query.allBackAliasMembers(); - const matchedTimeDimension = preAggregation.partitionGranularity && !this.hasCumulativeMeasures && - this.query.timeDimensions.find(td => { + let matchedTimeDimension: BaseTimeDimension | undefined; + + if (preAggregation.partitionGranularity && !this.hasCumulativeMeasures()) { + matchedTimeDimension = this.query.timeDimensions.find(td => { if (!td.dateRange) { return false; } @@ -173,7 +243,6 @@ export class PreAggregations { const timeDimensionReference = timeDimensionsReference[0]; // timeDimensionsReference[*].dimension can contain full join path, so we should trim it - // TODO check full join path match here const timeDimensionReferenceDimension = CubeSymbols.joinHintFromPath(timeDimensionReference.dimension).path; if (td.dimension === timeDimensionReferenceDimension) { @@ -183,20 +252,25 @@ export class PreAggregations { // Handling for views return td.dimension === allBackAliasMembers[timeDimensionReferenceDimension]; }); + } - const filters = preAggregation.partitionGranularity && this.query.filters.filter(td => { - // TODO support all date operators - if (td.isDateOperator() && td.camelizeOperator === 'inDateRange') { - if (td.dimension === foundPreAggregation.references.timeDimensions[0].dimension) { - return true; - } + let filters: BaseFilter[] | undefined; - // Handling for views - return td.dimension === allBackAliasMembers[foundPreAggregation.references.timeDimensions[0].dimension]; - } + if (preAggregation.partitionGranularity) { + filters = this.query.filters?.filter((td): td is BaseFilter => { + // TODO support all date operators + if (td.isDateOperator() && 'camelizeOperator' in td && td.camelizeOperator === 'inDateRange') { + if (td.dimension === foundPreAggregation.references.timeDimensions[0].dimension) { + return true; + } - return false; - }); + // Handling for views + return td.dimension === allBackAliasMembers[foundPreAggregation.references.timeDimensions[0].dimension]; + } + + return false; + }); + } const uniqueKeyColumnsDefault = () => null; const uniqueKeyColumns = ({ @@ -208,12 +282,11 @@ export class PreAggregations { return { preAggregationId: `${cube}.${preAggregationName}`, - timezone: this.query.options && this.query.options.timezone, + timezone: this.query.options?.timezone, timestampFormat: queryForSqlEvaluation.timestampFormat(), timestampPrecision: queryForSqlEvaluation.timestampPrecision(), tableName, invalidateKeyQueries, - partitionInvalidateKeyQueries, type: preAggregation.type, external: preAggregation.external, previewSql: queryForSqlEvaluation.preAggregationPreviewSql(tableName), @@ -224,18 +297,20 @@ export class PreAggregations { uniqueKeyColumns, aggregationsColumns, dataSource: queryForSqlEvaluation.dataSource, - // in fact we can reference preAggregation.granularity however accessing timeDimensions is more strict and consistent + // in fact, we can reference preAggregation.granularity however accessing timeDimensions is more strict and consistent granularity: references.timeDimensions[0]?.granularity, partitionGranularity: preAggregation.partitionGranularity, - updateWindowSeconds: preAggregation.refreshKey && preAggregation.refreshKey.updateWindow && + updateWindowSeconds: preAggregation.refreshKey && + 'updateWindow' in preAggregation.refreshKey && + preAggregation.refreshKey?.updateWindow && queryForSqlEvaluation.parseSecondDuration(preAggregation.refreshKey.updateWindow), preAggregationStartEndQueries: (preAggregation.partitionGranularity || references.timeDimensions[0]?.granularity) && this.refreshRangeQuery(cube).preAggregationStartEndQueries(cube, preAggregation), matchedTimeDimensionDateRange: preAggregation.partitionGranularity && ( - matchedTimeDimension && matchedTimeDimension.boundaryDateRangeFormatted() || - filters && filters[0] && filters[0].formattedDateRange() // TODO intersect all date ranges + matchedTimeDimension?.boundaryDateRangeFormatted() || + filters?.[0]?.formattedDateRange() // TODO intersect all date ranges ), indexesSql: Object.keys(preAggregation.indexes || {}) .map( @@ -247,7 +322,7 @@ export class PreAggregations { sql: queryForSqlEvaluation.indexSql( cube, preAggregation, - preAggregation.indexes[index], + preAggregation.indexes?.[index], indexName, tableName ) @@ -261,8 +336,8 @@ export class PreAggregations { const indexName = this.preAggregationTableName(cube, `${foundPreAggregation.sqlAlias || preAggregationName}_${index}`, preAggregation, true); return { indexName, - type: preAggregation.indexes[index].type, - columns: queryForSqlEvaluation.evaluateIndexColumns(cube, preAggregation.indexes[index]) + type: preAggregation.indexes?.[index].type, + columns: queryForSqlEvaluation.evaluateIndexColumns(cube, preAggregation.indexes?.[index]) }; } ), @@ -274,7 +349,7 @@ export class PreAggregations { }; } - preAggregationTableName(cube, preAggregationName, preAggregation, skipSchema) { + private preAggregationTableName(cube: string, preAggregationName: string, preAggregation: PreAggregationDefinitionExtended | PreAggregationForQuery, skipSchema: boolean = false): string { const name = preAggregation.sqlAlias || preAggregationName; return this.query.preAggregationTableName( cube, @@ -283,7 +358,7 @@ export class PreAggregations { ); } - findPreAggregationToUseForCube(cube) { + public findPreAggregationToUseForCube(cube: string): PreAggregationForCube | null { const preAggregates = this.query.cubeEvaluator.preAggregationsForCube(cube); const originalSqlPreAggregations = R.pipe( R.toPairs, @@ -301,12 +376,12 @@ export class PreAggregations { return null; } - static transformQueryToCanUseForm(query) { + public static transformQueryToCanUseForm(query: BaseQuery): TransformedQuery { const flattenDimensionMembers = this.flattenDimensionMembers(query); - const sortedDimensions = this.squashDimensions(query, flattenDimensionMembers); + const sortedDimensions = this.squashDimensions(flattenDimensionMembers); const allBackAliasMembers = query.allBackAliasMembers(); - const measures = query.measures.concat(query.measureFilters); - const measurePaths = R.uniq(this.flattenMembers(measures).map(m => m.expressionPath())); + const measures: (BaseMeasure | BaseFilter | BaseGroupFilter)[] = [...query.measures, ...query.measureFilters]; + const measurePaths = R.uniq(this.flattenMembers(measures).filter((m): m is BaseMeasure => m instanceof BaseMeasure).map(m => m.expressionPath())); const collectLeafMeasures = query.collectLeafMeasures.bind(query); const dimensionsList = query.dimensions.map(dim => dim.expressionPath()); const segmentsList = query.segments.map(s => s.expressionPath()); @@ -341,7 +416,7 @@ export class PreAggregations { const leafMeasurePaths = R.pipe( - R.map(m => { + R.map((m: { measure: string }) => { const leafMeasures = query.collectFrom([m], collectLeafMeasures, 'collectLeafMeasures'); measureToLeafMeasures[m.measure] = leafMeasures.map((measure) => { const baseMeasure = query.newMeasure(measure); @@ -376,7 +451,7 @@ export class PreAggregations { const hasNoTimeDimensionsWithoutGranularity = !query.timeDimensions.filter(d => !d.granularity).length; const allFiltersWithinSelectedDimensions = - R.all(d => dimensionsList.indexOf(d) !== -1)( + R.all((d: string) => dimensionsList.indexOf(d) !== -1)( R.flatten( query.filters.map(f => f.getMembers()) ).map(f => query.cubeEvaluator.pathFromArray(f.path())) @@ -430,14 +505,7 @@ export class PreAggregations { }; } - /** - * - * @param query - * @param members - * @param {Map>} cubeToJoinPrefix - * @returns {Array} - */ - static ownedMembers(query, members) { + public static ownedMembers(query: BaseQuery, members): string[] { return R.pipe(R.uniq, R.sortBy(R.identity))( query .collectFrom(members, query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor') @@ -445,21 +513,21 @@ export class PreAggregations { ); } - static sortTimeDimensionsWithRollupGranularity(timeDimensions) { + public static sortTimeDimensionsWithRollupGranularity(timeDimensions: BaseTimeDimension[] | undefined): [expressionPath: string, rollupGranularity: string | null][] { return timeDimensions && R.sortBy( - R.prop(0), - timeDimensions.map(d => [d.expressionPath(), d.rollupGranularity()]) + ([exprPath]) => exprPath, + timeDimensions.map(d => [d.expressionPath(), d.rollupGranularity()] as [string, string | null]) ) || []; } - static timeDimensionsAsIs(timeDimensions) { + public static timeDimensionsAsIs(timeDimensions: BaseTimeDimension[] | undefined): [expressionPath: string, resolvedGranularity: string | null][] { return timeDimensions && R.sortBy( - R.prop(0), - timeDimensions.map(d => [d.expressionPath(), d.resolvedGranularity()]), + ([exprPath]) => exprPath, + timeDimensions.map(d => [d.expressionPath(), d.resolvedGranularity()] as [string, string | null]), ) || []; } - static collectFilterDimensionsWithSingleValueEqual(filters, map) { + public static collectFilterDimensionsWithSingleValueEqual(filters, map) { // eslint-disable-next-line no-restricted-syntax for (const f of filters) { if (f.operator === 'equals') { @@ -475,7 +543,8 @@ export class PreAggregations { return map; } - static transformedQueryToReferences(query) { + // FIXME: It seems to be not used at all + public static transformedQueryToReferences(query) { return { measures: query.measures, dimensions: query.sortedDimensions, @@ -483,7 +552,7 @@ export class PreAggregations { }; } - canUsePreAggregationFn(query, refs) { + private canUsePreAggregationFn(query: BaseQuery, refs: PreAggregationReferences | null = null) { return PreAggregations.canUsePreAggregationForTransformedQueryFn( PreAggregations.transformQueryToCanUseForm(query), refs, @@ -493,21 +562,14 @@ export class PreAggregations { /** * Returns function to determine whether pre-aggregation can be used or not * for specified query, or its value for `refs` if specified. - * @param {Object} transformedQuery transformed query - * @param {PreAggregationReferences?} refs pre-aggs reference - * @returns {function(preagg: Object): boolean} */ - static canUsePreAggregationForTransformedQueryFn(transformedQuery, refs) { + public static canUsePreAggregationForTransformedQueryFn(transformedQuery: TransformedQuery, refs: PreAggregationReferences | null = null): CanUsePreAggregationFn { // TODO this needs to check not only members list, but their join paths as well: // query can have same members as pre-agg, but different calculated join path - // `refs` will come from preagg references, and would contain full join paths + // `refs` will come from pre-agg references, and would contain full join paths // TODO remove this in favor of matching with join path - /** - * @param {PreAggregationReferences} references - * @returns {PreAggregationReferences} - */ - function trimmedReferences(references) { + function trimmedReferences(references: PreAggregationReferences): PreAggregationReferences { const timeDimensionsTrimmed = references .timeDimensions .map(td => ({ @@ -532,10 +594,8 @@ export class PreAggregations { /** * Returns an array of 2-elements arrays with the dimension and granularity * sorted by the concatenated dimension + granularity key. - * @param {Array<{dimension: string, granularity: string}>} timeDimensions - * @returns {Array>} */ - const sortTimeDimensions = (timeDimensions) => ( + const sortTimeDimensions = (timeDimensions: ({ dimension: string; granularity?: string }[] | undefined)): [string, string][] => ( timeDimensions && R.sortBy( d => d.join('.'), @@ -548,10 +608,7 @@ export class PreAggregations { ) || [] ); - /** - * @type {Set} - */ - const filterDimensionsSingleValueEqual = + const filterDimensionsSingleValueEqual: Set = transformedQuery.filterDimensionsSingleValueEqual && (transformedQuery.filterDimensionsSingleValueEqual instanceof Set ? transformedQuery.filterDimensionsSingleValueEqual : new Set( @@ -559,8 +616,8 @@ export class PreAggregations { transformedQuery.filterDimensionsSingleValueEqual || {}, ) )); - - const backAlias = (references) => references.map(r => ( + + const backAlias = (references: ([string, string | undefined] | string)[]) => references.map(r => ( Array.isArray(r) ? [transformedQuery.allBackAliasMembers[r[0]] || r[0], r[1]] : transformedQuery.allBackAliasMembers[r] || r @@ -568,10 +625,8 @@ export class PreAggregations { /** * Determine whether pre-aggregation can be used or not. - * @param {PreAggregationReferences} references - * @returns {boolean} */ - const canUsePreAggregationNotAdditive = (references) => { + const canUsePreAggregationNotAdditive: CanUsePreAggregationFn = (references: PreAggregationReferences): boolean => { // TODO remove this in favor of matching with join path const referencesTrimmed = trimmedReferences(references); @@ -651,13 +706,7 @@ export class PreAggregations { .map((newGranularity) => [dimension, newGranularity]); }; - /** - * Determine whether pre-aggregation can be used or not. - * TODO: revisit cumulative leaf measure matches. - * @param {PreAggregationReferences} references - * @returns {boolean} - */ - const canUsePreAggregationLeafMeasureAdditive = (references) => { + const canUsePreAggregationLeafMeasureAdditive: CanUsePreAggregationFn = (references): boolean => { /** * Array of 2-element arrays with dimension and granularity. * @type {Array>} @@ -684,9 +733,9 @@ export class PreAggregations { const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass( timeDimensionsList.map( - tds => R.anyPass(tds.map(td => { + tds => R.anyPass(tds.map((td: [string, string]) => { if (td[1] === '*') { - return R.any(tdtc => tdtc[0] === td[0]); // need to match the dimension at least + return R.any((tdtc: [string, string]) => tdtc[0] === td[0]); // need to match the dimension at least } else { return R.contains(td); } @@ -699,9 +748,11 @@ export class PreAggregations { ); if (transformedQuery.ungrouped) { - const allReferenceCubes = R.pipe(R.map(m => (m.dimension || m).split('.')[0]), R.uniq, R.sortBy(R.identity))( - referencesTrimmed.measures.concat(referencesTrimmed.dimensions).concat(referencesTrimmed.timeDimensions) - ); + const allReferenceCubes = R.pipe(R.map((name: string) => name?.split('.')[0]), R.uniq, R.sortBy(R.identity))([ + ...referencesTrimmed.measures, + ...referencesTrimmed.dimensions, + ...referencesTrimmed.timeDimensions.map(td => td.dimension), + ]); if ( !R.equals(transformedQuery.sortedAllCubeNames, allReferenceCubes) || !( @@ -717,7 +768,7 @@ export class PreAggregations { windowGranularityMatches(references) ) && ( R.all( - m => referencesTrimmed.measures.indexOf(m) !== -1, + (m: string) => referencesTrimmed.measures.indexOf(m) !== -1, transformedQuery.leafMeasures, ) || R.all( m => backAliasMeasures.indexOf(m) !== -1, @@ -729,47 +780,44 @@ export class PreAggregations { )); }; - /** - * Determine whether pre-aggregation can be used or not. - * @returns {boolean} - */ - const canUseFn = + const canUseFn: CanUsePreAggregationFn = ( transformedQuery.leafMeasureAdditive && !transformedQuery.hasMultipliedMeasures && !transformedQuery.hasMultiStage || transformedQuery.ungrouped - ) ? (r) => canUsePreAggregationLeafMeasureAdditive(r) || - canUsePreAggregationNotAdditive(r) + ) ? ((r: PreAggregationReferences): boolean => canUsePreAggregationLeafMeasureAdditive(r) || + canUsePreAggregationNotAdditive(r)) : canUsePreAggregationNotAdditive; if (refs) { + // @ts-ignore TS think it is boolean here return canUseFn(refs); } else { return canUseFn; } } - static squashDimensions(query, flattenDimensionMembers) { + private static squashDimensions(flattenDimensionMembers: BaseMember[]): string[] { return R.pipe(R.uniq, R.sortBy(R.identity))( - flattenDimensionMembers.map(d => d.expressionPath()) + flattenDimensionMembers + .filter((member: BaseMember): member is BaseMeasure => typeof (member as any).expressionPath === 'function') + .map(d => d.expressionPath()) ); } - static flattenMembers(members) { + private static flattenMembers(members: BaseMember[]): BaseMember[] { return R.flatten( members.map(m => m.getMembers()), ); } - static flattenDimensionMembers(query) { - return this.flattenMembers( - query.dimensions - .concat(query.filters) - .concat(query.segments) - ); + private static flattenDimensionMembers(query: BaseQuery): BaseMember[] { + return this.flattenMembers([ + ...query.dimensions, + ...query.filters, + ...query.segments, + ]); } - // eslint-disable-next-line no-unused-vars - // eslint-disable-next-line @typescript-eslint/no-unused-vars - getCubeLattice(cube, preAggregationName, preAggregation) { + public getCubeLattice(_cube, _preAggregationName, _preAggregation): unknown { throw new UserError('Auto rollups supported only in Enterprise version'); } @@ -778,9 +826,8 @@ export class PreAggregations { * from the list of potentially applicable pre-aggs). The order of the * potentially applicable pre-aggs is the same as the order in which these * pre-aggs appear in the schema file. - * @returns {Object} */ - findPreAggregationForQuery() { + public findPreAggregationForQuery(): PreAggregationForQuery | undefined { if (!this.preAggregationForQuery) { if (this.query.useNativeSqlPlanner && this.query.canUseNativeSqlPlannerPreAggregation) { this.preAggregationForQuery = this.query.findPreAggregationForQueryRust(); @@ -795,11 +842,17 @@ export class PreAggregations { return this.preAggregationForQuery; } - findAutoRollupPreAggregationsForCube(cube, preAggregations) { + private findAutoRollupPreAggregationsForCube(cube: string, preAggregations: PreAggregationDefinitions): PreAggregationForQuery[] { if ( - R.any(m => m.path() && m.path()[0] === cube, this.query.measures) || + this.query.measures.some((m) => { + const path = m.path(); + return path !== null && path[0] === cube; + }) || !this.query.measures.length && !this.query.timeDimensions.length && - R.all(d => d.path() && d.path()[0] === cube, this.query.dimensions) + this.query.dimensions.every((d) => { + const path = d.path(); + return path !== null && path[0] === cube; + }) ) { return R.pipe( R.toPairs, @@ -807,10 +860,10 @@ export class PreAggregations { // eslint-disable-next-line @typescript-eslint/no-unused-vars R.filter(([k, a]) => a.type === 'autoRollup'), R.map(([preAggregationName, preAggregation]) => { - const cubeLattice = this.getCubeLattice(cube, preAggregationName, preAggregation); + const cubeLattice: any = this.getCubeLattice(cube, preAggregationName, preAggregation); const optimalPreAggregation = cubeLattice.findOptimalPreAggregationFromLattice(this.query); return optimalPreAggregation && { - preAggregationName: preAggregationName + this.autoRollupNameSuffix(cube, optimalPreAggregation), + preAggregationName: preAggregationName + this.autoRollupNameSuffix(optimalPreAggregation), preAggregation: Object.assign( optimalPreAggregation, preAggregation @@ -826,17 +879,16 @@ export class PreAggregations { } /** - * Returns an array of potentially applicable for the query preaggs in the + * Returns an array of potentially applicable for the query pre-aggs in the * same order they appear in the schema file. - * @returns {Array} */ - rollupMatchResults() { + private rollupMatchResults(): PreAggregationForQuery[] { const { query } = this; const canUsePreAggregation = this.canUsePreAggregationFn(query); return R.pipe( - R.map(cube => { + R.map((cube: string) => { const preAggregations = this.query.cubeEvaluator.preAggregationsForCube(cube); @@ -860,7 +912,7 @@ export class PreAggregations { )(query.collectCubeNames()); } - findRollupPreAggregationsForCube(cube, canUsePreAggregation, preAggregations) { + private findRollupPreAggregationsForCube(cube: string, canUsePreAggregation: CanUsePreAggregationFn, preAggregations: PreAggregationDefinitions): PreAggregationForQuery[] { return R.pipe( R.toPairs, // eslint-disable-next-line no-unused-vars @@ -870,7 +922,7 @@ export class PreAggregations { )(preAggregations); } - getRollupPreAggregationByName(cube, preAggregationName) { + public getRollupPreAggregationByName(cube, preAggregationName) { const canUsePreAggregation = () => true; const preAggregation = R.pipe( R.toPairs, @@ -890,7 +942,7 @@ export class PreAggregations { } // TODO check multiplication factor didn't change - buildRollupJoin(preAggObj, preAggObjsToJoin) { + private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): RollupJoin { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { @@ -924,7 +976,7 @@ export class PreAggregations { ); } - preAggObjForJoin(preAggObjsToJoin, joinMembers, join) { + private preAggObjForJoin(preAggObjsToJoin: PreAggregationForQuery[], joinMembers, join): PreAggregationForQuery { const fromPreAggObj = preAggObjsToJoin .filter(p => joinMembers.every(m => !!p.references.dimensions.find(d => m === d))); if (!fromPreAggObj.length) { @@ -938,7 +990,7 @@ export class PreAggregations { return fromPreAggObj[0]; } - resolveJoinMembers(join) { + private resolveJoinMembers(join) { return join.joins.map(j => { const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); @@ -961,7 +1013,7 @@ export class PreAggregations { }); } - cubesFromPreAggregation(preAggObj) { + private cubesFromPreAggregation(preAggObj: PreAggregationForQuery): string[] { return R.uniq( preAggObj.references.measures.map(m => this.query.cubeEvaluator.parsePath('measures', m)).concat( preAggObj.references.dimensions.map(m => this.query.cubeEvaluator.parsePathAnyType(m)) @@ -969,9 +1021,14 @@ export class PreAggregations { ); } - evaluatedPreAggregationObj(cube, preAggregationName, preAggregation, canUsePreAggregation) { + private evaluatedPreAggregationObj( + cube: string, + preAggregationName: string, + preAggregation: PreAggregationDefinitionExtended, + canUsePreAggregation: CanUsePreAggregationFn + ): PreAggregationForQuery { const references = this.evaluateAllReferences(cube, preAggregation, preAggregationName); - const preAggObj = { + const preAggObj: PreAggregationForQuery = { preAggregationName, preAggregation, cube, @@ -1050,16 +1107,16 @@ export class PreAggregations { } } - static checkPartitionGranularityDefined(cube, preAggregationName, preAggregation) { + public static checkPartitionGranularityDefined(cube: string, preAggregationName: string, preAggregation: PreAggregationForQuery): string { if (!preAggregation.preAggregation.partitionGranularity) { throw new UserError(`'${preAggregation.cube}.${preAggregation.preAggregationName}' referenced by '${cube}.${preAggregationName}' rollupLambda doesn't have partition granularity. Partition granularity is required if multiple rollups are provided.`); } return preAggregation.preAggregation.partitionGranularity; } - static memberNameMismatchValidation(preAggA, preAggB, memberType) { - const preAggAMemberNames = PreAggregations.memberShortNames(preAggA.references[memberType], memberType === 'timeDimensions'); - const preAggBMemberNames = PreAggregations.memberShortNames(preAggB.references[memberType], memberType === 'timeDimensions'); + public static memberNameMismatchValidation(preAggA: PreAggregationForQuery, preAggB: PreAggregationForQuery, memberType: 'measures' | 'dimensions' | 'timeDimensions') { + const preAggAMemberNames = PreAggregations.memberShortNames(preAggA.references[memberType]); + const preAggBMemberNames = PreAggregations.memberShortNames(preAggB.references[memberType]); if (!R.equals( preAggAMemberNames, preAggBMemberNames @@ -1068,11 +1125,17 @@ export class PreAggregations { } } - static memberShortNames(memberArray, isTimeDimension) { - return memberArray.map(member => (isTimeDimension ? `${member.dimension.split('.')[1]}.${member.granularity}` : member.split('.')[1])); + private static memberShortNames(memberArray: (string | PreAggregationTimeDimensionReference)[]): string[] { + return memberArray.map(member => { + if (typeof member !== 'string') { + return `${member.dimension.split('.')[1]}.${member.granularity}`; + } else { + return member.split('.')[1]; + } + }); } - rollupMatchResultDescriptions() { + public rollupMatchResultDescriptions() { return this.rollupMatchResults().map(p => ({ name: this.query.cubeEvaluator.pathFromArray([p.cube, p.preAggregationName]), tableName: this.preAggregationTableName(p.cube, p.preAggregationName, p.preAggregation), @@ -1081,12 +1144,12 @@ export class PreAggregations { })); } - canUseTransformedQuery() { + public canUseTransformedQuery(): TransformedQuery { return PreAggregations.transformQueryToCanUseForm(this.query); } - static hasCumulativeMeasures(query) { - const measures = (query.measures.concat(query.measureFilters)); + public static hasCumulativeMeasures(query: BaseQuery): boolean { + const measures = [...query.measures, ...query.measureFilters]; const collectLeafMeasures = query.collectLeafMeasures.bind(query); return R.pipe( R.map(m => query.collectFrom([m], collectLeafMeasures, 'collectLeafMeasures')), @@ -1097,17 +1160,17 @@ export class PreAggregations { )(measures); } - castGranularity(granularity) { + public castGranularity(granularity) { return granularity; } - collectOriginalSqlPreAggregations(fn) { + public collectOriginalSqlPreAggregations(fn) { const preAggregations = []; const result = this.query.evaluateSymbolSqlWithContext(fn, { collectOriginalSqlPreAggregations: preAggregations }); return { preAggregations, result }; } - refreshRangeQuery(cube) { + private refreshRangeQuery(cube): BaseQuery { return this.query.newSubQueryForCube( cube, { @@ -1118,7 +1181,7 @@ export class PreAggregations { ); } - originalSqlPreAggregationQuery(cube, aggregation) { + public originalSqlPreAggregationQuery(cube, aggregation): BaseQuery { return this.query.newSubQueryForCube( cube, { @@ -1130,7 +1193,7 @@ export class PreAggregations { ); } - rollupPreAggregationQuery(cube, aggregation) { + public rollupPreAggregationQuery(cube: string, aggregation): BaseQuery { // `this.evaluateAllReferences` will retain not only members, but their join path as well, and pass join hints // to subquery. Otherwise, members in subquery would regenerate new join tree from clean state, // and it can be different from expected by join path in pre-aggregation declaration @@ -1162,7 +1225,7 @@ export class PreAggregations { }); } - autoRollupPreAggregationQuery(cube, aggregation) { + public autoRollupPreAggregationQuery(cube: string, aggregation): BaseQuery { return this.query.newSubQueryForCube( cube, { @@ -1178,7 +1241,7 @@ export class PreAggregations { ); } - mergePartitionTimeDimensions(aggregation, partitionTimeDimensions) { + private mergePartitionTimeDimensions(aggregation: PreAggregationReferences, partitionTimeDimensions: BaseTimeDimension[]) { if (!partitionTimeDimensions) { return aggregation.timeDimensions; } @@ -1191,7 +1254,7 @@ export class PreAggregations { }); } - autoRollupNameSuffix(cube, aggregation) { + private autoRollupNameSuffix(aggregation): string { // eslint-disable-next-line prefer-template return '_' + aggregation.dimensions.concat( aggregation.timeDimensions.map(d => `${d.dimension}${d.granularity.substring(0, 1)}`) @@ -1203,14 +1266,7 @@ export class PreAggregations { .toLowerCase(); } - /** - * - * @param {string} cube - * @param aggregation - * @param {string} [preAggregationName] - * @returns {PreAggregationReferences} - */ - evaluateAllReferences(cube, aggregation, preAggregationName) { + private evaluateAllReferences(cube: string, aggregation: PreAggregationDefinition, preAggregationName: string | null = null): PreAggregationReferences { // TODO build a join tree for all references, so they would always include full join path // Even for preaggregation references without join path // It is necessary to be able to match query and preaggregation based on full join tree @@ -1240,12 +1296,12 @@ export class PreAggregations { return this.query.cacheValue(['evaluateAllReferences', cube, preAggregationName], evaluateReferences); } - originalSqlPreAggregationTable(preAggregationDescription) { + public originalSqlPreAggregationTable(preAggregationDescription: PreAggregationForCube): string { // eslint-disable-next-line prefer-const let { preAggregationName, preAggregation } = preAggregationDescription; // @todo Dont use sqlAlias directly, we needed to move it in preAggregationTableName - if (preAggregation && preAggregation.sqlAlias) { + if (preAggregation?.sqlAlias) { preAggregationName = preAggregation.sqlAlias; } @@ -1255,7 +1311,7 @@ export class PreAggregations { ); } - rollupLambdaUnion(preAggregationForQuery, rollupGranularity) { + private rollupLambdaUnion(preAggregationForQuery: PreAggregationForQuery, rollupGranularity: string): string { if (!preAggregationForQuery.referencedPreAggregations) { return this.preAggregationTableName( preAggregationForQuery.cube, @@ -1311,7 +1367,7 @@ export class PreAggregations { return `(${union})`; } - rollupPreAggregation(preAggregationForQuery, measures, isFullSimpleQuery, filters) { + public rollupPreAggregation(preAggregationForQuery: PreAggregationForQuery, measures: BaseMeasure[], isFullSimpleQuery: boolean, filters): string { let toJoin; // TODO granularity shouldn't be null? const rollupGranularity = preAggregationForQuery.references.timeDimensions[0]?.granularity || 'day'; @@ -1352,16 +1408,15 @@ export class PreAggregations { const from = this.query.joinSql(toJoin); const replacedFilters = - filters || this.query.segments - .concat(this.query.filters).concat( - this.query.timeDimensions.map(dimension => dimension.dateRange && ({ - filterToWhere: () => this.query.timeRangeFilter( - this.query.dimensionSql(dimension), - dimension.localDateTimeFromParam(), - dimension.localDateTimeToParam(), - ), - })) - ).filter(f => !!f); + filters || [...this.query.segments, ...this.query.filters, ...( + this.query.timeDimensions.map(dimension => dimension.dateRange && ({ + filterToWhere: () => this.query.timeRangeFilter( + this.query.dimensionSql(dimension), + dimension.localDateTimeFromParam(), + dimension.localDateTimeToParam(), + ), + })) + )].filter(f => !!f); const renderedReference = { ...(this.measuresRenderedReference(preAggregationForQuery)), @@ -1387,9 +1442,13 @@ export class PreAggregations { ); } - measuresRenderedReference(preAggregationForQuery) { - return R.pipe( - R.map(path => { + private measuresRenderedReference(preAggregationForQuery: PreAggregationForQuery): Record { + return R.pipe< + string[], + Array<[string, string]>, + Record + >( + R.map((path: string): [string, string] => { const measure = this.query.newMeasure(path); return [ path, @@ -1405,9 +1464,13 @@ export class PreAggregations { )(this.rollupMeasures(preAggregationForQuery)); } - measureAliasesRenderedReference(preAggregationForQuery) { - return R.pipe( - R.map(path => { + private measureAliasesRenderedReference(preAggregationForQuery: PreAggregationForQuery): Record { + return R.pipe< + string[], + Array<[string, string]>, + Record + >( + R.map((path: string): [string, string] => { const measure = this.query.newMeasure(path); return [ path, @@ -1418,9 +1481,13 @@ export class PreAggregations { )(this.rollupMeasures(preAggregationForQuery)); } - dimensionsRenderedReference(preAggregationForQuery) { - return R.pipe( - R.map(path => { + private dimensionsRenderedReference(preAggregationForQuery: PreAggregationForQuery): Record { + return R.pipe< + string[], + Array<[string, string]>, + Record + >( + R.map((path: string): [string, string] => { const dimension = this.query.newDimension(path); return [ path, @@ -1431,9 +1498,13 @@ export class PreAggregations { )(this.rollupDimensions(preAggregationForQuery)); } - timeDimensionsRenderedReference(rollupGranularity, preAggregationForQuery) { - return R.pipe( - R.map((td) => { + private timeDimensionsRenderedReference(rollupGranularity: string, preAggregationForQuery: PreAggregationForQuery): Record { + return R.pipe< + PreAggregationTimeDimensionReference[], + Array<[string, string]>, + Record + >( + R.map((td: PreAggregationTimeDimensionReference): [string, string] => { const timeDimension = this.query.newTimeDimension(td); return [ td.dimension, @@ -1444,25 +1515,25 @@ export class PreAggregations { )(this.rollupTimeDimensions(preAggregationForQuery)); } - rollupMembers(preAggregationForQuery, type) { + private rollupMembers(preAggregationForQuery: PreAggregationForQuery, type: 'measures' | 'dimensions' | 'timeDimensions'): string[] | PreAggregationTimeDimensionReference[] { return preAggregationForQuery.preAggregation.type === 'autoRollup' ? preAggregationForQuery.preAggregation[type] : this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation, preAggregationForQuery.preAggregationName)[type]; } - rollupMeasures(preAggregationForQuery) { - return this.rollupMembers(preAggregationForQuery, 'measures'); + public rollupMeasures(preAggregationForQuery: PreAggregationForQuery): string[] { + return this.rollupMembers(preAggregationForQuery, 'measures') as string[]; } - rollupDimensions(preAggregationForQuery) { - return this.rollupMembers(preAggregationForQuery, 'dimensions'); + public rollupDimensions(preAggregationForQuery: PreAggregationForQuery): string[] { + return this.rollupMembers(preAggregationForQuery, 'dimensions') as string[]; } - rollupTimeDimensions(preAggregationForQuery) { - return this.rollupMembers(preAggregationForQuery, 'timeDimensions'); + public rollupTimeDimensions(preAggregationForQuery: PreAggregationForQuery): PreAggregationTimeDimensionReference[] { + return this.rollupMembers(preAggregationForQuery, 'timeDimensions') as PreAggregationTimeDimensionReference[]; } - preAggregationId(preAggregation) { + public preAggregationId(preAggregation: PreAggregationForQuery): string { return `${preAggregation.cube}.${preAggregation.preAggregationName}`; } } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index aec3b2e76f917..86a597e74046d 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -66,7 +66,33 @@ export type PreAggregationFilters = { scheduled?: boolean, }; -type PreAggregationDefinition = { +export type EveryInterval = string; +type EveryCronInterval = string; +type EveryCronTimeZone = string; + +export type CubeRefreshKeySqlVariant = { + sql: () => string; + every?: EveryInterval; +}; + +export type CubeRefreshKeyEveryVariant = { + every: EveryInterval | EveryCronInterval; + timezone?: EveryCronTimeZone; + incremental?: boolean; + updateWindow?: EveryInterval; +}; + +export type CubeRefreshKeyImmutableVariant = { + immutable: true; +}; + +export type CubeRefreshKey = + | CubeRefreshKeySqlVariant + | CubeRefreshKeyEveryVariant + | CubeRefreshKeyImmutableVariant; + +export type PreAggregationDefinition = { + type: 'autoRollup' | 'originalSql' | 'rollupJoin' | 'rollupLambda' | 'rollup', allowNonStrictDateRangeMatch?: boolean, timeDimensionReference?: () => ToString, granularity: string, @@ -75,9 +101,15 @@ type PreAggregationDefinition = { segmentReferences: () => Array, measureReferences: () => Array, rollupReferences: () => Array, + indexes?: Record, + refreshKey?: CubeRefreshKey, + scheduledRefresh: boolean, + external: boolean, }; -type PreAggregationTimeDimensionReference = { +export type PreAggregationDefinitions = Record; + +export type PreAggregationTimeDimensionReference = { dimension: string, granularity: string, }; @@ -521,7 +553,7 @@ export class CubeEvaluator extends CubeSymbols { ); } - public preAggregationsForCube(path: string) { + public preAggregationsForCube(path: string): Record { return this.cubeFromPath(path).preAggregations || {}; } @@ -663,7 +695,7 @@ export class CubeEvaluator extends CubeSymbols { throw new UserError(`Can't resolve member '${Array.isArray(path) ? path.join('.') : path}'`); } - public byPath(type: 'measures' | 'dimensions' | 'segments', path: string | string[]) { + public byPath(type: 'measures' | 'dimensions' | 'segments' | 'preAggregations', path: string | string[]) { if (!type) { throw new Error(`Type can't be undefined for '${path}'`); } @@ -688,7 +720,7 @@ export class CubeEvaluator extends CubeSymbols { return this.evaluatedCubes[cubeAndName[0]][type][cubeAndName[1]]; } - public parsePath(type, path) { + public parsePath(type: 'measures' | 'dimensions' | 'segments' | 'preAggregations', path: string): string[] { // Should throw UserError in case of parse error this.byPath(type, path); return path.split('.'); @@ -707,7 +739,7 @@ export class CubeEvaluator extends CubeSymbols { return this.isRbacEnabledCache; } - protected parsePathAnyType(path) { + public parsePathAnyType(path: string): string[] { // Should throw UserError in case of parse error this.byPathAnyType(path); return path.split('.'); @@ -746,7 +778,7 @@ export class CubeEvaluator extends CubeSymbols { return { cubeReferencesUsed, pathReferencesUsed, evaluatedSql }; } - protected evaluatePreAggregationReferences(cube: string, aggregation: PreAggregationDefinition): PreAggregationReferences { + public evaluatePreAggregationReferences(cube: string, aggregation: PreAggregationDefinition): PreAggregationReferences { const timeDimensions: Array = []; if (aggregation.timeDimensionReference) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index f50e469f56b70..0625d05988c8b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -702,7 +702,7 @@ export class CubeSymbols { return options.originalSorting ? references : R.sortBy(R.identity, references) as any; } - public pathFromArray(array) { + public pathFromArray(array: string[]): string { return array.join('.'); } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 819d27317ac0d..0c1accf8c60d2 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -533,7 +533,7 @@ const CubeRefreshKeySchema = condition( (s) => defined(s.sql), Joi.object().keys({ sql: Joi.func().required(), - // We dont support timezone for this, because it's useless + // We don't support timezone for this, because it's useless // We cannot support cron interval every: everyInterval, }), diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index fc22bcedee4e8..a5ab7c78f2022 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -2496,9 +2496,8 @@ describe('PreAggregations', () => { console.log(queryAndParams); const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); console.log(JSON.stringify(preAggregationsDescription, null, 2)); - const { partitionInvalidateKeyQueries, loadSql } = preAggregationsDescription.find(p => p.preAggregationId === 'RealTimeLambdaVisitors.partitioned'); + const { loadSql } = preAggregationsDescription.find(p => p.preAggregationId === 'RealTimeLambdaVisitors.partitioned'); - expect(partitionInvalidateKeyQueries).toStrictEqual([]); expect(loadSql[0]).not.toMatch(/GROUP BY/); expect(loadSql[0]).toMatch(/THEN 1 END `real_time_lambda_visitors__count`/); });