fix: Views with proxy dimensions and non-additive measures don't not match pre-aggregations (#7374)

Fixes #7099

Author: paveltiunov

packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

@@ -266,13 +266,15 @@ export class PreAggregations {
   }
 
   static transformQueryToCanUseForm(query) {
-    const sortedDimensions = this.squashDimensions(query);
+    const flattenDimensionMembers = this.flattenDimensionMembers(query);
+    const sortedDimensions = this.squashDimensions(query, flattenDimensionMembers);
+    const aliasedToFirstNonAliasedDimension = this.aliasedToFirstNonAliasedDimension(query, flattenDimensionMembers);
     const measures = query.measures.concat(query.measureFilters);
-    const measurePaths = R.uniq(measures.map(m => m.measure));
+    const measurePaths = R.uniq(PreAggregations.firstNonAliasMember(query, measures));
     const collectLeafMeasures = query.collectLeafMeasures.bind(query);
-    const dimensionsList = query.dimensions.map(dim => dim.dimension);
-    const segmentsList = query.segments.map(s => s.segment);
-    const ownedDimensions = PreAggregations.ownedMembers(query, PreAggregations.concatDimensionMembers(query));
+    const dimensionsList = PreAggregations.firstNonAliasMember(query, query.dimensions);
+    const segmentsList = PreAggregations.firstNonAliasMember(query, query.segments);
+    const ownedDimensions = PreAggregations.ownedMembers(query, flattenDimensionMembers);
     const ownedTimeDimensions = query.timeDimensions.map(td => {
       const owned = PreAggregations.ownedMembers(query, [td]);
       let { dimension } = td;
@@ -288,6 +290,19 @@ export class PreAggregations {
       });
     }).map(d => query.newTimeDimension(d));
 
+    const firstNonAliasTimeDimensions = query.timeDimensions.map(td => {
+      const nonAlias = PreAggregations.firstNonAliasMember(query, [td]);
+      let { dimension } = td;
+      if (nonAlias.length === 1) {
+        [dimension] = nonAlias;
+      }
+      return query.newTimeDimension({
+        dimension,
+        dateRange: td.dateRange,
+        granularity: td.granularity,
+      });
+    }).map(d => query.newTimeDimension(d));
+
     const measureToLeafMeasures = {};
 
     const leafMeasurePaths =
@@ -319,16 +334,18 @@ export class PreAggregations {
       return true;
     }
 
-    const sortedTimeDimensions = PreAggregations.sortTimeDimensionsWithRollupGranularity(query.timeDimensions);
-    const timeDimensions = PreAggregations.timeDimensionsAsIs(query.timeDimensions);
+    const sortedTimeDimensions = PreAggregations.sortTimeDimensionsWithRollupGranularity(firstNonAliasTimeDimensions);
+    const timeDimensions = PreAggregations.timeDimensionsAsIs(firstNonAliasTimeDimensions);
     const ownedTimeDimensionsWithRollupGranularity = PreAggregations.sortTimeDimensionsWithRollupGranularity(ownedTimeDimensions);
     const ownedTimeDimensionsAsIs = PreAggregations.timeDimensionsAsIs(ownedTimeDimensions);
 
     const hasNoTimeDimensionsWithoutGranularity = !query.timeDimensions.filter(d => !d.granularity).length;
 
     const allFiltersWithinSelectedDimensions =
       R.all(d => dimensionsList.indexOf(d) !== -1)(
-        query.filters.map(f => f.dimension)
+        R.flatten(
+          query.filters.map(f => f.getMembers())
+        ).map(f => PreAggregations.firstNonAliasMember(query, [f])[0])
       );
 
     const isAdditive = R.all(m => m.isAdditive(), query.measures);
@@ -345,7 +362,8 @@ export class PreAggregations {
 
     let filterDimensionsSingleValueEqual = this.collectFilterDimensionsWithSingleValueEqual(
       query.filters,
-      dimensionsList.concat(segmentsList).reduce((map, d) => map.set(d, 1), new Map())
+      dimensionsList.concat(segmentsList).reduce((map, d) => map.set(d, 1), new Map()),
+      aliasedToFirstNonAliasedDimension
     );
 
     filterDimensionsSingleValueEqual =
@@ -381,6 +399,14 @@ export class PreAggregations {
     );
   }
 
+  static firstNonAliasMember(query, members) {
+    return members.map(
+      member => query
+        .collectFrom([member], query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor')
+        .find(d => !query.cubeEvaluator.byPathAnyType(d).aliasMember)
+    );
+  }
+
   static sortTimeDimensionsWithRollupGranularity(timeDimensions) {
     return timeDimensions && R.sortBy(
       R.prop(0),
@@ -395,13 +421,14 @@ export class PreAggregations {
     ) || [];
   }
 
-  static collectFilterDimensionsWithSingleValueEqual(filters, map) {
+  static collectFilterDimensionsWithSingleValueEqual(filters, map, aliasedToFirstNonAliasedDimension) {
     // eslint-disable-next-line no-restricted-syntax
     for (const f of filters) {
       if (f.operator === 'equals') {
-        map.set(f.dimension, Math.min(map.get(f.dimension) || 2, f.values.length));
+        const nonAliasedDimension = aliasedToFirstNonAliasedDimension[f.dimension] || f.dimension;
+        map.set(nonAliasedDimension, Math.min(map.get(nonAliasedDimension) || 2, f.values.length));
       } else if (f.operator === 'and') {
-        const res = this.collectFilterDimensionsWithSingleValueEqual(f.values, map);
+        const res = this.collectFilterDimensionsWithSingleValueEqual(f.values, map, aliasedToFirstNonAliasedDimension);
         if (res == null) return null;
       } else {
         return null;
@@ -632,14 +659,29 @@ export class PreAggregations {
     }
   }
 
-  static squashDimensions(query) {
+  static squashDimensions(query, flattenDimensionMembers) {
     return R.pipe(R.uniq, R.sortBy(R.identity))(
-      query.dimensions.concat(query.filters).map(d => d.dimension).concat(query.segments.map(s => s.segment))
+      PreAggregations.firstNonAliasMember(
+        query,
+        flattenDimensionMembers
+      )
     );
   }
 
-  static concatDimensionMembers(query) {
-    return query.dimensions.concat(query.filters).concat(query.segments);
+  static aliasedToFirstNonAliasedDimension(query, flattenDimensionMembers) {
+    return flattenDimensionMembers
+      .map(member => (
+        { [member.dimension]: PreAggregations.firstNonAliasMember(query, [member])[0] }
+      ));
+  }
+
+  static flattenDimensionMembers(query) {
+    return R.flatten(
+      query.dimensions
+        .concat(query.filters)
+        .concat(query.segments)
+        .map(m => m.getMembers()),
+    );
   }
 
   // eslint-disable-next-line no-unused-vars

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js

@@ -138,17 +138,23 @@ export class CubeEvaluator extends CubeSymbols {
 
     for (const memberName of Object.keys(members)) {
       let ownedByCube = true;
+      let aliasMember = false;
 
       const member = members[memberName];
       if (member.sql && !member.subQuery) {
         const funcArgs = this.funcArguments(member.sql);
-        const cubeReferences = this.collectUsedCubeReferences(cube.name, member.sql);
+        const { cubeReferencesUsed, evaluatedSql, pathReferencesUsed } = this.collectUsedCubeReferences(cube.name, member.sql);
         // We won't check for FILTER_PARAMS here as it shouldn't affect ownership and it should obey the same reference rules.
         // To affect ownership FILTER_PARAMS can be declared as `${FILTER_PARAMS.Foo.bar.filter(`${Foo.bar}`)}`.
-        if (funcArgs.length > 0 && cubeReferences.length === 0) {
+        // It isn't owned if there are non {CUBE} references
+        if (funcArgs.length > 0 && cubeReferencesUsed.length === 0) {
           ownedByCube = false;
         }
-        const foreignCubes = cubeReferences.filter(usedCube => usedCube !== cube.name);
+        // Aliases one to one some another member as in case of views
+        if (!ownedByCube && pathReferencesUsed.length === 1 && this.pathFromArray(pathReferencesUsed[0]) === evaluatedSql) {
+          aliasMember = true;
+        }
+        const foreignCubes = cubeReferencesUsed.filter(usedCube => usedCube !== cube.name);
         if (foreignCubes.length > 0) {
           errorReporter.error(`Member '${cube.name}.${memberName}' references foreign cubes: ${foreignCubes.join(', ')}. Please split and move this definition to corresponding cubes.`);
         }
@@ -158,7 +164,7 @@ export class CubeEvaluator extends CubeSymbols {
         errorReporter.error(`View '${cube.name}' defines own member '${cube.name}.${memberName}'. Please move this member definition to one of the cubes.`);
       }
 
-      members[memberName] = { ...members[memberName], ownedByCube };
+      members[memberName] = { ...members[memberName], ownedByCube, aliasMember };
     }
   }
 
@@ -357,8 +363,9 @@ export class CubeEvaluator extends CubeSymbols {
     const cubeEvaluator = this;
 
     const cubeReferencesUsed = [];
+    const pathReferencesUsed = [];
 
-    cubeEvaluator.resolveSymbolsCall(sqlFn, (name) => {
+    const evaluatedSql = cubeEvaluator.resolveSymbolsCall(sqlFn, (name) => {
       const referencedCube = cubeEvaluator.symbols[name] && name || cube;
       const resolvedSymbol =
         cubeEvaluator.resolveSymbol(
@@ -369,14 +376,20 @@ export class CubeEvaluator extends CubeSymbols {
       if (resolvedSymbol._objectWithResolvedProperties) {
         return resolvedSymbol;
       }
-      return cubeEvaluator.pathFromArray([referencedCube, name]);
+      const path = [referencedCube, name];
+      pathReferencesUsed.push(path);
+      return cubeEvaluator.pathFromArray(path);
     }, {
       // eslint-disable-next-line no-shadow
-      sqlResolveFn: (symbol, cube, n) => cubeEvaluator.pathFromArray([cube, n]),
+      sqlResolveFn: (symbol, cube, n) => {
+        const path = [cube, n];
+        pathReferencesUsed.push(path);
+        return cubeEvaluator.pathFromArray(path);
+      },
       contextSymbols: BaseQuery.emptyParametrizedContextSymbols(this, () => '$empty_param$'),
       cubeReferencesUsed,
     });
-    return cubeReferencesUsed;
+    return { cubeReferencesUsed, pathReferencesUsed, evaluatedSql };
   }
 
   evaluatePreAggregationReferences(cube, aggregation) {

packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts

@@ -82,6 +82,10 @@ describe('PreAggregations', () => {
           type: 'time',
           sql: 'created_at'
         },
+        signedUpAt: {
+          type: 'time',
+          sql: \`\${createdAt}\`
+        },
         checkinsCount: {
           type: 'number',
           sql: \`\${visitor_checkins.count}\`,
@@ -450,6 +454,13 @@ describe('PreAggregations', () => {
         }
       }
     });
+    
+    view('visitors_view', {
+      cubes: [{
+        join_path: visitors,
+        includes: ['uniqueSourceCount', 'ratio', 'createdAt', 'signedUpAt']
+      }]
+    });
     `);
 
   it('simple pre-aggregation', () => compiler.compile().then(() => {
@@ -540,6 +551,100 @@ describe('PreAggregations', () => {
     });
   }));
 
+  it('leaf measure view pre-aggregation', () => compiler.compile().then(() => {
+    const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
+      measures: [
+        'visitors_view.ratio'
+      ],
+      timeDimensions: [{
+        dimension: 'visitors_view.createdAt',
+        granularity: 'day',
+        dateRange: ['2017-01-01', '2017-01-30']
+      }],
+      timezone: 'America/Los_Angeles',
+      order: [{
+        id: 'visitors_view.createdAt'
+      }],
+      preAggregationsSchema: ''
+    });
+
+    const queryAndParams = query.buildSqlAndParams();
+    console.log(queryAndParams);
+    const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
+    console.log(preAggregationsDescription);
+    expect((<any>preAggregationsDescription)[0].loadSql[0]).toMatch(/visitors_ratio/);
+
+    return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
+      expect(res).toEqual(
+        [
+          {
+            visitors_view__created_at_day: '2017-01-02T00:00:00.000Z',
+            visitors_view__ratio: '0.33333333333333333333'
+          },
+          {
+            visitors_view__created_at_day: '2017-01-04T00:00:00.000Z',
+            visitors_view__ratio: '0.50000000000000000000'
+          },
+          {
+            visitors_view__created_at_day: '2017-01-05T00:00:00.000Z',
+            visitors_view__ratio: '1.00000000000000000000'
+          },
+          {
+            visitors_view__created_at_day: '2017-01-06T00:00:00.000Z',
+            visitors_view__ratio: null
+          }
+        ]
+      );
+    });
+  }));
+
+  it('non-additive measure view pre-aggregation', () => compiler.compile().then(() => {
+    const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
+      measures: [
+        'visitors_view.uniqueSourceCount'
+      ],
+      timeDimensions: [{
+        dimension: 'visitors_view.signedUpAt',
+        granularity: 'day',
+        dateRange: ['2017-01-01', '2017-01-30']
+      }],
+      timezone: 'America/Los_Angeles',
+      order: [{
+        id: 'visitors_view.createdAt'
+      }],
+      preAggregationsSchema: ''
+    });
+
+    const queryAndParams = query.buildSqlAndParams();
+    console.log(queryAndParams);
+    const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
+    console.log(preAggregationsDescription);
+    expect((<any>preAggregationsDescription)[0].loadSql[0]).toMatch(/visitors_ratio/);
+
+    return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
+      expect(res).toEqual(
+        [
+          {
+            visitors_view__signed_up_at_day: '2017-01-02T00:00:00.000Z',
+            visitors_view__unique_source_count: '1'
+          },
+          {
+            visitors_view__signed_up_at_day: '2017-01-04T00:00:00.000Z',
+            visitors_view__unique_source_count: '1'
+          },
+          {
+            visitors_view__signed_up_at_day: '2017-01-05T00:00:00.000Z',
+            visitors_view__unique_source_count: '1'
+          },
+          {
+            visitors_view__signed_up_at_day: '2017-01-06T00:00:00.000Z',
+            visitors_view__unique_source_count: '0'
+          }
+        ]
+      );
+    });
+  }));
+
   it('inherited original sql', () => compiler.compile().then(() => {
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
       measures: [