fix: Recursive pre-aggregation description generation: support propagateFiltersToSubQuery with partitioned originalSql

Author: paveltiunov

packages/cubejs-schema-compiler/adapter/BaseQuery.js

@@ -1533,6 +1533,11 @@ class BaseQuery {
         };
       }
     }
+
+    return this.refreshKeysByCubes(this.allCubeNames, transformFn);
+  }
+
+  refreshKeysByCubes(cubes, transformFn) {
     let refreshKeyAllSetManually = true;
     const refreshKeyQueryByCube = cube => {
       const cubeFromPath = this.cubeEvaluator.cubeFromPath(cube);
@@ -1562,15 +1567,14 @@ class BaseQuery {
         { preAggregationQuery: true }
       );
     };
-    const cubeNames = this.allCubeNames;
-    const queries = cubeNames
+    const queries = cubes
       .map(cube => [cube, refreshKeyQueryByCube(cube)])
       .map(([cube, sql]) => (transformFn ? transformFn(sql, cube) : sql))
       .map(paramAnnotatedSql => this.paramAllocator.buildSqlAndParams(paramAnnotatedSql));
     return {
       queries,
       renewalThreshold: this.renewalThreshold(refreshKeyAllSetManually),
-      refreshKeyRenewalThresholds: cubeNames.map(c => {
+      refreshKeyRenewalThresholds: cubes.map(c => {
         const cubeFromPath = this.cubeEvaluator.cubeFromPath(c);
         if (cubeFromPath.refreshKey && cubeFromPath.refreshKey.every) {
           return this.refreshKeyRenewalThresholdForInterval(cubeFromPath.refreshKey.every);
@@ -1760,7 +1764,7 @@ class BaseQuery {
             if (!preAggregation.partitionGranularity) {
               throw new UserError(`Incremental refresh key can only be used for partitioned pre-aggregations`);
             }
-            // TOOD Case when partitioned originalSql is resolved for query without time dimension.
+            // TODO Case when partitioned originalSql is resolved for query without time dimension.
             // Consider fallback to not using such originalSql for consistency?
             if (preAggregationQueryForSql.timeDimensions.length) {
               refreshKey = this.incrementalRefreshKey(
@@ -1777,6 +1781,12 @@ class BaseQuery {
             };
           }
         }
+        if (preAggregation.type === 'originalSql') {
+          return this.evaluateSymbolSqlWithContext(
+            () => this.refreshKeysByCubes([cube]),
+            { preAggregationQuery: true }
+          );
+        }
         if (
           preAggregation.partitionGranularity &&
           !preAggregationQueryForSql.allCubeNames.find(c => {

packages/cubejs-schema-compiler/adapter/PreAggregations.js

@@ -19,42 +19,29 @@ class PreAggregations {
   }
 
   preAggregationsDescriptionLocal() {
-    const preAggregationForQuery = this.findPreAggregationForQuery();
-    if (preAggregationForQuery) {
-      if (preAggregationForQuery.preAggregation.useOriginalSqlPreAggregations) {
-        const { preAggregations, result } =
-          this.collectOriginalSqlPreAggregations(
-            () => this.preAggregationDescriptionsFor(preAggregationForQuery.cube, preAggregationForQuery)
-          );
-
-        const queryForEval = this.query.preAggregationQueryForSqlEvaluation(
-          preAggregationForQuery.cube,
-          preAggregationForQuery.preAggregation
-        );
-
-        // TODO consider recursive pre-aggregation descriptions instead of duplication of sub query logic
-        return R.unnest(preAggregations.map(
-          p => this.preAggregationDescriptionsFor(p.cube, p).concat(
-            R.unnest(
-              queryForEval.subQueryDimensions.map(d => queryForEval.subQueryDescription(d).subQuery)
-                .map(q => q.preAggregations.preAggregationDescriptionsFor(p.cube, p))
-            )
-          )
-        )).concat(result);
+    const isInPreAggregationQuery = this.query.options.preAggregationQuery;
+    if (!isInPreAggregationQuery) {
+      const preAggregationForQuery = this.findPreAggregationForQuery();
+      if (preAggregationForQuery) {
+        return this.preAggregationDescriptionsFor(preAggregationForQuery.cube, preAggregationForQuery);
       }
-      return this.preAggregationDescriptionsFor(preAggregationForQuery.cube, preAggregationForQuery);
     }
-    return R.pipe(
-      R.map(cube => {
-        const foundPreAggregation = this.findPreAggregationToUseForCube(cube);
-        if (foundPreAggregation) {
-          return this.preAggregationDescriptionsFor(cube, foundPreAggregation);
-        }
-        return null;
-      }),
-      R.filter(R.identity),
-      R.unnest
-    )(this.preAggregationCubes());
+    if (
+      !isInPreAggregationQuery ||
+      isInPreAggregationQuery && this.query.options.useOriginalSqlPreAggregationsInPreAggregation) {
+      return R.pipe(
+        R.map(cube => {
+          const foundPreAggregation = this.findPreAggregationToUseForCube(cube);
+          if (foundPreAggregation) {
+            return this.preAggregationDescriptionsFor(cube, foundPreAggregation);
+          }
+          return null;
+        }),
+        R.filter(R.identity),
+        R.unnest
+      )(this.preAggregationCubes());
+    }
+    return [];
   }
 
   preAggregationCubes() {
@@ -65,13 +52,13 @@ class PreAggregations {
   preAggregationDescriptionsFor(cube, foundPreAggregation) {
     if (this.canPartitionsBeUsed(foundPreAggregation)) {
       const { dimension, partitionDimension } = this.partitionDimension(foundPreAggregation);
-      return partitionDimension.timeSeries().map(
-        range => this.preAggregationDescriptionFor(
+      return R.unnest(partitionDimension.timeSeries().map(
+        range => this.preAggregationDescriptionsForRecursive(
           cube, this.addPartitionRangeTo(foundPreAggregation, dimension, range)
         )
-      );
+      ));
     }
-    return [this.preAggregationDescriptionFor(cube, foundPreAggregation)];
+    return this.preAggregationDescriptionsForRecursive(cube, foundPreAggregation);
   }
 
   canPartitionsBeUsed(foundPreAggregation) {
@@ -103,6 +90,12 @@ class PreAggregations {
     return { dimension, partitionDimension };
   }
 
+  preAggregationDescriptionsForRecursive(cube, foundPreAggregation) {
+    const query = this.query.preAggregationQueryForSqlEvaluation(cube, foundPreAggregation.preAggregation);
+    const descriptions = query !== this.query ? query.preAggregations.preAggregationsDescription() : [];
+    return descriptions.concat(this.preAggregationDescriptionFor(cube, foundPreAggregation));
+  }
+
   preAggregationDescriptionFor(cube, foundPreAggregation) {
     const { preAggregationName, preAggregation } = foundPreAggregation;
     const tableName = this.preAggregationTableName(cube, preAggregationName, preAggregation);

packages/cubejs-schema-compiler/test/PreAggregationsTest.js

@@ -73,7 +73,8 @@ describe('PreAggregations', function test() {
         checkinsCount: {
           type: 'number',
           sql: \`\${visitor_checkins.count}\`,
-          subQuery: true
+          subQuery: true,
+          propagateFiltersToSubQuery: true
         }
       },
       
@@ -248,8 +249,11 @@ describe('PreAggregations', function test() {
     console.log(toReplace);
     preAggregation = Array.isArray(preAggregation) ? preAggregation : [preAggregation];
     return [
-      preAggregation.reduce((replacedQuery, desc) =>
-        replacedQuery.replace(new RegExp(desc.tableName, 'g'), desc.tableName + '_' + suffix), toReplace
+      preAggregation.reduce(
+        (replacedQuery, desc) => replacedQuery
+          .replace(new RegExp(desc.tableName, 'g'), desc.tableName + '_' + suffix)
+          .replace(/CREATE INDEX (?!i_)/, `CREATE INDEX i_${suffix}_`),
+        toReplace
       ),
       params
     ];