chore(api-gateway): Response size optimization, pre-aggregations system API (#6728)

Author: RusovDmitriy

packages/cubejs-api-gateway/src/gateway.ts

@@ -618,14 +618,29 @@ class ApiGateway {
           compilerApi.preAggregationsSchema
         );
 
-      const mergePartitionsAndVersionEntries = () => ({ preAggregation, partitions, ...props }) => ({
-        ...props,
+      const checkExpand = (path: string | RegExp) => !query.expand ||
+        (path instanceof RegExp
+          ? query.expand.some((p: string) => path.test(p))
+          : query.expand.includes(path));
+
+      const mergePartitionsAndVersionEntries = () => ({ errors, preAggregation, partitions, invalidateKeyQueries, timezones }) => ({
+        errors,
+        invalidateKeyQueries,
         preAggregation,
-        partitions: partitions.map(partition => {
-          partition.versionEntries = versionEntriesResult?.versionEntriesByTableName[partition?.tableName] || [];
-          partition.structureVersion = versionEntriesResult?.structureVersionsByTableName[partition?.tableName];
-          return partition;
-        }),
+        timezones,
+        partitions: partitions.map(partition => ({
+          ...(checkExpand('partitions.details') ? partition : {}),
+          ...(checkExpand('partitions.meta') ? {
+            dataSource: partition.dataSource,
+            preAggregationId: partition.preAggregationId,
+            tableName: partition.tableName,
+            type: partition.type,
+          } : {}),
+          ...(checkExpand('partitions.versions') ? {
+            versionEntries: versionEntriesResult?.versionEntriesByTableName[partition?.tableName] || [],
+            structureVersion: versionEntriesResult?.structureVersionsByTableName[partition?.tableName] || [],
+          } : {}),
+        })),
       });
 
       res({

packages/cubejs-api-gateway/src/query.js

@@ -254,6 +254,7 @@ const remapToQueryAdapterFormat = (query) => (query ? {
 } : query);
 
 const queryPreAggregationsSchema = Joi.object().keys({
+  expand: Joi.array().items(Joi.string()),
   metadata: Joi.object(),
   timezone: Joi.string(),
   timezones: Joi.array().items(Joi.string()),
@@ -274,7 +275,8 @@ const normalizeQueryPreAggregations = (query, defaultValues) => {
   return {
     metadata: query.metadata,
     timezones: query.timezones || (query.timezone && [query.timezone]) || defaultValues?.timezones || ['UTC'],
-    preAggregations: query.preAggregations
+    preAggregations: query.preAggregations,
+    expand: query.expand
   };
 };
 

packages/cubejs-api-gateway/test/mocks.ts

@@ -28,7 +28,7 @@ export const preAggregationsResultFactory = () => ([
 
 export const preAggregationPartitionsResultFactory = () => ([
   {
-    timezone: 'UTC',
+    timezones: ['UTC'],
     preAggregation: preAggregationsResultFactory()[0],
     partitions: [{
       tableName: 'dev_pre_aggregations.usage_usages20210430'