feat: Cube based includes and meta exposure (#6380)

* feat: Cube based includes and meta exposure

* chore: Fix mssql test

* Join path support in includes

Author: paveltiunov

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js

@@ -123,14 +123,16 @@ export class CubeEvaluator extends CubeSymbols {
   }
 
   addIncludes(cube, errorReporter) {
-    if (!cube.includes) {
+    if (!cube.includes && !cube.cubes) {
       return;
     }
     const types = ['measures', 'dimensions', 'segments'];
     for (const type of types) {
+      const cubeIncludes = cube.cubes && this.membersFromCubes(cube.cubes, type, errorReporter) || [];
       const includes = cube.includes && this.membersFromIncludeExclude(cube.includes, cube.name, type) || [];
       const excludes = cube.excludes && this.membersFromIncludeExclude(cube.excludes, cube.name, type) || [];
-      const finalIncludes = R.difference(includes, excludes);
+      // cube includes will take precedence in case of member clash
+      const finalIncludes = this.diffByMember(this.diffByMember(includes, cubeIncludes).concat(cubeIncludes), excludes);
       const includeMembers = this.generateIncludeMembers(finalIncludes, cube.name, type);
       for (const [memberName, memberDefinition] of includeMembers) {
         if (cube[type]?.[memberName]) {
@@ -142,16 +144,71 @@ export class CubeEvaluator extends CubeSymbols {
     }
   }
 
+  membersFromCubes(cubes, type, errorReporter) {
+    return R.unnest(cubes.map(cubeInclude => {
+      const fullPath = this.evaluateReferences(null, cubeInclude.cube, { collectJoinHints: true });
+      const split = fullPath.split('.');
+      const cubeReference = split[split.length - 1];
+      const cubeName = cubeInclude.name || cubeReference;
+      let includes;
+      if (cubeInclude.includes === '*') {
+        const membersObj = this.symbols[cubeReference]?.cubeObj()?.[type] || {};
+        includes = Object.keys(membersObj).map(memberName => ({ member: `${fullPath}.${memberName}` }));
+      } else {
+        includes = cubeInclude.includes.map(include => {
+          const member = include.member || include;
+          if (member.indexOf('.') !== -1) {
+            errorReporter.error(`Paths aren't allowed in cube includes but '${member}' provided as include member`);
+          }
+          let name = include.name || member;
+          name = cubeInclude.prefix ? `${cubeName}_${name}` : name;
+          if (include.member) {
+            const resolvedMember = this.symbols[cubeReference]?.cubeObj()?.[type]?.[include.member];
+            return resolvedMember ? {
+              member: `${fullPath}.${include.member}`,
+              name,
+            } : undefined;
+          } else {
+            const resolvedMember = this.symbols[cubeReference]?.cubeObj()?.[type]?.[include];
+            return resolvedMember ? {
+              member: `${fullPath}.${include}`,
+              name
+            } : undefined;
+          }
+        });
+      }
+
+      const excludes = (cubeInclude.excludes || []).map(exclude => {
+        if (exclude.indexOf('.') !== -1) {
+          errorReporter.error(`Paths aren't allowed in cube excludes but '${exclude}' provided as exclude member`);
+        }
+        const resolvedMember = this.symbols[cubeReference]?.cubeObj()?.[type]?.[exclude];
+        return resolvedMember ? {
+          member: `${cubeReference}.${exclude}`
+        } : undefined;
+      });
+      return this.diffByMember(includes.filter(Boolean), excludes.filter(Boolean));
+    }));
+  }
+
+  diffByMember(includes, excludes) {
+    const excludesMap = new Map();
+    for (const exclude of excludes) {
+      excludesMap.set(exclude.member, true);
+    }
+    return includes.filter(include => !excludesMap.get(include.member));
+  }
+
   membersFromIncludeExclude(referencesFn, cubeName, type) {
     const references = this.evaluateReferences(cubeName, referencesFn);
     return R.unnest(references.map(ref => {
       const path = ref.split('.');
       if (path.length === 1) {
         const membersObj = this.symbols[path[0]]?.cubeObj()?.[type] || {};
-        return Object.keys(membersObj).map(memberName => `${ref}.${memberName}`);
+        return Object.keys(membersObj).map(memberName => ({ member: `${ref}.${memberName}` }));
       } else if (path.length === 2) {
         const resolvedMember = this.symbols[path[0]]?.cubeObj()?.[type]?.[path[1]];
-        return resolvedMember ? [ref] : undefined;
+        return resolvedMember ? [{ member: ref }] : undefined;
       } else {
         throw new Error(`Unexpected path length ${path.length} for ${ref}`);
       }
@@ -160,33 +217,40 @@ export class CubeEvaluator extends CubeSymbols {
 
   generateIncludeMembers(members, cubeName, type) {
     return members.map(memberRef => {
-      const path = memberRef.split('.');
-      const resolvedMember = this.symbols[path[0]]?.cubeObj()?.[type]?.[path[1]];
+      const path = memberRef.member.split('.');
+      const resolvedMember = this.symbols[path[path.length - 2]]?.cubeObj()?.[type]?.[path[path.length - 1]];
       if (!resolvedMember) {
-        throw new Error(`Can't resolve '${memberRef}' while generating include members`);
+        throw new Error(`Can't resolve '${memberRef.member}' while generating include members`);
       }
 
       // eslint-disable-next-line no-new-func
-      const sql = new Function(path[0], `return \`\${${path[0]}.${path[1]}}\`;`);
+      const sql = new Function(path[0], `return \`\${${memberRef.member}}\`;`);
       let memberDefinition;
       if (type === 'measures') {
         memberDefinition = {
           sql,
-          type: 'number'
+          type: 'number',
+          aggType: resolvedMember.type,
+          meta: resolvedMember.meta,
+          description: resolvedMember.description,
         };
       } else if (type === 'dimensions') {
         memberDefinition = {
           sql,
-          type: resolvedMember.type
+          type: resolvedMember.type,
+          meta: resolvedMember.meta,
+          description: resolvedMember.description,
         };
       } else if (type === 'segments') {
         memberDefinition = {
-          sql
+          sql,
+          meta: resolvedMember.meta,
+          description: resolvedMember.description,
         };
       } else {
         throw new Error(`Unexpected member type: ${type}`);
       }
-      return [path[1], memberDefinition];
+      return [memberRef.name || path[path.length - 1], memberDefinition];
     });
   }
 
@@ -408,6 +472,14 @@ export class CubeEvaluator extends CubeSymbols {
   evaluateReferences(cube, referencesFn, options = {}) {
     const cubeEvaluator = this;
 
+    const fullPath = (joinHints, path) => {
+      if (joinHints?.length > 0) {
+        return R.uniq(joinHints.concat(path));
+      } else {
+        return path;
+      }
+    };
+
     const arrayOrSingle = cubeEvaluator.resolveSymbolsCall(referencesFn, (name) => {
       const referencedCube = cubeEvaluator.symbols[name] && name || cube;
       const resolvedSymbol =
@@ -419,10 +491,13 @@ export class CubeEvaluator extends CubeSymbols {
       if (resolvedSymbol._objectWithResolvedProperties) {
         return resolvedSymbol;
       }
-      return cubeEvaluator.pathFromArray([referencedCube, name]);
+      return cubeEvaluator.pathFromArray(fullPath(cubeEvaluator.joinHints(), [referencedCube, name]));
     }, {
       // eslint-disable-next-line no-shadow
-      sqlResolveFn: (symbol, cube, n) => cubeEvaluator.pathFromArray([cube, n])
+      sqlResolveFn: (symbol, cube, n) => cubeEvaluator.pathFromArray(fullPath(cubeEvaluator.joinHints(), [cube, n])),
+      // eslint-disable-next-line no-shadow
+      cubeAliasFn: (cube) => cubeEvaluator.pathFromArray(fullPath(cubeEvaluator.joinHints(), [cube])),
+      collectJoinHints: options.collectJoinHints,
     });
     if (!Array.isArray(arrayOrSingle)) {
       return arrayOrSingle.toString();

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -260,6 +260,7 @@ export class CubeSymbols {
       joinHints = joinHints.concat(cubeName);
     }
     const self = this;
+    const { sqlResolveFn, cubeAliasFn, query, cubeReferencesUsed } = self.resolveSymbolsCallContext || {};
     return new Proxy({}, {
       get: (v, propertyName) => {
         if (propertyName === '__cubeName') {
@@ -273,7 +274,6 @@ export class CubeSymbols {
           }
           return undefined;
         }
-        const { sqlResolveFn, cubeAliasFn, query, cubeReferencesUsed } = self.resolveSymbolsCallContext || {};
         if (propertyName === 'toString') {
           return () => {
             if (query) {
@@ -283,7 +283,10 @@ export class CubeSymbols {
             if (cubeReferencesUsed) {
               cubeReferencesUsed.push(cube.cubeName());
             }
-            return cubeAliasFn && cubeAliasFn(cube.cubeName()) || cube.cubeName();
+            return cubeAliasFn && this.withSymbolsCallContext(
+              () => cubeAliasFn(cube.cubeName()),
+              { ...this.resolveSymbolsCallContext, joinHints }
+            ) || cube.cubeName();
           };
         }
         if (propertyName === 'sql') {

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js

@@ -138,7 +138,7 @@ export class CubeToMetaTransformer {
       cumulativeTotal: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]),
       cumulative: nameToMetric[1].cumulative || BaseMeasure.isCumulative(nameToMetric[1]),
       type,
-      aggType: nameToMetric[1].type,
+      aggType: nameToMetric[1].aggType || nameToMetric[1].type,
       drillMembers: drillMembersArray,
       drillMembersGrouped: {
         measures: drillMembersArray.filter((member) => this.cubeEvaluator.isMeasure(member)),

packages/cubejs-schema-compiler/src/compiler/CubeValidator.js

@@ -21,7 +21,8 @@ export const nonStringFields = new Set([
   'incremental',
   'external',
   'useOriginalSqlPreAggregations',
-  'readOnly'
+  'readOnly',
+  'prefix'
 ]);
 
 const identifierRegex = /^[_a-zA-Z][_a-zA-Z0-9]*$/;
@@ -528,6 +529,24 @@ const viewSchema = inherit(baseSchema, {
   isView: Joi.boolean().strict(),
   includes: Joi.func(),
   excludes: Joi.func(),
+  cubes: Joi.array().items(
+    Joi.object().keys({
+      cube: Joi.func().required(),
+      prefix: Joi.boolean(),
+      name: Joi.string(),
+      includes: Joi.alternatives([
+        Joi.string().valid('*'),
+        Joi.array().items(Joi.alternatives([
+          Joi.string().required(),
+          Joi.object().keys({
+            member: Joi.string().required(),
+            name: Joi.string()
+          })
+        ]))
+      ]).required(),
+      excludes: Joi.array().items(Joi.string().required()),
+    })
+  ),
 });
 
 function formatErrorMessageFromDetails(explain, d) {

packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts

@@ -17,6 +17,7 @@ export const transpiledFieldsPatterns: Array<RegExp> = [
   /^contextMembers$/,
   /^includes$/,
   /^excludes$/,
+  /^cubes\.[0-9]+\.cube$/,
 ];
 
 export const transpiledFields: Set<String> = new Set<String>();

packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts

@@ -346,18 +346,24 @@ cubes:
         
 views:
   - name: orders_view
-    includes: 
-      - orders.count
-      - orders.time
-      - customers.name
+    cubes:
+      - cube: orders
+        prefix: true
+        includes:
+          - count
+          - member: time
+            name: date
+      - cube: orders.customers
+        includes:
+          - name
     `);
     await compiler.compile();
 
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
-      measures: ['orders_view.count'],
+      measures: ['orders_view.orders_count'],
       dimensions: ['orders_view.name'],
       timeDimensions: [{
-        dimension: 'orders_view.time',
+        dimension: 'orders_view.orders_date',
         granularity: 'day',
         dateRange: ['2022-01-01', '2022-01-03']
       }],
@@ -372,9 +378,9 @@ views:
 
     expect(res).toEqual(
       [{
-        orders_view__count: '1',
+        orders_view__orders_count: '1',
         orders_view__name: 'Foo',
-        orders_view__time_day: '2022-01-01T00:00:00.000Z',
+        orders_view__orders_date_day: '2022-01-01T00:00:00.000Z',
       }]
     );
   });

packages/cubejs-testing-shared/src/db/mssql.ts

@@ -8,7 +8,7 @@ type MssqlStartOptions = DBRunnerContainerOptions & {
 
 export class MssqlDbRunner extends DbRunnerAbstract {
   public static startContainer(options: MssqlStartOptions) {
-    const version = process.env.TEST_MSSQL_VERSION || options.version || '2022-latest';
+    const version = process.env.TEST_MSSQL_VERSION || options.version || '2017-latest';
 
     const container = new GenericContainer(`mcr.microsoft.com/mssql/server:${version}`)
       .withEnv('ACCEPT_EULA', 'Y')

packages/cubejs-testing/birdbox-fixtures/driver-test-data/ECommerce.js

@@ -46,6 +46,9 @@ cube(`ECommerce`, {
   measures: {
     count: {
       type: `count`,
+      meta: {
+        foo: `bar`
+      }
     },
     totalQuantity: {
       sql: 'quantity',
@@ -114,3 +117,11 @@ cube(`ECommerce`, {
     },
   },
 });
+
+view(`ECommerceView`, {
+  cubes: [{
+    cube: ECommerce,
+    includes: `*`,
+    excludes: [`orderDate`]
+  }]
+});

packages/cubejs-testing/test/driverTests/testSets.ts

@@ -54,7 +54,7 @@ import {
   filteringECommerceEndsWithDimensionsFirst,
   filteringECommerceEndsWithDimensionsSecond,
   filteringECommerceEndsWithDimensionsThird,
-  queryingEcommerceTotalQuantifyAvgDiscountTotal, hiddenMember, hiddenCube, preAggsCustomersRunningTotal,
+  queryingEcommerceTotalQuantifyAvgDiscountTotal, hiddenMember, hiddenCube, viewMetaExposed, preAggsCustomersRunningTotal,
 } from './tests';
 import { testSet } from './driverTest';
 
@@ -133,6 +133,7 @@ export const mainTestSet = testSet([
   ...withoutOrderingTestSet,
   hiddenMember,
   hiddenCube,
+  viewMetaExposed
 ]);
 
 export const preAggsTestSet = testSet([

packages/cubejs-testing/test/driverTests/tests.ts

@@ -1291,3 +1291,14 @@ export const hiddenCube = driverTestFn({
     expect(meta.cubes.find(cube => cube.name === 'HiddenECommerce')).toBe(undefined);
   }
 });
+
+export const viewMetaExposed = driverTestFn({
+  name: 'view meta exposed',
+  schemas: commonSchemas,
+  testFn: async (client) => {
+    const meta = await client.meta();
+    const view = meta.cubes.find(cube => cube.name === 'ECommerceView');
+    expect(view?.measures?.find(m => m.name === 'ECommerceView.count')?.aggType).toBe('count');
+    expect(view?.measures?.find(m => m.name === 'ECommerceView.count')?.meta?.foo).toBe('bar');
+  }
+});