feat(schema-compiler): auto include hierarchy dimensions (#9288)

Author: vasilev-alex

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -263,7 +263,8 @@ export class CubeEvaluator extends CubeSymbols {
                 }
 
                 return null;
-              }).filter(Boolean);
+              })
+                .filter(Boolean);
 
               const name = hierarchyPathToName[[cubeName, it.name].join('.')];
               if (!name) {

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -243,10 +243,39 @@ export class CubeSymbols {
       allMembers: new Set(),
     };
 
-    const types = ['measures', 'dimensions', 'segments', 'hierarchies'];
+    const autoIncludeMembers = new Set();
+    // `hierarchies` must be processed first
+    const types = ['hierarchies', 'measures', 'dimensions', 'segments'];
+
     for (const type of types) {
-      const cubeIncludes = cube.cubes && this.membersFromCubes(cube, cube.cubes, type, errorReporter, splitViews, memberSets) || [];
+      let cubeIncludes = [];
+      if (cube.cubes) {
+        // If the hierarchy is included all members from it should be included as well
+        // Extend `includes` with members from hierarchies that should be auto-included
+        const cubes = type === 'dimensions' ? cube.cubes.map((it) => {
+          const fullPath = this.evaluateReferences(null, it.joinPath, { collectJoinHints: true });
+          const split = fullPath.split('.');
+          const cubeRef = split[split.length - 1];
+
+          if (it.includes === '*') {
+            return it;
+          }
+
+          const currentCubeAutoIncludeMembers = Array.from(autoIncludeMembers)
+            .filter((path) => path.startsWith(`${cubeRef}.`))
+            .map((path) => path.split('.')[1])
+            .filter(memberName => !it.includes.find((include) => (include.name || include) === memberName));
 
+          return {
+            ...it,
+            includes: (it.includes || []).concat(currentCubeAutoIncludeMembers),
+          };
+        }) : cube.cubes;
+
+        cubeIncludes = this.membersFromCubes(cube, cubes, type, errorReporter, splitViews, memberSets) || [];
+      }
+
+      // This is the deprecated approach
       const includes = cube.includes && this.membersFromIncludeExclude(cube.includes, cube.name, type) || [];
       const excludes = cube.excludes && this.membersFromIncludeExclude(cube.excludes, cube.name, type) || [];
 
@@ -256,6 +285,21 @@ export class CubeSymbols {
         excludes
       );
 
+      if (type === 'hierarchies') {
+        for (const member of finalIncludes) {
+          const path = member.member.split('.');
+          const cubeName = path[path.length - 2];
+          const hierarchyName = path[path.length - 1];
+          const hierarchy = this.getResolvedMember(type, cubeName, hierarchyName);
+
+          if (hierarchy) {
+            const levels = this.evaluateReferences(cubeName, this.getResolvedMember('hierarchies', cubeName, hierarchyName).levels, { originalSorting: true });
+
+            levels.forEach((level) => autoIncludeMembers.add(level));
+          }
+        }
+      }
+
       const includeMembers = this.generateIncludeMembers(finalIncludes, cube.name, type);
       this.applyIncludeMembers(includeMembers, cube, type, errorReporter);
 
@@ -278,7 +322,7 @@ export class CubeSymbols {
   applyIncludeMembers(includeMembers, cube, type, errorReporter) {
     for (const [memberName, memberDefinition] of includeMembers) {
       if (cube[type]?.[memberName]) {
-        errorReporter.error(`Included member '${memberName}' conflicts with existing member of '${cube.name}'. Please consider excluding this member.`);
+        errorReporter.error(`Included member '${memberName}' conflicts with existing member of '${cube.name}'. Please consider excluding this member or assigning it an alias.`);
       } else if (type !== 'hierarchies') {
         cube[type][memberName] = memberDefinition;
       }
@@ -300,11 +344,7 @@ export class CubeSymbols {
 
       if (cubeInclude.includes === '*') {
         const membersObj = this.symbols[cubeReference]?.cubeObj()?.[type] || {};
-        if (Array.isArray(membersObj)) {
-          includes = membersObj.map(it => ({ member: `${fullPath}.${it.name}`, name: fullMemberName(it.name) }));
-        } else {
-          includes = Object.keys(membersObj).map(memberName => ({ member: `${fullPath}.${memberName}`, name: fullMemberName(memberName) }));
-        }
+        includes = Object.keys(membersObj).map(memberName => ({ member: `${fullPath}.${memberName}`, name: fullMemberName(memberName) }));
       } else {
         includes = cubeInclude.includes.map(include => {
           const member = include.alias || include;
@@ -397,10 +437,6 @@ export class CubeSymbols {
    * @protected
    */
   getResolvedMember(type, cubeName, memberName) {
-    if (Array.isArray(this.symbols[cubeName]?.cubeObj()?.[type])) {
-      return this.symbols[cubeName]?.cubeObj()?.[type]?.find((it) => it.name === memberName);
-    }
-
     return this.symbols[cubeName]?.cubeObj()?.[type]?.[memberName];
   }
 

packages/cubejs-schema-compiler/test/unit/fixtures/folders.yml

@@ -22,10 +22,6 @@ cubes:
       - name: status
         sql: status
         type: string
-
-      - name: city
-        sql: city
-        type: string
     hierarchies:
       - name: orders_hierarchy
         levels:

packages/cubejs-schema-compiler/test/unit/fixtures/hierarchies.yml

@@ -66,6 +66,8 @@ views:
         includes:
           - age
           - state
+          - name: city
+            alias: user_city
   - name: orders_includes_excludes_view
     cubes:
       - join_path: orders
@@ -85,5 +87,17 @@ views:
       - join_path: users
         prefix: true
         includes: "*"
-
-
+  - name: only_hierarchy_included_view
+    cubes:
+      - join_path: orders
+        includes:
+          - orders_hierarchy
+      - join_path: users
+        includes:
+          - city
+  - name: auto_include_view
+    cubes:
+      - join_path: orders
+        includes:
+          - orders_hierarchy
+          - some_other_hierarchy

packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts

@@ -25,14 +25,15 @@ describe('Cube hierarchies', () => {
         public: true,
         levels: [
           'orders_users_view.status',
-          'orders_users_view.number'
+          'orders_users_view.number',
+          'orders_users_view.user_city'
         ],
       },
       {
         name: 'orders_users_view.some_other_hierarchy',
         public: true,
         title: 'Some other hierarchy',
-        levels: ['orders_users_view.state']
+        levels: ['orders_users_view.state', 'orders_users_view.user_city']
       }
     ]);
 
@@ -56,6 +57,40 @@ describe('Cube hierarchies', () => {
     expect(prefixedHierarchy?.levels).toEqual(['all_hierarchy_view.users_age', 'all_hierarchy_view.users_city']);
   });
 
+  it('auto include hierarchy members', async () => {
+    const modelContent = fs.readFileSync(
+      path.join(process.cwd(), '/test/unit/fixtures/hierarchies.yml'),
+      'utf8'
+    );
+    const { compiler, metaTransformer } = prepareYamlCompiler(modelContent);
+
+    await compiler.compile();
+
+    const view1 = metaTransformer.cubes.find(
+      (it) => it.config.name === 'only_hierarchy_included_view'
+    );
+
+    expect(view1.config.dimensions).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ name: 'only_hierarchy_included_view.status' }),
+        expect.objectContaining({ name: 'only_hierarchy_included_view.number' }),
+        expect.objectContaining({ name: 'only_hierarchy_included_view.city' })
+      ])
+    );
+
+    // Members from the `users` cube are not included as `users` is not selected (not joined)
+    const view2 = metaTransformer.cubes.find(
+      (it) => it.config.name === 'auto_include_view'
+    );
+    expect(view2.config.dimensions.length).toEqual(2);
+    expect(view2.config.dimensions).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ name: 'auto_include_view.status' }),
+        expect.objectContaining({ name: 'auto_include_view.number' }),
+      ])
+    );
+  });
+
   it(('hierarchy with measure'), async () => {
     const modelContent = fs.readFileSync(
       path.join(process.cwd(), '/test/unit/fixtures/hierarchy-with-measure.yml'),