feat(schema-compiler): folders support, hierarchies improvements (#9018)

Author: vasilev-alex

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -112,7 +112,8 @@ export class CubeEvaluator extends CubeSymbols {
     this.evaluateMultiStageReferences(cube.name, cube.measures);
     this.evaluateMultiStageReferences(cube.name, cube.dimensions);
 
-    this.prepareHierarchies(cube);
+    this.prepareHierarchies(cube, errorReporter);
+    this.prepareFolders(cube, errorReporter);
 
     this.prepareAccessPolicy(cube, errorReporter);
 
@@ -179,36 +180,91 @@ export class CubeEvaluator extends CubeSymbols {
     }
   }
 
-  private prepareHierarchies(cube: any) {
+  private prepareFolders(cube: any, errorReporter: ErrorReporter) {
+    if (Array.isArray(cube.folders)) {
+      cube.folders = cube.folders.map(it => {
+        const includedMembers = this.allMembersOrList(cube, it.includes);
+        const includes = includedMembers.map(memberName => {
+          if (memberName.includes('.')) {
+            errorReporter.error(
+              `Paths aren't allowed in the 'folders' but '${memberName}' has been provided for ${cube.name}`
+            );
+          }
+
+          const member = cube.includedMembers.find(m => m.name === memberName);
+          if (!member) {
+            errorReporter.error(
+              `Member '${memberName}' included in folder '${it.name}' not found`
+            );
+            return null;
+          }
+
+          return member;
+        })
+          .filter(Boolean);
+
+        return ({
+          ...it,
+          includes
+        });
+      });
+    }
+
+    return [];
+  }
+
+  private prepareHierarchies(cube: any, errorReporter: ErrorReporter) {
+    const uniqueHierarchyNames = new Set();
     if (Array.isArray(cube.hierarchies)) {
-      cube.hierarchies = cube.hierarchies.map(hierarchy => ({
-        ...hierarchy,
-        levels: this.evaluateReferences(
-          cube.name, hierarchy.levels, { originalSorting: true }
-        )
-      }));
+      cube.hierarchies = cube.hierarchies.map(hierarchy => {
+        if (uniqueHierarchyNames.has(hierarchy.name)) {
+          errorReporter.error(`Duplicate hierarchy name '${hierarchy.name}' in cube '${cube.name}'`);
+        }
+        uniqueHierarchyNames.add(hierarchy.name);
+
+        return ({
+          ...hierarchy,
+          levels: this.evaluateReferences(
+            cube.name,
+            hierarchy.levels,
+            { originalSorting: true }
+          )
+        });
+      });
     }
 
     if (cube.isView && (cube.includedMembers || []).length) {
-      const includedCubeNames: string[] = R.uniq(cube.includedMembers.map(it => it.memberPath.split('.')[0]));
       const includedMemberPaths: string[] = R.uniq(cube.includedMembers.map(it => it.memberPath));
-
-      if (!cube.hierarchies) {
-        for (const cubeName of includedCubeNames) {
-          const { hierarchies } = this.evaluatedCubes[cubeName] || {};
-
-          if (Array.isArray(hierarchies) && hierarchies.length) {
-            const filteredHierarchies = hierarchies.map(it => {
-              const levels = it.levels.filter(level => includedMemberPaths.includes(level));
+      const includedCubeNames: string[] = R.uniq(includedMemberPaths.map(it => it.split('.')[0]));
+      const includedHierarchyNames = cube.includedMembers.filter(it => it.type === 'hierarchies').map(it => it.memberPath.split('.')[1]);
+
+      for (const cubeName of includedCubeNames) {
+        const { hierarchies } = this.evaluatedCubes[cubeName] || {};
+
+        if (Array.isArray(hierarchies) && hierarchies.length) {
+          const filteredHierarchies = hierarchies
+            .filter(it => includedHierarchyNames.includes(it.name))
+            .map(it => {
+              const levels = it.levels.filter(level => {
+                const member = cube.includedMembers.find(m => m.memberPath === level);
+                if (member && member.type !== 'dimensions') {
+                  const memberName = level.split('.')[1] || level;
+                  errorReporter.error(`Only dimensions can be part of a hierarchy. Please remove the '${memberName}' member from the '${it.name}' hierarchy.`);
+                } else if (member) {
+                  return includedMemberPaths.includes(level);
+                }
+
+                return null;
+              }).filter(Boolean);
 
               return {
                 ...it,
                 levels
               };
-            }).filter(it => it.levels.length);
+            })
+            .filter(it => it.levels.length);
 
-            cube.hierarchies = [...(cube.hierarchies || []), ...filteredHierarchies];
-          }
+          cube.hierarchies = [...(cube.hierarchies || []), ...filteredHierarchies];
         }
       }
 

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -226,9 +226,15 @@ export class CubeSymbols {
       return;
     }
 
-    const types = ['measures', 'dimensions', 'segments'];
+    const memberSets = {
+      resolvedMembers: new Set(),
+      allMembers: new Set(),
+    };
+
+    const types = ['measures', 'dimensions', 'segments', 'hierarchies'];
     for (const type of types) {
-      const cubeIncludes = cube.cubes && this.membersFromCubes(cube, cube.cubes, type, errorReporter, splitViews) || [];
+      const cubeIncludes = cube.cubes && this.membersFromCubes(cube, cube.cubes, type, errorReporter, splitViews, memberSets) || [];
+
       const includes = cube.includes && this.membersFromIncludeExclude(cube.includes, cube.name, type) || [];
       const excludes = cube.excludes && this.membersFromIncludeExclude(cube.excludes, cube.name, type) || [];
 
@@ -245,18 +251,23 @@ export class CubeSymbols {
         const split = it.member.split('.');
         const memberPath = this.pathFromArray([split[split.length - 2], split[split.length - 1]]);
         return {
+          type,
           memberPath,
           name: it.name
         };
       })))];
     }
+
+    [...memberSets.allMembers].filter(it => !memberSets.resolvedMembers.has(it)).forEach(it => {
+      errorReporter.error(`Member '${it}' is included in '${cube.name}' but not defined in any cube`);
+    });
   }
 
   applyIncludeMembers(includeMembers, cube, type, errorReporter) {
     for (const [memberName, memberDefinition] of includeMembers) {
       if (cube[type]?.[memberName]) {
         errorReporter.error(`Included member '${memberName}' conflicts with existing member of '${cube.name}'. Please consider excluding this member.`);
-      } else {
+      } else if (type !== 'hierarchies') {
         cube[type][memberName] = memberDefinition;
       }
     }
@@ -265,7 +276,7 @@ export class CubeSymbols {
   /**
    * @protected
    */
-  membersFromCubes(parentCube, cubes, type, errorReporter, splitViews) {
+  membersFromCubes(parentCube, cubes, type, errorReporter, splitViews, memberSets) {
     return R.unnest(cubes.map(cubeInclude => {
       const fullPath = this.evaluateReferences(null, cubeInclude.joinPath, { collectJoinHints: true });
       const split = fullPath.split('.');
@@ -277,37 +288,43 @@ export class CubeSymbols {
 
       if (cubeInclude.includes === '*') {
         const membersObj = this.symbols[cubeReference]?.cubeObj()?.[type] || {};
-        includes = Object.keys(membersObj).map(memberName => ({ member: `${fullPath}.${memberName}`, name: fullMemberName(memberName) }));
+        if (Array.isArray(membersObj)) {
+          includes = membersObj.map(it => ({ member: `${fullPath}.${it.name}`, name: fullMemberName(it.name) }));
+        } else {
+          includes = Object.keys(membersObj).map(memberName => ({ member: `${fullPath}.${memberName}`, name: fullMemberName(memberName) }));
+        }
       } else {
         includes = cubeInclude.includes.map(include => {
           const member = include.alias || include;
-          if (member.indexOf('.') !== -1) {
+
+          if (member.includes('.')) {
             errorReporter.error(`Paths aren't allowed in cube includes but '${member}' provided as include member`);
           }
 
           const name = fullMemberName(include.alias || member);
-          if (include.name) {
-            const resolvedMember = this.symbols[cubeReference]?.cubeObj()?.[type]?.[include.name];
-            return resolvedMember ? {
-              member: `${fullPath}.${include.name}`,
-              name,
-            } : undefined;
-          } else {
-            const resolvedMember = this.symbols[cubeReference]?.cubeObj()?.[type]?.[include];
-            return resolvedMember ? {
-              member: `${fullPath}.${include}`,
-              name
-            } : undefined;
+          memberSets.allMembers.add(name);
+
+          const includedMemberName = include.name || include;
+
+          const resolvedMember = this.getResolvedMember(type, cubeReference, includedMemberName) ? {
+            member: `${fullPath}.${includedMemberName}`,
+            name,
+          } : undefined;
+
+          if (resolvedMember) {
+            memberSets.resolvedMembers.add(name);
           }
+
+          return resolvedMember;
         });
       }
 
       const excludes = (cubeInclude.excludes || []).map(exclude => {
-        if (exclude.indexOf('.') !== -1) {
+        if (exclude.includes('.')) {
           errorReporter.error(`Paths aren't allowed in cube excludes but '${exclude}' provided as exclude member`);
         }
 
-        const resolvedMember = this.symbols[cubeReference]?.cubeObj()?.[type]?.[exclude];
+        const resolvedMember = this.getResolvedMember(type, cubeReference, exclude);
         return resolvedMember ? {
           member: `${fullPath}.${exclude}`
         } : undefined;
@@ -356,21 +373,32 @@ export class CubeSymbols {
         const membersObj = this.symbols[path[0]]?.cubeObj()?.[type] || {};
         return Object.keys(membersObj).map(memberName => ({ member: `${ref}.${memberName}` }));
       } else if (path.length === 2) {
-        const resolvedMember = this.symbols[path[0]]?.cubeObj()?.[type]?.[path[1]];
+        const resolvedMember = this.getResolvedMember(type, path[0], path[1]);
         return resolvedMember ? [{ member: ref }] : undefined;
       } else {
         throw new Error(`Unexpected path length ${path.length} for ${ref}`);
       }
     })).filter(Boolean);
   }
 
+  /**
+   * @protected
+   */
+  getResolvedMember(type, cubeName, memberName) {
+    if (Array.isArray(this.symbols[cubeName]?.cubeObj()?.[type])) {
+      return this.symbols[cubeName]?.cubeObj()?.[type]?.find((it) => it.name === memberName);
+    }
+
+    return this.symbols[cubeName]?.cubeObj()?.[type]?.[memberName];
+  }
+
   /**
    * @protected
    */
   generateIncludeMembers(members, cubeName, type) {
     return members.map(memberRef => {
       const path = memberRef.member.split('.');
-      const resolvedMember = this.symbols[path[path.length - 2]]?.cubeObj()?.[type]?.[path[path.length - 1]];
+      const resolvedMember = this.getResolvedMember(type, path[path.length - 2], path[path.length - 1]);
       if (!resolvedMember) {
         throw new Error(`Can't resolve '${memberRef.member}' while generating include members`);
       }
@@ -404,6 +432,11 @@ export class CubeSymbols {
           meta: resolvedMember.meta,
           description: resolvedMember.description,
         };
+      } else if (type === 'hierarchies') {
+        memberDefinition = {
+          title: resolvedMember.title,
+          levels: resolvedMember.levels,
+        };
       } else {
         throw new Error(`Unexpected member type: ${type}`);
       }
@@ -458,6 +491,7 @@ export class CubeSymbols {
           name
         );
       // eslint-disable-next-line no-underscore-dangle
+      // if (resolvedSymbol && resolvedSymbol._objectWithResolvedProperties) {
       if (resolvedSymbol._objectWithResolvedProperties) {
         return resolvedSymbol;
       }

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js

@@ -106,7 +106,15 @@ export class CubeToMetaTransformer {
           })),
           R.toPairs
         )(cube.segments || {}),
-        hierarchies: cube.hierarchies || []
+        hierarchies: (cube.hierarchies || []).map((it) => ({
+          ...it,
+          public: it.public ?? true,
+          name: `${cube.name}.${it.name}`,
+        })),
+        folders: (cube.folders || []).map((it) => ({
+          name: it.name,
+          members: it.includes.map(member => `${cube.name}.${member.name}`),
+        })),
       },
     };
   }

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -745,17 +745,25 @@ const baseSchema = {
   )),
   segments: SegmentsSchema,
   preAggregations: PreAggregationsAlternatives,
-  hierarchies: Joi.array().items(Joi.object().keys({
+  folders: Joi.array().items(Joi.object().keys({
     name: Joi.string().required(),
-    title: Joi.string(),
-    levels: Joi.func()
+    includes: Joi.alternatives([
+      Joi.string().valid('*'),
+      Joi.array().items(Joi.string().required())
+    ]).required(),
   })),
   accessPolicy: Joi.array().items(RolePolicySchema.required()),
 };
 
 const cubeSchema = inherit(baseSchema, {
   sql: Joi.func(),
   sqlTable: Joi.func(),
+  hierarchies: Joi.array().items(Joi.object().keys({
+    name: identifier,
+    title: Joi.string(),
+    public: Joi.boolean().strict(),
+    levels: Joi.func()
+  })),
 }).xor('sql', 'sqlTable').messages({
   'object.xor': 'You must use either sql or sqlTable within a model, but not both'
 });

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -131,6 +131,7 @@ export class YamlCompiler {
     cubeObj.segments = this.yamlArrayToObj(cubeObj.segments || [], 'segment', errorsReport);
     cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations || [], 'preAggregation', errorsReport);
     cubeObj.joins = this.yamlArrayToObj(cubeObj.joins || [], 'join', errorsReport);
+    // cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport);
 
     return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport);
   }

packages/cubejs-schema-compiler/test/unit/__snapshots__/views.test.ts.snap

@@ -40,6 +40,7 @@ Object {
       "type": "number",
     },
   ],
+  "folders": Array [],
   "hierarchies": Array [],
   "isVisible": true,
   "measures": Array [
@@ -118,6 +119,7 @@ Object {
       "type": "number",
     },
   ],
+  "folders": Array [],
   "hierarchies": Array [],
   "isVisible": true,
   "measures": Array [
@@ -230,6 +232,7 @@ Object {
       "type": "number",
     },
   ],
+  "folders": Array [],
   "hierarchies": Array [],
   "isVisible": true,
   "measures": Array [
@@ -308,6 +311,7 @@ Object {
       "type": "number",
     },
   ],
+  "folders": Array [],
   "hierarchies": Array [],
   "isVisible": true,
   "measures": Array [