fix(schema-compiler): Fix cubes inheritance (cube-js#9386)

* some improvements in transformYamlCubeObj()

* implement inheritance for preaggs & DAPs

* rename prepareCompiler → prepareJsCompiler() in tests

* add tests for inheritance

* hide sql prop from parent cube if needed

* add tests for sql override

* fix pre-aggs inheritance

* fix test

* remove unneeded

* add more tests for views and errors

* fix preaggs again

* mooooore tests :)

* more test coverage

* more tests for folders

* remove useless check

* just refactoring

* fix schema validation for extended cubes

* add tests

* update joi to the latest

* fix tests

* add tests for pre-aggs

* fix test

* fix tests

* fix snapshots after merge changes

Author: KSDaemon

packages/cubejs-api-gateway/package.json

@@ -40,7 +40,7 @@
     "graphql-tag": "^2.12.6",
     "http-proxy-middleware": "^3.0.0",
     "inflection": "^1.12.0",
-    "joi": "^17.8.3",
+    "joi": "^17.13.3",
     "jsonwebtoken": "^9.0.2",
     "jwk-to-pem": "^2.0.4",
     "moment": "^2.24.0",

packages/cubejs-schema-compiler/package.json

@@ -47,7 +47,7 @@
     "cron-parser": "^4.9.0",
     "humps": "^2.0.1",
     "inflection": "^1.12.0",
-    "joi": "^17.8.3",
+    "joi": "^17.13.3",
     "js-yaml": "^4.1.0",
     "lru-cache": "^11.1.0",
     "moment-timezone": "^0.5.46",

packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts

@@ -1,6 +1,6 @@
 import { UserError } from '../compiler/UserError';
 import type { BaseQuery } from './BaseQuery';
-import { MeasureDefinition } from "../compiler/CubeEvaluator";
+import { MeasureDefinition } from '../compiler/CubeEvaluator';
 
 export class BaseMeasure {
   public readonly expression: any;

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -256,25 +256,17 @@ export class CubeEvaluator extends CubeSymbols {
   }
 
   private prepareHierarchies(cube: any, errorReporter: ErrorReporter): void {
-    const uniqueHierarchyNames = new Set();
     if (Object.keys(cube.hierarchies).length) {
-      cube.evaluatedHierarchies = Object.entries(cube.hierarchies).map(([name, hierarchy]) => {
-        if (uniqueHierarchyNames.has(name)) {
-          errorReporter.error(`Duplicate hierarchy name '${name}' in cube '${cube.name}'`);
-        }
-        uniqueHierarchyNames.add(name);
-
-        return ({
-          name,
-          ...(typeof hierarchy === 'object' ? hierarchy : {}),
-          levels: this.evaluateReferences(
-            cube.name,
-            // @ts-ignore
-            hierarchy.levels,
-            { originalSorting: true }
-          )
-        });
-      });
+      cube.evaluatedHierarchies = Object.entries(cube.hierarchies).map(([name, hierarchy]) => ({
+        name,
+        ...(typeof hierarchy === 'object' ? hierarchy : {}),
+        levels: this.evaluateReferences(
+          cube.name,
+          // @ts-ignore
+          hierarchy.levels,
+          { originalSorting: true }
+        )
+      }));
     }
 
     if (cube.isView && (cube.includedMembers || []).length) {

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -14,13 +14,19 @@ export type ToString = { toString(): string };
 interface CubeDefinition {
   name: string;
   extends?: (...args: Array<unknown>) => { __cubeName: string };
+  sql?: string | (() => string);
+  // eslint-disable-next-line camelcase
+  sql_table?: string | (() => string);
+  sqlTable?: string | (() => string);
   measures?: Record<string, any>;
   dimensions?: Record<string, any>;
   segments?: Record<string, any>;
   hierarchies?: Record<string, any>;
   preAggregations?: Record<string, any>;
+  // eslint-disable-next-line camelcase
+  pre_aggregations?: Record<string, any>;
   joins?: Record<string, any>;
-  accessPolicy?: Record<string, any>;
+  accessPolicy?: any[];
   includes?: any;
   excludes?: any;
   cubes?: any;
@@ -36,7 +42,7 @@ interface SplitViews {
 const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/;
 export const CONTEXT_SYMBOLS = {
   SECURITY_CONTEXT: 'securityContext',
-  // SECURITY_CONTEXT has been deprecated, however security_context (lowecase)
+  // SECURITY_CONTEXT has been deprecated, however security_context (lowercase)
   // is allowed in RBAC policies for query-time attribute matching
   security_context: 'securityContext',
   securityContext: 'securityContext',
@@ -106,10 +112,13 @@ export class CubeSymbols {
   }
 
   public createCube(cubeDefinition: CubeDefinition) {
+    let preAggregations: any;
+    let joins: any;
     let measures: any;
     let dimensions: any;
     let segments: any;
     let hierarchies: any;
+    let accessPolicy: any;
 
     const cubeObject = Object.assign({
       allDefinitions(type: string) {
@@ -122,6 +131,37 @@ export class CubeSymbols {
           return { ...cubeDefinition[type] };
         }
       },
+
+      get preAggregations() {
+        // For preAggregations order is important, and destructing parents cube pre-aggs first will lead to
+        // unexpected results, so we can not use common approach with allDefinitions('preAggregations') here.
+        if (!preAggregations) {
+          const parentPreAggregations = cubeDefinition.extends ? super.preAggregations : null;
+          // Unfortunately, cube is not camelized yet at this point :(
+          const localPreAggregations = cubeDefinition.preAggregations || cubeDefinition.pre_aggregations;
+
+          if (parentPreAggregations) {
+            preAggregations = { ...localPreAggregations, ...parentPreAggregations, ...localPreAggregations };
+          } else {
+            preAggregations = { ...localPreAggregations };
+          }
+        }
+        return preAggregations;
+      },
+      set preAggregations(v) {
+        // Dont allow to modify
+      },
+
+      get joins() {
+        if (!joins) {
+          joins = this.allDefinitions('joins');
+        }
+        return joins;
+      },
+      set joins(v) {
+        // Dont allow to modify
+      },
+
       get measures() {
         if (!measures) {
           measures = this.allDefinitions('measures');
@@ -159,18 +199,41 @@ export class CubeSymbols {
         return hierarchies;
       },
       set hierarchies(v) {
-        //
+        // Dont allow to modify
+      },
+
+      get accessPolicy() {
+        if (!accessPolicy) {
+          const parentAcls = cubeDefinition.extends ? super.accessPolicy : [];
+          accessPolicy = [...(parentAcls || []), ...(cubeDefinition.accessPolicy || [])];
+        }
+        // Schema validator expects accessPolicy to be not empty if defined
+        if (accessPolicy.length) {
+          return accessPolicy;
+        } else {
+          return undefined;
+        }
+      },
+      set accessPolicy(v) {
+        // Dont allow to modify
       }
     },
     cubeDefinition);
 
     if (cubeDefinition.extends) {
       const superCube = this.resolveSymbolsCall(cubeDefinition.extends, (name: string) => this.cubeReferenceProxy(name));
-      Object.setPrototypeOf(
-        cubeObject,
-        // eslint-disable-next-line no-underscore-dangle
-        superCube.__cubeName ? this.getCubeDefinition(superCube.__cubeName) : superCube
-      );
+      // eslint-disable-next-line no-underscore-dangle
+      const parentCube = superCube.__cubeName ? this.getCubeDefinition(superCube.__cubeName) : superCube;
+      Object.setPrototypeOf(cubeObject, parentCube);
+
+      // We have 2 different properties that are mutually exclusive: `sqlTable` & `sql`
+      // And if in extending cube one of them is defined - we need to hide the other from parent cube definition
+      // Unfortunately, cube is not camelized yet at this point :(
+      if ((cubeDefinition.sqlTable || cubeDefinition.sql_table) && parentCube.sql) {
+        cubeObject.sql = undefined;
+      } else if (cubeDefinition.sql && (parentCube.sqlTable || parentCube.sql_table)) {
+        cubeObject.sqlTable = undefined;
+      }
     }
 
     return cubeObject;