feat(schema-compiler): Introduce support for sqlTable (syntactic sugar) (#6360)

ovr · web-flow · commit c73b368ca207 · 2023-03-28T23:35:24.000+03:00
diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js
@@ -1418,11 +1418,18 @@ class BaseQuery {
       }
       return this.preAggregations.originalSqlPreAggregationTable(foundPreAggregation);
     }
-    const evaluatedSql = this.evaluateSql(cube, this.cubeEvaluator.cubeFromPath(cube).sql);
+
+    const fromPath = this.cubeEvaluator.cubeFromPath(cube);
+    if (fromPath.sqlTable) {
+      return this.evaluateSql(cube, fromPath.sqlTable);
+    }
+
+    const evaluatedSql = this.evaluateSql(cube, fromPath.sql);
     const selectAsterisk = evaluatedSql.match(/^\s*select\s+\*\s+from\s+([a-zA-Z0-9_\-`".*]+)\s*$/i);
     if (selectAsterisk) {
       return selectAsterisk[1];
     }
+
     return `(${evaluatedSql})`;
   }
 
diff --git a/packages/cubejs-schema-compiler/src/adapter/index.ts b/packages/cubejs-schema-compiler/src/adapter/index.ts
@@ -15,6 +15,7 @@ export * from './CubeStoreQuery';
 // Base queries that can be re-used across different drivers
 export * from './MysqlQuery';
 export * from './PostgresQuery';
+export * from './MssqlQuery';
 
 // Candidates to move from this package to drivers packages
 // export * from './PrestodbQuery';
diff --git a/packages/cubejs-schema-compiler/src/compiler/CompilerCache.ts b/packages/cubejs-schema-compiler/src/compiler/CompilerCache.ts
@@ -2,20 +2,29 @@ import LRUCache from 'lru-cache';
 import { QueryCache } from '../adapter/QueryCache';
 
 export class CompilerCache extends QueryCache {
-  constructor({ maxQueryCacheSize, maxQueryCacheAge }) {
+  protected readonly queryCache: LRUCache<string, QueryCache>;
+
+  public constructor({ maxQueryCacheSize, maxQueryCacheAge }) {
     super();
+
     this.queryCache = new LRUCache({
       max: maxQueryCacheSize || 10000,
       maxAge: (maxQueryCacheAge * 1000) || 1000 * 60 * 10,
       updateAgeOnGet: true
     });
   }
 
-  getQueryCache(key) {
+  public getQueryCache(key: unknown): QueryCache {
     const keyString = JSON.stringify(key);
-    if (!this.queryCache.get(keyString)) {
-      this.queryCache.set(keyString, new QueryCache());
+
+    const exist = this.queryCache.get(keyString);
+    if (exist) {
+      return exist;
     }
-    return this.queryCache.get(keyString);
+
+    const result = new QueryCache();
+    this.queryCache.set(keyString, result);
+
+    return result;
   }
 }
diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js
@@ -97,8 +97,9 @@ export class CubeEvaluator extends CubeSymbols {
   transformMembers(members, cube, errorReporter) {
     members = members || {};
     for (const memberName of Object.keys(members)) {
-      const member = members[memberName];
       let ownedByCube = true;
+
+      const member = members[memberName];
       if (member.sql && !member.subQuery) {
         const funcArgs = this.funcArguments(member.sql);
         const cubeReferences = this.collectUsedCubeReferences(cube.name, member.sql);
@@ -112,9 +113,11 @@ export class CubeEvaluator extends CubeSymbols {
           errorReporter.error(`Member '${cube.name}.${memberName}' references foreign cubes: ${foreignCubes.join(', ')}. Please split and move this definition to corresponding cubes.`);
         }
       }
+
       if (ownedByCube && cube.isView) {
         errorReporter.error(`View '${cube.name}' defines own member '${cube.name}.${memberName}'. Please move this member definition to one of the cubes.`);
       }
+
       members[memberName] = { ...members[memberName], ownedByCube };
     }
   }
diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js
@@ -450,17 +450,8 @@ const MeasuresSchema = Joi.object().pattern(identifierRegex, Joi.alternatives().
  * and update CubePropContextTranspiler.transpiledFieldsPatterns
  **************************** */
 
-const cubeSchema = Joi.object().keys({
+const baseSchema = {
   name: identifier,
-  sql: Joi.alternatives().conditional(
-    Joi.ref('..isView'), [
-      {
-        is: true,
-        then: Joi.forbidden(),
-        otherwise: Joi.func().required()
-      }
-    ]
-  ),
   refreshKey: CubeRefreshKeySchema,
   fileName: Joi.string().required(),
   extends: Joi.func(),
@@ -470,7 +461,6 @@ const cubeSchema = Joi.object().keys({
   dataSource: Joi.string(),
   description: Joi.string(),
   rewriteQueries: Joi.boolean().strict(),
-  isView: Joi.boolean().strict(),
   shown: Joi.boolean().strict(),
   joins: Joi.object().pattern(identifierRegex, Joi.object().keys({
     sql: Joi.func().required(),
@@ -521,6 +511,17 @@ const cubeSchema = Joi.object().keys({
   preAggregations: PreAggregationsAlternatives,
   includes: Joi.func(),
   excludes: Joi.func(),
+};
+
+const cubeSchema = inherit(baseSchema, {
+  sql: Joi.func(),
+  sqlTable: Joi.func(),
+}).xor('sql', 'sqlTable').messages({
+  'object.xor': 'You must use either sql or sqlTable within a model, but not both'
+});
+
+const viewSchema = inherit(baseSchema, {
+  isView: Joi.boolean().strict(),
 });
 
 function formatErrorMessageFromDetails(explain, d) {
@@ -578,7 +579,7 @@ function collectFunctionFieldsPatterns(patterns, path, o) {
 
 export function functionFieldsPatterns() {
   const functionPatterns = new Set();
-  collectFunctionFieldsPatterns(functionPatterns, '', cubeSchema);
+  collectFunctionFieldsPatterns(functionPatterns, '', { ...cubeSchema, ...viewSchema });
   return Array.from(functionPatterns);
 }
 
@@ -595,7 +596,7 @@ export class CubeValidator {
   }
 
   validate(cube, errorReporter) {
-    const result = cubeSchema.validate(cube);
+    const result = cube.isView ? viewSchema.validate(cube) : cubeSchema.validate(cube);
 
     if (result.error != null) {
       errorReporter.error(formatErrorMessage(result.error), result.error);
diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js
@@ -181,6 +181,7 @@ export class DataSchemaCompiler {
     if (err) {
       errorsReport.error(err.toString());
     }
+
     try {
       vm.runInNewContext(file.content, {
         view: (name, cube) => (
diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts
@@ -94,7 +94,7 @@ export class YamlCompiler {
       for (const p of transpiledFieldsPatterns) {
         const fullPath = propertyPath.join('.');
         if (fullPath.match(p)) {
-          if (typeof obj === 'string' && propertyPath[propertyPath.length - 1] === 'sql') {
+          if (typeof obj === 'string' && ['sql', 'sqlTable'].includes(propertyPath[propertyPath.length - 1])) {
             return this.parsePythonIntoArrowFunction(`f"${this.escapeDoubleQuotes(obj)}"`, cubeName, obj, errorsReport);
           } else if (typeof obj === 'string') {
             return this.parsePythonIntoArrowFunction(obj, cubeName, obj, errorsReport);
diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts
@@ -10,6 +10,7 @@ import type { CubeDictionary } from '../CubeDictionary';
 export const transpiledFieldsPatterns: Array<RegExp> = [
   /\.sql$/,
   /sql$/,
+  /sqlTable$/,
   /^measures\.[_a-zA-Z][_a-zA-Z0-9]*\.(drillMemberReferences|drillMembers)$/,
   /^preAggregations\.[_a-zA-Z][_a-zA-Z0-9]*\.indexes\.[_a-zA-Z][_a-zA-Z0-9]*\.columns$/,
   /^preAggregations\.[_a-zA-Z][_a-zA-Z0-9]*\.(timeDimensionReference|timeDimension|segments|dimensions|measures|rollups|segmentReferences|dimensionReferences|measureReferences|rollupReferences)$/,
diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts
@@ -1,13 +1,53 @@
 import moment from 'moment-timezone';
-import { UserError } from '../../src/compiler/UserError';
-import { PostgresQuery } from '../../src/adapter/PostgresQuery';
-import { prepareCompiler } from './PrepareCompiler';
-import { MssqlQuery } from '../../src/adapter/MssqlQuery';
-import { BaseQuery } from '../../src';
-import { createCubeSchema, createJoinedCubesSchema } from './utils';
+import { BaseQuery, PostgresQuery, MssqlQuery, UserError } from '../../src';
+import { prepareCompiler, prepareYamlCompiler } from './PrepareCompiler';
+import { createCubeSchema, createCubeSchemaYaml, createJoinedCubesSchema } from './utils';
 
 describe('SQL Generation', () => {
-  describe('Common', () => {
+  describe('Common - Yaml - syntax sugar', () => {
+    const compilers = /** @type Compilers */ prepareYamlCompiler(
+      createCubeSchemaYaml({ name: 'cards', sqlTable: 'card_tbl' })
+    );
+
+    it('Simple query', async () => {
+      await compilers.compiler.compile();
+
+      const query = new PostgresQuery(compilers, {
+        measures: [
+          'cards.count'
+        ],
+        timeDimensions: [],
+        filters: [],
+      });
+      const queryAndParams = query.buildSqlAndParams();
+      expect(queryAndParams[0]).toContain('card_tbl');
+    });
+  });
+
+  describe('Common - JS - syntax sugar', () => {
+    const compilers = /** @type Compilers */ prepareCompiler(
+      createCubeSchema({
+        name: 'cards',
+        sqlTable: 'card_tbl'
+      })
+    );
+
+    it('Simple query', async () => {
+      await compilers.compiler.compile();
+
+      const query = new PostgresQuery(compilers, {
+        measures: [
+          'cards.count'
+        ],
+        timeDimensions: [],
+        filters: [],
+      });
+      const queryAndParams = query.buildSqlAndParams();
+      expect(queryAndParams[0]).toContain('card_tbl');
+    });
+  });
+
+  describe('Common - JS', () => {
     const compilers = /** @type Compilers */ prepareCompiler(
       createCubeSchema({
         name: 'cards',
diff --git a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts
@@ -15,6 +15,77 @@ describe('Cube Validation', () => {
     console.log('CubePropContextTranspiler.transpiledFieldsPatterns =', transpiledFieldsPatterns);
   });
 
+  it('cube defined with sql - correct', async () => {
+    const cubeValidator = new CubeValidator(new CubeSymbols());
+    const cube = {
+      name: 'name',
+      sql: () => 'SELECT * FROM public.Users',
+      fileName: 'fileName',
+    };
+
+    const validationResult = cubeValidator.validate(cube, {
+      error: (message, e) => {
+        console.log(message);
+      }
+    });
+
+    expect(validationResult.error).toBeFalsy();
+  });
+
+  it('cube defined with sqlTable - correct', async () => {
+    const cubeValidator = new CubeValidator(new CubeSymbols());
+    const cube = {
+      name: 'name',
+      sqlTable: () => 'public.Users',
+      fileName: 'fileName',
+    };
+
+    const validationResult = cubeValidator.validate(cube, {
+      error: (message, e) => {
+        console.log(message);
+      }
+    });
+
+    expect(validationResult.error).toBeFalsy();
+  });
+
+  it('cube defined with sql and sqlTable - fail', async () => {
+    const cubeValidator = new CubeValidator(new CubeSymbols());
+    const cube = {
+      name: 'name',
+      sql: () => 'SELECT * FROM public.Users',
+      sqlTable: () => 'public.Users',
+      fileName: 'fileName',
+    };
+
+    const validationResult = cubeValidator.validate(cube, {
+      error: (message, e) => {
+        console.log(message);
+        expect(message).toContain('You must use either sql or sqlTable within a model, but not both');
+      }
+    });
+
+    expect(validationResult.error).toBeTruthy();
+  });
+
+  it('view defined by includes - correct', async () => {
+    const cubeValidator = new CubeValidator(new CubeSymbols());
+    const cube = {
+      name: 'name',
+      // it's a hidden field which we use internally
+      isView: true,
+      fileName: 'fileName',
+    };
+
+    const validationResult = cubeValidator.validate(cube, {
+      error: (message, e) => {
+        console.log(message);
+      }
+    });
+
+    expect(validationResult.error).toBeFalsy();
+  });
+
   it('refreshKey alternatives', async () => {
     const cubeValidator = new CubeValidator(new CubeSymbols());
     const cube = {
diff --git a/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts
@@ -1,11 +1,9 @@
-import { CubeValidator } from '../../src/compiler/CubeValidator';
-import { CubeSymbols } from '../../src/compiler/CubeSymbols';
 import { prepareCompiler } from './PrepareCompiler';
 
 describe('pre-aggregations', () => {
   it('rollupJoin scheduledRefresh', async () => {
     process.env.CUBEJS_SCHEDULED_REFRESH_DEFAULT = 'true';
-    const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(
+    const { compiler, cubeEvaluator } = prepareCompiler(
       `
         cube(\`Users\`, {
           sql: \`SELECT * FROM public.users\`,
diff --git a/packages/cubejs-schema-compiler/test/unit/utils.ts b/packages/cubejs-schema-compiler/test/unit/utils.ts
@@ -1,21 +1,15 @@
 interface CreateCubeSchemaOptions {
   name: string,
+  sqlTable?: string,
   refreshKey?: string,
   preAggregations?: string,
 }
 
-/**
- * Returns test cube schema based on incoming parameters.
- * @param {CreateCubeSchemaOptions} param
- * @returns {string}
- */
-export function createCubeSchema({ name, refreshKey = '', preAggregations = '' }: CreateCubeSchemaOptions) {
+export function createCubeSchema({ name, refreshKey = '', preAggregations = '', sqlTable }: CreateCubeSchemaOptions): string {
   return ` 
     cube('${name}', {
-        sql: \`
-        select * from cards
-        \`,
-        
+        ${sqlTable ? `sqlTable: \`${sqlTable}\`` : 'sql: `select * from cards`'},
+
         ${refreshKey}
    
         measures: {
@@ -55,11 +49,39 @@ export function createCubeSchema({ name, refreshKey = '', preAggregations = '' }
   `;
 }
 
+export function createCubeSchemaYaml({ name, sqlTable }: CreateCubeSchemaOptions): string {
+  return ` 
+    cubes:
+      - name: ${name}
+        sql_table: ${sqlTable}
+    
+        measures:
+          - name: count
+            type: count
+          - name: sum
+            type: sum
+            sql: amount
+          - name: min
+            sql: amount
+            type: min
+          - name: max
+            sql: amount
+            type: max
+        dimensions:
+          - name: id
+            sql: id
+            type: number
+            primary_key: true
+          - name: createdAt
+            sql: created_at
+            type: time
+  `;
+}
+
 /**
  * Returns joined test cubes schema. Schema looks like: A -< B -< C >- D >- E.
  * The original data set can be found under the link.
  * {@link https://docs.google.com/spreadsheets/d/1BNDpA7x4JLhlvvPdrQIC0c0PH4xZhdRrEFfXdRW1j4U/edit?usp=sharing|Dataset}
- * @returns {string}
  */
 export function createJoinedCubesSchema(): string {
   return `

Original file line number	Diff line number	Diff line change
`@@ -1418,11 +1418,18 @@ class BaseQuery {`
`1418`	`1418`	`}`
`1419`	`1419`	`return this.preAggregations.originalSqlPreAggregationTable(foundPreAggregation);`
`1420`	`1420`	`}`
`1421`		`- const evaluatedSql = this.evaluateSql(cube, this.cubeEvaluator.cubeFromPath(cube).sql);`
	`1421`	`+`
	`1422`	`+ const fromPath = this.cubeEvaluator.cubeFromPath(cube);`
	`1423`	`+ if (fromPath.sqlTable) {`
	`1424`	`+ return this.evaluateSql(cube, fromPath.sqlTable);`
	`1425`	`+ }`
	`1426`	`+`
	`1427`	`+ const evaluatedSql = this.evaluateSql(cube, fromPath.sql);`
`1422`	`1428`	const selectAsterisk = evaluatedSql.match(/^\sselect\s+\\s+from\s+([a-zA-Z0-9_\-`".]+)\s$/i);
`1423`	`1429`	`if (selectAsterisk) {`
`1424`	`1430`	`return selectAsterisk[1];`
`1425`	`1431`	`}`
	`1432`	`+`
`1426`	`1433`	return `(${evaluatedSql})`;
`1427`	`1434`	`}`
`1428`	`1435`
Original file line number	Diff line number	Diff line change
`@@ -97,8 +97,9 @@ export class CubeEvaluator extends CubeSymbols {`
`97`	`97`	`transformMembers(members, cube, errorReporter) {`
`98`	`98`	`members = members \|\| {};`
`99`	`99`	`for (const memberName of Object.keys(members)) {`
`100`		`- const member = members[memberName];`
`101`	`100`	`let ownedByCube = true;`
	`101`	`+`
	`102`	`+ const member = members[memberName];`
`102`	`103`	`if (member.sql && !member.subQuery) {`
`103`	`104`	`const funcArgs = this.funcArguments(member.sql);`
`104`	`105`	`const cubeReferences = this.collectUsedCubeReferences(cube.name, member.sql);`
`@@ -112,9 +113,11 @@ export class CubeEvaluator extends CubeSymbols {`
`112`	`113`	errorReporter.error(`Member '${cube.name}.${memberName}' references foreign cubes: ${foreignCubes.join(', ')}. Please split and move this definition to corresponding cubes.`);
`113`	`114`	`}`
`114`	`115`	`}`
	`116`	`+`
`115`	`117`	`if (ownedByCube && cube.isView) {`
`116`	`118`	errorReporter.error(`View '${cube.name}' defines own member '${cube.name}.${memberName}'. Please move this member definition to one of the cubes.`);
`117`	`119`	`}`
	`120`	`+`
`118`	`121`	`members[memberName] = { ...members[memberName], ownedByCube };`
`119`	`122`	`}`
`120`	`123`	`}`
Original file line number	Diff line number	Diff line change
`@@ -181,6 +181,7 @@ export class DataSchemaCompiler {`
`181`	`181`	`if (err) {`
`182`	`182`	`errorsReport.error(err.toString());`
`183`	`183`	`}`
	`184`	`+`
`184`	`185`	`try {`
`185`	`186`	`vm.runInNewContext(file.content, {`
`186`	`187`	`view: (name, cube) => (`