feat(schema-compiler): generate join sql using dimension refs when po… (cube-js#9413)

* feat(schema-compiler): generate join sql using dimension refs when possible

* tests

* lint

* name fix

Author: vasilev-alex

packages/cubejs-backend-shared/src/helpers.ts

@@ -74,3 +74,7 @@ export async function streamToArray<T>(stream: Readable): Promise<T[]> {
 export async function oldStreamToArray<T>(stream: NodeJS.ReadableStream): Promise<T[]> {
   return streamToArray(new Readable().wrap(stream));
 }
+
+export function notEmpty<T>(value: T | null | undefined): value is T {
+  return value !== null && value !== undefined;
+}

packages/cubejs-schema-compiler/src/scaffolding/ScaffoldingSchema.ts

@@ -1,5 +1,6 @@
 import inflection from 'inflection';
 import R from 'ramda';
+import { notEmpty } from '@cubejs-backend/shared';
 import { UserError } from '../compiler';
 import { toSnakeCase } from './utils';
 
@@ -26,7 +27,7 @@ export type Dimension = {
 
 export type TableName = string | [string, string];
 
-type JoinRelationship = 'hasOne' | 'hasMany' | 'belongsTo';
+export type JoinRelationship = 'hasOne' | 'hasMany' | 'belongsTo';
 
 type ColumnsToJoin = {
   cubeToJoin: string;
@@ -45,11 +46,13 @@ export type CubeDescriptorMember = {
   isPrimaryKey?: boolean;
 };
 
-type Join = {
+export type Join = {
   thisTableColumn: string;
+  thisTableColumnIncludedAsDimension?: boolean;
   tableName: TableName;
   cubeToJoin: string;
   columnToJoin: string;
+  columnToJoinIncludedAsDimension?: boolean;
   relationship: JoinRelationship;
 };
 
@@ -234,7 +237,7 @@ export class ScaffoldingSchema {
     };
   }
 
-  protected parseTableName(tableName) {
+  protected parseTableName(tableName: TableName) {
     let schemaAndTable;
     if (Array.isArray(tableName)) {
       schemaAndTable = tableName;
@@ -247,7 +250,7 @@ export class ScaffoldingSchema {
     return schemaAndTable;
   }
 
-  protected dimensions(tableDefinition): Dimension[] {
+  protected dimensions(tableDefinition: ColumnData[]): Dimension[] {
     return this.dimensionColumns(tableDefinition).map(column => {
       const res: Dimension = {
         name: column.name,
@@ -280,7 +283,7 @@ export class ScaffoldingSchema {
     return !column.name.match(new RegExp(idRegex, 'i')) && !!MEASURE_DICTIONARY.find(word => this.fixCase(column.name).endsWith(word));
   }
 
-  protected dimensionColumns(tableDefinition: any) {
+  protected dimensionColumns(tableDefinition: ColumnData[]): Array<ColumnData & { columnType?: string }> {
     const dimensionColumns = tableDefinition.filter(
       column => !column.name.startsWith('_') && ['string', 'boolean'].includes(this.columnType(column)) ||
         column.attributes?.includes('primaryKey') ||
@@ -307,7 +310,7 @@ export class ScaffoldingSchema {
     return value.toLocaleLowerCase();
   }
 
-  protected joins(tableName: TableName, tableDefinition: ColumnData[]) {
+  protected joins(tableName: TableName, tableDefinition: ColumnData[]): Join[] {
     const cubeName = (name: string) => (this.options.snakeCase ? toSnakeCase(name) : inflection.camelize(name));
 
     return R.unnest(tableDefinition
@@ -336,7 +339,7 @@ export class ScaffoldingSchema {
             return null;
           }
 
-          columnsToJoin = tablesToJoin.map<any>(definition => {
+          columnsToJoin = tablesToJoin.map(definition => {
             if (tableName === definition.tableName) {
               return null;
             }
@@ -350,22 +353,22 @@ export class ScaffoldingSchema {
               columnToJoin: columnForJoin.name,
               tableName: definition.tableName
             };
-          }).filter(R.identity);
+          }).filter(notEmpty);
         }
 
         if (!columnsToJoin.length) {
           return null;
         }
 
-        return columnsToJoin.map(columnToJoin => ({
+        return columnsToJoin.map<Join>(columnToJoin => ({
           thisTableColumn: column.name,
           tableName: columnToJoin.tableName,
           cubeToJoin: columnToJoin.cubeToJoin,
           columnToJoin: columnToJoin.columnToJoin,
           relationship: 'belongsTo'
         }));
       })
-      .filter(R.identity)) as Join[];
+      .filter(notEmpty));
   }
 
   protected timeColumnIndex(column): number {

packages/cubejs-schema-compiler/src/scaffolding/formatters/BaseSchemaFormatter.ts

@@ -3,6 +3,7 @@ import { CubeMembers, SchemaContext } from '../ScaffoldingTemplate';
 import {
   CubeDescriptor,
   DatabaseSchema,
+  Dimension,
   MemberType,
   ScaffoldingSchema,
   TableName,
@@ -51,21 +52,39 @@ export abstract class BaseSchemaFormatter {
     tableNames: TableName[],
     schemaContext: SchemaContext = {}
   ): SchemaFile[] {
-    const schemaForTables = this.scaffoldingSchema.generateForTables(
+    const tableSchemas = this.scaffoldingSchema.generateForTables(
       tableNames.map((n) => this.scaffoldingSchema.resolveTableName(n))
     );
 
-    return schemaForTables.map((tableSchema) => ({
-      fileName: `${tableSchema.cube}.${this.fileExtension()}`,
-      content: this.renderFile(this.schemaDescriptorForTable(tableSchema, schemaContext)),
-    }));
+    return this.generateFilesByTableSchemas(tableSchemas, schemaContext);
   }
 
   public generateFilesByCubeDescriptors(
     cubeDescriptors: CubeDescriptor[],
     schemaContext: SchemaContext = {}
   ): SchemaFile[] {
-    return this.schemaForTablesByCubeDescriptors(cubeDescriptors).map((tableSchema) => ({
+    return this.generateFilesByTableSchemas(this.tableSchemasByCubeDescriptors(cubeDescriptors), schemaContext);
+  }
+
+  protected generateFilesByTableSchemas(tableSchemas: TableSchema[], schemaContext: SchemaContext = {}): SchemaFile[] {
+    const cubeToDimensionNamesMap = new Map(
+      tableSchemas.map(tableSchema => [tableSchema.cube, tableSchema.dimensions.map(d => d.name)])
+    );
+
+    tableSchemas = tableSchemas.map((tableSchema) => {
+      const updatedJoins = tableSchema.joins.map((join) => ({
+        ...join,
+        thisTableColumnIncludedAsDimension: !!cubeToDimensionNamesMap.get(tableSchema.cube)?.includes(join.thisTableColumn),
+        columnToJoinIncludedAsDimension: !!cubeToDimensionNamesMap.get(join.cubeToJoin)?.includes(join.columnToJoin)
+      }));
+
+      return {
+        ...tableSchema,
+        joins: updatedJoins
+      };
+    });
+
+    return tableSchemas.map((tableSchema) => ({
       fileName: `${tableSchema.cube}.${this.fileExtension()}`,
       content: this.renderFile(this.schemaDescriptorForTable(tableSchema, schemaContext)),
     }));
@@ -85,7 +104,7 @@ export abstract class BaseSchemaFormatter {
       : undefined;
   }
 
-  protected memberName(member) {
+  protected memberName(member: { title: string }) {
     const title = member.title.replace(/[^A-Za-z0-9]+/g, '_').toLowerCase();
 
     if (this.options.snakeCase) {
@@ -106,7 +125,7 @@ export abstract class BaseSchemaFormatter {
     return !!name.match(/^[a-z0-9_]+$/);
   }
 
-  public schemaDescriptorForTable(tableSchema: TableSchema, schemaContext: SchemaContext = {}) {
+  protected schemaDescriptorForTable(tableSchema: TableSchema, schemaContext: SchemaContext = {}) {
     let table = `${
       tableSchema.schema?.length ? `${this.escapeName(tableSchema.schema)}.` : ''
     }${this.escapeName(tableSchema.table)}`;
@@ -130,23 +149,39 @@ export abstract class BaseSchemaFormatter {
         sql: `SELECT * FROM ${table}`,
       };
 
-    return {
-      cube: tableSchema.cube,
-      ...sqlOption,
-      ...dataSourceProp,
+    // Try to use dimension refs if possible
+    // Source and target columns must be included in the respective cubes as dimensions
+    // {CUBE.dimension_name} = {other_cube.other_dimension_name}
+    // instead of
+    // {CUBE}.dimension_name = {other_cube}.other_dimension_name
+    const joins = tableSchema.joins
+      .map((j) => {
+        const thisTableColumnRef = j.thisTableColumnIncludedAsDimension
+          ? this.cubeReference(`CUBE.${this.memberName({ title: j.thisTableColumn })}`)
+          : `${this.cubeReference('CUBE')}.${this.escapeName(
+            j.thisTableColumn
+          )}`;
+        const columnToJoinRef = j.columnToJoinIncludedAsDimension
+          ? this.cubeReference(`${j.cubeToJoin}.${this.memberName({ title: j.columnToJoin })}`)
+          : `${this.cubeReference(j.cubeToJoin)}.${this.escapeName(j.columnToJoin)}`;
 
-      joins: tableSchema.joins
-        .map((j) => ({
+        return ({
           [j.cubeToJoin]: {
-            sql: `${this.cubeReference('CUBE')}.${this.escapeName(
-              j.thisTableColumn
-            )} = ${this.cubeReference(j.cubeToJoin)}.${this.escapeName(j.columnToJoin)}`,
+            sql: `${thisTableColumnRef} = ${columnToJoinRef}`,
             relationship: this.options.snakeCase
               ? (JOIN_RELATIONSHIP_MAP[j.relationship] ?? j.relationship)
               : j.relationship,
           },
-        }))
-        .reduce((a, b) => ({ ...a, ...b }), {}),
+        });
+      })
+      .reduce((a, b) => ({ ...a, ...b }), {});
+
+    return {
+      cube: tableSchema.cube,
+      ...sqlOption,
+      ...dataSourceProp,
+
+      joins,
       dimensions: tableSchema.dimensions.sort((a) => (a.isPrimaryKey ? -1 : 0))
         .map((m) => ({
           [this.memberName(m)]: {
@@ -189,7 +224,7 @@ export abstract class BaseSchemaFormatter {
     };
   }
 
-  protected schemaForTablesByCubeDescriptors(cubeDescriptors: CubeDescriptor[]) {
+  protected tableSchemasByCubeDescriptors(cubeDescriptors: CubeDescriptor[]) {
     const tableNames = cubeDescriptors.map(({ tableName }) => tableName);
     const generatedSchemaForTables = this.scaffoldingSchema.generateForTables(
       tableNames.map((n) => this.scaffoldingSchema.resolveTableName(n))