chore(base-driver,redshift-driver,bigquery-driver,materialize-driver): Speed up table schema retrospection

Speed up by one time inplace sorting in stead of a few and with new objects creation

Author: KSDaemon

packages/cubejs-base-driver/src/BaseDriver.ts

@@ -53,7 +53,7 @@ import {
   TableMemoryData,
   PrimaryKeysQueryResult,
   ForeignKeysQueryResult,
-  DatabaseStructure,
+  DatabaseStructure, InformationSchemaColumn,
 } from './driver.interface';
 
 /**
@@ -427,28 +427,43 @@ export abstract class BaseDriver implements DriverInterface {
     return false;
   }
 
-  protected informationColumnsSchemaReducer(result: any, i: any): DatabaseStructure {
-    let schema = (result[i.table_schema] || {});
-    const columns = (schema[i.table_name] || []);
+  protected informationColumnsSchemaReducer(result: DatabaseStructure, i: InformationSchemaColumn): DatabaseStructure {
+    if (!result[i.table_schema]) {
+      result[i.table_schema] = {};
+    }
+
+    if (!result[i.table_schema][i.table_name]) {
+      result[i.table_schema][i.table_name] = [];
+    }
 
-    columns.push({
+    result[i.table_schema][i.table_name].push({
       name: i.column_name,
       type: i.data_type,
       attributes: i.key_type ? ['primaryKey'] : []
     });
 
-    columns.sort();
-    schema[i.table_name] = columns;
-    schema = sortByKeys(schema);
-    result[i.table_schema] = schema;
+    return result;
+  }
 
-    return sortByKeys(result);
+  protected informationColumnsSchemaSorter(data: InformationSchemaColumn[]) {
+    return data
+      .map((i) => ({
+        ...i,
+        sortedKeyServiceField: `${i.table_schema}.${i.table_name}.${i.column_name}`,
+      }))
+      .sort((a, b) => a.sortedKeyServiceField.localeCompare(b.sortedKeyServiceField));
   }
 
-  public tablesSchema(): Promise<DatabaseStructure> {
+  public async tablesSchema(): Promise<DatabaseStructure> {
     const query = this.informationSchemaQuery();
+    const data: InformationSchemaColumn[] = await this.query(query, []);
+
+    if (!data.length) {
+      return {};
+    }
 
-    return this.query(query, []).then(data => data.reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {}));
+    const sortedData = this.informationColumnsSchemaSorter(data);
+    return sortedData.reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {});
   }
 
   // Extended version of tablesSchema containing primary and foreign keys

packages/cubejs-base-driver/src/driver.interface.ts

@@ -212,6 +212,18 @@ export type ForeignKeysQueryResult = {
   target_column: string
 };
 
+export type InformationSchemaColumn = {
+  // eslint-disable-next-line camelcase
+  table_schema: string;
+  // eslint-disable-next-line camelcase
+  table_name: string;
+  // eslint-disable-next-line camelcase
+  column_name: string;
+  // eslint-disable-next-line camelcase
+  data_type: string;
+  [key: string]: string
+};
+
 export type TableKeysFilter = {
   tableSchema: string,
   tableName: string[]

packages/cubejs-bigquery-driver/src/BigQueryDriver.ts

@@ -202,7 +202,7 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface {
 
       if (result.length) {
         return R.reduce(
-          this.informationColumnsSchemaReducer, {}, result[0]
+          this.informationColumnsSchemaReducer, {}, this.informationColumnsSchemaSorter(result[0])
         );
       }
 

packages/cubejs-materialize-driver/src/MaterializeDriver.ts

@@ -10,6 +10,7 @@ import {
   DatabaseStructure,
   DownloadTableMemoryData,
   IndexesSQL,
+  InformationSchemaColumn,
   StreamOptions,
   StreamTableDataWithTypes,
   TableStructure
@@ -173,11 +174,13 @@ export class MaterializeDriver extends PostgresDriver {
     return version.split(' ')[0];
   }
 
-  public async tablesSchema(): Promise<DatabaseStructure> {
+  public override async tablesSchema(): Promise<DatabaseStructure> {
     const version = await this.getMaterializeVersion();
     const query = this.informationSchemaQueryWithFilter(version);
+    const data: InformationSchemaColumn[] = await this.query(query, []);
+    const sortedData = this.informationColumnsSchemaSorter(data);
 
-    return this.query(query, []).then(data => data.reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {}));
+    return sortedData.reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {});
   }
 
   protected async* asyncFetcher<R extends unknown>(conn: PoolClient, cursorId: string): AsyncGenerator<R> {

packages/cubejs-redshift-driver/src/RedshiftDriver.ts

@@ -10,6 +10,7 @@ import {
   DatabaseStructure,
   DownloadTableCSVData,
   DriverCapabilities,
+  InformationSchemaColumn,
   QueryColumnsResult,
   QuerySchemasResult,
   QueryTablesResult,
@@ -137,7 +138,9 @@ export class RedshiftDriver extends PostgresDriver<RedshiftDriverConfiguration>
    */
   public override async tablesSchema(): Promise<DatabaseStructure> {
     const query = this.informationSchemaQuery();
-    const tablesSchema = await this.query(query, []).then(data => data.reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {}));
+    const data: InformationSchemaColumn[] = await this.query(query, []);
+    const tablesSchema = this.informationColumnsSchemaSorter(data)
+      .reduce<DatabaseStructure>(this.informationColumnsSchemaReducer, {});
 
     const allSchemas = await this.getSchemas();
     const externalSchemas = allSchemas.filter(s => !tablesSchema[s.schema_name]).map(s => s.schema_name);