feat(api-gateway, server-core): Added dataSources method (#5789)

Author: wolfeeeg

packages/cubejs-api-gateway/src/gateway.ts

@@ -366,6 +366,18 @@ class ApiGateway {
           });
         }
       );
+      
+      app.get(
+        '/cubejs-system/v1/data-sources',
+        jsonParser,
+        systemMiddlewares,
+        async (req: Request, res: Response) => {
+          await this.dataSources({
+            context: req.context,
+            res: this.resToResultFn(res),
+          });
+        }
+      );
     }
 
     app.get('/readyz', guestMiddlewares, cachedHandler(this.readiness));
@@ -1005,6 +1017,24 @@ class ApiGateway {
       });
     }
   }
+  
+  protected async dataSources({ context, res }: { context?: RequestContext, res: ResponseResultFn }) {
+    const requestStarted = new Date();
+
+    try {
+      const orchestratorApi = await this.getAdapterApi(context);
+      const { dataSources } = await this.getCompilerApi(context).dataSources(orchestratorApi);
+
+      res({ dataSources });
+    } catch (e) {
+      this.handleError({
+        e,
+        context,
+        res,
+        requestStarted,
+      });
+    }
+  }
 
   protected async sqlRunner({ query, context, res }: QueryRequest) {
     const requestStarted = new Date();

packages/cubejs-api-gateway/test/index.test.ts

@@ -592,6 +592,7 @@ describe('API Gateway', () => {
           }
         }]
       },
+      { route: 'data-sources', successResult: { dataSources: ['default'] } },
     ];
 
     testConfigs.forEach((config) => {

packages/cubejs-api-gateway/test/mocks.ts

@@ -139,7 +139,13 @@ export const compilerApi = jest.fn().mockImplementation(() => ({
 
   async preAggregations() {
     return preAggregationsResultFactory();
-  }
+  },
+
+  async dataSources() {
+    return {
+      dataSources: ['default']
+    };
+  },
 }));
 
 export class RefreshSchedulerMock {

packages/cubejs-server-core/src/core/CompilerApi.js

@@ -196,6 +196,54 @@ export class CompilerApi {
     };
   }
 
+  async dataSources(orchestratorApi) {
+    let compilerVersion = (
+      this.schemaVersion && await this.schemaVersion() ||
+      'default_schema_version'
+    );
+
+    if (typeof compilerVersion === 'object') {
+      compilerVersion = JSON.stringify(compilerVersion);
+    }
+
+    let { compilers } = this;
+    if (!compilers || this.compilerVersion !== compilerVersion) {
+      compilers = await compile(this.repository, {
+        allowNodeRequire: this.allowNodeRequire,
+        compileContext: this.compileContext,
+        allowJsDuplicatePropsInSchema: this.allowJsDuplicatePropsInSchema,
+        standalone: this.standalone,
+      });
+    }
+
+    const { cubeEvaluator } = await compilers;
+
+    let dataSources = await Promise.all(
+      cubeEvaluator
+        .cubeNames()
+        .map(
+          async (cube) => cubeEvaluator.cubeFromPath(cube).dataSource ?? 'default'
+        )
+    );
+
+    dataSources = [...new Set(dataSources)];
+
+    dataSources = await Promise.all(
+      dataSources.map(async (dataSource) => {
+        try {
+          await orchestratorApi.driverFactory(dataSource);
+          return dataSource;
+        } catch (err) {
+          return null;
+        }
+      })
+    );
+
+    return {
+      dataSources: dataSources.filter((source) => source),
+    };
+  }
+
   canUsePreAggregationForTransformedQuery(transformedQuery, refs) {
     return PreAggregations.canUsePreAggregationForTransformedQueryFn(transformedQuery, refs);
   }

packages/cubejs-server-core/test/unit/index.test.ts

@@ -39,7 +39,7 @@ cube('Bar', {
       type: 'count'
     }
   },
-  
+
   dimensions: {
     time: {
       sql: 'timestamp',
@@ -57,6 +57,28 @@ const repositoryWithoutContent: SchemaFileRepository = {
   dataSchemaFiles: () => Promise.resolve([{ fileName: 'main.js', content: '' }]),
 };
 
+const repositoryWithDataSource: SchemaFileRepository = {
+  localPath: () => __dirname,
+  dataSchemaFiles: () => Promise.resolve([{ fileName: 'main.js', content: `
+cube('Bar', {
+  sql: 'select * from bar',
+  
+  measures: {
+    count: {
+      type: 'count'
+    }
+  },
+  dimensions: {
+    time: {
+      sql: 'timestamp',
+      type: 'time'
+    }
+  }, 
+  dataSource: 'main'
+});
+` }]),
+};
+
 describe('index.test', () => {
   beforeEach(() => {
     delete process.env.CUBEJS_EXT_DB_TYPE;
@@ -366,6 +388,49 @@ describe('index.test', () => {
       expect(metaConfigExtendedSpy).toHaveBeenCalled();
       metaConfigExtendedSpy.mockClear();
     });
+
+    test('CompilerApi dataSources default', async () => {
+      const dataSources = await compilerApi.dataSources({
+        driverFactory: jest.fn(async () => true)
+      });
+
+      expect(dataSources).toHaveProperty('dataSources');
+      expect(dataSources.dataSources).toEqual([]);
+    });
+  });
+  
+  describe('CompilerApi dataSources method', () => {
+    const logger = jest.fn(() => {});
+    const compilerApi = new CompilerApi(
+      repositoryWithDataSource,
+      async () => 'mysql',
+      { logger }
+    );
+
+    const dataSourcesSpy = jest.spyOn(compilerApi, 'dataSources');
+    test('CompilerApi dataSources', async () => {
+      const dataSources = await compilerApi.dataSources({
+        driverFactory: jest.fn(async () => true)
+      });
+
+      expect(dataSources).toHaveProperty('dataSources');
+      expect(dataSources.dataSources).toEqual(['main']);
+      expect(dataSourcesSpy).toHaveBeenCalled();
+      dataSourcesSpy.mockClear();
+    });
+
+    test('CompilerApi dataSources with driverFactory error', async () => {
+      const dataSources = await compilerApi.dataSources({
+        driverFactory: jest.fn(async () => {
+          throw new Error('Some driverFactory error');
+        })
+      });
+
+      expect(dataSources).toHaveProperty('dataSources');
+      expect(dataSources.dataSources).toEqual([]);
+      expect(dataSourcesSpy).toHaveBeenCalled();
+      dataSourcesSpy.mockClear();
+    });
   });
 
   test('Should create instance of CubejsServerCore, dbType from process.env.CUBEJS_DB_TYPE', () => {