Merge branch 'feature/gcs-export-bucket' of github.com:qiao-x/cube into qiao-x-feature/gcs-export-bucket

Author: KSDaemon

docs/pages/product/configuration/data-sources/databricks-jdbc.mdx

@@ -147,6 +147,17 @@ CUBEJS_DB_EXPORT_BUCKET_AZURE_CLIENT_ID=<AZURE_CLIENT_ID>
 CUBEJS_DB_EXPORT_BUCKET_AZURE_CLIENT_SECRET=<AZURE_CLIENT_SECRET>
 ```
 
+#### Google Cloud Storage
+
+To use Google Cloud Storage as an export bucket, follow [the Databricks guide on
+connecting to Google Cloud Storage][databricks-docs-uc-gcs].
+
+```dotenv
+CUBEJS_DB_EXPORT_BUCKET=gs://my-bucket-on-gcs
+CUBEJS_DB_EXPORT_BUCKET_TYPE=gcs
+CUBEJS_DB_EXPORT_GCS_CREDENTIALS=<BASE64_ENCODED_SERVICE_CREDENTIALS_JSON>
+```
+
 ## SSL/TLS
 
 Cube does not require any additional configuration to enable SSL/TLS for
@@ -173,6 +184,8 @@ bucket][self-preaggs-export-bucket] **must be** configured.
   https://docs.databricks.com/data/data-sources/azure/azure-storage.html
 [databricks-docs-uc-s3]:
   https://docs.databricks.com/en/connect/unity-catalog/index.html
+[databricks-docs-uc-gcs]:
+  https://docs.databricks.com/gcp/en/connect/unity-catalog/cloud-storage.html
 [databricks-docs-jdbc-url]:
   https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url
 [databricks-docs-pat]:

packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts

@@ -26,6 +26,8 @@ import {
 import { DatabricksQuery } from './DatabricksQuery';
 import { resolveJDBCDriver, extractUidFromJdbcUrl } from './helpers';
 
+const SUPPORTED_BUCKET_TYPES = ['s3', 'gcs', 'azure'];
+
 export type DatabricksDriverConfiguration = JDBCDriverConfiguration &
   {
     /**
@@ -103,6 +105,11 @@ export type DatabricksDriverConfiguration = JDBCDriverConfiguration &
      * Azure service principal client secret
      */
     azureClientSecret?: string,
+
+    /**
+     * GCS credentials JSON content
+     */
+    gcsCredentials?: string,
   };
 
 type ShowTableRow = {
@@ -209,7 +216,7 @@ export class DatabricksDriver extends JDBCDriver {
       // common export bucket config
       bucketType:
         conf?.bucketType ||
-        getEnv('dbExportBucketType', { supported: ['s3', 'azure'], dataSource }),
+        getEnv('dbExportBucketType', { supported: SUPPORTED_BUCKET_TYPES, dataSource }),
       exportBucket:
         conf?.exportBucket ||
         getEnv('dbExportBucket', { dataSource }),
@@ -246,6 +253,10 @@ export class DatabricksDriver extends JDBCDriver {
       azureClientSecret:
         conf?.azureClientSecret ||
         getEnv('dbExportBucketAzureClientSecret', { dataSource }),
+      // GCS credentials
+      gcsCredentials:
+        conf?.gcsCredentials ||
+        getEnv('dbExportGCSCredentials', { dataSource }),
     };
     if (config.readOnly === undefined) {
       // we can set readonly to true if there is no bucket config provided
@@ -643,7 +654,7 @@ export class DatabricksDriver extends JDBCDriver {
    * export bucket data.
    */
   public async unload(tableName: string, options: UnloadOptions) {
-    if (!['azure', 's3'].includes(this.config.bucketType as string)) {
+    if (!SUPPORTED_BUCKET_TYPES.includes(this.config.bucketType as string)) {
       throw new Error(`Unsupported export bucket type: ${
         this.config.bucketType
       }`);
@@ -733,6 +744,15 @@ export class DatabricksDriver extends JDBCDriver {
         url.host,
         objectSearchPrefix,
       );
+    } else if (this.config.bucketType === 'gcs') {
+      return this.extractFilesFromGCS(
+        { credentials: this.config.gcsCredentials },
+        url.host,
+        objectSearchPrefix+".csv",
+      ).then(files => files.filter(file => 
+        decodeURIComponent(new URL(file).pathname).endsWith('.csv') || 
+        decodeURIComponent(new URL(file).pathname).endsWith('.csv.gz')
+      ));
     } else {
       throw new Error(`Unsupported export bucket type: ${
         this.config.bucketType
@@ -769,6 +789,9 @@ export class DatabricksDriver extends JDBCDriver {
    *
    * `fs.s3a.access.key <aws-access-key>`
    * `fs.s3a.secret.key <aws-secret-key>`
+   * For Google cloud storage you can configure storage credentials and create an external location to access it
+   * (https://docs.databricks.com/gcp/en/connect/unity-catalog/cloud-storage/storage-credentials
+   * https://docs.databricks.com/gcp/en/connect/unity-catalog/cloud-storage/external-locations)
    */
   private async createExternalTableFromSql(tableFullName: string, sql: string, params: unknown[], columns: ColumnInfo[]) {
     let select = sql;