fix(databricks-jdbc-driver): Allow paths in s3/azure export buckets (cube-js#9365)

* fix(databricks-jdbc-driver): Allow paths in s3/azure export buckets

* Add tests for s3 export with prefix

* Add tests for azure export with prefix

* minor improvement

Author: KSDaemon

.github/workflows/drivers-tests.yml

@@ -207,7 +207,9 @@ jobs:
         clickhouse-export-bucket-s3
         databricks-jdbc
         databricks-jdbc-export-bucket-s3
+        databricks-jdbc-export-bucket-s3-prefix
         databricks-jdbc-export-bucket-azure
+        databricks-jdbc-export-bucket-azure-prefix
         redshift
         redshift-export-bucket-s3
         snowflake
@@ -231,7 +233,9 @@ jobs:
           - clickhouse-export-bucket-s3
           - databricks-jdbc
           - databricks-jdbc-export-bucket-s3
+          - databricks-jdbc-export-bucket-s3-prefix
           - databricks-jdbc-export-bucket-azure
+          - databricks-jdbc-export-bucket-azure-prefix
           - mssql
           - mysql
           - postgres

packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts

@@ -703,6 +703,9 @@ export class DatabricksDriver extends JDBCDriver {
     // wasbs://[email protected]
     // The extractors in BaseDriver expect just clean bucket name
     const url = new URL(this.config.exportBucket || '');
+    const prefix = url.pathname.slice(1);
+    const delimiter = (prefix && !prefix.endsWith('/')) ? '/' : '';
+    const objectSearchPrefix = `${prefix}${delimiter}${tableName}`;
 
     if (this.config.bucketType === 'azure') {
       const {
@@ -716,7 +719,7 @@ export class DatabricksDriver extends JDBCDriver {
         // Databricks uses different bucket address form, so we need to transform it
         // to the one understandable by extractFilesFromAzure implementation
         `${url.host}/${url.username}`,
-        tableName,
+        objectSearchPrefix,
       );
     } else if (this.config.bucketType === 's3') {
       return this.extractUnloadedFilesFromS3(
@@ -728,7 +731,7 @@ export class DatabricksDriver extends JDBCDriver {
           region: this.config.awsRegion || '',
         },
         url.host,
-        tableName,
+        objectSearchPrefix,
       );
     } else {
       throw new Error(`Unsupported export bucket type: ${

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -11,6 +11,17 @@
         }
       }
     },
+    "export-bucket-s3-prefix": {
+      "cube": {
+        "environment": {
+          "CUBEJS_DB_EXPORT_BUCKET_TYPE": "s3",
+          "CUBEJS_DB_EXPORT_BUCKET": "s3://databricks-drivers-tests-preaggs/testing_prefix/for_export_buckets/",
+          "CUBEJS_DB_EXPORT_BUCKET_AWS_KEY": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY}",
+          "CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET}",
+          "CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-east-1"
+        }
+      }
+    },
     "export-bucket-azure": {
       "cube": {
         "environment": {
@@ -19,6 +30,15 @@
           "CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY}"
         }
       }
+    },
+    "export-bucket-azure-prefix": {
+      "cube": {
+        "environment": {
+          "CUBEJS_DB_EXPORT_BUCKET_TYPE": "azure",
+          "CUBEJS_DB_EXPORT_BUCKET": "abfss://databricks-drivers-tests-preaggs@coreteamdevtest.dfs.core.windows.net/testing_prefix/for_export_buckets",
+          "CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY}"
+        }
+      }
     }
   },
   "cube": {

packages/cubejs-testing-drivers/package.json

@@ -29,7 +29,9 @@
     "databricks-jdbc-core": "yarn test-driver -i dist/test/databricks-jdbc-core.test.js",
     "databricks-jdbc-full": "yarn test-driver -i dist/test/databricks-jdbc-full.test.js",
     "databricks-jdbc-export-bucket-s3-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-s3-full.test.js",
+    "databricks-jdbc-export-bucket-s3-prefix-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-s3-prefix-full.test.js",
     "databricks-jdbc-export-bucket-azure-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-azure-full.test.js",
+    "databricks-jdbc-export-bucket-azure-prefix-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-azure-prefix-full.test.js",
     "mssql-driver": "yarn test-driver -i dist/test/mssql-driver.test.js",
     "mssql-core": "yarn test-driver -i dist/test/mssql-core.test.js",
     "mssql-full": "yarn test-driver -i dist/test/mssql-full.test.js",