feat(databricks-jdbc-driver): Switch to the latest OSS Databricks JDBC driver

packages/cubejs-backend-shared/src/http-utils.ts

@@ -66,9 +66,11 @@ export async function streamWithProgress(
 type DownloadAndExtractFile = {
   showProgress: boolean;
   cwd: string;
+  noExtract?: boolean;
+  dstFileName?: string;
 };
 
-export async function downloadAndExtractFile(url: string, { cwd }: DownloadAndExtractFile) {
+export async function downloadAndExtractFile(url: string, { cwd, noExtract, dstFileName }: DownloadAndExtractFile) {
   const request = new Request(url, {
     headers: new Headers({
       'Content-Type': 'application/octet-stream',
@@ -99,7 +101,15 @@ export async function downloadAndExtractFile(url: string, { cwd }: DownloadAndEx
     });
   });
 
-  await decompress(savedFilePath, cwd);
+  if (noExtract) {
+    if (dstFileName) {
+      fs.copyFileSync(savedFilePath, path.resolve(path.join(cwd, dstFileName)));
+    } else {
+      fs.copyFileSync(savedFilePath, cwd);
+    }
+  } else {
+    await decompress(savedFilePath, cwd);
+  }
 
   try {
     fs.unlinkSync(savedFilePath);

packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts

@@ -4,27 +4,21 @@
  * @fileoverview The `DatabricksDriver` and related types declaration.
  */
 
+import { assertDataSource, getEnv, } from '@cubejs-backend/shared';
 import {
-  getEnv,
-  assertDataSource,
-} from '@cubejs-backend/shared';
-import {
+  DatabaseStructure,
   DriverCapabilities,
+  GenericDataBaseType,
   QueryColumnsResult,
   QueryOptions,
   QuerySchemasResult,
   QueryTablesResult,
-  UnloadOptions,
-  GenericDataBaseType,
   TableColumn,
-  DatabaseStructure,
+  UnloadOptions,
 } from '@cubejs-backend/base-driver';
-import {
-  JDBCDriver,
-  JDBCDriverConfiguration,
-} from '@cubejs-backend/jdbc-driver';
+import { JDBCDriver, JDBCDriverConfiguration, } from '@cubejs-backend/jdbc-driver';
 import { DatabricksQuery } from './DatabricksQuery';
-import { resolveJDBCDriver, extractUidFromJdbcUrl } from './helpers';
+import { extractUidFromJdbcUrl, resolveJDBCDriver } from './helpers';
 
 export type DatabricksDriverConfiguration = JDBCDriverConfiguration &
   {
@@ -132,7 +126,7 @@ export class DatabricksDriver extends JDBCDriver {
   /**
    * Show warning message flag.
    */
-  private showSparkProtocolWarn: boolean;
+  private readonly showSparkProtocolWarn: boolean;
 
   /**
    * Driver Configuration.
@@ -429,8 +423,7 @@ export class DatabricksDriver extends JDBCDriver {
         metadata[database] = {};
       }
 
-      const columns = await this.tableColumnTypes(`${database}.${tableName}`);
-      metadata[database][tableName] = columns;
+      metadata[database][tableName] = await this.tableColumnTypes(`${database}.${tableName}`);
     }));
 
     return metadata;

packages/cubejs-databricks-jdbc-driver/src/helpers.ts

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 
-import { downloadJDBCDriver } from './installer';
+import { downloadJDBCDriver, OSS_DRIVER_VERSION } from './installer';
 
 async function fileExistsOr(
   fsPath: string,
@@ -15,16 +15,16 @@ async function fileExistsOr(
 
 export async function resolveJDBCDriver(): Promise<string> {
   return fileExistsOr(
-    path.join(process.cwd(), 'DatabricksJDBC42.jar'),
+    path.join(process.cwd(), `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`),
     async () => fileExistsOr(
-      path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'),
+      path.join(__dirname, '..', 'download', `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`),
       async () => {
         const pathOrNull = await downloadJDBCDriver();
         if (pathOrNull) {
           return pathOrNull;
         }
         throw new Error(
-          'Please download and place DatabricksJDBC42.jar inside your ' +
+          `Please download and place databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar inside your ` +
           'project directory'
         );
       }

packages/cubejs-databricks-jdbc-driver/src/installer.ts

@@ -1,6 +1,8 @@
 import path from 'path';
 import { downloadAndExtractFile, getEnv } from '@cubejs-backend/shared';
 
+export const OSS_DRIVER_VERSION = '1.0.2';
+
 function acceptedByEnv() {
   const acceptStatus = getEnv('databrickAcceptPolicy');
   if (acceptStatus) {
@@ -15,23 +17,32 @@ function acceptedByEnv() {
   return acceptStatus;
 }
 
+/**
+ * In the beginning of 2025 Databricks released their open-source version of JDBC driver and encourage
+ * all users to migrate to it as company plans to focus on improving and evolving it over legacy simba driver.
+ * More info about OSS Driver could be found at https://docs.databricks.com/aws/en/integrations/jdbc/oss
+ * As of March 2025 To use the Databricks JDBC Driver (OSS), the following requirements must be met:
+ * Java Runtime Environment (JRE) 11.0 or above. CI testing is supported on JRE 11, 17, and 21.
+ */
 export async function downloadJDBCDriver(): Promise<string | null> {
   const driverAccepted = acceptedByEnv();
 
   if (driverAccepted) {
-    console.log('Downloading DatabricksJDBC42-2.6.29.1051');
+    console.log(`Downloading databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`);
 
     await downloadAndExtractFile(
-      'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/DatabricksJDBC42-2.6.29.1051.zip',
+      `https://repo1.maven.org/maven2/com/databricks/databricks-jdbc/${OSS_DRIVER_VERSION}-oss/databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`,
       {
         showProgress: true,
         cwd: path.resolve(path.join(__dirname, '..', 'download')),
+        noExtract: true,
+        dstFileName: `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`,
       }
     );
 
-    console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/docs/release-notes.txt');
+    console.log(`Release notes: https://mvnrepository.com/artifact/com.databricks/databricks-jdbc/${OSS_DRIVER_VERSION}-oss`);
 
-    return path.resolve(path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'));
+    return path.resolve(path.join(__dirname, '..', 'download', `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`));
   }
 
   return null;