Skip to content

Commit 29fdd61

Browse files
KSDaemonKSDaemon
authored
feat(databricks-jdbc-driver): Switch to the latest OSS Databricks JDBC driver (#9376)
* add noExtract flag to downloadAndExtractFile() * code polishment * switch to oss driver version * add dstFileName option to downloadAndExtractFile() * extract driver version to constant * remove databrickAcceptPolicy() flow as new OSS driver is Apache 2.0 Lic * fix postinstall
1 parent 78ac7b2 commit 29fdd61

File tree

8 files changed

+58
-116
lines changed

8 files changed

+58
-116
lines changed

‎packages/cubejs-backend-shared/src/env.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -952,9 +952,15 @@ const variables: Record<string, (...args: any) => any> = {
952952
* Accept Databricks policy flag. This environment variable doesn't
953953
* need to be split by the data source.
954954
*/
955-
databrickAcceptPolicy: () => (
956-
get('CUBEJS_DB_DATABRICKS_ACCEPT_POLICY').asBoolStrict()
957-
),
955+
databrickAcceptPolicy: () => {
956+
const val = get('CUBEJS_DB_DATABRICKS_ACCEPT_POLICY').asBoolStrict();
957+
958+
if (val !== undefined) {
959+
console.warn(
960+
'The CUBEJS_DB_DATABRICKS_ACCEPT_POLICY is not needed anymore. Please, remove it'
961+
);
962+
}
963+
},
958964

959965
/**
960966
* Databricks jdbc-connection url.

‎packages/cubejs-backend-shared/src/http-utils.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,11 @@ export async function streamWithProgress(
6666
type DownloadAndExtractFile = {
6767
showProgress: boolean;
6868
cwd: string;
69+
noExtract?: boolean;
70+
dstFileName?: string;
6971
};
7072

71-
export async function downloadAndExtractFile(url: string, { cwd }: DownloadAndExtractFile) {
73+
export async function downloadAndExtractFile(url: string, { cwd, noExtract, dstFileName }: DownloadAndExtractFile) {
7274
const request = new Request(url, {
7375
headers: new Headers({
7476
'Content-Type': 'application/octet-stream',
@@ -99,7 +101,15 @@ export async function downloadAndExtractFile(url: string, { cwd }: DownloadAndEx
99101
});
100102
});
101103

102-
await decompress(savedFilePath, cwd);
104+
if (noExtract) {
105+
if (dstFileName) {
106+
fs.copyFileSync(savedFilePath, path.resolve(path.join(cwd, dstFileName)));
107+
} else {
108+
fs.copyFileSync(savedFilePath, cwd);
109+
}
110+
} else {
111+
await decompress(savedFilePath, cwd);
112+
}
103113

104114
try {
105115
fs.unlinkSync(savedFilePath);

‎packages/cubejs-backend-shared/test/db_env_multi.test.ts‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1192,34 +1192,6 @@ describe('Multiple datasources', () => {
11921192
);
11931193
});
11941194

1195-
test('getEnv("databrickAcceptPolicy")', () => {
1196-
process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'true';
1197-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(true);
1198-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(true);
1199-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(true);
1200-
1201-
process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'false';
1202-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(false);
1203-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(false);
1204-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(false);
1205-
1206-
process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'wrong';
1207-
expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toThrow(
1208-
'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"'
1209-
);
1210-
expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toThrow(
1211-
'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"'
1212-
);
1213-
expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toThrow(
1214-
'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"'
1215-
);
1216-
1217-
delete process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY;
1218-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toBeUndefined();
1219-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toBeUndefined();
1220-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toBeUndefined();
1221-
});
1222-
12231195
test('getEnv("athenaAwsKey")', () => {
12241196
process.env.CUBEJS_AWS_KEY = 'default1';
12251197
process.env.CUBEJS_DS_POSTGRES_AWS_KEY = 'postgres1';

‎packages/cubejs-backend-shared/test/db_env_single.test.ts‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -760,34 +760,6 @@ describe('Single datasources', () => {
760760
expect(getEnv('databricksCatalog', { dataSource: 'wrong' })).toBeUndefined();
761761
});
762762

763-
test('getEnv("databrickAcceptPolicy")', () => {
764-
process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'true';
765-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(true);
766-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(true);
767-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(true);
768-
769-
process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'false';
770-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(false);
771-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(false);
772-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(false);
773-
774-
process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'wrong';
775-
expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toThrow(
776-
'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"'
777-
);
778-
expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toThrow(
779-
'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"'
780-
);
781-
expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toThrow(
782-
'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"'
783-
);
784-
785-
delete process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY;
786-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toBeUndefined();
787-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toBeUndefined();
788-
expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toBeUndefined();
789-
});
790-
791763
test('getEnv("athenaAwsKey")', () => {
792764
process.env.CUBEJS_AWS_KEY = 'default1';
793765
expect(getEnv('athenaAwsKey', { dataSource: 'default' })).toEqual('default1');

‎packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,21 @@
44
* @fileoverview The `DatabricksDriver` and related types declaration.
55
*/
66

7+
import { assertDataSource, getEnv, } from '@cubejs-backend/shared';
78
import {
8-
getEnv,
9-
assertDataSource,
10-
} from '@cubejs-backend/shared';
11-
import {
9+
DatabaseStructure,
1210
DriverCapabilities,
11+
GenericDataBaseType,
1312
QueryColumnsResult,
1413
QueryOptions,
1514
QuerySchemasResult,
1615
QueryTablesResult,
17-
UnloadOptions,
18-
GenericDataBaseType,
1916
TableColumn,
20-
DatabaseStructure,
17+
UnloadOptions,
2118
} from '@cubejs-backend/base-driver';
22-
import {
23-
JDBCDriver,
24-
JDBCDriverConfiguration,
25-
} from '@cubejs-backend/jdbc-driver';
19+
import { JDBCDriver, JDBCDriverConfiguration, } from '@cubejs-backend/jdbc-driver';
2620
import { DatabricksQuery } from './DatabricksQuery';
27-
import { resolveJDBCDriver, extractUidFromJdbcUrl } from './helpers';
21+
import { extractUidFromJdbcUrl, resolveJDBCDriver } from './helpers';
2822

2923
export type DatabricksDriverConfiguration = JDBCDriverConfiguration &
3024
{
@@ -132,7 +126,7 @@ export class DatabricksDriver extends JDBCDriver {
132126
/**
133127
* Show warning message flag.
134128
*/
135-
private showSparkProtocolWarn: boolean;
129+
private readonly showSparkProtocolWarn: boolean;
136130

137131
/**
138132
* Driver Configuration.
@@ -429,8 +423,7 @@ export class DatabricksDriver extends JDBCDriver {
429423
metadata[database] = {};
430424
}
431425

432-
const columns = await this.tableColumnTypes(`${database}.${tableName}`);
433-
metadata[database][tableName] = columns;
426+
metadata[database][tableName] = await this.tableColumnTypes(`${database}.${tableName}`);
434427
}));
435428

436429
return metadata;

‎packages/cubejs-databricks-jdbc-driver/src/helpers.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import fs from 'fs';
22
import path from 'path';
33

4-
import { downloadJDBCDriver } from './installer';
4+
import { downloadJDBCDriver, OSS_DRIVER_VERSION } from './installer';
55

66
async function fileExistsOr(
77
fsPath: string,
@@ -15,16 +15,16 @@ async function fileExistsOr(
1515

1616
export async function resolveJDBCDriver(): Promise<string> {
1717
return fileExistsOr(
18-
path.join(process.cwd(), 'DatabricksJDBC42.jar'),
18+
path.join(process.cwd(), `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`),
1919
async () => fileExistsOr(
20-
path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'),
20+
path.join(__dirname, '..', 'download', `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`),
2121
async () => {
2222
const pathOrNull = await downloadJDBCDriver();
2323
if (pathOrNull) {
2424
return pathOrNull;
2525
}
2626
throw new Error(
27-
'Please download and place DatabricksJDBC42.jar inside your ' +
27+
`Please download and place databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar inside your ` +
2828
'project directory'
2929
);
3030
}

‎packages/cubejs-databricks-jdbc-driver/src/installer.ts‎

Lines changed: 23 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,32 @@
11
import path from 'path';
22
import { downloadAndExtractFile, getEnv } from '@cubejs-backend/shared';
33

4-
function acceptedByEnv() {
5-
const acceptStatus = getEnv('databrickAcceptPolicy');
6-
if (acceptStatus) {
7-
console.log('You accepted Terms & Conditions for JDBC driver from DataBricks by CUBEJS_DB_DATABRICKS_ACCEPT_POLICY');
8-
}
9-
10-
if (acceptStatus === false) {
11-
console.log('You declined Terms & Conditions for JDBC driver from DataBricks by CUBEJS_DB_DATABRICKS_ACCEPT_POLICY');
12-
console.log('Installation will be skipped');
13-
}
14-
15-
return acceptStatus;
16-
}
17-
4+
export const OSS_DRIVER_VERSION = '1.0.2';
5+
6+
/**
7+
* In the beginning of 2025 Databricks released their open-source version of JDBC driver and encourage
8+
* all users to migrate to it as company plans to focus on improving and evolving it over legacy simba driver.
9+
* More info about OSS Driver could be found at https://docs.databricks.com/aws/en/integrations/jdbc/oss
10+
* As of March 2025 To use the Databricks JDBC Driver (OSS), the following requirements must be met:
11+
* Java Runtime Environment (JRE) 11.0 or above. CI testing is supported on JRE 11, 17, and 21.
12+
*/
1813
export async function downloadJDBCDriver(): Promise<string | null> {
19-
const driverAccepted = acceptedByEnv();
20-
21-
if (driverAccepted) {
22-
console.log('Downloading DatabricksJDBC42-2.6.29.1051');
14+
// TODO: Just to throw a console warning that this ENV is obsolete and could be safely removed
15+
getEnv('databrickAcceptPolicy');
2316

24-
await downloadAndExtractFile(
25-
'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/DatabricksJDBC42-2.6.29.1051.zip',
26-
{
27-
showProgress: true,
28-
cwd: path.resolve(path.join(__dirname, '..', 'download')),
29-
}
30-
);
17+
console.log(`Downloading databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`);
3118

32-
console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/docs/release-notes.txt');
19+
await downloadAndExtractFile(
20+
`https://repo1.maven.org/maven2/com/databricks/databricks-jdbc/${OSS_DRIVER_VERSION}-oss/databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`,
21+
{
22+
showProgress: true,
23+
cwd: path.resolve(path.join(__dirname, '..', 'download')),
24+
noExtract: true,
25+
dstFileName: `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`,
26+
}
27+
);
3328

34-
return path.resolve(path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'));
35-
}
29+
console.log(`Release notes: https://mvnrepository.com/artifact/com.databricks/databricks-jdbc/${OSS_DRIVER_VERSION}-oss`);
3630

37-
return null;
31+
return path.resolve(path.join(__dirname, '..', 'download', `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`));
3832
}

‎packages/cubejs-databricks-jdbc-driver/src/post-install.ts‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,11 @@
11
import 'source-map-support/register';
22

33
import { displayCLIError } from '@cubejs-backend/shared';
4-
5-
import fs from 'fs';
6-
import path from 'path';
7-
import { downloadJDBCDriver } from './installer';
4+
import { resolveJDBCDriver } from './helpers';
85

96
(async () => {
107
try {
11-
if (!fs.existsSync(path.join(__dirname, '..', 'download', 'SparkJDBC42.jar'))) {
12-
await downloadJDBCDriver();
13-
}
8+
await resolveJDBCDriver();
149
} catch (e: any) {
1510
await displayCLIError(e, 'Cube.js Databricks JDBC Installer');
1611
}

0 commit comments

Comments
 (0)