diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index dff50a243251b..eaad63fc5d03f 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -952,15 +952,9 @@ const variables: Record any> = { * Accept Databricks policy flag. This environment variable doesn't * need to be split by the data source. */ - databrickAcceptPolicy: () => { - const val = get('CUBEJS_DB_DATABRICKS_ACCEPT_POLICY').asBoolStrict(); - - if (val !== undefined) { - console.warn( - 'The CUBEJS_DB_DATABRICKS_ACCEPT_POLICY is not needed anymore. Please, remove it' - ); - } - }, + databrickAcceptPolicy: () => ( + get('CUBEJS_DB_DATABRICKS_ACCEPT_POLICY').asBoolStrict() + ), /** * Databricks jdbc-connection url. diff --git a/packages/cubejs-backend-shared/src/http-utils.ts b/packages/cubejs-backend-shared/src/http-utils.ts index b75b0eb884c44..a69de11928854 100644 --- a/packages/cubejs-backend-shared/src/http-utils.ts +++ b/packages/cubejs-backend-shared/src/http-utils.ts @@ -66,11 +66,9 @@ export async function streamWithProgress( type DownloadAndExtractFile = { showProgress: boolean; cwd: string; - noExtract?: boolean; - dstFileName?: string; }; -export async function downloadAndExtractFile(url: string, { cwd, noExtract, dstFileName }: DownloadAndExtractFile) { +export async function downloadAndExtractFile(url: string, { cwd }: DownloadAndExtractFile) { const request = new Request(url, { headers: new Headers({ 'Content-Type': 'application/octet-stream', @@ -101,15 +99,7 @@ export async function downloadAndExtractFile(url: string, { cwd, noExtract, dstF }); }); - if (noExtract) { - if (dstFileName) { - fs.copyFileSync(savedFilePath, path.resolve(path.join(cwd, dstFileName))); - } else { - fs.copyFileSync(savedFilePath, cwd); - } - } else { - await decompress(savedFilePath, cwd); - } + await decompress(savedFilePath, cwd); try { fs.unlinkSync(savedFilePath); diff --git a/packages/cubejs-backend-shared/test/db_env_multi.test.ts b/packages/cubejs-backend-shared/test/db_env_multi.test.ts index 7219d039e422a..f42e177fbbb83 100644 --- a/packages/cubejs-backend-shared/test/db_env_multi.test.ts +++ b/packages/cubejs-backend-shared/test/db_env_multi.test.ts @@ -1192,6 +1192,34 @@ describe('Multiple datasources', () => { ); }); + test('getEnv("databrickAcceptPolicy")', () => { + process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'true'; + expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(true); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(true); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(true); + + process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'false'; + expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(false); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(false); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(false); + + process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'wrong'; + expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toThrow( + 'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"' + ); + expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toThrow( + 'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"' + ); + expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toThrow( + 'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"' + ); + + delete process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY; + expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toBeUndefined(); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toBeUndefined(); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toBeUndefined(); + }); + test('getEnv("athenaAwsKey")', () => { process.env.CUBEJS_AWS_KEY = 'default1'; process.env.CUBEJS_DS_POSTGRES_AWS_KEY = 'postgres1'; diff --git a/packages/cubejs-backend-shared/test/db_env_single.test.ts b/packages/cubejs-backend-shared/test/db_env_single.test.ts index 1dd5612309f32..f5de389afe9a2 100644 --- a/packages/cubejs-backend-shared/test/db_env_single.test.ts +++ b/packages/cubejs-backend-shared/test/db_env_single.test.ts @@ -760,6 +760,34 @@ describe('Single datasources', () => { expect(getEnv('databricksCatalog', { dataSource: 'wrong' })).toBeUndefined(); }); + test('getEnv("databrickAcceptPolicy")', () => { + process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'true'; + expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(true); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(true); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(true); + + process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'false'; + expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toEqual(false); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toEqual(false); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toEqual(false); + + process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY = 'wrong'; + expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toThrow( + 'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"' + ); + expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toThrow( + 'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"' + ); + expect(() => getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toThrow( + 'env-var: "CUBEJS_DB_DATABRICKS_ACCEPT_POLICY" should be either "true", "false", "TRUE", or "FALSE"' + ); + + delete process.env.CUBEJS_DB_DATABRICKS_ACCEPT_POLICY; + expect(getEnv('databrickAcceptPolicy', { dataSource: 'default' })).toBeUndefined(); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'postgres' })).toBeUndefined(); + expect(getEnv('databrickAcceptPolicy', { dataSource: 'wrong' })).toBeUndefined(); + }); + test('getEnv("athenaAwsKey")', () => { process.env.CUBEJS_AWS_KEY = 'default1'; expect(getEnv('athenaAwsKey', { dataSource: 'default' })).toEqual('default1'); diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts index d035b75f562c2..b1b54cf35db0f 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts @@ -4,21 +4,27 @@ * @fileoverview The `DatabricksDriver` and related types declaration. */ -import { assertDataSource, getEnv, } from '@cubejs-backend/shared'; import { - DatabaseStructure, + getEnv, + assertDataSource, +} from '@cubejs-backend/shared'; +import { DriverCapabilities, - GenericDataBaseType, QueryColumnsResult, QueryOptions, QuerySchemasResult, QueryTablesResult, - TableColumn, UnloadOptions, + GenericDataBaseType, + TableColumn, + DatabaseStructure, } from '@cubejs-backend/base-driver'; -import { JDBCDriver, JDBCDriverConfiguration, } from '@cubejs-backend/jdbc-driver'; +import { + JDBCDriver, + JDBCDriverConfiguration, +} from '@cubejs-backend/jdbc-driver'; import { DatabricksQuery } from './DatabricksQuery'; -import { extractUidFromJdbcUrl, resolveJDBCDriver } from './helpers'; +import { resolveJDBCDriver, extractUidFromJdbcUrl } from './helpers'; export type DatabricksDriverConfiguration = JDBCDriverConfiguration & { @@ -126,7 +132,7 @@ export class DatabricksDriver extends JDBCDriver { /** * Show warning message flag. */ - private readonly showSparkProtocolWarn: boolean; + private showSparkProtocolWarn: boolean; /** * Driver Configuration. @@ -423,7 +429,8 @@ export class DatabricksDriver extends JDBCDriver { metadata[database] = {}; } - metadata[database][tableName] = await this.tableColumnTypes(`${database}.${tableName}`); + const columns = await this.tableColumnTypes(`${database}.${tableName}`); + metadata[database][tableName] = columns; })); return metadata; diff --git a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts index 8eca5896ba84e..b864d07b4c007 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; -import { downloadJDBCDriver, OSS_DRIVER_VERSION } from './installer'; +import { downloadJDBCDriver } from './installer'; async function fileExistsOr( fsPath: string, @@ -15,16 +15,16 @@ async function fileExistsOr( export async function resolveJDBCDriver(): Promise { return fileExistsOr( - path.join(process.cwd(), `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`), + path.join(process.cwd(), 'DatabricksJDBC42.jar'), async () => fileExistsOr( - path.join(__dirname, '..', 'download', `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`), + path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'), async () => { const pathOrNull = await downloadJDBCDriver(); if (pathOrNull) { return pathOrNull; } throw new Error( - `Please download and place databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar inside your ` + + 'Please download and place DatabricksJDBC42.jar inside your ' + 'project directory' ); } diff --git a/packages/cubejs-databricks-jdbc-driver/src/installer.ts b/packages/cubejs-databricks-jdbc-driver/src/installer.ts index 35eddbbc24957..fb405e325a701 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/installer.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/installer.ts @@ -1,32 +1,38 @@ import path from 'path'; import { downloadAndExtractFile, getEnv } from '@cubejs-backend/shared'; -export const OSS_DRIVER_VERSION = '1.0.2'; - -/** - * In the beginning of 2025 Databricks released their open-source version of JDBC driver and encourage - * all users to migrate to it as company plans to focus on improving and evolving it over legacy simba driver. - * More info about OSS Driver could be found at https://docs.databricks.com/aws/en/integrations/jdbc/oss - * As of March 2025 To use the Databricks JDBC Driver (OSS), the following requirements must be met: - * Java Runtime Environment (JRE) 11.0 or above. CI testing is supported on JRE 11, 17, and 21. - */ +function acceptedByEnv() { + const acceptStatus = getEnv('databrickAcceptPolicy'); + if (acceptStatus) { + console.log('You accepted Terms & Conditions for JDBC driver from DataBricks by CUBEJS_DB_DATABRICKS_ACCEPT_POLICY'); + } + + if (acceptStatus === false) { + console.log('You declined Terms & Conditions for JDBC driver from DataBricks by CUBEJS_DB_DATABRICKS_ACCEPT_POLICY'); + console.log('Installation will be skipped'); + } + + return acceptStatus; +} + export async function downloadJDBCDriver(): Promise { - // TODO: Just to throw a console warning that this ENV is obsolete and could be safely removed - getEnv('databrickAcceptPolicy'); + const driverAccepted = acceptedByEnv(); + + if (driverAccepted) { + console.log('Downloading DatabricksJDBC42-2.6.29.1051'); - console.log(`Downloading databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`); + await downloadAndExtractFile( + 'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/DatabricksJDBC42-2.6.29.1051.zip', + { + showProgress: true, + cwd: path.resolve(path.join(__dirname, '..', 'download')), + } + ); - await downloadAndExtractFile( - `https://repo1.maven.org/maven2/com/databricks/databricks-jdbc/${OSS_DRIVER_VERSION}-oss/databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`, - { - showProgress: true, - cwd: path.resolve(path.join(__dirname, '..', 'download')), - noExtract: true, - dstFileName: `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`, - } - ); + console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/docs/release-notes.txt'); - console.log(`Release notes: https://mvnrepository.com/artifact/com.databricks/databricks-jdbc/${OSS_DRIVER_VERSION}-oss`); + return path.resolve(path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar')); + } - return path.resolve(path.join(__dirname, '..', 'download', `databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`)); + return null; } diff --git a/packages/cubejs-databricks-jdbc-driver/src/post-install.ts b/packages/cubejs-databricks-jdbc-driver/src/post-install.ts index 51e4a506d4588..8f934ef9b9103 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/post-install.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/post-install.ts @@ -1,11 +1,16 @@ import 'source-map-support/register'; import { displayCLIError } from '@cubejs-backend/shared'; -import { resolveJDBCDriver } from './helpers'; + +import fs from 'fs'; +import path from 'path'; +import { downloadJDBCDriver } from './installer'; (async () => { try { - await resolveJDBCDriver(); + if (!fs.existsSync(path.join(__dirname, '..', 'download', 'SparkJDBC42.jar'))) { + await downloadJDBCDriver(); + } } catch (e: any) { await displayCLIError(e, 'Cube.js Databricks JDBC Installer'); }