Skip to content

tmp/databricks3 #9654

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

tmp/databricks3 #9654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
307e801
remove databrickAcceptPolicy env
KSDaemon Jun 3, 2025
3be8a22
fix databricks env var names
KSDaemon Jun 3, 2025
7c6342c
prepare CI
KSDaemon Jun 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/drivers-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,8 @@ jobs:
DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN }}
DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY }}
DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET }}
DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_ID: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_ID }}
DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_SECRET }}

# Redshift
DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST }}
Expand Down
19 changes: 2 additions & 17 deletions packages/cubejs-backend-shared/src/env.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -951,25 +951,10 @@ const variables: Record<string, (...args: any) => any> = {
* Databricks Driver *
***************************************************************** */

/**
* Accept Databricks policy flag. This environment variable doesn't
* need to be split by the data source.
* TODO: Tech-debt: Remove totally someday
*/
databrickAcceptPolicy: () => {
const val = get('CUBEJS_DB_DATABRICKS_ACCEPT_POLICY').asBoolStrict();

if (val !== undefined) {
console.warn(
'The CUBEJS_DB_DATABRICKS_ACCEPT_POLICY is not needed anymore. Please, remove it'
);
}
},

/**
* Databricks jdbc-connection url.
*/
databrickUrl: ({
databricksUrl: ({
dataSource,
}: {
dataSource: string,
Expand All @@ -990,7 +975,7 @@ const variables: Record<string, (...args: any) => any> = {
/**
* Databricks jdbc-connection token.
*/
databrickToken: ({
databricksToken: ({
dataSource,
}: {
dataSource: string,
Expand Down
36 changes: 18 additions & 18 deletions packages/cubejs-backend-shared/test/db_env_multi.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1105,31 +1105,31 @@ describe('Multiple datasources', () => {
process.env.CUBEJS_DB_DATABRICKS_URL = 'default1';
process.env.CUBEJS_DS_POSTGRES_DB_DATABRICKS_URL = 'postgres1';
process.env.CUBEJS_DS_WRONG_DB_DATABRICKS_URL = 'wrong1';
expect(getEnv('databrickUrl', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databrickUrl', { dataSource: 'postgres' })).toEqual('postgres1');
expect(() => getEnv('databrickUrl', { dataSource: 'wrong' })).toThrow(
expect(getEnv('databricksUrl', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databricksUrl', { dataSource: 'postgres' })).toEqual('postgres1');
expect(() => getEnv('databricksUrl', { dataSource: 'wrong' })).toThrow(
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
);

process.env.CUBEJS_DB_DATABRICKS_URL = 'default2';
process.env.CUBEJS_DS_POSTGRES_DB_DATABRICKS_URL = 'postgres2';
process.env.CUBEJS_DS_WRONG_DB_DATABRICKS_URL = 'wrong2';
expect(getEnv('databrickUrl', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databrickUrl', { dataSource: 'postgres' })).toEqual('postgres2');
expect(() => getEnv('databrickUrl', { dataSource: 'wrong' })).toThrow(
expect(getEnv('databricksUrl', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databricksUrl', { dataSource: 'postgres' })).toEqual('postgres2');
expect(() => getEnv('databricksUrl', { dataSource: 'wrong' })).toThrow(
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
);

delete process.env.CUBEJS_DB_DATABRICKS_URL;
delete process.env.CUBEJS_DS_POSTGRES_DB_DATABRICKS_URL;
delete process.env.CUBEJS_DS_WRONG_DB_DATABRICKS_URL;
expect(() => getEnv('databrickUrl', { dataSource: 'default' })).toThrow(
expect(() => getEnv('databricksUrl', { dataSource: 'default' })).toThrow(
'The CUBEJS_DB_DATABRICKS_URL is required and missing.'
);
expect(() => getEnv('databrickUrl', { dataSource: 'postgres' })).toThrow(
expect(() => getEnv('databricksUrl', { dataSource: 'postgres' })).toThrow(
'The CUBEJS_DS_POSTGRES_DB_DATABRICKS_URL is required and missing.'
);
expect(() => getEnv('databrickUrl', { dataSource: 'wrong' })).toThrow(
expect(() => getEnv('databricksUrl', { dataSource: 'wrong' })).toThrow(
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
);
});
Expand All @@ -1138,27 +1138,27 @@ describe('Multiple datasources', () => {
process.env.CUBEJS_DB_DATABRICKS_TOKEN = 'default1';
process.env.CUBEJS_DS_POSTGRES_DB_DATABRICKS_TOKEN = 'postgres1';
process.env.CUBEJS_DS_WRONG_DB_DATABRICKS_TOKEN = 'wrong1';
expect(getEnv('databrickToken', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databrickToken', { dataSource: 'postgres' })).toEqual('postgres1');
expect(() => getEnv('databrickToken', { dataSource: 'wrong' })).toThrow(
expect(getEnv('databricksToken', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databricksToken', { dataSource: 'postgres' })).toEqual('postgres1');
expect(() => getEnv('databricksToken', { dataSource: 'wrong' })).toThrow(
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
);

process.env.CUBEJS_DB_DATABRICKS_TOKEN = 'default2';
process.env.CUBEJS_DS_POSTGRES_DB_DATABRICKS_TOKEN = 'postgres2';
process.env.CUBEJS_DS_WRONG_DB_DATABRICKS_TOKEN = 'wrong2';
expect(getEnv('databrickToken', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databrickToken', { dataSource: 'postgres' })).toEqual('postgres2');
expect(() => getEnv('databrickToken', { dataSource: 'wrong' })).toThrow(
expect(getEnv('databricksToken', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databricksToken', { dataSource: 'postgres' })).toEqual('postgres2');
expect(() => getEnv('databricksToken', { dataSource: 'wrong' })).toThrow(
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
);

delete process.env.CUBEJS_DB_DATABRICKS_TOKEN;
delete process.env.CUBEJS_DS_POSTGRES_DB_DATABRICKS_TOKEN;
delete process.env.CUBEJS_DS_WRONG_DB_DATABRICKS_TOKEN;
expect(getEnv('databrickToken', { dataSource: 'default' })).toBeUndefined();
expect(getEnv('databrickToken', { dataSource: 'postgres' })).toBeUndefined();
expect(() => getEnv('databrickToken', { dataSource: 'wrong' })).toThrow(
expect(getEnv('databricksToken', { dataSource: 'default' })).toBeUndefined();
expect(getEnv('databricksToken', { dataSource: 'postgres' })).toBeUndefined();
expect(() => getEnv('databricksToken', { dataSource: 'wrong' })).toThrow(
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
);
});
Expand Down
36 changes: 18 additions & 18 deletions packages/cubejs-backend-shared/test/db_env_single.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -705,42 +705,42 @@ describe('Single datasources', () => {

test('getEnv("databrickUrl")', () => {
process.env.CUBEJS_DB_DATABRICKS_URL = 'default1';
expect(getEnv('databrickUrl', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databrickUrl', { dataSource: 'postgres' })).toEqual('default1');
expect(getEnv('databrickUrl', { dataSource: 'wrong' })).toEqual('default1');
expect(getEnv('databricksUrl', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databricksUrl', { dataSource: 'postgres' })).toEqual('default1');
expect(getEnv('databricksUrl', { dataSource: 'wrong' })).toEqual('default1');

process.env.CUBEJS_DB_DATABRICKS_URL = 'default2';
expect(getEnv('databrickUrl', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databrickUrl', { dataSource: 'postgres' })).toEqual('default2');
expect(getEnv('databrickUrl', { dataSource: 'wrong' })).toEqual('default2');
expect(getEnv('databricksUrl', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databricksUrl', { dataSource: 'postgres' })).toEqual('default2');
expect(getEnv('databricksUrl', { dataSource: 'wrong' })).toEqual('default2');

delete process.env.CUBEJS_DB_DATABRICKS_URL;
expect(() => getEnv('databrickUrl', { dataSource: 'default' })).toThrow(
expect(() => getEnv('databricksUrl', { dataSource: 'default' })).toThrow(
'The CUBEJS_DB_DATABRICKS_URL is required and missing.'
);
expect(() => getEnv('databrickUrl', { dataSource: 'postgres' })).toThrow(
expect(() => getEnv('databricksUrl', { dataSource: 'postgres' })).toThrow(
'The CUBEJS_DB_DATABRICKS_URL is required and missing.'
);
expect(() => getEnv('databrickUrl', { dataSource: 'wrong' })).toThrow(
expect(() => getEnv('databricksUrl', { dataSource: 'wrong' })).toThrow(
'The CUBEJS_DB_DATABRICKS_URL is required and missing.'
);
});

test('getEnv("databrickToken")', () => {
process.env.CUBEJS_DB_DATABRICKS_TOKEN = 'default1';
expect(getEnv('databrickToken', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databrickToken', { dataSource: 'postgres' })).toEqual('default1');
expect(getEnv('databrickToken', { dataSource: 'wrong' })).toEqual('default1');
expect(getEnv('databricksToken', { dataSource: 'default' })).toEqual('default1');
expect(getEnv('databricksToken', { dataSource: 'postgres' })).toEqual('default1');
expect(getEnv('databricksToken', { dataSource: 'wrong' })).toEqual('default1');

process.env.CUBEJS_DB_DATABRICKS_TOKEN = 'default2';
expect(getEnv('databrickToken', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databrickToken', { dataSource: 'postgres' })).toEqual('default2');
expect(getEnv('databrickToken', { dataSource: 'wrong' })).toEqual('default2');
expect(getEnv('databricksToken', { dataSource: 'default' })).toEqual('default2');
expect(getEnv('databricksToken', { dataSource: 'postgres' })).toEqual('default2');
expect(getEnv('databricksToken', { dataSource: 'wrong' })).toEqual('default2');

delete process.env.CUBEJS_DB_DATABRICKS_TOKEN;
expect(getEnv('databrickToken', { dataSource: 'default' })).toBeUndefined();
expect(getEnv('databrickToken', { dataSource: 'postgres' })).toBeUndefined();
expect(getEnv('databrickToken', { dataSource: 'wrong' })).toBeUndefined();
expect(getEnv('databricksToken', { dataSource: 'default' })).toBeUndefined();
expect(getEnv('databricksToken', { dataSource: 'postgres' })).toBeUndefined();
expect(getEnv('databricksToken', { dataSource: 'wrong' })).toBeUndefined();
});

test('getEnv("databricksCatalog")', () => {
Expand Down
4 changes: 2 additions & 2 deletions packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ export class DatabricksDriver extends JDBCDriver {
let showSparkProtocolWarn = false;
let url: string =
conf?.url ||
getEnv('databrickUrl', { dataSource }) ||
getEnv('databricksUrl', { dataSource }) ||
getEnv('jdbcUrl', { dataSource });
if (url.indexOf('jdbc:spark://') !== -1) {
showSparkProtocolWarn = true;
Expand All @@ -211,7 +211,7 @@ export class DatabricksDriver extends JDBCDriver {
UID: uid,
PWD:
conf?.token ||
getEnv('databrickToken', { dataSource }) ||
getEnv('databricksToken', { dataSource }) ||
pwd,
UserAgentEntry: 'CubeDev_Cube',
},
Expand Down
3 changes: 0 additions & 3 deletions packages/cubejs-databricks-jdbc-driver/src/installer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ export const OSS_DRIVER_VERSION = '1.0.2';
* Java Runtime Environment (JRE) 11.0 or above. CI testing is supported on JRE 11, 17, and 21.
*/
export async function downloadJDBCDriver(): Promise<string | null> {
// TODO: Just to throw a console warning that this ENV is obsolete and could be safely removed
getEnv('databrickAcceptPolicy');

console.log(`Downloading databricks-jdbc-${OSS_DRIVER_VERSION}-oss.jar`);

await downloadAndExtractFile(
Expand Down
6 changes: 5 additions & 1 deletion packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
"CUBEJS_DB_EXPORT_BUCKET": "s3://databricks-drivers-tests-preaggs",
"CUBEJS_DB_EXPORT_BUCKET_AWS_KEY": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY}",
"CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET}",
"CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-east-1"
"CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-east-1",
"Cannot_left_comments_in_json": "Use OAuth machine-to-machine (M2M) authentication here for testing it too",
"CUBEJS_DB_DATABRICKS_TOKEN": "",
"CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_ID": "${DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_ID}",
"CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_SECRET": "${DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_SECRET}"
}
}
},
Expand Down
Loading