diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 775d3cfee7269..e7e364fdc7fdb 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -330,7 +330,11 @@ jobs: retry_wait_seconds: 15 timeout_minutes: 20 command: | + dig ohio.cloud.databricks.com. + curl -v https://ohio.cloud.databricks.com + curl -v https://ohio.cloud.databricks.com/sql/1.0/warehouses/ + CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true node --eval='require("https").get("https://ohio.cloud.databricks.com/sql/1.0/warehouses/", (res) => {console.log("res status and headers", res.statusCode, res.statusMessage, res.headers); res.on("data", (chunk) => console.log("res data", chunk.toString()))})' cd ./packages/cubejs-testing-drivers export DEBUG=testcontainers - yarn ${{ matrix.database }}-full + CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true yarn ${{ matrix.database }}-full diff --git a/packages/cubejs-databricks-jdbc-driver/src/installer.ts b/packages/cubejs-databricks-jdbc-driver/src/installer.ts index fb405e325a701..c1910ac53c24a 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/installer.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/installer.ts @@ -2,7 +2,10 @@ import path from 'path'; import { downloadAndExtractFile, getEnv } from '@cubejs-backend/shared'; function acceptedByEnv() { + console.log('acceptedByEnv call'); + const acceptStatus = getEnv('databrickAcceptPolicy'); + console.log('acceptedByEnv call acceptStatus', acceptStatus); if (acceptStatus) { console.log('You accepted Terms & Conditions for JDBC driver from DataBricks by CUBEJS_DB_DATABRICKS_ACCEPT_POLICY'); } @@ -16,20 +19,23 @@ function acceptedByEnv() { } export async function downloadJDBCDriver(): Promise { + console.log('downloadJDBCDriver call'); const driverAccepted = acceptedByEnv(); + console.log('downloadJDBCDriver call driverAccepted', driverAccepted); + if (driverAccepted) { - console.log('Downloading DatabricksJDBC42-2.6.29.1051'); + console.log('Downloading DatabricksJDBC42-2.6.40.1071'); await downloadAndExtractFile( - 'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/DatabricksJDBC42-2.6.29.1051.zip', + 'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.40/DatabricksJDBC42-2.6.40.1071.zip', { showProgress: true, cwd: path.resolve(path.join(__dirname, '..', 'download')), } ); - console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/docs/release-notes.txt'); + console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.40/docs/release-notes.txt'); return path.resolve(path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar')); } diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index 136d21c289e07..298c693e56b4b 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -37,14 +37,26 @@ let mvnPromise: Promise | null = null; const initMvn = (customClassPath: any) => { if (!mvnPromise) { mvnPromise = new Promise((resolve, reject) => { + console.log('new mvnPromise'); + const options = { packageJsonPath: `${path.join(__dirname, '../..')}/package.json`, }; + console.log('mvnPromise options', options); mvn(options, (err: any, mvnResults: any) => { + console.log('mvnPromise callback err', err); + console.log('mvnPromise callback mvnResults', mvnResults); + + console.log('mvnPromise callback err.message.includes', err.message.includes('Could not find java property')); + if (err && !err.message.includes('Could not find java property')) { reject(err); } else { + console.log('jinst.isJvmCreated()', jinst.isJvmCreated()); + if (!jinst.isJvmCreated()) { + console.log('before jinst.addOption'); + jinst.addOption('-Xrs'); jinst.addOption('-Dfile.encoding=UTF8'); @@ -53,6 +65,8 @@ const initMvn = (customClassPath: any) => { jinst.addOption('--add-opens=java.base/java.nio=ALL-UNNAMED'); const classPath = (mvnResults && mvnResults.classpath || []).concat(customClassPath || []); + + console.log('before jinst.setupClasspath'); jinst.setupClasspath(classPath); } resolve(); @@ -137,7 +151,10 @@ export class JDBCDriver extends BaseDriver { this.pool = genericPool.createPool({ create: async () => { - await initMvn(await this.getCustomClassPath()); + // console.log('before await this.getCustomClassPath()', new Error().stack); + const ccp = await this.getCustomClassPath(); + // console.log('before await initMvn(ccp)'); + await initMvn(ccp); if (!this.jdbcProps) { /** @protected */ @@ -145,6 +162,7 @@ export class JDBCDriver extends BaseDriver { } const getConnection = promisify(DriverManager.getConnection.bind(DriverManager)); + // console.log('new Connection this.config.url', this.config.url); return new Connection(await getConnection(this.config.url, this.jdbcProps)); }, // @ts-expect-error Promise vs Promise @@ -183,6 +201,10 @@ export class JDBCDriver extends BaseDriver { acquireTimeoutMillis: 120000, ...(poolOptions || {}) }) as ExtendedPool; + + this.pool.on('factoryCreateError', (err) => { + console.log('pool factoryCreateError', err); + }); } protected async getCustomClassPath() { diff --git a/packages/cubejs-testing-drivers/src/tests/testQueries.ts b/packages/cubejs-testing-drivers/src/tests/testQueries.ts index 7accddba8baa9..d89211e22c8c2 100644 --- a/packages/cubejs-testing-drivers/src/tests/testQueries.ts +++ b/packages/cubejs-testing-drivers/src/tests/testQueries.ts @@ -121,7 +121,7 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten } console.log(`Creating ${queries.length} fixture tables completed`); } catch (e: any) { - console.log('Error creating fixtures', e.stack); + console.log('Error creating TEST fixtures', e.stack); throw e; } }); diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index 1a92f73fbf5ad..6073ca8b911ac 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -1,5 +1,67 @@ +// import * as https from 'node:https'; +// +// import D from '@cubejs-backend/databricks-jdbc-driver'; + import { testQueries } from '../src/tests/testQueries'; +// https.get( +// 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', +// (res) => { +// console.log( +// 'res status and headers', +// res.statusCode, +// res.statusMessage, +// res.headers +// ); +// res.on('data', (chunk) => console.log('res data', chunk.toString())); +// } +// ); +// +// https.get( +// 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', +// { +// headers: { +// authorization: `Bearer ${process.env.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN}`, +// } +// }, +// (res) => { +// console.log( +// 'res with token status and headers', +// res.statusCode, +// res.statusMessage, +// res.headers +// ); +// res.on('data', (chunk) => console.log('res with token data', chunk.toString())); +// } +// ); +// +// async function f() { +// const source = new D({ +// url: 'jdbc:databricks://ohio.cloud.databricks.com/default;httpPath=/sql/1.0/warehouses/;', +// maxPoolSize: 1, +// }); +// console.log('test driver constructed'); +// const qRes = await source.query('SELECT 1 as foo;', []); +// console.log('qRes', qRes); +// } +// +// f() +// .then( +// () => { +// console.log('driver test query succ'); +// process.exit(0); +// }, +// e => { +// console.log('driver test query failed', e); +// process.exit(1); +// } +// ); +// +// setTimeout(() => { +// console.log('TIMING OUT'); +// process.exit(1); +// }, 3 * 60 * 1000); + testQueries('databricks-jdbc', { includeIncrementalSchemaSuite: true, includeHLLSuite: true,