From 93cb5dd5bea46d3d173d92bef27275d99da62baa Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Thu, 31 Oct 2024 18:16:50 +0200 Subject: [PATCH 01/27] [DO NOT MERGE] tiny diff to trigger tests --- packages/cubejs-testing-drivers/src/tests/testQueries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/src/tests/testQueries.ts b/packages/cubejs-testing-drivers/src/tests/testQueries.ts index 7accddba8baa9..d89211e22c8c2 100644 --- a/packages/cubejs-testing-drivers/src/tests/testQueries.ts +++ b/packages/cubejs-testing-drivers/src/tests/testQueries.ts @@ -121,7 +121,7 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten } console.log(`Creating ${queries.length} fixture tables completed`); } catch (e: any) { - console.log('Error creating fixtures', e.stack); + console.log('Error creating TEST fixtures', e.stack); throw e; } }); From cf875b16121b8ba0b59ee5ba210e197c4d31cd79 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Fri, 1 Nov 2024 14:42:03 +0200 Subject: [PATCH 02/27] [DO NOT MERGE] Add debug dig and curl calls --- .github/workflows/drivers-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 775d3cfee7269..381381b1dba9f 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -330,6 +330,8 @@ jobs: retry_wait_seconds: 15 timeout_minutes: 20 command: | + dig https://ohio.cloud.databricks.com. + curl -v https://ohio.cloud.databricks.com. cd ./packages/cubejs-testing-drivers export DEBUG=testcontainers yarn ${{ matrix.database }}-full From fa9d6bc9dbb5fb44f6aef824505141429e5152e2 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Fri, 1 Nov 2024 14:56:39 +0200 Subject: [PATCH 03/27] [DO NOT MERGE] fix dig call --- .github/workflows/drivers-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 381381b1dba9f..00d6e41259666 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -330,7 +330,7 @@ jobs: retry_wait_seconds: 15 timeout_minutes: 20 command: | - dig https://ohio.cloud.databricks.com. + dig ohio.cloud.databricks.com. curl -v https://ohio.cloud.databricks.com. cd ./packages/cubejs-testing-drivers export DEBUG=testcontainers From 71f8df9c43522db736b6f1477739bd3db361c796 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Fri, 1 Nov 2024 15:01:39 +0200 Subject: [PATCH 04/27] [DO NOT MERGE] more curl calls --- .github/workflows/drivers-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 00d6e41259666..3301871848ca2 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -331,7 +331,8 @@ jobs: timeout_minutes: 20 command: | dig ohio.cloud.databricks.com. - curl -v https://ohio.cloud.databricks.com. + curl -v https://ohio.cloud.databricks.com + curl -v https://ohio.cloud.databricks.com/sql/1.0/warehouses/ cd ./packages/cubejs-testing-drivers export DEBUG=testcontainers yarn ${{ matrix.database }}-full From bd08e7706f94dd788e01edf0870260a0c15916db Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Fri, 1 Nov 2024 15:35:32 +0200 Subject: [PATCH 05/27] [DO NOT MERGE] add debug calls from node --- .github/workflows/drivers-tests.yml | 1 + .../test/databricks-jdbc-full.test.ts | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 3301871848ca2..cf873934b37ea 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -333,6 +333,7 @@ jobs: dig ohio.cloud.databricks.com. curl -v https://ohio.cloud.databricks.com curl -v https://ohio.cloud.databricks.com/sql/1.0/warehouses/ + node --eval='require("https").get("https://ohio.cloud.databricks.com/sql/1.0/warehouses/", (res) => {console.log("res status and headers", res.statusCode, res.statusMessage, res.headers); res.on("data", (chunk) => console.log("res data", chunk.toString()))})' cd ./packages/cubejs-testing-drivers export DEBUG=testcontainers yarn ${{ matrix.database }}-full diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index 1a92f73fbf5ad..a60ab385e5528 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -1,5 +1,20 @@ +import * as https from "node:https"; + import { testQueries } from '../src/tests/testQueries'; +https.get( + 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', + (res) => { + console.log( + 'res status and headers', + res.statusCode, + res.statusMessage, + res.headers + ); + res.on('data', (chunk) => console.log('res data', chunk.toString())); + } +); + testQueries('databricks-jdbc', { includeIncrementalSchemaSuite: true, includeHLLSuite: true, From 558de586fc4cebd1374eb199a59e64defa8f1084 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 04:05:12 +0200 Subject: [PATCH 06/27] [DO NOT MERGE] add debug calls from node with token --- .../test/databricks-jdbc-full.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index a60ab385e5528..f4f23d46529cc 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -15,6 +15,24 @@ https.get( } ); +https.get( + 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', + { + headers: { + authorization: process.env.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN, + } + }, + (res) => { + console.log( + 'res with token status and headers', + res.statusCode, + res.statusMessage, + res.headers + ); + res.on('data', (chunk) => console.log('res with token data', chunk.toString())); + } +); + testQueries('databricks-jdbc', { includeIncrementalSchemaSuite: true, includeHLLSuite: true, From 4368ed2fd5bcb0aecec7b902464a6a6554249933 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 04:58:31 +0200 Subject: [PATCH 07/27] [DO NOT MERGE] fix debug calls from node with token --- .../cubejs-testing-drivers/test/databricks-jdbc-full.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index f4f23d46529cc..90205c64be2ff 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -19,7 +19,7 @@ https.get( 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', { headers: { - authorization: process.env.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN, + authorization: `Bearer ${process.env.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN}`, } }, (res) => { From 4ab47ed4007ef3c3322391c04872c5ad39df01c0 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 05:18:23 +0200 Subject: [PATCH 08/27] [DO NOT MERGE] add debug calls from node thru driver --- .../test/databricks-jdbc-full.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index 90205c64be2ff..a1dc7557730cc 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -1,4 +1,6 @@ -import * as https from "node:https"; +import * as https from 'node:https'; + +import D from '@cubejs-backend/databricks-jdbc-driver'; import { testQueries } from '../src/tests/testQueries'; @@ -33,6 +35,15 @@ https.get( } ); +async function f() { + const source = new D(); + console.log('test driver constructed'); + const qRes = await source.query('SELECT 1 as foo;', []); + console.log('qRes', qRes); +} + +f().catch(e => console.log('driver test query failed', e)); + testQueries('databricks-jdbc', { includeIncrementalSchemaSuite: true, includeHLLSuite: true, From dba68084e5e1297442a1145a0fd4fd259b8157f6 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 05:51:07 +0200 Subject: [PATCH 09/27] [DO NOT MERGE] tune debug calls from node thru driver --- .../cubejs-testing-drivers/test/databricks-jdbc-full.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index a1dc7557730cc..eda6394252338 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -36,7 +36,9 @@ https.get( ); async function f() { - const source = new D(); + const source = new D({ + url: 'ohio.cloud.databricks.com', + }); console.log('test driver constructed'); const qRes = await source.query('SELECT 1 as foo;', []); console.log('qRes', qRes); From fc1274cfc59de19da8b5292563f68a73f524c719 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 16:13:08 +0200 Subject: [PATCH 10/27] [DO NOT MERGE] tune debug calls from node thru driver --- .../cubejs-testing-drivers/test/databricks-jdbc-full.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index eda6394252338..b4eca917139d7 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -37,7 +37,7 @@ https.get( async function f() { const source = new D({ - url: 'ohio.cloud.databricks.com', + url: 'https://ohio.cloud.databricks.com', }); console.log('test driver constructed'); const qRes = await source.query('SELECT 1 as foo;', []); From 185c62c5e25b4200f3cac896ef6dbc617a53295a Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 16:54:53 +0200 Subject: [PATCH 11/27] [DO NOT MERGE] bump JDBC driver version --- packages/cubejs-databricks-jdbc-driver/src/installer.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-databricks-jdbc-driver/src/installer.ts b/packages/cubejs-databricks-jdbc-driver/src/installer.ts index fb405e325a701..ccfd09682a640 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/installer.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/installer.ts @@ -19,17 +19,17 @@ export async function downloadJDBCDriver(): Promise { const driverAccepted = acceptedByEnv(); if (driverAccepted) { - console.log('Downloading DatabricksJDBC42-2.6.29.1051'); + console.log('Downloading DatabricksJDBC42-2.6.40.1071'); await downloadAndExtractFile( - 'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/DatabricksJDBC42-2.6.29.1051.zip', + 'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.40/DatabricksJDBC42-2.6.40.1071.zip', { showProgress: true, cwd: path.resolve(path.join(__dirname, '..', 'download')), } ); - console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/docs/release-notes.txt'); + console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.40/docs/release-notes.txt'); return path.resolve(path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar')); } From ca63c63025e4b4a880bdc988ce4e56ec69478ee5 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 17:22:22 +0200 Subject: [PATCH 12/27] [DO NOT MERGE] some debug for JDBC --- packages/cubejs-jdbc-driver/src/JDBCDriver.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index 136d21c289e07..ea2b71b428ce3 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -137,7 +137,10 @@ export class JDBCDriver extends BaseDriver { this.pool = genericPool.createPool({ create: async () => { - await initMvn(await this.getCustomClassPath()); + console.log("before await this.getCustomClassPath()"); + const ccp = await this.getCustomClassPath(); + console.log("before await initMvn(ccp)"); + await initMvn(ccp); if (!this.jdbcProps) { /** @protected */ @@ -145,6 +148,7 @@ export class JDBCDriver extends BaseDriver { } const getConnection = promisify(DriverManager.getConnection.bind(DriverManager)); + console.log("new Connection this.config.url", this.config.url); return new Connection(await getConnection(this.config.url, this.jdbcProps)); }, // @ts-expect-error Promise vs Promise From d1bc137e0862a3ed80b3832327d7741b0f883046 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 18:07:06 +0200 Subject: [PATCH 13/27] [DO NOT MERGE] more debug for mvn in JDBC --- packages/cubejs-jdbc-driver/src/JDBCDriver.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index ea2b71b428ce3..3d64a54927978 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -37,9 +37,12 @@ let mvnPromise: Promise | null = null; const initMvn = (customClassPath: any) => { if (!mvnPromise) { mvnPromise = new Promise((resolve, reject) => { + console.log('new mvnPromise'); + const options = { packageJsonPath: `${path.join(__dirname, '../..')}/package.json`, }; + console.log('mvnPromise options', options); mvn(options, (err: any, mvnResults: any) => { if (err && !err.message.includes('Could not find java property')) { reject(err); From 2f454785be9f1420dc0f231c776fa02b2348ce88 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 18:09:49 +0200 Subject: [PATCH 14/27] [DO NOT MERGE] more debug for mvn in JDBC --- packages/cubejs-jdbc-driver/src/JDBCDriver.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index 3d64a54927978..b8705cd1e163e 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -44,6 +44,9 @@ const initMvn = (customClassPath: any) => { }; console.log('mvnPromise options', options); mvn(options, (err: any, mvnResults: any) => { + console.log('mvnPromise callback err', err); + console.log('mvnPromise callback mvnResults', mvnResults); + if (err && !err.message.includes('Could not find java property')) { reject(err); } else { From cdef2e01d3857eaee2cbd8dd7abd4f489b25b509 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 18:47:59 +0200 Subject: [PATCH 15/27] [DO NOT MERGE] more debug for jinst in JDBC --- packages/cubejs-jdbc-driver/src/JDBCDriver.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index b8705cd1e163e..592fd118d3337 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -47,10 +47,16 @@ const initMvn = (customClassPath: any) => { console.log('mvnPromise callback err', err); console.log('mvnPromise callback mvnResults', mvnResults); + console.log('mvnPromise callback err.message.includes', err.message.includes('Could not find java property')); + if (err && !err.message.includes('Could not find java property')) { reject(err); } else { + console.log('jinst.isJvmCreated()', jinst.isJvmCreated()); + if (!jinst.isJvmCreated()) { + console.log('before jinst.addOption'); + jinst.addOption('-Xrs'); jinst.addOption('-Dfile.encoding=UTF8'); @@ -59,6 +65,8 @@ const initMvn = (customClassPath: any) => { jinst.addOption('--add-opens=java.base/java.nio=ALL-UNNAMED'); const classPath = (mvnResults && mvnResults.classpath || []).concat(customClassPath || []); + + console.log('before jinst.setupClasspath'); jinst.setupClasspath(classPath); } resolve(); From 79fa08f5436ba762b07dda89c5d83db6f5f45d7c Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 18:49:30 +0200 Subject: [PATCH 16/27] [DO NOT MERGE] more debug from node --- .../test/databricks-jdbc-full.test.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index b4eca917139d7..58bfa781b56be 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -44,9 +44,19 @@ async function f() { console.log('qRes', qRes); } -f().catch(e => console.log('driver test query failed', e)); +f() + .then( + () => { + console.log('driver test query succ'); + process.exit(0); + }, + e => { + console.log('driver test query failed', e); + process.exit(1); + } + ); -testQueries('databricks-jdbc', { - includeIncrementalSchemaSuite: true, - includeHLLSuite: true, -}); +// testQueries('databricks-jdbc', { +// includeIncrementalSchemaSuite: true, +// includeHLLSuite: true, +// }); From 553fdff060cb4d6989fdd2e58a72a038ffec36bf Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 19:07:19 +0200 Subject: [PATCH 17/27] [DO NOT MERGE] simplify test script --- packages/cubejs-testing-drivers/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/package.json b/packages/cubejs-testing-drivers/package.json index 3c5d600a948e3..222c93b062f5d 100644 --- a/packages/cubejs-testing-drivers/package.json +++ b/packages/cubejs-testing-drivers/package.json @@ -27,7 +27,7 @@ "clickhouse-export-bucket-s3-full": "yarn test-driver -i dist/test/clickhouse-export-bucket-s3-full.test.js", "databricks-jdbc-driver": "yarn test-driver -i dist/test/databricks-jdbc-driver.test.js", "databricks-jdbc-core": "yarn test-driver -i dist/test/databricks-jdbc-core.test.js", - "databricks-jdbc-full": "yarn test-driver -i dist/test/databricks-jdbc-full.test.js", + "databricks-jdbc-full": "node dist/test/databricks-jdbc-full.test.js", "databricks-jdbc-export-bucket-s3-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-s3-full.test.js", "databricks-jdbc-export-bucket-azure-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-azure-full.test.js", "mssql-driver": "yarn test-driver -i dist/test/mssql-driver.test.js", From 1f3d7a71b57e70ad1cdd3b45601a6a475a8186ac Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 20:58:29 +0200 Subject: [PATCH 18/27] [DO NOT MERGE] simplify test script --- .../cubejs-testing-drivers/test/databricks-jdbc-full.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index 58bfa781b56be..df6deca2dd998 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -38,6 +38,7 @@ https.get( async function f() { const source = new D({ url: 'https://ohio.cloud.databricks.com', + maxPoolSize: 1, }); console.log('test driver constructed'); const qRes = await source.query('SELECT 1 as foo;', []); From a4a9681de13a659a0525aff6da278c5ffb2674d5 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 21:17:20 +0200 Subject: [PATCH 19/27] [DO NOT MERGE] more debug from JDBC --- packages/cubejs-jdbc-driver/src/JDBCDriver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index 592fd118d3337..c45fd385b63ce 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -151,7 +151,7 @@ export class JDBCDriver extends BaseDriver { this.pool = genericPool.createPool({ create: async () => { - console.log("before await this.getCustomClassPath()"); + console.log("before await this.getCustomClassPath()", new Error().stack); const ccp = await this.getCustomClassPath(); console.log("before await initMvn(ccp)"); await initMvn(ccp); From b34c26afbde217d6c4a523d25eb10e12f9ecb158 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 22:31:32 +0200 Subject: [PATCH 20/27] [DO NOT MERGE] more debug from JDBC --- packages/cubejs-jdbc-driver/src/JDBCDriver.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index c45fd385b63ce..2aaff204228b8 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -151,9 +151,9 @@ export class JDBCDriver extends BaseDriver { this.pool = genericPool.createPool({ create: async () => { - console.log("before await this.getCustomClassPath()", new Error().stack); + console.log('before await this.getCustomClassPath()', new Error().stack); const ccp = await this.getCustomClassPath(); - console.log("before await initMvn(ccp)"); + console.log('before await initMvn(ccp)'); await initMvn(ccp); if (!this.jdbcProps) { @@ -162,7 +162,7 @@ export class JDBCDriver extends BaseDriver { } const getConnection = promisify(DriverManager.getConnection.bind(DriverManager)); - console.log("new Connection this.config.url", this.config.url); + console.log('new Connection this.config.url', this.config.url); return new Connection(await getConnection(this.config.url, this.jdbcProps)); }, // @ts-expect-error Promise vs Promise @@ -201,6 +201,10 @@ export class JDBCDriver extends BaseDriver { acquireTimeoutMillis: 120000, ...(poolOptions || {}) }) as ExtendedPool; + + this.pool.on('factoryCreateError', (err) => { + console.log('pool factoryCreateError', err); + }); } protected async getCustomClassPath() { From 04d9dab5d89f9bf87d68a0e162794706a61b6fd7 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sat, 2 Nov 2024 23:56:00 +0200 Subject: [PATCH 21/27] [DO NOT MERGE] more debug from JDBC --- .github/workflows/drivers-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index cf873934b37ea..62c92876630fc 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -333,7 +333,7 @@ jobs: dig ohio.cloud.databricks.com. curl -v https://ohio.cloud.databricks.com curl -v https://ohio.cloud.databricks.com/sql/1.0/warehouses/ - node --eval='require("https").get("https://ohio.cloud.databricks.com/sql/1.0/warehouses/", (res) => {console.log("res status and headers", res.statusCode, res.statusMessage, res.headers); res.on("data", (chunk) => console.log("res data", chunk.toString()))})' + CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true node --eval='require("https").get("https://ohio.cloud.databricks.com/sql/1.0/warehouses/", (res) => {console.log("res status and headers", res.statusCode, res.statusMessage, res.headers); res.on("data", (chunk) => console.log("res data", chunk.toString()))})' cd ./packages/cubejs-testing-drivers export DEBUG=testcontainers yarn ${{ matrix.database }}-full From 35a012e56fc9a089da69451d3e368f40e2dc7537 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sun, 3 Nov 2024 01:28:41 +0200 Subject: [PATCH 22/27] [DO NOT MERGE] more debug from JDBC --- packages/cubejs-databricks-jdbc-driver/src/installer.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/cubejs-databricks-jdbc-driver/src/installer.ts b/packages/cubejs-databricks-jdbc-driver/src/installer.ts index ccfd09682a640..c1910ac53c24a 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/installer.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/installer.ts @@ -2,7 +2,10 @@ import path from 'path'; import { downloadAndExtractFile, getEnv } from '@cubejs-backend/shared'; function acceptedByEnv() { + console.log('acceptedByEnv call'); + const acceptStatus = getEnv('databrickAcceptPolicy'); + console.log('acceptedByEnv call acceptStatus', acceptStatus); if (acceptStatus) { console.log('You accepted Terms & Conditions for JDBC driver from DataBricks by CUBEJS_DB_DATABRICKS_ACCEPT_POLICY'); } @@ -16,8 +19,11 @@ function acceptedByEnv() { } export async function downloadJDBCDriver(): Promise { + console.log('downloadJDBCDriver call'); const driverAccepted = acceptedByEnv(); + console.log('downloadJDBCDriver call driverAccepted', driverAccepted); + if (driverAccepted) { console.log('Downloading DatabricksJDBC42-2.6.40.1071'); From fc8e33176bb439b29844aff77987488011c797e1 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sun, 3 Nov 2024 02:24:01 +0200 Subject: [PATCH 23/27] [DO NOT MERGE] fix databricks accept --- .github/workflows/drivers-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 62c92876630fc..e7e364fdc7fdb 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -336,5 +336,5 @@ jobs: CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true node --eval='require("https").get("https://ohio.cloud.databricks.com/sql/1.0/warehouses/", (res) => {console.log("res status and headers", res.statusCode, res.statusMessage, res.headers); res.on("data", (chunk) => console.log("res data", chunk.toString()))})' cd ./packages/cubejs-testing-drivers export DEBUG=testcontainers - yarn ${{ matrix.database }}-full + CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true yarn ${{ matrix.database }}-full From 80137b316839eaca2112227e57af32ec40aca5e3 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sun, 3 Nov 2024 02:38:33 +0200 Subject: [PATCH 24/27] [DO NOT MERGE] fix databricks test jdbc url schema --- .../cubejs-testing-drivers/test/databricks-jdbc-full.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index df6deca2dd998..d12bd9ffc08ff 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -37,7 +37,7 @@ https.get( async function f() { const source = new D({ - url: 'https://ohio.cloud.databricks.com', + url: 'jdbc:databricks://ohio.cloud.databricks.com', maxPoolSize: 1, }); console.log('test driver constructed'); From f3764fe01c233445c7eb02a66133d4b1c9f2e192 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sun, 3 Nov 2024 02:39:51 +0200 Subject: [PATCH 25/27] [DO NOT MERGE] add debug timeout --- .../cubejs-testing-drivers/test/databricks-jdbc-full.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index d12bd9ffc08ff..263b4c55c713e 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -57,6 +57,11 @@ f() } ); +setTimeout(() => { + console.log('TIMING OUT'); + process.exit(1); +}, 3 * 60 * 1000); + // testQueries('databricks-jdbc', { // includeIncrementalSchemaSuite: true, // includeHLLSuite: true, From ade37eab4cf11334825ef74447451d8df93d5297 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sun, 3 Nov 2024 02:53:52 +0200 Subject: [PATCH 26/27] [DO NOT MERGE] add fake http path --- .../cubejs-testing-drivers/test/databricks-jdbc-full.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index 263b4c55c713e..453ad8eb81b4e 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -37,7 +37,7 @@ https.get( async function f() { const source = new D({ - url: 'jdbc:databricks://ohio.cloud.databricks.com', + url: 'jdbc:databricks://ohio.cloud.databricks.com/default;httpPath=/sql/1.0/warehouses/;', maxPoolSize: 1, }); console.log('test driver constructed'); From a7f46646685d542a6724d658b482fcf9b4bc10c9 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Sun, 3 Nov 2024 03:11:19 +0200 Subject: [PATCH 27/27] [DO NOT MERGE] hide some logs and just run tests --- packages/cubejs-jdbc-driver/src/JDBCDriver.ts | 6 +- packages/cubejs-testing-drivers/package.json | 2 +- .../test/databricks-jdbc-full.test.ts | 128 +++++++++--------- 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts index 2aaff204228b8..298c693e56b4b 100644 --- a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts +++ b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts @@ -151,9 +151,9 @@ export class JDBCDriver extends BaseDriver { this.pool = genericPool.createPool({ create: async () => { - console.log('before await this.getCustomClassPath()', new Error().stack); + // console.log('before await this.getCustomClassPath()', new Error().stack); const ccp = await this.getCustomClassPath(); - console.log('before await initMvn(ccp)'); + // console.log('before await initMvn(ccp)'); await initMvn(ccp); if (!this.jdbcProps) { @@ -162,7 +162,7 @@ export class JDBCDriver extends BaseDriver { } const getConnection = promisify(DriverManager.getConnection.bind(DriverManager)); - console.log('new Connection this.config.url', this.config.url); + // console.log('new Connection this.config.url', this.config.url); return new Connection(await getConnection(this.config.url, this.jdbcProps)); }, // @ts-expect-error Promise vs Promise diff --git a/packages/cubejs-testing-drivers/package.json b/packages/cubejs-testing-drivers/package.json index 222c93b062f5d..3c5d600a948e3 100644 --- a/packages/cubejs-testing-drivers/package.json +++ b/packages/cubejs-testing-drivers/package.json @@ -27,7 +27,7 @@ "clickhouse-export-bucket-s3-full": "yarn test-driver -i dist/test/clickhouse-export-bucket-s3-full.test.js", "databricks-jdbc-driver": "yarn test-driver -i dist/test/databricks-jdbc-driver.test.js", "databricks-jdbc-core": "yarn test-driver -i dist/test/databricks-jdbc-core.test.js", - "databricks-jdbc-full": "node dist/test/databricks-jdbc-full.test.js", + "databricks-jdbc-full": "yarn test-driver -i dist/test/databricks-jdbc-full.test.js", "databricks-jdbc-export-bucket-s3-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-s3-full.test.js", "databricks-jdbc-export-bucket-azure-full": "yarn test-driver -i dist/test/databricks-jdbc-export-bucket-azure-full.test.js", "mssql-driver": "yarn test-driver -i dist/test/mssql-driver.test.js", diff --git a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts index 453ad8eb81b4e..6073ca8b911ac 100644 --- a/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts +++ b/packages/cubejs-testing-drivers/test/databricks-jdbc-full.test.ts @@ -1,68 +1,68 @@ -import * as https from 'node:https'; - -import D from '@cubejs-backend/databricks-jdbc-driver'; +// import * as https from 'node:https'; +// +// import D from '@cubejs-backend/databricks-jdbc-driver'; import { testQueries } from '../src/tests/testQueries'; -https.get( - 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', - (res) => { - console.log( - 'res status and headers', - res.statusCode, - res.statusMessage, - res.headers - ); - res.on('data', (chunk) => console.log('res data', chunk.toString())); - } -); - -https.get( - 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', - { - headers: { - authorization: `Bearer ${process.env.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN}`, - } - }, - (res) => { - console.log( - 'res with token status and headers', - res.statusCode, - res.statusMessage, - res.headers - ); - res.on('data', (chunk) => console.log('res with token data', chunk.toString())); - } -); - -async function f() { - const source = new D({ - url: 'jdbc:databricks://ohio.cloud.databricks.com/default;httpPath=/sql/1.0/warehouses/;', - maxPoolSize: 1, - }); - console.log('test driver constructed'); - const qRes = await source.query('SELECT 1 as foo;', []); - console.log('qRes', qRes); -} - -f() - .then( - () => { - console.log('driver test query succ'); - process.exit(0); - }, - e => { - console.log('driver test query failed', e); - process.exit(1); - } - ); - -setTimeout(() => { - console.log('TIMING OUT'); - process.exit(1); -}, 3 * 60 * 1000); +// https.get( +// 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', +// (res) => { +// console.log( +// 'res status and headers', +// res.statusCode, +// res.statusMessage, +// res.headers +// ); +// res.on('data', (chunk) => console.log('res data', chunk.toString())); +// } +// ); +// +// https.get( +// 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/', +// { +// headers: { +// authorization: `Bearer ${process.env.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN}`, +// } +// }, +// (res) => { +// console.log( +// 'res with token status and headers', +// res.statusCode, +// res.statusMessage, +// res.headers +// ); +// res.on('data', (chunk) => console.log('res with token data', chunk.toString())); +// } +// ); +// +// async function f() { +// const source = new D({ +// url: 'jdbc:databricks://ohio.cloud.databricks.com/default;httpPath=/sql/1.0/warehouses/;', +// maxPoolSize: 1, +// }); +// console.log('test driver constructed'); +// const qRes = await source.query('SELECT 1 as foo;', []); +// console.log('qRes', qRes); +// } +// +// f() +// .then( +// () => { +// console.log('driver test query succ'); +// process.exit(0); +// }, +// e => { +// console.log('driver test query failed', e); +// process.exit(1); +// } +// ); +// +// setTimeout(() => { +// console.log('TIMING OUT'); +// process.exit(1); +// }, 3 * 60 * 1000); -// testQueries('databricks-jdbc', { -// includeIncrementalSchemaSuite: true, -// includeHLLSuite: true, -// }); +testQueries('databricks-jdbc', { + includeIncrementalSchemaSuite: true, + includeHLLSuite: true, +});