From 70beacc2ce1382599de2a37a6ed034e0c6e9ef80 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Wed, 12 Mar 2025 23:20:26 +0200 Subject: [PATCH 1/2] [DEBUG] Add more logs to debug athena --- .github/workflows/drivers-tests.yml | 2 +- packages/cubejs-athena-driver/src/AthenaDriver.ts | 1 + .../src/helpers/buildPreaggs.ts | 11 ++++++++++- .../cubejs-testing-drivers/src/tests/testQueries.ts | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index ec9d3a16d5526..764d90a84d014 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -343,6 +343,6 @@ jobs: timeout_minutes: 20 command: | cd ./packages/cubejs-testing-drivers - export DEBUG=testcontainers + export DEBUG=testcontainers* yarn ${{ matrix.database }}-full diff --git a/packages/cubejs-athena-driver/src/AthenaDriver.ts b/packages/cubejs-athena-driver/src/AthenaDriver.ts index aaea16c04a275..818694be315f9 100644 --- a/packages/cubejs-athena-driver/src/AthenaDriver.ts +++ b/packages/cubejs-athena-driver/src/AthenaDriver.ts @@ -480,6 +480,7 @@ export class AthenaDriver extends BaseDriver implements DriverInterface { ...(this.config.catalog != null ? { QueryExecutionContext: { Catalog: this.config.catalog } } : {}) }; const { QueryExecutionId } = await this.athena.startQueryExecution(request); + console.log('athena started query ', query, QueryExecutionId); return { QueryExecutionId: checkNonNullable('StartQueryExecution', QueryExecutionId) }; } diff --git a/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts b/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts index 4570e8bbc949a..5eaec7f732875 100644 --- a/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts +++ b/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts @@ -53,14 +53,22 @@ export async function buildPreaggs( selector: any, ) { return new Promise((resolve, reject) => { + console.log('Starting building pre-agg', selector); + postRequest( port, '/cubejs-api/v1/pre-aggregations/jobs', token, { action: 'post', selector }, ).then((post) => { + console.log('Building pre-agg postRequest callback', selector); + readData(post).then((_jobs) => { + console.log('Building pre-agg readData callback', selector); + const jobs = JSON.parse(_jobs.toString()); + + console.log('Building pre-agg readData jobs', selector, jobs); if (jobs.length === 0) { resolve(true); } else { @@ -73,6 +81,7 @@ export async function buildPreaggs( { action: 'get', resType: 'object', tokens: jobs }, ); const statuses = JSON.parse((await readData(get)).toString()); + console.log('Building pre-agg readData jobs statuses', selector, statuses); Object.keys(statuses).forEach((t: string) => { const { status } = statuses[t]; if (status.indexOf('failure') >= 0) { @@ -135,7 +144,7 @@ export async function hookPreaggs( }, tokens, ); - + selectors.forEach((info) => { const { status } = info; if (status.indexOf('failure') >= 0) { diff --git a/packages/cubejs-testing-drivers/src/tests/testQueries.ts b/packages/cubejs-testing-drivers/src/tests/testQueries.ts index b755aea1772be..09a9ca39ca991 100644 --- a/packages/cubejs-testing-drivers/src/tests/testQueries.ts +++ b/packages/cubejs-testing-drivers/src/tests/testQueries.ts @@ -141,6 +141,7 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten // MUST be the first test in the list! execute('must built pre-aggregations', async () => { + console.log('Starting building pre-aggs'); await buildPreaggs(env.cube.port, apiToken, { timezones: ['UTC'], preAggregations: ['Customers.RAExternal'], From 4b82829f4c2d5ca12acb2ae1bb84d62d3c731da7 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Thu, 13 Mar 2025 10:25:26 +0200 Subject: [PATCH 2/2] [DEBUG] Add more logs to debug athena --- .../cubejs-testing-drivers/src/helpers/buildPreaggs.ts | 9 --------- .../src/tests/testIncrementalSchemaLoading.ts | 9 +++++++++ packages/cubejs-testing-drivers/src/tests/testQueries.ts | 1 - 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts b/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts index 5eaec7f732875..f752bcaa3e576 100644 --- a/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts +++ b/packages/cubejs-testing-drivers/src/helpers/buildPreaggs.ts @@ -53,22 +53,14 @@ export async function buildPreaggs( selector: any, ) { return new Promise((resolve, reject) => { - console.log('Starting building pre-agg', selector); - postRequest( port, '/cubejs-api/v1/pre-aggregations/jobs', token, { action: 'post', selector }, ).then((post) => { - console.log('Building pre-agg postRequest callback', selector); - readData(post).then((_jobs) => { - console.log('Building pre-agg readData callback', selector); - const jobs = JSON.parse(_jobs.toString()); - - console.log('Building pre-agg readData jobs', selector, jobs); if (jobs.length === 0) { resolve(true); } else { @@ -81,7 +73,6 @@ export async function buildPreaggs( { action: 'get', resType: 'object', tokens: jobs }, ); const statuses = JSON.parse((await readData(get)).toString()); - console.log('Building pre-agg readData jobs statuses', selector, statuses); Object.keys(statuses).forEach((t: string) => { const { status } = statuses[t]; if (status.indexOf('failure') >= 0) { diff --git a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts index 8cfcfe3e0916c..a29d71d6a8ed5 100644 --- a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts +++ b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts @@ -26,7 +26,11 @@ export function incrementalSchemaLoadingSuite( }); execute('should load schemas', async () => { + const start = performance.now(); + console.log('should load schemas', 'start'); const inputSchemas: QuerySchemasResult[] = await driver().getSchemas(); + console.log('should load schemas', 'after getSchemas', performance.now() - start); + console.log('should load schemas', 'inputSchemas', inputSchemas); expect(inputSchemas).toBeInstanceOf(Array); expect(inputSchemas.length).toBeGreaterThan(0); expect(inputSchemas).toContainEqual({ @@ -35,9 +39,14 @@ export function incrementalSchemaLoadingSuite( }); execute('should load tables for specific schemas', async () => { + const start = performance.now(); + console.log('should load tables for specific schemas', 'start'); let inputSchemas: QuerySchemasResult[] = await driver().getSchemas(); + console.log('should load schemas', 'after getSchemas', performance.now() - start); inputSchemas = inputSchemas.filter((s) => !!s.schema_name); const inputTables = await driver().getTablesForSpecificSchemas(inputSchemas); + console.log('should load schemas', 'after getTablesForSpecificSchemas', performance.now() - start); + console.log('should load schemas', 'inputTables', inputTables); expect(inputTables).toBeInstanceOf(Array); expect(inputTables.length).toBeGreaterThan(0); expect(inputTables).toContainEqual({ diff --git a/packages/cubejs-testing-drivers/src/tests/testQueries.ts b/packages/cubejs-testing-drivers/src/tests/testQueries.ts index 09a9ca39ca991..b755aea1772be 100644 --- a/packages/cubejs-testing-drivers/src/tests/testQueries.ts +++ b/packages/cubejs-testing-drivers/src/tests/testQueries.ts @@ -141,7 +141,6 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten // MUST be the first test in the list! execute('must built pre-aggregations', async () => { - console.log('Starting building pre-aggs'); await buildPreaggs(env.cube.port, apiToken, { timezones: ['UTC'], preAggregations: ['Customers.RAExternal'],