Skip to content

Commit 46f7dbd

Browse files
paveltiunovpaveltiunov
authored
fix(athena-driver): Internal: Error during planning: Coercion from [Utf8, Utf8] to the signature Exact([Utf8, Timestamp(Nanosecond, None)]) failed for athena pre-aggregations (#6655)
* fix(athena-driver): Internal: Error during planning: Coercion from [Utf8, Utf8] to the signature Exact([Utf8, Timestamp(Nanosecond, None)]) failed for athena pre-aggregations * Increase JDBC test connection timeout to address ResourceRequest timed out databricks issues * Try sequential fixture table creation to address databricks timeout
1 parent 05fcdca commit 46f7dbd

File tree

11 files changed

+401
-63
lines changed

11 files changed

+401
-63
lines changed

‎packages/cubejs-athena-driver/src/AthenaDriver.ts‎

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -428,47 +428,7 @@ export class AthenaDriver extends BaseDriver implements DriverInterface {
428428
* fields meta info.
429429
*/
430430
public mapTypes(fields: ColumnInfo[]): TableStructure {
431-
return fields.map((field) => {
432-
let type;
433-
switch (field.Type) {
434-
case 'boolean':
435-
type = 'boolean';
436-
break;
437-
// integers
438-
case 'tinyint':
439-
case 'smallint':
440-
case 'int':
441-
case 'integer':
442-
case 'bigint':
443-
type = 'int';
444-
break;
445-
// float
446-
case 'float':
447-
type = 'float';
448-
break;
449-
// double
450-
case 'double':
451-
type = 'double';
452-
break;
453-
// strings
454-
case 'char':
455-
case 'varchar':
456-
case 'string':
457-
case 'binary':
458-
type = 'text';
459-
break;
460-
// date and time
461-
case 'date':
462-
case 'timestamp':
463-
type = 'date';
464-
break;
465-
// unknown
466-
default:
467-
type = 'text';
468-
break;
469-
}
470-
return { name: <string>field.Name, type };
471-
});
431+
return fields.map((field) => ({ name: <string>field.Name, type: this.toGenericType(field.Type || 'text') }));
472432
}
473433

474434
/**

‎packages/cubejs-jdbc-driver/src/JDBCDriver.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,13 @@ export class JDBCDriver extends BaseDriver {
8888

8989
/**
9090
* Time to wait for a response from a connection after validation
91-
* request before determining it as not valid. Default - 30000 ms.
91+
* request before determining it as not valid. Default - 60000 ms.
9292
*/
9393
testConnectionTimeout?: number,
9494
} = {}
9595
) {
9696
super({
97-
testConnectionTimeout: config.testConnectionTimeout || 30000,
97+
testConnectionTimeout: config.testConnectionTimeout || 60000,
9898
});
9999

100100
const dataSource =

‎packages/cubejs-testing-drivers/fixtures/bigquery.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
9797
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
9898
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
99-
"querying ECommerce: partitioned pre-agg"
99+
"querying ECommerce: partitioned pre-agg",
100+
"querying ECommerce: partitioned pre-agg higher granularity"
100101
]
101102
}

‎packages/cubejs-testing-drivers/fixtures/clickhouse.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
9999
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
100100
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
101-
"querying ECommerce: partitioned pre-agg"
101+
"querying ECommerce: partitioned pre-agg",
102+
"querying ECommerce: partitioned pre-agg higher granularity"
102103
]
103104
}

‎packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
7272
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
7373
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
74-
"querying ECommerce: partitioned pre-agg"
74+
"querying ECommerce: partitioned pre-agg",
75+
"querying ECommerce: partitioned pre-agg higher granularity"
7576
]
7677
}

‎packages/cubejs-testing-drivers/fixtures/mssql.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@
9090
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
9191
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
9292
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
93-
"querying ECommerce: partitioned pre-agg"
93+
"querying ECommerce: partitioned pre-agg",
94+
"querying ECommerce: partitioned pre-agg higher granularity"
9495
]
9596
}

‎packages/cubejs-testing-drivers/fixtures/mysql.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
8484
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
8585
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
86-
"querying ECommerce: partitioned pre-agg"
86+
"querying ECommerce: partitioned pre-agg",
87+
"querying ECommerce: partitioned pre-agg higher granularity"
8788
]
8889
}

‎packages/cubejs-testing-drivers/fixtures/postgres.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
8080
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
8181
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
82-
"querying ECommerce: partitioned pre-agg"
82+
"querying ECommerce: partitioned pre-agg",
83+
"querying ECommerce: partitioned pre-agg higher granularity"
8384
]
8485
}

‎packages/cubejs-testing-drivers/src/tests/testQueries.ts‎

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export function testQueries(type: string): void {
1818
const fixtures = getFixtures(type);
1919
let client: CubejsApi;
2020
let driver: BaseDriver;
21-
let query: string[];
21+
let queries: string[];
2222
let env: Environment;
2323

2424
function execute(name: string, test: () => Promise<void>) {
@@ -30,7 +30,7 @@ export function testQueries(type: string): void {
3030
}
3131
const apiToken = sign({}, 'mysupersecret');
3232

33-
const suffix = `_${new Date().getTime().toString(32)}`;
33+
const suffix = new Date().getTime().toString(32);
3434
beforeAll(async () => {
3535
env = await runEnvironment(type, suffix);
3636
process.env.CUBEJS_REFRESH_WORKER = 'true';
@@ -47,23 +47,28 @@ export function testQueries(type: string): void {
4747
apiUrl: `http://127.0.0.1:${env.cube.port}/cubejs-api/v1`,
4848
});
4949
driver = (await getDriver(type)).source;
50-
query = getCreateQueries(type, suffix);
51-
await Promise.all(query.map(async (q) => {
50+
queries = getCreateQueries(type, suffix);
51+
console.log(`Creating ${queries.length} fixture tables`);
52+
for (const q of queries) {
5253
await driver.query(q);
53-
}));
54+
}
55+
console.log(`Creating ${queries.length} fixture tables completed`);
5456
});
5557

5658
afterAll(async () => {
57-
const tables = Object
58-
.keys(fixtures.tables)
59-
.map((key: string) => `${fixtures.tables[<'products' | 'customers' | 'ecommerce'>key]}${suffix}`);
60-
await Promise.all(
61-
tables.map(async (t) => {
59+
try {
60+
const tables = Object
61+
.keys(fixtures.tables)
62+
.map((key: string) => `${fixtures.tables[<'products' | 'customers' | 'ecommerce'>key]}${suffix}`);
63+
console.log(`Dropping ${tables.length} fixture tables`);
64+
for (const t of tables) {
6265
await driver.dropTable(t);
63-
})
64-
);
65-
await driver.release();
66-
await env.stop();
66+
}
67+
console.log(`Dropping ${tables.length} fixture tables completed`);
68+
} finally {
69+
await driver.release();
70+
await env.stop();
71+
}
6772
});
6873

6974
// MUST be the first test in the list!
@@ -1304,5 +1309,26 @@ export function testQueries(type: string): void {
13041309
});
13051310
expect(response.rawData()).toMatchSnapshot();
13061311
});
1312+
1313+
execute('querying ECommerce: partitioned pre-agg higher granularity', async () => {
1314+
const response = await client.load({
1315+
dimensions: [
1316+
'ECommerce.productName'
1317+
],
1318+
measures: [
1319+
'ECommerce.totalQuantity',
1320+
],
1321+
timeDimensions: [{
1322+
dimension: 'ECommerce.orderDate',
1323+
granularity: 'year'
1324+
}],
1325+
order: {
1326+
'ECommerce.totalProfit': 'desc',
1327+
'ECommerce.productName': 'asc'
1328+
},
1329+
total: true
1330+
});
1331+
expect(response.rawData()).toMatchSnapshot();
1332+
});
13071333
});
13081334
}

‎packages/cubejs-testing-drivers/test/__snapshots__/athena-full.test.ts.snap‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5772,6 +5772,179 @@ Array [
57725772
]
57735773
`;
57745774

5775+
exports[`Queries with the @cubejs-backend/athena-driver querying ECommerce: partitioned pre-agg higher granularity 1`] = `
5776+
Array [
5777+
Object {
5778+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5779+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5780+
"ECommerce.productName": "Anderson Hickey Conga Table Tops & Accessories",
5781+
"ECommerce.totalQuantity": "2",
5782+
},
5783+
Object {
5784+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5785+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5786+
"ECommerce.productName": "Balt Solid Wood Rectangular Table",
5787+
"ECommerce.totalQuantity": "2",
5788+
},
5789+
Object {
5790+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5791+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5792+
"ECommerce.productName": "Canon PC1080F Personal Copier",
5793+
"ECommerce.totalQuantity": "7",
5794+
},
5795+
Object {
5796+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5797+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5798+
"ECommerce.productName": "DMI Eclipse Executive Suite Bookcases",
5799+
"ECommerce.totalQuantity": "1",
5800+
},
5801+
Object {
5802+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5803+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5804+
"ECommerce.productName": "Global Adaptabilites Bookcase, Cherry/Storm Gray Finish",
5805+
"ECommerce.totalQuantity": "8",
5806+
},
5807+
Object {
5808+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5809+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5810+
"ECommerce.productName": "Google Nexus 5",
5811+
"ECommerce.totalQuantity": "11",
5812+
},
5813+
Object {
5814+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5815+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5816+
"ECommerce.productName": "Google Nexus 6",
5817+
"ECommerce.totalQuantity": "3",
5818+
},
5819+
Object {
5820+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5821+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5822+
"ECommerce.productName": "Google Nexus 7",
5823+
"ECommerce.totalQuantity": "3",
5824+
},
5825+
Object {
5826+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5827+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5828+
"ECommerce.productName": "HTC One",
5829+
"ECommerce.totalQuantity": "3",
5830+
},
5831+
Object {
5832+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5833+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5834+
"ECommerce.productName": "Harbour Creations 67200 Series Stacking Chairs",
5835+
"ECommerce.totalQuantity": "11",
5836+
},
5837+
Object {
5838+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5839+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5840+
"ECommerce.productName": "Hewlett Packard 610 Color Digital Copier / Printer",
5841+
"ECommerce.totalQuantity": "3",
5842+
},
5843+
Object {
5844+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5845+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5846+
"ECommerce.productName": "Iceberg Nesting Folding Chair, 19w x 6d x 43h",
5847+
"ECommerce.totalQuantity": "4",
5848+
},
5849+
Object {
5850+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5851+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5852+
"ECommerce.productName": "Kingston Digital DataTraveler 16GB USB 2.0",
5853+
"ECommerce.totalQuantity": "5",
5854+
},
5855+
Object {
5856+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5857+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5858+
"ECommerce.productName": "Kingston Digital DataTraveler 16GB USB 2.1",
5859+
"ECommerce.totalQuantity": "5",
5860+
},
5861+
Object {
5862+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5863+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5864+
"ECommerce.productName": "Kingston Digital DataTraveler 16GB USB 2.2",
5865+
"ECommerce.totalQuantity": "8",
5866+
},
5867+
Object {
5868+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5869+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5870+
"ECommerce.productName": "Lexmark 20R1285 X6650 Wireless All-in-One Printer",
5871+
"ECommerce.totalQuantity": "6",
5872+
},
5873+
Object {
5874+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5875+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5876+
"ECommerce.productName": "Linden 10 Round Wall Clock, Black",
5877+
"ECommerce.totalQuantity": "14",
5878+
},
5879+
Object {
5880+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5881+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5882+
"ECommerce.productName": "Logitech diNovo Edge Keyboard",
5883+
"ECommerce.totalQuantity": "9",
5884+
},
5885+
Object {
5886+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5887+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5888+
"ECommerce.productName": "Magna Visual Magnetic Picture Hangers",
5889+
"ECommerce.totalQuantity": "4",
5890+
},
5891+
Object {
5892+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5893+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5894+
"ECommerce.productName": "OIC #2 Pencils, Medium Soft",
5895+
"ECommerce.totalQuantity": "9",
5896+
},
5897+
Object {
5898+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5899+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5900+
"ECommerce.productName": "Okidata C610n Printer",
5901+
"ECommerce.totalQuantity": "2",
5902+
},
5903+
Object {
5904+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5905+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5906+
"ECommerce.productName": "Panasonic KP-380BK Classic Electric Pencil Sharpener",
5907+
"ECommerce.totalQuantity": "8",
5908+
},
5909+
Object {
5910+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5911+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5912+
"ECommerce.productName": "Plymouth Boxed Rubber Bands by Plymouth",
5913+
"ECommerce.totalQuantity": "11",
5914+
},
5915+
Object {
5916+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5917+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5918+
"ECommerce.productName": "Project Tote Personal File",
5919+
"ECommerce.totalQuantity": "1",
5920+
},
5921+
Object {
5922+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5923+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5924+
"ECommerce.productName": "Recycled Eldon Regeneration Jumbo File",
5925+
"ECommerce.totalQuantity": "4",
5926+
},
5927+
Object {
5928+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5929+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5930+
"ECommerce.productName": "Tyvek Side-Opening Peel & Seel Expanding Envelopes",
5931+
"ECommerce.totalQuantity": "2",
5932+
},
5933+
Object {
5934+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5935+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5936+
"ECommerce.productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box",
5937+
"ECommerce.totalQuantity": "11",
5938+
},
5939+
Object {
5940+
"ECommerce.orderDate": "2020-01-01T00:00:00.000",
5941+
"ECommerce.orderDate.year": "2020-01-01T00:00:00.000",
5942+
"ECommerce.productName": "Wausau Papers Astrobrights Colored Envelopes",
5943+
"ECommerce.totalQuantity": "3",
5944+
},
5945+
]
5946+
`;
5947+
57755948
exports[`Queries with the @cubejs-backend/athena-driver querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test 1`] = `
57765949
Array [
57775950
Object {

0 commit comments

Comments
 (0)