Skip to content

Commit e1a1f02

Browse files
KSDaemonKSDaemon
authored
chore(snowflake-driver): Update snowflake-sdk to 2.0.x version (#9370)
* update snowflake-sdk * code polish * missed version up * fix HLL_SNOWFLAKE serialization * remove util.construct_hostname
1 parent a93ae85 commit e1a1f02

File tree

8 files changed

+1033
-1089
lines changed

8 files changed

+1033
-1089
lines changed

‎packages/cubejs-bigquery-driver/src/BigQueryDriver.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,12 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface {
146146
}
147147
}
148148

149+
/**
150+
* Returns the configurable driver options
151+
* Note: It returns the unprefixed option names.
152+
* In case of using multisources options need to be prefixed manually.
153+
*/
149154
public static driverEnvVariables() {
150-
// TODO (buntarb): check how this method can/must be used with split
151-
// names by the data source.
152155
return [
153156
'CUBEJS_DB_BQ_PROJECT_ID',
154157
'CUBEJS_DB_BQ_KEY_FILE',

‎packages/cubejs-elasticsearch-driver/driver/ElasticSearchDriver.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,12 @@ class ElasticSearchDriver extends BaseDriver {
8484
}
8585
}
8686

87+
/**
88+
* Returns the configurable driver options
89+
* Note: It returns the unprefixed option names.
90+
* In case of using multisources options need to be prefixed manually.
91+
*/
8792
static driverEnvVariables() {
88-
// TODO (buntarb): check how this method can/must be used with split
89-
// names by the data source.
9093
return [
9194
'CUBEJS_DB_URL',
9295
'CUBEJS_DB_ELASTIC_QUERY_FORMAT',

‎packages/cubejs-ksql-driver/src/KsqlDriver.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -346,9 +346,12 @@ export class KsqlDriver extends BaseDriver implements DriverInterface {
346346
return `\`${identifier}\``;
347347
}
348348

349+
/**
350+
* Returns the configurable driver options
351+
* Note: It returns the unprefixed option names.
352+
* In case of using multisources options need to be prefixed manually.
353+
*/
349354
public static driverEnvVariables() {
350-
// TODO (buntarb): check how this method can/must be used with split
351-
// names by the data source.
352355
return [
353356
'CUBEJS_DB_URL',
354357
'CUBEJS_DB_USER',

‎packages/cubejs-mssql-driver/driver/MSSqlDriver.js‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,19 @@ class MSSqlDriver extends BaseDriver {
8282
this.initialConnectPromise = this.connectionPool.connect();
8383
}
8484

85+
/**
86+
* Returns the configurable driver options
87+
* Note: It returns the unprefixed option names.
88+
* In case of using multisources options need to be prefixed manually.
89+
*/
8590
static driverEnvVariables() {
86-
// TODO (buntarb): check how this method can/must be used with split
87-
// names by the data source.
8891
return [
89-
'CUBEJS_DB_HOST', 'CUBEJS_DB_NAME', 'CUBEJS_DB_PORT', 'CUBEJS_DB_USER', 'CUBEJS_DB_PASS', 'CUBEJS_DB_DOMAIN'
92+
'CUBEJS_DB_HOST',
93+
'CUBEJS_DB_NAME',
94+
'CUBEJS_DB_PORT',
95+
'CUBEJS_DB_USER',
96+
'CUBEJS_DB_PASS',
97+
'CUBEJS_DB_DOMAIN',
9098
];
9199
}
92100

@@ -97,8 +105,8 @@ class MSSqlDriver extends BaseDriver {
97105
/**
98106
* Executes query in streaming mode.
99107
*
100-
* @param {string} query
101-
* @param {Array} values
108+
* @param {string} query
109+
* @param {Array} values
102110
* @param {{ highWaterMark: number? }} options
103111
* @return {Promise<StreamTableDataWithTypes>}
104112
*/
@@ -151,7 +159,7 @@ class MSSqlDriver extends BaseDriver {
151159
* scale: number?,
152160
* precision: number?
153161
* }
154-
* }} fields
162+
* }} fields
155163
*/
156164
mapFields(fields) {
157165
return Object.keys(fields).map((field) => {

‎packages/cubejs-mysql-driver/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"@cubejs-backend/base-driver": "1.2.25",
3131
"@cubejs-backend/shared": "1.2.25",
3232
"@types/mysql": "^2.15.21",
33-
"generic-pool": "^3.6.0",
33+
"generic-pool": "^3.8.2",
3434
"mysql": "^2.18.1"
3535
},
3636
"devDependencies": {

‎packages/cubejs-snowflake-driver/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
"@cubejs-backend/base-driver": "1.2.25",
2929
"@cubejs-backend/shared": "1.2.25",
3030
"date-fns-timezone": "^0.1.4",
31-
"snowflake-sdk": "^1.13.1"
31+
"snowflake-sdk": "^2.0.3"
3232
},
3333
"license": "Apache-2.0",
3434
"publishConfig": {

‎packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts‎

Lines changed: 32 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -4,51 +4,28 @@
44
* @fileoverview The `SnowflakeDriver` and related types declaration.
55
*/
66

7-
import {
8-
getEnv,
9-
assertDataSource,
10-
} from '@cubejs-backend/shared';
7+
import { assertDataSource, getEnv, } from '@cubejs-backend/shared';
118
import snowflake, { Column, Connection, RowStatement } from 'snowflake-sdk';
129
import {
1310
BaseDriver,
11+
DownloadQueryResultsOptions,
12+
DownloadQueryResultsResult,
1413
DownloadTableCSVData,
14+
DownloadTableMemoryData,
15+
DriverCapabilities,
1516
DriverInterface,
1617
GenericDataBaseType,
17-
TableStructure,
18-
UnloadOptions,
1918
StreamOptions,
2019
StreamTableDataWithTypes,
21-
DownloadTableMemoryData,
22-
DownloadQueryResultsResult,
23-
DownloadQueryResultsOptions,
24-
DriverCapabilities,
20+
TableStructure,
21+
UnloadOptions,
2522
} from '@cubejs-backend/base-driver';
2623
import { formatToTimeZone } from 'date-fns-timezone';
2724
import fs from 'fs/promises';
2825
import { HydrationMap, HydrationStream } from './HydrationStream';
2926

30-
// eslint-disable-next-line import/order
31-
const util = require('snowflake-sdk/lib/util');
32-
3327
const SUPPORTED_BUCKET_TYPES = ['s3', 'gcs', 'azure'];
3428

35-
// TODO Remove when https://github.com/snowflakedb/snowflake-connector-nodejs/pull/158 is resolved
36-
util.construct_hostname = (region: any, account: any) => {
37-
let host;
38-
if (region === 'us-west-2') {
39-
region = null;
40-
}
41-
if (account.indexOf('.') > 0) {
42-
account = account.substring(0, account.indexOf('.'));
43-
}
44-
if (region) {
45-
host = `${account}.${region}.snowflakecomputing.com`;
46-
} else {
47-
host = `${account}.snowflakecomputing.com`;
48-
}
49-
return host;
50-
};
51-
5229
type HydrationConfiguration = {
5330
types: string[], toValue: (column: Column) => ((value: any) => any) | null
5431
};
@@ -104,11 +81,21 @@ const hydrators: HydrationConfiguration[] = [
10481
}
10582
);
10683
},
84+
},
85+
{
86+
types: ['object'], // Workaround for HLL_SNOWFLAKE
87+
toValue: () => (value) => {
88+
if (!value) {
89+
return null;
90+
}
91+
92+
return JSON.stringify(value);
93+
},
10794
}
10895
];
10996

11097
const SnowflakeToGenericType: Record<string, GenericDataBaseType> = {
111-
// It's a limitation for now, because anyway we dont work with JSON objects in Cube Store.
98+
// It's a limitation for now, because anyway we don't work with JSON objects in Cube Store.
11299
object: 'HLL_SNOWFLAKE',
113100
number: 'decimal',
114101
timestamp_ntz: 'timestamp'
@@ -198,9 +185,12 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
198185
return 8;
199186
}
200187

188+
/**
189+
* Returns the configurable driver options
190+
* Note: It returns the unprefixed option names.
191+
* In case of using multisources options need to be prefixed manually.
192+
*/
201193
public static driverEnvVariables() {
202-
// TODO (buntarb): check how this method can/must be used with split
203-
// names by the data source.
204194
return [
205195
'CUBEJS_DB_NAME',
206196
'CUBEJS_DB_USER',
@@ -211,9 +201,11 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
211201
'CUBEJS_DB_SNOWFLAKE_ROLE',
212202
'CUBEJS_DB_SNOWFLAKE_CLIENT_SESSION_KEEP_ALIVE',
213203
'CUBEJS_DB_SNOWFLAKE_AUTHENTICATOR',
204+
'CUBEJS_DB_SNOWFLAKE_OAUTH_TOKEN_PATH',
205+
'CUBEJS_DB_SNOWFLAKE_HOST',
206+
'CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY',
214207
'CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PATH',
215208
'CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PASS',
216-
'CUBEJS_DB_SNOWFLAKE_OAUTH_TOKEN_PATH',
217209
'CUBEJS_DB_SNOWFLAKE_QUOTED_IDENTIFIERS_IGNORE_CASE',
218210
];
219211
}
@@ -257,10 +249,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
257249
privateKey += '\n';
258250
}
259251

260-
snowflake.configure({
261-
// TODO: Remove after release of https://github.com/snowflakedb/snowflake-connector-nodejs/pull/912
262-
logLevel: 'OFF' as any
263-
});
252+
snowflake.configure({ logLevel: 'OFF' });
264253

265254
this.config = {
266255
readOnly: false,
@@ -416,9 +405,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
416405
this.config.token = await this.readOAuthToken();
417406
}
418407

419-
const connection = snowflake.createConnection(this.config);
420-
421-
return connection;
408+
return snowflake.createConnection(this.config);
422409
}
423410

424411
/**
@@ -475,8 +462,8 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
475462
}
476463
}
477464

478-
// eslint-disable-next-line no-return-assign
479-
return this.connection = this.initConnection();
465+
this.connection = this.initConnection();
466+
return this.connection;
480467
}
481468

482469
/**
@@ -758,7 +745,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
758745
const hydrationMap = this.generateHydrationMap(stmt.getColumns());
759746
const types: {name: string, type: string}[] =
760747
this.getTypes(stmt);
761-
if (rows && rows.length && Object.keys(hydrationMap).length) {
748+
if (rows?.length && Object.keys(hydrationMap).length) {
762749
for (const row of rows) {
763750
for (const [field, toValue] of Object.entries(hydrationMap)) {
764751
if (row.hasOwnProperty(field)) {
@@ -922,6 +909,6 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
922909

923910
public async getTablesQuery(schemaName: string) {
924911
const tables = await super.getTablesQuery(schemaName.toUpperCase());
925-
return tables.map(t => ({ table_name: t.TABLE_NAME && t.TABLE_NAME.toLowerCase() }));
912+
return tables.map(t => ({ table_name: t.TABLE_NAME?.toLowerCase() }));
926913
}
927914
}

0 commit comments

Comments
 (0)