Skip to content

Commit 3710b11

Browse files
paveltiunovpaveltiunov
authored
fix(snowflake-driver): Int is exported to pre-aggregations as decimal… (#6513)
* fix(snowflake-driver): Int is exported to pre-aggregations as decimal: Internal: Execution error: Internal: Error during planning: Extension planner for ClusterSendNode * Fix wrong package ordering * Touch snowflake * Add client build * Remove obsolete snapshot * Fix driver test use incorrect API token * Fix athena driver test
1 parent ff374fb commit 3710b11

File tree

13 files changed

+182
-58
lines changed

13 files changed

+182
-58
lines changed

‎.github/actions/integration/snowflake.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
set -eo pipefail
3+
4+
# Debug log for test containers
5+
export DEBUG=testcontainers
6+
7+
echo "::group::Snowflake [cloud]"
8+
export CUBEJS_DB_NAME=DEMO_DB
9+
export CUBEJS_DB_SNOWFLAKE_ACCOUNT=qna80818
10+
export CUBEJS_DB_SNOWFLAKE_REGION=us-east-1
11+
export CUBEJS_DB_SNOWFLAKE_WAREHOUSE=COMPUTE_WH
12+
13+
yarn lerna run --concurrency 1 --stream --no-prefix smoke:snowflake
14+
15+
echo "::endgroup::"

‎.github/workflows/cloud.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
paths:
88
- '.github/actions/integration/athena.sh'
99
- '.github/actions/integration/bigquery.sh'
10+
- '.github/actions/integration/snowflake.sh'
1011
- '.github/workflows/cloud.yml'
1112
- 'packages/cubejs-testing/**'
1213
- 'packages/cubejs-testing-shared/**'
@@ -17,6 +18,7 @@ on:
1718
# drivers
1819
- 'packages/cubejs-athena-driver/**'
1920
- 'packages/cubejs-bigquery-driver/**'
21+
- 'packages/cubejs-snowflake-driver/**'
2022
- 'lerna.json'
2123
- 'package.json'
2224
- 'yarn.lock'
@@ -29,7 +31,7 @@ jobs:
2931
strategy:
3032
matrix:
3133
node-version: [ 16.x ]
32-
db: [ 'athena', 'bigquery' ]
34+
db: [ 'athena', 'bigquery', 'snowflake' ]
3335
fail-fast: false
3436

3537
steps:
@@ -67,6 +69,8 @@ jobs:
6769
run: yarn policies set-version v1.22.19
6870
- name: Yarn install
6971
run: CUBESTORE_SKIP_POST_INSTALL=true yarn install --frozen-lockfile
72+
- name: Build client
73+
run: yarn build
7074
- name: Lerna tsc
7175
run: yarn tsc
7276
- name: Run Integration tests for ${{ matrix.db }} matrix
@@ -75,4 +79,6 @@ jobs:
7579
CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
7680
CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
7781
CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
82+
CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_USER }}
83+
CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_PASS }}
7884
run: ./.github/actions/integration/${{ matrix.db }}.sh

‎packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts‎

Lines changed: 23 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -466,23 +466,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
466466
return;
467467
}
468468
const types: {name: string, type: string}[] =
469-
stmt.getColumns().map((column) => {
470-
const type = {
471-
name: column.getName().toLowerCase(),
472-
type: '',
473-
};
474-
if (column.isNumber()) {
475-
// @ts-ignore
476-
if (column.getPrecision() === 0) {
477-
type.type = 'int';
478-
} else {
479-
type.type = 'decimal';
480-
}
481-
} else {
482-
type.type = this.toGenericType(column.getType());
483-
}
484-
return type;
485-
});
469+
this.getTypes(stmt);
486470
resolve(types);
487471
},
488472
}));
@@ -718,23 +702,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
718702
}
719703
const hydrationMap = this.generateHydrationMap(stmt.getColumns());
720704
const types: {name: string, type: string}[] =
721-
stmt.getColumns().map((column) => {
722-
const type = {
723-
name: column.getName().toLowerCase(),
724-
type: '',
725-
};
726-
if (column.isNumber()) {
727-
// @ts-ignore
728-
if (column.getPrecision() === 0) {
729-
type.type = 'int';
730-
} else {
731-
type.type = 'decimal';
732-
}
733-
} else {
734-
type.type = this.toGenericType(column.getType());
735-
}
736-
return type;
737-
});
705+
this.getTypes(stmt);
738706
if (rows && rows.length && Object.keys(hydrationMap).length) {
739707
for (const row of rows) {
740708
for (const [field, toValue] of Object.entries(hydrationMap)) {
@@ -779,23 +747,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
779747
}
780748
}));
781749
const types: {name: string, type: string}[] =
782-
stmt.getColumns().map((column) => {
783-
const type = {
784-
name: column.getName().toLowerCase(),
785-
type: '',
786-
};
787-
if (column.isNumber()) {
788-
// @ts-ignore
789-
if (column.getPrecision() === 0) {
790-
type.type = 'int';
791-
} else {
792-
type.type = 'decimal';
793-
}
794-
} else {
795-
type.type = this.toGenericType(column.getType());
796-
}
797-
return type;
798-
});
750+
this.getTypes(stmt);
799751
const hydrationMap = this.generateHydrationMap(stmt.getColumns());
800752
if (Object.keys(hydrationMap).length) {
801753
const rowStream = new HydrationStream(hydrationMap);
@@ -817,6 +769,26 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
817769
};
818770
}
819771

772+
private getTypes(stmt: Statement) {
773+
return stmt.getColumns().map((column) => {
774+
const type = {
775+
name: column.getName().toLowerCase(),
776+
type: '',
777+
};
778+
if (column.isNumber()) {
779+
// @ts-ignore
780+
if (column.getScale() === 0) {
781+
type.type = 'int';
782+
} else {
783+
type.type = 'decimal';
784+
}
785+
} else {
786+
type.type = this.toGenericType(column.getType());
787+
}
788+
return type;
789+
});
790+
}
791+
820792
protected generateHydrationMap(columns: Column[]): HydrationMap {
821793
const hydrationMap: Record<string, any> = {};
822794

‎packages/cubejs-testing-drivers/fixtures/athena.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@
7474
"---------------------------------------",
7575
"Full tests ",
7676
"---------------------------------------",
77-
"must built pre-aggregations",
7877

7978
"---------------------------------------",
8079
"SKIPED FOR ALL ",

‎packages/cubejs-testing-drivers/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
"dotenv": "^16.0.3",
6363
"fs-extra": "^11.1.1",
6464
"jest": "^26.6.3",
65+
"jsonwebtoken": "^8.5.1",
6566
"ramda": "^0.28.0",
6667
"testcontainers": "^9.3.0",
6768
"typescript": "~4.9.5",

‎packages/cubejs-testing-drivers/src/tests/testQueries.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { jest, expect, beforeAll, afterAll } from '@jest/globals';
22
import { BaseDriver } from '@cubejs-backend/base-driver';
33
import cubejs, { CubejsApi } from '@cubejs-client/core';
4+
import { sign } from 'jsonwebtoken';
45
import { Environment } from '../types/Environment';
56
import {
67
getFixtures,
@@ -27,6 +28,7 @@ export function testQueries(type: string): void {
2728
it(name, test);
2829
}
2930
}
31+
const apiToken = sign({}, 'mysupersecret');
3032

3133
beforeAll(async () => {
3234
env = await runEnvironment(type);
@@ -40,7 +42,7 @@ export function testQueries(type: string): void {
4042
process.env.CUBEJS_DB_HOST = '127.0.0.1';
4143
process.env.CUBEJS_DB_PORT = `${env.data.port}`;
4244
}
43-
client = cubejs('mysupersecret', {
45+
client = cubejs(apiToken, {
4446
apiUrl: `http://127.0.0.1:${env.cube.port}/cubejs-api/v1`,
4547
});
4648
driver = (await getDriver(type)).source;
@@ -65,19 +67,19 @@ export function testQueries(type: string): void {
6567

6668
// MUST be the first test in the list!
6769
execute('must built pre-aggregations', async () => {
68-
await buildPreaggs(env.cube.port, 'mysupersecret', {
70+
await buildPreaggs(env.cube.port, apiToken, {
6971
timezones: ['UTC'],
7072
preAggregations: ['Customers.RAExternal'],
7173
contexts: [{ securityContext: { tenant: 't1' } }],
7274
});
7375

74-
await buildPreaggs(env.cube.port, 'mysupersecret', {
76+
await buildPreaggs(env.cube.port, apiToken, {
7577
timezones: ['UTC'],
7678
preAggregations: ['ECommerce.SAExternal'],
7779
contexts: [{ securityContext: { tenant: 't1' } }],
7880
});
7981

80-
await buildPreaggs(env.cube.port, 'mysupersecret', {
82+
await buildPreaggs(env.cube.port, apiToken, {
8183
timezones: ['UTC'],
8284
preAggregations: ['ECommerce.TAExternal'],
8385
contexts: [{ securityContext: { tenant: 't1' } }],

‎packages/cubejs-testing/birdbox-fixtures/snowflake/schema/Orders.js‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
cube(`Orders`, {
2+
sql: `
3+
select 1 as id, 100 as amount, 'new' status
4+
UNION ALL
5+
select 2 as id, 200 as amount, 'new' status
6+
UNION ALL
7+
select 3 as id, 300 as amount, 'processed' status
8+
UNION ALL
9+
select 4 as id, 500 as amount, 'processed' status
10+
UNION ALL
11+
select 5 as id, 600 as amount, 'shipped' status
12+
`,
13+
measures: {
14+
count: {
15+
type: `count`,
16+
},
17+
totalAmount: {
18+
sql: `amount`,
19+
type: `sum`,
20+
},
21+
toRemove: {
22+
type: `count`,
23+
},
24+
},
25+
dimensions: {
26+
status: {
27+
sql: `status`,
28+
type: `string`,
29+
},
30+
},
31+
32+
preAggregations: {
33+
amount: {
34+
measures: [totalAmount],
35+
dimensions: [status],
36+
}
37+
}
38+
});

‎packages/cubejs-testing/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
"smoke:athena:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-athena.test.js",
5353
"smoke:bigquery": "jest --verbose -i dist/test/smoke-bigquery.test.js",
5454
"smoke:bigquery:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-bigquery.test.js",
55+
"smoke:snowflake": "jest --verbose -i dist/test/smoke-snowflake.test.js",
56+
"smoke:snowflake:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-snowflake.test.js",
5557
"smoke:crate": "jest --verbose -i dist/test/smoke-crate.test.js",
5658
"smoke:crate:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-crate.test.js",
5759
"smoke:firebolt": "jest --verbose -i dist/test/smoke-firebolt.test.js",

‎packages/cubejs-testing/src/REQUIRED_ENV_VARS.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@ export const REQUIRED_ENV_VARS: {[key: string]: string[]} = {
1515
'CUBEJS_DB_EXPORT_BUCKET',
1616
'CUBEJS_DB_BQ_CREDENTIALS',
1717
],
18+
snowflake: [
19+
'CUBEJS_DB_USER',
20+
'CUBEJS_DB_PASS',
21+
'CUBEJS_DB_NAME',
22+
'CUBEJS_DB_SNOWFLAKE_ACCOUNT',
23+
'CUBEJS_DB_SNOWFLAKE_REGION',
24+
'CUBEJS_DB_SNOWFLAKE_WAREHOUSE',
25+
],
1826
crate: [],
1927
firebolt: [
2028
'CUBEJS_DB_USER',

‎packages/cubejs-testing/src/birdbox.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ interface Args {
4848
log: Log,
4949
}
5050

51-
export type DriverType = 'postgresql' | 'postgres' | 'multidb' | 'materialize' | 'crate' | 'bigquery' | 'athena' | 'postgresql-cubestore' | 'firebolt' | 'questdb' | 'redshift' | 'databricks-jdbc' | 'prestodb' | 'mssql' | 'trino' | 'oracle' | 'duckdb';
51+
export type DriverType = 'postgresql' | 'postgres' | 'multidb' | 'materialize' | 'crate' | 'bigquery' | 'athena' | 'postgresql-cubestore' | 'firebolt' | 'questdb' | 'redshift' | 'databricks-jdbc' | 'prestodb' | 'mssql' | 'trino' | 'oracle' | 'duckdb' | 'snowflake';
5252

5353
export type Schemas = string[];
5454

@@ -108,6 +108,7 @@ const driverNameToFolderNameMapper: Record<DriverType, string> = {
108108
trino: 'postgresql',
109109
oracle: 'oracle',
110110
duckdb: 'postgresql',
111+
snowflake: 'snowflake',
111112
};
112113

113114
/**

0 commit comments

Comments
 (0)