Skip to content

Commit 4e2732a

Browse files
paveltiunovpaveltiunov
authored
fix(cubesql): Quote FROM alias for SQL push down to avoid name clas… (#7755)
* fix(cubesql): Quote `FROM` alias for SQL push down to avoid name clashes with keywords * Add pg dependency * Build backend-native for tests * Remove not needed architectures * Fix download path * Fix No port binding found for :NaN * Fix No port binding found for :NaN
1 parent 49acad5 commit 4e2732a

File tree

14 files changed

+584
-22
lines changed

14 files changed

+584
-22
lines changed

‎.github/workflows/drivers-tests.yml‎

Lines changed: 85 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,86 @@ jobs:
6262
env:
6363
OUT: ${{ steps.get-tag.outputs.sha }}
6464

65+
native_linux:
66+
runs-on: ubuntu-20.04
67+
timeout-minutes: 60
68+
name: Build native Linux ${{ matrix.node-version }} ${{ matrix.target }} Python ${{ matrix.python-version }}
69+
strategy:
70+
matrix:
71+
node-version: [ 16 ]
72+
python-version: [ "fallback" ]
73+
target: [ "x86_64-unknown-linux-gnu" ]
74+
fail-fast: false
75+
container:
76+
image: cubejs/rust-cross:${{ matrix.target }}-30052023
77+
78+
steps:
79+
- name: Checkout
80+
uses: actions/checkout@v4
81+
- name: Install Rust
82+
uses: actions-rs/toolchain@v1
83+
with:
84+
toolchain: nightly-2023-12-13
85+
override: true
86+
components: rustfmt
87+
target: ${{ matrix.target }}
88+
- name: Install Node.js ${{ matrix.node-version }}
89+
uses: actions/setup-node@v3
90+
with:
91+
node-version: ${{ matrix.node-version }}
92+
- name: Install Yarn
93+
run: npm install -g yarn
94+
- name: Set Yarn version
95+
run: yarn policies set-version v1.22.19
96+
- name: Get yarn cache directory path
97+
id: yarn-cache-dir-path
98+
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
99+
shell: bash
100+
- name: Restore yarn cache
101+
uses: actions/cache@v3
102+
with:
103+
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
104+
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
105+
restore-keys: |
106+
${{ runner.os }}-yarn-
107+
- name: Copy yarn.lock file
108+
run: cp yarn.lock packages/cubejs-docker
109+
- name: Yarn install
110+
uses: nick-invision/retry@v2
111+
env:
112+
CUBESTORE_SKIP_POST_INSTALL: true
113+
with:
114+
max_attempts: 3
115+
retry_on: error
116+
retry_wait_seconds: 15
117+
timeout_minutes: 20
118+
command: yarn install --frozen-lockfile
119+
- name: Compile TypeScript
120+
run: yarn tsc
121+
- name: Build native (fallback)
122+
if: (matrix.python-version == 'fallback')
123+
env:
124+
CARGO_BUILD_TARGET: ${{ matrix.target }}
125+
run: cd packages/cubejs-backend-native && npm run native:build-release
126+
- name: Setup cross compilation
127+
if: (matrix.target == 'aarch64-unknown-linux-gnu')
128+
uses: allenevans/[email protected]
129+
with:
130+
PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }}
131+
- name: Build native (with Python)
132+
if: (matrix.python-version != 'fallback')
133+
env:
134+
PYO3_PYTHON: python${{ matrix.python-version }}
135+
CARGO_BUILD_TARGET: ${{ matrix.target }}
136+
run: cd packages/cubejs-backend-native && npm run native:build-release-python
137+
- name: Upload native build
138+
uses: actions/upload-artifact@v4
139+
with:
140+
name: backend-native
141+
path: packages/cubejs-backend-native/index.node
142+
65143
build:
66-
needs: latest-tag-sha
144+
needs: [latest-tag-sha, native_linux]
67145
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
68146
runs-on: ubuntu-20.04
69147
timeout-minutes: 30
@@ -81,6 +159,12 @@ jobs:
81159
- name: Set up QEMU
82160
uses: docker/setup-qemu-action@v2
83161

162+
- name: Download native build
163+
uses: actions/download-artifact@v4
164+
with:
165+
name: backend-native
166+
path: packages/cubejs-backend-native/
167+
84168
- name: Set up Docker Buildx
85169
uses: docker/setup-buildx-action@v2
86170
with:

‎packages/cubejs-testing-drivers/fixtures/postgres.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,15 @@
88
"CUBEJS_DB_HOST": "data",
99
"CUBEJS_DB_NAME": "test",
1010
"CUBEJS_DB_USER": "test",
11-
"CUBEJS_DB_PASS": "test"
11+
"CUBEJS_DB_PASS": "test",
12+
"CUBEJS_PG_SQL_PORT": "5656",
13+
"CUBEJS_SQL_USER": "admin",
14+
"CUBEJS_SQL_PASSWORD": "admin_password",
15+
"CUBESQL_SQL_PUSH_DOWN": "true"
1216
},
1317
"depends_on": ["data"],
1418
"links": ["data"],
15-
"ports" : ["4000"]
19+
"ports" : ["4000", "5656"]
1620
},
1721
"data": {
1822
"image": "postgres:13",

‎packages/cubejs-testing-drivers/fixtures/snowflake.json‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,21 @@
1111
"CUBEJS_DB_NAME": "DEMO_DB",
1212
"CUBEJS_PRE_AGGREGATIONS_SCHEMA": "drivers_tests_preaggs",
1313

14-
"CUBEJS_DB_SNOWFLAKE_ACCOUNT": "lxb31104.us-west-2",
14+
"CUBEJS_DB_SNOWFLAKE_ACCOUNT": "lxb31104",
1515
"CUBEJS_DB_SNOWFLAKE_REGION": "us-west-2",
1616
"CUBEJS_DB_SNOWFLAKE_WAREHOUSE": "COMPUTE_WH",
1717
"CUBEJS_DB_SNOWFLAKE_ROLE": "PUBLIC",
1818
"CUBEJS_DB_EXPORT_BUCKET": "athena-drivers-tests-preaggs",
1919
"CUBEJS_DB_EXPORT_BUCKET_TYPE": "s3",
2020
"CUBEJS_DB_EXPORT_BUCKET_AWS_KEY": "${DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY}",
2121
"CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET": "${DRIVERS_TESTS_ATHENA_CUBEJS_AWS_SECRET}",
22-
"CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-east-1"
22+
"CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-east-1",
23+
"CUBEJS_PG_SQL_PORT": "5656",
24+
"CUBEJS_SQL_USER": "admin",
25+
"CUBEJS_SQL_PASSWORD": "admin_password",
26+
"CUBESQL_SQL_PUSH_DOWN": "true"
2327
},
24-
"ports" : ["4000"]
28+
"ports" : ["4000", "5656"]
2529
},
2630
"cast": {
2731
"SELECT_PREFIX": "",

‎packages/cubejs-testing-drivers/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
"fs-extra": "^11.1.1",
6666
"jest": "^27",
6767
"jsonwebtoken": "^8.5.1",
68+
"pg": "^8.7.3",
6869
"ramda": "^0.28.0",
6970
"testcontainers": "^9.3.0",
7071
"typescript": "~5.2.2",

‎packages/cubejs-testing-drivers/src/helpers/runEnvironment.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,11 +174,15 @@ export async function runEnvironment(
174174
}
175175
const cube = cliEnv ? {
176176
port: 4000,
177+
pgPort: parseInt(fixtures.cube.ports[1], 10),
177178
logs: cliEnv.cli?.stdout || process.stdout
178179
} : {
179180
port: environment.getContainer('cube').getMappedPort(
180181
parseInt(fixtures.cube.ports[0], 10),
181182
),
183+
pgPort: fixtures.cube.ports[1] && environment.getContainer('cube').getMappedPort(
184+
parseInt(fixtures.cube.ports[1], 10),
185+
) || undefined,
182186
logs: await environment.getContainer('cube').logs(),
183187
};
184188

‎packages/cubejs-testing-drivers/src/tests/testQueries.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { jest, expect, beforeAll, afterAll } from '@jest/globals';
22
import { randomBytes } from 'crypto';
3+
import { Client as PgClient } from 'pg';
34
import { BaseDriver } from '@cubejs-backend/base-driver';
45
import cubejs, { CubejsApi } from '@cubejs-client/core';
56
import { sign } from 'jsonwebtoken';
@@ -22,6 +23,38 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
2223
let driver: BaseDriver;
2324
let queries: string[];
2425
let env: Environment;
26+
let connection: PgClient;
27+
28+
let connectionId = 0;
29+
30+
async function createPostgresClient(user: string, password: string, pgPort: number | undefined) {
31+
if (!pgPort) {
32+
return <any>undefined;
33+
}
34+
connectionId++;
35+
const currentConnId = connectionId;
36+
37+
console.debug(`[pg] new connection ${currentConnId}`);
38+
39+
const conn = new PgClient({
40+
database: 'db',
41+
port: pgPort,
42+
host: 'localhost',
43+
user,
44+
password,
45+
ssl: false,
46+
});
47+
conn.on('error', (err) => {
48+
console.log(err);
49+
});
50+
conn.on('end', () => {
51+
console.debug(`[pg] end ${currentConnId}`);
52+
});
53+
54+
await conn.connect();
55+
56+
return conn;
57+
}
2558

2659
function execute(name: string, test: () => Promise<void>) {
2760
if (fixtures.skip && fixtures.skip.indexOf(name) >= 0) {
@@ -30,6 +63,15 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
3063
it(name, test);
3164
}
3265
}
66+
67+
function executePg(name: string, test: () => Promise<void>) {
68+
if (!fixtures.cube.ports[1] || fixtures.skip && fixtures.skip.indexOf(name) >= 0) {
69+
it.skip(name, test);
70+
} else {
71+
it(name, test);
72+
}
73+
}
74+
3375
const apiToken = sign({}, 'mysupersecret');
3476

3577
const suffix = randomBytes(8).toString('hex');
@@ -64,6 +106,7 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
64106
console.log('Error creating fixtures', e.stack);
65107
throw e;
66108
}
109+
connection = await createPostgresClient('admin', 'admin_password', env.cube.pgPort);
67110
});
68111

69112
afterAll(async () => {
@@ -1441,5 +1484,21 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
14411484
if (includeIncrementalSchemaSuite) {
14421485
incrementalSchemaLoadingSuite(execute, () => driver, tables);
14431486
}
1487+
1488+
executePg('SQL API: powerbi min max push down', async () => {
1489+
const res = await connection.query(`
1490+
select
1491+
max("rows"."orderDate") as "a0",
1492+
min("rows"."orderDate") as "a1"
1493+
from
1494+
(
1495+
select
1496+
"orderDate"
1497+
from
1498+
"public"."ECommerce" "$Table"
1499+
) "rows"
1500+
`);
1501+
expect(res.rows).toMatchSnapshot('powerbi_min_max_push_down');
1502+
});
14441503
});
14451504
}

‎packages/cubejs-testing-drivers/src/types/Environment.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { Readable } from 'stream';
33
export type Environment = {
44
cube: {
55
port: number;
6+
pgPort?: number;
67
logs: Readable;
78
};
89
store: {

0 commit comments

Comments
 (0)