Skip to content

Commit 253f2c2

Browse files
authored
Merge branch 'master' into feat/pinot-rest-driver
2 parents 13008ef + 0b00351 commit 253f2c2

File tree

511 files changed

+168471
-13695
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

511 files changed

+168471
-13695
lines changed

.editorconfig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,6 @@ insert_final_newline = true
1111

1212
[*.rs]
1313
indent_size = 4
14+
15+
[rust/cubesql/cubesql/egraph-debug-template/**/*.{js,jsx,ts,tsx}]
16+
indent_size = 4

.github/workflows/drivers-tests.yml

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,8 @@ jobs:
144144
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
145145
runs-on: ubuntu-20.04
146146
timeout-minutes: 30
147+
env:
148+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
147149
steps:
148150
- name: Check out the repo
149151
uses: actions/checkout@v4
@@ -154,6 +156,7 @@ jobs:
154156
with:
155157
username: ${{ secrets.DOCKERHUB_USERNAME }}
156158
password: ${{ secrets.DOCKERHUB_TOKEN }}
159+
if: (env.DOCKERHUB_USERNAME != '')
157160
- name: Set up QEMU
158161
uses: docker/setup-qemu-action@v3
159162
- name: Download native build
@@ -167,15 +170,39 @@ jobs:
167170
context: .
168171
file: ./packages/cubejs-docker/testing-drivers.Dockerfile
169172
tags: cubejs/cube:testing-drivers
170-
push: true
173+
push: ${{ (env.DOCKERHUB_USERNAME != '') }}
174+
- name: Save Docker image as artifact
175+
run: |
176+
IMAGE_TAG=cubejs/cube:testing-drivers
177+
docker save -o image.tar $IMAGE_TAG
178+
gzip image.tar
179+
continue-on-error: true
180+
- name: Upload Docker image artifact
181+
uses: actions/upload-artifact@v4
182+
with:
183+
name: docker-image
184+
path: image.tar.gz
171185

172186
tests:
173187
runs-on: ubuntu-20.04
174188
timeout-minutes: 30
175189
needs: [latest-tag-sha, build]
176190
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
177191
env:
178-
CLOUD_DATABASES: athena bigquery databricks-jdbc databricks-jdbc-export-bucket snowflake
192+
CLOUD_DATABASES: >
193+
athena-export-bucket-s3
194+
bigquery-export-bucket-gcs
195+
clickhouse-export-bucket-s3
196+
databricks-jdbc
197+
databricks-jdbc-export-bucket-s3
198+
databricks-jdbc-export-bucket-azure
199+
redshift
200+
redshift-export-bucket-s3
201+
snowflake
202+
snowflake-export-bucket-s3
203+
snowflake-export-bucket-azure
204+
snowflake-export-bucket-azure-via-storage-integration
205+
snowflake-export-bucket-gcs
179206
# As per docs:
180207
# Secrets cannot be directly referenced in if: conditionals. Instead, consider setting
181208
# secrets as job-level environment variables, then referencing the environment variables
@@ -186,15 +213,23 @@ jobs:
186213
node:
187214
- 20.x
188215
database:
189-
- athena
190-
- bigquery
216+
- athena-export-bucket-s3
217+
- bigquery-export-bucket-gcs
191218
- clickhouse
219+
- clickhouse-export-bucket-s3
192220
- databricks-jdbc
193-
- databricks-jdbc-export-bucket
221+
- databricks-jdbc-export-bucket-s3
222+
- databricks-jdbc-export-bucket-azure
194223
- mssql
195224
- mysql
196225
- postgres
226+
- redshift
227+
- redshift-export-bucket-s3
197228
- snowflake
229+
- snowflake-export-bucket-s3
230+
- snowflake-export-bucket-azure
231+
- snowflake-export-bucket-azure-via-storage-integration
232+
- snowflake-export-bucket-gcs
198233
fail-fast: false
199234

200235
steps:
@@ -244,6 +279,16 @@ jobs:
244279
cd packages/cubejs-testing-drivers
245280
yarn tsc
246281
282+
- name: Download Docker image artifact
283+
uses: actions/download-artifact@v4
284+
with:
285+
name: docker-image
286+
287+
- name: Load Docker image into Docker Daemon
288+
run: |
289+
gunzip image.tar.gz
290+
docker load -i image.tar
291+
247292
- name: Run tests
248293
uses: nick-fields/retry@v3
249294
# It's enough to test for any one secret because they are set all at once or not set all
@@ -258,12 +303,24 @@ jobs:
258303
# BigQuery
259304
DRIVERS_TESTS_CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_BQ_CREDENTIALS }}
260305

306+
#GCS
307+
DRIVERS_TESTS_CUBEJS_DB_EXPORT_GCS_CREDENTIALS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_GCS_CREDENTIALS }}
308+
309+
# Azure
310+
DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY }}
311+
DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_SAS_TOKEN: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_SAS_TOKEN }}
312+
261313
# Databricks
262314
DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_URL: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_URL }}
263315
DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN }}
264316
DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY }}
265317
DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET }}
266318

319+
# Redshift
320+
DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST }}
321+
DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER }}
322+
DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS }}
323+
267324
# Snowflake
268325
DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
269326
DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
@@ -276,3 +333,4 @@ jobs:
276333
cd ./packages/cubejs-testing-drivers
277334
export DEBUG=testcontainers
278335
yarn ${{ matrix.database }}-full
336+

.github/workflows/master.yml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,19 @@ jobs:
137137
with:
138138
github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
139139
script: |
140+
const prUrl = context.payload.pull_request ? context.payload.pull_request.html_url : '';
141+
const commitUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`;
142+
140143
await github.rest.actions.createWorkflowDispatch({
141144
owner: 'cubedevinc',
142145
repo: 'sql-api-test-suite',
143146
workflow_id: 'test_and_run_test_suites.yml',
144147
ref: 'main',
145148
inputs: {
146-
'cube-image': 'cubejs/cube:dev'
149+
'cube-image': 'cubejs/cube:dev',
150+
'source-repo': context.repo.repo,
151+
'source-pr-url': prUrl,
152+
'source-commit-url': commitUrl,
153+
'initiator': '${{ github.actor }}'
147154
}
148155
})

.github/workflows/post-release.yml

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -205,22 +205,3 @@ jobs:
205205
color: danger
206206
env:
207207
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
208-
209-
trigger-repo-sync:
210-
runs-on: ubuntu-20.04
211-
name: 'Sync runtime repo'
212-
timeout-minutes: 60
213-
steps:
214-
- name: Checkout
215-
uses: actions/checkout@v4
216-
- name: Trigger runtime
217-
uses: actions/github-script@v6
218-
with:
219-
github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
220-
script: |
221-
await github.rest.actions.createWorkflowDispatch({
222-
owner: 'cubedevinc',
223-
repo: 'cube-runtime',
224-
workflow_id: 'sync.yml',
225-
ref: 'master'
226-
})

.github/workflows/publish.yml

Lines changed: 79 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ jobs:
305305
overwrite: true
306306

307307
docker-default:
308-
needs: [npm, cubestore_linux, native_linux]
308+
needs: [npm, cubestore_linux, native_linux, detect_branch]
309309
name: Debian docker image
310310
runs-on: ${{ matrix.os }}
311311
strategy:
@@ -345,7 +345,11 @@ jobs:
345345
if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
346346
MINOR=${VERSION%.*}
347347
MAJOR=${MINOR%.*}
348-
TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR},${DOCKER_IMAGE}:${MAJOR},${DOCKER_IMAGE}:latest"
348+
TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR},${DOCKER_IMAGE}:${MAJOR}"
349+
fi
350+
351+
if [ "${{ needs['detect_branch'].outputs.is_master }}" = "true" ]; then
352+
TAGS="$TAGS,${DOCKER_IMAGE}:latest"
349353
fi
350354
351355
{
@@ -385,7 +389,7 @@ jobs:
385389
IMAGE_VERSION=${{ steps.prep.outputs.version }}
386390
387391
docker-debian-jdk:
388-
needs: [npm, cubestore_linux, native_linux]
392+
needs: [npm, cubestore_linux, native_linux, detect_branch]
389393
name: Debian with jdk docker image
390394
runs-on: ubuntu-24.04
391395
timeout-minutes: 30
@@ -420,11 +424,15 @@ jobs:
420424
if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
421425
MINOR=${VERSION%.*}
422426
MAJOR=${MINOR%.*}
423-
TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR}-jdk,${DOCKER_IMAGE}:${MAJOR}-jdk,${DOCKER_IMAGE}:jdk"
427+
TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR}-jdk,${DOCKER_IMAGE}:${MAJOR}-jdk"
424428
elif [ "${{ github.event_name }}" = "push" ]; then
425429
TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}-jdk"
426430
fi
427431
432+
if [ "${{ needs['detect_branch'].outputs.is_master }}" = "true" ]; then
433+
TAGS="$TAGS,${DOCKER_IMAGE}:jdk"
434+
fi
435+
428436
{
429437
echo "version=${VERSION}"
430438
echo "tags=${TAGS}"
@@ -464,6 +472,7 @@ jobs:
464472
docker-cubestore:
465473
name: Cube Store Docker
466474
runs-on: ${{ matrix.os }}
475+
needs: [detect_branch]
467476
strategy:
468477
fail-fast: false
469478
matrix:
@@ -521,7 +530,11 @@ jobs:
521530
if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
522531
MINOR=${VERSION%.*}
523532
MAJOR=${MINOR%.*}
524-
TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR}${{ matrix.postfix }},${DOCKER_IMAGE}:${MAJOR}${{ matrix.postfix }},${DOCKER_IMAGE}:${{ matrix.tag }}"
533+
TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR}${{ matrix.postfix }},${DOCKER_IMAGE}:${MAJOR}${{ matrix.postfix }}"
534+
fi
535+
536+
if [ "${{ needs['detect_branch'].outputs.is_master }}" = "true" ]; then
537+
TAGS="$TAGS,${DOCKER_IMAGE}:${{ matrix.tag }}"
525538
fi
526539
527540
{
@@ -744,7 +757,7 @@ jobs:
744757
name: Trigger test suites run
745758
runs-on: ubuntu-20.04
746759
needs: [docker-default]
747-
timeout-minutes: 60
760+
timeout-minutes: 15
748761
steps:
749762
- name: Checkout
750763
uses: actions/checkout@v4
@@ -753,12 +766,71 @@ jobs:
753766
with:
754767
github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
755768
script: |
769+
const prUrl = context.payload.pull_request ? context.payload.pull_request.html_url : '';
770+
const commitUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`;
771+
756772
await github.rest.actions.createWorkflowDispatch({
757773
owner: 'cubedevinc',
758774
repo: 'sql-api-test-suite',
759775
workflow_id: 'test_and_run_test_suites.yml',
760776
ref: 'main',
761777
inputs: {
762-
'cube-image': 'cubejs/cube:latest'
778+
'cube-image': 'cubejs/cube:latest',
779+
'source-repo': context.repo.repo,
780+
'source-pr-url': prUrl,
781+
'source-commit-url': commitUrl,
782+
'initiator': '${{ github.actor }}'
783+
}
784+
})
785+
786+
detect_branch:
787+
runs-on: ubuntu-20.04
788+
outputs:
789+
is_master: ${{ steps.detect_branch.outputs.is_master }}
790+
branch: ${{ steps.detect_branch.outputs.branch }}
791+
steps:
792+
- name: Checkout
793+
uses: actions/checkout@v4
794+
- name: Detect branch name
795+
id: detect_branch
796+
env:
797+
EVENT_BASE_REF: ${{ github.event.base_ref || 'master' }}
798+
run: |
799+
BRANCH=${EVENT_BASE_REF#refs/heads/}
800+
echo "Detected branch: $BRANCH"
801+
802+
IS_MASTER=false
803+
if [ "$BRANCH" == "master" ]; then
804+
IS_MASTER=true
805+
fi
806+
echo "Detected is_master: $IS_MASTER"
807+
808+
echo "branch=$BRANCH" >> "$GITHUB_OUTPUT"
809+
echo "is_master=$IS_MASTER" >> "$GITHUB_OUTPUT"
810+
811+
trigger-repo-sync:
812+
name: Trigger runtime repo
813+
runs-on: ubuntu-20.04
814+
needs: [docker-default, detect_branch]
815+
timeout-minutes: 15
816+
steps:
817+
- name: Checkout
818+
uses: actions/checkout@v4
819+
- name: Trigger runtime
820+
uses: actions/github-script@v6
821+
with:
822+
github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
823+
script: |
824+
const tagName = process.env.GITHUB_REF.replace('refs/tags/', '');
825+
const branchName = '${{ needs['detect_branch'].outputs.branch }}';
826+
827+
await github.rest.actions.createWorkflowDispatch({
828+
owner: 'cubedevinc',
829+
repo: 'cube-runtime',
830+
workflow_id: 'sync.yml',
831+
ref: 'master',
832+
inputs: {
833+
'tag': tagName,
834+
'branch': branchName,
763835
}
764836
})

.github/workflows/push.yml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -363,8 +363,13 @@ jobs:
363363
- name: Lerna tsc
364364
run: yarn tsc
365365
- name: Run Integration tests for ${{ matrix.db }} matrix
366-
timeout-minutes: 30
367-
run: ./.github/actions/integration/${{ matrix.db }}.sh
366+
uses: nick-fields/retry@v3
367+
with:
368+
max_attempts: 3
369+
retry_on: error
370+
retry_wait_seconds: 15
371+
timeout_minutes: 30
372+
command: ./.github/actions/integration/${{ matrix.db }}.sh
368373

369374
integration-smoke:
370375
needs: [ latest-tag-sha, build-cubestore ]

.github/workflows/rust-cubesql.yml

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,17 @@ on:
55
paths:
66
- '.github/workflows/rust-cubesql.yml'
77
- 'packages/cubejs-backend-native/**'
8+
- 'rust/cubenativeutils/**'
9+
- 'rust/cubesqlplanner/**'
810
- 'rust/cubesql/**'
911
branches:
1012
- 'master'
1113
pull_request:
1214
paths:
1315
- '.github/workflows/rust-cubesql.yml'
1416
- 'packages/cubejs-backend-native/**'
17+
- 'rust/cubenativeutils/**'
18+
- 'rust/cubesqlplanner/**'
1519
- 'rust/cubesql/**'
1620

1721
jobs:
@@ -43,16 +47,16 @@ jobs:
4347
run: cd rust/cubesql && cargo fmt --all -- --check
4448
- name: Lint Native
4549
run: cd packages/cubejs-backend-native && cargo fmt --all -- --check
46-
# TODO replace with clippy once cubesql is ready
47-
- name: Check CubeSQL
48-
run: cd rust/cubesql && cargo check --locked --workspace --all-targets --keep-going
50+
- name: Lint cubenativeutils
51+
run: cd rust/cubenativeutils && cargo fmt --all -- --check
52+
- name: Lint cubesqlplanner
53+
run: cd rust/cubesqlplanner && cargo fmt --all -- --check
54+
- name: Clippy CubeSQL
55+
run: cd rust/cubesql && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings
4956
- name: Clippy Native
5057
run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings
5158
- name: Clippy Native (with Python)
5259
run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going --features python -- -D warnings
53-
# CubeSQL is not ready for Clippy
54-
#- name: Clippy CubeSQL
55-
# run: cd rust/cubesql && cargo clippy -- -D warnings
5660

5761
unit:
5862
# We use host instead of cross container, because it's much faster

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,4 @@ testings/
2323
rust/cubesql/profile.json
2424
.cubestore
2525
.env
26-
26+
.vimspector.json

0 commit comments

Comments
 (0)