Merge branch 'master' into master

Author: KSDaemon

.editorconfig

@@ -11,3 +11,6 @@ insert_final_newline = true
 
 [*.rs]
 indent_size = 4
+
+[rust/cubesql/cubesql/egraph-debug-template/**/*.{js,jsx,ts,tsx}]
+indent_size = 4

.github/ISSUE_TEMPLATE/sql_api_query_issue.md

@@ -2,7 +2,7 @@
 name: SQL API Query Issue
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: 'api:sql'
 assignees: ''
 
 ---
@@ -13,8 +13,11 @@ Search for `Failed SQL` log message.
 **Logical Plan**
 Search for `Can't rewrite plan` log message.
 
+**Tool**
+Was thethe SQL query above generated by some BI tool or any other tool? Did you write it yourself?
+
 **Version:**
-[e.g. 0.4.5]
+E.g., v1.1.0.
 
 **Additional context**
 Add any other context about the problem here.

.github/actions/integration/dremio.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Dremio [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:dremio
+
+echo "::endgroup::"

.github/actions/integration/firebolt.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Firebolt [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:firebolt
+
+echo "::endgroup::"

.github/actions/integration/vertica.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+export TEST_VERTICA_VERSION=12.0.4-0
+
+echo "::group::Vertica ${TEST_VERTICA_VERSION}"
+docker pull vertica/vertica-ce:${TEST_VERTICA_VERSION}
+yarn lerna run --concurrency 1 --stream --no-prefix integration:vertica
+echo "::endgroup::"

.github/actions/smoke.sh

@@ -55,4 +55,12 @@ echo "::endgroup::"
 
 echo "::group::MongoBI"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:mongobi
-echo "::endgroup::"
+echo "::endgroup::"
+
+echo "::group::Vertica"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:vertica
+echo "::endgroup::"
+
+echo "::group::RBAC"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:rbac
+echo "::endgroup::"

.github/pull_request_template.md

@@ -1,5 +1,5 @@
 **Check List**
-- [ ] Tests has been run in packages where changes made if available
+- [ ] Tests have been run in packages where changes made if available
 - [ ] Linter has been run for changed code
 - [ ] Tests for the changes have been added if not covered yet
 - [ ] Docs have been added / updated if required

.github/workflows/cloud.yml

@@ -25,6 +25,9 @@ on:
       - 'package.json'
       - 'yarn.lock'
 
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
+
 jobs:
   latest-tag-sha:
     runs-on: ubuntu-20.04
@@ -59,11 +62,20 @@ jobs:
       matrix:
         node-version: [ 20.x ]
         db: [ 'athena', 'bigquery', 'snowflake' ]
+        target: [ "x86_64-unknown-linux-gnu" ]
       fail-fast: false
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: 1.84.1
+          # override: true # this is by default on
+          rustflags: ""
+          components: rustfmt
+          target: ${{ matrix.target }}
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -82,17 +94,37 @@ jobs:
       - name: Set Yarn version
         run: yarn policies set-version v1.22.22
       - name: Yarn install
-        run: CUBESTORE_SKIP_POST_INSTALL=true yarn install --frozen-lockfile
-      - name: Build client
+        uses: nick-fields/retry@v3
+        env:
+          CUBESTORE_SKIP_POST_INSTALL: true
+        with:
+          max_attempts: 3
+          retry_on: error
+          retry_wait_seconds: 15
+          timeout_minutes: 20
+          command: yarn install --frozen-lockfile
+      - name: Build Core Client libraries
         run: yarn build
+      - name: Build other packages
+        run: yarn lerna run --concurrency 1 build
+        env:
+          NODE_OPTIONS: --max_old_space_size=4096
       - name: Lerna tsc
         run: yarn tsc
+      - name: Build cubejs-backend-native (without Python)
+        run: yarn run native:build-release
+        working-directory: ./packages/cubejs-backend-native
       - name: Run Integration tests for ${{ matrix.db }} matrix
-        timeout-minutes: 30
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 3
+          retry_on: error
+          retry_wait_seconds: 15
+          timeout_minutes: 30
+          command: ./.github/actions/integration/${{ matrix.db }}.sh
         env:
           CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
           CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
           CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
           CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
           CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
-        run: ./.github/actions/integration/${{ matrix.db }}.sh

.github/workflows/drivers-tests.yml

@@ -23,7 +23,9 @@ on:
       - 'packages/cubejs-mssql-driver/**'
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
+      - 'packages/cubejs-redshift-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
+      - 'packages/cubejs-vertica-driver/**'
 
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'
@@ -48,12 +50,17 @@ on:
       - 'packages/cubejs-mssql-driver/**'
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
+      - 'packages/cubejs-redshift-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
+      - 'packages/cubejs-vertica-driver/**'
 
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'
       - 'rust/cubesql/**'
 
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
+
 jobs:
   latest-tag-sha:
     runs-on: ubuntu-20.04
@@ -97,7 +104,7 @@ jobs:
       - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: nightly-2024-07-15
+          toolchain: 1.84.1
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
@@ -115,8 +122,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: ./packages/cubejs-backend-native
-          key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
-          shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
+          key: native-${{ runner.OS }}-${{ matrix.target }}
+          shared-key: native-${{ runner.OS }}-${{ matrix.target }}
       - name: Build native (fallback)
         if: (matrix.python-version == 'fallback')
         env:
@@ -163,7 +170,12 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: backend-native
-          path: packages/cubejs-backend-native/
+          path: packages/cubejs-backend-native
+      # current .dockerignore prevents use of native build
+      - name: Unignore native from .dockerignore
+        run: |
+          grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp
+          mv .dockerignore.tmp .dockerignore
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
@@ -196,6 +208,8 @@ jobs:
         databricks-jdbc
         databricks-jdbc-export-bucket-s3
         databricks-jdbc-export-bucket-azure
+        redshift
+        redshift-export-bucket-s3
         snowflake
         snowflake-export-bucket-s3
         snowflake-export-bucket-azure
@@ -221,6 +235,8 @@ jobs:
           - mssql
           - mysql
           - postgres
+          - redshift
+          - redshift-export-bucket-s3
           - snowflake
           - snowflake-export-bucket-s3
           - snowflake-export-bucket-azure
@@ -312,6 +328,11 @@ jobs:
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY }}
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET }}
 
+          # Redshift
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST }}
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER }}
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS }}
+
           # Snowflake
           DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
           DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
@@ -324,3 +345,4 @@ jobs:
             cd ./packages/cubejs-testing-drivers
             export DEBUG=testcontainers
             yarn ${{ matrix.database }}-full
+

.github/workflows/master.yml

@@ -16,6 +16,8 @@ on:
       - 'rust/cubesql/**'
     branches:
       - master
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
 jobs:
   latest-tag-sha:
     runs-on: ubuntu-20.04
@@ -45,15 +47,15 @@ jobs:
     timeout-minutes: 60
     name: Build Linux Native backend for Dev image
     container:
-      image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024-python-3.9
+      image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024-python-3.11
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: nightly-2024-07-15
+          toolchain: 1.84.1
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
@@ -77,14 +79,14 @@ jobs:
         run: npm install -g [email protected]
       - name: Build native (with Python)
         env:
-          PYO3_PYTHON: python3.9
+          PYO3_PYTHON: python3.11
           CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu
         working-directory: ./packages/cubejs-backend-native
         run: yarn run native:build-debug-python
       - name: Store build artifact for dev image
         uses: actions/upload-artifact@v4
         with:
-          name: "native-linux-x64-glibc-3.9.node"    # this name is referenced below in docker-image-dev
+          name: "native-linux-x64-glibc-3.11.node"    # this name is referenced below in docker-image-dev
           path: ./packages/cubejs-backend-native/index.node
           overwrite: true
 
@@ -99,7 +101,7 @@ jobs:
       - name: Download backend-native artifact
         uses: actions/download-artifact@v4
         with:
-          name: "native-linux-x64-glibc-3.9.node"    # this name is referenced in above in native_linux
+          name: "native-linux-x64-glibc-3.11.node"    # this name is referenced in above in native_linux
           path: ./packages/cubejs-backend-native/
       - name: Login to DockerHub
         uses: docker/login-action@v3
@@ -108,6 +110,11 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+      # current .dockerignore prevents use of native build
+      - name: Unignore native from .dockerignore
+        run: |
+          grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp
+          mv .dockerignore.tmp .dockerignore
       - name: Push to Docker Hub
         uses: docker/build-push-action@v6
         with:
@@ -137,12 +144,19 @@ jobs:
         with:
           github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
           script: |
+            const prUrl = context.payload.pull_request ? context.payload.pull_request.html_url : '';
+            const commitUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`;
+
             await github.rest.actions.createWorkflowDispatch({
               owner: 'cubedevinc',
               repo: 'sql-api-test-suite',
               workflow_id: 'test_and_run_test_suites.yml',
               ref: 'main',
               inputs: {
-                'cube-image': 'cubejs/cube:dev'
+                'cube-image': 'cubejs/cube:dev',
+                'source-repo': context.repo.repo,
+                'source-pr-url': prUrl,
+                'source-commit-url': commitUrl,
+                'initiator': '${{ github.actor }}'
               }
             })