Merge branch 'master' into patch-1

Author: KSDaemon

.github/ISSUE_TEMPLATE/sql_api_query_issue.md

@@ -2,7 +2,7 @@
 name: SQL API Query Issue
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: 'api:sql'
 assignees: ''
 
 ---
@@ -13,8 +13,11 @@ Search for `Failed SQL` log message.
 **Logical Plan**
 Search for `Can't rewrite plan` log message.
 
+**Tool**
+Was thethe SQL query above generated by some BI tool or any other tool? Did you write it yourself?
+
 **Version:**
-[e.g. 0.4.5]
+E.g., v1.1.0.
 
 **Additional context**
 Add any other context about the problem here.

.github/actions/integration/dremio.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Dremio [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:dremio
+
+echo "::endgroup::"

.github/actions/integration/firebolt.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Firebolt [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:firebolt
+
+echo "::endgroup::"

.github/actions/smoke.sh

@@ -55,4 +55,8 @@ echo "::endgroup::"
 
 echo "::group::MongoBI"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:mongobi
-echo "::endgroup::"
+echo "::endgroup::"
+
+echo "::group::RBAC"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:rbac
+echo "::endgroup::"

.github/workflows/publish.yml

@@ -152,7 +152,7 @@ jobs:
     strategy:
       matrix:
         node-version: [20.x]
-        os-version: ["macos-12"]
+        os-version: ["macos-13"]
         target: ["x86_64-apple-darwin", "aarch64-apple-darwin"]
         python-version: ["3.9", "3.10", "3.11", "3.12", "fallback"]
         include:
@@ -191,7 +191,7 @@ jobs:
           components: rustfmt
           target: ${{ matrix.target }}
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         if: (matrix.python-version != 'fallback')
         with:
           python-version: ${{ matrix.python-version }}
@@ -265,7 +265,7 @@ jobs:
           rustflags: ""
           components: rustfmt
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         if: (matrix.python-version != 'fallback')
         with:
           python-version: ${{ matrix.python-version }}
@@ -666,17 +666,26 @@ jobs:
         target:
           - x86_64-pc-windows-msvc
           - x86_64-apple-darwin
+          - aarch64-apple-darwin
         include:
           - target: x86_64-pc-windows-msvc
             os: windows-2019
             executable_name: cubestored.exe
             strip: true
             # cubestored.exe: CantPackException: superfluous data between sections
             compress: false
-            # Please use minimal possible version of macOS, because it produces constraint on libstdc++
             tar_executable: tar
+            # Please use minimal possible version of macOS, because it produces constraint on libstdc++
           - target: x86_64-apple-darwin
-            os: macos-12
+            os: macos-13
+            executable_name: cubestored
+            # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.
+            strip: false
+            compress: false
+            # bsd tar has a different format with Sparse files which breaks download script
+            tar_executable: gtar
+          - os: macos-14
+            target: aarch64-apple-darwin
             executable_name: cubestored
             # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.
             strip: false
@@ -784,7 +793,7 @@ jobs:
             })
 
   detect_branch:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     outputs:
       is_master: ${{ steps.detect_branch.outputs.is_master }}
       branch: ${{ steps.detect_branch.outputs.branch }}
@@ -793,20 +802,10 @@ jobs:
         uses: actions/checkout@v4
       - name: Detect branch name
         id: detect_branch
-        env:
-          EVENT_BASE_REF: ${{ github.event.base_ref || 'master' }}
         run: |
-          BRANCH=${EVENT_BASE_REF#refs/heads/}
-          echo "Detected branch: $BRANCH"
-
-          IS_MASTER=false
-          if [ "$BRANCH" == "master" ]; then
-            IS_MASTER=true
-          fi
-          echo "Detected is_master: $IS_MASTER"
-
-          echo "branch=$BRANCH" >> "$GITHUB_OUTPUT"
-          echo "is_master=$IS_MASTER" >> "$GITHUB_OUTPUT"
+          # Git & GitHub (for now) doesn't provide an easy/stable way to detect branch name from tag.
+          echo "branch=master" >> "$GITHUB_OUTPUT"
+          echo "is_master=true" >> "$GITHUB_OUTPUT"
 
   trigger-repo-sync:
     name: Trigger runtime repo
@@ -817,7 +816,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Trigger runtime
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
           script: |

.github/workflows/push.yml

@@ -42,6 +42,7 @@ jobs:
       matrix:
         # Current docker version + next LTS
         node-version: [20.x, 22.x]
+        python-version: [3.11]
       fail-fast: false
 
     steps:
@@ -65,6 +66,10 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -92,6 +97,12 @@ jobs:
         run: yarn tsc
       - name: Build client
         run: yarn build
+      - name: Build cubejs-backend-native (with Python)
+        run: yarn run native:build-release-python
+        working-directory: ./packages/cubejs-backend-native
+        env:
+          PYO3_PYTHON: python${{ matrix.python-version }}
+
       - name: Lerna test
         run: yarn lerna run --concurrency 1 --stream --no-prefix unit
 #      - uses: codecov/codecov-action@v1
@@ -313,13 +324,19 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     if: (needs['latest-tag-sha'].outputs.sha != github.sha)
+    env:
+      CLOUD_DATABASES: >
+        firebolt
+        dremio
+      # Athena (just to check for secrets availability)
+      DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
 
     strategy:
       matrix:
         node-version: [20.x]
         db: [
             'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
-            'mysql-aurora-serverless', 'crate', 'mongobi'
+            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio'
         ]
       fail-fast: false
 
@@ -370,6 +387,21 @@ jobs:
           retry_wait_seconds: 15
           timeout_minutes: 30
           command: ./.github/actions/integration/${{ matrix.db }}.sh
+        # It's enough to test for any one secret because they are set all at once or not set all
+        if: |
+          (contains(env.CLOUD_DATABASES, matrix.db) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
+          (!contains(env.CLOUD_DATABASES, matrix.db))
+        env:
+          # Firebolt Integration
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }}
+          # Dremio Integration
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }}
 
   integration-smoke:
     needs: [ latest-tag-sha, build-cubestore ]
@@ -380,6 +412,7 @@ jobs:
     strategy:
       matrix:
         node-version: [ 20.x ]
+        python-version: [ 3.11 ]
       fail-fast: false
 
     steps:
@@ -405,6 +438,10 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -432,6 +469,11 @@ jobs:
         uses: GoodManWEN/oracle-client-action@main
       - name: Build client
         run: yarn build
+      - name: Build cubejs-backend-native (with Python)
+        run: yarn run native:build-release-python
+        working-directory: ./packages/cubejs-backend-native
+        env:
+          PYO3_PYTHON: python${{ matrix.python-version }}
       - name: Lerna tsc
         run: yarn tsc
       - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact

.github/workflows/rust-cubesql.yml

@@ -62,7 +62,11 @@ jobs:
     #  We use host instead of cross container, because it's much faster
     runs-on: ubuntu-24.04
     timeout-minutes: 60
-    name: Unit (Rewrite Engine)
+    name: Unit (Rewrite Engine) (CUBESQL_TOP_DOWN_EXTRACTOR=${{ matrix.top-down-extractor }})
+    strategy:
+      matrix:
+        top-down-extractor: ['true', 'false']
+      fail-fast: false
 
     steps:
       - name: Checkout
@@ -90,6 +94,7 @@ jobs:
           CUBESQL_TESTING_CUBE_TOKEN: ${{ secrets.CUBESQL_TESTING_CUBE_TOKEN }}
           CUBESQL_TESTING_CUBE_URL: ${{ secrets.CUBESQL_TESTING_CUBE_URL }}
           CUBESQL_SQL_PUSH_DOWN: true
+          CUBESQL_TOP_DOWN_EXTRACTOR: ${{ matrix.top-down-extractor }}
           CUBESQL_REWRITE_CACHE: true
           CUBESQL_REWRITE_TIMEOUT: 60
         run: |
@@ -245,7 +250,7 @@ jobs:
           components: rustfmt
           target: ${{ matrix.target }}
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         if: (matrix.python-version != 'fallback')
         with:
           python-version: ${{ matrix.python-version }}
@@ -327,7 +332,7 @@ jobs:
           rustflags: ""
           components: rustfmt
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         if: (matrix.python-version != 'fallback')
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/rust-cubestore-master.yml

@@ -149,7 +149,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Push to Docker Hub
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: ./rust/cubestore
           file: ./rust/cubestore/Dockerfile
@@ -187,6 +187,7 @@ jobs:
         target:
           - x86_64-pc-windows-msvc
           - x86_64-apple-darwin
+          - aarch64-apple-darwin
         include:
           - os: windows-2019
             target: x86_64-pc-windows-msvc
@@ -195,12 +196,18 @@ jobs:
             # cubestored.exe: CantPackException: superfluous data between sections
             compress: false
             # Please use minimal possible version of macOS, because it produces constraint on libstdc++
-          - os: macos-12
+          - os: macos-13
             target: x86_64-apple-darwin
             executable_name: cubestored
             # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.
             strip: false
             compress: false
+          - os: macos-14
+            target: aarch64-apple-darwin
+            executable_name: cubestored
+            # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.
+            strip: false
+            compress: false
       fail-fast: false
     steps:
       - uses: actions/checkout@v4

.github/workflows/rust-cubestore.yml

@@ -97,7 +97,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Build only
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: ./rust/cubestore/
           file: ./rust/cubestore/Dockerfile
@@ -118,6 +118,7 @@ jobs:
         target:
           - x86_64-pc-windows-msvc
           - x86_64-apple-darwin
+          - aarch64-apple-darwin
         include:
           - os: windows-2019
             target: x86_64-pc-windows-msvc
@@ -126,12 +127,18 @@ jobs:
             # cubestored.exe: CantPackException: superfluous data between sections
             compress: false
             # Please use minimal possible version of macOS, because it produces constraint on libstdc++
-          - os: macos-12
+          - os: macos-13
             target: x86_64-apple-darwin
             executable_name: cubestored
             # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.
             strip: false
             compress: false
+          - os: macos-14
+            target: aarch64-apple-darwin
+            executable_name: cubestored
+            # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.
+            strip: false
+            compress: false
       fail-fast: false
     steps:
       - uses: actions/checkout@v4