Merge branch 'cube-js:master' into master

Author: Helge-TXS

.dockerignore

@@ -1,6 +1,7 @@
 **
 !package.json
 !tsconfig.base.json
+!tsconfig.json
 !rollup.config.js
 !yarn.lock
 !lerna.json

.github/actions/codecov-fix.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
+
+for path in "$repo_root"/packages/*/coverage/lcov.info; do
+  [ -f "$path" ] || continue
+
+  pkg_dir="$(dirname "$path")"          # packages/foo/coverage
+  pkg_root="$(dirname "$pkg_dir")"      # packages/foo
+  tmp_path="$pkg_dir/lcov.fixed.info"
+
+  sed "s|SF:src/|SF:packages/$(basename "$pkg_root")/src/|g" "$path" > "$tmp_path"
+done

.github/actions/integration/trino.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+export TEST_PRESTO_VERSION=341-SNAPSHOT
+export TEST_PGSQL_VERSION=12.4
+
+echo "::group::Trino ${TEST_PRESTO_VERSION} with PostgreSQL ${TEST_PGSQL_VERSION}"
+docker pull lewuathe/presto-coordinator:${TEST_PRESTO_VERSION}
+docker pull lewuathe/presto-worker:${TEST_PRESTO_VERSION}
+docker pull postgres:${TEST_PGSQL_VERSION}
+yarn lerna run --concurrency 1 --stream --no-prefix integration:trino
+echo "::endgroup::"

.github/workflows/drivers-tests.yml

@@ -57,9 +57,16 @@ on:
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'
       - 'rust/cubesql/**'
-
-env:
-  CUBEJS_TESSERACT_ORCHESTRATOR: true
+  workflow_dispatch:
+    inputs:
+      use_tesseract_sql_planner:
+        description: 'Enable TESSERACT_SQL_PLANNER?'
+        required: true
+        default: 'false'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
 
 jobs:
   latest-tag-sha:
@@ -205,6 +212,7 @@ jobs:
         athena-export-bucket-s3
         bigquery-export-bucket-gcs
         clickhouse-export-bucket-s3
+        clickhouse-export-bucket-s3-prefix
         databricks-jdbc
         databricks-jdbc-export-bucket-s3
         databricks-jdbc-export-bucket-s3-prefix
@@ -217,9 +225,12 @@ jobs:
         snowflake
         snowflake-encrypted-pk
         snowflake-export-bucket-s3
+        snowflake-export-bucket-s3-prefix
         snowflake-export-bucket-azure
+        snowflake-export-bucket-azure-prefix
         snowflake-export-bucket-azure-via-storage-integration
         snowflake-export-bucket-gcs
+        snowflake-export-bucket-gcs-prefix
       # As per docs:
       # Secrets cannot be directly referenced in if: conditionals. Instead, consider setting
       # secrets as job-level environment variables, then referencing the environment variables
@@ -232,6 +243,7 @@ jobs:
           - bigquery-export-bucket-gcs
           - clickhouse
           - clickhouse-export-bucket-s3
+          - clickhouse-export-bucket-s3-prefix
           - databricks-jdbc
           - databricks-jdbc-export-bucket-s3
           - databricks-jdbc-export-bucket-s3-prefix
@@ -247,9 +259,20 @@ jobs:
           - snowflake
           - snowflake-encrypted-pk
           - snowflake-export-bucket-s3
+          - snowflake-export-bucket-s3-prefix
           - snowflake-export-bucket-azure
+          - snowflake-export-bucket-azure-prefix
           - snowflake-export-bucket-azure-via-storage-integration
           - snowflake-export-bucket-gcs
+          - snowflake-export-bucket-gcs-prefix
+        use_tesseract_sql_planner: [ false ]
+        include:
+          - database: postgres
+            use_tesseract_sql_planner: true
+          - database: bigquery-export-bucket-gcs
+            use_tesseract_sql_planner: true
+          - database: athena-export-bucket-s3
+            use_tesseract_sql_planner: true
       fail-fast: false
 
     steps:
@@ -316,6 +339,8 @@ jobs:
           (contains(env.CLOUD_DATABASES, matrix.database) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
           (!contains(env.CLOUD_DATABASES, matrix.database))
         env:
+          DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_tesseract_sql_planner) || matrix.use_tesseract_sql_planner }}
+
           # Athena
           DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
           DRIVERS_TESTS_ATHENA_CUBEJS_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_SECRET }}
@@ -335,6 +360,8 @@ jobs:
           DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN }}
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY }}
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET }}
+          DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_ID: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_ID }}
+          DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_OAUTH_CLIENT_SECRET }}
 
           # Redshift
           DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST }}

.github/workflows/master.yml

@@ -155,7 +155,7 @@ jobs:
               ref: 'main',
               inputs: {
                 'cube-image': 'cubejs/cube:dev',
-                'source-repo': context.repo.repo,
+                'source-repo': `${context.repo.owner}/${context.repo.repo}`,
                 'source-pr-url': prUrl,
                 'source-commit-url': commitUrl,
                 'initiator': '${{ github.actor }}'

.github/workflows/publish.yml

@@ -242,9 +242,9 @@ jobs:
       matrix:
         node-version: [22.x]
         python-version: ["fallback"]
-        os-version: [windows-2019]
+        os-version: [windows-2022]
         include:
-          - os-version: windows-2019
+          - os-version: windows-2022
             package_target_arch: x64
             package_target_platform: win32
             package_target_libc: unknown
@@ -670,7 +670,7 @@ jobs:
           - aarch64-apple-darwin
         include:
           - target: x86_64-pc-windows-msvc
-            os: windows-2019
+            os: windows-2022
             executable_name: cubestored.exe
             strip: true
             # cubestored.exe: CantPackException: superfluous data between sections

.github/workflows/push.yml

@@ -19,6 +19,7 @@ on:
       - 'package.json'
       - 'lerna.json'
       - 'rollup.config.js'
+      - 'jest.base.config.js'
       - 'yarn.lock'
     branches:
       - 'master'
@@ -40,6 +41,7 @@ on:
       - 'package.json'
       - 'lerna.json'
       - 'rollup.config.js'
+      - 'jest.base.config.js'
       - 'yarn.lock'
 
 env:
@@ -118,15 +120,21 @@ jobs:
         run: yarn build
       - name: Lerna test
         run: yarn lerna run --concurrency 1 --stream --no-prefix unit
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v5
-        if: (matrix.node-version == '22.x')
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./packages/*/coverage/*
-          verbose: true
-          flags: cube-backend
-          fail_ci_if_error: false
+      - name: Fix lcov paths
+        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
+        run: |
+          ./.github/actions/codecov-fix.sh
+      - name: Combine all fixed LCOV files
+        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
+        run: |
+          echo "" > ./combined-unit.lcov
+          find ./packages -type f -name lcov.fixed.info -exec cat {} + >> ./combined-unit.lcov || true
+      - name: Upload coverage artifact
+        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-unit
+          path: ./combined-unit.lcov
 
   lint:
     runs-on: ubuntu-24.04
@@ -404,10 +412,15 @@ jobs:
       matrix:
         node-version: [22.x]
         db: [
-            'athena', 'bigquery', 'snowflake',
+            'athena', 'bigquery', 'snowflake', 'trino',
             'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
             'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio', 'vertica'
         ]
+        use_tesseract_sql_planner: [ false ]
+        include:
+          - db: postgres
+            node-version: 22.x
+            use_tesseract_sql_planner: true
       fail-fast: false
 
     steps:
@@ -473,6 +486,8 @@ jobs:
           (contains(env.CLOUD_DATABASES, matrix.db) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
           (!contains(env.CLOUD_DATABASES, matrix.db))
         env:
+          CUBEJS_TESSERACT_SQL_PLANNER: ${{ matrix.use_tesseract_sql_planner }}
+
           # Firebolt Integration
           DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME }}
           DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME }}
@@ -494,6 +509,18 @@ jobs:
           # Snowflake
           DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
           DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
+      - name: Fix lcov paths
+        run: |
+          ./.github/actions/codecov-fix.sh
+      - name: Combine all fixed LCOV files
+        run: |
+          echo "" > ./combined-integration-${{ matrix.db }}-${{ matrix.use_tesseract_sql_planner }}.lcov
+          find ./packages -type f -name lcov.fixed.info -exec cat {} + >> ./combined-integration-${{ matrix.db }}.lcov || true
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-integration-${{ matrix.db }}-${{ matrix.use_tesseract_sql_planner }}
+          path: ./combined-integration-${{ matrix.db }}-${{ matrix.use_tesseract_sql_planner }}.lcov
 
   integration-smoke:
     needs: [latest-tag-sha, build-cubestore, build-native-linux]
@@ -776,3 +803,29 @@ jobs:
         with:
           name: cypress-screenshots-docker-dev-${{ matrix.name }}
           path: packages/cubejs-testing/cypress/screenshots
+
+  upload-coverage:
+    name: Upload merged coverage to Codecov
+    needs: [unit, integration]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Install Codecov CLI
+        run: |
+          curl -Os https://uploader.codecov.io/latest/linux/codecov
+          chmod +x codecov
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-coverage
+      - name: Merge all coverage files
+        run: |
+          find all-coverage -name '*.lcov' -exec cat {} + > merged-coverage.info
+      - name: Upload merged coverage to Codecov
+        run: |
+          ./codecov --token "${{ secrets.CODECOV_TOKEN }}" \
+              --file ./merged-coverage.info \
+              --flags cube-backend \
+              --name "cube backend coverage"

.github/workflows/rust-cubesql.yml

@@ -57,6 +57,10 @@ jobs:
         run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings
       - name: Clippy Native (with Python)
         run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going --features python -- -D warnings
+      - name: Clippy cubenativeutils
+        run: cd rust/cubenativeutils && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings
+      - name: Clippy cubesqlplanner
+        run: cd rust/cubesqlplanner && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings
 
   unit:
     #  We use host instead of cross container, because it's much faster
@@ -85,6 +89,11 @@ jobs:
         with:
           crate: cargo-llvm-cov
           version: "0.6.10"
+      - name: Install [email protected]
+        uses: baptiste0928/cargo-install@v3
+        with:
+          crate: cargo-insta
+          version: "1.42.0"
       - name: Unit tests (Rewrite Engine)
         env:
           CUBESQL_TESTING_CUBE_TOKEN: ${{ secrets.CUBESQL_TESTING_CUBE_TOKEN }}
@@ -94,7 +103,11 @@ jobs:
           CUBESQL_REWRITE_TIMEOUT: 60
         run: |
           cd rust/cubesql
-          cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
+          # See https://github.com/taiki-e/cargo-llvm-cov/blob/main/README.md#get-coverage-of-external-tests
+          # shellcheck source=/dev/null
+          source <(cargo llvm-cov show-env --export-prefix)
+          cargo insta test --all-features --workspace --unreferenced reject
+          cargo llvm-cov report --lcov --output-path lcov.info
       - name: Upload code coverage
         uses: codecov/codecov-action@v5
         with:
@@ -312,7 +325,7 @@ jobs:
       matrix:
         # We do not need to test under all versions, we do it under linux
         node-version: [22.x]
-        os-version: [windows-2019]
+        os-version: [windows-2022]
         python-version: ["fallback"]
       fail-fast: false
 

.github/workflows/rust-cubestore-master.yml

@@ -189,7 +189,7 @@ jobs:
           - x86_64-apple-darwin
           - aarch64-apple-darwin
         include:
-          - os: windows-2019
+          - os: windows-2022
             target: x86_64-pc-windows-msvc
             executable_name: cubestored.exe
             strip: true

.github/workflows/rust-cubestore.yml

@@ -114,7 +114,7 @@ jobs:
           - x86_64-apple-darwin
           - aarch64-apple-darwin
         include:
-          - os: windows-2019
+          - os: windows-2022
             target: x86_64-pc-windows-msvc
             executable_name: cubestored.exe
             strip: true