Merge branch 'master' into chore-optimize-pre-aggrgations-api

Author: RusovDmitriy

.github/actions/integration/mssql.sh

@@ -4,13 +4,6 @@ set -eo pipefail
 # Debug log for test containers
 export DEBUG=testcontainers
 
-export TEST_MSSQL_VERSION=2017-latest
-
-echo "::group::MSSQL ${TEST_MSSQL_VERSION}";
-docker pull mcr.microsoft.com/mssql/server:${TEST_MSSQL_VERSION}
-yarn lerna run --concurrency 1 --stream --no-prefix integration:mssql
-echo "::endgroup::"
-
 export TEST_MSSQL_VERSION=2019-latest
 
 echo "::group::MSSQL ${TEST_MSSQL_VERSION}";

.github/workflows/cloud.yml

@@ -25,9 +25,12 @@ on:
       - 'package.json'
       - 'yarn.lock'
 
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
+
 jobs:
   latest-tag-sha:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     outputs:
       sha: ${{ steps.get-tag.outputs.sha }}
     steps:
@@ -52,18 +55,27 @@ jobs:
   integration-cloud:
     needs: latest-tag-sha
     if: (needs['latest-tag-sha'].outputs.sha != github.sha)
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
 
     strategy:
       matrix:
         node-version: [ 20.x ]
         db: [ 'athena', 'bigquery', 'snowflake' ]
+        target: [ "x86_64-unknown-linux-gnu" ]
       fail-fast: false
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: 1.84.1
+          # override: true # this is by default on
+          rustflags: ""
+          components: rustfmt
+          target: ${{ matrix.target }}
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -82,17 +94,37 @@ jobs:
       - name: Set Yarn version
         run: yarn policies set-version v1.22.22
       - name: Yarn install
-        run: CUBESTORE_SKIP_POST_INSTALL=true yarn install --frozen-lockfile
-      - name: Build client
+        uses: nick-fields/retry@v3
+        env:
+          CUBESTORE_SKIP_POST_INSTALL: true
+        with:
+          max_attempts: 3
+          retry_on: error
+          retry_wait_seconds: 15
+          timeout_minutes: 20
+          command: yarn install --frozen-lockfile
+      - name: Build Core Client libraries
         run: yarn build
+      - name: Build other packages
+        run: yarn lerna run --concurrency 1 build
+        env:
+          NODE_OPTIONS: --max_old_space_size=4096
       - name: Lerna tsc
         run: yarn tsc
+      - name: Build cubejs-backend-native (without Python)
+        run: yarn run native:build-release
+        working-directory: ./packages/cubejs-backend-native
       - name: Run Integration tests for ${{ matrix.db }} matrix
-        timeout-minutes: 30
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 3
+          retry_on: error
+          retry_wait_seconds: 15
+          timeout_minutes: 30
+          command: ./.github/actions/integration/${{ matrix.db }}.sh
         env:
           CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
           CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
           CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
           CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
           CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
-        run: ./.github/actions/integration/${{ matrix.db }}.sh

.github/workflows/cross-images.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   docker-dev:
     name: Build cross image for ${{ matrix.target }} target
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 120
     strategy:
       matrix:

.github/workflows/drivers-tests.yml

@@ -23,6 +23,7 @@ on:
       - 'packages/cubejs-mssql-driver/**'
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
+      - 'packages/cubejs-redshift-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
       - 'packages/cubejs-vertica-driver/**'
 
@@ -49,16 +50,20 @@ on:
       - 'packages/cubejs-mssql-driver/**'
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
+      - 'packages/cubejs-redshift-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
       - 'packages/cubejs-vertica-driver/**'
 
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'
       - 'rust/cubesql/**'
 
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
+
 jobs:
   latest-tag-sha:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     outputs:
       sha: ${{ steps.get-tag.outputs.sha }}
     steps:
@@ -81,7 +86,7 @@ jobs:
           OUT: ${{ steps.get-tag.outputs.sha }}
 
   native_linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     name: Build native Linux ${{ matrix.node-version }} ${{ matrix.target }} Python ${{ matrix.python-version }}
     strategy:
@@ -99,7 +104,7 @@ jobs:
       - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: nightly-2024-07-15
+          toolchain: 1.84.1
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
@@ -117,8 +122,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: ./packages/cubejs-backend-native
-          key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
-          shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
+          key: native-${{ runner.OS }}-${{ matrix.target }}
+          shared-key: native-${{ runner.OS }}-${{ matrix.target }}
       - name: Build native (fallback)
         if: (matrix.python-version == 'fallback')
         env:
@@ -144,7 +149,7 @@ jobs:
   build:
     needs: [latest-tag-sha, native_linux]
     if: (needs['latest-tag-sha'].outputs.sha != github.sha)
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -165,7 +170,12 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: backend-native
-          path: packages/cubejs-backend-native/
+          path: packages/cubejs-backend-native
+      # current .dockerignore prevents use of native build
+      - name: Unignore native from .dockerignore
+        run: |
+          grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp
+          mv .dockerignore.tmp .dockerignore
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
@@ -186,7 +196,7 @@ jobs:
           path: image.tar.gz
 
   tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     needs: [latest-tag-sha, build]
     if: (needs['latest-tag-sha'].outputs.sha != github.sha)