Merge remote-tracking branch 'upstream/master'

Author: ramalingamt

.editorconfig

@@ -11,3 +11,6 @@ insert_final_newline = true
 
 [*.rs]
 indent_size = 4
+
+[rust/cubesql/cubesql/egraph-debug-template/**/*.{js,jsx,ts,tsx}]
+indent_size = 4

.github/ISSUE_TEMPLATE/sql_api_query_issue.md

@@ -2,7 +2,7 @@
 name: SQL API Query Issue
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: 'api:sql'
 assignees: ''
 
 ---
@@ -13,8 +13,11 @@ Search for `Failed SQL` log message.
 **Logical Plan**
 Search for `Can't rewrite plan` log message.
 
+**Tool**
+Was thethe SQL query above generated by some BI tool or any other tool? Did you write it yourself?
+
 **Version:**
-[e.g. 0.4.5]
+E.g., v1.1.0.
 
 **Additional context**
 Add any other context about the problem here.

.github/actions/smoke.sh

@@ -55,4 +55,8 @@ echo "::endgroup::"
 
 echo "::group::MongoBI"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:mongobi
-echo "::endgroup::"
+echo "::endgroup::"
+
+echo "::group::RBAC"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:rbac
+echo "::endgroup::"

.github/workflows/drivers-tests.yml

@@ -144,6 +144,8 @@ jobs:
     if: (needs['latest-tag-sha'].outputs.sha != github.sha)
     runs-on: ubuntu-20.04
     timeout-minutes: 30
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -154,6 +156,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+        if: (env.DOCKERHUB_USERNAME != '')
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Download native build
@@ -167,27 +170,66 @@ jobs:
           context: .
           file: ./packages/cubejs-docker/testing-drivers.Dockerfile
           tags: cubejs/cube:testing-drivers
-          push: true
+          push: ${{ (env.DOCKERHUB_USERNAME != '') }}
+      - name: Save Docker image as artifact
+        run: |
+          IMAGE_TAG=cubejs/cube:testing-drivers
+          docker save -o image.tar $IMAGE_TAG
+          gzip image.tar
+        continue-on-error: true
+      - name: Upload Docker image artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image
+          path: image.tar.gz
 
   tests:
     runs-on: ubuntu-20.04
     timeout-minutes: 30
     needs: [latest-tag-sha, build]
     if: (needs['latest-tag-sha'].outputs.sha != github.sha)
+    env:
+      CLOUD_DATABASES: >
+        athena-export-bucket-s3
+        bigquery-export-bucket-gcs
+        clickhouse-export-bucket-s3
+        databricks-jdbc
+        databricks-jdbc-export-bucket-s3
+        databricks-jdbc-export-bucket-azure
+        redshift
+        redshift-export-bucket-s3
+        snowflake
+        snowflake-export-bucket-s3
+        snowflake-export-bucket-azure
+        snowflake-export-bucket-azure-via-storage-integration
+        snowflake-export-bucket-gcs
+      # As per docs:
+      # Secrets cannot be directly referenced in if: conditionals. Instead, consider setting
+      # secrets as job-level environment variables, then referencing the environment variables
+      # to conditionally run steps in the job.
+      DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
     strategy:
       matrix:
         node:
           - 20.x
         database:
-          - athena
-          - bigquery
+          - athena-export-bucket-s3
+          - bigquery-export-bucket-gcs
           - clickhouse
+          - clickhouse-export-bucket-s3
           - databricks-jdbc
-          - databricks-jdbc-export-bucket
+          - databricks-jdbc-export-bucket-s3
+          - databricks-jdbc-export-bucket-azure
           - mssql
           - mysql
           - postgres
+          - redshift
+          - redshift-export-bucket-s3
           - snowflake
+          - snowflake-export-bucket-s3
+          - snowflake-export-bucket-azure
+          - snowflake-export-bucket-azure-via-storage-integration
+          - snowflake-export-bucket-gcs
       fail-fast: false
 
     steps:
@@ -237,7 +279,22 @@ jobs:
           cd packages/cubejs-testing-drivers
           yarn tsc
 
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker image into Docker Daemon
+        run: |
+          gunzip image.tar.gz
+          docker load -i image.tar
+
       - name: Run tests
+        uses: nick-fields/retry@v3
+        # It's enough to test for any one secret because they are set all at once or not set all
+        if: |
+          (contains(env.CLOUD_DATABASES, matrix.database) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
+          (!contains(env.CLOUD_DATABASES, matrix.database))
         env:
           # Athena
           DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
@@ -246,16 +303,34 @@ jobs:
           # BigQuery
           DRIVERS_TESTS_CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_BQ_CREDENTIALS }}
 
+          #GCS
+          DRIVERS_TESTS_CUBEJS_DB_EXPORT_GCS_CREDENTIALS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_GCS_CREDENTIALS }}
+
+          # Azure
+          DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY }}
+          DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_SAS_TOKEN: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_SAS_TOKEN }}
+
           # Databricks
           DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_URL: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_URL }}
           DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN }}
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY }}
           DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET }}
 
+          # Redshift
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_HOST }}
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_USER }}
+          DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_REDSHIFT_PASS }}
+
           # Snowflake
           DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }}
           DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }}
-        run: |
-          cd ./packages/cubejs-testing-drivers
-          export DEBUG=testcontainers
-          yarn ${{ matrix.database }}-full
+        with:
+          max_attempts: 3
+          retry_on: error
+          retry_wait_seconds: 15
+          timeout_minutes: 20
+          command: |
+            cd ./packages/cubejs-testing-drivers
+            export DEBUG=testcontainers
+            yarn ${{ matrix.database }}-full
+

.github/workflows/master.yml

@@ -137,12 +137,19 @@ jobs:
         with:
           github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
           script: |
+            const prUrl = context.payload.pull_request ? context.payload.pull_request.html_url : '';
+            const commitUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`;
+
             await github.rest.actions.createWorkflowDispatch({
               owner: 'cubedevinc',
               repo: 'sql-api-test-suite',
               workflow_id: 'test_and_run_test_suites.yml',
               ref: 'main',
               inputs: {
-                'cube-image': 'cubejs/cube:dev'
+                'cube-image': 'cubejs/cube:dev',
+                'source-repo': context.repo.repo,
+                'source-pr-url': prUrl,
+                'source-commit-url': commitUrl,
+                'initiator': '${{ github.actor }}'
               }
             })

.github/workflows/post-release.yml

@@ -205,22 +205,3 @@ jobs:
           color: danger
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-
-  trigger-repo-sync:
-    runs-on: ubuntu-20.04
-    name: 'Sync runtime repo'
-    timeout-minutes: 60
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Trigger runtime
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.GH_TRIGGER_TOKEN }}
-          script: |
-            await github.rest.actions.createWorkflowDispatch({
-              owner: 'cubedevinc',
-              repo: 'cube-runtime',
-              workflow_id: 'sync.yml',
-              ref: 'master'
-            })