Merge branch 'cube-js:master' into master

Author: ramalingamt

.github/actions/integration/dremio.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Dremio [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:dremio
+
+echo "::endgroup::"

.github/workflows/push.yml

@@ -316,6 +316,7 @@ jobs:
     env:
       CLOUD_DATABASES: >
         firebolt
+        dremio
       # Athena (just to check for secrets availability)
       DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
 
@@ -324,7 +325,7 @@ jobs:
         node-version: [20.x]
         db: [
             'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
-            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt'
+            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio'
         ]
       fail-fast: false
 
@@ -386,6 +387,10 @@ jobs:
           DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }}
           DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }}
           DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }}
+          # Dremio Integration
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }}
 
   integration-smoke:
     needs: [ latest-tag-sha, build-cubestore ]

CHANGELOG.md

@@ -3,6 +3,39 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.1.11](https://github.com/cube-js/cube/compare/v1.1.10...v1.1.11) (2024-12-16)
+
+
+### Bug Fixes
+
+* TypeError: Cannot read properties of undefined (reading 'joins') ([14adaeb](https://github.com/cube-js/cube/commit/14adaebdd1c3d398bcd2997012da070999e47d9d))
+
+
+
+
+
+## [1.1.10](https://github.com/cube-js/cube/compare/v1.1.9...v1.1.10) (2024-12-16)
+
+
+### Bug Fixes
+
+* **api-gateway:** allow switch sql user when the new user is the same ([#9037](https://github.com/cube-js/cube/issues/9037)) ([a69c28f](https://github.com/cube-js/cube/commit/a69c28f524fa0625b825b98a38e7f5a211a98f74))
+* **api-gateway:** make sure DAP works sql pushdown ([#9021](https://github.com/cube-js/cube/issues/9021)) ([23695b2](https://github.com/cube-js/cube/commit/23695b2b5e886b5b7daf8b3f74003bb04e5b2e0b))
+* **cubestore:** Allow create an index from expressions ([#9006](https://github.com/cube-js/cube/issues/9006)) ([222cab8](https://github.com/cube-js/cube/commit/222cab897c289bfc929f217483e4905204bac12f))
+* **schema-compiler:** fix DAP with query_rewrite and python config ([#9033](https://github.com/cube-js/cube/issues/9033)) ([849790f](https://github.com/cube-js/cube/commit/849790f965dd0d9fddba11e3d8d124b84397ca9b))
+* **schema-compiler:** join relationship aliases ([ad4e8e3](https://github.com/cube-js/cube/commit/ad4e8e3872307ab77e035709e5208b0191f87f5b))
+
+
+### Features
+
+* **cubesql:** Basic VALUES support in rewrite engine ([#9041](https://github.com/cube-js/cube/issues/9041)) ([368671f](https://github.com/cube-js/cube/commit/368671fd1b53b2ed5ad8df6af113492982f23c0c))
+* **dremio-driver:** Add Dremio Cloud Support ([#8956](https://github.com/cube-js/cube/issues/8956)) ([d2c2fcd](https://github.com/cube-js/cube/commit/d2c2fcdaf8944ea7dd27e73b63c0b151c317022e))
+* **tesseract:** Support multiple join paths within single query ([#9047](https://github.com/cube-js/cube/issues/9047)) ([b62446e](https://github.com/cube-js/cube/commit/b62446e3c3893068f8dd8aa32d7204ea06a16f98))
+
+
+
+
+
 ## [1.1.9](https://github.com/cube-js/cube/compare/v1.1.8...v1.1.9) (2024-12-08)
 
 

docs/pages/product/apis-integrations/ai-api.mdx

@@ -181,6 +181,44 @@ to give the AI context on possible values in a categorical dimension:
               - completed
 ```
 
+### Value search
+
+By default, the AI API has no ability to see the contents of your data (for privacy reasons).
+However, this makes it difficult for the AI API to generate correct filters for some queries.
+
+Imagine you have a categorical `order_status` dimension with the possible values "shipped",
+"processing", and "completed". Without value search, asking "how many complete orders did
+we have today" might get you a query filtering on `order_status = 'Complete'` instead of
+the correct `order_status = 'completed'`.
+
+To solve this, the AI API can perform "value searches" where it introspects the values in
+selected categorical dimensions before running a query. Value search is opt-in and dimensions
+must be enabled for it individually. Currently, the AI API performs value search by running
+Cube queries using the `contains` filter operator against one or more chosen dimensions.
+The LLM will select dimensions from among those you have based on the question asked and
+generate possible values dynamically.
+
+<InfoBox>
+  When running value search queries, the AI API passes through the security context used
+  for the AI API request, so security is maintained and only dimensions the end user has
+  access to are able to be searched.
+</InfoBox>
+
+To enable value search on a dimension, set the `searchable` field to true under the `ai`
+meta tag, as shown below:
+```yaml
+    - name: order_status
+      sql: order_status
+      type: string
+      meta:
+        ai:
+          searchable: true
+```
+
+Note that enabling Value Search may lead to slightly longer AI API response times when it
+is used but should result in significantly more accurate queries in many situations. Value
+Search can only be used on string dimensions.
+
 ### Other LLM providers
 
 <InfoBox>

docs/pages/product/workspace/ai-assistant.mdx

@@ -97,6 +97,11 @@ to give the AI context on possible values in a categorical dimension:
               - completed
 ```
 
+### Value search
+
+Value Search can be enabled for AI Assistant in the same way as for the AI API. See the 
+[AI API's documentation][ref-ai-api-value-search] for details and instructions.
+
 ### Other LLM providers
 
 See the [AI API's documentation][ref-ai-api-providers] for information on how to "bring your own" LLM. 
@@ -127,3 +132,4 @@ See the [AI API's documentation][ref-ai-api-providers] for information on how to
 [ref-playground]: /product/workspace/playground
 [ref-catalog-downstream]: /product/workspace/semantic-catalog#connecting-downstream-tools
 [ref-ai-api-providers]: /product/apis-integrations/ai-api#other-llm-providers
+[ref-ai-api-value-search]: /product/apis-integrations/ai-api#value-search

docs/pages/reference/data-model/joins.mdx

@@ -409,6 +409,9 @@ cubes:
 Cube automatically detects chasm and fan traps based on the `many_to_one` and `one_to_many` relationships defined in join.
 When detected, Cube generates a deduplication query that evaluates all distinct primary keys within the multiplied measure's cube and then joins distinct primary keys to this cube on itself to calculate the aggregation result.
 If there's more than one multiplied measure in a query, then such query is generated for every such multiplied measure, and results are joined.
+Cube solves for chasm and fan traps during query time.
+If there's pre-aggregregation that fits measure multiplication requirements it'd be leveraged to serve such a query.
+Such pre-aggregations and queries are always considered non-additive for the purpose of pre-aggregation matching.
 
 Let's consider an example data model:
 

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.1.9",
+  "version": "1.1.11",
   "npmClient": "yarn",
   "useWorkspaces": true,
   "packages": [

packages/cubejs-api-gateway/CHANGELOG.md

@@ -3,6 +3,19 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.1.10](https://github.com/cube-js/cube/compare/v1.1.9...v1.1.10) (2024-12-16)
+
+
+### Bug Fixes
+
+* **api-gateway:** allow switch sql user when the new user is the same ([#9037](https://github.com/cube-js/cube/issues/9037)) ([a69c28f](https://github.com/cube-js/cube/commit/a69c28f524fa0625b825b98a38e7f5a211a98f74))
+* **api-gateway:** make sure DAP works sql pushdown ([#9021](https://github.com/cube-js/cube/issues/9021)) ([23695b2](https://github.com/cube-js/cube/commit/23695b2b5e886b5b7daf8b3f74003bb04e5b2e0b))
+* **schema-compiler:** fix DAP with query_rewrite and python config ([#9033](https://github.com/cube-js/cube/issues/9033)) ([849790f](https://github.com/cube-js/cube/commit/849790f965dd0d9fddba11e3d8d124b84397ca9b))
+
+
+
+
+
 ## [1.1.9](https://github.com/cube-js/cube/compare/v1.1.8...v1.1.9) (2024-12-08)
 
 **Note:** Version bump only for package @cubejs-backend/api-gateway

packages/cubejs-api-gateway/package.json

@@ -2,7 +2,7 @@
   "name": "@cubejs-backend/api-gateway",
   "description": "Cube.js API Gateway",
   "author": "Cube Dev, Inc.",
-  "version": "1.1.9",
+  "version": "1.1.10",
   "repository": {
     "type": "git",
     "url": "https://github.com/cube-js/cube.git",
@@ -27,8 +27,8 @@
     "dist/src/*"
   ],
   "dependencies": {
-    "@cubejs-backend/native": "1.1.9",
-    "@cubejs-backend/shared": "1.1.8",
+    "@cubejs-backend/native": "1.1.10",
+    "@cubejs-backend/shared": "1.1.10",
     "@ungap/structured-clone": "^0.3.4",
     "body-parser": "^1.19.0",
     "chrono-node": "^2.6.2",

packages/cubejs-api-gateway/src/gateway.ts

@@ -1207,16 +1207,16 @@ class ApiGateway {
           }
 
           // First apply cube/view level security policies
-          const queryWithRlsFilters = await compilerApi.applyRowLevelSecurity(
+          const { query: queryWithRlsFilters, denied } = await compilerApi.applyRowLevelSecurity(
             normalizedQuery,
             evaluatedQuery,
             context
           );
           // Then apply user-supplied queryRewrite
-          let rewrittenQuery = await this.queryRewrite(
+          let rewrittenQuery = !denied ? await this.queryRewrite(
             queryWithRlsFilters,
             context
-          );
+          ) : queryWithRlsFilters;
 
           // applyRowLevelSecurity may add new filters which may contain raw member expressions
           // if that's the case, we should run an extra pass of parsing here to make sure