Merge branch 'master' into fix-index-and-expression

Author: RusovDmitriy

.github/actions/integration/firebolt.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Firebolt [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:firebolt
+
+echo "::endgroup::"

.github/workflows/push.yml

@@ -319,7 +319,7 @@ jobs:
         node-version: [20.x]
         db: [
             'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
-            'mysql-aurora-serverless', 'crate', 'mongobi'
+            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt'
         ]
       fail-fast: false
 
@@ -370,6 +370,13 @@ jobs:
           retry_wait_seconds: 15
           timeout_minutes: 30
           command: ./.github/actions/integration/${{ matrix.db }}.sh
+        env:
+          # Firebolt Integration
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }}
 
   integration-smoke:
     needs: [ latest-tag-sha, build-cubestore ]

docs/pages/product/apis-integrations/ai-api.mdx

@@ -11,6 +11,8 @@ Specifically, you can send the AI API a message (or conversation of messages) an
   your Cube representative to have it enabled for your account.
 </WarningBox>
 
+See [AI API reference][ref-ref-ai-api] for the list of supported API endpoinsts.
+
 ## Configuration
 
 While the AI API is in preview, your Cube account team will enable and configure it for you.
@@ -217,4 +219,7 @@ deployment. See below for required variables by provider (required unless noted)
 #### OpenAI
 
 - `OPENAI_MODEL` - An OpenAI chat model ID, for example `gpt-4o`
-- `OPENAI_API_KEY` - An OpenAI API key (we recommend creating a service account for the AI API)
+- `OPENAI_API_KEY` - An OpenAI API key (we recommend creating a service account for the AI API)
+
+
+[ref-ref-ai-api]: /reference/ai-api

docs/pages/product/apis-integrations/rest-api.mdx

@@ -16,6 +16,16 @@ data applications, including but not limited to the following ones:
 Often, the REST API is used to enable [embedded analytics][cube-ea] and
 [real-time analytics][cube-rta] use cases.
 
+See [REST API reference][ref-ref-rest-api] for the list of supported API endpoinsts.
+Also, check [query format][ref-rest-query-format] for details about query syntax.
+
+<InfoBox>
+
+You can find a mostly complete OpenAPI documentation for the REST API in the
+linked [`openspec.yml` file][gh-cube-openspec] that you can use with tools like Swagger.
+
+</InfoBox>
+
 <InfoBox>
 
 If you've chosen [GraphQL][graphql] as a query language for your front-end
@@ -24,10 +34,9 @@ provides.
 
 </InfoBox>
 
-Under the hood, REST API also provides endpoints for [GraphQL
-API][ref-graphql-api] and [Orchestration API][ref-orchestration-api]. However,
-they target specific use cases and are not usually considered part of the REST
-API.
+REST API also provides endpoints for [GraphQL API][ref-graphql-api] and
+[Orchestration API][ref-orchestration-api]. However, they target specific use
+cases and are not considered part of the REST API.
 
 ## Example request
 
@@ -84,10 +93,6 @@ curl \
   http://localhost:4000/cubejs-api/v1/meta | jq
 ```
 
-Check [query format][ref-rest-query-format] for details about supported queries.
-Also, check [REST API reference][ref-ref-rest-api] for the list of supported API
-endpoints.
-
 ## Configuration
 
 REST API is enabled by default and secured using [API scopes][self-api-scopes]
@@ -291,3 +296,4 @@ example, the following query will retrieve rows 101-200 from the `Orders` cube:
 [self-cors]: #configuration-cors
 [ref-ref-rest-api]: /reference/rest-api
 [link-jq-utility]: https://jqlang.github.io/jq/
+[gh-cube-openspec]: https://github.com/cube-js/cube/blob/master/packages/cubejs-api-gateway/openspec.yml

docs/pages/product/apis-integrations/sql-api.mdx

@@ -20,6 +20,10 @@ but not limited to the following ones:
 Often, the SQL API is used to enable internal or self-serve [business
 intelligence][cube-bi-use-case] use cases.
 
+See [SQL API reference][ref-ref-sql-api] for the list of supported SQL commands,
+functions, and operators. Also, check [query format][ref-sql-query-format] for
+details about supported queries.
+
 <InfoBox>
 
 Please use [this GitHub issue](https://github.com/cube-js/cube/issues/3906) to
@@ -78,10 +82,6 @@ WHERE table_schema = 'public';
 \d
 ```
 
-Check [query format][ref-sql-query-format] for details about supported queries.
-Also, check [SQL API reference][ref-ref-sql-api] for the list of supported SQL
-commands, functions, and operators.
-
 ## Fundamentals
 
 In the SQL API, each cube or view from the [data model][ref-data-model-concepts]

docs/pages/product/caching/using-pre-aggregations.mdx

@@ -601,6 +601,12 @@ The data for `zip_code_index` would look as follows:
 | 88523    | 3800        |
 | 88524    | 5000        |
 
+### Compaction
+
+Whenever a newer version of pre-aggregation is just built and becomes available its performance would be suboptimal as it's pending compaction.
+Most of the essential compaction process usually takes several seconds to several minutes for bigger partitions after pre-aggregation creation, depending on the size of the partition and the Cube Store workers' processing power available.
+This compaction process is usually unnoticeable for queries that are optimal in terms of index usage, so it's always best practice to make sure all of your queries match an index.
+
 ## Inspecting pre-aggregations
 
 Cube Store partially supports the MySQL protocol. This allows you to execute

packages/cubejs-firebolt-driver/package.json

@@ -23,7 +23,7 @@
     "watch": "tsc -w",
     "test": "yarn integration",
     "integration": "npm run integration:firebolt",
-    "integration:firebolt": "jest --verbose dist/test",
+    "integration:firebolt": "jest --verbose dist/test --runInBand",
     "lint": "eslint src/* --ext .ts",
     "lint:fix": "eslint --fix src/* --ext .ts"
   },
@@ -43,7 +43,10 @@
     "access": "public"
   },
   "jest": {
-    "testEnvironment": "node"
+    "testEnvironment": "node",
+    "setupFiles": [
+      "./test/test-env.js"
+    ]
   },
   "eslintConfig": {
     "extends": "../cubejs-linter"

packages/cubejs-firebolt-driver/test/Firebolt.test.ts renamed to packages/cubejs-firebolt-driver/test/FireboltDriver.test.ts

@@ -70,50 +70,48 @@ cube(\`sales\`, {
         measures: ['sales.count'],
         filters: [
           {
-            member: "sales.isShiped",
-            operator: "equals",
-            values: ["true"]
+            member: 'sales.isShiped',
+            operator: 'equals',
+            values: ['true']
           }
         ]
       }
-    )
+    );
 
     const queryAndParams = query.buildSqlAndParams();
 
     expect(queryAndParams[0]).toContain(
       '("sales".is_shiped = CAST(? AS BOOLEAN))'
     );
 
-    expect(queryAndParams[1]).toEqual(["true"]);
-  }))
-    it("should cast timestamp", () =>
-    compiler.compile().then(() => {
-      const query = new FireboltQuery(
-        { joinGraph, cubeEvaluator, compiler },
-        {
-          measures: ["sales.count"],
-          timeDimensions: [
-            {
-              dimension: "sales.salesDatetime",
-              granularity: "day",
-              dateRange: ["2017-01-01", "2017-01-02"],
-            },
-          ],
-          timezone: "America/Los_Angeles",
-          order: [
-            {
-              id: "sales.salesDatetime",
-            },
-          ],
-        }
-      );
-
-      const queryAndParams = query.buildSqlAndParams();
+    expect(queryAndParams[1]).toEqual(['true']);
+  }));
 
-      expect(queryAndParams[0]).toContain(
-        '("sales".sales_datetime >= ?::timestamptz AND "sales".sales_datetime <= ?::timestamptz)'
-      );
-    }));
+  it('should cast timestamp', () => compiler.compile().then(() => {
+    const query = new FireboltQuery(
+      { joinGraph, cubeEvaluator, compiler },
+      {
+        measures: ['sales.count'],
+        timeDimensions: [
+          {
+            dimension: 'sales.salesDatetime',
+            granularity: 'day',
+            dateRange: ['2017-01-01', '2017-01-02'],
+          },
+        ],
+        timezone: 'America/Los_Angeles',
+        order: [
+          {
+            id: 'sales.salesDatetime',
+          },
+        ],
+      }
+    );
 
+    const queryAndParams = query.buildSqlAndParams();
 
-})
+    expect(queryAndParams[0]).toContain(
+      '("sales".sales_datetime >= ?::timestamptz AND "sales".sales_datetime <= ?::timestamptz)'
+    );
+  }));
+});

packages/cubejs-firebolt-driver/test/FireboltQuery.test.ts

@@ -1,9 +1,9 @@
-import {assertDataSource, getEnv} from '@cubejs-backend/shared';
+import { assertDataSource, getEnv } from '@cubejs-backend/shared';
 import { DriverTests } from '@cubejs-backend/testing-shared';
 
-import { FireboltDriver } from '../src';
 import { Firebolt } from 'firebolt-sdk';
 import { version } from 'firebolt-sdk/package.json';
+import { FireboltDriver } from '../src';
 
 describe('FireboltDriver autostart', () => {
   let tests: DriverTests;
@@ -43,7 +43,7 @@ describe('FireboltDriver autostart', () => {
     const engineName = getEnv('fireboltEngineName', { dataSource });
     const firebolt = Firebolt({
       apiEndpoint: getEnv('fireboltApiEndpoint', { dataSource }) || 'api.app.firebolt.io',
-    })
+    });
     await firebolt.connect({
       auth,
       database: getEnv('dbName', { dataSource }),