feat(pinot-driver): Add enableNullHandling to query options using env var (#9310)

* feat(pinot-driver): Add enableNullHandling=true to query options

* Add enableNullHandling=true to query options using env var CUBEJS_DB_NULL_HANDLING

* Add 'The Startree/Pinot null value support' to docs

* fix reference env var

* Update docs/pages/product/configuration/data-sources/pinot.mdx

Co-authored-by: Konstantin Burkalev <[email protected]>

* Update docs/pages/reference/configuration/environment-variables.mdx

Co-authored-by: ron-damon <[email protected]>

* Update docs/pages/reference/configuration/environment-variables.mdx

Co-authored-by: ron-damon <[email protected]>

* Update packages/cubejs-backend-shared/src/env.ts

Co-authored-by: Konstantin Burkalev <[email protected]>

* fix env var name

* Add docs for auth token config

---------

Co-authored-by: Igor Lukanin <[email protected]>
Co-authored-by: sergisulca <[email protected]>
Co-authored-by: Sergio Sulca <[email protected]>
Co-authored-by: Konstantin Burkalev <[email protected]>
Co-authored-by: ron-damon <[email protected]>

Author: 5 people

docs/pages/product/configuration/data-sources/pinot.mdx

@@ -14,8 +14,9 @@ workloads. [StarTree][link-startree] is a fully-managed platform for Pinot.
 - The hostname for the [Pinot][pinot] broker
 - The port for the [Pinot][pinot] broker
 
-With the current implementation of the Pinot driver, you have to enable the
-[multi-stage query engine][link-pinot-msqe] in your Pinot cluster.
+Note that the following features should be enabled in your Pinot cluster:
+- [Multi-stage query engine][link-pinot-msqe].
+- [Advanced null value support][link-pinot-nvs].
 
 ## Setup
 
@@ -41,14 +42,15 @@ CUBEJS_DB_PASS=**********
 
 ## Environment Variables
 
-| Environment Variable | Description                                | Possible Values     | Required |
-|----------------------|--------------------------------------------|---------------------|:--------:|
-| `CUBEJS_DB_HOST`     | The host URL for your Pinot broker         | A valid host URL    |    ✅    |
-| `CUBEJS_DB_PORT`     | The port for the database connection       | A valid port number |    ✅    |
-| `CUBEJS_DB_USER`     | The username used to connect to the broker | A valid username    |    ❌    |
-| `CUBEJS_DB_PASS`     | The password used to connect to the broker | A valid password    |    ❌    |
-| `CUBEJS_DB_NAME`     | The database name for StarTree             | A valid name        |    ❌    |
-| `CUBEJS_DB_PINOT_AUTH_TOKEN` | The authentication token for StarTree | A valid token    |    ❌    |
+| Environment Variable            | Description                                           | Possible Values     | Required |
+|---------------------------------|-------------------------------------------------------|---------------------|:--------:|
+| `CUBEJS_DB_HOST`                | The host URL for your Pinot broker                    | A valid host URL    |    ✅    |
+| `CUBEJS_DB_PORT`                | The port for the database connection                  | A valid port number |    ✅    |
+| `CUBEJS_DB_USER`                | The username used to connect to the broker            | A valid username    |    ❌    |
+| `CUBEJS_DB_PASS`                | The password used to connect to the broker            | A valid password    |    ❌    |
+| `CUBEJS_DB_NAME`                | The database name for StarTree                        | A valid name        |    ❌    |
+| `CUBEJS_DB_PINOT_NULL_HANDLING` | If `true`, enables null handling. Default is `false`  | `true`, `false`     |    ❌    |
+| `CUBEJS_DB_PINOT_AUTH_TOKEN`    | The authentication token for StarTree                 | A valid token       |    ❌    |
 
 ## Pre-Aggregation Feature Support
 
@@ -97,6 +99,7 @@ Cube does not require any additional configuration to enable SSL as Pinot connec
 [link-pinot]: https://pinot.apache.org/
 [pinot]: https://docs.pinot.apache.org/
 [link-pinot-msqe]: https://docs.pinot.apache.org/reference/multi-stage-engine
+[link-pinot-nvs]: https://docs.pinot.apache.org/developers/advanced/null-value-support#advanced-null-handling-support
 [pinot-docs-approx-agg-fns]:
   https://docs.pinot.apache.org/users/user-guide-query/query-syntax/how-to-handle-unique-counting
 [ref-recipe-enable-ssl]:

docs/pages/reference/configuration/environment-variables.mdx

@@ -740,6 +740,22 @@ The Snowflake warehouse to use when connecting to the database.
 | ---------------------------------------------------------------------- | ---------------------- | --------------------- |
 | [A valid Snowflake warehouse][snowflake-docs-warehouse] in the account | N/A                    | N/A                   |
 
+## `CUBEJS_DB_PINOT_NULL_HANDLING`
+
+The Startree/Pinot null value support. If `true`, enables null handling.
+
+| Possible Values | Default in Development | Default in Production |
+| --------------- | ---------------------- | --------------------- |
+| `true`, `false` |	`false`                | `false`               |
+
+## `CUBEJS_DB_PINOT_AUTH_TOKEN`
+
+The authentication token for StarTree to be passed as HTTP headers.
+
+| Possible Values                     | Default in Development | Default in Production |
+| ----------------------------------- | ---------------------- | --------------------- |
+| A valid string containing the token |	N/A                    | N/A                   |
+
 ## `CUBEJS_DB_SSL`
 
 If `true`, enables SSL encryption for database connections from Cube.

packages/cubejs-backend-shared/src/env.ts

@@ -1784,6 +1784,35 @@ const variables: Record<string, (...args: any) => any> = {
     ]
   ),
 
+  /**
+   * Pinot / Startree Null value support
+   */
+
+  pinotNullHandling: ({ dataSource }: { dataSource: string }) => {
+    const val = process.env[
+      keyByDataSource('CUBEJS_DB_PINOT_NULL_HANDLING', dataSource)
+    ];
+
+    if (val) {
+      if (val.toLocaleLowerCase() === 'true') {
+        return true;
+      } else if (val.toLowerCase() === 'false') {
+        return false;
+      } else {
+        throw new TypeError(
+          `The ${
+            keyByDataSource(
+              'CUBEJS_DB_PINOT_NULL_HANDLING',
+              dataSource,
+            )
+          } must be either 'true' or 'false'.`
+        );
+      }
+    } else {
+      return false;
+    }
+  },
+
   /** ****************************************************************
    * Dremio Driver                                                   *
    ***************************************************************** */

packages/cubejs-pinot-driver/src/PinotDriver.ts

@@ -33,6 +33,7 @@ export type PinotDriverConfiguration = {
   ssl?: string | TLSConnectionOptions;
   dataSource?: string;
   queryTimeout?: number;
+  nullHandling?: boolean;
 };
 
 type AuthorizationHeaders = {
@@ -108,6 +109,7 @@ export class PinotDriver extends BaseDriver implements DriverInterface {
         : undefined,
       authToken: getEnv('pinotAuthToken', { dataSource }),
       ssl: this.getSslOptions(dataSource),
+      nullHandling: getEnv('pinotNullHandling', { dataSource }),
       queryTimeout: getEnv('dbQueryTimeout', { dataSource }),
       ...config
     };
@@ -167,7 +169,7 @@ export class PinotDriver extends BaseDriver implements DriverInterface {
       }),
       body: JSON.stringify({
         sql: query,
-        queryOptions: `useMultistageEngine=true;timeoutMs=${this.config.queryTimeout}`
+        queryOptions: `useMultistageEngine=true;enableNullHandling=${this.config.nullHandling};timeoutMs=${this.config.queryTimeout}`
       })
     });