Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions docs/pages/product/configuration/data-sources/pinot.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ redirect_from:

# Pinot

Apache Pinot is a real-time distributed OLAP datastore purpose-built for low-latency, high-throughput analytics, and perfect for user-facing analytical workloads
Apache Pinot is a real-time distributed OLAP datastore purpose-built for low-latency, high-throughput analytics,
and perfect for user-facing analytical workloads.

## Prerequisites

Expand All @@ -28,12 +29,12 @@ CUBEJS_DB_PASS=**********

## Environment Variables

| Environment Variable | Description | Possible Values | Required |
| -------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------- | :------: |
| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ |
| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ |
| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ |
| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ |
| Environment Variable | Description | Possible Values | Required |
|----------------------|--------------------------------------------|---------------------|:--------:|
| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ |
| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ |
| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ |
| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ |

## Pre-Aggregation Feature Support

Expand All @@ -55,8 +56,8 @@ here][ref-caching-using-preaggs-build-strats].
</InfoBox>

| Feature | Works with read-only mode? | Is default? |
| ------------- | :------------------------: | :---------: |
| Simple | ✅ | |
|---------------|:--------------------------:|:-----------:|
| Simple | ✅ | |
| Batching | - | - |
| Export bucket | - | - |

Expand All @@ -77,7 +78,7 @@ Pinot does not support export buckets.

## SSL

Cube does not require any additional configuration to enable SSL as Elasticsearch connections are made over HTTPS.
Cube does not require any additional configuration to enable SSL as Pinot connections are made over HTTPS.

[pinot]: https://docs.pinot.apache.org/
[pinot-docs-approx-agg-fns]:
Expand Down
2 changes: 2 additions & 0 deletions packages/cubejs-pinot-driver/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Change Log

34 changes: 2 additions & 32 deletions packages/cubejs-pinot-driver/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,6 @@ Cube.js Pinot driver.

[Learn more](https://github.com/cube-js/cube.js#getting-started)

### Project Status
# License

In Review. [#8689](https://github.com/cube-js/cube/pull/8689)

### Installation

`npm i @inthememory/pinot-driver`

### Usage
#### For Docker

Build development [docker image](https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/DEVELOPMENT.md).

Assuming the built image is tagged `cubejs/cube:dev`

```
FROM cubejs/cube:dev
RUN npm i @inthememory/pinot-driver
```

```
environment:
- CUBEJS_DB_TYPE=pinot
- CUBEJS_DB_HOST= #broker_host
- CUBEJS_DB_PORT= #broker_port
- CUBEJS_DB_USER= #database user
- CUBEJS_DB_PASS= #database password
- CUBEJS_DEV_MODE=true #if running locally
```

### License

Cube.js Pinot Database Driver is [Apache 2.0 licensed](./LICENSE).
Cube.js Pinot Database Driver is [Apache 2.0 licensed](./LICENSE).
24 changes: 12 additions & 12 deletions packages/cubejs-pinot-driver/package.json
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
"name": "@inthememory/pinot-driver",
"name": "@cubejs-backend/pinot-driver",
"description": "Cube.js Pinot database driver",
"author": "Julian Ronsse, InTheMemory",
"version": "0.35.80",
"author": "Julian Ronsse, InTheMemory, Cube Dev, Inc.",
"version": "1.1.0",
"repository": {
"type": "git",
"url": "https://github.com/cube-js/cube.git",
"directory": "packages/cubejs-pinotdb-driver"
"directory": "packages/cubejs-pinot-driver"
},
"engines": {
"node": "^14.0.0 || ^16.0.0 || >=17.0.0"
Expand All @@ -21,29 +21,29 @@
"build": "rm -rf dist && npm run tsc",
"tsc": "tsc",
"watch": "tsc -w",
"integration": "jest dist/test",
"integration:pinot": "jest dist/test",
"integration": "npm run integration:pinot",
"integration:pinot": "jest --verbose dist/test",
"lint": "eslint src/* --ext .ts",
"lint:fix": "eslint --fix src/* --ext .ts"
},
"dependencies": {
"@cubejs-backend/base-driver": "^0.36.0",
"@cubejs-backend/schema-compiler": "^0.36.0",
"@cubejs-backend/shared": "^0.36.0",
"@cubejs-backend/base-driver": "1.0.3",
"@cubejs-backend/schema-compiler": "1.1.0",
"@cubejs-backend/shared": "1.0.3",
"node-fetch": "^2.6.1",
"ramda": "^0.27.0",
"ramda": "^0.27.2",
"sqlstring": "^2.3.3"
},
"license": "Apache-2.0",
"publishConfig": {
"access": "public"
},
"devDependencies": {
"@cubejs-backend/linter": "^0.36.0",
"@cubejs-backend/linter": "^1.0.0",
"@types/jest": "^27",
"jest": "^27",
"should": "^13.2.3",
"testcontainers": "^10.13.0",
"testcontainers": "^10.10.4",
"typescript": "~5.2.2"
},
"jest": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ controller.data.dir=/var/pinot/controller/data
controller.zk.str=zookeeper:2181
pinot.set.instance.id.to.hostname=true
controller.admin.access.control.principals=admin
controller.admin.access.control.principals.admin.password=mysecret
controller.admin.access.control.principals.admin.password=mysecret
7 changes: 5 additions & 2 deletions packages/cubejs-pinot-driver/src/PinotDriver.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/**
* @copyright Cube Dev, Inc.
* @license Apache-2.0
* @fileoverview The `PrestoDriver` and related types declaration.
* @fileoverview The `PinotDriver` and related types declaration.
*/

import {
Expand Down Expand Up @@ -146,7 +146,10 @@ export class PinotDriver extends BaseDriver implements DriverInterface {
'Content-Type': 'application/json',
...this.authorizationHeaders()
}),
body: JSON.stringify({ sql: query, queryOptions: `useMultistageEngine=true;timeoutMs=${this.config.queryTimeout}` })
body: JSON.stringify({
sql: query,
queryOptions: `useMultistageEngine=true;timeoutMs=${this.config.queryTimeout}`
})
});

return new Promise((resolve, reject) => {
Expand Down
11 changes: 2 additions & 9 deletions packages/cubejs-pinot-driver/src/PinotQuery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ export class PinotQuery extends BaseQuery {
const values = timeDimension.timeSeries().map(
([from, to]) => `select '${from}' f, '${to}' t`
).join(' UNION ALL ');
return `SELECT ${this.timeStampCast('dates.f')} date_from, ${this.timeStampCast('dates.t')} date_to FROM (${values}) AS dates`;
return `SELECT ${this.timeStampCast('dates.f')} date_from, ${this.timeStampCast('dates.t')} date_to
FROM (${values}) AS dates`;
}

public applyMeasureFilters(evaluateSql: '*' | string, symbol: any, cubeName: string) {
Expand Down Expand Up @@ -123,14 +124,6 @@ export class PinotQuery extends BaseQuery {
};
}

public hllInit(sql: string) {
return this.countDistinctApprox(sql); // todo: ensure the correct way to do so in pinot
}

public hllMerge(sql: string) {
return this.countDistinctApprox(sql); // todo: ensure the correct way to do so in pinot
}

public countDistinctApprox(sql: string) {
return `DistinctCountHLLPlus(${sql})`;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ module.exports = {
ksql: '@cubejs-backend/ksql-driver',
questdb: '@cubejs-backend/questdb-driver',
materialize: '@cubejs-backend/materialize-driver',
pinot: '@inthememory/pinot-driver',
pinot: '@cubejs-backend/pinot-driver',
// List for JDBC drivers
'databricks-jdbc': '@cubejs-backend/databricks-jdbc-driver',
};
Loading