Skip to content

Commit 0c41721

Browse files
GraphmaxerGraphmaxer
committed
feat(clickhouse-driver): allow to enable compression
1 parent 6ea1817 commit 0c41721

File tree

6 files changed

+89
-11
lines changed

6 files changed

+89
-11
lines changed

‎docs/pages/product/configuration/data-sources/clickhouse.mdx‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@ CUBEJS_DB_PASS=**********
3131

3232
## Environment Variables
3333

34-
| Environment Variable | Description | Possible Values | Required |
35-
| ------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: |
36-
| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ |
37-
| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ |
38-
| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ |
39-
| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ |
40-
| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ |
41-
| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ |
42-
| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ |
43-
| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ |
34+
| Environment Variable | Description | Possible Values | Required |
35+
| ---------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: |
36+
| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ |
37+
| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ |
38+
| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ |
39+
| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ |
40+
| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ |
41+
| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ |
42+
| `CUBEJS_DB_CLICKHOUSE_COMPRESSION` | Whether the ClickHouse client has compression enabled or not | `true`, `false` | ❌ |
43+
| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ |
44+
| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ |
4445

4546
[ref-data-source-concurrency]: /product/configuration/concurrency#data-source-concurrency
4647

@@ -130,6 +131,9 @@ You can connect to a ClickHouse database when your user's permissions are
130131
[restricted][clickhouse-readonly] to read-only, by setting
131132
`CUBEJS_DB_CLICKHOUSE_READONLY` to `true`.
132133

134+
You can connect to a ClickHouse database with compression enabled, by setting
135+
`CUBEJS_DB_CLICKHOUSE_COMPRESSION` to `true`.
136+
133137
[clickhouse]: https://clickhouse.tech/
134138
[clickhouse-docs-users]:
135139
https://clickhouse.tech/docs/en/operations/settings/settings-users/
@@ -144,4 +148,4 @@ You can connect to a ClickHouse database when your user's permissions are
144148
[self-preaggs-batching]: #batching
145149
[ref-preaggs]: /product/caching/using-pre-aggregations
146150
[ref-preaggs-indexes]: /reference/data-model/pre-aggregations#indexes
147-
[ref-preaggs-rollup-join]: /reference/data-model/pre-aggregations#rollup_join
151+
[ref-preaggs-rollup-join]: /reference/data-model/pre-aggregations#rollup_join

‎docs/pages/reference/configuration/environment-variables.mdx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,14 @@ Whether the ClickHouse user has read-only access or not.
217217
| --------------- | ---------------------- | --------------------- |
218218
| `true`, `false` | N/A | N/A |
219219

220+
## `CUBEJS_DB_CLICKHOUSE_COMPRESSION`
221+
222+
Whether the ClickHouse client has compression enabled or not.
223+
224+
| Possible Values | Default in Development | Default in Production |
225+
| --------------- | ---------------------- | --------------------- |
226+
| `true`, `false` | N/A | N/A |
227+
220228
## `CUBEJS_DB_DATABRICKS_ACCEPT_POLICY`
221229

222230
To accept the license terms for the Databricks JDBC driver, this must be set to

‎packages/cubejs-backend-shared/src/env.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1169,6 +1169,19 @@ const variables: Record<string, (...args: any) => any> = {
11691169
]
11701170
),
11711171

1172+
/**
1173+
* ClickHouse compression flag.
1174+
*/
1175+
clickhouseCompression: ({
1176+
dataSource
1177+
}: {
1178+
dataSource: string,
1179+
}) => (
1180+
process.env[
1181+
keyByDataSource('CUBEJS_DB_CLICKHOUSE_COMPRESSION', dataSource)
1182+
]
1183+
),
1184+
11721185
/** ****************************************************************
11731186
* ElasticSearch Driver *
11741187
***************************************************************** */

‎packages/cubejs-backend-shared/test/db_env_multi.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1539,6 +1539,35 @@ describe('Multiple datasources', () => {
15391539
);
15401540
});
15411541

1542+
test('getEnv("clickhouseCompression")', () => {
1543+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'default1';
1544+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'postgres1';
1545+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'wrong1';
1546+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual('default1');
1547+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual('postgres1');
1548+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1549+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1550+
);
1551+
1552+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'default2';
1553+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'postgres2';
1554+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'wrong2';
1555+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual('default2');
1556+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual('postgres2');
1557+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1558+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1559+
);
1560+
1561+
delete process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION;
1562+
delete process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION;
1563+
delete process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION;
1564+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toBeUndefined();
1565+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toBeUndefined();
1566+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1567+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1568+
);
1569+
});
1570+
15421571
test('getEnv("elasticApiId")', () => {
15431572
process.env.CUBEJS_DB_ELASTIC_APIKEY_ID = 'default1';
15441573
process.env.CUBEJS_DS_POSTGRES_DB_ELASTIC_APIKEY_ID = 'postgres1';

‎packages/cubejs-backend-shared/test/db_env_single.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -975,6 +975,23 @@ describe('Single datasources', () => {
975975
expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toBeUndefined();
976976
});
977977

978+
test('getEnv("clickhouseCompression")', () => {
979+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'default1';
980+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual('default1');
981+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual('default1');
982+
expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual('default1');
983+
984+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'default2';
985+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual('default2');
986+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual('default2');
987+
expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual('default2');
988+
989+
delete process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION;
990+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toBeUndefined();
991+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toBeUndefined();
992+
expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toBeUndefined();
993+
});
994+
978995
test('getEnv("elasticApiId")', () => {
979996
process.env.CUBEJS_DB_ELASTIC_APIKEY_ID = 'default1';
980997
expect(getEnv('elasticApiId', { dataSource: 'default' })).toEqual('default1');

‎packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ type ClickHouseDriverConfig = {
111111
database: string,
112112
requestTimeout: number,
113113
exportBucket: ClickhouseDriverExportAWS | null,
114+
compression: { response?: boolean; request?: boolean },
114115
clickhouseSettings: ClickHouseSettings,
115116
};
116117

@@ -165,6 +166,11 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
165166
exportBucket: this.getExportBucket(dataSource),
166167
readOnly: !!config.readOnly,
167168
requestTimeout,
169+
compression: {
170+
// Response compression can't be enabled for a user with readonly=1, as ClickHouse will not allow settings modifications for such user.
171+
response: this.readOnlyMode ? false : getEnv('clickhouseCompression', { dataSource }) === 'true',
172+
request: getEnv('clickhouseCompression', { dataSource }) === 'true',
173+
},
168174
clickhouseSettings: {
169175
// If ClickHouse user's permissions are restricted with "readonly = 1",
170176
// change settings queries are not allowed. Thus, "join_use_nulls" setting
@@ -224,6 +230,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
224230
username: this.config.username,
225231
password: this.config.password,
226232
database: this.config.database,
233+
compression: this.config.compression,
227234
clickhouse_settings: this.config.clickhouseSettings,
228235
request_timeout: this.config.requestTimeout,
229236
max_open_connections: maxPoolSize,

0 commit comments

Comments
 (0)