Skip to content

Commit 78ac7b2

Browse files
GraphmaxerGraphmaxer
authored
feat(clickhouse-driver): Allow to enable compression, thanks @Graphmaxer (#9341)
1 parent 7e363f7 commit 78ac7b2

File tree

6 files changed

+165
-40
lines changed

6 files changed

+165
-40
lines changed

‎docs/pages/product/configuration/data-sources/clickhouse.mdx‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@ CUBEJS_DB_PASS=**********
3131

3232
## Environment Variables
3333

34-
| Environment Variable | Description | Possible Values | Required |
35-
| ------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: |
36-
| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ |
37-
| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ |
38-
| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ |
39-
| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ |
40-
| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ |
41-
| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ |
42-
| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ |
43-
| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ |
34+
| Environment Variable | Description | Possible Values | Required |
35+
| ---------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: |
36+
| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ |
37+
| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ |
38+
| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ |
39+
| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ |
40+
| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ |
41+
| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ |
42+
| `CUBEJS_DB_CLICKHOUSE_COMPRESSION` | Whether the ClickHouse client has compression enabled or not | `true`, `false` | ❌ |
43+
| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ |
44+
| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ |
4445

4546
[ref-data-source-concurrency]: /product/configuration/concurrency#data-source-concurrency
4647

@@ -130,6 +131,9 @@ You can connect to a ClickHouse database when your user's permissions are
130131
[restricted][clickhouse-readonly] to read-only, by setting
131132
`CUBEJS_DB_CLICKHOUSE_READONLY` to `true`.
132133

134+
You can connect to a ClickHouse database with compression enabled, by setting
135+
`CUBEJS_DB_CLICKHOUSE_COMPRESSION` to `true`.
136+
133137
[clickhouse]: https://clickhouse.tech/
134138
[clickhouse-docs-users]:
135139
https://clickhouse.tech/docs/en/operations/settings/settings-users/
@@ -144,4 +148,4 @@ You can connect to a ClickHouse database when your user's permissions are
144148
[self-preaggs-batching]: #batching
145149
[ref-preaggs]: /product/caching/using-pre-aggregations
146150
[ref-preaggs-indexes]: /reference/data-model/pre-aggregations#indexes
147-
[ref-preaggs-rollup-join]: /reference/data-model/pre-aggregations#rollup_join
151+
[ref-preaggs-rollup-join]: /reference/data-model/pre-aggregations#rollup_join

‎docs/pages/reference/configuration/environment-variables.mdx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,14 @@ Whether the ClickHouse user has read-only access or not.
217217
| --------------- | ---------------------- | --------------------- |
218218
| `true`, `false` | N/A | N/A |
219219

220+
## `CUBEJS_DB_CLICKHOUSE_COMPRESSION`
221+
222+
Whether the ClickHouse client has compression enabled or not.
223+
224+
| Possible Values | Default in Development | Default in Production |
225+
| --------------- | ---------------------- | --------------------- |
226+
| `true`, `false` | `false` | `false` |
227+
220228
## `CUBEJS_DB_DATABRICKS_ACCEPT_POLICY`
221229

222230
To accept the license terms for the Databricks JDBC driver, this must be set to

‎packages/cubejs-backend-shared/src/env.ts‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1164,9 +1164,22 @@ const variables: Record<string, (...args: any) => any> = {
11641164
}: {
11651165
dataSource: string,
11661166
}) => (
1167-
process.env[
1168-
keyByDataSource('CUBEJS_DB_CLICKHOUSE_READONLY', dataSource)
1169-
]
1167+
get(keyByDataSource('CUBEJS_DB_CLICKHOUSE_READONLY', dataSource))
1168+
.default('false')
1169+
.asBool()
1170+
),
1171+
1172+
/**
1173+
* ClickHouse compression flag.
1174+
*/
1175+
clickhouseCompression: ({
1176+
dataSource
1177+
}: {
1178+
dataSource: string,
1179+
}) => (
1180+
get(keyByDataSource('CUBEJS_DB_CLICKHOUSE_COMPRESSION', dataSource))
1181+
.default('false')
1182+
.asBool()
11701183
),
11711184

11721185
/** ****************************************************************

‎packages/cubejs-backend-shared/test/db_env_multi.test.ts‎

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1511,34 +1511,89 @@ describe('Multiple datasources', () => {
15111511
});
15121512

15131513
test('getEnv("clickhouseReadOnly")', () => {
1514-
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default1';
1515-
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'postgres1';
1516-
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'wrong1';
1517-
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default1');
1518-
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('postgres1');
1514+
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'true';
1515+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'true';
1516+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'true';
1517+
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(true);
1518+
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(true);
15191519
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow(
15201520
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
15211521
);
15221522

1523-
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default2';
1524-
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'postgres2';
1525-
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'wrong2';
1526-
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default2');
1527-
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('postgres2');
1523+
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'false';
1524+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'false';
1525+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'false';
1526+
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false);
1527+
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false);
1528+
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow(
1529+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1530+
);
1531+
1532+
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'wrong';
1533+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'wrong';
1534+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'wrong';
1535+
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'default' })).toThrow(
1536+
'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
1537+
);
1538+
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toThrow(
1539+
'env-var: "CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
1540+
);
15281541
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow(
15291542
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
15301543
);
15311544

15321545
delete process.env.CUBEJS_DB_CLICKHOUSE_READONLY;
15331546
delete process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY;
15341547
delete process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY;
1535-
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toBeUndefined();
1536-
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toBeUndefined();
1548+
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false);
1549+
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false);
15371550
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow(
15381551
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
15391552
);
15401553
});
15411554

1555+
test('getEnv("clickhouseCompression")', () => {
1556+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'true';
1557+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'true';
1558+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'true';
1559+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(true);
1560+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(true);
1561+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1562+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1563+
);
1564+
1565+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'false';
1566+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'false';
1567+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'false';
1568+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false);
1569+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false);
1570+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1571+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1572+
);
1573+
1574+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'wrong';
1575+
process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'wrong';
1576+
process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'wrong';
1577+
expect(() => getEnv('clickhouseCompression', { dataSource: 'default' })).toThrow(
1578+
'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
1579+
);
1580+
expect(() => getEnv('clickhouseCompression', { dataSource: 'postgres' })).toThrow(
1581+
'env-var: "CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
1582+
);
1583+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1584+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1585+
);
1586+
1587+
delete process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION;
1588+
delete process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION;
1589+
delete process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION;
1590+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false);
1591+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false);
1592+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1593+
'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
1594+
);
1595+
});
1596+
15421597
test('getEnv("elasticApiId")', () => {
15431598
process.env.CUBEJS_DB_ELASTIC_APIKEY_ID = 'default1';
15441599
process.env.CUBEJS_DS_POSTGRES_DB_ELASTIC_APIKEY_ID = 'postgres1';

‎packages/cubejs-backend-shared/test/db_env_single.test.ts‎

Lines changed: 51 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -959,20 +959,59 @@ describe('Single datasources', () => {
959959
});
960960

961961
test('getEnv("clickhouseReadOnly")', () => {
962-
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default1';
963-
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default1');
964-
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('default1');
965-
expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual('default1');
966-
967-
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default2';
968-
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default2');
969-
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('default2');
970-
expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual('default2');
962+
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'true';
963+
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(true);
964+
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(true);
965+
expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual(true);
966+
967+
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'false';
968+
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false);
969+
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false);
970+
expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual(false);
971+
972+
process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'wrong';
973+
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'default' })).toThrow(
974+
'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
975+
);
976+
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toThrow(
977+
'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
978+
);
979+
expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow(
980+
'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
981+
);
971982

972983
delete process.env.CUBEJS_DB_CLICKHOUSE_READONLY;
973-
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toBeUndefined();
974-
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toBeUndefined();
975-
expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toBeUndefined();
984+
expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false);
985+
expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false);
986+
expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual(false);
987+
});
988+
989+
test('getEnv("clickhouseCompression")', () => {
990+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'true';
991+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(true);
992+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(true);
993+
expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual(true);
994+
995+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'false';
996+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false);
997+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false);
998+
expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual(false);
999+
1000+
process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'wrong';
1001+
expect(() => getEnv('clickhouseCompression', { dataSource: 'default' })).toThrow(
1002+
'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
1003+
);
1004+
expect(() => getEnv('clickhouseCompression', { dataSource: 'postgres' })).toThrow(
1005+
'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
1006+
);
1007+
expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow(
1008+
'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0'
1009+
);
1010+
1011+
delete process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION;
1012+
expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false);
1013+
expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false);
1014+
expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual(false);
9761015
});
9771016

9781017
test('getEnv("elasticApiId")', () => {

‎packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ type ClickHouseDriverConfig = {
111111
database: string,
112112
requestTimeout: number,
113113
exportBucket: ClickhouseDriverExportAWS | null,
114+
compression: { response?: boolean; request?: boolean },
114115
clickhouseSettings: ClickHouseSettings,
115116
};
116117

@@ -150,8 +151,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
150151
const database = config.database ?? (getEnv('dbName', { dataSource }) as string) ?? 'default';
151152

152153
// TODO this is a bit inconsistent with readOnly
153-
this.readOnlyMode =
154-
getEnv('clickhouseReadOnly', { dataSource }) === 'true';
154+
this.readOnlyMode = getEnv('clickhouseReadOnly', { dataSource });
155155

156156
// Expect that getEnv('dbQueryTimeout') will always return a value
157157
const requestTimeoutEnv: number = getEnv('dbQueryTimeout', { dataSource }) * 1000;
@@ -165,6 +165,11 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
165165
exportBucket: this.getExportBucket(dataSource),
166166
readOnly: !!config.readOnly,
167167
requestTimeout,
168+
compression: {
169+
// Response compression can't be enabled for a user with readonly=1, as ClickHouse will not allow settings modifications for such user.
170+
response: this.readOnlyMode ? false : getEnv('clickhouseCompression', { dataSource }),
171+
request: getEnv('clickhouseCompression', { dataSource }),
172+
},
168173
clickhouseSettings: {
169174
// If ClickHouse user's permissions are restricted with "readonly = 1",
170175
// change settings queries are not allowed. Thus, "join_use_nulls" setting
@@ -224,6 +229,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
224229
username: this.config.username,
225230
password: this.config.password,
226231
database: this.config.database,
232+
compression: this.config.compression,
227233
clickhouse_settings: this.config.clickhouseSettings,
228234
request_timeout: this.config.requestTimeout,
229235
max_open_connections: maxPoolSize,

0 commit comments

Comments
 (0)