diff --git a/docs/pages/product/configuration/data-sources/clickhouse.mdx b/docs/pages/product/configuration/data-sources/clickhouse.mdx index 5c839ce73f2a1..68d598ad2c789 100644 --- a/docs/pages/product/configuration/data-sources/clickhouse.mdx +++ b/docs/pages/product/configuration/data-sources/clickhouse.mdx @@ -31,16 +31,17 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| ------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | -| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | +| ---------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | +| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ | +| `CUBEJS_DB_CLICKHOUSE_COMPRESSION` | Whether the ClickHouse client has compression enabled or not | `true`, `false` | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | +| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ | [ref-data-source-concurrency]: /product/configuration/concurrency#data-source-concurrency @@ -130,6 +131,9 @@ You can connect to a ClickHouse database when your user's permissions are [restricted][clickhouse-readonly] to read-only, by setting `CUBEJS_DB_CLICKHOUSE_READONLY` to `true`. +You can connect to a ClickHouse database with compression enabled, by setting +`CUBEJS_DB_CLICKHOUSE_COMPRESSION` to `true`. + [clickhouse]: https://clickhouse.tech/ [clickhouse-docs-users]: https://clickhouse.tech/docs/en/operations/settings/settings-users/ @@ -144,4 +148,4 @@ You can connect to a ClickHouse database when your user's permissions are [self-preaggs-batching]: #batching [ref-preaggs]: /product/caching/using-pre-aggregations [ref-preaggs-indexes]: /reference/data-model/pre-aggregations#indexes -[ref-preaggs-rollup-join]: /reference/data-model/pre-aggregations#rollup_join \ No newline at end of file +[ref-preaggs-rollup-join]: /reference/data-model/pre-aggregations#rollup_join diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index 2a957a893acc4..e9fddf073d06a 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -217,6 +217,14 @@ Whether the ClickHouse user has read-only access or not. | --------------- | ---------------------- | --------------------- | | `true`, `false` | N/A | N/A | +## `CUBEJS_DB_CLICKHOUSE_COMPRESSION` + +Whether the ClickHouse client has compression enabled or not. + +| Possible Values | Default in Development | Default in Production | +| --------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + ## `CUBEJS_DB_DATABRICKS_ACCEPT_POLICY` To accept the license terms for the Databricks JDBC driver, this must be set to diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 6b6337856d03a..eaad63fc5d03f 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1164,9 +1164,22 @@ const variables: Record any> = { }: { dataSource: string, }) => ( - process.env[ - keyByDataSource('CUBEJS_DB_CLICKHOUSE_READONLY', dataSource) - ] + get(keyByDataSource('CUBEJS_DB_CLICKHOUSE_READONLY', dataSource)) + .default('false') + .asBool() + ), + + /** + * ClickHouse compression flag. + */ + clickhouseCompression: ({ + dataSource + }: { + dataSource: string, + }) => ( + get(keyByDataSource('CUBEJS_DB_CLICKHOUSE_COMPRESSION', dataSource)) + .default('false') + .asBool() ), /** **************************************************************** diff --git a/packages/cubejs-backend-shared/test/db_env_multi.test.ts b/packages/cubejs-backend-shared/test/db_env_multi.test.ts index b37ce923c20c4..f42e177fbbb83 100644 --- a/packages/cubejs-backend-shared/test/db_env_multi.test.ts +++ b/packages/cubejs-backend-shared/test/db_env_multi.test.ts @@ -1511,20 +1511,33 @@ describe('Multiple datasources', () => { }); test('getEnv("clickhouseReadOnly")', () => { - process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default1'; - process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'postgres1'; - process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'wrong1'; - expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default1'); - expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('postgres1'); + process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'true'; + process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'true'; + process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'true'; + expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(true); + expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(true); expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow( 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' ); - process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default2'; - process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'postgres2'; - process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'wrong2'; - expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default2'); - expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('postgres2'); + process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'false'; + process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'false'; + process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'false'; + expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false); + expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow( + 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' + ); + + process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'wrong'; + process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY = 'wrong'; + process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY = 'wrong'; + expect(() => getEnv('clickhouseReadOnly', { dataSource: 'default' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + expect(() => getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toThrow( + 'env-var: "CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow( 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' ); @@ -1532,13 +1545,55 @@ describe('Multiple datasources', () => { delete process.env.CUBEJS_DB_CLICKHOUSE_READONLY; delete process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_READONLY; delete process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_READONLY; - expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toBeUndefined(); - expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toBeUndefined(); + expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false); expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow( 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' ); }); + test('getEnv("clickhouseCompression")', () => { + process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'true'; + process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'true'; + process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'true'; + expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(true); + expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(true); + expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow( + 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' + ); + + process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'false'; + process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'false'; + process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'false'; + expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false); + expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow( + 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' + ); + + process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'wrong'; + process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION = 'wrong'; + process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION = 'wrong'; + expect(() => getEnv('clickhouseCompression', { dataSource: 'default' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + expect(() => getEnv('clickhouseCompression', { dataSource: 'postgres' })).toThrow( + 'env-var: "CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow( + 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' + ); + + delete process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION; + delete process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_COMPRESSION; + delete process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_COMPRESSION; + expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false); + expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow( + 'The wrong data source is missing in the declared CUBEJS_DATASOURCES.' + ); + }); + test('getEnv("elasticApiId")', () => { process.env.CUBEJS_DB_ELASTIC_APIKEY_ID = 'default1'; process.env.CUBEJS_DS_POSTGRES_DB_ELASTIC_APIKEY_ID = 'postgres1'; diff --git a/packages/cubejs-backend-shared/test/db_env_single.test.ts b/packages/cubejs-backend-shared/test/db_env_single.test.ts index 765e94b7c175d..f5de389afe9a2 100644 --- a/packages/cubejs-backend-shared/test/db_env_single.test.ts +++ b/packages/cubejs-backend-shared/test/db_env_single.test.ts @@ -959,20 +959,59 @@ describe('Single datasources', () => { }); test('getEnv("clickhouseReadOnly")', () => { - process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default1'; - expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default1'); - expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('default1'); - expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual('default1'); - - process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'default2'; - expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual('default2'); - expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual('default2'); - expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual('default2'); + process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'true'; + expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(true); + expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(true); + expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual(true); + + process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'false'; + expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false); + expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual(false); + + process.env.CUBEJS_DB_CLICKHOUSE_READONLY = 'wrong'; + expect(() => getEnv('clickhouseReadOnly', { dataSource: 'default' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + expect(() => getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + expect(() => getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_READONLY" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); delete process.env.CUBEJS_DB_CLICKHOUSE_READONLY; - expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toBeUndefined(); - expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toBeUndefined(); - expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toBeUndefined(); + expect(getEnv('clickhouseReadOnly', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseReadOnly', { dataSource: 'postgres' })).toEqual(false); + expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toEqual(false); + }); + + test('getEnv("clickhouseCompression")', () => { + process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'true'; + expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(true); + expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(true); + expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual(true); + + process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'false'; + expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false); + expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual(false); + + process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION = 'wrong'; + expect(() => getEnv('clickhouseCompression', { dataSource: 'default' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + expect(() => getEnv('clickhouseCompression', { dataSource: 'postgres' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + expect(() => getEnv('clickhouseCompression', { dataSource: 'wrong' })).toThrow( + 'env-var: "CUBEJS_DB_CLICKHOUSE_COMPRESSION" should be either "true", "false", "TRUE", "FALSE", 1, or 0' + ); + + delete process.env.CUBEJS_DB_CLICKHOUSE_COMPRESSION; + expect(getEnv('clickhouseCompression', { dataSource: 'default' })).toEqual(false); + expect(getEnv('clickhouseCompression', { dataSource: 'postgres' })).toEqual(false); + expect(getEnv('clickhouseCompression', { dataSource: 'wrong' })).toEqual(false); }); test('getEnv("elasticApiId")', () => { diff --git a/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts b/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts index 48c029e45ba64..e35cd35c4fe0b 100644 --- a/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts +++ b/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts @@ -111,6 +111,7 @@ type ClickHouseDriverConfig = { database: string, requestTimeout: number, exportBucket: ClickhouseDriverExportAWS | null, + compression: { response?: boolean; request?: boolean }, clickhouseSettings: ClickHouseSettings, }; @@ -150,8 +151,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { const database = config.database ?? (getEnv('dbName', { dataSource }) as string) ?? 'default'; // TODO this is a bit inconsistent with readOnly - this.readOnlyMode = - getEnv('clickhouseReadOnly', { dataSource }) === 'true'; + this.readOnlyMode = getEnv('clickhouseReadOnly', { dataSource }); // Expect that getEnv('dbQueryTimeout') will always return a value const requestTimeoutEnv: number = getEnv('dbQueryTimeout', { dataSource }) * 1000; @@ -165,6 +165,11 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { exportBucket: this.getExportBucket(dataSource), readOnly: !!config.readOnly, requestTimeout, + compression: { + // Response compression can't be enabled for a user with readonly=1, as ClickHouse will not allow settings modifications for such user. + response: this.readOnlyMode ? false : getEnv('clickhouseCompression', { dataSource }), + request: getEnv('clickhouseCompression', { dataSource }), + }, clickhouseSettings: { // If ClickHouse user's permissions are restricted with "readonly = 1", // change settings queries are not allowed. Thus, "join_use_nulls" setting @@ -224,6 +229,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { username: this.config.username, password: this.config.password, database: this.config.database, + compression: this.config.compression, clickhouse_settings: this.config.clickhouseSettings, request_timeout: this.config.requestTimeout, max_open_connections: maxPoolSize,