From d2b587bafa4d9a98b33e094cd310a8a1a8ad9885 Mon Sep 17 00:00:00 2001 From: Gabriel Rodriguez Date: Wed, 6 Nov 2024 21:31:17 -0500 Subject: [PATCH 1/3] use default credential provider chain if access key and secret key are not provided --- packages/cubejs-backend-shared/src/env.ts | 10 ++++++++++ .../cubejs-duckdb-driver/src/DuckDBDriver.ts | 14 +++++++++++++- .../test/DuckDBDriver.test.ts | 19 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 3d85a197b7add..6bee25489a2bc 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1613,6 +1613,16 @@ const variables: Record any> = { ] ), + duckdbS3UseCredentialChain: ({ + dataSource + }: { + dataSource: string, + }) => ( + process.env[ + keyByDataSource('CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN', dataSource) + ] + ), + /** * Presto catalog. */ diff --git a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts index 9eeba91b27ac1..4e392337ff222 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts @@ -20,6 +20,7 @@ export type DuckDBDriverConfiguration = { dataSource?: string, initSql?: string, schema?: string, + duckdbS3UseCredentialChain?: boolean, }; type InitPromise = { @@ -145,7 +146,7 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { value: getEnv('duckdbS3SessionToken', this.config), } ]; - + for (const { key, value } of configuration) { if (value) { try { @@ -159,6 +160,17 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { } } } + const useCredentialChain = getEnv('duckdbS3UseCredentialChain', this.config); + if (useCredentialChain === 'true' || useCredentialChain === true) { + try { + await execAsync('CREATE SECRET (TYPE S3, PROVIDER \'CREDENTIAL_CHAIN\')'); + } catch (e) { + if (this.logger) { + console.error('DuckDB - error on creating S3 credential chain secret', { e }); + } + throw e; + } + } if (this.config.initSql) { try { diff --git a/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts b/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts index b31bfc17af272..17f8244b7b049 100644 --- a/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts +++ b/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts @@ -73,4 +73,23 @@ describe('DuckDBDriver', () => { { id: '3', created: '2020-03-03T03:03:03.333Z', created_date: '2020-03-03T00:00:00.000Z', price: '300' } ]); }); + + test('should execute CREATE SECRET when duckdbS3UseCredentialChain is set', async () => { + process.env.duckdbS3UseCredentialChain = 'true'; + + // Create a new driver instance to pick up the environment variable + const driverWithCredentialChain = new DuckDBDriver({}); + + // Mock the execAsync method to spy on it + const execAsyncSpy = jest.spyOn((driverWithCredentialChain as any), 'execAsync'); + + await driverWithCredentialChain.testConnection(); + + expect(execAsyncSpy).toHaveBeenCalledWith(`CREATE SECRET (TYPE S3, PROVIDER 'CREDENTIAL_CHAIN')`); + + // Clean up + delete process.env.duckdbS3UseCredentialChain; + execAsyncSpy.mockRestore(); + await driverWithCredentialChain.release(); + }); }); From e4a8ecf83c7c6d293bfa40cd7dcb89d5f2a1c7d2 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 13 Jun 2025 19:40:12 +0300 Subject: [PATCH 2/3] some polishment --- packages/cubejs-backend-shared/src/env.ts | 27 ++++++++++++++++--- .../cubejs-duckdb-driver/src/DuckDBDriver.ts | 5 ++-- .../test/DuckDBDriver.test.ts | 19 ------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index d154cba4bd308..706875862ddb4 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1803,11 +1803,30 @@ const variables: Record any> = { dataSource }: { dataSource: string, - }) => ( - process.env[ + }) => { + const val = process.env[ keyByDataSource('CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN', dataSource) - ] - ), + ]; + + if (val) { + if (val.toLocaleLowerCase() === 'true') { + return true; + } else if (val.toLowerCase() === 'false') { + return false; + } else { + throw new TypeError( + `The ${ + keyByDataSource( + 'CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN', + dataSource, + ) + } must be either 'true' or 'false'.` + ); + } + } else { + return false; + } + }, /** *************************************************************** * Presto/Trino Driver * diff --git a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts index ecf9bf7cc00c4..bddb2f1d55e9e 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts @@ -164,8 +164,9 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { } } } - const useCredentialChain = getEnv('duckdbS3UseCredentialChain', this.config); - if (useCredentialChain === 'true' || useCredentialChain === true) { + + const useCredentialChain = this.config.duckdbS3UseCredentialChain || getEnv('duckdbS3UseCredentialChain', this.config); + if (useCredentialChain) { try { await execAsync('CREATE SECRET (TYPE S3, PROVIDER \'CREDENTIAL_CHAIN\')'); } catch (e) { diff --git a/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts b/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts index 17f8244b7b049..b31bfc17af272 100644 --- a/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts +++ b/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts @@ -73,23 +73,4 @@ describe('DuckDBDriver', () => { { id: '3', created: '2020-03-03T03:03:03.333Z', created_date: '2020-03-03T00:00:00.000Z', price: '300' } ]); }); - - test('should execute CREATE SECRET when duckdbS3UseCredentialChain is set', async () => { - process.env.duckdbS3UseCredentialChain = 'true'; - - // Create a new driver instance to pick up the environment variable - const driverWithCredentialChain = new DuckDBDriver({}); - - // Mock the execAsync method to spy on it - const execAsyncSpy = jest.spyOn((driverWithCredentialChain as any), 'execAsync'); - - await driverWithCredentialChain.testConnection(); - - expect(execAsyncSpy).toHaveBeenCalledWith(`CREATE SECRET (TYPE S3, PROVIDER 'CREDENTIAL_CHAIN')`); - - // Clean up - delete process.env.duckdbS3UseCredentialChain; - execAsyncSpy.mockRestore(); - await driverWithCredentialChain.release(); - }); }); From ee9ba7b0523d593fdc48c37293c43acbbe3a8315 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 13 Jun 2025 19:48:58 +0300 Subject: [PATCH 3/3] update docs --- .../configuration/data-sources/duckdb.mdx | 33 ++++++++++--------- .../reference/environment-variables.mdx | 8 +++++ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/duckdb.mdx b/docs/pages/product/configuration/data-sources/duckdb.mdx index af4e71f6c92df..89d65ffed0139 100644 --- a/docs/pages/product/configuration/data-sources/duckdb.mdx +++ b/docs/pages/product/configuration/data-sources/duckdb.mdx @@ -59,22 +59,23 @@ deployment][ref-demo-deployment] in Cube Cloud. ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | :------: | -| `CUBEJS_DB_DUCKDB_MEMORY_LIMIT` | The maximum memory limit for DuckDB. Equivalent to `SET memory_limit=`. Default is 75% of available RAM | A valid memory limit | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_SCHEMA` | The [default search schema][link-duckdb-configuration-ref] | A valid schema name | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_MOTHERDUCK_TOKEN` | The service token to use for connections to MotherDuck | A valid [MotherDuck service token][motherduck-docs-svc-token] | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_DATABASE_PATH` | The database filepath to use for connection to a local database. | A valid duckdb database file path | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_ACCESS_KEY_ID` | The Access Key ID to use for database connections | A valid Access Key ID | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_SECRET_ACCESS_KEY` | The Secret Access Key to use for database connections | A valid Secret Access Key | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_ENDPOINT` | The S3 endpoint | A valid [S3 endpoint][duckdb-docs-s3-import] | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_REGION` | The [region of the bucket][duckdb-docs-s3-import] | A valid AWS region | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_USE_SSL` | Use SSL for connection | A boolean | ❌ | ❌ | -| `CUBEJS_DB_DUCKDB_S3_URL_STYLE` | To choose the S3 URL style(vhost or path) | 'vhost' or 'path' | ❌ | ❌ | -| `CUBEJS_DB_DUCKDB_S3_SESSION_TOKEN` | The token for the S3 session | A valid Session Token | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_EXTENSIONS` | A comma-separated list of DuckDB extensions to install and load | A comma-separated list of DuckDB extensions | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_COMMUNITY_EXTENSIONS` | A comma-separated list of DuckDB community extensions to install and load | A comma-separated list of DuckDB community extensions | ❌ | ✅ | -| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | +|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------|:--------:| +| `CUBEJS_DB_DUCKDB_MEMORY_LIMIT` | The maximum memory limit for DuckDB. Equivalent to `SET memory_limit=`. Default is 75% of available RAM | A valid memory limit | ❌ | +| `CUBEJS_DB_DUCKDB_SCHEMA` | The [default search schema][link-duckdb-configuration-ref] | A valid schema name | ❌ | +| `CUBEJS_DB_DUCKDB_MOTHERDUCK_TOKEN` | The service token to use for connections to MotherDuck | A valid [MotherDuck service token][motherduck-docs-svc-token] | ❌ | +| `CUBEJS_DB_DUCKDB_DATABASE_PATH` | The database filepath to use for connection to a local database. | A valid duckdb database file path | ❌ | +| `CUBEJS_DB_DUCKDB_S3_ACCESS_KEY_ID` | The Access Key ID to use for database connections | A valid Access Key ID | ❌ | +| `CUBEJS_DB_DUCKDB_S3_SECRET_ACCESS_KEY` | The Secret Access Key to use for database connections | A valid Secret Access Key | ❌ | +| `CUBEJS_DB_DUCKDB_S3_ENDPOINT` | The S3 endpoint | A valid [S3 endpoint][duckdb-docs-s3-import] | ❌ | +| `CUBEJS_DB_DUCKDB_S3_REGION` | The [region of the bucket][duckdb-docs-s3-import] | A valid AWS region | ❌ | +| `CUBEJS_DB_DUCKDB_S3_USE_SSL` | Use SSL for connection | A boolean | ❌ | +| `CUBEJS_DB_DUCKDB_S3_URL_STYLE` | To choose the S3 URL style(vhost or path) | `vhost` or `path` | ❌ | +| `CUBEJS_DB_DUCKDB_S3_SESSION_TOKEN` | The token for the S3 session | A valid Session Token | ❌ | +| `CUBEJS_DB_DUCKDB_EXTENSIONS` | A comma-separated list of DuckDB extensions to install and load | A comma-separated list of DuckDB extensions | ❌ | +| `CUBEJS_DB_DUCKDB_COMMUNITY_EXTENSIONS` | A comma-separated list of DuckDB community extensions to install and load | A comma-separated list of DuckDB community extensions | ❌ | +| `CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN` | A flag to use credentials chain for secrets for S3 connections | `true`, `false`. Defaults to `false` | ❌ | +| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ | [ref-data-source-concurrency]: /product/configuration/concurrency#data-source-concurrency diff --git a/docs/pages/product/configuration/reference/environment-variables.mdx b/docs/pages/product/configuration/reference/environment-variables.mdx index 24e79dbaa158d..0a1320c211afe 100644 --- a/docs/pages/product/configuration/reference/environment-variables.mdx +++ b/docs/pages/product/configuration/reference/environment-variables.mdx @@ -375,6 +375,14 @@ A comma-separated list of DuckDB community extensions to install and load. | ----------------------------------------------------- | ---------------------- | --------------------- | | A comma-separated list of DuckDB community extensions | N/A | N/A | +## `CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN` + +A flag to use credentials chain for secrets for S3 connections. + +| Possible Values | Default in Development | Default in Production | +| ----------------------------------------------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + ## `CUBEJS_DB_ELASTIC_APIKEY_ID` The [ID of the API key from elastic.co][elastic-docs-api-keys]. Required when