diff --git a/docs/pages/product/configuration/data-sources/duckdb.mdx b/docs/pages/product/configuration/data-sources/duckdb.mdx index af4e71f6c92df..89d65ffed0139 100644 --- a/docs/pages/product/configuration/data-sources/duckdb.mdx +++ b/docs/pages/product/configuration/data-sources/duckdb.mdx @@ -59,22 +59,23 @@ deployment][ref-demo-deployment] in Cube Cloud. ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | :------: | -| `CUBEJS_DB_DUCKDB_MEMORY_LIMIT` | The maximum memory limit for DuckDB. Equivalent to `SET memory_limit=`. Default is 75% of available RAM | A valid memory limit | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_SCHEMA` | The [default search schema][link-duckdb-configuration-ref] | A valid schema name | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_MOTHERDUCK_TOKEN` | The service token to use for connections to MotherDuck | A valid [MotherDuck service token][motherduck-docs-svc-token] | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_DATABASE_PATH` | The database filepath to use for connection to a local database. | A valid duckdb database file path | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_ACCESS_KEY_ID` | The Access Key ID to use for database connections | A valid Access Key ID | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_SECRET_ACCESS_KEY` | The Secret Access Key to use for database connections | A valid Secret Access Key | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_ENDPOINT` | The S3 endpoint | A valid [S3 endpoint][duckdb-docs-s3-import] | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_REGION` | The [region of the bucket][duckdb-docs-s3-import] | A valid AWS region | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_S3_USE_SSL` | Use SSL for connection | A boolean | ❌ | ❌ | -| `CUBEJS_DB_DUCKDB_S3_URL_STYLE` | To choose the S3 URL style(vhost or path) | 'vhost' or 'path' | ❌ | ❌ | -| `CUBEJS_DB_DUCKDB_S3_SESSION_TOKEN` | The token for the S3 session | A valid Session Token | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_EXTENSIONS` | A comma-separated list of DuckDB extensions to install and load | A comma-separated list of DuckDB extensions | ❌ | ✅ | -| `CUBEJS_DB_DUCKDB_COMMUNITY_EXTENSIONS` | A comma-separated list of DuckDB community extensions to install and load | A comma-separated list of DuckDB community extensions | ❌ | ✅ | -| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | +|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------|:--------:| +| `CUBEJS_DB_DUCKDB_MEMORY_LIMIT` | The maximum memory limit for DuckDB. Equivalent to `SET memory_limit=`. Default is 75% of available RAM | A valid memory limit | ❌ | +| `CUBEJS_DB_DUCKDB_SCHEMA` | The [default search schema][link-duckdb-configuration-ref] | A valid schema name | ❌ | +| `CUBEJS_DB_DUCKDB_MOTHERDUCK_TOKEN` | The service token to use for connections to MotherDuck | A valid [MotherDuck service token][motherduck-docs-svc-token] | ❌ | +| `CUBEJS_DB_DUCKDB_DATABASE_PATH` | The database filepath to use for connection to a local database. | A valid duckdb database file path | ❌ | +| `CUBEJS_DB_DUCKDB_S3_ACCESS_KEY_ID` | The Access Key ID to use for database connections | A valid Access Key ID | ❌ | +| `CUBEJS_DB_DUCKDB_S3_SECRET_ACCESS_KEY` | The Secret Access Key to use for database connections | A valid Secret Access Key | ❌ | +| `CUBEJS_DB_DUCKDB_S3_ENDPOINT` | The S3 endpoint | A valid [S3 endpoint][duckdb-docs-s3-import] | ❌ | +| `CUBEJS_DB_DUCKDB_S3_REGION` | The [region of the bucket][duckdb-docs-s3-import] | A valid AWS region | ❌ | +| `CUBEJS_DB_DUCKDB_S3_USE_SSL` | Use SSL for connection | A boolean | ❌ | +| `CUBEJS_DB_DUCKDB_S3_URL_STYLE` | To choose the S3 URL style(vhost or path) | `vhost` or `path` | ❌ | +| `CUBEJS_DB_DUCKDB_S3_SESSION_TOKEN` | The token for the S3 session | A valid Session Token | ❌ | +| `CUBEJS_DB_DUCKDB_EXTENSIONS` | A comma-separated list of DuckDB extensions to install and load | A comma-separated list of DuckDB extensions | ❌ | +| `CUBEJS_DB_DUCKDB_COMMUNITY_EXTENSIONS` | A comma-separated list of DuckDB community extensions to install and load | A comma-separated list of DuckDB community extensions | ❌ | +| `CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN` | A flag to use credentials chain for secrets for S3 connections | `true`, `false`. Defaults to `false` | ❌ | +| `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ | [ref-data-source-concurrency]: /product/configuration/concurrency#data-source-concurrency diff --git a/docs/pages/product/configuration/reference/environment-variables.mdx b/docs/pages/product/configuration/reference/environment-variables.mdx index 24e79dbaa158d..0a1320c211afe 100644 --- a/docs/pages/product/configuration/reference/environment-variables.mdx +++ b/docs/pages/product/configuration/reference/environment-variables.mdx @@ -375,6 +375,14 @@ A comma-separated list of DuckDB community extensions to install and load. | ----------------------------------------------------- | ---------------------- | --------------------- | | A comma-separated list of DuckDB community extensions | N/A | N/A | +## `CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN` + +A flag to use credentials chain for secrets for S3 connections. + +| Possible Values | Default in Development | Default in Production | +| ----------------------------------------------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + ## `CUBEJS_DB_ELASTIC_APIKEY_ID` The [ID of the API key from elastic.co][elastic-docs-api-keys]. Required when diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 94bbdea86ad3f..706875862ddb4 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1784,6 +1784,7 @@ const variables: Record any> = { } return []; }, + duckdbCommunityExtensions: ({ dataSource }: { @@ -1797,6 +1798,36 @@ const variables: Record any> = { } return []; }, + + duckdbS3UseCredentialChain: ({ + dataSource + }: { + dataSource: string, + }) => { + const val = process.env[ + keyByDataSource('CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN', dataSource) + ]; + + if (val) { + if (val.toLocaleLowerCase() === 'true') { + return true; + } else if (val.toLowerCase() === 'false') { + return false; + } else { + throw new TypeError( + `The ${ + keyByDataSource( + 'CUBEJS_DB_DUCKDB_S3_USE_CREDENTIAL_CHAIN', + dataSource, + ) + } must be either 'true' or 'false'.` + ); + } + } else { + return false; + } + }, + /** *************************************************************** * Presto/Trino Driver * **************************************************************** */ diff --git a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts index d5eb73d1ecc96..bddb2f1d55e9e 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts @@ -22,6 +22,7 @@ export type DuckDBDriverConfiguration = { initSql?: string, motherDuckToken?: string, schema?: string, + duckdbS3UseCredentialChain?: boolean, }; type InitPromise = { @@ -164,6 +165,18 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { } } + const useCredentialChain = this.config.duckdbS3UseCredentialChain || getEnv('duckdbS3UseCredentialChain', this.config); + if (useCredentialChain) { + try { + await execAsync('CREATE SECRET (TYPE S3, PROVIDER \'CREDENTIAL_CHAIN\')'); + } catch (e) { + if (this.logger) { + console.error('DuckDB - error on creating S3 credential chain secret', { e }); + } + throw e; + } + } + // Install & load extensions if configured in env variable. const officialExtensions = getEnv('duckdbExtensions', this.config); await this.installExtensions(officialExtensions, execAsync);