diff --git a/docs/pages/product/configuration/data-sources/duckdb.mdx b/docs/pages/product/configuration/data-sources/duckdb.mdx index 7f9daa2cb3bdb..d5050c5efe5b8 100644 --- a/docs/pages/product/configuration/data-sources/duckdb.mdx +++ b/docs/pages/product/configuration/data-sources/duckdb.mdx @@ -73,6 +73,7 @@ deployment][ref-demo-deployment] in Cube Cloud. | `CUBEJS_DB_DUCKDB_S3_URL_STYLE` | To choose the S3 URL style(vhost or path) | 'vhost' or 'path' | ❌ | ❌ | | `CUBEJS_DB_DUCKDB_S3_SESSION_TOKEN` | The token for the S3 session | A valid Session Token | ❌ | ✅ | | `CUBEJS_DB_DUCKDB_EXTENSIONS` | A comma-separated list of DuckDB extensions to install and load | A comma-separated list of DuckDB extensions | ❌ | ✅ | +| `CUBEJS_DB_DUCKDB_COMMUNITY_EXTENSIONS` | A comma-separated list of DuckDB community extensions to install and load | A comma-separated list of DuckDB community extensions | ❌ | ✅ | | `CUBEJS_CONCURRENCY` | The number of [concurrent queries][ref-data-source-concurrency] to the data source | A valid number | ❌ | [ref-data-source-concurrency]: /product/configuration/concurrency#data-source-concurrency @@ -133,4 +134,4 @@ connections are made over HTTPS. /product/caching/using-pre-aggregations#pre-aggregation-build-strategies [ref-schema-ref-types-formats-countdistinctapprox]: /reference/data-model/types-and-formats#count_distinct_approx [self-preaggs-batching]: #batching -[ref-demo-deployment]: /product/deployment/cloud/deployments#demo-deployments \ No newline at end of file +[ref-demo-deployment]: /product/deployment/cloud/deployments#demo-deployments diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index 948b2de9a4c7b..f542d6c03cfb0 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -372,6 +372,14 @@ A comma-separated list of DuckDB extensions to install and load. | ------------------------------------------- | ---------------------- | --------------------- | | A comma-separated list of DuckDB extensions | N/A | N/A | +## `CUBEJS_DB_DUCKDB_COMMUNITY_EXTENSIONS` + +A comma-separated list of DuckDB community extensions to install and load. + +| Possible Values | Default in Development | Default in Production | +| ----------------------------------------------------- | ---------------------- | --------------------- | +| A comma-separated list of DuckDB community extensions | N/A | N/A | + ## `CUBEJS_DB_ELASTIC_APIKEY_ID` The [ID of the API key from elastic.co][elastic-docs-api-keys]. Required when diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 9e48c7fc1fbb5..c7b0c2e3bd584 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1773,6 +1773,19 @@ const variables: Record any> = { } return []; }, + duckdbCommunityExtensions: ({ + dataSource + }: { + dataSource: string, + }) => { + const extensions = process.env[ + keyByDataSource('CUBEJS_DB_DUCKDB_COMMUNITY_EXTENSIONS', dataSource) + ]; + if (extensions) { + return extensions.split(',').map(e => e.trim()); + } + return []; + }, /** *************************************************************** * Presto Driver * **************************************************************** */ diff --git a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts index e933d777f50b2..d5eb73d1ecc96 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts @@ -57,10 +57,38 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { return super.toGenericType(columnType.toLowerCase()); } + private async installExtensions(extensions: string[], execAsync: (sql: string, ...params: any[]) => Promise, repository: string = ''): Promise { + repository = repository ? ` FROM ${repository}` : ''; + for (const extension of extensions) { + try { + await execAsync(`INSTALL ${extension}${repository}`); + } catch (e) { + if (this.logger) { + console.error(`DuckDB - error on installing ${extension}`, { e }); + } + // DuckDB will lose connection_ref on connection on error, this will lead to broken connection object + throw e; + } + } + } + + private async loadExtensions(extensions: string[], execAsync: (sql: string, ...params: any[]) => Promise): Promise { + for (const extension of extensions) { + try { + await execAsync(`LOAD ${extension}`); + } catch (e) { + if (this.logger) { + console.error(`DuckDB - error on loading ${extension}`, { e }); + } + // DuckDB will lose connection_ref on connection on error, this will lead to broken connection object + throw e; + } + } + } + protected async init(): Promise { const token = this.config.motherDuckToken || getEnv('duckdbMotherDuckToken', this.config); const dbPath = this.config.databasePath || getEnv('duckdbDatabasePath', this.config); - // Determine the database URL based on the provided db_path or token let dbUrl: string; if (dbPath) { @@ -121,7 +149,7 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { value: getEnv('duckdbS3SessionToken', this.config), } ]; - + for (const { key, value } of configuration) { if (value) { try { @@ -137,34 +165,13 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { } // Install & load extensions if configured in env variable. - const extensions = getEnv('duckdbExtensions', this.config); - for (const extension of extensions) { - try { - await execAsync(`INSTALL ${extension}`); - } catch (e) { - if (this.logger) { - console.error(`DuckDB - error on installing ${extension}`, { - e - }); - } - - // DuckDB will lose connection_ref on connection on error, this will lead to broken connection object - throw e; - } - - try { - await execAsync(`LOAD ${extension}`); - } catch (e) { - if (this.logger) { - console.error(`DuckDB - error on loading ${extension}`, { - e - }); - } - - // DuckDB will lose connection_ref on connection on error, this will lead to broken connection object - throw e; - } - } + const officialExtensions = getEnv('duckdbExtensions', this.config); + await this.installExtensions(officialExtensions, execAsync); + await this.loadExtensions(officialExtensions, execAsync); + const communityExtensions = getEnv('duckdbCommunityExtensions', this.config); + // @see https://duckdb.org/community_extensions/ + await this.installExtensions(communityExtensions, execAsync, 'community'); + await this.loadExtensions(communityExtensions, execAsync); if (this.config.initSql) { try { @@ -177,7 +184,7 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { } } } - + return { defaultConnection, db