diff --git a/README.md b/README.md index df64228..f948fa1 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ docker run -p 5522:5522 \ ghcr.io/ibero-data/duck-ui:latest ``` -| Variable | Description | Default | +| Runtime Variable | Description | Default | |----------|-------------|---------| | `DUCK_UI_EXTERNAL_CONNECTION_NAME` | Name for the external connection | "" | | `DUCK_UI_EXTERNAL_HOST` | Host URL for external DuckDB | "" | @@ -50,6 +50,14 @@ docker run -p 5522:5522 \ | `DUCK_UI_EXTERNAL_PASS` | Password for external connection | "" | | `DUCK_UI_EXTERNAL_DATABASE_NAME` | Database name for external connection | "" | | `DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS` | Allow unsigned extensions in DuckDB | false | +| `DUCK_UI_DUCKDB_WASM_USE_CDN` | Load DuckDB WASM from CDN (ignored when build-time `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true`) | false | +| `DUCK_UI_DUCKDB_WASM_BASE_URL` | Custom CDN base URL (used when `DUCK_UI_DUCKDB_WASM_USE_CDN=true`) | auto jsDelivr | + +| Build-time Variable | Description | Default | +|----------|-------------|---------| +| `DUCK_UI_DUCKDB_WASM_CDN_ONLY` | Build a CDN-only artifact (local DuckDB WASM assets are not bundled). | false | + +When `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true`, runtime `DUCK_UI_DUCKDB_WASM_USE_CDN=false` cannot switch back to local WASM. @@ -197,4 +205,4 @@ This project is sponsored by:
-Want to be a sponsor? [Contact us](mailto:caio.ricciuti+sponsorship@outlook.com). \ No newline at end of file +Want to be a sponsor? [Contact us](mailto:caio.ricciuti+sponsorship@outlook.com). diff --git a/docs/environment-variables.md b/docs/environment-variables.md index c63dc00..803a789 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -17,11 +17,21 @@ These variables allow you to configure Duck-UI to connect to an external DuckDB | `DUCK_UI_EXTERNAL_PASS` | Password for authentication | No | `""` | `"your-password"` | | `DUCK_UI_EXTERNAL_DATABASE_NAME` | Database name to connect to | No | `""` | `"analytics"` | -### Extension Settings +### Runtime Settings | Variable | Description | Required | Default | Example | |----------|-------------|----------|---------|---------| | `DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS` | Allow loading unsigned DuckDB extensions | No | `false` | `"true"` | +| `DUCK_UI_DUCKDB_WASM_USE_CDN` | Enable loading DuckDB WASM and worker files from CDN (ignored when `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true` at build time) | No | `false` | `"true"` | +| `DUCK_UI_DUCKDB_WASM_BASE_URL` | Custom CDN base URL (when `DUCK_UI_DUCKDB_WASM_USE_CDN=true`) | No | Auto (`duckdb.getJsDelivrBundles()`) | `"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.33.1-dev19.0/dist"` | + +### Build-time Settings + +| Variable | Description | Required | Default | Example | +|----------|-------------|----------|---------|---------| +| `DUCK_UI_DUCKDB_WASM_CDN_ONLY` | Build a CDN-only artifact (local DuckDB WASM files are not bundled). Suitable for edge platforms with strict asset size limits. | No | `false` | `"true"` | + +When `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true`, runtime `DUCK_UI_DUCKDB_WASM_USE_CDN=false` cannot switch back to local WASM because local assets are not included in the build. ::: warning Security Note Enabling unsigned extensions may pose security risks. Only enable this in trusted environments. diff --git a/inject-env.js b/inject-env.js index b5372d6..4d71d10 100644 --- a/inject-env.js +++ b/inject-env.js @@ -16,7 +16,11 @@ const envVars = { process.env.DUCK_UI_EXTERNAL_DATABASE_NAME || "", // Add new configuration for DuckDB settings DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS: - process.env.DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS === "true" || false + process.env.DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS === "true" || false, + DUCK_UI_DUCKDB_WASM_USE_CDN: + process.env.DUCK_UI_DUCKDB_WASM_USE_CDN === "true" || false, + DUCK_UI_DUCKDB_WASM_BASE_URL: + process.env.DUCK_UI_DUCKDB_WASM_BASE_URL || "" }; const scriptContent = ` diff --git a/src/services/duckdb/index.ts b/src/services/duckdb/index.ts index cf81b5e..44012d1 100644 --- a/src/services/duckdb/index.ts +++ b/src/services/duckdb/index.ts @@ -2,7 +2,7 @@ // Extracted from the monolithic store for testability and modularity. export { rawResultToJSON, resultToJSON } from "./resultParser"; -export { initializeWasmConnection, loadEmbeddedDatabases, MANUAL_BUNDLES } from "./wasmConnection"; +export { initializeWasmConnection, loadEmbeddedDatabases, resolveDuckdbBundles } from "./wasmConnection"; export { cleanupOPFSConnection, testOPFSConnection, opfsActivePaths } from "./opfsConnection"; export { executeExternalQuery, diff --git a/src/services/duckdb/opfsConnection.ts b/src/services/duckdb/opfsConnection.ts index 1f882f8..1eaed61 100644 --- a/src/services/duckdb/opfsConnection.ts +++ b/src/services/duckdb/opfsConnection.ts @@ -1,6 +1,6 @@ import * as duckdb from "@duckdb/duckdb-wasm"; import { retryWithBackoff, validateConnection } from "./utils"; -import { MANUAL_BUNDLES } from "./wasmConnection"; +import { createDuckdbWorker, resolveDuckdbBundles } from "./wasmConnection"; import type { ConnectionProvider } from "@/store/types"; // OPFS connection tracking to prevent concurrent access @@ -61,12 +61,17 @@ export const testOPFSConnection = async ( ); } - const bundle = await duckdb.selectBundle(MANUAL_BUNDLES); - const worker = new Worker(bundle.mainWorker!); + const bundles = await resolveDuckdbBundles(); + const bundle = await duckdb.selectBundle(bundles); + const { worker, revoke } = createDuckdbWorker(bundle.mainWorker!); const logger = new duckdb.VoidLogger(); const db = new duckdb.AsyncDuckDB(logger, worker); - await db.instantiate(bundle.mainModule); + try { + await db.instantiate(bundle.mainModule); + } finally { + revoke(); + } // Use retry with exponential backoff for OPFS access handle conflicts await retryWithBackoff( diff --git a/src/services/duckdb/wasmConnection.ts b/src/services/duckdb/wasmConnection.ts index f01cc25..45b5e8d 100644 --- a/src/services/duckdb/wasmConnection.ts +++ b/src/services/duckdb/wasmConnection.ts @@ -1,21 +1,87 @@ import * as duckdb from "@duckdb/duckdb-wasm"; import { validateConnection } from "./utils"; -// Import WASM bundles -import duckdb_wasm from "@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url"; -import mvp_worker from "@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url"; -import duckdb_wasm_eh from "@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url"; -import eh_worker from "@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url"; - -export const MANUAL_BUNDLES: duckdb.DuckDBBundles = { - mvp: { - mainModule: duckdb_wasm, - mainWorker: mvp_worker, - }, - eh: { - mainModule: duckdb_wasm_eh, - mainWorker: eh_worker, - }, +const getRuntimeEnv = (): Partial> => + (window.env ?? {}) as Partial>; + +const getCdnBundles = (runtimeEnv: Partial>): duckdb.DuckDBBundles => { + const configuredBaseUrl = runtimeEnv.DUCK_UI_DUCKDB_WASM_BASE_URL ?? ""; + if (!configuredBaseUrl) { + return duckdb.getJsDelivrBundles(); + } + + const baseUrl = configuredBaseUrl.replace(/\/+$/, ""); + + return { + mvp: { + mainModule: `${baseUrl}/duckdb-mvp.wasm`, + mainWorker: `${baseUrl}/duckdb-browser-mvp.worker.js`, + }, + eh: { + mainModule: `${baseUrl}/duckdb-eh.wasm`, + mainWorker: `${baseUrl}/duckdb-browser-eh.worker.js`, + }, + }; +}; + +export const resolveDuckdbBundles: () => Promise = + __DUCK_UI_BUILD_DUCKDB_CDN_ONLY__ + ? async () => getCdnBundles(getRuntimeEnv()) + : (() => { + let localBundlesPromise: Promise | null = null; + + const getLocalBundles = async (): Promise => { + if (!localBundlesPromise) { + localBundlesPromise = Promise.all([ + import("@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url"), + import("@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url"), + import("@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url"), + import("@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url"), + ]).then(([duckdbWasm, mvpWorker, duckdbWasmEh, ehWorker]) => ({ + mvp: { + mainModule: duckdbWasm.default, + mainWorker: mvpWorker.default, + }, + eh: { + mainModule: duckdbWasmEh.default, + mainWorker: ehWorker.default, + }, + })).catch((error) => { + // Allow retry after transient chunk/load failures. + localBundlesPromise = null; + throw error; + }); + } + + return localBundlesPromise; + }; + + return async () => { + const runtimeEnv = getRuntimeEnv(); + if (runtimeEnv.DUCK_UI_DUCKDB_WASM_USE_CDN === true) { + return getCdnBundles(runtimeEnv); + } + return getLocalBundles(); + }; + })(); + +export const createDuckdbWorker = ( + mainWorkerUrl: string +): { worker: Worker; revoke: () => void } => { + if (/^https?:\/\//i.test(mainWorkerUrl)) { + const workerUrl = URL.createObjectURL( + new Blob([`importScripts(${JSON.stringify(mainWorkerUrl)});`], { type: "text/javascript" }) + ); + return { + worker: new Worker(workerUrl), + revoke: () => URL.revokeObjectURL(workerUrl), + }; + } + + return { + worker: new Worker(mainWorkerUrl), + revoke: () => {}, + }; }; /** @@ -25,8 +91,9 @@ export const initializeWasmConnection = async (): Promise<{ db: duckdb.AsyncDuckDB; connection: duckdb.AsyncDuckDBConnection; }> => { - const bundle = await duckdb.selectBundle(MANUAL_BUNDLES); - const worker = new Worker(bundle.mainWorker!); + const bundles = await resolveDuckdbBundles(); + const bundle = await duckdb.selectBundle(bundles); + const { worker, revoke } = createDuckdbWorker(bundle.mainWorker!); const logger = new duckdb.VoidLogger(); // Check if unsigned extensions are allowed from environment @@ -35,7 +102,11 @@ export const initializeWasmConnection = async (): Promise<{ // Create database with configuration const db = new duckdb.AsyncDuckDB(logger, worker); - await db.instantiate(bundle.mainModule); + try { + await db.instantiate(bundle.mainModule); + } finally { + revoke(); + } const dbConfig: duckdb.DuckDBConfig = { allowUnsignedExtensions: allowUnsignedExtensions, diff --git a/src/services/persistence/systemDb.ts b/src/services/persistence/systemDb.ts index cd6b986..75588bd 100644 --- a/src/services/persistence/systemDb.ts +++ b/src/services/persistence/systemDb.ts @@ -7,7 +7,7 @@ */ import * as duckdb from "@duckdb/duckdb-wasm"; -import { MANUAL_BUNDLES } from "@/services/duckdb/wasmConnection"; +import { createDuckdbWorker, resolveDuckdbBundles } from "@/services/duckdb/wasmConnection"; import { runMigrations } from "./migrations"; // Singleton state @@ -62,12 +62,17 @@ export async function initializeSystemDb(): Promise { try { console.info("[SystemDB] Initializing system database (OPFS)..."); - const bundle = await duckdb.selectBundle(MANUAL_BUNDLES); - const worker = new Worker(bundle.mainWorker!); + const bundles = await resolveDuckdbBundles(); + const bundle = await duckdb.selectBundle(bundles); + const { worker, revoke } = createDuckdbWorker(bundle.mainWorker!); const logger = new duckdb.VoidLogger(); systemDb = new duckdb.AsyncDuckDB(logger, worker); - await systemDb.instantiate(bundle.mainModule); + try { + await systemDb.instantiate(bundle.mainModule); + } finally { + revoke(); + } await systemDb.open({ path: `opfs://${SYSTEM_DB_PATH}`, diff --git a/src/store/types.ts b/src/store/types.ts index 6e2123c..09fe44e 100644 --- a/src/store/types.ts +++ b/src/store/types.ts @@ -15,6 +15,8 @@ declare global { DUCK_UI_EXTERNAL_PASS: string; DUCK_UI_EXTERNAL_DATABASE_NAME: string; DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS: boolean; + DUCK_UI_DUCKDB_WASM_USE_CDN?: boolean; + DUCK_UI_DUCKDB_WASM_BASE_URL?: string; }; } } diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index 973db8e..1eb9bda 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -2,3 +2,4 @@ declare const __DUCK_UI_VERSION__: string; declare const __DUCK_UI_RELEASE_DATE__: string; +declare const __DUCK_UI_BUILD_DUCKDB_CDN_ONLY__: boolean; diff --git a/vite.config.ts b/vite.config.ts index e14fbf9..9ac0655 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -6,6 +6,7 @@ import tailwindcss from '@tailwindcss/vite'; export default defineConfig(({ mode }) => { const env = loadEnv(mode, process.cwd(), ''); + const buildDuckdbCdnOnly = env.DUCK_UI_DUCKDB_WASM_CDN_ONLY === 'true'; // Manually construct the object to be defined // Filter out keys with invalid JS identifier characters (fixes Windows builds where @@ -28,7 +29,8 @@ export default defineConfig(({ mode }) => { define: { __DUCK_UI_VERSION__: JSON.stringify(pkg.version), __DUCK_UI_RELEASE_DATE__: JSON.stringify(pkg.release_date), + __DUCK_UI_BUILD_DUCKDB_CDN_ONLY__: JSON.stringify(buildDuckdbCdnOnly), ...processEnvValues // Spread the processed variables }, }; -}); \ No newline at end of file +});