Merge pull request #31 from xxxbrian/main

Support CDN loading for DuckDB WASM

Author: caioricciuti

README.md

@@ -41,7 +41,7 @@ docker run -p 5522:5522 \
   ghcr.io/ibero-data/duck-ui:latest
 ```
 
-| Variable | Description | Default |
+| Runtime Variable | Description | Default |
 |----------|-------------|---------|
 | `DUCK_UI_EXTERNAL_CONNECTION_NAME` | Name for the external connection | "" |
 | `DUCK_UI_EXTERNAL_HOST` | Host URL for external DuckDB | "" |
@@ -50,6 +50,14 @@ docker run -p 5522:5522 \
 | `DUCK_UI_EXTERNAL_PASS` | Password for external connection | "" |
 | `DUCK_UI_EXTERNAL_DATABASE_NAME` | Database name for external connection | "" |
 | `DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS` | Allow unsigned extensions in DuckDB | false |
+| `DUCK_UI_DUCKDB_WASM_USE_CDN` | Load DuckDB WASM from CDN (ignored when build-time `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true`) | false |
+| `DUCK_UI_DUCKDB_WASM_BASE_URL` | Custom CDN base URL (used when `DUCK_UI_DUCKDB_WASM_USE_CDN=true`) | auto jsDelivr |
+
+| Build-time Variable | Description | Default |
+|----------|-------------|---------|
+| `DUCK_UI_DUCKDB_WASM_CDN_ONLY` | Build a CDN-only artifact (local DuckDB WASM assets are not bundled). | false |
+
+When `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true`, runtime `DUCK_UI_DUCKDB_WASM_USE_CDN=false` cannot switch back to local WASM.
 
 
 
@@ -197,4 +205,4 @@ This project is sponsored by:
 
 <br/>
 
-Want to be a sponsor? [Contact us](mailto:caio.ricciuti+sponsorship@outlook.com).
+Want to be a sponsor? [Contact us](mailto:caio.ricciuti+sponsorship@outlook.com).

docs/environment-variables.md

@@ -17,11 +17,21 @@ These variables allow you to configure Duck-UI to connect to an external DuckDB
 | `DUCK_UI_EXTERNAL_PASS` | Password for authentication | No | `""` | `"your-password"` |
 | `DUCK_UI_EXTERNAL_DATABASE_NAME` | Database name to connect to | No | `""` | `"analytics"` |
 
-### Extension Settings
+### Runtime Settings
 
 | Variable | Description | Required | Default | Example |
 |----------|-------------|----------|---------|---------|
 | `DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS` | Allow loading unsigned DuckDB extensions | No | `false` | `"true"` |
+| `DUCK_UI_DUCKDB_WASM_USE_CDN` | Enable loading DuckDB WASM and worker files from CDN (ignored when `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true` at build time) | No | `false` | `"true"` |
+| `DUCK_UI_DUCKDB_WASM_BASE_URL` | Custom CDN base URL (when `DUCK_UI_DUCKDB_WASM_USE_CDN=true`) | No | Auto (`duckdb.getJsDelivrBundles()`) | `"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.33.1-dev19.0/dist"` |
+
+### Build-time Settings
+
+| Variable | Description | Required | Default | Example |
+|----------|-------------|----------|---------|---------|
+| `DUCK_UI_DUCKDB_WASM_CDN_ONLY` | Build a CDN-only artifact (local DuckDB WASM files are not bundled). Suitable for edge platforms with strict asset size limits. | No | `false` | `"true"` |
+
+When `DUCK_UI_DUCKDB_WASM_CDN_ONLY=true`, runtime `DUCK_UI_DUCKDB_WASM_USE_CDN=false` cannot switch back to local WASM because local assets are not included in the build.
 
 ::: warning Security Note
 Enabling unsigned extensions may pose security risks. Only enable this in trusted environments.

inject-env.js

@@ -16,7 +16,11 @@ const envVars = {
     process.env.DUCK_UI_EXTERNAL_DATABASE_NAME || "",
   // Add new configuration for DuckDB settings
   DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS:
-    process.env.DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS === "true" || false
+    process.env.DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS === "true" || false,
+  DUCK_UI_DUCKDB_WASM_USE_CDN:
+    process.env.DUCK_UI_DUCKDB_WASM_USE_CDN === "true" || false,
+  DUCK_UI_DUCKDB_WASM_BASE_URL:
+    process.env.DUCK_UI_DUCKDB_WASM_BASE_URL || ""
 };
 
 const scriptContent = `

src/services/duckdb/index.ts

@@ -2,7 +2,7 @@
 // Extracted from the monolithic store for testability and modularity.
 
 export { rawResultToJSON, resultToJSON } from "./resultParser";
-export { initializeWasmConnection, loadEmbeddedDatabases, MANUAL_BUNDLES } from "./wasmConnection";
+export { initializeWasmConnection, loadEmbeddedDatabases, resolveDuckdbBundles } from "./wasmConnection";
 export { cleanupOPFSConnection, testOPFSConnection, opfsActivePaths } from "./opfsConnection";
 export {
   executeExternalQuery,

src/services/duckdb/opfsConnection.ts

@@ -1,6 +1,6 @@
 import * as duckdb from "@duckdb/duckdb-wasm";
 import { retryWithBackoff, validateConnection } from "./utils";
-import { MANUAL_BUNDLES } from "./wasmConnection";
+import { createDuckdbWorker, resolveDuckdbBundles } from "./wasmConnection";
 import type { ConnectionProvider } from "@/store/types";
 
 // OPFS connection tracking to prevent concurrent access
@@ -61,12 +61,17 @@ export const testOPFSConnection = async (
     );
   }
 
-  const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
-  const worker = new Worker(bundle.mainWorker!);
+  const bundles = await resolveDuckdbBundles();
+  const bundle = await duckdb.selectBundle(bundles);
+  const { worker, revoke } = createDuckdbWorker(bundle.mainWorker!);
   const logger = new duckdb.VoidLogger();
   const db = new duckdb.AsyncDuckDB(logger, worker);
 
-  await db.instantiate(bundle.mainModule);
+  try {
+    await db.instantiate(bundle.mainModule);
+  } finally {
+    revoke();
+  }
 
   // Use retry with exponential backoff for OPFS access handle conflicts
   await retryWithBackoff(

src/services/duckdb/wasmConnection.ts

@@ -1,21 +1,87 @@
 import * as duckdb from "@duckdb/duckdb-wasm";
 import { validateConnection } from "./utils";
 
-// Import WASM bundles
-import duckdb_wasm from "@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url";
-import mvp_worker from "@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url";
-import duckdb_wasm_eh from "@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url";
-import eh_worker from "@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url";
-
-export const MANUAL_BUNDLES: duckdb.DuckDBBundles = {
-  mvp: {
-    mainModule: duckdb_wasm,
-    mainWorker: mvp_worker,
-  },
-  eh: {
-    mainModule: duckdb_wasm_eh,
-    mainWorker: eh_worker,
-  },
+const getRuntimeEnv = (): Partial<NonNullable<Window["env"]>> =>
+  (window.env ?? {}) as Partial<NonNullable<Window["env"]>>;
+
+const getCdnBundles = (runtimeEnv: Partial<NonNullable<Window["env"]>>): duckdb.DuckDBBundles => {
+  const configuredBaseUrl = runtimeEnv.DUCK_UI_DUCKDB_WASM_BASE_URL ?? "";
+  if (!configuredBaseUrl) {
+    return duckdb.getJsDelivrBundles();
+  }
+
+  const baseUrl = configuredBaseUrl.replace(/\/+$/, "");
+
+  return {
+    mvp: {
+      mainModule: `${baseUrl}/duckdb-mvp.wasm`,
+      mainWorker: `${baseUrl}/duckdb-browser-mvp.worker.js`,
+    },
+    eh: {
+      mainModule: `${baseUrl}/duckdb-eh.wasm`,
+      mainWorker: `${baseUrl}/duckdb-browser-eh.worker.js`,
+    },
+  };
+};
+
+export const resolveDuckdbBundles: () => Promise<duckdb.DuckDBBundles> =
+  __DUCK_UI_BUILD_DUCKDB_CDN_ONLY__
+    ? async () => getCdnBundles(getRuntimeEnv())
+    : (() => {
+        let localBundlesPromise: Promise<duckdb.DuckDBBundles> | null = null;
+
+        const getLocalBundles = async (): Promise<duckdb.DuckDBBundles> => {
+          if (!localBundlesPromise) {
+            localBundlesPromise = Promise.all([
+              import("@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url"),
+              import("@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url"),
+              import("@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url"),
+              import("@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url"),
+            ]).then(([duckdbWasm, mvpWorker, duckdbWasmEh, ehWorker]) => ({
+              mvp: {
+                mainModule: duckdbWasm.default,
+                mainWorker: mvpWorker.default,
+              },
+              eh: {
+                mainModule: duckdbWasmEh.default,
+                mainWorker: ehWorker.default,
+              },
+            })).catch((error) => {
+              // Allow retry after transient chunk/load failures.
+              localBundlesPromise = null;
+              throw error;
+            });
+          }
+
+          return localBundlesPromise;
+        };
+
+        return async () => {
+          const runtimeEnv = getRuntimeEnv();
+          if (runtimeEnv.DUCK_UI_DUCKDB_WASM_USE_CDN === true) {
+            return getCdnBundles(runtimeEnv);
+          }
+          return getLocalBundles();
+        };
+      })();
+
+export const createDuckdbWorker = (
+  mainWorkerUrl: string
+): { worker: Worker; revoke: () => void } => {
+  if (/^https?:\/\//i.test(mainWorkerUrl)) {
+    const workerUrl = URL.createObjectURL(
+      new Blob([`importScripts(${JSON.stringify(mainWorkerUrl)});`], { type: "text/javascript" })
+    );
+    return {
+      worker: new Worker(workerUrl),
+      revoke: () => URL.revokeObjectURL(workerUrl),
+    };
+  }
+
+  return {
+    worker: new Worker(mainWorkerUrl),
+    revoke: () => {},
+  };
 };
 
 /**
@@ -25,8 +91,9 @@ export const initializeWasmConnection = async (): Promise<{
   db: duckdb.AsyncDuckDB;
   connection: duckdb.AsyncDuckDBConnection;
 }> => {
-  const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
-  const worker = new Worker(bundle.mainWorker!);
+  const bundles = await resolveDuckdbBundles();
+  const bundle = await duckdb.selectBundle(bundles);
+  const { worker, revoke } = createDuckdbWorker(bundle.mainWorker!);
   const logger = new duckdb.VoidLogger();
 
   // Check if unsigned extensions are allowed from environment
@@ -35,7 +102,11 @@ export const initializeWasmConnection = async (): Promise<{
   // Create database with configuration
   const db = new duckdb.AsyncDuckDB(logger, worker);
 
-  await db.instantiate(bundle.mainModule);
+  try {
+    await db.instantiate(bundle.mainModule);
+  } finally {
+    revoke();
+  }
 
   const dbConfig: duckdb.DuckDBConfig = {
     allowUnsignedExtensions: allowUnsignedExtensions,

src/services/persistence/systemDb.ts

@@ -7,7 +7,7 @@
  */
 
 import * as duckdb from "@duckdb/duckdb-wasm";
-import { MANUAL_BUNDLES } from "@/services/duckdb/wasmConnection";
+import { createDuckdbWorker, resolveDuckdbBundles } from "@/services/duckdb/wasmConnection";
 import { runMigrations } from "./migrations";
 
 // Singleton state
@@ -62,12 +62,17 @@ export async function initializeSystemDb(): Promise<void> {
   try {
     console.info("[SystemDB] Initializing system database (OPFS)...");
 
-    const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
-    const worker = new Worker(bundle.mainWorker!);
+    const bundles = await resolveDuckdbBundles();
+    const bundle = await duckdb.selectBundle(bundles);
+    const { worker, revoke } = createDuckdbWorker(bundle.mainWorker!);
     const logger = new duckdb.VoidLogger();
 
     systemDb = new duckdb.AsyncDuckDB(logger, worker);
-    await systemDb.instantiate(bundle.mainModule);
+    try {
+      await systemDb.instantiate(bundle.mainModule);
+    } finally {
+      revoke();
+    }
 
     await systemDb.open({
       path: `opfs://${SYSTEM_DB_PATH}`,

src/store/types.ts

@@ -15,6 +15,8 @@ declare global {
       DUCK_UI_EXTERNAL_PASS: string;
       DUCK_UI_EXTERNAL_DATABASE_NAME: string;
       DUCK_UI_ALLOW_UNSIGNED_EXTENSIONS: boolean;
+      DUCK_UI_DUCKDB_WASM_USE_CDN?: boolean;
+      DUCK_UI_DUCKDB_WASM_BASE_URL?: string;
     };
   }
 }

src/vite-env.d.ts

@@ -2,3 +2,4 @@
 
 declare const __DUCK_UI_VERSION__: string;
 declare const __DUCK_UI_RELEASE_DATE__: string;
+declare const __DUCK_UI_BUILD_DUCKDB_CDN_ONLY__: boolean;

vite.config.ts

@@ -6,6 +6,7 @@ import tailwindcss from '@tailwindcss/vite';
 
 export default defineConfig(({ mode }) => {
   const env = loadEnv(mode, process.cwd(), '');
+  const buildDuckdbCdnOnly = env.DUCK_UI_DUCKDB_WASM_CDN_ONLY === 'true';
 
   // Manually construct the object to be defined
   // Filter out keys with invalid JS identifier characters (fixes Windows builds where
@@ -28,7 +29,8 @@ export default defineConfig(({ mode }) => {
     define: {
       __DUCK_UI_VERSION__: JSON.stringify(pkg.version),
       __DUCK_UI_RELEASE_DATE__: JSON.stringify(pkg.release_date),
+      __DUCK_UI_BUILD_DUCKDB_CDN_ONLY__: JSON.stringify(buildDuckdbCdnOnly),
       ...processEnvValues // Spread the processed variables
     },
   };
-});
+});