Remove backward compatibility with useIndexedDBCache flag

Author: akaashrp

examples/cache-usage/README.md

@@ -1,8 +1,7 @@
 # WebLLM Cache Usage
 
 WebLLM supports multiple persistent cache backends. You can pick the classic Cache API, IndexedDB, or the experimental Chrome [Cross-Origin Storage](https://github.com/explainers-by-googlers/cross-origin-storage) extension by
-setting `AppConfig.cacheBackend` to `"cache"`, `"indexeddb"`, or `"cross-origin"`. (`AppConfig.useIndexedDBCache`
-is still honored for backward compatibility.)
+setting `AppConfig.cacheBackend` to `"cache"`, `"indexeddb"`, or `"cross-origin"`.
 This folder provides an example on how different caches are used in WebLLM. We also
 demonstrate the utility cache functions such as deleting models, checking if models are in cache, etc.
 

examples/cache-usage/package.json

@@ -3,12 +3,12 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "start": "parcel src/cache_usage.html  --port 8888",
+    "start": "parcel src/cache_usage.html  --port 8889",
     "build": "parcel build src/cache_usage.html --dist-dir lib"
   },
   "devDependencies": {
     "buffer": "^5.7.1",
-    "parcel": "^2.8.3",
+    "parcel": "2.8.3",
     "process": "^0.11.10",
     "tslib": "^2.3.1",
     "typescript": "^4.9.5",

src/cache_util.ts

@@ -15,6 +15,26 @@ import CrossOriginStorageCache from "./cross_origin_storage_cache";
 type CacheScope = "webllm/model" | "webllm/config" | "webllm/wasm";
 
 let crossOriginUnavailableLogged = false;
+let crossOriginAvailabilityWait: Promise<void> | null = null;
+
+function scheduleCrossOriginFallbackWarning(
+  logger: (msg: string) => void,
+): void {
+  if (crossOriginUnavailableLogged || crossOriginAvailabilityWait) {
+    return;
+  }
+  crossOriginAvailabilityWait = (async () => {
+    const availableSoon = await CrossOriginStorage.waitForAvailability();
+    crossOriginAvailabilityWait = null;
+    if (availableSoon || crossOriginUnavailableLogged) {
+      return;
+    }
+    logger(
+      "Cross-origin storage backend is not yet available; temporarily falling back to the Cache API.",
+    );
+    crossOriginUnavailableLogged = true;
+  })();
+}
 
 function shouldUseCrossOrigin(appConfig: AppConfig): boolean {
   return (
@@ -33,13 +53,7 @@ export function getArtifactCache(
     if (CrossOriginStorage.isAvailable()) {
       return new CrossOriginStorageCache(scope);
     }
-    // Fallback to Cache API
-    if (!crossOriginUnavailableLogged) {
-      logger(
-        "Cross-origin storage backend requested but unavailable; falling back to Cache API.",
-      );
-      crossOriginUnavailableLogged = true;
-    }
+    scheduleCrossOriginFallbackWarning(logger);
   }
   if (backend === "indexeddb") {
     return new tvmjs.ArtifactIndexedDBCache(scope);
@@ -81,6 +95,10 @@ async function deleteTensorCacheEntries(
   try {
     manifest = await cache.fetchWithCache(jsonUrl, "json");
   } catch (err) {
+    console.warn(
+      `Failed to load tensor cache manifest at ${jsonUrl}; skipping deletion.`,
+      err,
+    );
     return;
   }
   const records = manifest?.records ?? [];

src/config.ts

@@ -269,8 +269,8 @@ export interface ModelRecord {
  * passed to the load.
  *
  * @param model_list: models to be used.
- * @param useIndexedDBCache: if true, will use IndexedDBCache to cache models and other artifacts.
- * If false or unspecified, will use the Cache API. For more information of the two, see:
+ * @param cacheBackend: the backend to use for caching models and other artifacts.
+ * If unspecified, will use the Cache API. For more information, see:
  * https://developer.mozilla.org/en-US/docs/Web/API/Storage_API/Storage_quotas_and_eviction_criteria#what_technologies_store_data_in_the_browser
  *
  * @note Note that the Cache API is more well-tested in WebLLM as of now.
@@ -279,15 +279,14 @@ export type CacheBackend = "cache" | "indexeddb" | "cross-origin";
 
 export interface AppConfig {
   model_list: Array<ModelRecord>;
-  useIndexedDBCache?: boolean;
   cacheBackend?: CacheBackend;
 }
 
 export function getCacheBackend(appConfig: AppConfig): CacheBackend {
   if (appConfig.cacheBackend !== undefined) {
     return appConfig.cacheBackend;
   }
-  return appConfig.useIndexedDBCache ? "indexeddb" : "cache";
+  return "cache";
 }
 
 /**
@@ -319,7 +318,7 @@ export const functionCallingModelIds = [
  * current WebLLM npm version.
  */
 export const prebuiltAppConfig: AppConfig = {
-  useIndexedDBCache: false,
+  cacheBackend: "cache",
   model_list: [
     // Llama-3.2
     {

src/cross_origin_storage.ts

@@ -1,5 +1,17 @@
 const HASH_ALGORITHM = "SHA-256";
 const HASH_MATCH_REGEX = /[A-Fa-f0-9]{64}/;
+const AVAILABILITY_POLL_INTERVAL_MS = 100;
+const DEFAULT_AVAILABILITY_TIMEOUT_MS = 3000;
+const HASH_CACHE_SYMBOL = Symbol.for("mlc.crossOriginStorage.hashCache");
+
+const globalScope = globalThis as Record<PropertyKey, unknown>;
+if (!globalScope[HASH_CACHE_SYMBOL]) {
+  globalScope[HASH_CACHE_SYMBOL] = new Map<string, CrossOriginHashDescriptor>();
+}
+const GLOBAL_HASH_CACHE = globalScope[HASH_CACHE_SYMBOL] as Map<
+  string,
+  CrossOriginHashDescriptor
+>;
 
 export interface CrossOriginHashDescriptor {
   algorithm: string;
@@ -16,7 +28,6 @@ interface CrossOriginStorageAPI {
     descriptors: CrossOriginHashDescriptor[],
     options?: { create?: boolean },
   ): Promise<CrossOriginStorageHandle[]>;
-  removeFileHandles?(descriptors: CrossOriginHashDescriptor[]): Promise<void>;
 }
 
 type RequestLike = string | URL | Request | { url?: string };
@@ -25,13 +36,16 @@ declare global {
   interface Navigator {
     crossOriginStorage?: CrossOriginStorageAPI;
   }
+  interface WorkerNavigator {
+    crossOriginStorage?: CrossOriginStorageAPI;
+  }
 }
 
 export default class CrossOriginStorage {
   private hashCache: Map<string, CrossOriginHashDescriptor>;
 
   constructor() {
-    this.hashCache = new Map();
+    this.hashCache = GLOBAL_HASH_CACHE;
   }
 
   static isAvailable(): boolean {
@@ -42,6 +56,35 @@ export default class CrossOriginStorage {
     );
   }
 
+  static async waitForAvailability(
+    timeoutMs: number = DEFAULT_AVAILABILITY_TIMEOUT_MS,
+  ): Promise<boolean> {
+    if (CrossOriginStorage.isAvailable()) {
+      return true;
+    }
+    if (typeof navigator === "undefined") {
+      return false;
+    }
+    if (typeof setTimeout === "undefined") {
+      return false;
+    }
+    return new Promise((resolve) => {
+      const deadline = Date.now() + timeoutMs;
+      const tick = () => {
+        if (CrossOriginStorage.isAvailable()) {
+          resolve(true);
+          return;
+        }
+        if (Date.now() >= deadline) {
+          resolve(false);
+          return;
+        }
+        setTimeout(tick, AVAILABILITY_POLL_INTERVAL_MS);
+      };
+      setTimeout(tick, AVAILABILITY_POLL_INTERVAL_MS);
+    });
+  }
+
   async match(request: RequestLike): Promise<Response | undefined> {
     const url = this.normalizeRequest(request);
     const hash = await this.resolveHashDescriptor(url);
@@ -84,17 +127,10 @@ export default class CrossOriginStorage {
     this.hashCache.set(url, hash);
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async delete(request: RequestLike): Promise<void> {
-    const url = this.normalizeRequest(request);
-    const hash = await this.resolveHashDescriptor(url);
-    if (!hash) {
-      return;
-    }
-    const api = this.getApi();
-    if (api && typeof api.removeFileHandles === "function") {
-      await api.removeFileHandles([hash]);
-    }
-    this.hashCache.delete(url);
+    // Currently no delete API provided by Cross-Origin Storage Extension
+    return;
   }
 
   private getApi(): CrossOriginStorageAPI | undefined {
@@ -145,7 +181,7 @@ export default class CrossOriginStorage {
     if (metadataHash) {
       return metadataHash;
     }
-    if (/\/resolve\/main\//.test(url)) {
+    if (/\/resolve\//.test(url)) {
       const pointerHash = await this.extractHashFromPointer(url);
       if (pointerHash) {
         return pointerHash;

src/cross_origin_storage_cache.ts

@@ -67,8 +67,7 @@ export class CrossOriginStorageCache implements tvmjs.ArtifactCacheTemplate {
   }
 
   async deleteInCache(_url: string): Promise<void> {
-    // no delete API currently provided by Cross-Origin Storage
-    return;
+    await this.storage.delete(_url);
   }
 
   private async responseToStoreType(

src/utils.ts

@@ -76,10 +76,7 @@ export function areAppConfigsEqual(
     return config1 === config2;
   }
 
-  // Check if both configurations have the same IndexedDB cache usage
-  if (config1.useIndexedDBCache !== config2.useIndexedDBCache) {
-    return false;
-  }
+  // Check if both configurations have the same cache backend
   if (config1.cacheBackend !== config2.cacheBackend) {
     return false;
   }

tests/scripts/sanity_checks/sanity_checks.ts

@@ -157,7 +157,7 @@ async function testLogprobs(modelId: string, appConfig: webllm.AppConfig) {
 async function main() {
   const modelId = "Qwen3-0.6B-q0f32-MLC";
   const appConfig = webllm.prebuiltAppConfig;
-  appConfig.useIndexedDBCache = true;
+  appConfig.cacheBackend = "indexeddb";
   setLabel("gpu-test-label", "Running tests...");
   let passed = 0,
     total = 0;