Transformers.js line items shouldn't load resources in the benchmark

Right now some of the resources are loaded via relative paths in the
benchmark. Instead of trying to patch the benchmark to use the blob,
this patch makes it easier to redirect `fetch` so that it can find
the already loaded data.

I did this by adding a `JetStream.resources` object that maps from
the original relative path resource to the blobURLOrPath vended
by the preloader.

Author: kmiller68

JetStreamDriver.js

@@ -560,20 +560,35 @@ const BenchmarkState = Object.freeze({
 
 
 class Scripts {
-    constructor() {
+    constructor(preloads) {
         this.scripts = [];
+
+        let preloadsCode = "";
+        let resourcesCode = "";
+        for (let { name, resource, blobURLOrPath } of preloads) {
+            console.assert(name?.length > 0, "Invalid preload name.");
+            console.assert(resource?.length > 0, "Invalid preload resource.");
+            console.assert(blobURLOrPath?.length > 0, "Invalid preload data.");
+            preloadsCode += `${JSON.stringify(name)}: "${blobURLOrPath}",\n`;
+            resourcesCode += `${JSON.stringify(resource)}: "${blobURLOrPath}",\n`;
+        }
         // Expose a globalThis.JetStream object to the workload. We use
         // a proxy to prevent prototype access and throw on unknown properties.
         this.add(`
             const throwOnAccess = (name) => new Proxy({},  {
                 get(target, property, receiver) {
                     throw new Error(name + "." + property + " is not defined.");
                 }
-            }); 
+            });
             globalThis.JetStream = {
                 __proto__: throwOnAccess("JetStream"),
                 preload: {
                     __proto__: throwOnAccess("JetStream.preload"),
+                    ${preloadsCode}
+                },
+                resources: {
+                    __proto__: throwOnAccess("JetStream.preload"),
+                    ${resourcesCode}
                 },
             };
             `);
@@ -631,8 +646,8 @@ class Scripts {
 }
 
 class ShellScripts extends Scripts {
-    constructor() {
-        super();
+    constructor(preloads) {
+        super(preloads);
         this.prefetchedResources = Object.create(null);;
     }
 
@@ -698,8 +713,8 @@ class ShellScripts extends Scripts {
 }
 
 class BrowserScripts extends Scripts {
-    constructor() {
-        super();
+    constructor(preloads) {
+        super(preloads);
         this.add("window.onerror = top.currentReject;");
     }
 
@@ -898,7 +913,7 @@ class Benchmark {
         if (this.isDone)
             throw new Error(`Cannot run Benchmark ${this.name} twice`);
         this._state = BenchmarkState.PREPARE;
-        const scripts = isInBrowser ? new BrowserScripts() : new ShellScripts();
+        const scripts = isInBrowser ? new BrowserScripts(this.preloads) : new ShellScripts(this.preloads);
 
         if (!!this.plan.deterministicRandom)
             scripts.addDeterministicRandom()
@@ -908,15 +923,6 @@ class Benchmark {
         if (this.shellPrefetchedResources) {
             scripts.addPrefetchedResources(this.shellPrefetchedResources);
         }
-        if (this.plan.preload) {
-            let preloadCode = "";
-            for (let [ variableName, blobURLOrPath ] of this.preloads) {
-                console.assert(variableName?.length > 0, "Invalid preload name.");
-                console.assert(blobURLOrPath?.length > 0, "Invalid preload data.");
-                preloadCode += `JetStream.preload[${JSON.stringify(variableName)}] = "${blobURLOrPath}";\n`;
-            }
-            scripts.add(preloadCode);
-        }
 
         const prerunCode = this.prerunCode;
         if (prerunCode)
@@ -1072,7 +1078,7 @@ class Benchmark {
                 promises.push(this.loadBlob("preload", name, resource).then((blobData) => {
                     if (!globalThis.allIsGood)
                         return;
-                    this.preloads.push([ blobData.prop, blobData.blobURL ]);
+                    this.preloads.push({ name: blobData.prop, resource: blobData.resource, blobURLOrPath: blobData.blobURL });
                     this.updateCounter();
                 }).catch((error) => {
                     // We'll try again later in retryPrefetchResourceForBrowser(). Don't throw an error.
@@ -1099,7 +1105,7 @@ class Benchmark {
             if (type == "preload") {
                 if (this.failedPreloads && this.failedPreloads[blobData.prop]) {
                     this.failedPreloads[blobData.prop] = false;
-                    this.preloads.push([ blobData.prop, blobData.blobURL ]);
+                    this.preloads.push({ name: blobData.prop, resource: blobData.resource, blobURLOrPath: blobData.blobURL });
                     counter.failedPreloadResources--;
                 }
             }
@@ -1112,7 +1118,7 @@ class Benchmark {
             if (!globalThis.allIsGood)
                 return;
             if (blobData.type == "preload")
-                this.preloads.push([ blobData.prop, blobData.blobURL ]);
+                this.preloads.push({ name: blobData.prop, resource: blobData.resource, blobURLOrPath: blobData.blobURL });
             this.updateCounter();
         });
 
@@ -1170,7 +1176,7 @@ class Benchmark {
                 this.shellPrefetchedResources[resource] = bytes;
             }
 
-            this.preloads.push([name, resource]);
+            this.preloads.push({ name, resource, blobURLOrPath: resource });
         }
     }
 

transformersjs/benchmark.js

@@ -16,13 +16,14 @@ globalThis.URL = URL;
 
 // Polyfill fetch for shell-compatibility and to cache / preload model weights etc.
 let preload = { /* Initialized in init() below due to async. */ };
-const originalFetch = globalThis.fetch ?? function(url) {
-  throw new Error("no fetch available");
-}
-globalThis.fetch = async function(url) {
+
+async function redirectingFetch(url) {
   // DEBUG
   // console.log('fetch', url);
 
+  if (url.startsWith("./"))
+    url = JetStream.resources[url];
+
   // Redirect some paths to cached/preloaded resources.
   if (preload[url]) {
     return {
@@ -38,8 +39,7 @@ globalThis.fetch = async function(url) {
     };
   }
 
-  // This should only be called in the browser, where fetch() is available.
-  return originalFetch(url);
+  throw new Error(`Unexpected resource requested in benchmark: ${url}`);
 };
 
 // JetStream benchmark harness. Reuse for two different Transformers.js tasks.
@@ -66,6 +66,9 @@ class Benchmark {
       // DEBUG
       // console.log('inputFile', this.inputFile.byteLength, 'bytes');
     }
+
+    // After we have loaded everything close the door behind us to make sure no other network requests happen.
+    globalThis.fetch = redirectingFetch;
   }
 
   async runIteration() {