Support manifest for WASMFS fetchfs

JoeOsborn · JoeOsborn · commit 4720bceeae25 · 2025-01-24T15:43:08.000-08:00
This way, a fetchfs WASMFS backend can have a mapping from paths to
URLs used to satisfy those paths.  I think this makes more sense than
creating a separate fetchfs backend for each individual file.

In the absence of a manifest, the old behavior of using the file path
as a relative path against the baseURL will still be used.

NOTE, it's important that the fetchfs have a dedicated directory
created/"mounted" or else the "create file with path" code path will
never be used.  In other words, paths at `/whatever.txt` can never
work given the way WASMFS is implemented; the accessed files don't get
paths unless the parent directory and child path have the same
backend.  So, creating the directory at `/dat` and having a manifest
entry like `/whatever.txt` is the way to go for a file at
`/dat/whatever.txt`.

Manifest files are automatically used to mount/create file contents,
so there's no need to use multipl calls to wasmfs_create_file or
wasmfs_create_directory, just the one directory needs to be
created (which is what FS.mount does in the JS API).
diff --git a/src/library_fetchfs.js b/src/library_fetchfs.js
@@ -5,11 +5,25 @@
  */
 
 addToLibrary({
-  $FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'],
+  $FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend', 'wasmfs_fetch_create_manifest', 'wasmfs_fetch_add_to_manifest'],
   $FETCHFS: {
     createBackend(opts) {
-      return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0);
-    }
+      if(opts.base_url === undefined) {
+        opts.base_url = "";
+      }
+      return withStackSave(() => {
+        var manifest = 0;
+        if (opts['manifest']) {
+          manifest = _wasmfs_fetch_create_manifest();
+          Object.entries(opts['manifest']).forEach(([path, url]) => {
+            _wasmfs_fetch_add_to_manifest(manifest,
+                                          stringToUTF8OnStack(path),
+                                          stringToUTF8OnStack(url));
+          });
+        }
+        return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0, manifest);
+      });
+    },
   },
 });
 
diff --git a/src/library_wasmfs_fetch.js b/src/library_wasmfs_fetch.js
@@ -31,7 +31,8 @@ addToLibrary({
         try {
           var u = new URL(fileUrl, self.location.origin);
           url = u.toString();
-        } catch (e) {
+        } catch(_e) {
+          throw {status: 404};
         }
       }
       var chunkSize = __wasmfs_fetch_get_chunk_size(file);
@@ -41,15 +42,15 @@ addToLibrary({
       var lastChunk = ((offset+len) / chunkSize) | 0;
       if (!(file in wasmFS$JSMemoryRanges)) {
         var fileInfo = await fetch(url,{method:"HEAD", headers:{"Range": "bytes=0-"}});
-        if(fileInfo.ok &&
+        if (fileInfo.ok &&
            fileInfo.headers.has("Content-Length") &&
            fileInfo.headers.get("Accept-Ranges") == "bytes" &&
            (parseInt(fileInfo.headers.get("Content-Length")) > chunkSize*2)) {
           wasmFS$JSMemoryRanges[file] = {size:parseInt(fileInfo.headers.get("Content-Length")), chunks:[], chunkSize:chunkSize};
         } else {
           // may as well/forced to download the whole file
           var wholeFileReq = await fetch(url);
-          if(!wholeFileReq.ok) {
+          if (!wholeFileReq.ok) {
             throw wholeFileReq;
           }
           var wholeFileData = new Uint8Array(await wholeFileReq.arrayBuffer());
@@ -60,11 +61,11 @@ addToLibrary({
       }
       var allPresent = true;
       var i;
-      if(lastChunk * chunkSize < offset+len) {
+      if (lastChunk * chunkSize < offset+len) {
         lastChunk += 1;
       }
-      for(i = firstChunk; i < lastChunk; i++) {
-        if(!wasmFS$JSMemoryRanges[file].chunks[i]) {
+      for (i = firstChunk; i < lastChunk; i++) {
+        if (!wasmFS$JSMemoryRanges[file].chunks[i]) {
           allPresent = false;
           break;
         }
@@ -78,14 +79,13 @@ addToLibrary({
       var start = firstChunk*chunkSize;
       var end = lastChunk*chunkSize;
       var response = await fetch(url, {headers:{"Range": `bytes=${start}-${end-1}`}});
-      if (response.ok) {
-        var bytes = new Uint8Array(await response['arrayBuffer']());
-        for (i = firstChunk; i < lastChunk; i++) {
-          wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start);
-        }
-      } else {
+      if (!response.ok) {
         throw response;
       }
+      var bytes = new Uint8Array(await response['arrayBuffer']());
+      for (i = firstChunk; i < lastChunk; i++) {
+        wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start);
+      }
       return Promise.resolve();
     }
 
@@ -110,24 +110,21 @@ addToLibrary({
       read: async (file, buffer, length, offset) => {
         try {
           await getFileRange(file, offset || 0, length);
-        } catch (response) {
-          return response.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}};
+        } catch (failedResponse) {
+          return failedResponse.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}};
         }
         var fileInfo = wasmFS$JSMemoryRanges[file];
         var fileData = fileInfo.chunks;
         var chunkSize = fileInfo.chunkSize;
         var firstChunk = (offset / chunkSize) | 0;
         var lastChunk = ((offset+length) / chunkSize) | 0;
-        if(offset + length > lastChunk * chunkSize) {
+        if (offset + length > lastChunk * chunkSize) {
           lastChunk += 1;
         }
         var readLength = 0;
         for (var i = firstChunk; i < lastChunk; i++) {
           var chunk = fileData[i];
           var start = Math.max(i*chunkSize, offset);
-          if(!chunk) {
-            throw [fileData.length, firstChunk, lastChunk, i];
-          }
           var chunkStart = i*chunkSize;
           var end = Math.min(chunkStart+chunkSize, offset+length);
           HEAPU8.set(chunk.subarray(start-chunkStart, end-chunkStart), buffer+(start-offset));
@@ -138,7 +135,7 @@ addToLibrary({
       getSize: async (file) => {
         try {
           await getFileRange(file, 0, 0);
-        } catch (response) {
+        } catch (failedResponse) {
           return 0;
         }
         return wasmFS$JSMemoryRanges[file].size;
diff --git a/system/include/emscripten/wasmfs.h b/system/include/emscripten/wasmfs.h
@@ -57,7 +57,18 @@ backend_t wasmfs_create_memory_backend(void);
 //
 // TODO: Add an async version of this function that will work on the main
 // thread.
-backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t);
+backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)),
+                                      uint32_t chunkSize,
+                                      void *manifest);
+
+// Create a FetchFS manifest record that can be populated and passed into
+// wasmfs_fetch_create_backend.
+void *wasmfs_fetch_create_manifest();
+
+// Add a path to URL mapping to the given manifest.
+void wasmfs_fetch_add_to_manifest(void *manifest __attribute__((nonnull)),
+                                  const char *path __attribute__((nonnull)),
+                                  const char *url __attribute__((nonnull)));
 
 backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull)));
 
diff --git a/system/lib/wasmfs/backends/fetch_backend.cpp b/system/lib/wasmfs/backends/fetch_backend.cpp
@@ -14,23 +14,35 @@ namespace wasmfs {
 
 const uint32_t DEFAULT_CHUNK_SIZE = 16*1024*1024;
 
+typedef std::map<std::string,std::string> FetchManifest;
+
 class FetchBackend : public wasmfs::ProxiedAsyncJSBackend {
   std::string baseUrl;
   uint32_t chunkSize;
+  FetchManifest *manifest;
  public:
+  // Takes ownership of manifest
   FetchBackend(const std::string& baseUrl,
                uint32_t chunkSize,
+               FetchManifest *manifest,
                std::function<void(backend_t)> setupOnThread)
-    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize)
-      // TODO manifest
+    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize), manifest(manifest)
   {}
+  ~FetchBackend() {
+    if(manifest != NULL) {
+      delete manifest;
+    }
+  }
   std::shared_ptr<DataFile> createFile(mode_t mode) override;
   std::shared_ptr<Directory> createDirectory(mode_t mode) override;
   const std::string getFileURL(const std::string& filePath);
   uint32_t getChunkSize();
+  const FetchManifest *getManifest() {
+    return manifest;
+  }
 };
 
-  
+
 class FetchFile : public ProxiedAsyncJSImplFile {
   std::string filePath;
   std::string fileUrl;
@@ -58,7 +70,49 @@ class FetchDirectory : public MemoryDirectory {
                  mode_t mode,
                  backend_t backend,
                  emscripten::ProxyWorker& proxy)
-    : MemoryDirectory(mode, backend), dirPath(path), proxy(proxy) {}
+    : MemoryDirectory(mode, backend), dirPath(path), proxy(proxy) {
+    auto manifest = dynamic_cast<FetchBackend*>(getBackend())->getManifest();
+    if (manifest && path == "") {
+      for (const auto& pair : *manifest) {
+        auto path = pair.first;
+        assert(path[0] == '/');
+        char delimiter = '/';
+        std::string path_so_far = "";
+        std::string tmp = "";
+        std::shared_ptr<FetchDirectory> dir = NULL;
+        std::istringstream iss(path);
+        while(std::getline(iss, tmp, delimiter)) {
+          path_so_far += tmp;
+          if (path_so_far == path) {
+            if(!dir) {
+              assert(this->insertDataFile(tmp, 0777));
+            } else {
+              assert(dir->insertDataFile(tmp, 0777));
+            }
+          } else if (path_so_far != "") {
+            std::shared_ptr<FetchDirectory> next = NULL;
+            if(!dir) {
+              next = std::dynamic_pointer_cast<FetchDirectory>(this->getChild(tmp));
+            } else {
+              next = std::dynamic_pointer_cast<FetchDirectory>(dir->getChild(tmp));
+            }
+            if (next) {
+              dir = next;
+              assert(dir);
+            } else {
+              if(!dir) {
+                dir = std::dynamic_pointer_cast<FetchDirectory>(this->insertDirectory(tmp, 0777));
+              } else {
+                dir = std::dynamic_pointer_cast<FetchDirectory>(dir->insertDirectory(tmp, 0777));
+              }
+              assert(dir);
+            }
+          }
+          path_so_far += delimiter;
+        }
+      }
+    }
+  }
 
   std::shared_ptr<DataFile> insertDataFile(const std::string& name,
                                            mode_t mode) override {
@@ -81,6 +135,10 @@ class FetchDirectory : public MemoryDirectory {
   std::string getChildPath(const std::string& name) const {
     return dirPath + '/' + name;
   }
+
+  std::shared_ptr<File> getChild(const std::string& name) override {
+    return MemoryDirectory::getChild(name);
+  }
 };
 
 std::shared_ptr<DataFile> FetchBackend::createFile(mode_t mode) {
@@ -92,41 +150,60 @@ std::shared_ptr<Directory> FetchBackend::createDirectory(mode_t mode) {
 }
 
 const std::string FetchBackend::getFileURL(const std::string& filePath) {
-  // TODO use manifest
+  if(manifest) {
+    if (auto search = manifest->find(filePath); search != manifest->end()) {
+      return baseUrl + "/" + search->second;
+    }
+  }
   if(filePath == "") {
     return baseUrl;
   }
   return baseUrl + "/" + filePath;
 }
+
 uint32_t FetchBackend::getChunkSize() {
   return chunkSize;
 }
 
 extern "C" {
-  backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize /* TODO manifest */) {
+  backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize, FetchManifest *manifest) {
   // ProxyWorker cannot safely be synchronously spawned from the main browser
   // thread. See comment in thread_utils.h for more details.
   assert(!emscripten_is_main_browser_thread() &&
          "Cannot safely create fetch backend on main browser thread");
   return wasmFS.addBackend(std::make_unique<FetchBackend>(
     base_url ? base_url : "",
-    chunkSize != 0 ? chunkSize : DEFAULT_CHUNK_SIZE,
-    /* TODO manifest */
+    chunkSize ? chunkSize : DEFAULT_CHUNK_SIZE,
+    manifest,
     [](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); }));
   }
 
-const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_path(void* ptr) {
+const char* _wasmfs_fetch_get_file_path(void* ptr) {
   auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
   return file ? file->getPath().data() : nullptr;
 }
-const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_url(void* ptr) {
+
+const char* _wasmfs_fetch_get_file_url(void* ptr) {
   auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
   return file ? file->getURL().data() : nullptr;
 }
-uint32_t EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_chunk_size(void* ptr) {
+
+uint32_t _wasmfs_fetch_get_chunk_size(void* ptr) {
   auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
   return file ? file->getChunkSize() : DEFAULT_CHUNK_SIZE;
 }
+
+void *EMSCRIPTEN_KEEPALIVE wasmfs_fetch_create_manifest() {
+  return new FetchManifest();
+}
+
+void EMSCRIPTEN_KEEPALIVE wasmfs_fetch_add_to_manifest(void *manifest_ptr, const char *path, const char *url) {
+  auto* manifest = reinterpret_cast<FetchManifest *>(manifest_ptr);
+  auto path_str = std::string(path);
+  auto url_str = std::string(url);
+  manifest->insert(std::pair(path_str, url_str));
+}
+
 }
 
 } // namespace wasmfs
diff --git a/system/lib/wasmfs/backends/fetch_backend.h b/system/lib/wasmfs/backends/fetch_backend.h
@@ -6,6 +6,6 @@
 #include "wasmfs.h"
 
 extern "C" {
-  // See library_wasmfs_fetch.js
-  void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
+// See library_wasmfs_fetch.js
+void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
 }
diff --git a/test/common.py b/test/common.py
@@ -2013,7 +2013,7 @@ def send_head(self):
         f = open(path, 'rb')
         fs = os.fstat(f.fileno())
       except IOError:
-        self.send_error(404, "File not found: " + path)
+        self.send_error(404, f'File not found {path}')
         return None
       if self.path.endswith('.js'):
         self.send_response(200)
@@ -2022,16 +2022,16 @@ def send_head(self):
         self.send_header('Content-length', fs[6])
         self.end_headers()
         return f
-      elif self.headers.get("Range"):
+      elif self.headers.get('Range'):
         self.send_response(206)
         ctype = self.guess_type(path)
         self.send_header('Content-Type', ctype)
-        pieces = self.headers.get("Range").split("=")[1].split("-")
+        pieces = self.headers.get('Range').split('=')[1].split('-')
         start = int(pieces[0]) if pieces[0] != '' else 0
         end = int(pieces[1]) if pieces[1] != '' else fs[6] - 1
         end = min(fs[6] - 1, end)
         length = end - start + 1
-        self.send_header('Content-Range', "bytes " + str(start) + "-" + str(end) + "/" + str(fs[6]))
+        self.send_header('Content-Range', f'bytes {start}-{end}/{fs[6]}')
         self.send_header('Content-Length', str(length))
         self.end_headers()
         return f
@@ -2135,19 +2135,19 @@ def do_GET(self):
         # Use SimpleHTTPServer default file serving operation for GET.
         if DEBUG:
           print('[simple HTTP serving:', unquote_plus(self.path), ']')
-        if self.headers.get("Range"):
+        if self.headers.get('Range'):
           self.send_response(206)
           path = self.translate_path(self.path)
           data = read_binary(path)
           ctype = self.guess_type(path)
           self.send_header('Content-type', ctype)
-          pieces = self.headers.get("range").split("=")[1].split("-")
+          pieces = self.headers.get('Range').split('=')[1].split('-')
           start = int(pieces[0]) if pieces[0] != '' else 0
           end = int(pieces[1]) if pieces[1] != '' else len(data) - 1
           end = min(len(data) - 1, end)
           length = end - start + 1
           self.send_header('Content-Length', str(length))
-          self.send_header('Content-Range', "bytes " + str(start) + "-" + str(end) + "/" + str(len(data)))
+          self.send_header('Content-Range', f'bytes {start}-{end}/{len(data)}')
           self.end_headers()
           self.wfile.write(data[start:end + 1])
         else:
diff --git a/test/wasmfs/wasmfs_fetch.c b/test/wasmfs/wasmfs_fetch.c

Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,6 @@`
`6`	`6`	`#include "wasmfs.h"`
`7`	`7`
`8`	`8`	`extern "C" {`
`9`		`- // See library_wasmfs_fetch.js`
`10`		`- void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);`
	`9`	`+// See library_wasmfs_fetch.js`
	`10`	`+void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);`
`11`	`11`	`}`