Support manifest for WASMFS fetchfs

This way, a fetchfs WASMFS backend can have a mapping from paths to
URLs used to satisfy those paths.  I think this makes more sense than
creating a separate fetchfs backend for each individual file.

In the absence of a manifest, the old behavior of using the file path
as a relative path against the baseURL will still be used.

NOTE, it's important that the fetchfs have a dedicated directory
created/"mounted" or else the "create file with path" code path will
never be used.  In other words, paths at `/whatever.txt` can never
work given the way WASMFS is implemented; the accessed files don't get
paths unless the parent directory and child path have the same
backend.  So, creating the directory at `/dat` and having a manifest
entry like `/whatever.txt` is the way to go for a file at
`/dat/whatever.txt`.

Author: JoeOsborn

src/library_fetchfs.js

@@ -8,7 +8,7 @@ addToLibrary({
   $FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'],
   $FETCHFS: {
     createBackend(opts) {
-      return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0);
+      return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0, 0);
     }
   },
 });

system/include/emscripten/wasmfs.h

@@ -57,7 +57,10 @@ backend_t wasmfs_create_memory_backend(void);
 //
 // TODO: Add an async version of this function that will work on the main
 // thread.
-backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t);
+backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t, void *);
+
+void *wasmfs_fetch_create_manifest();
+void wasmfs_fetch_add_to_manifest(void *, const char *, const char *);
 
 backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull)));
 

system/lib/wasmfs/backends/fetch_backend.cpp

@@ -15,23 +15,32 @@ namespace wasmfs {
 
 const uint32_t DEFAULT_CHUNK_SIZE = 16*1024*1024;
 
+typedef std::map<std::string,std::string> FetchManifest;
+
 class FetchBackend : public wasmfs::ProxiedAsyncJSBackend {
   std::string baseUrl;
   uint32_t chunkSize;
+  FetchManifest *manifest;
  public:
+  // Takes ownership of manifest
   FetchBackend(const std::string& baseUrl,
                uint32_t chunkSize,
+               FetchManifest *manifest,
                std::function<void(backend_t)> setupOnThread)
-    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize)
-      // TODO manifest
+    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize), manifest(manifest)
   {}
+  ~FetchBackend() {
+    if(manifest != NULL) {
+      delete manifest;
+    }
+  }
   std::shared_ptr<DataFile> createFile(mode_t mode) override;
   std::shared_ptr<Directory> createDirectory(mode_t mode) override;
   const std::string getFileURL(const std::string& filePath);
   uint32_t getChunkSize();
 };
 
-  
+
 class FetchFile : public ProxiedAsyncJSImplFile {
   std::string filePath;
   std::string fileUrl;
@@ -93,7 +102,11 @@ std::shared_ptr<Directory> FetchBackend::createDirectory(mode_t mode) {
 }
 
 const std::string FetchBackend::getFileURL(const std::string& filePath) {
-  // TODO use manifest
+  if(manifest) {
+    if (auto search = manifest->find(filePath); search != manifest->end()) {
+      return baseUrl + "/" + search->second;
+    }
+  }
   if(filePath == "") {
     return baseUrl;
   }
@@ -104,15 +117,15 @@ uint32_t FetchBackend::getChunkSize() {
 }
 
 extern "C" {
-  backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize /* TODO manifest */) {
+  backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize, FetchManifest *manifest) {
   // ProxyWorker cannot safely be synchronously spawned from the main browser
   // thread. See comment in thread_utils.h for more details.
   assert(!emscripten_is_main_browser_thread() &&
          "Cannot safely create fetch backend on main browser thread");
   return wasmFS.addBackend(std::make_unique<FetchBackend>(
     base_url ? base_url : "",
     chunkSize != 0 ? chunkSize : DEFAULT_CHUNK_SIZE,
-    /* TODO manifest */
+    manifest,
     [](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); }));
   }
 
@@ -128,6 +141,17 @@ uint32_t EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_chunk_size(void* ptr) {
   auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
   return file ? file->getChunkSize() : DEFAULT_CHUNK_SIZE;
 }
+
+void *EMSCRIPTEN_KEEPALIVE wasmfs_fetch_create_manifest() {
+  return new FetchManifest();
+}
+void EMSCRIPTEN_KEEPALIVE wasmfs_fetch_add_to_manifest(void *manifest_ptr, const char *path, const char *url) {
+  auto* manifest = reinterpret_cast<FetchManifest *>(manifest_ptr);
+  auto path_str = std::string(path);
+  auto url_str = std::string(url);
+  manifest->insert(std::pair(path_str, url_str));
+}
+
 }
 
 } // namespace wasmfs

test/wasmfs/wasmfs_fetch.c

@@ -42,12 +42,24 @@ void check_file(int fd, const char* content) {
 void test_url_relative() {
   printf("Running %s...\n", __FUNCTION__);
 
-  backend_t backend2 = wasmfs_create_fetch_backend("test.txt",0);
+  backend_t backend2 = wasmfs_create_fetch_backend("test.txt",0,NULL);
   int fd = wasmfs_create_file("/file_rel", 0777, backend2);
   check_file(fd, "fetch 2");
   assert(close(fd) == 0);
 }
 
+void test_manifest() {
+  printf("Running %s...\n", __FUNCTION__);
+  void *manifest = wasmfs_fetch_create_manifest();
+  wasmfs_fetch_add_to_manifest(manifest, "/file", "test.txt");
+  backend_t backend = wasmfs_create_fetch_backend("",0,manifest);
+  // Create a directory with the fetchfs backend
+  wasmfs_create_directory("/dat", 0777, backend);
+  int fd = wasmfs_create_file("/dat/file", 0777, backend);
+  check_file(fd, "fetch 2");
+  assert(close(fd) == 0);
+}
+
 void test_url_absolute() {
   printf("Running %s...\n", __FUNCTION__);
 
@@ -56,7 +68,7 @@ void test_url_absolute() {
   char url[200];
   snprintf(url, sizeof(url), "%s%s", url_orig, file_name);
 
-  backend_t backend = wasmfs_create_fetch_backend(url,0);
+  backend_t backend = wasmfs_create_fetch_backend(url,0,NULL);
   int fd = wasmfs_create_file(file_name, 0777, backend);
   check_file(fd, "fetch 2");
   assert(close(fd) == 0);
@@ -69,7 +81,7 @@ void test_directory_abs() {
   char url[200];
   snprintf(url, sizeof(url), "%s%s", url_orig, dir_path);
 
-  backend_t backend = wasmfs_create_fetch_backend(url,0);
+  backend_t backend = wasmfs_create_fetch_backend(url,0,NULL);
   int res = wasmfs_create_directory(dir_path, 0777, backend);
   if (errno)
     perror("wasmfs_create_directory");
@@ -100,7 +112,7 @@ void test_directory_abs() {
 
 void test_default() {
   printf("Running %s...\n", __FUNCTION__);
-  backend_t backend = wasmfs_create_fetch_backend("data.dat",0);
+  backend_t backend = wasmfs_create_fetch_backend("data.dat",0,NULL);
 
   // Create a file in that backend.
   int fd = wasmfs_create_file("/testfile", 0777, backend);
@@ -136,7 +148,7 @@ void test_small_reads() {
   char expected[] = "hello";
   size_t size = 5;
 
-  backend_t backend = wasmfs_create_fetch_backend("small.dat",0);
+  backend_t backend = wasmfs_create_fetch_backend("small.dat",0,NULL);
   int fd = wasmfs_create_file("/testfile3", 0777, backend);
   char buf[size + 1];
   for (size_t i = 0; i < size; i++) {
@@ -157,7 +169,7 @@ void test_small_chunks() {
   char expected[] = "hello";
   size_t size = 5;
 
-  backend_t backend = wasmfs_create_fetch_backend("small.dat",2);
+  backend_t backend = wasmfs_create_fetch_backend("small.dat",2,NULL);
   int fd;
   char buf[size + 1];
   fd = wasmfs_create_file("/testfile4", 0777, backend);
@@ -203,7 +215,7 @@ void test_small_chunks_divisor_of_size() {
   char expected[] = "hello, fetch";
   size_t size = 12;
 
-  backend_t backend = wasmfs_create_fetch_backend("data.dat",4);
+  backend_t backend = wasmfs_create_fetch_backend("data.dat",4,NULL);
   int fd;
   char buf[size + 1];
   fd = wasmfs_create_file("/testfile7", 0777, backend);
@@ -283,6 +295,7 @@ void test_nonexistent() {
 int main() {
   getUrlOrigin(url_orig, sizeof(url_orig));
   test_default();
+  test_manifest();
   test_url_relative();
   test_url_absolute();
   test_directory_abs();