chunked backend, works for single chunks and simple backend url mapping

Author: JoeOsborn

src/library_wasmfs_fetch.js

@@ -5,28 +5,24 @@
  */
 
 addToLibrary({
+  $wasmFS$JSMemoryRanges: {},
+
   // Fetch backend: On first access of the file (either a read or a getSize), it
   // will fetch() the data from the network asynchronously. Otherwise, after
   // that fetch it behaves just like JSFile (and it reuses the code from there).
 
   _wasmfs_create_fetch_backend_js__deps: [
     '$wasmFS$backends',
-    '$wasmFS$JSMemoryFiles',
-    '_wasmfs_create_js_file_backend_js',
-    '_wasmfs_fetch_get_file_path',
+    '$wasmFS$JSMemoryRanges',
+    '_wasmfs_fetch_get_file_url',
+    '_wasmfs_fetch_get_chunk_size',
   ],
   _wasmfs_create_fetch_backend_js: async function(backend) {
     // Get a promise that fetches the data and stores it in JS memory (if it has
     // not already been fetched).
-    async function getFile(file) {
-      if (wasmFS$JSMemoryFiles[file]) {
-        // The data is already here, so nothing to do before we continue on to
-        // the actual read below.
-        return Promise.resolve();
-      }
-      // This is the first time we want the file's data.
+    async function getFileRange(file, offset, len) {
       var url = '';
-      var fileUrl_p = __wasmfs_fetch_get_file_path(file);
+      var fileUrl_p = __wasmfs_fetch_get_file_url(file);
       var fileUrl = UTF8ToString(fileUrl_p);
       var isAbs = fileUrl.indexOf('://') !== -1;
       if (isAbs) {
@@ -38,31 +34,68 @@ addToLibrary({
         } catch (e) {
         }
       }
-      var response = await fetch(url);
+      var chunkSize = __wasmfs_fetch_get_chunk_size();
+      offset = offset || 0;
+      len = len || chunkSize;
+      var firstChunk = (offset / chunkSize) | 0;
+      var lastChunk = ((offset+len) / chunkSize) | 0;
+      if (!(file in wasmFS$JSMemoryRanges)) {
+        var fileInfo = await fetch(url,{method:"HEAD",headers:{"Range":"bytes=0-1"}});
+        if(fileInfo.ok && fileInfo.headers.has("Content-Length") && fileInfo.headers.get("Accept-Ranges") == "bytes" && (parseInt(fileInfo.headers.get("Content-Length")) > chunkSize*2)) {
+          wasmFS$JSMemoryRanges[file] = {size:parseInt(fileInfo.headers.get("Content-Length")), chunks:[], chunkSize:chunkSize};
+        } else {
+          // may as well/forced to download the whole file
+          var wholeFileReq = await fetch(url);
+          if(!wholeFileReq.ok) {
+            throw wholeFileReq;
+          }
+          var wholeFileData = new Uint8Array(await wholeFileReq.arrayBuffer());
+          var text = new TextDecoder().decode(wholeFileData);
+          wasmFS$JSMemoryRanges[file] = {size:wholeFileData.byteLength, chunks:[wholeFileData], chunkSize:wholeFileData.byteLength};
+          return Promise.resolve();
+        }
+      }
+      var allPresent = true;
+      var i;
+      if(lastChunk * chunkSize < offset+len) {
+        lastChunk += 1;
+      }
+      for(i = firstChunk; i < lastChunk; i++) {
+        if(!wasmFS$JSMemoryRanges[file].chunks[i]) {
+          allPresent = false;
+          break;
+        }
+      }
+      if (allPresent) {
+        // The data is already here, so nothing to do before we continue on to
+        // the actual read.
+        return Promise.resolve();
+      }
+      // This is the first time we want the chunk's data.
+      var start = firstChunk*chunkSize;
+      var end = lastChunk*chunkSize;
+      var response = await fetch(url, {headers:{"Range": `bytes=${start}-${end}`}});
       if (response.ok) {
-        var buffer = await response['arrayBuffer']();
-        wasmFS$JSMemoryFiles[file] = new Uint8Array(buffer);
+        var bytes = new Uint8Array(await response['arrayBuffer']());
+        for (i = firstChunk; i < lastChunk; i++) {
+          wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize,(i+1)*chunkSize);
+        }
       } else {
         throw response;
       }
+      return Promise.resolve();
     }
 
-    // Start with the normal JSFile operations. This sets
-    //   wasmFS$backends[backend]
-    // which we will then augment.
-    __wasmfs_create_js_file_backend_js(backend);
-
-    // Add the async operations on top.
-    var jsFileOps = wasmFS$backends[backend];
     wasmFS$backends[backend] = {
       // alloc/free operations are not actually async. Just forward to the
       // parent class, but we must return a Promise as the caller expects.
       allocFile: async (file) => {
-        jsFileOps.allocFile(file);
+        // nop
         return Promise.resolve();
       },
       freeFile: async (file) => {
-        jsFileOps.freeFile(file);
+        // free memory
+        wasmFS$JSMemoryRanges[file] = undefined;
         return Promise.resolve();
       },
 
@@ -73,17 +106,39 @@ addToLibrary({
       // read/getSize fetch the data, then forward to the parent class.
       read: async (file, buffer, length, offset) => {
         try {
-          await getFile(file);
+          await getFileRange(file, offset || 0, length);
         } catch (response) {
           return response.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}};
         }
-        return jsFileOps.read(file, buffer, length, offset);
+        var fileInfo = wasmFS$JSMemoryRanges[file];
+        var fileData = fileInfo.chunks;
+        var chunkSize = fileInfo.chunkSize;
+        var firstChunk = (offset / chunkSize) | 0;
+        var lastChunk = ((offset+length) / chunkSize) | 0;
+        if(offset + length > lastChunk * chunkSize) {
+          lastChunk += 1;
+        }
+        var readLength = 0;
+        for (var i = firstChunk; i < lastChunk; i++) {
+          var chunk = fileData[i];
+          var start = Math.max(i*chunkSize, offset);
+          if(!chunk) {
+            throw [fileData.length, firstChunk, lastChunk, i];
+          }
+          var end = Math.min(start+chunk.byteLength, offset+length);
+          var chunkStart = i*chunkSize;
+          HEAPU8.set(chunk.subarray(start-chunkStart, end-chunkStart), buffer+(start-offset));
+          readLength = end - offset;
+        }
+        return readLength;
       },
       getSize: async (file) => {
         try {
-          await getFile(file);
-        } catch (response) {}
-        return jsFileOps.getSize(file);
+          await getFileRange(file, 0, 0);
+        } catch (response) {
+          return 0;
+        }
+        return wasmFS$JSMemoryRanges[file].size;
       },
     };
   },

system/lib/wasmfs/backends/fetch_backend.cpp

@@ -13,17 +13,41 @@
 
 namespace wasmfs {
 
+const uint32_t DEFAULT_CHUNK_SIZE = 16*1024*1024;
+
+class FetchBackend : public wasmfs::ProxiedAsyncJSBackend {
+  std::string baseUrl;
+  uint32_t chunkSize;
+ public:
+  FetchBackend(const std::string& baseUrl,
+               uint32_t chunkSize,
+               std::function<void(backend_t)> setupOnThread)
+    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize)
+      // TODO manifest
+  {}
+  std::shared_ptr<DataFile> createFile(mode_t mode) override;
+  std::shared_ptr<Directory> createDirectory(mode_t mode) override;
+  const std::string getFileURL(const std::string& filePath);
+  uint32_t getChunkSize();
+};
+
+  
 class FetchFile : public ProxiedAsyncJSImplFile {
   std::string filePath;
+  std::string fileUrl;
 
 public:
   FetchFile(const std::string& path,
             mode_t mode,
             backend_t backend,
             emscripten::ProxyWorker& proxy)
-    : ProxiedAsyncJSImplFile(mode, backend, proxy), filePath(path) {}
+    : ProxiedAsyncJSImplFile(mode, backend, proxy), filePath(path) {
+    this->fileUrl = dynamic_cast<FetchBackend*>(getBackend())->getFileURL(filePath);
+  }
 
   const std::string& getPath() const { return filePath; }
+  const std::string& getURL() const { return fileUrl; }
+  const uint32_t getChunkSize() const { return dynamic_cast<FetchBackend*>(getBackend())->getChunkSize(); }
 };
 
 class FetchDirectory : public MemoryDirectory {
@@ -60,38 +84,50 @@ class FetchDirectory : public MemoryDirectory {
   }
 };
 
-class FetchBackend : public ProxiedAsyncJSBackend {
-  std::string baseUrl;
-
-public:
-  FetchBackend(const std::string& baseUrl,
-               std::function<void(backend_t)> setupOnThread)
-    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl) {}
+std::shared_ptr<DataFile> FetchBackend::createFile(mode_t mode) {
+  return std::make_shared<FetchFile>("", mode, this, proxy);
+}
 
-  std::shared_ptr<DataFile> createFile(mode_t mode) override {
-    return std::make_shared<FetchFile>(baseUrl, mode, this, proxy);
-  }
+std::shared_ptr<Directory> FetchBackend::createDirectory(mode_t mode) {
+  return std::make_shared<FetchDirectory>("", mode, this, proxy);
+}
 
-  std::shared_ptr<Directory> createDirectory(mode_t mode) override {
-    return std::make_shared<FetchDirectory>(baseUrl, mode, this, proxy);
+const std::string FetchBackend::getFileURL(const std::string& filePath) {
+  // TODO use manifest
+  if(filePath == "") {
+    return baseUrl;
   }
-};
+  return baseUrl + "/" + filePath;
+}
+uint32_t FetchBackend::getChunkSize() {
+  return chunkSize;
+}
 
 extern "C" {
-backend_t wasmfs_create_fetch_backend(const char* base_url) {
+  backend_t wasmfs_create_fetch_backend(const char* base_url /* TODO manifest */) {
   // ProxyWorker cannot safely be synchronously spawned from the main browser
   // thread. See comment in thread_utils.h for more details.
   assert(!emscripten_is_main_browser_thread() &&
          "Cannot safely create fetch backend on main browser thread");
   return wasmFS.addBackend(std::make_unique<FetchBackend>(
     base_url ? base_url : "",
+    DEFAULT_CHUNK_SIZE,
+    /* TODO manifest */
     [](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); }));
-}
+  }
 
 const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_path(void* ptr) {
   auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
   return file ? file->getPath().data() : nullptr;
 }
+const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_url(void* ptr) {
+  auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
+  return file ? file->getURL().data() : nullptr;
+}
+uint32_t EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_chunk_size(void* ptr) {
+  auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
+  return file ? file->getChunkSize() : DEFAULT_CHUNK_SIZE;
+}
 }
 
 } // namespace wasmfs

system/lib/wasmfs/backends/fetch_backend.h

@@ -6,7 +6,6 @@
 #include "wasmfs.h"
 
 extern "C" {
-
-// See library_wasmfs_fetch.js
-void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
+  // See library_wasmfs_fetch.js
+  void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
 }