add chunk size parameter for wasm fetchfs, fix bugs

JoeOsborn · JoeOsborn · commit dd04ca958946 · 2024-11-26T12:31:08.000-08:00
diff --git a/src/library_fetchfs.js b/src/library_fetchfs.js
@@ -8,7 +8,7 @@ addToLibrary({
   $FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'],
   $FETCHFS: {
     createBackend(opts) {
-      return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url));
+      return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0);
     }
   },
 });
diff --git a/src/library_wasmfs_fetch.js b/src/library_wasmfs_fetch.js
@@ -34,14 +34,17 @@ addToLibrary({
         } catch (e) {
         }
       }
-      var chunkSize = __wasmfs_fetch_get_chunk_size();
+      var chunkSize = __wasmfs_fetch_get_chunk_size(file);
       offset = offset || 0;
       len = len || chunkSize;
       var firstChunk = (offset / chunkSize) | 0;
       var lastChunk = ((offset+len) / chunkSize) | 0;
       if (!(file in wasmFS$JSMemoryRanges)) {
-        var fileInfo = await fetch(url,{method:"HEAD",headers:{"Range":"bytes=0-1"}});
-        if(fileInfo.ok && fileInfo.headers.has("Content-Length") && fileInfo.headers.get("Accept-Ranges") == "bytes" && (parseInt(fileInfo.headers.get("Content-Length")) > chunkSize*2)) {
+        var fileInfo = await fetch(url,{method:"HEAD", headers:{"Range": "bytes=0-"}});
+        if(fileInfo.ok &&
+           fileInfo.headers.has("Content-Length") &&
+           fileInfo.headers.get("Accept-Ranges") == "bytes" &&
+           (parseInt(fileInfo.headers.get("Content-Length")) > chunkSize*2)) {
           wasmFS$JSMemoryRanges[file] = {size:parseInt(fileInfo.headers.get("Content-Length")), chunks:[], chunkSize:chunkSize};
         } else {
           // may as well/forced to download the whole file
@@ -74,11 +77,11 @@ addToLibrary({
       // This is the first time we want the chunk's data.
       var start = firstChunk*chunkSize;
       var end = lastChunk*chunkSize;
-      var response = await fetch(url, {headers:{"Range": `bytes=${start}-${end}`}});
+      var response = await fetch(url, {headers:{"Range": `bytes=${start}-${end-1}`}});
       if (response.ok) {
         var bytes = new Uint8Array(await response['arrayBuffer']());
         for (i = firstChunk; i < lastChunk; i++) {
-          wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize,(i+1)*chunkSize);
+          wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start);
         }
       } else {
         throw response;
@@ -125,8 +128,8 @@ addToLibrary({
           if(!chunk) {
             throw [fileData.length, firstChunk, lastChunk, i];
           }
-          var end = Math.min(start+chunk.byteLength, offset+length);
           var chunkStart = i*chunkSize;
+          var end = Math.min(chunkStart+chunkSize, offset+length);
           HEAPU8.set(chunk.subarray(start-chunkStart, end-chunkStart), buffer+(start-offset));
           readLength = end - offset;
         }
diff --git a/system/include/emscripten/wasmfs.h b/system/include/emscripten/wasmfs.h
@@ -57,7 +57,7 @@ backend_t wasmfs_create_memory_backend(void);
 //
 // TODO: Add an async version of this function that will work on the main
 // thread.
-backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)));
+backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t);
 
 backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull)));
 
diff --git a/system/lib/wasmfs/backends/fetch_backend.cpp b/system/lib/wasmfs/backends/fetch_backend.cpp
@@ -104,14 +104,14 @@ uint32_t FetchBackend::getChunkSize() {
 }
 
 extern "C" {
-  backend_t wasmfs_create_fetch_backend(const char* base_url /* TODO manifest */) {
+  backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize /* TODO manifest */) {
   // ProxyWorker cannot safely be synchronously spawned from the main browser
   // thread. See comment in thread_utils.h for more details.
   assert(!emscripten_is_main_browser_thread() &&
          "Cannot safely create fetch backend on main browser thread");
   return wasmFS.addBackend(std::make_unique<FetchBackend>(
     base_url ? base_url : "",
-    DEFAULT_CHUNK_SIZE,
+    chunkSize != 0 ? chunkSize : DEFAULT_CHUNK_SIZE,
     /* TODO manifest */
     [](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); }));
   }
diff --git a/test/wasmfs/wasmfs_fetch.c b/test/wasmfs/wasmfs_fetch.c
@@ -42,7 +42,7 @@ void check_file(int fd, const char* content) {
 void test_url_relative() {
   printf("Running %s...\n", __FUNCTION__);
 
-  backend_t backend2 = wasmfs_create_fetch_backend("test.txt");
+  backend_t backend2 = wasmfs_create_fetch_backend("test.txt",0);
   int fd = wasmfs_create_file("/file_rel", 0777, backend2);
   check_file(fd, "fetch 2");
   assert(close(fd) == 0);
@@ -56,7 +56,7 @@ void test_url_absolute() {
   char url[200];
   snprintf(url, sizeof(url), "%s%s", url_orig, file_name);
 
-  backend_t backend = wasmfs_create_fetch_backend(url);
+  backend_t backend = wasmfs_create_fetch_backend(url,0);
   int fd = wasmfs_create_file(file_name, 0777, backend);
   check_file(fd, "fetch 2");
   assert(close(fd) == 0);
@@ -69,7 +69,7 @@ void test_directory_abs() {
   char url[200];
   snprintf(url, sizeof(url), "%s%s", url_orig, dir_path);
 
-  backend_t backend = wasmfs_create_fetch_backend(url);
+  backend_t backend = wasmfs_create_fetch_backend(url,0);
   int res = wasmfs_create_directory(dir_path, 0777, backend);
   if (errno)
     perror("wasmfs_create_directory");
@@ -100,7 +100,7 @@ void test_directory_abs() {
 
 void test_default() {
   printf("Running %s...\n", __FUNCTION__);
-  backend_t backend = wasmfs_create_fetch_backend("data.dat");
+  backend_t backend = wasmfs_create_fetch_backend("data.dat",0);
 
   // Create a file in that backend.
   int fd = wasmfs_create_file("/testfile", 0777, backend);
@@ -136,7 +136,7 @@ void test_small_reads() {
   char expected[] = "hello";
   size_t size = 5;
 
-  backend_t backend = wasmfs_create_fetch_backend("small.dat");
+  backend_t backend = wasmfs_create_fetch_backend("small.dat",0);
   int fd = wasmfs_create_file("/testfile3", 0777, backend);
   char buf[size + 1];
   for (size_t i = 0; i < size; i++) {
@@ -150,6 +150,108 @@ void test_small_reads() {
   assert(close(fd) == 0);
 }
 
+void test_small_chunks() {
+  // Read the file in small amounts.
+  printf("Running %s...\n", __FUNCTION__);
+
+  char expected[] = "hello";
+  size_t size = 5;
+
+  backend_t backend = wasmfs_create_fetch_backend("small.dat",2);
+  int fd;
+  char buf[size + 1];
+  fd = wasmfs_create_file("/testfile4", 0777, backend);
+  for (size_t i = 0; i < size; i+=1) {
+    int read_now = read(fd, buf + i, 1);
+    assert(read_now <= 1);
+    printf("read some bytes smaller than chunk size\n");
+  }
+  buf[size] = 0;
+  printf("buf %s\n",buf);
+  assert(strcmp(buf, "hello") == 0);
+
+  assert(close(fd) == 0);
+
+  fd = wasmfs_create_file("/testfile5", 0777, backend);
+  for (size_t i = 0; i < size; i+=2) {
+    int read_now = read(fd, buf + i, 2);
+    assert(read_now <= 2);
+    printf("read some bytes equal to chunk size\n");
+  }
+  buf[size] = 0;
+  printf("buf %s\n",buf);
+  assert(strcmp(buf, "hello") == 0);
+
+  assert(close(fd) == 0);
+
+  fd = wasmfs_create_file("/testfile6", 0777, backend);
+  for (size_t i = 0; i < size; i+=5) {
+    int read_now = read(fd, buf + i, 5);
+    assert(read_now <= 5);
+    printf("read some bytes much larger than chunk size\n");
+  }
+  buf[size] = 0;
+  printf("buf %s\n",buf);
+  assert(strcmp(buf, "hello") == 0);
+
+  assert(close(fd) == 0);
+}
+
+void test_small_chunks_divisor_of_size() {
+  printf("Running %s...\n", __FUNCTION__);
+
+  char expected[] = "hello, fetch";
+  size_t size = 12;
+
+  backend_t backend = wasmfs_create_fetch_backend("data.dat",4);
+  int fd;
+  char buf[size + 1];
+  fd = wasmfs_create_file("/testfile7", 0777, backend);
+  for (size_t i = 0; i < size; i+=3) {
+    int read_now = read(fd, buf + i, 3);
+    assert(read_now <= 3);
+    printf("read some bytes less than chunk size\n");
+  }
+  buf[size] = 0;
+  printf("buf %s\n",buf);
+  assert(strcmp(buf, "hello, fetch") == 0);
+  assert(close(fd) == 0);
+
+  fd = wasmfs_create_file("/testfile8", 0777, backend);
+  for (size_t i = 0; i < size; i+=4) {
+    int read_now = read(fd, buf + i, 4);
+    assert(read_now <= 4);
+    printf("read some bytes equal to chunk size\n");
+  }
+  buf[size] = 0;
+  printf("buf %s\n",buf);
+  assert(strcmp(buf, "hello, fetch") == 0);
+
+  assert(close(fd) == 0);
+
+  fd = wasmfs_create_file("/testfile9", 0777, backend);
+  for (size_t i = 0; i < size; i+=5) {
+    int read_now = read(fd, buf + i, 5);
+    assert(read_now <= 5);
+    printf("read some bytes greater than chunk size\n");
+  }
+  buf[size] = 0;
+  printf("buf %s\n",buf);
+  assert(strcmp(buf, "hello, fetch") == 0);
+
+  assert(close(fd) == 0);
+
+  fd = wasmfs_create_file("/testfile10", 0777, backend);
+  int read_now = read(fd, buf, 12);
+  assert(read_now == 12);
+  printf("read some bytes much greater than chunk size\n");
+  buf[size] = 0;
+  printf("buf %s\n",buf);
+  assert(strcmp(buf, "hello, fetch") == 0);
+
+  assert(close(fd) == 0);
+}
+
 void test_nonexistent() {
   printf("Running %s...\n", __FUNCTION__);
 
@@ -185,6 +287,8 @@ int main() {
   test_url_absolute();
   test_directory_abs();
   test_small_reads();
+  test_small_chunks();
+  test_small_chunks_divisor_of_size();
   test_nonexistent();
 
   return 0;

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@ addToLibrary({`
`8`	`8`	`$FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'],`
`9`	`9`	`$FETCHFS: {`
`10`	`10`	`createBackend(opts) {`
`11`		`- return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url));`
	`11`	`+ return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize \| 0);`
`12`	`12`	`}`
`13`	`13`	`},`
`14`	`14`	`});`