WasmFS JS API: Implement mount and unmount (#19825)

jameshu15869 · web-flow · commit 48e8a04805e1 · 2023-08-01T13:22:07.000-07:00
This PR implements FS.mount and FS.unmount. There were existing tests, so I just added some additional test cases to those files. Tests that now pass in WasmFS mode were also added to CI. There are some differences in the behavior of the WasmFS mount and the legacy API. For example, WasmFS mount will create a new directory to mount to if one does not already exist. Existing directories must be nonempty, since WasmFS will unlink the directory before creating a new directory with the specified backend. Currently, WasmFS does not support mounting IDBFS, WORKERFS, and PROXYFS, although those may be added in the future using a bridge to legacy backends (#18935)
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -544,13 +544,29 @@ jobs:
             wasmfs.test_readdir_unlink
             wasmfs.test_unistd_pipe
             wasmfs.test_fs_write
+            wasmfs.test_fs_writev
+            wasmfs.test_fs_writev_rawfs
+            wasmfs.test_fs_writeFile
+            wasmfs.test_fs_writeFile_rawfs
+            wasmfs.test_fs_readv
+            wasmfs.test_fs_write
+            wasmfs.test_fs_readv_rawfs
+            wasmfs.test_fs_nodefs_nofollow
+            wasmfs.test_fs_nodefs_readdir
+            wasmfs.test_fs_nodefs_home
+            wasmfs.test_fs_nodefs_cloexec
+            wasmfs.test_fs_nodefs_cloexec_rawfs
+            wasmfs.test_fs_errorstack
+            wasmfs.test_fs_errorstack_rawfs
+            wasmfs.test_fs_emptyPath
             wasmfs.test_webidl
             wasmfs.test_dlfcn_self
             wasmfs.test_dlfcn_unique_sig
             wasmfs.test_dylink_basics
             wasmfs.test_exit_status
             wasmfs.test_minimal_runtime_memorygrowth
             wasmfs.test_mmap_anon*
+            wasmfs.test_mount
             wasmfs.test_getcwd_with_non_ascii_name
             wasmfs.test_stat
             wasmfs.test_fstatat
@@ -559,6 +575,7 @@ jobs:
             wasmfs.test_fcntl_open
             wasmfs.test_fs_js_api
             wasmfs.test_fs_llseek
+            wasmfs.test_fs_llseek_rawfs
             wasmfs.test_freetype"
   test-wasm2js1:
     environment:
diff --git a/emcc.py b/emcc.py
@@ -2339,6 +2339,8 @@ def phase_linker_setup(options, state, newargs):
       # included, as the entire JS library can refer to things that require
       # these exports.)
       settings.REQUIRED_EXPORTS += [
+        '_wasmfs_mount',
+        '_wasmfs_unmount',
         '_wasmfs_read_file',
         '_wasmfs_write_file',
         '_wasmfs_open',
diff --git a/src/library_fetchfs.js b/src/library_fetchfs.js
@@ -0,0 +1,18 @@
+/**
+ * @license
+ * Copyright 2023 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+mergeInto(LibraryManager.library, {
+  $FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'],
+  $FETCHFS: {
+    createBackend(opts) {
+      return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url));
+    }
+  },
+});
+
+if (!WASMFS) {
+  error("using -lfetchfs.js requires using WasmFS (-sWASMFS)");
+}
diff --git a/src/library_icasefs.js b/src/library_icasefs.js
@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2023 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+mergeInto(LibraryManager.library, {
+  $ICASEFS__deps: ['wasmfs_create_icase_backend'],
+  $ICASEFS: {
+    createBackend(opts) {
+      if (typeof opts.backend === "undefined") {
+        throw new Error("Underlying backend is not valid.");
+      }
+      var underlyingBackend = opts.backend.createBackend(opts);
+      return _wasmfs_create_icase_backend(underlyingBackend);
+    }
+  },
+});
+
+if (!WASMFS) {
+  error("using -licasefs.js requires using WasmFS (-sWASMFS)");
+}
diff --git a/src/library_idbfs.js b/src/library_idbfs.js
@@ -312,3 +312,7 @@ mergeInto(LibraryManager.library, {
     }
   }
 });
+
+if (WASMFS) {
+  error("using -lidbfs is not currently supported in WasmFS.");
+}
diff --git a/src/library_jsfilefs.js b/src/library_jsfilefs.js
@@ -0,0 +1,18 @@
+/**
+ * @license
+ * Copyright 2023 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+mergeInto(LibraryManager.library, {
+  $JSFILEFS__deps: ['wasmfs_create_js_file_backend'],
+  $JSFILEFS: {
+    createBackend(opts) {
+      return _wasmfs_create_js_file_backend();
+    }
+  },
+});
+
+if (!WASMFS) {
+  error("using -ljsfile.js requires using WasmFS (-sWASMFS)");
+}
diff --git a/src/library_nodefs.js b/src/library_nodefs.js
@@ -5,6 +5,14 @@
  */
 
 mergeInto(LibraryManager.library, {
+#if WASMFS
+  $NODEFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_node_backend'],
+  $NODEFS: {
+    createBackend(opts) {
+      return _wasmfs_create_node_backend(stringToUTF8OnStack(opts.root));
+    }
+  }
+#else
   $NODEFS__deps: ['$FS', '$PATH', '$ERRNO_CODES', '$mmapAlloc'],
   $NODEFS__postset: 'if (ENVIRONMENT_IS_NODE) { NODEFS.staticInit(); }',
   $NODEFS: {
@@ -314,4 +322,5 @@ mergeInto(LibraryManager.library, {
       }
     }
   }
+#endif
 });
diff --git a/src/library_opfs.js b/src/library_opfs.js
@@ -0,0 +1,18 @@
+/**
+ * @license
+ * Copyright 2023 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+mergeInto(LibraryManager.library, {
+  $OPFS__deps: ['wasmfs_create_opfs_backend'],
+  $OPFS: {
+    createBackend(opts) {
+      return _wasmfs_create_opfs_backend();
+    }
+  },
+});
+
+if (!WASMFS) {
+  error("using -lopfs.js requires using WasmFS (-sWASMFS)");
+}
diff --git a/src/library_proxyfs.js b/src/library_proxyfs.js
@@ -218,3 +218,7 @@ mergeInto(LibraryManager.library, {
     }
   }
 });
+
+if (WASMFS) {
+  error("using -lproxyfs is not currently supported in WasmFS.");
+}
diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js
@@ -5,6 +5,12 @@
  */
 
 mergeInto(LibraryManager.library, {
+  $MEMFS__deps: ['wasmfs_create_memory_backend'],
+  $MEMFS: {
+    createBackend(opts) {
+      return _wasmfs_create_memory_backend();
+    }
+  },
   $wasmFSPreloadedFiles: [],
   $wasmFSPreloadedDirs: [],
   // We must note when preloading has been "flushed", that is, the time at which
@@ -18,6 +24,7 @@ FS.init();
 FS.createPreloadedFile = FS_createPreloadedFile;
 `,
   $FS__deps: [
+    '$MEMFS',
     '$wasmFSPreloadedFiles',
     '$wasmFSPreloadedDirs',
     '$wasmFSPreloadingFlushed',
@@ -35,6 +42,21 @@ FS.createPreloadedFile = FS_createPreloadedFile;
                                              // up requiring all of our code
                                              // here.
     '$FS_modeStringToFlags',
+#if LibraryManager.has('library_icasefs.js')
+    '$ICASEFS',
+#endif
+#if LibraryManager.has('library_nodefs.js')
+    '$NODEFS',
+#endif
+#if LibraryManager.has('library_opfs.js')
+    '$OPFS',
+#endif
+#if LibraryManager.has('library_jsfilefs.js')
+    '$JSFILEFS',
+#endif
+#if LibraryManager.has('library_fetchfs.js')
+    '$FETCHFS',
+#endif
     'malloc',
     'free',
 #endif
@@ -349,8 +371,20 @@ FS.createPreloadedFile = FS_createPreloadedFile;
       __wasmfs_readdir_finish(state);
       return entries;
     }),
-    // TODO: mount
-    // TODO: unmount
+    mount: (type, opts, mountpoint) => {
+#if ASSERTIONS
+      if (typeof type == 'string') {
+        // The filesystem was not included, and instead we have an error
+        // message stored in the variable.
+        throw type;
+      }
+#endif
+      var backendPointer = type.createBackend(opts);
+      return FS.handleError(withStackSave(() => __wasmfs_mount(stringToUTF8OnStack(mountpoint), backendPointer)));
+    },
+    unmount: (mountpoint) => (
+      FS.handleError(withStackSave(() => __wasmfs_unmount(stringToUTF8OnStack(mountpoint))))
+    ),
     // TODO: lookup
     mknod(path, mode, dev) {
       return FS.handleError(withStackSave(() => {
diff --git a/src/library_workerfs.js b/src/library_workerfs.js
@@ -156,3 +156,7 @@ mergeInto(LibraryManager.library, {
     },
   },
 });
+
+if (WASMFS) {
+  error("using -lworkerfs is not currently supported in WasmFS.");
+}
diff --git a/src/shell.js b/src/shell.js
@@ -485,6 +485,11 @@ assert(typeof Module['TOTAL_MEMORY'] == 'undefined', 'Module.TOTAL_MEMORY has be
 {{{ makeRemovedFSAssert('IDBFS') }}}
 {{{ makeRemovedFSAssert('PROXYFS') }}}
 {{{ makeRemovedFSAssert('WORKERFS') }}}
+{{{ makeRemovedFSAssert('FETCHFS') }}}
+{{{ makeRemovedFSAssert('ICASEFS') }}}
+{{{ makeRemovedFSAssert('JSFILEFS') }}}
+{{{ makeRemovedFSAssert('OPFS') }}}
+
 #if !NODERAWFS
 {{{ makeRemovedFSAssert('NODEFS') }}}
 #endif
diff --git a/system/include/emscripten/wasmfs.h b/system/include/emscripten/wasmfs.h
@@ -26,12 +26,18 @@ backend_t wasmfs_get_backend_by_fd(int fd);
 // Returns the file descriptor for the new file like `open`. Returns a negative
 // value on error. TODO: It might be worth returning a more specialized type
 // like __wasi_fd_t here.
+// TODO: Remove this function so that only directories can be mounted.
 int wasmfs_create_file(const char* pathname __attribute__((nonnull)), mode_t mode, backend_t backend);
 
 // Creates a new directory in the new file system under a specific backend.
 // Returns 0 on success like `mkdir`, or a negative value on error.
+// TODO: Add an alias with wasmfs_mount.
 int wasmfs_create_directory(const char* path __attribute__((nonnull)), mode_t mode, backend_t backend);
 
+// Unmounts the directory (Which must be a valid mountpoint) at a specific path.
+// Returns 0 on success, or a negative value on error.
+int wasmfs_unmount(intptr_t path);
+
 // Backend creation
 
 // Creates a JSFile Backend in the new file system.
@@ -66,8 +72,7 @@ backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull)));
 // thread.
 backend_t wasmfs_create_opfs_backend(void);
 
-backend_t wasmfs_create_icase_backend(backend_constructor_t create_backend,
-                                      void* arg);
+backend_t wasmfs_create_icase_backend(backend_t backend);
 
 // Similar to fflush(0), but also flushes all internal buffers inside WasmFS.
 // This is necessary because in a Web environment we must buffer at an
diff --git a/system/lib/wasmfs/backends/ignore_case_backend.cpp b/system/lib/wasmfs/backends/ignore_case_backend.cpp
@@ -234,11 +234,11 @@ backend_t createIgnoreCaseBackend(std::function<backend_t()> createBackend) {
 
 extern "C" {
 
-// C API for creating ignore case backend.
-backend_t wasmfs_create_icase_backend(backend_constructor_t create_backend,
-                                      void* arg) {
+// C API FOR creating an ignore case backend by supplying a pointer to another backend.
+backend_t wasmfs_create_icase_backend(backend_t backend) {
   return createIgnoreCaseBackend(
-    [create_backend, arg]() { return create_backend(arg); });
+    [backend]() { return backend; }
+  );
 }
 
 } // extern "C"
diff --git a/system/lib/wasmfs/js_api.cpp b/system/lib/wasmfs/js_api.cpp
@@ -22,11 +22,16 @@ using namespace wasmfs;
 
 extern "C" {
 
+// TODO: Replace forward declarations with #include <emscripten/wasmfs.h> and 
+// resolve wasmfs::backend_t namespace conflicts.
 __wasi_fd_t wasmfs_create_file(char* pathname, mode_t mode, backend_t backend);
+int wasmfs_create_directory(char* path, int mode, backend_t backend);
+int wasmfs_unmount(intptr_t path);
 
 // Copy the file specified by the pathname into JS.
 // Return a pointer to the JS buffer in HEAPU8.
 // The buffer will also contain the file length.
+// TODO: Use WasmFS ErrnoError handling instead of aborting on failure.
 void* _wasmfs_read_file(char* path) {
   static_assert(sizeof(off_t) == 8, "File offset type must be 64-bit");
 
@@ -207,7 +212,7 @@ int _wasmfs_llseek(int fd, off_t offset, int whence) {
 
 int _wasmfs_rename(char* oldpath, char* newpath) {
   return __syscall_renameat(AT_FDCWD, (intptr_t)oldpath, AT_FDCWD, (intptr_t)newpath);
-};
+}
 
 int _wasmfs_read(int fd, void *buf, size_t count) {
   __wasi_iovec_t iovs[1];
@@ -265,6 +270,24 @@ int _wasmfs_lstat(char* path, struct stat* statBuf) {
   return __syscall_lstat64((intptr_t)path, (intptr_t)statBuf);
 }
 
+// The legacy JS API requires a mountpoint to already exist, so  WasmFS will attempt to remove 
+// the target directory if it exists before replacing it with a mounted directory.
+int _wasmfs_mount(char* path, wasmfs::backend_t created_backend) {
+  int err = __syscall_rmdir((intptr_t)path);
+
+  // The legacy JS API mount requires the directory to already exist, but we will also allow it to be missing.
+  if (err && err != -ENOENT) {
+    return err;
+  }
+
+  return wasmfs_create_directory(path, 0777, created_backend);
+}
+
+// WasmFS will always remove the mounted directory, regardless of if the directory existed before.
+int _wasmfs_unmount(char* path) {
+  return wasmfs_unmount((intptr_t)path);
+}
+
 // Helper method that identifies what a path is:
 //   ENOENT - if nothing exists there
 //   EISDIR - if it is a directory
diff --git a/system/lib/wasmfs/syscalls.cpp b/system/lib/wasmfs/syscalls.cpp
@@ -861,6 +861,39 @@ int __syscall_rmdir(intptr_t path) {
   return __syscall_unlinkat(AT_FDCWD, path, AT_REMOVEDIR);
 }
 
+// wasmfs_unmount is similar to __syscall_unlinkat, but assumes AT_REMOVEDIR is true
+// and will only unlink mountpoints (Empty and nonempty).
+int wasmfs_unmount(intptr_t path) {
+  auto parsed = path::parseParent((char*)path, AT_FDCWD);
+  if (auto err = parsed.getError()) {
+    return err;
+  }
+  auto& [parent, childNameView] = parsed.getParentChild();
+  std::string childName(childNameView);
+  auto lockedParent = parent->locked();
+  auto file = lockedParent.getChild(childName);
+  if (!file) {
+    return -ENOENT;
+  }
+  // Disallow removing the root directory, even if it is empty.
+  if (file == wasmFS.getRootDirectory()) {
+    return -EBUSY;
+  }
+
+  if (auto dir = file->dynCast<Directory>()) {
+    if (parent->getBackend() == dir->getBackend()) {
+      // The child is not a valid mountpoint.
+      return -EINVAL;
+    }
+  } else {
+    // A normal file or symlink.
+    return -ENOTDIR;
+  }
+
+  // Input is valid, perform the unlink.
+  return lockedParent.removeChild(childName);
+}
+
 int __syscall_getdents64(int fd, intptr_t dirp, size_t count) {
   dirent* result = (dirent*)dirp;
 
diff --git a/test/fs/test_fs_js_api.c b/test/fs/test_fs_js_api.c
@@ -1,3 +1,10 @@
+/*
+ * Copyright 2023 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
 #include <emscripten/emscripten.h>
 #include <stdio.h>
 #include <unistd.h>
diff --git a/test/fs/test_mount.c b/test/fs/test_mount.c
diff --git a/test/test_core.py b/test/test_core.py

Original file line number	Diff line number	Diff line change
`@@ -312,3 +312,7 @@ mergeInto(LibraryManager.library, {`
`312`	`312`	`}`
`313`	`313`	`}`
`314`	`314`	`});`
	`315`	`+`
	`316`	`+if (WASMFS) {`
	`317`	`+ error("using -lidbfs is not currently supported in WasmFS.");`
	`318`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,14 @@`
`5`	`5`	`*/`
`6`	`6`
`7`	`7`	`mergeInto(LibraryManager.library, {`
	`8`	`+#if WASMFS`
	`9`	`+ $NODEFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_node_backend'],`
	`10`	`+ $NODEFS: {`
	`11`	`+ createBackend(opts) {`
	`12`	`+ return _wasmfs_create_node_backend(stringToUTF8OnStack(opts.root));`
	`13`	`+ }`
	`14`	`+ }`
	`15`	`+#else`
`8`	`16`	`$NODEFS__deps: ['$FS', '$PATH', '$ERRNO_CODES', '$mmapAlloc'],`
`9`	`17`	`$NODEFS__postset: 'if (ENVIRONMENT_IS_NODE) { NODEFS.staticInit(); }',`
`10`	`18`	`$NODEFS: {`
`@@ -314,4 +322,5 @@ mergeInto(LibraryManager.library, {`
`314`	`322`	`}`
`315`	`323`	`}`
`316`	`324`	`}`
	`325`	`+#endif`
`317`	`326`	`});`
Original file line number	Diff line number	Diff line change
`@@ -218,3 +218,7 @@ mergeInto(LibraryManager.library, {`
`218`	`218`	`}`
`219`	`219`	`}`
`220`	`220`	`});`
	`221`	`+`
	`222`	`+if (WASMFS) {`
	`223`	`+ error("using -lproxyfs is not currently supported in WasmFS.");`
	`224`	`+}`