diff --git a/src/lib/libmapfs.js b/src/lib/libmapfs.js new file mode 100644 index 0000000000000..aa5bd9958a474 --- /dev/null +++ b/src/lib/libmapfs.js @@ -0,0 +1,29 @@ +/** + * @license + * Copyright 2025 The Emscripten Authors + * SPDX-License-Identifier: MIT + */ + +addToLibrary({ + $MAPFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_map_backend', 'wasmfs_map_create_manifest', 'wasmfs_map_add_to_manifest'], + $MAPFS: { + createBackend(opts) { + var manifest = 0; + if (opts['manifest']) { + manifest = _wasmfs_map_create_manifest(); + Object.entries(opts['manifest']).forEach(([path, dest]) => { + withStackSave(() => { + _wasmfs_map_add_to_manifest(manifest, + stringToUTF8OnStack(path), + stringToUTF8OnStack(dest)); + }); + }); + return _wasmfs_create_map_backend(manifest); + }; + }, + }, +}); + +if (!WASMFS) { + error("using -lmapfs.js requires using WasmFS (-sWASMFS)"); +} diff --git a/src/lib/libwasmfs.js b/src/lib/libwasmfs.js index 63a14510767bd..db584316a26a9 100644 --- a/src/lib/libwasmfs.js +++ b/src/lib/libwasmfs.js @@ -61,6 +61,9 @@ addToLibrary({ #endif #if LibraryManager.has('libfetchfs.js') '$FETCHFS', +#endif +#if LibraryManager.has('libmapfs.js') + '$MAPFS', #endif 'malloc', 'free', diff --git a/system/include/emscripten/wasmfs.h b/system/include/emscripten/wasmfs.h index f7025dc6eb6cb..94dd229b75de5 100644 --- a/system/include/emscripten/wasmfs.h +++ b/system/include/emscripten/wasmfs.h @@ -78,6 +78,45 @@ backend_t wasmfs_create_memory_backend(void); backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t chunk_size); +// Mapping backend +// +// Creates a new path mapping backend based on a provided `manifest`, +// which is a mapping from file paths (relative to where the mapfs is +// mounted, but starting with a leading /) to absolute file paths. +// This is useful if your C program expects files at certain +// locations, but might not handle symlinks properly. It can also be +// combined with e.g. fetchfs, in case your web server has files at +// several different paths but the C program wants to see all the +// files within one directory. The manifest can be built using +// `wasmfs_map_create_manifest` and `wasmfs_map_add_to_manifest`. + +// When the mapfs is mounted, directories and files corresponding to +// the manifest entries will be automatically created. Importantly, +// only individual files can be mapped through mapfs, not whole +// directories---a manifest can't map one directory to another, just +// one file to another. +// +// For example, a manifest like `{'/test.txt':'/resources/file.txt'}` +// mounted at `/dat` (using either `wasmfs_mount` or +// `wasmfs_create_directory`) will let you write +// `open("/dat/test.txt", O_RDONLY)` and get the contents of +// `/resources/file.txt`. +// +typedef struct MapManifest *manifest_t; +backend_t wasmfs_create_map_backend(manifest_t manifest __attribute__((nonnull))); + +// Create a MapFS manifest record that can be populated with +// wasmfs_map_add_to_manifest and passed into +// wasmfs_map_create_backend. +manifest_t wasmfs_map_create_manifest(); + +// Add a path-to-URL mapping to the given manifest. +void wasmfs_map_add_to_manifest(manifest_t manifest __attribute__((nonnull)), + const char *from_path __attribute__((nonnull)), + const char *to_path __attribute__((nonnull))); + + + backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull))); // Note: this cannot be called on the browser main thread because it might diff --git a/system/lib/wasmfs/backends/map_backend.cpp b/system/lib/wasmfs/backends/map_backend.cpp new file mode 100644 index 0000000000000..6bf29c716d1f6 --- /dev/null +++ b/system/lib/wasmfs/backends/map_backend.cpp @@ -0,0 +1,171 @@ +// Copyright 2025 The Emscripten Authors. All rights reserved. +// Emscripten is available under two separate licenses, the MIT license and the +// University of Illinois/NCSA Open Source License. Both these licenses can be +// found in the LICENSE file. + +// This file defines the MAPFS backend. + +#include "backend.h" +#include "wasmfs.h" +#include "virtual.h" +#include "file.h" +#include "paths.h" +#include "memory_backend.h" +#include + +namespace wasmfs { + +using MapManifest=std::map; + +class MapBackend : public Backend { + MapManifest *manifest; + public: + // Takes ownership of manifest + MapBackend(MapManifest *manifest): manifest(manifest) { + assert(manifest && "Mapfs: null manifest not supported"); + } + ~MapBackend() { + if (manifest != NULL) { + delete manifest; + } + } + std::shared_ptr createFile(mode_t mode) override; + std::shared_ptr createDirectory(mode_t mode) override; + std::shared_ptr createSymlink(std::string target) override { + fprintf(stderr, "MAPFS doesn't support creating symlinks"); + abort(); + return NULL; + } + const std::string getTargetPath(const std::string& filePath); + const MapManifest *getManifest() { + return manifest; + } +}; + + +class MapDirectory : public MemoryDirectory { + std::string virtualPath; + +public: + MapDirectory(const std::string& path, + mode_t mode, + backend_t backend) + : MemoryDirectory(mode, backend), virtualPath(path) { + auto manifest = dynamic_cast(getBackend())->getManifest(); + if (manifest && path == "") { + for (const auto& pair : *manifest) { + auto path = pair.first; + assert(path[0] == '/'); + char delimiter = '/'; + std::string pathSoFar = ""; + std::string tmp = ""; + std::shared_ptr dir = NULL; + std::istringstream iss(path); + while(std::getline(iss, tmp, delimiter)) { + pathSoFar += tmp; + if (pathSoFar == path) { + if(!dir) { + assert(this->insertDataFile(tmp, 0777)); + } else { + assert(dir->insertDataFile(tmp, 0777)); + } + } else if (pathSoFar != "") { + std::shared_ptr next = NULL; + if(!dir) { + next = std::dynamic_pointer_cast(this->getChild(tmp)); + } else { + next = std::dynamic_pointer_cast(dir->getChild(tmp)); + } + if (next) { + dir = next; + assert(dir); + } else { + if(!dir) { + dir = std::dynamic_pointer_cast(this->insertDirectory(tmp, 0777)); + } else { + dir = std::dynamic_pointer_cast(dir->insertDirectory(tmp, 0777)); + } + assert(dir); + } + } + pathSoFar += delimiter; + } + } + } + } + + std::shared_ptr insertDataFile(const std::string& name, + mode_t mode) override { + auto backend = dynamic_cast(getBackend()); + auto childPath = getChildPath(name); + auto targetPath = backend->getTargetPath(childPath); + auto parsed = path::parseFile(targetPath); + if (auto err = parsed.getError()) { + fprintf(stderr, "error %ld\n", err); + abort(); + } + auto file = parsed.getFile(); + if (!file) { + fprintf(stderr, "no datafile for %s\n", targetPath.c_str()); + return nullptr; + } + auto virtualFile = std::make_shared(file->cast(), file->getBackend()); + insertChild(name, virtualFile); + return virtualFile->cast(); + } + + std::shared_ptr insertDirectory(const std::string& name, + mode_t mode) override { + auto childPath = getChildPath(name); + auto childDir = + std::make_shared(childPath, mode, getBackend()); + insertChild(name, childDir); + return childDir; + } + + std::string getChildPath(const std::string& name) const { + return virtualPath + '/' + name; + } + + std::shared_ptr getChild(const std::string& name) override { + return MemoryDirectory::getChild(name); + } +}; + +std::shared_ptr MapBackend::createFile(mode_t mode) { + assert(false && "Can't create freestanding file with MAPFS"); + return NULL; +} + +std::shared_ptr MapBackend::createDirectory(mode_t mode) { + return std::make_shared("", mode, this); +} + +const std::string MapBackend::getTargetPath(const std::string& filePath) { + if (auto search = manifest->find(filePath); search != manifest->end()) { + return search->second; + } + fprintf(stderr, "File %s not found in manifest", filePath.c_str()); + abort(); +} + +extern "C" { + +backend_t wasmfs_create_map_backend(MapManifest *manifest) { + return wasmFS.addBackend(std::make_unique(manifest)); +} + +void *EMSCRIPTEN_KEEPALIVE wasmfs_map_create_manifest() { + return new MapManifest(); +} + +void EMSCRIPTEN_KEEPALIVE wasmfs_map_add_to_manifest(MapManifest *manifest, const char *virtualPath, const char *targetPath) { + assert(manifest && "wasmfs_map_add_to_manifest: null manifest"); + auto virtualStr = std::string(virtualPath); + auto targetStr = std::string(targetPath); + manifest->insert(std::pair(virtualStr, targetStr)); +} + +} + +} // namespace wasmfs diff --git a/test/test_browser.py b/test/test_browser.py index dae4276795343..85f4e0e8136f6 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5348,6 +5348,17 @@ def test_wasmfs_fetch_backend(self, args): '-sFORCE_FILESYSTEM', '-lfetchfs.js', '--js-library', test_file('wasmfs/wasmfs_fetch.js')] + args) + @no_wasm64() + @parameterized({ + # using BigInt support affects the ABI, and should not break things. (this + # could be tested on either thread; do the main thread for simplicity) + 'bigint': (['-sWASM_BIGINT'],), + }) + def test_wasmfs_map(self, args): + self.btest_exit('wasmfs/wasmfs_map.c', + args=['-sWASMFS', '-sFORCE_FILESYSTEM', '-lmapfs.js'] + args) + + @no_firefox('no OPFS support yet') @no_wasm64() @parameterized({ diff --git a/test/wasmfs/wasmfs_map.c b/test/wasmfs/wasmfs_map.c new file mode 100644 index 0000000000000..bed2fd83d2c3c --- /dev/null +++ b/test/wasmfs/wasmfs_map.c @@ -0,0 +1,88 @@ +/* + * Copyright 2025 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void check_file(int fd, const char* content) { + assert(fd >= 0); + struct stat st; + assert(fstat(fd, &st) != -1); + printf("file size: %lld\n", st.st_size); + assert(st.st_size > 0); + + char buf[st.st_size]; + int bytes_read = read(fd, buf, st.st_size); + printf("read size: %d\n", bytes_read); + buf[bytes_read] = 0; + printf("'%s'\n", buf); + assert(strcmp(buf, content) == 0); +} + +void test_manifest() { + printf("Running %s...\n", __FUNCTION__); + void *manifest = wasmfs_map_create_manifest(); + wasmfs_map_add_to_manifest(manifest, "/file", "/test.txt"); + wasmfs_map_add_to_manifest(manifest, "/sub/file", "/subdir/test2.txt"); + backend_t backend = wasmfs_create_map_backend(manifest); + assert(wasmfs_create_directory("/dat", 0777, backend) >= 0); + int fd = open("/dat/file", O_RDONLY); + printf("Loaded %d from %s\n",fd,"/dat/file"); + check_file(fd, "mapfs"); + assert(close(fd) == 0); + int fd2 = open("/dat/sub/file", O_RDONLY); + check_file(fd2, "mapfs 2"); + assert(close(fd2) == 0); +} + +void test_manifest_js() { + EM_ASM({ + var contents; + FS.mount(MAPFS, {"manifest":{"/file":"/test.txt", "/sub/file":"/subdir/test2.txt"}}, "/dat2"); + contents = FS.readFile("/dat2/file", {encoding:'utf8'}); + if(contents != "mapfs") { + throw "Wrong file contents "+contents; + } + contents = FS.readFile("/dat2/sub/file", {encoding:'utf8'}); + if(contents != "mapfs 2") { + throw "Wrong file contents "+contents; + } + }); +} + +void test_nonexistent() { + printf("Running %s...\n", __FUNCTION__); + backend_t backend = wasmfs_get_backend_by_path("/dat/sub"); + const char* file_name = "test.txt"; + int fd = open("/dat/sub/test.txt", O_RDONLY); + assert(fd < 0); +} + +int main() { + FILE *test_txt = fopen("/test.txt", "w"); + fputs("mapfs", test_txt); + fclose(test_txt); + mkdir("/subdir", 0777); + FILE *test2_txt = fopen("/subdir/test2.txt", "w"); + fputs("mapfs 2", test2_txt); + fclose(test2_txt); + + test_manifest(); + test_manifest_js(); + test_nonexistent(); + + return 0; +} diff --git a/tools/system_libs.py b/tools/system_libs.py index 9b3a3c2308dfd..939c6de143dd6 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2013,6 +2013,7 @@ def get_files(self): filenames=['fetch_backend.cpp', 'ignore_case_backend.cpp', 'js_file_backend.cpp', + 'map_backend.cpp', 'memory_backend.cpp', 'node_backend.cpp', 'opfs_backend.cpp'])