Skip to content

Commit 0b6e63e

Browse files
JoeOsbornJoeOsborn
committed
chunked backend, works for single chunks and simple backend url maps
1 parent cbf40dc commit 0b6e63e

File tree

7 files changed

+252
-54
lines changed

7 files changed

+252
-54
lines changed

‎src/library_fetchfs.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ addToLibrary({
88
$FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'],
99
$FETCHFS: {
1010
createBackend(opts) {
11-
return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url));
11+
return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0);
1212
}
1313
},
1414
});

‎src/library_wasmfs_fetch.js‎

Lines changed: 86 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,24 @@
55
*/
66

77
addToLibrary({
8+
$wasmFS$JSMemoryRanges: {},
9+
810
// Fetch backend: On first access of the file (either a read or a getSize), it
911
// will fetch() the data from the network asynchronously. Otherwise, after
1012
// that fetch it behaves just like JSFile (and it reuses the code from there).
1113

1214
_wasmfs_create_fetch_backend_js__deps: [
1315
'$wasmFS$backends',
14-
'$wasmFS$JSMemoryFiles',
15-
'_wasmfs_create_js_file_backend_js',
16-
'_wasmfs_fetch_get_file_path',
16+
'$wasmFS$JSMemoryRanges',
17+
'_wasmfs_fetch_get_file_url',
18+
'_wasmfs_fetch_get_chunk_size',
1719
],
1820
_wasmfs_create_fetch_backend_js: async function(backend) {
1921
// Get a promise that fetches the data and stores it in JS memory (if it has
2022
// not already been fetched).
21-
async function getFile(file) {
22-
if (wasmFS$JSMemoryFiles[file]) {
23-
// The data is already here, so nothing to do before we continue on to
24-
// the actual read below.
25-
return Promise.resolve();
26-
}
27-
// This is the first time we want the file's data.
23+
async function getFileRange(file, offset, len) {
2824
var url = '';
29-
var fileUrl_p = __wasmfs_fetch_get_file_path(file);
25+
var fileUrl_p = __wasmfs_fetch_get_file_url(file);
3026
var fileUrl = UTF8ToString(fileUrl_p);
3127
var isAbs = fileUrl.indexOf('://') !== -1;
3228
if (isAbs) {
@@ -38,31 +34,71 @@ addToLibrary({
3834
} catch (e) {
3935
}
4036
}
41-
var response = await fetch(url);
37+
var chunkSize = __wasmfs_fetch_get_chunk_size(file);
38+
offset = offset || 0;
39+
len = len || chunkSize;
40+
var firstChunk = (offset / chunkSize) | 0;
41+
var lastChunk = ((offset+len) / chunkSize) | 0;
42+
if (!(file in wasmFS$JSMemoryRanges)) {
43+
var fileInfo = await fetch(url,{method:"HEAD", headers:{"Range": "bytes=0-"}});
44+
if(fileInfo.ok &&
45+
fileInfo.headers.has("Content-Length") &&
46+
fileInfo.headers.get("Accept-Ranges") == "bytes" &&
47+
(parseInt(fileInfo.headers.get("Content-Length")) > chunkSize*2)) {
48+
wasmFS$JSMemoryRanges[file] = {size:parseInt(fileInfo.headers.get("Content-Length")), chunks:[], chunkSize:chunkSize};
49+
} else {
50+
// may as well/forced to download the whole file
51+
var wholeFileReq = await fetch(url);
52+
if(!wholeFileReq.ok) {
53+
throw wholeFileReq;
54+
}
55+
var wholeFileData = new Uint8Array(await wholeFileReq.arrayBuffer());
56+
var text = new TextDecoder().decode(wholeFileData);
57+
wasmFS$JSMemoryRanges[file] = {size:wholeFileData.byteLength, chunks:[wholeFileData], chunkSize:wholeFileData.byteLength};
58+
return Promise.resolve();
59+
}
60+
}
61+
var allPresent = true;
62+
var i;
63+
if(lastChunk * chunkSize < offset+len) {
64+
lastChunk += 1;
65+
}
66+
for(i = firstChunk; i < lastChunk; i++) {
67+
if(!wasmFS$JSMemoryRanges[file].chunks[i]) {
68+
allPresent = false;
69+
break;
70+
}
71+
}
72+
if (allPresent) {
73+
// The data is already here, so nothing to do before we continue on to
74+
// the actual read.
75+
return Promise.resolve();
76+
}
77+
// This is the first time we want the chunk's data.
78+
var start = firstChunk*chunkSize;
79+
var end = lastChunk*chunkSize;
80+
var response = await fetch(url, {headers:{"Range": `bytes=${start}-${end-1}`}});
4281
if (response.ok) {
43-
var buffer = await response['arrayBuffer']();
44-
wasmFS$JSMemoryFiles[file] = new Uint8Array(buffer);
82+
var bytes = new Uint8Array(await response['arrayBuffer']());
83+
for (i = firstChunk; i < lastChunk; i++) {
84+
wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start);
85+
}
4586
} else {
4687
throw response;
4788
}
89+
return Promise.resolve();
4890
}
4991

50-
// Start with the normal JSFile operations. This sets
51-
// wasmFS$backends[backend]
52-
// which we will then augment.
53-
__wasmfs_create_js_file_backend_js(backend);
54-
55-
// Add the async operations on top.
56-
var jsFileOps = wasmFS$backends[backend];
5792
wasmFS$backends[backend] = {
5893
// alloc/free operations are not actually async. Just forward to the
5994
// parent class, but we must return a Promise as the caller expects.
6095
allocFile: async (file) => {
61-
jsFileOps.allocFile(file);
96+
// nop
6297
return Promise.resolve();
6398
},
6499
freeFile: async (file) => {
65-
jsFileOps.freeFile(file);
100+
// free memory
101+
wasmFS$JSMemoryRanges[file] = undefined;
66102
return Promise.resolve();
67103
},
68104

@@ -73,17 +109,39 @@ addToLibrary({
73109
// read/getSize fetch the data, then forward to the parent class.
74110
read: async (file, buffer, length, offset) => {
75111
try {
76-
await getFile(file);
112+
await getFileRange(file, offset || 0, length);
77113
} catch (response) {
78114
return response.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}};
79115
}
80-
return jsFileOps.read(file, buffer, length, offset);
116+
var fileInfo = wasmFS$JSMemoryRanges[file];
117+
var fileData = fileInfo.chunks;
118+
var chunkSize = fileInfo.chunkSize;
119+
var firstChunk = (offset / chunkSize) | 0;
120+
var lastChunk = ((offset+length) / chunkSize) | 0;
121+
if(offset + length > lastChunk * chunkSize) {
122+
lastChunk += 1;
123+
}
124+
var readLength = 0;
125+
for (var i = firstChunk; i < lastChunk; i++) {
126+
var chunk = fileData[i];
127+
var start = Math.max(i*chunkSize, offset);
128+
if(!chunk) {
129+
throw [fileData.length, firstChunk, lastChunk, i];
130+
}
131+
var chunkStart = i*chunkSize;
132+
var end = Math.min(chunkStart+chunkSize, offset+length);
133+
HEAPU8.set(chunk.subarray(start-chunkStart, end-chunkStart), buffer+(start-offset));
134+
readLength = end - offset;
135+
}
136+
return readLength;
81137
},
82138
getSize: async (file) => {
83139
try {
84-
await getFile(file);
85-
} catch (response) {}
86-
return jsFileOps.getSize(file);
140+
await getFileRange(file, 0, 0);
141+
} catch (response) {
142+
return 0;
143+
}
144+
return wasmFS$JSMemoryRanges[file].size;
87145
},
88146
};
89147
},

‎system/include/emscripten/wasmfs.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ backend_t wasmfs_create_memory_backend(void);
5757
//
5858
// TODO: Add an async version of this function that will work on the main
5959
// thread.
60-
backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)));
60+
backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t);
6161

6262
backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull)));
6363

‎system/lib/wasmfs/backends/fetch_backend.cpp‎

Lines changed: 52 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,41 @@
1212

1313
namespace wasmfs {
1414

15+
const uint32_t DEFAULT_CHUNK_SIZE = 16*1024*1024;
16+
17+
class FetchBackend : public wasmfs::ProxiedAsyncJSBackend {
18+
std::string baseUrl;
19+
uint32_t chunkSize;
20+
public:
21+
FetchBackend(const std::string& baseUrl,
22+
uint32_t chunkSize,
23+
std::function<void(backend_t)> setupOnThread)
24+
: ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize)
25+
// TODO manifest
26+
{}
27+
std::shared_ptr<DataFile> createFile(mode_t mode) override;
28+
std::shared_ptr<Directory> createDirectory(mode_t mode) override;
29+
const std::string getFileURL(const std::string& filePath);
30+
uint32_t getChunkSize();
31+
};
32+
33+
1534
class FetchFile : public ProxiedAsyncJSImplFile {
1635
std::string filePath;
36+
std::string fileUrl;
1737

1838
public:
1939
FetchFile(const std::string& path,
2040
mode_t mode,
2141
backend_t backend,
2242
emscripten::ProxyWorker& proxy)
23-
: ProxiedAsyncJSImplFile(mode, backend, proxy), filePath(path) {}
43+
: ProxiedAsyncJSImplFile(mode, backend, proxy), filePath(path) {
44+
this->fileUrl = dynamic_cast<FetchBackend*>(getBackend())->getFileURL(filePath);
45+
}
2446

2547
const std::string& getPath() const { return filePath; }
48+
const std::string& getURL() const { return fileUrl; }
49+
const uint32_t getChunkSize() const { return dynamic_cast<FetchBackend*>(getBackend())->getChunkSize(); }
2650
};
2751

2852
class FetchDirectory : public MemoryDirectory {
@@ -59,38 +83,50 @@ class FetchDirectory : public MemoryDirectory {
5983
}
6084
};
6185

62-
class FetchBackend : public ProxiedAsyncJSBackend {
63-
std::string baseUrl;
64-
65-
public:
66-
FetchBackend(const std::string& baseUrl,
67-
std::function<void(backend_t)> setupOnThread)
68-
: ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl) {}
86+
std::shared_ptr<DataFile> FetchBackend::createFile(mode_t mode) {
87+
return std::make_shared<FetchFile>("", mode, this, proxy);
88+
}
6989

70-
std::shared_ptr<DataFile> createFile(mode_t mode) override {
71-
return std::make_shared<FetchFile>(baseUrl, mode, this, proxy);
72-
}
90+
std::shared_ptr<Directory> FetchBackend::createDirectory(mode_t mode) {
91+
return std::make_shared<FetchDirectory>("", mode, this, proxy);
92+
}
7393

74-
std::shared_ptr<Directory> createDirectory(mode_t mode) override {
75-
return std::make_shared<FetchDirectory>(baseUrl, mode, this, proxy);
94+
const std::string FetchBackend::getFileURL(const std::string& filePath) {
95+
// TODO use manifest
96+
if(filePath == "") {
97+
return baseUrl;
7698
}
77-
};
99+
return baseUrl + "/" + filePath;
100+
}
101+
uint32_t FetchBackend::getChunkSize() {
102+
return chunkSize;
103+
}
78104

79105
extern "C" {
80-
backend_t wasmfs_create_fetch_backend(const char* base_url) {
106+
backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize /* TODO manifest */) {
81107
// ProxyWorker cannot safely be synchronously spawned from the main browser
82108
// thread. See comment in thread_utils.h for more details.
83109
assert(!emscripten_is_main_browser_thread() &&
84110
"Cannot safely create fetch backend on main browser thread");
85111
return wasmFS.addBackend(std::make_unique<FetchBackend>(
86112
base_url ? base_url : "",
113+
chunkSize != 0 ? chunkSize : DEFAULT_CHUNK_SIZE,
114+
/* TODO manifest */
87115
[](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); }));
88-
}
116+
}
89117

90118
const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_path(void* ptr) {
91119
auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
92120
return file ? file->getPath().data() : nullptr;
93121
}
122+
const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_url(void* ptr) {
123+
auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
124+
return file ? file->getURL().data() : nullptr;
125+
}
126+
uint32_t EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_chunk_size(void* ptr) {
127+
auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
128+
return file ? file->getChunkSize() : DEFAULT_CHUNK_SIZE;
129+
}
94130
}
95131

96132
} // namespace wasmfs

‎system/lib/wasmfs/backends/fetch_backend.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
#include "wasmfs.h"
77

88
extern "C" {
9-
10-
// See library_wasmfs_fetch.js
11-
void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
9+
// See library_wasmfs_fetch.js
10+
void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
1211
}

‎test/test_browser.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5285,6 +5285,7 @@ def test_wasmfs_fetch_backend(self, args):
52855285
create_file('subdir/backendfile2', 'file 2')
52865286
self.btest_exit('wasmfs/wasmfs_fetch.c',
52875287
args=['-sWASMFS', '-pthread', '-sPROXY_TO_PTHREAD',
5288+
'-sFORCE_FILESYSTEM', '-lfetchfs.js',
52885289
'--js-library', test_file('wasmfs/wasmfs_fetch.js')] + args)
52895290

52905291
@no_firefox('no OPFS support yet')

0 commit comments

Comments
 (0)