Skip to content

[file_packager.py] Convert fetchRemotePackage to promise #24892

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[file_packager.py] Convert fetchRemotePackage to promise #24892

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ede1781
change fetchRemotePackage to be async instead of callback errback
lkwinta Aug 8, 2025
fb97ad1
apply ruff fix
lkwinta Aug 8, 2025
39e4335
fix from emcc
lkwinta Aug 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 79 additions & 68 deletions tools/file_packager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -948,73 +948,81 @@ def generate_js(data_target, data_files, metadata):
node_support_code = ''
if options.support_node:
node_support_code = '''
if (isNode) {
require('fs').readFile(packageName, (err, contents) => {
if (err) {
errback(err);
} else {
callback(contents.buffer);
}
});
return;
}'''.strip()
if (isNode) {
require('fs').readFile(packageName, (err, contents) => {
if (err) {
fetchPackageReject(err);
} else {
fetchPackageResolve(contents.buffer);
}
});
return;
}'''.strip()

ret += '''
function fetchRemotePackage(packageName, packageSize, callback, errback) {
%(node_support_code)s
Module['dataFileDownloads'] ??= {};
fetch(packageName)
.catch((cause) => Promise.reject(new Error(`Network Error: ${packageName}`, {cause}))) // If fetch fails, rewrite the error to include the failing URL & the cause.
.then((response) => {
if (!response.ok) {
return Promise.reject(new Error(`${response.status}: ${response.url}`));
}
function fetchRemotePackage(packageName, packageSize) {
return new Promise((fetchPackageResolve, fetchPackageReject) => {
%(node_support_code)s
Module['dataFileDownloads'] ??= {};
fetch(packageName)
.then((response) => {
if (!response.ok) {
fetchPackageReject(new Error(`${response.status}: ${response.url}`));
return;
}

if (!response.body && response.arrayBuffer) { // If we're using the polyfill, readers won't be available...
return response.arrayBuffer().then(callback);
}
if (!response.body && response.arrayBuffer) { // If we're using the polyfill, readers won't be available...
fetchPackageResolve(response.arrayBuffer());
return;
}

const reader = response.body.getReader();
const iterate = () => reader.read().then(handleChunk).catch((cause) => {
return Promise.reject(new Error(`Unexpected error while handling : ${response.url} ${cause}`, {cause}));
});
const reader = response.body.getReader();
const iterate = () => reader.read().then(handleChunk).catch((cause) => {
fetchPackageReject(new Error(`Unexpected error while handling : ${response.url} ${cause}`, {cause}));
return;
});

const chunks = [];
const headers = response.headers;
const total = Number(headers.get('Content-Length') ?? packageSize);
let loaded = 0;
const chunks = [];
const headers = response.headers;
const total = Number(headers.get('Content-Length') ?? packageSize);
let loaded = 0;

const handleChunk = ({done, value}) => {
if (!done) {
chunks.push(value);
loaded += value.length;
Module['dataFileDownloads'][packageName] = {loaded, total};
const handleChunk = ({done, value}) => {
if (!done) {
chunks.push(value);
loaded += value.length;
Module['dataFileDownloads'][packageName] = {loaded, total};

let totalLoaded = 0;
let totalSize = 0;
let totalLoaded = 0;
let totalSize = 0;

for (const download of Object.values(Module['dataFileDownloads'])) {
totalLoaded += download.loaded;
totalSize += download.total;
}
for (const download of Object.values(Module['dataFileDownloads'])) {
totalLoaded += download.loaded;
totalSize += download.total;
}

Module['setStatus']?.(`Downloading data... (${totalLoaded}/${totalSize})`);
return iterate();
} else {
const packageData = new Uint8Array(chunks.map((c) => c.length).reduce((a, b) => a + b, 0));
let offset = 0;
for (const chunk of chunks) {
packageData.set(chunk, offset);
offset += chunk.length;
Module['setStatus']?.(`Downloading data... (${totalLoaded}/${totalSize})`);
return iterate();
} else {
const packageData = new Uint8Array(chunks.map((c) => c.length).reduce((a, b) => a + b, 0));
let offset = 0;
for (const chunk of chunks) {
packageData.set(chunk, offset);
offset += chunk.length;
}
fetchPackageResolve(packageData.buffer);
}
callback(packageData.buffer);
}
};

Module['setStatus']?.('Downloading data...');
return iterate();
};

Module['setStatus']?.('Downloading data...');
return iterate();
})
.catch((cause) => {
// If fetch fails, rewrite the error to include the failing URL & the cause.
fetchPackageReject(new Error(`Network Error: ${packageName}`, {cause}));
});
});
};
}

function handleError(error) {
console.error('package error:', error);
Expand All @@ -1040,7 +1048,7 @@ def generate_js(data_target, data_files, metadata):
function preloadFallback(error) {
console.error(error);
console.error('falling back to default preload behavior');
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, processPackageData, handleError);
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE).then(processPackageData).catch(handleError);
};

openDatabase()
Expand All @@ -1050,15 +1058,16 @@ def generate_js(data_target, data_files, metadata):
if (useCached) {
fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, metadata, processPackageData, preloadFallback);
} else {
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE,
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE)
.then(
(packageData) => {
cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID}, processPackageData,
(error) => {
console.error(error);
processPackageData(packageData);
});
}
, preloadFallback);
})
.catch(preloadFallback);
}
}, preloadFallback))
.catch(preloadFallback);
Expand All @@ -1073,14 +1082,16 @@ def generate_js(data_target, data_files, metadata):
var fetchedCallback = null;
var fetched = Module['getPreloadedPackage'] ? Module['getPreloadedPackage'](REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE) : null;

if (!fetched) fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, (data) => {
if (fetchedCallback) {
fetchedCallback(data);
fetchedCallback = null;
} else {
fetched = data;
}
}, handleError);\n'''
if (!fetched) fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE)
.then((data) => {
if (fetchedCallback) {
fetchedCallback(data);
fetchedCallback = null;
} else {
fetched = data;
}
})
.catch(handleError);\n'''

code += '''
Module['preloadResults'][PACKAGE_NAME] = {fromCache: false};
Expand Down