Skip to content

[file_packager.py] Add --export-es6 to file_packager.py #24737

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 13, 2025
Merged

[file_packager.py] Add --export-es6 to file_packager.py #24737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
115e93d
Add modularize to file_packager.py
lkwinta Jul 18, 2025
b62ddc7
simplify promise usage
lkwinta Aug 12, 2025
6707c83
simplyfi help code
lkwinta Aug 12, 2025
a25caf1
simplyfi test
lkwinta Aug 12, 2025
697c9e6
explicite promise resolving
lkwinta Aug 12, 2025
03095d0
address @sbc100 review
lkwinta Aug 13, 2025
2874708
fix indentation
lkwinta Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ChangeLog.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ See docs/process.md for more on how version tagging works.
- emcc will now error if `MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION` or
`MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION` are used with `SINGLE_FILE`.
These are fundamentally incompatible but were previously ignored. (#24849)
- `--export-es6` flag was added to `file_packager.py` available when run
standalone, to enable ES6 imports of generated JavaScript code (#24737)

4.0.12 - 08/01/25
-----------------
Expand Down
36 changes: 36 additions & 0 deletions test/test_other.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3919,6 +3919,11 @@ def test_file_packager_returns_error_if_target_equal_to_jsoutput(self):
err = self.expect_fail([FILE_PACKAGER, 'test.data', '--js-output=test.data'])
self.assertContained(MESSAGE, err)

def test_file_packager_returns_error_if_emcc_and_export_es6(self):
MESSAGE = 'error: Can\'t use --export-es6 option together with --from-emcc since the code should be embedded within emcc\'s code'
err = self.expect_fail([FILE_PACKAGER, 'test.data', '--export-es6', '--from-emcc'])
self.assertContained(MESSAGE, err)

def test_file_packager_embed(self):
create_file('data.txt', 'hello data')

Expand All @@ -3945,6 +3950,37 @@ def test_file_packager_embed(self):
output = self.run_js('a.out.js')
self.assertContained('hello data', output)

def test_file_packager_export_es6(self):
create_file('smth.txt', 'hello data')
self.run_process([FILE_PACKAGER, 'test.data', '--export-es6', '--preload', 'smth.txt', '--js-output=dataFileLoader.js'])

create_file('test.c', '''
#include <stdio.h>
#include <emscripten.h>

EMSCRIPTEN_KEEPALIVE int test_fun() {
FILE* f = fopen("smth.txt", "r");
char buf[64] = {0};
int rtn = fread(buf, 1, 64, f);
buf[rtn] = '\\0';
fclose(f);
printf("%s\\n", buf);
return 0;
}
''')
self.run_process([EMCC, 'test.c', '-sFORCE_FILESYSTEM', '-sMODULARIZE', '-sEXPORT_ES6', '-o', 'moduleFile.js'])

create_file('run.js', '''
import loadDataFile from './dataFileLoader.js'
import {default as loadModule} from './moduleFile.js'

var module = await loadModule();
await loadDataFile(module);
module._test_fun();
''')

self.assertContained('hello data', self.run_js('run.js'))

@crossplatform
def test_file_packager_depfile(self):
create_file('data1.txt', 'data1')
Expand Down
88 changes: 68 additions & 20 deletions tools/file_packager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

Usage:

file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--help]
file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--export-es6] [--help]

--preload ,
--embed See emcc --help for more details on those options.
Expand All @@ -41,6 +41,8 @@

--export-name=EXPORT_NAME Use custom export name (default is `Module`)

--export-es6 Wrap generated code inside ES6 exported function

--no-force Don't create output if no valid input file is specified.

--use-preload-cache Stores package in IndexedDB so that subsequent loads don't need to do XHR. Checks package version.
Expand Down Expand Up @@ -129,6 +131,7 @@ def __init__(self):
self.use_preload_plugins = False
self.support_node = True
self.wasm64 = False
self.export_es6 = False


class DataFile:
Expand Down Expand Up @@ -362,7 +365,7 @@ def main(): # noqa: C901, PLR0912, PLR0915
To revalidate these numbers, run `ruff check --select=C901,PLR091`.
"""
if len(sys.argv) == 1:
err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--help]
err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--export-es6] [--help]
Try 'file_packager --help' for more details.''')
return 1

Expand Down Expand Up @@ -391,6 +394,9 @@ def main(): # noqa: C901, PLR0912, PLR0915
elif arg == '--no-force':
options.force = False
leading = ''
elif arg == '--export-es6':
options.export_es6 = True
leading = ''
elif arg == '--use-preload-cache':
options.use_preload_cache = True
leading = ''
Expand Down Expand Up @@ -485,6 +491,11 @@ def main(): # noqa: C901, PLR0912, PLR0915
diagnostics.error('TARGET should not be the same value of --js-output')
return 1

if options.from_emcc and options.export_es6:
diagnostics.error('Can\'t use --export-es6 option together with --from-emcc since the code should be embedded '
'within emcc\'s code')
return 1

walked.append(__file__)
for file_ in data_files:
if not should_ignore(file_.srcpath):
Expand Down Expand Up @@ -621,20 +632,38 @@ def generate_js(data_target, data_files, metadata):
if options.from_emcc:
ret = ''
else:
ret = '''
if options.export_es6:
ret = 'export default async function loadDataFile(Module) {\n'
else:
ret = '''
var Module = typeof %(EXPORT_NAME)s != 'undefined' ? %(EXPORT_NAME)s : {};\n''' % {"EXPORT_NAME": options.export_name}

ret += '''
Module['expectedDataFileDownloads'] ??= 0;
Module['expectedDataFileDownloads']++;
(() => {
Module['expectedDataFileDownloads']++;'''

if not options.export_es6:
ret += '''
(() => {'''

ret += '''
// Do not attempt to redownload the virtual filesystem data when in a pthread or a Wasm Worker context.
var isPthread = typeof ENVIRONMENT_IS_PTHREAD != 'undefined' && ENVIRONMENT_IS_PTHREAD;
var isWasmWorker = typeof ENVIRONMENT_IS_WASM_WORKER != 'undefined' && ENVIRONMENT_IS_WASM_WORKER;
if (isPthread || isWasmWorker) return;\n'''

if options.support_node:
ret += " var isNode = typeof process === 'object' && typeof process.versions === 'object' && typeof process.versions.node === 'string';\n"

if options.support_node and options.export_es6:
ret += '''if (isNode) {
const { createRequire } = await import('module');
/** @suppress{duplicate} */
var require = createRequire(import.meta.url);
}\n'''

if options.export_es6:
ret += 'return new Promise((loadDataResolve, loadDataReject) => {\n'
ret += ' async function loadPackage(metadata) {\n'

code = '''
Expand Down Expand Up @@ -688,6 +717,10 @@ def generate_js(data_target, data_files, metadata):
Module['FS_createDataFile'](this.name, null, byteArray, true, true, true);
Module['removeRunDependency'](`fp ${that.name}`);'''

finish_handler = create_preloaded if options.use_preload_plugins else create_data
if options.export_es6:
finish_handler += '\nloadDataResolve();'

if not options.lz4:
# Data requests - for getting a block of data out of the big archive - have
# a similar API to XHRs
Expand Down Expand Up @@ -720,11 +753,18 @@ def generate_js(data_target, data_files, metadata):
var files = metadata['files'];
for (var i = 0; i < files.length; ++i) {
new DataRequest(files[i]['start'], files[i]['end'], files[i]['audio'] || 0).open('GET', files[i]['filename']);
}\n''' % (create_preloaded if options.use_preload_plugins else create_data)
}\n''' % finish_handler

if options.has_embedded and not options.obj_output:
diagnostics.warn('--obj-output is recommended when using --embed. This outputs an object file for linking directly into your application is more efficient than JS encoding')

catch_handler = ''
if options.export_es6:
catch_handler += '''
.catch((error) => {
loadDataReject(error);
})'''

for counter, file_ in enumerate(data_files):
filename = file_.dstpath
dirname = os.path.dirname(filename)
Expand Down Expand Up @@ -1049,10 +1089,10 @@ def generate_js(data_target, data_files, metadata):
}
}
} catch(e) {
await preloadFallback(e);
await preloadFallback(e)%s;
}

Module['setStatus']?.('Downloading...');\n'''
Module['setStatus']?.('Downloading...');\n''' % catch_handler
else:
# Not using preload cache, so we might as well start the xhr ASAP,
# potentially before JS parsing of the main codebase if it's after us.
Expand All @@ -1065,15 +1105,16 @@ def generate_js(data_target, data_files, metadata):
if (!fetched) {
// Note that we don't use await here because we want to execute the
// the rest of this function immediately.
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE).then((data) => {
if (fetchedCallback) {
fetchedCallback(data);
fetchedCallback = null;
} else {
fetched = data;
}
})
}\n'''
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE)
.then((data) => {
if (fetchedCallback) {
fetchedCallback(data);
fetchedCallback = null;
} else {
fetched = data;
}
})%s;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like there is no current error handling here, for this fetch. It looks like the code is trying to do a pre-fetch and maybe we can just ignore failures (like we currently do) for the pre-fetch?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In that case this block could be reverted

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, without this catch here in es6 mode, i can't handle error on my own, the whole js crashes because of 'unhandled error on promise'

Fine:

async function F1() {
    throw new Error();
}

F1().catch((err)=>console.log(err))

Unhandled error in promise:

async function F1() {
    async function F2(){
    throw new Error();
    }
    F2();
}

F1().catch((err)=>console.log(err))

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But isn't that just the existing behaviour? There is not catch mechanism for this particular fetch and so errors here will always go the to the global error handlers. I think that is the current/expected behaviour. We could change that perhaps?

Given that there is not really any way to recover from such errors why not just let the global error handler handle these errors? Or the global unhandledRejection handler.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I meant that when the throw is two or more level deep I cant catch the error even with top level try catch block which is an issue - most likely es6 mode will be used together with some frontend and we don't want to crash whole JS but rather give some information and maybe possibility to retry

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And inside webworker we can't access window context to get access to unhandledrejection handler

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can handle unhandled rejections using the worker global scope:

    // worker.js
    self.addEventListener('unhandledrejection', (event) => {
        console.error('Unhandled Promise Rejection in Worker:', event.reason);
    });

But I understand that it would be nice to correctly reject the promise here, so this part lgtm.

We can refactor later perhaps.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure

}\n''' % catch_handler

code += '''
Module['preloadResults'][PACKAGE_NAME] = {fromCache: false};
Expand All @@ -1090,10 +1131,10 @@ def generate_js(data_target, data_files, metadata):
ret += '''
}
if (Module['calledRun']) {
runWithFS(Module);
runWithFS(Module)%s;
} else {
(Module['preRun'] ??= []).push(runWithFS); // FS is not initialized yet, wait for it
}\n'''
}\n''' % catch_handler

if options.separate_metadata:
node_support_code = ''
Expand Down Expand Up @@ -1131,7 +1172,14 @@ def generate_js(data_target, data_files, metadata):
}
loadPackage(%s);\n''' % json.dumps(metadata)

ret += '''
if options.export_es6:
ret += '''
});
}
// END the loadDataFile function
'''
else:
ret += '''
})();\n'''

return ret
Expand Down