diff --git a/Cargo.lock b/Cargo.lock index 9078cb2..e4e827e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -188,6 +188,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "displaydoc" version = "0.2.5" @@ -199,6 +224,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "embedded-io" version = "0.4.0" @@ -227,12 +258,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - [[package]] name = "fastrand" version = "2.3.0" @@ -371,9 +396,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" dependencies = [ - "fallible-iterator", "indexmap", - "stable_deref_trait", ] [[package]] @@ -642,20 +665,6 @@ version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde51589ab56b20a6f686b2c68f7a0bd6add753d697abf720d63f8db3ab7b1ad" -[[package]] -name = "orca-wasm" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ce9b3c6e21f067e2c1c85a4478ec26dda0a51fec5bc932dfa77f016a880861" -dependencies = [ - "gimli", - "log", - "serde_json", - "tempfile", - "wasm-encoder 0.227.1", - "wasmparser 0.227.1", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -729,6 +738,26 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rustix" version = "1.0.2" @@ -824,10 +853,11 @@ dependencies = [ "clap", "heck", "js-component-bindgen", - "orca-wasm", "rand", + "serde_json", "wasm-encoder 0.227.1", - "wasmparser 0.227.1", + "wasmparser 0.239.0", + "wirm", "wit-bindgen", "wit-bindgen-core", "wit-component", @@ -996,6 +1026,16 @@ dependencies = [ "wasmparser 0.227.1", ] +[[package]] +name = "wasm-encoder" +version = "0.239.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be00faa2b4950c76fe618c409d2c3ea5a3c9422013e079482d78544bb2d184c" +dependencies = [ + "leb128fmt", + "wasmparser 0.239.0", +] + [[package]] name = "wasm-metadata" version = "0.227.1" @@ -1033,6 +1073,18 @@ name = "wasmparser" version = "0.227.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f51cad774fb3c9461ab9bccc9c62dfb7388397b5deda31bf40e8108ccd678b2" +dependencies = [ + "bitflags", + "hashbrown", + "indexmap", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.239.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9d90bb93e764f6beabf1d02028c70a2156a6583e63ac4218dd07ef733368b0" dependencies = [ "bitflags", "hashbrown", @@ -1187,6 +1239,20 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wirm" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14732cb9a0eaf9ec52ecd36b9394ade5c16eea5405d160d8829f0199d97d507d" +dependencies = [ + "log", + "rayon", + "serde_json", + "tempfile", + "wasm-encoder 0.239.0", + "wasmparser 0.239.0", +] + [[package]] name = "wit-bindgen" version = "0.41.0" diff --git a/Cargo.toml b/Cargo.toml index 39e8d17..80a49c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,11 @@ too_many_arguments = 'allow' anyhow = { version = "1.0.95", default-features = false } heck = { version = "0.5", default-features = false } js-component-bindgen = { version = "1.11.0" } -orca-wasm = { version = "0.9.2", default-features = false } +wirm = { version = "2.1.0", features = ["parallel"] } rand = { version = "0.8", default-features = false } +serde_json = { version = "1.0", default-features = false, features = ["alloc"] } wasm-encoder = { version = "0.227.1", features = [ "component-model", "std" ] } -wasmparser = { version = "0.227.1", features = ["features", +wasmparser = { version = "0.239.0", features = ["features", "component-model", "hash-collections", "serde", diff --git a/Makefile b/Makefile index 3ff0301..b9c52c5 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ endif STARLINGMONKEY_DEPS = $(STARLINGMONKEY_SRC)/cmake/* embedding/* $(STARLINGMONKEY_SRC)/runtime/* $(STARLINGMONKEY_SRC)/builtins/* $(STARLINGMONKEY_SRC)/builtins/*/* $(STARLINGMONKEY_SRC)/builtins/*/*/* $(STARLINGMONKEY_SRC)/include/* all: release -debug: lib/starlingmonkey_embedding.debug.wasm lib/spidermonkey-embedding-splicer.js -release: lib/starlingmonkey_embedding.wasm lib/spidermonkey-embedding-splicer.js +debug: lib/starlingmonkey_embedding.debug.wasm lib/spidermonkey-embedding-splicer.js target/debug/splicer +release: lib/starlingmonkey_embedding.wasm lib/spidermonkey-embedding-splicer.js target/release/splicer lib/spidermonkey-embedding-splicer.js: target/wasm32-wasip1/release/splicer_component.wasm crates/spidermonkey-embedding-splicer/wit/spidermonkey-embedding-splicer.wit | obj lib @$(JCO) new target/wasm32-wasip1/release/splicer_component.wasm -o obj/spidermonkey-embedding-splicer.wasm --wasi-reactor @@ -23,6 +23,12 @@ lib/spidermonkey-embedding-splicer.js: target/wasm32-wasip1/release/splicer_comp target/wasm32-wasip1/release/splicer_component.wasm: Cargo.toml crates/spidermonkey-embedding-splicer/Cargo.toml crates/spidermonkey-embedding-splicer/src/*.rs crates/splicer-component/src/*.rs cargo build --lib --release --target wasm32-wasip1 +target/release/splicer: Cargo.toml crates/spidermonkey-embedding-splicer/Cargo.toml crates/spidermonkey-embedding-splicer/src/*.rs crates/spidermonkey-embedding-splicer/src/bin/splicer.rs + cargo build --bin splicer --release + +target/debug/splicer: Cargo.toml crates/spidermonkey-embedding-splicer/Cargo.toml crates/spidermonkey-embedding-splicer/src/*.rs crates/spidermonkey-embedding-splicer/src/bin/splicer.rs + cargo build --bin splicer + lib/starlingmonkey_embedding.wasm: $(STARLINGMONKEY_DEPS) | lib cmake -B build-release -DCMAKE_BUILD_TYPE=Release make -j16 -C build-release starlingmonkey_embedding diff --git a/crates/spidermonkey-embedding-splicer/Cargo.toml b/crates/spidermonkey-embedding-splicer/Cargo.toml index be8d7e6..438408d 100644 --- a/crates/spidermonkey-embedding-splicer/Cargo.toml +++ b/crates/spidermonkey-embedding-splicer/Cargo.toml @@ -16,8 +16,9 @@ anyhow = { workspace = true } clap = { version = "4.5.31", features = ["suggestions", "color", "derive"] } heck = { workspace = true } js-component-bindgen = { workspace = true, features = [ "transpile-bindgen" ] } -orca-wasm = { workspace = true } +wirm = { workspace = true } rand = { workspace = true } +serde_json = { workspace = true } wasm-encoder = { workspace = true } wasmparser = { workspace = true } wit-bindgen = { workspace = true } diff --git a/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs b/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs index 7d62a8d..0912489 100644 --- a/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs +++ b/crates/spidermonkey-embedding-splicer/src/bin/splicer.rs @@ -5,7 +5,9 @@ use std::str::FromStr; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; -use spidermonkey_embedding_splicer::wit::exports::local::spidermonkey_embedding_splicer::splicer::Feature; +use spidermonkey_embedding_splicer::wit::exports::local::spidermonkey_embedding_splicer::splicer::{ + CoreFn, CoreTy, Feature, +}; use spidermonkey_embedding_splicer::{splice, stub_wasi}; #[derive(Parser, Debug)] @@ -139,8 +141,112 @@ fn main() -> Result<()> { out_dir.join("initializer.js").display() ) })?; + + // Write exports and imports as JSON (manual serialization) + let exports_json = serialize_exports(&result.exports); + fs::write(out_dir.join("exports.json"), exports_json).with_context(|| { + format!( + "Failed to write exports file: {}", + out_dir.join("exports.json").display() + ) + })?; + + let imports_json = serialize_imports(&result.imports); + fs::write(out_dir.join("imports.json"), imports_json).with_context(|| { + format!( + "Failed to write imports file: {}", + out_dir.join("imports.json").display() + ) + })?; + + println!( + "Successfully generated bindings and saved to {}", + out_dir.display() + ); } } Ok(()) } + +/// Manually serialize exports to JSON +fn serialize_exports(exports: &[(String, CoreFn)]) -> String { + let mut result = String::from("[\n"); + for (i, (name, core_fn)) in exports.iter().enumerate() { + if i > 0 { + result.push_str(",\n"); + } + result.push_str(" [\""); + result.push_str(&name.replace('\\', "\\\\").replace('"', "\\\"")); + result.push_str("\", "); + result.push_str(&serialize_core_fn(core_fn)); + result.push(']'); + } + result.push_str("\n]"); + result +} + +/// Manually serialize imports to JSON +fn serialize_imports(imports: &[(String, String, u32)]) -> String { + let mut result = String::from("[\n"); + for (i, (specifier, name, arg_count)) in imports.iter().enumerate() { + if i > 0 { + result.push_str(",\n"); + } + result.push_str(" [\""); + result.push_str(&specifier.replace('\\', "\\\\").replace('"', "\\\"")); + result.push_str("\", \""); + result.push_str(&name.replace('\\', "\\\\").replace('"', "\\\"")); + result.push_str("\", "); + result.push_str(&arg_count.to_string()); + result.push(']'); + } + result.push_str("\n]"); + result +} + +/// Manually serialize CoreFn to JSON +fn serialize_core_fn(core_fn: &CoreFn) -> String { + let mut result = String::from("{"); + + // params + result.push_str("\"params\": ["); + for (i, param) in core_fn.params.iter().enumerate() { + if i > 0 { + result.push_str(", "); + } + result.push_str(&serialize_core_ty(param)); + } + result.push_str("], "); + + // ret + result.push_str("\"ret\": "); + if let Some(ref ret) = core_fn.ret { + result.push_str(&serialize_core_ty(ret)); + } else { + result.push_str("null"); + } + result.push_str(", "); + + // retptr + result.push_str(&format!("\"retptr\": {}, ", core_fn.retptr)); + + // retsize + result.push_str(&format!("\"retsize\": {}, ", core_fn.retsize)); + + // paramptr + result.push_str(&format!("\"paramptr\": {}", core_fn.paramptr)); + + result.push('}'); + result +} + +/// Manually serialize CoreTy to JSON +fn serialize_core_ty(core_ty: &CoreTy) -> String { + match core_ty { + CoreTy::I32 => "\"i32\"".to_string(), + CoreTy::I64 => "\"i64\"".to_string(), + CoreTy::F32 => "\"f32\"".to_string(), + CoreTy::F64 => "\"f64\"".to_string(), + } +} diff --git a/crates/spidermonkey-embedding-splicer/src/splice.rs b/crates/spidermonkey-embedding-splicer/src/splice.rs index 38606a0..039d573 100644 --- a/crates/spidermonkey-embedding-splicer/src/splice.rs +++ b/crates/spidermonkey-embedding-splicer/src/splice.rs @@ -1,17 +1,17 @@ use std::path::PathBuf; use anyhow::Result; -use orca_wasm::ir::function::{FunctionBuilder, FunctionModifier}; -use orca_wasm::ir::id::{ExportsID, FunctionID, LocalID}; -use orca_wasm::ir::module::Module; -use orca_wasm::ir::types::{BlockType, ElementItems, InstrumentationMode}; -use orca_wasm::module_builder::AddLocal; -use orca_wasm::opcode::{Inject, InjectAt}; -use orca_wasm::{DataType, Opcode}; use wasm_encoder::{Encode, Section}; use wasmparser::ExternalKind; use wasmparser::MemArg; use wasmparser::Operator; +use wirm::ir::function::{FunctionBuilder, FunctionModifier}; +use wirm::ir::id::{ExportsID, FunctionID, LocalID}; +use wirm::ir::module::Module; +use wirm::ir::types::{BlockType, ElementItems, InstrumentationMode}; +use wirm::module_builder::AddLocal; +use wirm::opcode::Inject; +use wirm::{DataType, Opcode}; use wit_component::metadata::{decode, Bindgen}; use wit_component::StringEncoding; use wit_parser::Resolve; @@ -725,7 +725,7 @@ fn synthesize_import_functions( // create imported function table let els = module.elements.iter_mut().next().unwrap(); - if let ElementItems::Functions(ref mut funcs) = &mut els.1 { + if let ElementItems::Functions(ref mut funcs) = &mut els.items { for fid in import_fnids { funcs.push(fid); } @@ -757,32 +757,50 @@ fn synthesize_import_functions( .get_fn_modifier(coreabi_get_import_fid) .unwrap(); - // walk until we get to the const representing the table index - let mut table_instr_idx = 0; - for (idx, instr) in builder.body.instructions.iter_mut().enumerate() { - if let Operator::I32Const { value: ref mut v } = instr.op { - // we specifically need the const "around" 3393 - // which is the coreabi_sample_i32 table offset - if *v < 1000 || *v > 5000 { - continue; + // Find the I32Const base index and compute the delta to new base + let mut table_instr_idx = 0usize; + let mut delta: i32 = 0; + { + let ops_ro = builder.body.instructions.get_ops(); + for (idx, op) in ops_ro.iter().enumerate() { + if let Operator::I32Const { value } = op { + // we specifically need the const "around" 3393 + // which is the coreabi_sample_i32 table offset + if *value < 1000 || *value > 5000 { + continue; + } + delta = import_fn_table_start_idx - *value; + table_instr_idx = idx; + break; } - *v = import_fn_table_start_idx; - table_instr_idx = idx; - break; } } - builder.inject_at( + // Inject adjustments after the located instruction: add delta and add arg index + builder + .body + .instructions + .set_current_mode(table_instr_idx, InstrumentationMode::After); + if delta != 0 { + builder + .body + .instructions + .add_instr(table_instr_idx, Operator::I32Const { value: delta }); + builder + .body + .instructions + .add_instr(table_instr_idx, Operator::I32Add); + } + builder.body.instructions.add_instr( table_instr_idx, - InstrumentationMode::Before, Operator::LocalGet { local_index: *arg_idx, }, ); - builder.inject_at( - table_instr_idx + 1, - InstrumentationMode::Before, - Operator::I32Add, - ); + builder + .body + .instructions + .add_instr(table_instr_idx, Operator::I32Add); + builder.body.instructions.finish_instr(table_instr_idx); } // remove unnecessary exports diff --git a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs index 9fe125b..7484d40 100644 --- a/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs +++ b/crates/spidermonkey-embedding-splicer/src/stub_wasi.rs @@ -3,13 +3,13 @@ use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Result}; -use orca_wasm::ir::function::FunctionBuilder; -use orca_wasm::ir::id::{FunctionID, LocalID}; -use orca_wasm::ir::module::module_functions::FuncKind; -use orca_wasm::ir::types::{BlockType, InitExpr, Value}; -use orca_wasm::module_builder::AddLocal; -use orca_wasm::{DataType, Instructions, Module, Opcode}; use wasmparser::{MemArg, TypeRef}; +use wirm::ir::function::FunctionBuilder; +use wirm::ir::id::{FunctionID, LocalID}; +use wirm::ir::module::module_functions::FuncKind; +use wirm::ir::types::{BlockType, InitExpr, Value}; +use wirm::module_builder::AddLocal; +use wirm::{DataType, InitInstr, Module, Opcode}; use wit_parser::Resolve; use crate::parse_wit; @@ -44,14 +44,14 @@ where }; let ty = module.types.get(ty_id).unwrap(); - let (params, results) = (ty.params().to_vec(), ty.results().to_vec()); - let mut builder = FunctionBuilder::new(params.as_slice(), results.as_slice()); + let mut builder = FunctionBuilder::new(ty.params().as_slice(), ty.results().as_slice()); let _args = stub(&mut builder)?; builder.replace_import_in_module(module, iid); return Ok(Some(fid)); } + Ok(None) } @@ -206,7 +206,7 @@ fn stub_random(module: &mut Module) -> Result<()> { // create a mutable random seed global let seed_val: i64 = 0; let seed_global = module.add_global( - InitExpr::new(vec![Instructions::Value(Value::I64(seed_val))]), + InitExpr::new(vec![InitInstr::Value(Value::I64(seed_val))]), DataType::I64, true, false, diff --git a/package-lock.json b/package-lock.json index 3961e7b..c83bda2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bytecodealliance/componentize-js", - "version": "0.19.0", + "version": "0.19.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bytecodealliance/componentize-js", - "version": "0.19.0", + "version": "0.19.1", "workspaces": [ "." ], diff --git a/src/cli.js b/src/cli.js index 39a1846..5c22570 100755 --- a/src/cli.js +++ b/src/cli.js @@ -6,55 +6,60 @@ import { writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; export async function componentizeCmd(jsSource, opts) { - const { component } = await componentize({ - sourcePath: jsSource, - witPath: resolve(opts.wit), - worldName: opts.worldName, - runtimeArgs: opts.runtimeArgs, - disableFeatures: opts.disable, - preview2Adapter: opts.preview2Adapter, - debugBindings: opts.debugBindings, - debugBuild: opts.useDebugBuild, - enableWizerLogging: opts.enableWizerLogging, - }); - await writeFile(opts.out, component); + const { component } = await componentize({ + sourcePath: jsSource, + witPath: resolve(opts.wit), + worldName: opts.worldName, + runtimeArgs: opts.runtimeArgs, + disableFeatures: opts.disable, + preview2Adapter: opts.preview2Adapter, + debugBindings: opts.debugBindings, + debugBuild: opts.useDebugBuild, + enableWizerLogging: opts.enableWizerLogging, + splicerBin: opts.splicerBin, + }); + await writeFile(opts.out, component); } program - .version('0.19.1') - .description('Create a component from a JavaScript module') - .usage(' --wit wit-world.wit -o ') - .argument('', 'JS source file to build') - .requiredOption('-w, --wit ', 'WIT path to build with') - .option('-n, --world-name ', 'WIT world to build') - .option('--runtime-args ', 'arguments to pass to the runtime') - .addOption( - new Option('-d, --disable ', 'disable WASI features').choices( - DEFAULT_FEATURES, - ), - ) - .option( - '--preview2-adapter ', - 'provide a custom preview2 adapter path', - ) - .option('--use-debug-build', 'use a debug build of StarlingMonkey') - .option('--debug-bindings', 'enable debug logging for bindings generation') - .option( - '--enable-wizer-logging', - 'enable debug logging for calls in the generated component', - ) - .requiredOption('-o, --out ', 'output component file') - .action(asyncAction(componentizeCmd)); + .version('0.19.1') + .description('Create a component from a JavaScript module') + .usage(' --wit wit-world.wit -o ') + .argument('', 'JS source file to build') + .requiredOption('-w, --wit ', 'WIT path to build with') + .option('-n, --world-name ', 'WIT world to build') + .option('--runtime-args ', 'arguments to pass to the runtime') + .addOption( + new Option('-d, --disable ', 'disable WASI features').choices( + DEFAULT_FEATURES, + ), + ) + .option( + '--preview2-adapter ', + 'provide a custom preview2 adapter path', + ) + .option('--use-debug-build', 'use a debug build of StarlingMonkey') + .option('--debug-bindings', 'enable debug logging for bindings generation') + .option( + '--enable-wizer-logging', + 'enable debug logging for calls in the generated component', + ) + .option( + '--splicer-bin ', + 'use native CLI splicer for better performance', + ) + .requiredOption('-o, --out ', 'output component file') + .action(asyncAction(componentizeCmd)); program.showHelpAfterError(); program.parse(); function asyncAction(cmd) { - return function () { - const args = [...arguments]; - (async () => { - await cmd.apply(null, args); - })(); - }; + return function () { + const args = [...arguments]; + (async () => { + await cmd.apply(null, args); + })(); + }; } diff --git a/src/componentize.js b/src/componentize.js index 5bd6c0e..5a6c338 100644 --- a/src/componentize.js +++ b/src/componentize.js @@ -2,7 +2,7 @@ import { freemem } from 'node:os'; import { TextDecoder } from 'node:util'; import { Buffer } from 'node:buffer'; import { fileURLToPath, URL } from 'node:url'; -import { cwd, stdout, platform } from 'node:process'; +import { cwd, stdout } from 'node:process'; import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { resolve, join, dirname } from 'node:path'; @@ -13,17 +13,21 @@ import { createHash } from 'node:crypto'; import oxc from 'oxc-parser'; import wizer from '@bytecodealliance/wizer'; import { - componentNew, - metadataAdd, - preview1AdapterReactorPath, + componentNew, + metadataAdd, + preview1AdapterReactorPath, } from '@bytecodealliance/jco'; -import { splicer } from '../lib/spidermonkey-embedding-splicer.js'; - import { maybeWindowsPath } from './platform.js'; +import { + spliceBindingsCli, + stubWasiCli, + spliceBindingsWasm, + stubWasiWasm, +} from './splicer.js'; export const { version } = JSON.parse( - await readFile(new URL('../package.json', import.meta.url), 'utf8'), + await readFile(new URL('../package.json', import.meta.url), 'utf8'), ); /** Prefix into wizer error output that indicates a error/trap */ @@ -45,328 +49,382 @@ const CHECK_INIT_RETURN_TYPE_PARSE = 2; /** Default settings for debug options */ const DEFAULT_DEBUG_SETTINGS = { - bindings: false, - bindingsDir: null, + bindings: false, + bindingsDir: null, - binary: false, - binaryPath: null, + binary: false, + binaryPath: null, - wizerLogging: false, + wizerLogging: false, }; /** Features that are used by default if not explicitly disabled */ export const DEFAULT_FEATURES = ['stdio', 'random', 'clocks', 'http', 'fetch-event']; export async function componentize( - opts, - _deprecatedWitWorldOrOpts = undefined, - _deprecatedOpts = undefined, + opts, + _deprecatedWitWorldOrOpts = undefined, + _deprecatedOpts = undefined, ) { - let useOriginalSourceFile = true; - let jsSource; - - // Handle the two old signatures - // (jsSource, witWorld, opts?) - // (jsSource, opts) - if (typeof opts === 'string') { - jsSource = opts; - useOriginalSourceFile = false; - if (typeof _deprecatedWitWorldOrOpts === 'string') { - opts = _deprecatedOpts || {}; - opts.witWorld = _deprecatedWitWorldOrOpts; - } else { - if (typeof _deprecatedWitWorldOrOpts !== 'object') { - throw new Error( - `componentize: second argument must be an object or a string, but is ${typeof _deprecatedWitWorldOrOpts}`, - ); - } - opts = _deprecatedWitWorldOrOpts; - } - } - - // Prepare a working directory for use during componentization - const { sourcesDir, baseDir: workDir } = await prepWorkDir(); - - let { - sourceName = 'source.js', - sourcePath = maybeWindowsPath(join(sourcesDir, sourceName)), - preview2Adapter = preview1AdapterReactorPath(), - witPath, - witWorld, - worldName, - disableFeatures = [], - enableFeatures = [], - - debug = { ...DEFAULT_DEBUG_SETTINGS }, - debugBuild = false, - debugBindings = false, - enableWizerLogging = false, - - runtimeArgs, - - } = opts; - - debugBindings = debugBindings || debug?.bindings; - debugBuild = debugBuild || debug?.build; - enableWizerLogging = enableWizerLogging || debug?.enableWizerLogging; - - // Determine the path to the StarlingMonkey binary - const engine = getEnginePath(opts); - - // Determine the default features that should be included - const features = new Set(); - for (let f of DEFAULT_FEATURES) { - if (!disableFeatures.includes(f)) { - features.add(f); - } - } - - if (!jsSource && sourcePath) { - jsSource = await readFile(sourcePath, 'utf8'); - } - const detectedExports = await detectKnownSourceExportNames( - sourceName, - jsSource, - ); - - // If there is an export of incomingHandler, there is likely to be a - // manual implementation of wasi:http/incoming-handler, so we should - // disable fetch-event - if (features.has('http') && detectedExports.has('incomingHandler')) { - if (debugBindings) { - console.error( - 'Detected `incomingHandler` export, disabling fetch-event...', - ); - } - features.delete('fetch-event'); - } - - // Splice the bindigns for the given WIT world into the engine WASM - let { wasm, jsBindings, exports, imports } = splicer.spliceBindings( - await readFile(engine), - [...features], - witWorld, - maybeWindowsPath(witPath), - worldName, - false, - ); - - const inputWasmPath = join(workDir, 'in.wasm'); - const outputWasmPath = join(workDir, 'out.wasm'); - - await writeFile(inputWasmPath, Buffer.from(wasm)); - let initializerPath = maybeWindowsPath(join(sourcesDir, 'initializer.js')); - await writeFile(initializerPath, jsBindings); - - if (debugBindings) { - // If a bindings output directory was specified, output generated bindings to files - if (debug?.bindingsDir) { - console.error(`Storing debug files in "${debug?.bindingsDir}"\n`); - // Ensure the debug bindings dir exists, and is a directory - if (!(await stat(debug?.bindingsDir).then((s) => s.isDirectory()))) { - throw new Error( - `Missing/invalid debug bindings directory [${debug?.bindingsDir}]`, - ); - } - // Write debug to bindings debug directory - await Promise.all([ - writeFile(join(debug?.bindingsDir, 'source.debug.js'), jsSource), - writeFile(join(debug?.bindingsDir, 'bindings.debug.js'), jsBindings), - writeFile( - join(debug?.bindingsDir, 'imports.debug.json'), - JSON.stringify(imports, null, 2), - ), - writeFile( - join(debug?.bindingsDir, 'exports.debug.json'), - JSON.stringify(exports, null, 2), - ), - ]); - } else { - // If a bindings output directory was not specified, output to stdout - console.error('--- JS Bindings ---'); - console.error( - jsBindings - .split('\n') - .map((ln, idx) => `${(idx + 1).toString().padStart(4, ' ')} | ${ln}`) - .join('\n'), - ); - console.error('--- JS Imports ---'); - console.error(imports); - console.error('--- JS Exports ---'); - console.error(exports); - } - } - - if (!useOriginalSourceFile) { - if (debugBindings) { - console.error(`> Writing JS source to ${tmpDir}/sources`); - } - await writeFile(sourcePath, jsSource); - } - - let hostenv = {}; - - if (opts.env) { - hostenv = typeof opts.env === 'object' ? opts.env : process.env; - } - - const env = { - ...hostenv, - DEBUG: enableWizerLogging ? '1' : '', - SOURCE_NAME: sourceName, - EXPORT_CNT: exports.length.toString(), - FEATURE_CLOCKS: features.has('clocks') ? '1' : '', - }; - - for (const [idx, [export_name, expt]] of exports.entries()) { - env[`EXPORT${idx}_NAME`] = export_name; - env[`EXPORT${idx}_ARGS`] = - (expt.paramptr ? '*' : '') + expt.params.join(','); - env[`EXPORT${idx}_RET`] = (expt.retptr ? '*' : '') + (expt.ret || ''); - env[`EXPORT${idx}_RETSIZE`] = String(expt.retsize); - } - - for (let i = 0; i < imports.length; i++) { - env[`IMPORT${i}_NAME`] = imports[i][1]; - env[`IMPORT${i}_ARGCNT`] = String(imports[i][2]); - } - env['IMPORT_CNT'] = imports.length; - - if (debugBindings) { - console.error('--- Wizer Env ---'); - console.error(env); - } - - sourcePath = maybeWindowsPath(sourcePath); - let workspacePrefix = dirname(sourcePath); - - // If the source path is within the current working directory, strip the - // cwd as a prefix from the source path, and remap the paths seen by the - // component to be relative to the current working directory. - // This only works in wizer. - if (!useOriginalSourceFile) { - workspacePrefix = sourcesDir; - sourcePath = sourceName; - } - let currentDir = maybeWindowsPath(cwd()); - if (workspacePrefix.startsWith(currentDir)) { - workspacePrefix = currentDir; - sourcePath = sourcePath.slice(workspacePrefix.length + 1); - } - - let args = `--initializer-script-path ${initializerPath} --strip-path-prefix ${workspacePrefix}/ ${sourcePath}`; - runtimeArgs = runtimeArgs ? `${runtimeArgs} ${args}` : args; - - let preopens = [`--dir ${sourcesDir}`]; - preopens.push(`--mapdir /::${workspacePrefix}`); - - let postProcess; - - const wizerBin = opts.wizerBin ?? wizer; - postProcess = spawnSync( - wizerBin, - [ - '--allow-wasi', - '--init-func', - 'componentize.wizer', - ...preopens, - `--wasm-bulk-memory=true`, - '--inherit-env=true', - `-o=${outputWasmPath}`, - inputWasmPath, - ], - { - stdio: [null, stdout, 'pipe'], - env, - input: runtimeArgs, - shell: true, - encoding: 'utf-8', - }, - ); - - // If the wizer process failed, parse the output and display to the user - if (postProcess.status !== 0) { - let wizerErr = parseWizerStderr(postProcess.stderr); - let err = `Failed to initialize component:\n${wizerErr}`; - if (debugBindings) { - err += `\n\nBinary and sources available for debugging at ${workDir}\n`; - } else { - await rm(workDir, { recursive: true }); - } - throw new Error(err); - } - - // Read the generated WASM back into memory - const bin = await readFile(outputWasmPath); - - // Check for initialization errors, by actually executing the binary in - // a mini sandbox to get back the initialization state - const { - exports: { check_init }, - getStderr, - } = await initWasm(bin); - - // If not in debug mode, clean up - if (!debugBindings) { - await rm(workDir, { recursive: true }); - } - - /// Process output of check init, throwing if necessary - await handleCheckInitOutput( - check_init(), - initializerPath, - workDir, - getStderr, - ); - - // After wizening, stub out the wasi imports depending on what features are enabled - const finalBin = splicer.stubWasi( - bin, - [...features], - witWorld, - maybeWindowsPath(witPath), - worldName, - ); - - if (debugBindings) { - await writeFile('binary.wasm', finalBin); - } - - const component = await metadataAdd( - await componentNew( - finalBin, - Object.entries({ - wasi_snapshot_preview1: await readFile(preview2Adapter), - }), - false, - ), - Object.entries({ - language: [['JavaScript', '']], - 'processed-by': [['ComponentizeJS', version]], - }), - ); - - // Convert CABI import conventions to ESM import conventions - imports = imports.map(([specifier, impt]) => - specifier === '$root' ? [impt, 'default'] : [specifier, impt], - ); - - // Build debug object to return - let debugOutput; - if (debugBindings) { - debugOutput.bindings = debug.bindings; - debugOutput.workDir = workDir; - } - if (debug?.binary) { - debugOutput.binary = debug.binary; - debugOutput.binaryPath = debug.binaryPath; - } - - return { - component, - imports, - debug: debugOutput, - }; + const t_start = Date.now(); + let useOriginalSourceFile = true; + let jsSource; + + // Handle the two old signatures + // (jsSource, witWorld, opts?) + // (jsSource, opts) + if (typeof opts === 'string') { + jsSource = opts; + useOriginalSourceFile = false; + if (typeof _deprecatedWitWorldOrOpts === 'string') { + opts = _deprecatedOpts || {}; + opts.witWorld = _deprecatedWitWorldOrOpts; + } else { + if (typeof _deprecatedWitWorldOrOpts !== 'object') { + throw new Error( + `componentize: second argument must be an object or a string, but is ${typeof _deprecatedWitWorldOrOpts}`, + ); + } + opts = _deprecatedWitWorldOrOpts; + } + } + + // Prepare a working directory for use during componentization + const { sourcesDir, baseDir: workDir } = await prepWorkDir(); + + let { + sourceName = 'source.js', + sourcePath = maybeWindowsPath(join(sourcesDir, sourceName)), + preview2Adapter = preview1AdapterReactorPath(), + witPath, + witWorld, + worldName, + disableFeatures = [], + enableFeatures = [], + + debug = { ...DEFAULT_DEBUG_SETTINGS }, + debugBuild = false, + debugBindings = false, + enableWizerLogging = false, + splicerBin, + + runtimeArgs, + + } = opts; + + debugBindings = debugBindings || debug?.bindings; + debugBuild = debugBuild || debug?.build; + enableWizerLogging = enableWizerLogging || debug?.enableWizerLogging; + + // Determine the path to the StarlingMonkey binary + const engine = getEnginePath(opts); + + // Determine the default features that should be included + const features = new Set(); + for (let f of DEFAULT_FEATURES) { + if (!disableFeatures.includes(f)) { + features.add(f); + } + } + + if (!jsSource && sourcePath) { + jsSource = await readFile(sourcePath, 'utf8'); + } + const detectedExports = await detectKnownSourceExportNames( + sourceName, + jsSource, + ); + + // If there is an export of incomingHandler, there is likely to be a + // manual implementation of wasi:http/incoming-handler, so we should + // disable fetch-event + if (features.has('http') && detectedExports.has('incomingHandler')) { + if (debugBindings) { + console.error( + 'Detected `incomingHandler` export, disabling fetch-event...', + ); + } + features.delete('fetch-event'); + } + + // Splice the bindings for the given WIT world into the engine WASM + const engineWasm = await readFile(engine); + + let result; + try { + if (splicerBin != undefined) { + result = await spliceBindingsCli( + engineWasm, + [...features], + witPath, + worldName, + debugBindings, + workDir, + opts, + ); + } else { + result = await spliceBindingsWasm( + engineWasm, + [...features], + witWorld, + witPath, + worldName, + debugBindings, + ); + } + } catch (err) { + if (debugBindings) { + console.error(`\n\nBinary and sources available for debugging at ${workDir}\n`); + } else { + await rm(workDir, { recursive: true }); + } + throw err; + } + + let wasm = result.wasm; + let jsBindings = result.jsBindings; + let exports = result.exports; + let imports = result.imports; + + const inputWasmPath = join(workDir, 'in.wasm'); + const outputWasmPath = join(workDir, 'out.wasm'); + + await writeFile(inputWasmPath, Buffer.from(wasm)); + let initializerPath = maybeWindowsPath(join(sourcesDir, 'initializer.js')); + await writeFile(initializerPath, jsBindings); + + if (debugBindings) { + // If a bindings output directory was specified, output generated bindings to files + if (debug?.bindingsDir) { + console.error(`Storing debug files in "${debug?.bindingsDir}"\n`); + // Ensure the debug bindings dir exists, and is a directory + if (!(await stat(debug?.bindingsDir).then((s) => s.isDirectory()))) { + throw new Error( + `Missing/invalid debug bindings directory [${debug?.bindingsDir}]`, + ); + } + // Write debug to bindings debug directory + await Promise.all([ + writeFile(join(debug?.bindingsDir, 'source.debug.js'), jsSource), + writeFile(join(debug?.bindingsDir, 'bindings.debug.js'), jsBindings), + writeFile( + join(debug?.bindingsDir, 'imports.debug.json'), + JSON.stringify(imports, null, 2), + ), + writeFile( + join(debug?.bindingsDir, 'exports.debug.json'), + JSON.stringify(exports, null, 2), + ), + ]); + } else { + // If a bindings output directory was not specified, output to stdout + console.error('--- JS Bindings ---'); + console.error( + jsBindings + .split('\n') + .map((ln, idx) => `${(idx + 1).toString().padStart(4, ' ')} | ${ln}`) + .join('\n'), + ); + console.error('--- JS Imports ---'); + console.error(imports); + console.error('--- JS Exports ---'); + console.error(exports); + } + } + + if (!useOriginalSourceFile) { + if (debugBindings) { + console.error(`> Writing JS source to ${tmpDir}/sources`); + } + await writeFile(sourcePath, jsSource); + } + + let hostenv = {}; + + if (opts.env) { + hostenv = typeof opts.env === 'object' ? opts.env : process.env; + } + + const env = { + ...hostenv, + DEBUG: enableWizerLogging ? '1' : '', + SOURCE_NAME: sourceName, + EXPORT_CNT: exports.length.toString(), + FEATURE_CLOCKS: features.has('clocks') ? '1' : '', + }; + + for (const [idx, [export_name, expt]] of exports.entries()) { + env[`EXPORT${idx}_NAME`] = export_name; + env[`EXPORT${idx}_ARGS`] = + (expt.paramptr ? '*' : '') + expt.params.join(','); + env[`EXPORT${idx}_RET`] = (expt.retptr ? '*' : '') + (expt.ret || ''); + env[`EXPORT${idx}_RETSIZE`] = String(expt.retsize); + } + + for (let i = 0; i < imports.length; i++) { + env[`IMPORT${i}_NAME`] = imports[i][1]; + env[`IMPORT${i}_ARGCNT`] = String(imports[i][2]); + } + env['IMPORT_CNT'] = imports.length; + + if (debugBindings) { + console.error('--- Wizer Env ---'); + console.error(env); + } + + sourcePath = maybeWindowsPath(sourcePath); + let workspacePrefix = dirname(sourcePath); + + // If the source path is within the current working directory, strip the + // cwd as a prefix from the source path, and remap the paths seen by the + // component to be relative to the current working directory. + // This only works in wizer. + if (!useOriginalSourceFile) { + workspacePrefix = sourcesDir; + sourcePath = sourceName; + } + let currentDir = maybeWindowsPath(cwd()); + if (workspacePrefix.startsWith(currentDir)) { + workspacePrefix = currentDir; + sourcePath = sourcePath.slice(workspacePrefix.length + 1); + } + + let args = `--initializer-script-path ${initializerPath} --strip-path-prefix ${workspacePrefix}/ ${sourcePath}`; + runtimeArgs = runtimeArgs ? `${runtimeArgs} ${args}` : args; + + let preopens = [`--dir ${sourcesDir}`]; + preopens.push(`--mapdir /::${workspacePrefix}`); + + let postProcess; + + const wizerBin = opts.wizerBin ?? wizer; + postProcess = spawnSync( + wizerBin, + [ + '--allow-wasi', + '--init-func', + 'componentize.wizer', + ...preopens, + `--wasm-bulk-memory=true`, + '--inherit-env=true', + `-o=${outputWasmPath}`, + inputWasmPath, + ], + { + stdio: [null, stdout, 'pipe'], + env, + input: runtimeArgs, + shell: true, + encoding: 'utf-8', + }, + ); + + // If the wizer process failed, parse the output and display to the user + if (postProcess.status !== 0) { + let wizerErr = parseWizerStderr(postProcess.stderr); + let err = `Failed to initialize component:\n${wizerErr}`; + if (debugBindings) { + err += `\n\nBinary and sources available for debugging at ${workDir}\n`; + } else { + await rm(workDir, { recursive: true }); + } + throw new Error(err); + } + + // Read the generated WASM back into memory + const bin = await readFile(outputWasmPath); + + // Check for initialization errors, by actually executing the binary in + // a mini sandbox to get back the initialization state + const { + exports: { check_init }, + getStderr, + } = await initWasm(bin); + + /// Process output of check init, throwing if necessary + await handleCheckInitOutput( + check_init(), + initializerPath, + workDir, + getStderr, + ); + + // After wizening, stub out the wasi imports depending on what features are enabled + let finalBin; + try { + if (splicerBin != undefined) { + finalBin = await stubWasiCli( + bin, + [...features], + witPath, + worldName, + workDir, + opts, + ); + } else { + finalBin = await stubWasiWasm( + bin, + [...features], + witWorld, + witPath, + worldName, + ); + } + } catch (err) { + if (debugBindings) { + console.error(`\n\nBinary and sources available for debugging at ${workDir}\n`); + } else { + await rm(workDir, { recursive: true }); + } + throw err; + } + + if (debugBindings) { + await writeFile('binary.wasm', finalBin); + } + + const adapterBytes = await readFile(preview2Adapter); + const rt = await componentNew( + finalBin, + Object.entries({ + wasi_snapshot_preview1: adapterBytes, + }), + false, + ); + const component = await metadataAdd( + rt, + Object.entries({ + language: [['JavaScript', '']], + 'processed-by': [['ComponentizeJS', version]], + }), + ); + + // Convert CABI import conventions to ESM import conventions + imports = imports.map(([specifier, impt]) => + specifier === '$root' ? [impt, 'default'] : [specifier, impt], + ); + + // Clean up temporary directory unless in debug mode + if (!debugBindings) { + await rm(workDir, { recursive: true }); + } + + // Build debug object to return + let debugOutput; + if (debugBindings) { + debugOutput.bindings = debug.bindings; + debugOutput.workDir = workDir; + } + if (debug?.binary) { + debugOutput.binary = debug.binary; + debugOutput.binaryPath = debug.binaryPath; + } + + return { + component, + imports, + debug: debugOutput, + }; } /** @@ -376,8 +434,8 @@ export async function componentize( * @returns {number} The minimum stack size that should be used as a default. */ function defaultMinStackSize(freeMemoryBytes) { - freeMemoryBytes = freeMemoryBytes ?? freemem(); - return Math.max(8 * 1024 * 1024, Math.floor(freeMemoryBytes * 0.1)); + freeMemoryBytes = freeMemoryBytes ?? freemem(); + return Math.max(8 * 1024 * 1024, Math.floor(freeMemoryBytes * 0.1)); } /** @@ -385,13 +443,13 @@ function defaultMinStackSize(freeMemoryBytes) { * found as line prefixes. */ function stripLinesPrefixes(input, prefixPatterns) { - return input - .split('\n') - .map((line) => - prefixPatterns.reduce((line, n) => line.replace(n, ''), line), - ) - .join('\n') - .trim(); + return input + .split('\n') + .map((line) => + prefixPatterns.reduce((line, n) => line.replace(n, ''), line), + ) + .join('\n') + .trim(); } /** @@ -401,15 +459,15 @@ function stripLinesPrefixes(input, prefixPatterns) { * @returns {string} String that can be printed to describe error output */ function parseWizerStderr(stderr) { - let output = `${stderr}`; - let causeStart = output.indexOf(WIZER_ERROR_CAUSE_PREFIX); - let exitCodeStart = output.indexOf(WIZER_EXIT_CODE_PREFIX); - if (causeStart === -1 || exitCodeStart === -1) { - return output; - } - - let causeEnd = output.indexOf('\n', exitCodeStart + 1); - return `${output.substring(0, causeStart)}${output.substring(causeEnd)}`.trim(); + let output = `${stderr}`; + let causeStart = output.indexOf(WIZER_ERROR_CAUSE_PREFIX); + let exitCodeStart = output.indexOf(WIZER_EXIT_CODE_PREFIX); + if (causeStart === -1 || exitCodeStart === -1) { + return output; + } + + let causeEnd = output.indexOf('\n', exitCodeStart + 1); + return `${output.substring(0, causeStart)}${output.substring(causeEnd)}`.trim(); } /** @@ -419,43 +477,43 @@ function parseWizerStderr(stderr) { * @returns {boolean} whether the value is numeric */ function isNumeric(n) { - switch (typeof n) { - case 'bigint': - case 'number': - return true; - case 'object': - return n.constructor == BigInt || n.constructor == Number; - default: - return false; - } + switch (typeof n) { + case 'bigint': + case 'number': + return true; + case 'object': + return n.constructor == BigInt || n.constructor == Number; + default: + return false; + } } /** Determine the correct path for the engine */ function getEnginePath(opts) { - if (opts.engine) { - return opts.engine; - } - const debugSuffix = opts?.debugBuild ? '.debug' : ''; - let engineBinaryRelPath = `../lib/starlingmonkey_embedding${debugSuffix}.wasm`; + if (opts.engine) { + return opts.engine; + } + const debugSuffix = opts?.debugBuild ? '.debug' : ''; + let engineBinaryRelPath = `../lib/starlingmonkey_embedding${debugSuffix}.wasm`; - return fileURLToPath(new URL(engineBinaryRelPath, import.meta.url)); + return fileURLToPath(new URL(engineBinaryRelPath, import.meta.url)); } /** Prepare a work directory for use with componentization */ async function prepWorkDir() { - const baseDir = maybeWindowsPath( - join( - tmpdir(), - createHash('sha256') - .update(Math.random().toString()) - .digest('hex') - .slice(0, 12), - ), - ); - await mkdir(baseDir); - const sourcesDir = maybeWindowsPath(join(baseDir, 'sources')); - await mkdir(sourcesDir); - return { baseDir, sourcesDir }; + const baseDir = maybeWindowsPath( + join( + tmpdir(), + createHash('sha256') + .update(Math.random().toString()) + .digest('hex') + .slice(0, 12), + ), + ); + await mkdir(baseDir); + const sourcesDir = maybeWindowsPath(join(baseDir, 'sources')); + await mkdir(sourcesDir); + return { baseDir, sourcesDir }; } /** @@ -465,47 +523,47 @@ async function prepWorkDir() { * @throws If a binary is invalid */ async function initWasm(bin) { - const eep = (name) => () => { - throw new Error( - `Internal error: unexpected call to "${name}" during Wasm verification`, - ); - }; - - let stderr = ''; - const wasmModule = await WebAssembly.compile(bin); - - const mockImports = { - wasi_snapshot_preview1: { - fd_write: function (fd, iovs, iovs_len, nwritten) { - if (fd !== 2) return 0; - const mem = new DataView(exports.memory.buffer); - let written = 0; - for (let i = 0; i < iovs_len; i++) { - const bufPtr = mem.getUint32(iovs + i * 8, true); - const bufLen = mem.getUint32(iovs + 4 + i * 8, true); - stderr += new TextDecoder().decode( - new Uint8Array(exports.memory.buffer, bufPtr, bufLen), - ); - written += bufLen; - } - mem.setUint32(nwritten, written, true); - return 1; - }, - }, - }; - - for (const { module, name } of WebAssembly.Module.imports(wasmModule)) { - mockImports[module] = mockImports[module] || {}; - if (!mockImports[module][name]) mockImports[module][name] = eep(name); - } - - const { exports } = await WebAssembly.instantiate(wasmModule, mockImports); - return { - exports, - getStderr() { - return stderr; - }, - }; + const eep = (name) => () => { + throw new Error( + `Internal error: unexpected call to "${name}" during Wasm verification`, + ); + }; + + let stderr = ''; + const wasmModule = await WebAssembly.compile(bin); + + const mockImports = { + wasi_snapshot_preview1: { + fd_write: function (fd, iovs, iovs_len, nwritten) { + if (fd !== 2) return 0; + const mem = new DataView(exports.memory.buffer); + let written = 0; + for (let i = 0; i < iovs_len; i++) { + const bufPtr = mem.getUint32(iovs + i * 8, true); + const bufLen = mem.getUint32(iovs + 4 + i * 8, true); + stderr += new TextDecoder().decode( + new Uint8Array(exports.memory.buffer, bufPtr, bufLen), + ); + written += bufLen; + } + mem.setUint32(nwritten, written, true); + return 1; + }, + }, + }; + + for (const { module, name } of WebAssembly.Module.imports(wasmModule)) { + mockImports[module] = mockImports[module] || {}; + if (!mockImports[module][name]) mockImports[module][name] = eep(name); + } + + const { exports } = await WebAssembly.instantiate(wasmModule, mockImports); + return { + exports, + getStderr() { + return stderr; + }, + }; } /** @@ -517,33 +575,33 @@ async function initWasm(bin) { * @param {() => string} getStderr - A function that resolves to the stderr output of check init */ async function handleCheckInitOutput( - status, - initializerPath, - workDir, - getStderr, + status, + initializerPath, + workDir, + getStderr, ) { - let err = null; - switch (status) { - case CHECK_INIT_RETURN_OK: - break; - case CHECK_INIT_RETURN_FN_LIST: - err = `Unable to extract expected exports list`; - break; - case CHECK_INIT_RETURN_TYPE_PARSE: - err = `Unable to parse the core ABI export types`; - break; - default: - err = `Unknown error during initialization: ${status}`; - } - - if (err) { - let msg = err; - const stderr = getStderr(); - if (stderr) { - msg += `\n${stripLinesPrefixes(stderr, [new RegExp(`${initializerPath}[:\\d]* ?`)], workDir)}`; - } - throw new Error(msg); - } + let err = null; + switch (status) { + case CHECK_INIT_RETURN_OK: + break; + case CHECK_INIT_RETURN_FN_LIST: + err = `Unable to extract expected exports list`; + break; + case CHECK_INIT_RETURN_TYPE_PARSE: + err = `Unable to parse the core ABI export types`; + break; + default: + err = `Unknown error during initialization: ${status}`; + } + + if (err) { + let msg = err; + const stderr = getStderr(); + if (stderr) { + msg += `\n${stripLinesPrefixes(stderr, [new RegExp(`${initializerPath}[:\\d]* ?`)], workDir)}`; + } + throw new Error(msg); + } } /** @@ -554,27 +612,27 @@ async function handleCheckInitOutput( * @returns {Promise} A Promise that resolves to a list of string that represent unversioned interfaces */ async function detectKnownSourceExportNames(filename, code) { - if (!filename) { - throw new Error('missing filename'); - } - if (!code) { - throw new Error('missing JS code'); - } - - const names = new Set(); - - const results = await oxc.parseAsync(filename, code); - if (results.errors.length > 0) { - throw new Error( - `failed to parse JS source, encountered [${results.errors.length}] errors`, - ); - } - - for (const staticExport of results.module.staticExports) { - for (const entry of staticExport.entries) { - names.add(entry.exportName.name); - } - } - - return names; + if (!filename) { + throw new Error('missing filename'); + } + if (!code) { + throw new Error('missing JS code'); + } + + const names = new Set(); + + const results = await oxc.parseAsync(filename, code); + if (results.errors.length > 0) { + throw new Error( + `failed to parse JS source, encountered [${results.errors.length}] errors`, + ); + } + + for (const staticExport of results.module.staticExports) { + for (const entry of staticExport.entries) { + names.add(entry.exportName.name); + } + } + + return names; } diff --git a/src/splicer.js b/src/splicer.js new file mode 100644 index 0000000..aaf178c --- /dev/null +++ b/src/splicer.js @@ -0,0 +1,230 @@ +import { spawnSync } from 'node:child_process'; +import { join } from 'node:path'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { platform } from 'node:process'; +import fs from 'node:fs'; + +import { splicer as wasmSplicer } from '../lib/spidermonkey-embedding-splicer.js'; +import { maybeWindowsPath } from './platform.js'; + +/** + * Get the path to the native splicer binary + * @param {Object} opts - Options object + * @returns {string} Path to the splicer binary + */ +function getSplicerBinaryPath(opts) { + if (opts.splicerBin) { + return opts.splicerBin; + } + // The splicer binary should be in target/release or target/debug + const mode = opts?.debugBuild ? 'debug' : 'release'; + const binaryName = platform === 'win32' ? 'splicer.exe' : 'splicer'; + const splicerBinaryRelPath = `../target/${mode}/${binaryName}`; + + return fileURLToPath(new URL(splicerBinaryRelPath, import.meta.url)); +} + +/** + * Run the splicer CLI with the given command and options + * @param {string} command - The splicer command (e.g., 'splice-bindings', 'stub-wasi') + * @param {string[]} baseArgs - Command-specific base arguments + * @param {string[]} features - List of features to enable + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {Object} opts - Additional options (for binary path) + * @returns {Promise} Throws if the command fails + */ +async function runSplicerCli( + command, + baseArgs, + features, + witPath, + worldName, + opts = {}, +) { + const splicerBin = getSplicerBinaryPath(opts); + + const args = [command, ...baseArgs]; + + for (const feature of features) { + args.push('--features', feature); + } + + // Add WIT path if provided + if (witPath) { + args.push('--wit-path', maybeWindowsPath(witPath)); + } + + // Add world name if provided + if (worldName) { + args.push('--world-name', worldName); + } + + // Run splicer CLI + console.error(`trace(splicer-cli:${command}): starting`); + if (!fs.existsSync(splicerBin)) { + throw new Error(`Failed to run splicer '${splicerBin} ${command}': splicer binary not found at ${splicerBin}`); + } + + const process = spawnSync(splicerBin, args, { + stdio: ['pipe', 'inherit', 'inherit'], + encoding: 'utf-8', + }); + + if (process.status !== 0) { + throw new Error(`Failed to run '${splicerBin} ${command}': exited with status ${process.status}`); + } +} + +/** + * Splice bindings using the native CLI binary + * @param {Buffer} engineWasm - The engine WASM binary + * @param {string[]} features - List of features to enable + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {boolean} debug - Enable debug mode + * @param {string} workDir - Working directory for temporary files + * @param {Object} opts - Additional options (for binary path) + * @returns {Promise<{wasm: Buffer, jsBindings: string, exports: Array, imports: Array}>} + */ +export async function spliceBindingsCli( + engineWasm, + features, + witPath, + worldName, + debug, + workDir, + opts = {}, +) { + // Prepare temporary directory for splicer output + const splicerOutDir = join(workDir, 'splicer-out'); + await mkdir(splicerOutDir); + + // Write engine wasm to temp file + const engineInputPath = join(workDir, 'engine.wasm'); + await writeFile(engineInputPath, engineWasm); + + // Build command-specific arguments + const baseArgs = [ + '--input', engineInputPath, + '--out-dir', splicerOutDir, + ]; + + // Add debug flag if needed + if (debug) { + baseArgs.push('--debug'); + } + + // Run splicer CLI + await runSplicerCli('splice-bindings', baseArgs, features, witPath, worldName, opts); + + // Read the outputs + const wasm = await readFile(join(splicerOutDir, 'component.wasm')); + const jsBindings = await readFile(join(splicerOutDir, 'initializer.js'), 'utf8'); + const exports = JSON.parse(await readFile(join(splicerOutDir, 'exports.json'), 'utf8')); + const imports = JSON.parse(await readFile(join(splicerOutDir, 'imports.json'), 'utf8')); + + return { wasm, jsBindings, exports, imports }; +} + +/** + * Stub WASI imports using the native CLI binary + * @param {Buffer} wasmBinary - The WASM binary to stub + * @param {string[]} features - List of features to enable + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {string} workDir - Working directory for temporary files + * @param {Object} opts - Additional options (for binary path) + * @returns {Promise} The stubbed WASM binary + */ +export async function stubWasiCli( + wasmBinary, + features, + witPath, + worldName, + workDir, + opts = {}, +) { + // Write wasm to temp file for stubbing + const stubInputPath = join(workDir, 'stub-input.wasm'); + const stubOutputPath = join(workDir, 'stub-output.wasm'); + await writeFile(stubInputPath, wasmBinary); + + // Build command-specific arguments + const baseArgs = [ + '--input', stubInputPath, + '--output', stubOutputPath, + ]; + + // Run splicer CLI + await runSplicerCli('stub-wasi', baseArgs, features, witPath, worldName, opts); + + // Read the stubbed wasm + const finalBin = await readFile(stubOutputPath); + + return finalBin; +} + +/** + * Splice bindings using the WASM module + * @param {Buffer} engineWasm - The engine WASM binary + * @param {string[]} features - List of features to enable + * @param {string|null} witWorld - WIT world source + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @param {boolean} debug - Enable debug mode + * @returns {Promise<{wasm: Buffer, jsBindings: string, exports: Array, imports: Array}>} + */ +export async function spliceBindingsWasm( + engineWasm, + features, + witWorld, + witPath, + worldName, + debug, +) { + const result = wasmSplicer.spliceBindings( + engineWasm, + features, + witWorld, + maybeWindowsPath(witPath), + worldName, + debug, + ); + + return { + wasm: Buffer.from(result.wasm), + jsBindings: result.jsBindings, + exports: result.exports, + imports: result.imports, + }; +} + +/** + * Stub WASI imports using the WASM module + * @param {Buffer} wasmBinary - The WASM binary to stub + * @param {string[]} features - List of features to enable + * @param {string|null} witWorld - WIT world source + * @param {string|null} witPath - Path to WIT file or directory + * @param {string|null} worldName - Name of the world to use + * @returns {Promise} The stubbed WASM binary + */ +export async function stubWasiWasm( + wasmBinary, + features, + witWorld, + witPath, + worldName, +) { + const result = wasmSplicer.stubWasi( + wasmBinary, + features, + witWorld, + maybeWindowsPath(witPath), + worldName, + ); + + return Buffer.from(result); +} + diff --git a/test/splicer.js b/test/splicer.js new file mode 100644 index 0000000..73e6dae --- /dev/null +++ b/test/splicer.js @@ -0,0 +1,304 @@ +import { fileURLToPath } from 'node:url'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { platform } from 'node:process'; + +import { componentize } from '@bytecodealliance/componentize-js'; +import { transpile } from '@bytecodealliance/jco'; + +import { suite, test, expect } from 'vitest'; + +import { + DEBUG_TRACING_ENABLED, + DEBUG_TEST_ENABLED, + maybeLogging, +} from './util.js'; + +const DEBUG_BINDINGS = process.env.DEBUG_BINDINGS === '1'; + +/** + * Get the path to the splicer binary + */ +function getSplicerBinaryPath() { + const mode = DEBUG_TEST_ENABLED ? 'debug' : 'release'; + const binaryName = platform === 'win32' ? 'splicer.exe' : 'splicer'; + return fileURLToPath( + new URL(`../target/${mode}/${binaryName}`, import.meta.url), + ); +} + +/** + * Helper to create a simple test component + */ +async function createTestComponent(splicerBin, testName) { + const source = ` + export function add(a, b) { + return a + b; + } + `; + + let componentOpts = { + sourceName: `${testName}.js`, + disableFeatures: maybeLogging(['random', 'clocks', 'http', 'stdio']), + debugBuild: DEBUG_TEST_ENABLED, + debugBindings: DEBUG_BINDINGS, + splicerBin, + }; + + // CLI splicer needs a WIT file, WASM splicer can use inline WIT + if (splicerBin) { + // Write WIT to a temp file for CLI splicer + const witContent = ` + package test:test; + + world test { + export add: func(a: s32, b: s32) -> s32; + } + `; + + const witDir = fileURLToPath( + new URL(`./output/${testName}-wit`, import.meta.url), + ); + await mkdir(witDir, { recursive: true }); + await writeFile(`${witDir}/world.wit`, witContent); + + componentOpts.witPath = witDir; + } else { + // WASM splicer can use inline WIT + componentOpts.witWorld = ` + package test:test; + + world test { + export add: func(a: s32, b: s32) -> s32; + } + `; + } + + const { component } = await componentize(source, componentOpts); + + return component; +} + +/** + * Helper to verify a component works correctly + */ +async function verifyComponent(component, testName) { + const map = { + 'wasi:cli-base/*': '@bytecodealliance/preview2-shim/cli-base#*', + 'wasi:clocks/*': '@bytecodealliance/preview2-shim/clocks#*', + 'wasi:filesystem/*': '@bytecodealliance/preview2-shim/filesystem#*', + 'wasi:http/*': '@bytecodealliance/preview2-shim/http#*', + 'wasi:io/*': '@bytecodealliance/preview2-shim/io#*', + 'wasi:logging/*': '@bytecodealliance/preview2-shim/logging#*', + 'wasi:poll/*': '@bytecodealliance/preview2-shim/poll#*', + 'wasi:random/*': '@bytecodealliance/preview2-shim/random#*', + 'wasi:sockets/*': '@bytecodealliance/preview2-shim/sockets#*', + }; + + const { files } = await transpile(component, { + name: testName, + map, + wasiShim: true, + validLiftingOptimization: false, + tracing: DEBUG_TRACING_ENABLED, + }); + + await mkdir(new URL(`./output/${testName}/interfaces`, import.meta.url), { + recursive: true, + }); + + await writeFile( + new URL(`./output/${testName}.component.wasm`, import.meta.url), + component, + ); + + for (const file of Object.keys(files)) { + await writeFile( + new URL(`./output/${testName}/${file}`, import.meta.url), + files[file], + ); + } + + const outputPath = fileURLToPath( + new URL(`./output/${testName}/${testName}.js`, import.meta.url), + ); + + const instance = await import(outputPath); + + // Test the exported function + const result = instance.add(5, 3); + expect(result).toBe(8); + + return instance; +} + +suite('Splicer Integration', async () => { + const splicerBinPath = getSplicerBinaryPath(); + const hasSplicerBinary = existsSync(splicerBinPath); + + if (hasSplicerBinary) { + console.log(`Found splicer binary at: ${splicerBinPath}`); + } else { + console.warn(`Splicer binary not found at: ${splicerBinPath}`); + console.warn('CLI splicer tests will be skipped'); + } + + test.concurrent('CLI splicer produces valid component', async () => { + if (!hasSplicerBinary) { + console.log('Skipping CLI splicer test - binary not found'); + return; + } + + const component = await createTestComponent( + splicerBinPath, + 'splicer-cli-test', + ); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + + await verifyComponent(component, 'splicer-cli-test'); + }); + + test.concurrent('WASM splicer produces valid component', async () => { + const component = await createTestComponent( + undefined, + 'splicer-wasm-test', + ); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + + await verifyComponent(component, 'splicer-wasm-test'); + }); + + test.concurrent( + 'CLI and WASM splicers produce functionally equivalent components', + async () => { + if (!hasSplicerBinary) { + console.log( + 'Skipping equivalence test - CLI splicer binary not found', + ); + return; + } + + const cliComponent = await createTestComponent( + splicerBinPath, + 'splicer-cli-equiv', + ); + const wasmComponent = await createTestComponent( + undefined, + 'splicer-wasm-equiv', + ); + + // Both should produce valid components + expect(cliComponent).toBeInstanceOf(Uint8Array); + expect(wasmComponent).toBeInstanceOf(Uint8Array); + + // Verify both work correctly + const cliInstance = await verifyComponent( + cliComponent, + 'splicer-cli-equiv', + ); + const wasmInstance = await verifyComponent( + wasmComponent, + 'splicer-wasm-equiv', + ); + + // Both should produce the same result + expect(cliInstance.add(10, 20)).toBe(30); + expect(wasmInstance.add(10, 20)).toBe(30); + expect(cliInstance.add(10, 20)).toBe(wasmInstance.add(10, 20)); + }, + ); + + test.concurrent('custom splicer binary path works', async () => { + if (!hasSplicerBinary) { + console.log('Skipping custom path test - binary not found'); + return; + } + + // Test with explicit path + const component = await createTestComponent( + splicerBinPath, + 'splicer-custom-path', + ); + + expect(component).toBeInstanceOf(Uint8Array); + await verifyComponent(component, 'splicer-custom-path'); + }); + + test.concurrent('invalid splicer binary path throws error', async () => { + const invalidPath = '/nonexistent/path/to/splicer'; + + await expect( + createTestComponent(invalidPath, 'splicer-invalid-path'), + ).rejects.toThrow(/splicer binary not found/); + }); + + test.concurrent( + 'CLI splicer with WIT path produces valid component', + async () => { + if (!hasSplicerBinary) { + console.log('Skipping WIT path test - binary not found'); + return; + } + + const source = ` + export function add(a, b) { + return a + b; + } + + export function getResult() { + return 42; + } + `; + + const witPath = fileURLToPath(new URL('./wit', import.meta.url)); + + const { component } = await componentize(source, { + sourceName: 'cli-wit-path-test.js', + witPath, + worldName: 'test2', + disableFeatures: maybeLogging([]), + debugBuild: DEBUG_TEST_ENABLED, + debugBindings: DEBUG_BINDINGS, + splicerBin: splicerBinPath, + }); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + }, + ); + + test.concurrent( + 'WASM splicer with WIT path produces valid component', + async () => { + const source = ` + export function add(a, b) { + return a + b; + } + + export function getResult() { + return 42; + } + `; + + const witPath = fileURLToPath(new URL('./wit', import.meta.url)); + + const { component } = await componentize(source, { + sourceName: 'wasm-wit-path-test.js', + witPath, + worldName: 'test2', + disableFeatures: maybeLogging([]), + debugBuild: DEBUG_TEST_ENABLED, + debugBindings: DEBUG_BINDINGS, + splicerBin: undefined, + }); + + expect(component).toBeInstanceOf(Uint8Array); + expect(component.length).toBeGreaterThan(0); + }, + ); +}); +