diff --git a/.github/workflows/dep_cargo_publish.yml b/.github/workflows/dep_cargo_publish.yml index 6dcf8b8..a172361 100644 --- a/.github/workflows/dep_cargo_publish.yml +++ b/.github/workflows/dep_cargo_publish.yml @@ -24,18 +24,6 @@ jobs: with: rust-toolchain: "1.85.0" - - name: Download Wasm Host (debug) - uses: actions/download-artifact@v4 - with: - name: wasm-runtime-debug - path: ${{ github.workspace }}/src/hyperlight_wasm/redist/debug/ - - - name: Download Wasm Host (release) - uses: actions/download-artifact@v4 - with: - name: wasm-runtime-release - path: ${{ github.workspace }}/src/hyperlight_wasm/redist/release/ - # github actions that run against PRs check out a ref to the PR merge branch # we need to switch / create a branch for cargo ws to run late - name: set-branch-for-PRs diff --git a/Cargo.lock b/Cargo.lock index 82313be..fc1e43b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -895,6 +895,18 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "flatbuffers" version = "25.2.10" @@ -1333,6 +1345,7 @@ dependencies = [ "metrics-util", "once_cell", "page_size", + "tar", "toml", "tracing", "windows", @@ -1674,6 +1687,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.1", "libc", + "redox_syscall", ] [[package]] @@ -2241,6 +2255,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +dependencies = [ + "bitflags 2.9.1", +] + [[package]] name = "redox_users" version = "0.5.0" @@ -2705,6 +2728,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.13.2" @@ -3742,6 +3776,16 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "xattr" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" +dependencies = [ + "libc", + "rustix 1.0.7", +] + [[package]] name = "yoke" version = "0.8.0" diff --git a/Justfile b/Justfile index 72790cd..54259a9 100644 --- a/Justfile +++ b/Justfile @@ -7,6 +7,11 @@ wit-world := if os() == "windows" { "$env:WIT_WORLD=\"" + justfile_directory() + set windows-shell := ["pwsh.exe", "-NoLogo", "-Command"] +make-vendor-tar: + tar cf ./src/hyperlight_wasm/vendor.tar \ + --exclude-vcs-ignores \ + -C ./src wasm_runtime hyperlight_wasm_macro + ensure-tools: cargo install --locked wasm-tools --version 1.235.0 cargo install cargo-component --locked --version 0.21.1 @@ -19,16 +24,12 @@ build target=default-target features="": (build-wasm-runtime target) (fmt-check) mkdir-redist target=default-target: mkdir {{ mkdir-arg }} x64 mkdir {{ mkdir-arg }} x64/{{ target }} - mkdir {{ mkdir-arg }} src/hyperlight_wasm/redist - mkdir {{ mkdir-arg }} src/hyperlight_wasm/redist/{{ target }} -build-wasm-runtime target=default-target: (mkdir-redist target) - cd ./src/wasm_runtime && cargo build --verbose --profile={{ if target == "debug" {"dev"} else { target } }} - cp ./src/wasm_runtime/target/x86_64-unknown-none/{{target}}/wasm_runtime ./x64/{{target}}/wasm_runtime - cp ./src/wasm_runtime/target/x86_64-unknown-none/{{target}}/wasm_runtime ./src/hyperlight_wasm/redist/{{target}}/wasm_runtime +build-wasm-runtime target=default-target: + cd ./src/wasm_runtime && cargo build --verbose --profile={{ if target == "debug" {"dev"} else { target } }} && rm -R target build-wasm-examples target=default-target: - {{ build-wasm-examples-command}} {{target}} + {{ build-wasm-examples-command }} {{target}} build-rust-wasm-examples target=default-target: (mkdir-redist target) rustup target add wasm32-unknown-unknown @@ -50,24 +51,29 @@ check target=default-target: cd src/rust_wasm_samples && cargo check --profile={{ if target == "debug" {"dev"} else { target } }} cd src/component_sample && cargo check --profile={{ if target == "debug" {"dev"} else { target } }} cd src/wasm_runtime && cargo check --profile={{ if target == "debug" {"dev"} else { target } }} + cd src/hyperlight_wasm_macro && cargo check --profile={{ if target == "debug" {"dev"} else { target } }} fmt-check: rustup toolchain install nightly -c rustfmt && cargo +nightly fmt -v --all -- --check cd src/rust_wasm_samples && rustup toolchain install nightly -c rustfmt && cargo +nightly fmt -v --all -- --check cd src/component_sample && rustup toolchain install nightly -c rustfmt && cargo +nightly fmt -v --all -- --check cd src/wasm_runtime && rustup toolchain install nightly -c rustfmt && cargo +nightly fmt -v --all -- --check + cd src/hyperlight_wasm_macro && rustup toolchain install nightly -c rustfmt && cargo +nightly fmt -v --all -- --check + fmt: rustup toolchain install nightly -c rustfmt cargo +nightly fmt --all cd src/rust_wasm_samples && cargo +nightly fmt -v --all cd src/component_sample && cargo +nightly fmt -v --all cd src/wasm_runtime && cargo +nightly fmt -v --all + cd src/hyperlight_wasm_macro && cargo +nightly fmt -v --all clippy target=default-target: (check target) cargo clippy --profile={{ if target == "debug" {"dev"} else { target } }} --all-targets --all-features -- -D warnings cd src/rust_wasm_samples && cargo clippy --profile={{ if target == "debug" {"dev"} else { target } }} --all-targets --all-features -- -D warnings cd src/component_sample && cargo clippy --profile={{ if target == "debug" {"dev"} else { target } }} --all-targets --all-features -- -D warnings cd src/wasm_runtime && cargo clippy --profile={{ if target == "debug" {"dev"} else { target } }} --all-targets --all-features -- -D warnings + cd src/hyperlight_wasm_macro && cargo clippy --profile={{ if target == "debug" {"dev"} else { target } }} --all-targets --all-features -- -D warnings # TESTING # Metrics tests cannot run with other tests they are marked as ignored so that cargo test works diff --git a/src/hyperlight_wasm/.gitignore b/src/hyperlight_wasm/.gitignore new file mode 100644 index 0000000..d12c914 --- /dev/null +++ b/src/hyperlight_wasm/.gitignore @@ -0,0 +1 @@ +vendor.tar \ No newline at end of file diff --git a/src/hyperlight_wasm/Cargo.toml b/src/hyperlight_wasm/Cargo.toml index 561e4fe..2bf39c5 100644 --- a/src/hyperlight_wasm/Cargo.toml +++ b/src/hyperlight_wasm/Cargo.toml @@ -3,7 +3,17 @@ name = "hyperlight-wasm" version = "0.1.0" edition = "2024" rust-version = "1.85" -include = ["*"] # Make sure wasm_runtime is included! +description = "Library that enables wasm modules and components to be run inside lightweight Virtual Machine backed Sandbox. It is built on top of Hyperlight." +license = "Apache-2.0" +repository = "https://github.com/hyperlight-dev/hyperlight-wasm" +include = [ + "/src", + "/scripts", + "/examples", + "benches", + "/build.rs", + "/vendor.tar", # Make sure vendor.tar is included! +] [lib] name = "hyperlight_wasm" @@ -59,6 +69,7 @@ blake3 = "1.8" built = { version = "0.8.0", features = ["chrono", "git2"] } anyhow = { version = "1.0" } goblin = "0.10.0" +tar = "0.4.44" [features] default = ["function_call_metrics", "kvm", "mshv2"] diff --git a/src/hyperlight_wasm/build.rs b/src/hyperlight_wasm/build.rs index c997b5f..7ed72a5 100644 --- a/src/hyperlight_wasm/build.rs +++ b/src/hyperlight_wasm/build.rs @@ -30,25 +30,83 @@ use std::{env, fs}; use anyhow::Result; use built::write_built_file; +fn get_wasm_runtime_path() -> PathBuf { + let manifest_dir = env::var_os("CARGO_MANIFEST_DIR").unwrap(); + let manifest_dir = PathBuf::from(manifest_dir); + + let tar_path = manifest_dir.join("vendor.tar"); + + let out_dir = env::var_os("OUT_DIR").unwrap(); + let out_dir = PathBuf::from(out_dir); + let vendor_dir = out_dir.join("vendor"); + + if vendor_dir.exists() { + fs::remove_dir_all(&vendor_dir).unwrap(); + } + + println!("cargo::rerun-if-changed={}", tar_path.display()); + + // If the vendor.tar file exists, extract it to the OUT_DIR/vendor directory + // and return the wasm_runtime directory inside it. + // This is useful for vendoring the wasm_runtime crate in a release build, since crates.io + // does not allow vendoring folders with Cargo.toml files (i.e., other crates). + // The vendor.tar file is expected to be in the same directory as this build script. + if tar_path.exists() { + let out_dir = env::var_os("OUT_DIR").unwrap(); + let out_dir = PathBuf::from(out_dir); + let vendor_dir = out_dir.join("vendor"); + + let mut tar = tar::Archive::new(fs::File::open(&tar_path).unwrap()); + tar.unpack(&vendor_dir).unwrap(); + + let wasm_runtime_dir = vendor_dir.join("wasm_runtime"); + + println!( + "cargo::warning=using vendor wasm_runtime from {}", + tar_path.display() + ); + return wasm_runtime_dir; + } + + let crates_dir = manifest_dir.parent().unwrap(); + + #[cfg(unix)] + std::os::unix::fs::symlink(crates_dir, &vendor_dir).unwrap(); + + #[cfg(not(unix))] + std::os::windows::fs::symlink_dir(crates_dir, &vendor_dir).unwrap(); + + let wasm_runtime_dir = crates_dir.join("wasm_runtime"); + if wasm_runtime_dir.exists() { + return wasm_runtime_dir; + } + + panic!( + r#" + The wasm_runtime directory not found in the expected locations. + If you are using hyperlight-wasm from a crates.io release, please file an issue: https://github.com/hyperlight-dev/hyperlight-wasm/issues + "# + ); +} + fn build_wasm_runtime() -> PathBuf { - let proj_dir = env::var_os("CARGO_MANIFEST_DIR").unwrap(); + let cargo_bin = env::var_os("CARGO").unwrap(); let profile = env::var_os("PROFILE").unwrap(); + let out_dir = env::var_os("OUT_DIR").unwrap(); + + let target_dir = Path::new("").join(&out_dir).join("target"); + + let in_repo_dir = get_wasm_runtime_path(); - let in_repo_dir = PathBuf::from(&proj_dir) - .parent() - .unwrap_or_else(|| panic!("could not find parent of cargo manifest directory")) - .join("wasm_runtime"); if !in_repo_dir.exists() { - panic!("hyperlight_wasm does not yet support being compiled from a release package"); + panic!("missing wasm_runtime in-tree dependency"); } - print!("cargo::rerun-if-changed="); - let _ = std::io::stdout() - .write_all(AsRef::::as_ref(&in_repo_dir).as_encoded_bytes()); - println!(); + + println!("cargo::rerun-if-changed={}", in_repo_dir.display()); println!("cargo::rerun-if-env-changed=WIT_WORLD"); // the PROFILE env var unfortunately only gives us 1 bit of "dev or release" let cargo_profile = if profile == "debug" { "dev" } else { "release" }; - let mut cmd = std::process::Command::new("cargo"); + let mut cmd = std::process::Command::new(cargo_bin); // Clear the variables that control Cargo's behaviour (as listed // at https://doc.rust-lang.org/cargo/reference/environment-variables.html): @@ -57,7 +115,12 @@ fn build_wasm_runtime() -> PathBuf { env_vars.retain(|(key, _)| !key.starts_with("CARGO_")); let cmd = cmd - .args(["build", &format!("--profile={}", cargo_profile), "-v"]) + .arg("build") + .arg("--profile") + .arg(cargo_profile) + .arg("-v") + .arg("--target-dir") + .arg(&target_dir) .current_dir(&in_repo_dir) .env_clear() .envs(env_vars); @@ -68,8 +131,7 @@ fn build_wasm_runtime() -> PathBuf { if !status.success() { panic!("could not compile wasm_runtime"); } - let resource = in_repo_dir - .join("target") + let resource = target_dir .join("x86_64-unknown-none") .join(profile) .join("wasm_runtime"); diff --git a/src/hyperlight_wasm/src/lib.rs b/src/hyperlight_wasm/src/lib.rs index 53f8870..f974da6 100644 --- a/src/hyperlight_wasm/src/lib.rs +++ b/src/hyperlight_wasm/src/lib.rs @@ -72,10 +72,8 @@ mod tests { fn test_wasmtime_version() { let wasmtime_version = super::get_wasmtime_version(); // get the wasmtime version from the wasm_runtime binary's Cargo.toml - let proj_dir = env::var_os("CARGO_MANIFEST_DIR").unwrap(); - let cargo_toml_path = Path::new(&proj_dir) - .parent() - .unwrap() + let cargo_toml_path = Path::new(env!("OUT_DIR")) + .join("vendor") .join("wasm_runtime") .join("Cargo.toml"); let cargo_toml_content = diff --git a/src/hyperlight_wasm/src/sandbox/wasm_sandbox.rs b/src/hyperlight_wasm/src/sandbox/wasm_sandbox.rs index ee42e8f..43204a2 100644 --- a/src/hyperlight_wasm/src/sandbox/wasm_sandbox.rs +++ b/src/hyperlight_wasm/src/sandbox/wasm_sandbox.rs @@ -221,11 +221,7 @@ mod tests { .expect("Failed to get CARGO_MANIFEST_DIR or RUST_DIR_FOR_DEBUGGING_TESTS env var") }); - let relative_path = if filename == "wasm_runtime" { - "redist" - } else { - "../../x64" - }; + let relative_path = "../../x64"; let filename_path = Path::new(&proj_dir) .join(relative_path) diff --git a/src/hyperlight_wasm_macro/src/wasmguest.rs b/src/hyperlight_wasm_macro/src/wasmguest.rs index 8776680..ed13682 100644 --- a/src/hyperlight_wasm_macro/src/wasmguest.rs +++ b/src/hyperlight_wasm_macro/src/wasmguest.rs @@ -35,20 +35,18 @@ use hyperlight_component_util::hl::{ emit_hl_unmarshal_result, }; use hyperlight_component_util::{resource, rtypes}; -use proc_macro::Ident; use proc_macro2::TokenStream; use quote::{format_ident, quote}; -use syn::ext::IdentExt; // Emit code to register this particular extern definition with the // wasmtime linker, calling through Hyperlight. // // depth: how many instances deep (from the root component) this is, // used to keep track of which linker instance to register on -fn emit_import_extern_decl<'a, 'b, 'c>( - s: &'c mut State<'a, 'b>, +fn emit_import_extern_decl<'b>( + s: &mut State<'_, 'b>, depth: u32, - ed: &'c ExternDecl<'b>, + ed: &ExternDecl<'b>, ) -> TokenStream { match &ed.desc { ExternDesc::CoreModule(_) => panic!("core module (im/ex)ports are not supported"), @@ -70,10 +68,7 @@ fn emit_import_extern_decl<'a, 'b, 'c>( }) .unzip::<_, _, Vec<_>, Vec<_>>(); let ret = format_ident!("ret"); - let is_ret_empty = match &ft.result { - etypes::Result::Named(rs) if rs.len() == 0 => true, - _ => false, - }; + let is_ret_empty = matches!(&ft.result, etypes::Result::Named(rs) if rs.is_empty()); let ur = if is_ret_empty { quote! { () } } else { @@ -133,10 +128,10 @@ fn emit_import_extern_decl<'a, 'b, 'c>( // path: the instance path (from the root component) where this // definition may be found, used to locate the wasmtime function to // call. -fn emit_export_extern_decl<'a, 'b, 'c>( - s: &'c mut State<'a, 'b>, +fn emit_export_extern_decl<'b>( + s: &mut State<'_, 'b>, path: Vec, - ed: &'c ExternDecl<'b>, + ed: &ExternDecl<'b>, ) -> TokenStream { match &ed.desc { ExternDesc::CoreModule(_) => panic!("core module (im/ex)ports are not supported"), @@ -238,11 +233,11 @@ fn emit_wasm_function_call( // // depth: how many instances deep (from the root component) this is, // used to keep track of which linker instance to register on -fn emit_import_instance<'a, 'b, 'c>( - s: &'c mut State<'a, 'b>, +fn emit_import_instance<'b>( + s: &mut State<'_, 'b>, wn: WitName, depth: u32, - it: &'c Instance<'b>, + it: &Instance<'b>, ) -> TokenStream { let mut s = s.with_cursor(wn.namespace_idents()); s.cur_helper_mod = Some(kebab_to_namespace(wn.name)); @@ -261,11 +256,11 @@ fn emit_import_instance<'a, 'b, 'c>( // path: the instance path (from the root component) where this // definition may be found, used to locate the wasmtime function to // call. -fn emit_export_instance<'a, 'b, 'c>( - s: &'c mut State<'a, 'b>, +fn emit_export_instance<'b>( + s: &mut State<'_, 'b>, wn: WitName, path: Vec, - it: &'c Instance<'b>, + it: &Instance<'b>, ) -> TokenStream { let mut s = s.with_cursor(wn.namespace_idents()); s.cur_helper_mod = Some(kebab_to_namespace(wn.name)); @@ -283,11 +278,7 @@ fn emit_export_instance<'a, 'b, 'c>( // keep track of resources sent to the host // - code to register each import with the wasmtime linker // - code to register each export with Hyperlight -fn emit_component<'a, 'b, 'c>( - s: &'c mut State<'a, 'b>, - wn: WitName, - ct: &'c Component<'b>, -) -> TokenStream { +fn emit_component<'b>(s: &mut State<'_, 'b>, wn: WitName, ct: &Component<'b>) -> TokenStream { let mut s = s.with_cursor(wn.namespace_idents()); let ns = wn.namespace_path(); let r#trait = kebab_to_type(wn.name); @@ -344,7 +335,7 @@ fn emit_component<'a, 'b, 'c>( } } -pub fn emit_toplevel<'a, 'b, 'c>(s: &'c mut State<'a, 'b>, n: &str, ct: &'c Component<'b>) { +pub fn emit_toplevel<'b>(s: &mut State<'_, 'b>, n: &str, ct: &Component<'b>) { s.is_impl = true; let wn = split_wit_name(n); let tokens = emit_component(s, wn, ct); diff --git a/src/wasm_runtime/.gitignore b/src/wasm_runtime/.gitignore index faf853a..be1ad00 100644 --- a/src/wasm_runtime/.gitignore +++ b/src/wasm_runtime/.gitignore @@ -1 +1,2 @@ -src/include/wasmtime-platform.h +guest-toolchain +target diff --git a/src/wasm_runtime/Cargo.toml b/src/wasm_runtime/Cargo.toml index 8f823d3..4af9520 100644 --- a/src/wasm_runtime/Cargo.toml +++ b/src/wasm_runtime/Cargo.toml @@ -23,3 +23,5 @@ cc = "1.2" cfg-if = { version = "1" } cargo_metadata = "0.19" reqwest = {version = "0.12", default-features = false, features = ["blocking","rustls-tls"] } + +[workspace] # indicate that this crate is not part of any workspace diff --git a/src/wasm_runtime/build.rs b/src/wasm_runtime/build.rs index 7235292..59ccd2e 100644 --- a/src/wasm_runtime/build.rs +++ b/src/wasm_runtime/build.rs @@ -14,13 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -use std::fs::File; -use std::io::copy; use std::path::Path; use std::{env, fs, path}; use cargo_metadata::{MetadataCommand, Package}; -use reqwest::blocking::get; fn main() { println!("cargo:rerun-if-changed=."); @@ -45,32 +42,6 @@ fn main() { None => panic!("wasmtime dependency not found"), }; - // download the wasmtime wasm_platform.h header file from GitHub releases and write it to the src/include directory - - // ensure the include directory exists - let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); - let include_file = path::Path::new(&crate_dir) - .join("src") - .join("include") - .join("wasmtime-platform.h"); - std::fs::create_dir_all(include_file.parent().unwrap()).unwrap(); - - // get the wasm_platform.h header file from github release with matching version e.g. https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-platform.h - let path = format!( - "https://github.com/bytecodealliance/wasmtime/releases/download/v{}/wasmtime-platform.h", - version_number - ); - let response = get(&path).expect("Failed to download header file"); - - // write the include file to the src/include directory - let out_file = include_file.to_str().unwrap(); - let mut dest = File::create(out_file).expect("Failed to create header file"); - copy( - &mut response.bytes().expect("Failed to read response").as_ref(), - &mut dest, - ) - .expect("Failed to copy content"); - // Write the version number to the metadata.rs file so that it is included in the binary let out_dir = env::var_os("OUT_DIR").unwrap(); diff --git a/src/wasm_runtime/src/platform.c b/src/wasm_runtime/src/platform.c index 71f52ee..43241f9 100644 --- a/src/wasm_runtime/src/platform.c +++ b/src/wasm_runtime/src/platform.c @@ -15,7 +15,8 @@ limitations under the License. */ #include -#include "wasmtime-platform.h" +#include +#include typedef void (*wasmtime_setjmp_cb_t)(void *, void *);