From bdaa0b139400a20bcdde741fbe4efd08ed628bff Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 4 Dec 2024 13:19:43 +0100 Subject: [PATCH 1/5] refactors linking --- src/lib.rs | 28 +++------------------------- src/link.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 25 deletions(-) create mode 100644 src/link.rs diff --git a/src/lib.rs b/src/lib.rs index 9f945a7..5a9c1ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,8 +11,7 @@ use { serde::Deserialize, std::{ collections::HashMap, - env, fs, - io::Cursor, + fs, iter, ops::Deref, path::{Path, PathBuf}, @@ -34,6 +33,7 @@ mod abi; mod bindgen; mod bindings; pub mod command; +mod link; mod prelink; #[cfg(feature = "pyo3")] mod python; @@ -326,29 +326,7 @@ pub async fn componentize( dl_openable: false, }); - // Link all the libraries (including any native extensions) into a single component. - let mut linker = wit_component::Linker::default() - .validate(true) - .use_built_in_libdl(true); - - for Library { - name, - module, - dl_openable, - } in &libraries - { - linker = linker.library(name, module, *dl_openable)?; - } - - linker = linker.adapter( - "wasi_snapshot_preview1", - &zstd::decode_all(Cursor::new(include_bytes!(concat!( - env!("OUT_DIR"), - "/wasi_snapshot_preview1.reactor.wasm.zst" - ))))?, - )?; - - let component = linker.encode()?; + let component = link::link_libraries(&libraries)?; let stubbed_component = if stub_wasi { stubwasi::link_stub_modules(libraries)? diff --git a/src/link.rs b/src/link.rs new file mode 100644 index 0000000..308322a --- /dev/null +++ b/src/link.rs @@ -0,0 +1,30 @@ +use std::io::Cursor; + +use anyhow::Result; + +use crate::Library; + +pub fn link_libraries(libraries: &[Library]) -> Result> { + let mut linker = wit_component::Linker::default() + .validate(true) + .use_built_in_libdl(true); + + for Library { + name, + module, + dl_openable, + } in libraries + { + linker = linker.library(name, module, *dl_openable)?; + } + + linker = linker.adapter( + "wasi_snapshot_preview1", + &zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasi_snapshot_preview1.reactor.wasm.zst" + ))))?, + )?; + + return linker.encode().map_err(|e| anyhow::anyhow!(e)); +} From 52defe65e3220e5cc822ee7358ceb080fdca0271 Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 4 Dec 2024 13:22:12 +0100 Subject: [PATCH 2/5] remove , anyhow::Error from prelink --- src/prelink.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/prelink.rs b/src/prelink.rs index ce40684..7f86a47 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -21,7 +21,7 @@ static NATIVE_EXTENSION_SUFFIX: &str = ".cpython-312-wasm32-wasi.so"; type ConfigsMatchedWorlds<'a> = IndexMap, Option<&'a str>)>; -pub fn embedded_python_standard_library() -> Result { +pub fn embedded_python_standard_library() -> Result { // Untar the embedded copy of the Python standard library into a temporary directory let stdlib = tempfile::tempdir()?; @@ -35,7 +35,7 @@ pub fn embedded_python_standard_library() -> Result { Ok(stdlib) } -pub fn embedded_helper_utils() -> Result { +pub fn embedded_helper_utils() -> Result { // Untar the embedded copy of helper utilities into a temporary directory let bundled = tempfile::tempdir()?; @@ -51,7 +51,7 @@ pub fn embedded_helper_utils() -> Result { pub fn bundle_libraries( library_path: Vec<(&str, Vec)>, -) -> Result, anyhow::Error> { +) -> Result> { let mut libraries = vec![ Library { name: "libcomponentize_py_runtime.so".into(), @@ -152,7 +152,7 @@ pub fn search_for_libraries_and_configs<'a>( python_path: &'a Vec<&'a str>, module_worlds: &'a [(&'a str, &'a str)], world: Option<&'a str>, -) -> Result<(ConfigsMatchedWorlds<'a>, Vec), anyhow::Error> { +) -> Result<(ConfigsMatchedWorlds<'a>, Vec)> { let mut raw_configs: Vec> = Vec::new(); let mut library_path: Vec<(&str, Vec)> = Vec::with_capacity(python_path.len()); for path in python_path { @@ -219,7 +219,7 @@ fn search_directory( libraries: &mut Vec, configs: &mut Vec>, modules_seen: &mut HashSet, -) -> Result<(), anyhow::Error> { +) -> Result<()> { if path.is_dir() { for entry in fs::read_dir(path).with_context(|| path.display().to_string())? { search_directory(root, &entry?.path(), libraries, configs, modules_seen)?; From 3ac08dd02a0120fba216b4f23fd57c8d1247daeb Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 4 Dec 2024 13:22:43 +0100 Subject: [PATCH 3/5] fmt --- src/lib.rs | 3 +-- src/prelink.rs | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5a9c1ac..144b27a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,8 +11,7 @@ use { serde::Deserialize, std::{ collections::HashMap, - fs, - iter, + fs, iter, ops::Deref, path::{Path, PathBuf}, str, diff --git a/src/prelink.rs b/src/prelink.rs index 7f86a47..4470678 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -49,9 +49,7 @@ pub fn embedded_helper_utils() -> Result { Ok(bundled) } -pub fn bundle_libraries( - library_path: Vec<(&str, Vec)>, -) -> Result> { +pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result> { let mut libraries = vec![ Library { name: "libcomponentize_py_runtime.so".into(), From 78ec52aa4569d31f0ef7ecd77c9338f6d76b70b1 Mon Sep 17 00:00:00 2001 From: Phil Cummins Date: Wed, 4 Dec 2024 13:38:07 +0100 Subject: [PATCH 4/5] formatting --- src/link.rs | 2 +- src/prelink.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/link.rs b/src/link.rs index 308322a..b2286a0 100644 --- a/src/link.rs +++ b/src/link.rs @@ -26,5 +26,5 @@ pub fn link_libraries(libraries: &[Library]) -> Result> { ))))?, )?; - return linker.encode().map_err(|e| anyhow::anyhow!(e)); + linker.encode().map_err(|e| anyhow::anyhow!(e)) } diff --git a/src/prelink.rs b/src/prelink.rs index 4470678..5a6df15 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -3,7 +3,7 @@ use std::{ collections::{HashMap, HashSet}, fs::{self}, - io::{self, Cursor}, + io::Cursor, ops::Deref, path::{Path, PathBuf}, }; From 9c72fa3fc74c960d44077244665ab68cbe7734d3 Mon Sep 17 00:00:00 2001 From: philrhc <140902392+philrhc@users.noreply.github.com> Date: Thu, 5 Dec 2024 19:51:11 +0100 Subject: [PATCH 5/5] nightly compiler/lint fixes (#129) * test * trigger test workflow * remove lint from workflow * lint/compiler fixes * re-adds readme --------- Co-authored-by: Phil Cummins --- src/stubwasi.rs | 33 ++++++++++++++++++--------------- src/summary.rs | 2 +- src/test/tests.rs | 2 ++ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/stubwasi.rs b/src/stubwasi.rs index 5305bad..4b3e620 100644 --- a/src/stubwasi.rs +++ b/src/stubwasi.rs @@ -10,9 +10,9 @@ use wasmparser::{FuncType, Parser, Payload, TypeRef}; use crate::Library; -pub fn link_stub_modules( - libraries: Vec, -) -> Result, impl Fn(u32) -> u32)>, Error> { +type LinkedStubModules = Option<(Vec, Box u32>)>; + +pub fn link_stub_modules(libraries: Vec) -> Result { let mut wasi_imports = HashMap::new(); let mut linker = wit_component::Linker::default() .validate(true) @@ -47,18 +47,21 @@ pub fn link_stub_modules( let new_adapter_count = u32::try_from(wasi_imports.len())?; assert!(new_adapter_count >= old_adapter_count); - Ok(Some((component, move |index: u32| { - if index == 0 { - // `main` module - 0 - } else if index <= new_adapter_count { - // adapter module - old_adapter_count - } else { - // one of the other kinds of module - index + old_adapter_count - new_adapter_count - } - }))) + Ok(Some(( + component, + Box::new(move |index: u32| { + if index == 0 { + // `main` module + 0 + } else if index <= new_adapter_count { + // adapter module + old_adapter_count + } else { + // one of the other kinds of module + index + old_adapter_count - new_adapter_count + } + }), + ))) } fn add_wasi_imports<'a>( diff --git a/src/summary.rs b/src/summary.rs index 5b6d4e4..71162da 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -88,7 +88,7 @@ pub struct MyFunction<'a> { pub wit_kind: wit_parser::FunctionKind, } -impl<'a> MyFunction<'a> { +impl MyFunction<'_> { fn key(&self) -> WorldKey { if let Some(interface) = self.interface.as_ref() { WorldKey::Interface(interface.id) diff --git a/src/test/tests.rs b/src/test/tests.rs index d647ccc..9b2c9da 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -1,3 +1,5 @@ +#![allow(non_local_definitions)] + use { super::{Ctx, Tester, SEED}, anyhow::{anyhow, Error, Result},