diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 34ce899..2c9767d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,6 +14,59 @@ env: WASI_SDK_RELEASE: wasi-sockets-alpha-5 jobs: + linux: + name: Populate cache + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install latest Rust nightly toolchain + uses: dtolnay/rust-toolchain@nightly + with: + targets: wasm32-wasip1 wasm32-unknown-unknown + + - name: Install latest Rust stable toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 wasm32-unknown-unknown + components: clippy, rustfmt + + - name: Install Rust std source + shell: bash + run: rustup component add rust-src --toolchain nightly + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" + cache-on-failure: false + + - name: Install WASI-SDK + shell: bash + run: | + cd /tmp + curl -LO https://github.com/dicej/wasi-sdk/releases/download/${WASI_SDK_RELEASE}/wasi-sdk-${WASI_SDK_VERSION}-linux.tar.gz + tar xf wasi-sdk-${WASI_SDK_VERSION}-linux.tar.gz + mv wasi-sdk-${WASI_SDK_VERSION} /opt/wasi-sdk + + - name: Cache CPython + id: cache-cpython-wasi + uses: actions/cache@v4 + with: + path: cpython/builddir/wasi + key: cpython-wasi + enableCrossOsArchive: true + + - name: Build + shell: bash + run: cargo build --release + test: name: Test strategy: @@ -67,9 +120,9 @@ jobs: shell: bash run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION})" >> ${GITHUB_ENV} - - name: Cache CPython + - name: Restore CPython id: cache-cpython-wasi - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: path: cpython/builddir/wasi key: cpython-wasi diff --git a/Cargo.lock b/Cargo.lock index 2b2f30d..2186c29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,7 +418,7 @@ dependencies = [ [[package]] name = "componentize-py" -version = "0.15.2" +version = "0.16.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index a0ed2f3..9326b0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "componentize-py" -version = "0.15.2" +version = "0.16.0" edition = "2021" exclude = ["cpython"] diff --git a/examples/cli/README.md b/examples/cli/README.md index ca32872..0cd4ba5 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-cli] `command` world. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 ``` ## Running the demo diff --git a/examples/http/README.md b/examples/http/README.md index 85bdd94..8e88371 100644 --- a/examples/http/README.md +++ b/examples/http/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-http] `proxy` world. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 ``` ## Running the demo diff --git a/examples/matrix-math/README.md b/examples/matrix-math/README.md index c5c1d16..64872a9 100644 --- a/examples/matrix-math/README.md +++ b/examples/matrix-math/README.md @@ -11,7 +11,7 @@ within a guest component. ## Prerequisites * `wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 * `NumPy`, built for WASI Note that we use an unofficial build of NumPy since the upstream project does @@ -23,7 +23,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz tar xf numpy-wasi.tar.gz ``` diff --git a/examples/sandbox/README.md b/examples/sandbox/README.md index 30d9591..008cb5e 100644 --- a/examples/sandbox/README.md +++ b/examples/sandbox/README.md @@ -8,10 +8,10 @@ sandboxed Python code snippets from within a Python app. ## Prerequisites * `wasmtime-py` 25.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 ``` -pip install componentize-py==0.15.2 wasmtime==25.0.0 +pip install componentize-py==0.16.0 wasmtime==25.0.0 ``` ## Running the demo diff --git a/examples/tcp/README.md b/examples/tcp/README.md index 4849688..52ad715 100644 --- a/examples/tcp/README.md +++ b/examples/tcp/README.md @@ -11,7 +11,7 @@ making an outbound TCP request using `wasi-sockets`. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -19,7 +19,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 ``` ## Running the demo diff --git a/pyproject.toml b/pyproject.toml index c7cc8b0..ec5aca5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ features = ["pyo3/extension-module"] [project] name = "componentize-py" -version = "0.15.2" +version = "0.16.0" description = "Tool to package Python applications as WebAssembly components" readme = "README.md" license = { file = "LICENSE" } diff --git a/src/command.rs b/src/command.rs index f3227c1..51516d0 100644 --- a/src/command.rs +++ b/src/command.rs @@ -48,6 +48,24 @@ pub struct Common { /// This enables using `@unstable` annotations in WIT files. #[clap(long)] all_features: bool, + + /// Specify names to use for imported interfaces. May be specified more than once. + /// + /// By default, the python module name generated for a given interface will be the snake-case form of the WIT + /// interface name, possibly qualified with the package name and namespace and/or version if that name would + /// otherwise clash with another interface. With this option, you may override that name with your own, unique + /// name. + #[arg(long, value_parser = parse_key_value)] + pub import_interface_name: Vec<(String, String)>, + + /// Specify names to use for exported interfaces. May be specified more than once. + /// + /// By default, the python module name generated for a given interface will be the snake-case form of the WIT + /// interface name, possibly qualified with the package name and namespace and/or version if that name would + /// otherwise clash with another interface. With this option, you may override that name with your own, unique + /// name. + #[arg(long, value_parser = parse_key_value)] + pub export_interface_name: Vec<(String, String)>, } #[derive(clap::Subcommand, Debug)] @@ -82,7 +100,7 @@ pub struct Componentize { /// /// Note that these must be specified in topological order (i.e. if a module containing WIT files depends on /// other modules containing WIT files, it must be listed after all its dependencies). - #[arg(short = 'm', long, value_parser = parse_module_world)] + #[arg(short = 'm', long, value_parser = parse_key_value)] pub module_worlds: Vec<(String, String)>, /// Output file to which to write the resulting component @@ -112,7 +130,7 @@ pub struct Bindings { pub world_module: Option, } -fn parse_module_world(s: &str) -> Result<(String, String), String> { +fn parse_key_value(s: &str) -> Result<(String, String), String> { let (k, v) = s .split_once('=') .ok_or_else(|| format!("expected string of form `=`; got `{s}`"))?; @@ -137,6 +155,16 @@ fn generate_bindings(common: Common, bindings: Bindings) -> Result<()> { common.all_features, bindings.world_module.as_deref(), &bindings.output_dir, + &common + .import_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), + &common + .export_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), ) } @@ -167,6 +195,16 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> { &componentize.output, None, componentize.stub_wasi, + &common + .import_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), + &common + .export_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), ))?; if !common.quiet { @@ -300,6 +338,8 @@ mod tests { quiet: false, features: vec![], all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -328,6 +368,8 @@ mod tests { quiet: false, features: vec!["x".to_owned()], all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -356,6 +398,8 @@ mod tests { quiet: false, features: vec![], all_features: true, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -382,6 +426,8 @@ mod tests { quiet: false, features: vec!["x".to_owned()], all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), diff --git a/src/lib.rs b/src/lib.rs index 0156507..9f945a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,12 +73,18 @@ impl WasiView for Ctx { struct RawComponentizePyConfig { bindings: Option, wit_directory: Option, + #[serde(default)] + import_interface_names: HashMap, + #[serde(default)] + export_interface_names: HashMap, } #[derive(Debug)] struct ComponentizePyConfig { bindings: Option, wit_directory: Option, + import_interface_names: HashMap, + export_interface_names: HashMap, } impl TryFrom<(&Path, RawComponentizePyConfig)> for ComponentizePyConfig { @@ -97,6 +103,8 @@ impl TryFrom<(&Path, RawComponentizePyConfig)> for ComponentizePyConfig { Ok(Self { bindings: raw.bindings.map(convert).transpose()?, wit_directory: raw.wit_directory.map(convert).transpose()?, + import_interface_names: raw.import_interface_names, + export_interface_names: raw.export_interface_names, }) } } @@ -162,6 +170,7 @@ impl Invoker for MyInvoker { } } +#[allow(clippy::too_many_arguments)] pub fn generate_bindings( wit_path: &Path, world: Option<&str>, @@ -169,12 +178,19 @@ pub fn generate_bindings( all_features: bool, world_module: Option<&str>, output_dir: &Path, + import_interface_names: &HashMap<&str, &str>, + export_interface_names: &HashMap<&str, &str>, ) -> Result<()> { // TODO: Split out and reuse the code responsible for finding and using componentize-py.toml files in the // `componentize` function below, since that can affect the bindings we should be generating. let (resolve, world) = parse_wit(wit_path, world, features, all_features)?; - let summary = Summary::try_new(&resolve, &iter::once(world).collect())?; + let summary = Summary::try_new( + &resolve, + &iter::once(world).collect(), + import_interface_names, + export_interface_names, + )?; let world_name = resolve.worlds[world].name.to_snake_case().escape(); let world_module = world_module.unwrap_or(&world_name); let world_dir = output_dir.join(world_module.replace('.', "/")); @@ -202,6 +218,8 @@ pub async fn componentize( output_path: &Path, add_to_linker: Option<&dyn Fn(&mut Linker) -> Result<()>>, stub_wasi: bool, + import_interface_names: &HashMap<&str, &str>, + export_interface_names: &HashMap<&str, &str>, ) -> Result<()> { // Remove non-existent elements from `python_path` so we don't choke on them later: let python_path = &python_path @@ -224,6 +242,30 @@ pub async fn componentize( (None, None) }; + let import_interface_names = import_interface_names + .iter() + .map(|(a, b)| (*a, *b)) + .chain(configs.iter().flat_map(|(_, (config, _))| { + config + .config + .import_interface_names + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + })) + .collect(); + + let export_interface_names = export_interface_names + .iter() + .map(|(a, b)| (*a, *b)) + .chain(configs.iter().flat_map(|(_, (config, _))| { + config + .config + .export_interface_names + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + })) + .collect(); + let configs = configs .iter() .map(|(module, (config, world))| { @@ -271,7 +313,12 @@ pub async fn componentize( .chain(main_world) .collect::>(); - let summary = Summary::try_new(&resolve, &worlds)?; + let summary = Summary::try_new( + &resolve, + &worlds, + &import_interface_names, + &export_interface_names, + )?; libraries.push(Library { name: "libcomponentize_py_bindings.so".into(), diff --git a/src/prelink.rs b/src/prelink.rs index 345a307..ce40684 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -50,7 +50,7 @@ pub fn embedded_helper_utils() -> Result { } pub fn bundle_libraries( - library_path: Vec<(&str, Vec)>, + library_path: Vec<(&str, Vec)>, ) -> Result, anyhow::Error> { let mut libraries = vec![ Library { @@ -153,9 +153,8 @@ pub fn search_for_libraries_and_configs<'a>( module_worlds: &'a [(&'a str, &'a str)], world: Option<&'a str>, ) -> Result<(ConfigsMatchedWorlds<'a>, Vec), anyhow::Error> { - let mut raw_configs: Vec> = Vec::new(); - let mut library_path: Vec<(&str, Vec)> = - Vec::with_capacity(python_path.len()); + let mut raw_configs: Vec> = Vec::new(); + let mut library_path: Vec<(&str, Vec)> = Vec::with_capacity(python_path.len()); for path in python_path { let mut libraries = Vec::new(); search_directory( diff --git a/src/python.rs b/src/python.rs index 7929430..9d9c085 100644 --- a/src/python.rs +++ b/src/python.rs @@ -12,7 +12,7 @@ use { #[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "componentize")] -#[pyo3(signature = (wit_path, world, features, all_features, python_path, module_worlds, app_name, output_path, stub_wasi))] +#[pyo3(signature = (wit_path, world, features, all_features, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names))] fn python_componentize( wit_path: Option, world: Option<&str>, @@ -23,6 +23,8 @@ fn python_componentize( app_name: &str, output_path: PathBuf, stub_wasi: bool, + import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, + export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, ) -> PyResult<()> { (|| { Runtime::new()?.block_on(crate::componentize( @@ -39,14 +41,23 @@ fn python_componentize( &output_path, None, stub_wasi, + &import_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), + &export_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), )) })() .map_err(|e| PyAssertionError::new_err(format!("{e:?}"))) } +#[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "generate_bindings")] -#[pyo3(signature = (wit_path, world, features, all_features, world_module, output_dir))] +#[pyo3(signature = (wit_path, world, features, all_features, world_module, output_dir, import_interface_names, export_interface_names))] fn python_generate_bindings( wit_path: PathBuf, world: Option<&str>, @@ -54,6 +65,8 @@ fn python_generate_bindings( all_features: bool, world_module: Option<&str>, output_dir: PathBuf, + import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, + export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, ) -> PyResult<()> { crate::generate_bindings( &wit_path, @@ -62,6 +75,14 @@ fn python_generate_bindings( all_features, world_module, &output_dir, + &import_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), + &export_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), ) .map_err(|e| PyAssertionError::new_err(format!("{e:?}"))) } diff --git a/src/summary.rs b/src/summary.rs index d10238b..5b6d4e4 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -232,7 +232,12 @@ pub struct Summary<'a> { } impl<'a> Summary<'a> { - pub fn try_new(resolve: &'a Resolve, worlds: &IndexSet) -> Result { + pub fn try_new( + resolve: &'a Resolve, + worlds: &IndexSet, + import_interface_names: &HashMap<&str, &str>, + export_interface_names: &HashMap<&str, &str>, + ) -> Result { let mut me = Self { resolve, functions: Vec::new(), @@ -272,8 +277,14 @@ impl<'a> Summary<'a> { me.types = me.types_sorted(); - me.imported_interface_names = me.interface_names(me.imported_interfaces.keys().copied()); - me.exported_interface_names = me.interface_names(me.exported_interfaces.keys().copied()); + me.imported_interface_names = me.interface_names( + me.imported_interfaces.keys().copied(), + import_interface_names, + ); + me.exported_interface_names = me.interface_names( + me.exported_interfaces.keys().copied(), + export_interface_names, + ); Ok(me) } @@ -1054,6 +1065,7 @@ impl<'a> Summary<'a> { fn interface_names( &self, ids: impl Iterator, + interface_names: &HashMap<&str, &str>, ) -> HashMap { let mut tree = HashMap::<_, HashMap<_, HashMap<_, _>>>::new(); for id in ids { @@ -1083,7 +1095,14 @@ impl<'a> Summary<'a> { .insert( *id, if let Some(version) = version { - if versions.len() == 1 { + if let Some(name) = interface_names.get( + format!( + "{package_namespace}:{package_name}/{name}@{version}" + ) + .as_str(), + ) { + (*name).to_owned() + } else if versions.len() == 1 { if packages.len() == 1 { (*name).to_owned() } else { @@ -1097,6 +1116,10 @@ impl<'a> Summary<'a> { version.to_string().replace('.', "-") ) } + } else if let Some(name) = interface_names.get( + format!("{package_namespace}:{package_name}/{name}").as_str() + ) { + (*name).to_owned() } else if packages.len() == 1 { (*name).to_owned() } else { @@ -1107,7 +1130,10 @@ impl<'a> Summary<'a> { } } else { assert!(names - .insert(*versions.get(&None).unwrap(), (*name).to_owned()) + .insert( + *versions.get(&None).unwrap(), + (*interface_names.get(*name).unwrap_or(name)).to_owned() + ) .is_none()); } } diff --git a/src/test.rs b/src/test.rs index c72bd37..b87abf4 100644 --- a/src/test.rs +++ b/src/test.rs @@ -9,7 +9,7 @@ use { prelude::Strategy, test_runner::{self, TestRng, TestRunner}, }, - std::{env, fs, iter, marker::PhantomData}, + std::{collections::HashMap, env, fs, iter, marker::PhantomData}, tokio::runtime::Runtime, wasmtime::{ component::{Component, InstancePre, Linker, ResourceTable}, @@ -77,6 +77,8 @@ async fn make_component( &tempdir.path().join("app.wasm"), add_to_linker, false, + &HashMap::new(), + &HashMap::new(), ) .await?;