From 22b934f48924a9fc2f18cc3903003cc82d5ec533 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Wed, 27 Nov 2024 12:16:04 -0700 Subject: [PATCH 1/2] optionally override module names; bump version to 0.16.0 This adds CLI and componentize-py.toml options for overriding the generated Python module names for one or more WIT interfaces. By default, the name is the snake-case version of the WIT name, qualified as necessary with the package namespace and name and/or the version in cases of ambiguity. Sometimes that's not what you want, though, so now you can override the naming on an individual basis as long as the name(s) you pick are unique. This can be especially useful for backwards compatibility when adding new versions of WIT interfaces. In that case, the generated module name may go from unqualified to qualified, but you can now force the name of the original version to be unqualified for compatibility. For example: - You release an SDK with an interface called `foo:bar/baz@1.0.0`. Since that's the only interface with the name `baz`, `componentize-py` will name the generated module `baz` also. - Later, you release a new version of the SDK with support for _both_ `foo:bar/baz@1.0.0` _and_ `foo:bar/baz@2.0.0`. In that case, `componentize-py` will name the generated modules `foo_bar_baz_1_0_0` and `foo_bar_baz_2_0_0` by default. However, you don't want to force users of your SDK to use the new name, so you pass `--import-interface-name foo:bar/baz@1.0.0=baz` to `componentize-py`, which tells it to use the original name. Signed-off-by: Joel Dice --- Cargo.lock | 2 +- Cargo.toml | 2 +- examples/cli/README.md | 4 +-- examples/http/README.md | 4 +-- examples/matrix-math/README.md | 4 +-- examples/sandbox/README.md | 4 +-- examples/tcp/README.md | 4 +-- pyproject.toml | 2 +- src/command.rs | 50 +++++++++++++++++++++++++++++++-- src/lib.rs | 51 ++++++++++++++++++++++++++++++++-- src/prelink.rs | 7 ++--- src/python.rs | 25 +++++++++++++++-- src/summary.rs | 36 ++++++++++++++++++++---- src/test.rs | 4 ++- 14 files changed, 170 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b2f30d..2186c29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,7 +418,7 @@ dependencies = [ [[package]] name = "componentize-py" -version = "0.15.2" +version = "0.16.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index a0ed2f3..9326b0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "componentize-py" -version = "0.15.2" +version = "0.16.0" edition = "2021" exclude = ["cpython"] diff --git a/examples/cli/README.md b/examples/cli/README.md index ca32872..0cd4ba5 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-cli] `command` world. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 ``` ## Running the demo diff --git a/examples/http/README.md b/examples/http/README.md index 85bdd94..8e88371 100644 --- a/examples/http/README.md +++ b/examples/http/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-http] `proxy` world. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 ``` ## Running the demo diff --git a/examples/matrix-math/README.md b/examples/matrix-math/README.md index c5c1d16..64872a9 100644 --- a/examples/matrix-math/README.md +++ b/examples/matrix-math/README.md @@ -11,7 +11,7 @@ within a guest component. ## Prerequisites * `wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 * `NumPy`, built for WASI Note that we use an unofficial build of NumPy since the upstream project does @@ -23,7 +23,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz tar xf numpy-wasi.tar.gz ``` diff --git a/examples/sandbox/README.md b/examples/sandbox/README.md index 30d9591..008cb5e 100644 --- a/examples/sandbox/README.md +++ b/examples/sandbox/README.md @@ -8,10 +8,10 @@ sandboxed Python code snippets from within a Python app. ## Prerequisites * `wasmtime-py` 25.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 ``` -pip install componentize-py==0.15.2 wasmtime==25.0.0 +pip install componentize-py==0.16.0 wasmtime==25.0.0 ``` ## Running the demo diff --git a/examples/tcp/README.md b/examples/tcp/README.md index 4849688..52ad715 100644 --- a/examples/tcp/README.md +++ b/examples/tcp/README.md @@ -11,7 +11,7 @@ making an outbound TCP request using `wasi-sockets`. ## Prerequisites * `Wasmtime` 26.0.0 or later -* `componentize-py` 0.15.2 +* `componentize-py` 0.16.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -19,7 +19,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v26.0.0. ``` cargo install --version 26.0.0 wasmtime-cli -pip install componentize-py==0.15.2 +pip install componentize-py==0.16.0 ``` ## Running the demo diff --git a/pyproject.toml b/pyproject.toml index c7cc8b0..ec5aca5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ features = ["pyo3/extension-module"] [project] name = "componentize-py" -version = "0.15.2" +version = "0.16.0" description = "Tool to package Python applications as WebAssembly components" readme = "README.md" license = { file = "LICENSE" } diff --git a/src/command.rs b/src/command.rs index f3227c1..51516d0 100644 --- a/src/command.rs +++ b/src/command.rs @@ -48,6 +48,24 @@ pub struct Common { /// This enables using `@unstable` annotations in WIT files. #[clap(long)] all_features: bool, + + /// Specify names to use for imported interfaces. May be specified more than once. + /// + /// By default, the python module name generated for a given interface will be the snake-case form of the WIT + /// interface name, possibly qualified with the package name and namespace and/or version if that name would + /// otherwise clash with another interface. With this option, you may override that name with your own, unique + /// name. + #[arg(long, value_parser = parse_key_value)] + pub import_interface_name: Vec<(String, String)>, + + /// Specify names to use for exported interfaces. May be specified more than once. + /// + /// By default, the python module name generated for a given interface will be the snake-case form of the WIT + /// interface name, possibly qualified with the package name and namespace and/or version if that name would + /// otherwise clash with another interface. With this option, you may override that name with your own, unique + /// name. + #[arg(long, value_parser = parse_key_value)] + pub export_interface_name: Vec<(String, String)>, } #[derive(clap::Subcommand, Debug)] @@ -82,7 +100,7 @@ pub struct Componentize { /// /// Note that these must be specified in topological order (i.e. if a module containing WIT files depends on /// other modules containing WIT files, it must be listed after all its dependencies). - #[arg(short = 'm', long, value_parser = parse_module_world)] + #[arg(short = 'm', long, value_parser = parse_key_value)] pub module_worlds: Vec<(String, String)>, /// Output file to which to write the resulting component @@ -112,7 +130,7 @@ pub struct Bindings { pub world_module: Option, } -fn parse_module_world(s: &str) -> Result<(String, String), String> { +fn parse_key_value(s: &str) -> Result<(String, String), String> { let (k, v) = s .split_once('=') .ok_or_else(|| format!("expected string of form `=`; got `{s}`"))?; @@ -137,6 +155,16 @@ fn generate_bindings(common: Common, bindings: Bindings) -> Result<()> { common.all_features, bindings.world_module.as_deref(), &bindings.output_dir, + &common + .import_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), + &common + .export_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), ) } @@ -167,6 +195,16 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> { &componentize.output, None, componentize.stub_wasi, + &common + .import_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), + &common + .export_interface_name + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(), ))?; if !common.quiet { @@ -300,6 +338,8 @@ mod tests { quiet: false, features: vec![], all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -328,6 +368,8 @@ mod tests { quiet: false, features: vec!["x".to_owned()], all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -356,6 +398,8 @@ mod tests { quiet: false, features: vec![], all_features: true, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -382,6 +426,8 @@ mod tests { quiet: false, features: vec!["x".to_owned()], all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), }; let bindings = Bindings { output_dir: out_dir.path().into(), diff --git a/src/lib.rs b/src/lib.rs index 0156507..9f945a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,12 +73,18 @@ impl WasiView for Ctx { struct RawComponentizePyConfig { bindings: Option, wit_directory: Option, + #[serde(default)] + import_interface_names: HashMap, + #[serde(default)] + export_interface_names: HashMap, } #[derive(Debug)] struct ComponentizePyConfig { bindings: Option, wit_directory: Option, + import_interface_names: HashMap, + export_interface_names: HashMap, } impl TryFrom<(&Path, RawComponentizePyConfig)> for ComponentizePyConfig { @@ -97,6 +103,8 @@ impl TryFrom<(&Path, RawComponentizePyConfig)> for ComponentizePyConfig { Ok(Self { bindings: raw.bindings.map(convert).transpose()?, wit_directory: raw.wit_directory.map(convert).transpose()?, + import_interface_names: raw.import_interface_names, + export_interface_names: raw.export_interface_names, }) } } @@ -162,6 +170,7 @@ impl Invoker for MyInvoker { } } +#[allow(clippy::too_many_arguments)] pub fn generate_bindings( wit_path: &Path, world: Option<&str>, @@ -169,12 +178,19 @@ pub fn generate_bindings( all_features: bool, world_module: Option<&str>, output_dir: &Path, + import_interface_names: &HashMap<&str, &str>, + export_interface_names: &HashMap<&str, &str>, ) -> Result<()> { // TODO: Split out and reuse the code responsible for finding and using componentize-py.toml files in the // `componentize` function below, since that can affect the bindings we should be generating. let (resolve, world) = parse_wit(wit_path, world, features, all_features)?; - let summary = Summary::try_new(&resolve, &iter::once(world).collect())?; + let summary = Summary::try_new( + &resolve, + &iter::once(world).collect(), + import_interface_names, + export_interface_names, + )?; let world_name = resolve.worlds[world].name.to_snake_case().escape(); let world_module = world_module.unwrap_or(&world_name); let world_dir = output_dir.join(world_module.replace('.', "/")); @@ -202,6 +218,8 @@ pub async fn componentize( output_path: &Path, add_to_linker: Option<&dyn Fn(&mut Linker) -> Result<()>>, stub_wasi: bool, + import_interface_names: &HashMap<&str, &str>, + export_interface_names: &HashMap<&str, &str>, ) -> Result<()> { // Remove non-existent elements from `python_path` so we don't choke on them later: let python_path = &python_path @@ -224,6 +242,30 @@ pub async fn componentize( (None, None) }; + let import_interface_names = import_interface_names + .iter() + .map(|(a, b)| (*a, *b)) + .chain(configs.iter().flat_map(|(_, (config, _))| { + config + .config + .import_interface_names + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + })) + .collect(); + + let export_interface_names = export_interface_names + .iter() + .map(|(a, b)| (*a, *b)) + .chain(configs.iter().flat_map(|(_, (config, _))| { + config + .config + .export_interface_names + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + })) + .collect(); + let configs = configs .iter() .map(|(module, (config, world))| { @@ -271,7 +313,12 @@ pub async fn componentize( .chain(main_world) .collect::>(); - let summary = Summary::try_new(&resolve, &worlds)?; + let summary = Summary::try_new( + &resolve, + &worlds, + &import_interface_names, + &export_interface_names, + )?; libraries.push(Library { name: "libcomponentize_py_bindings.so".into(), diff --git a/src/prelink.rs b/src/prelink.rs index 345a307..ce40684 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -50,7 +50,7 @@ pub fn embedded_helper_utils() -> Result { } pub fn bundle_libraries( - library_path: Vec<(&str, Vec)>, + library_path: Vec<(&str, Vec)>, ) -> Result, anyhow::Error> { let mut libraries = vec![ Library { @@ -153,9 +153,8 @@ pub fn search_for_libraries_and_configs<'a>( module_worlds: &'a [(&'a str, &'a str)], world: Option<&'a str>, ) -> Result<(ConfigsMatchedWorlds<'a>, Vec), anyhow::Error> { - let mut raw_configs: Vec> = Vec::new(); - let mut library_path: Vec<(&str, Vec)> = - Vec::with_capacity(python_path.len()); + let mut raw_configs: Vec> = Vec::new(); + let mut library_path: Vec<(&str, Vec)> = Vec::with_capacity(python_path.len()); for path in python_path { let mut libraries = Vec::new(); search_directory( diff --git a/src/python.rs b/src/python.rs index 7929430..9d9c085 100644 --- a/src/python.rs +++ b/src/python.rs @@ -12,7 +12,7 @@ use { #[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "componentize")] -#[pyo3(signature = (wit_path, world, features, all_features, python_path, module_worlds, app_name, output_path, stub_wasi))] +#[pyo3(signature = (wit_path, world, features, all_features, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names))] fn python_componentize( wit_path: Option, world: Option<&str>, @@ -23,6 +23,8 @@ fn python_componentize( app_name: &str, output_path: PathBuf, stub_wasi: bool, + import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, + export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, ) -> PyResult<()> { (|| { Runtime::new()?.block_on(crate::componentize( @@ -39,14 +41,23 @@ fn python_componentize( &output_path, None, stub_wasi, + &import_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), + &export_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), )) })() .map_err(|e| PyAssertionError::new_err(format!("{e:?}"))) } +#[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "generate_bindings")] -#[pyo3(signature = (wit_path, world, features, all_features, world_module, output_dir))] +#[pyo3(signature = (wit_path, world, features, all_features, world_module, output_dir, import_interface_names, export_interface_names))] fn python_generate_bindings( wit_path: PathBuf, world: Option<&str>, @@ -54,6 +65,8 @@ fn python_generate_bindings( all_features: bool, world_module: Option<&str>, output_dir: PathBuf, + import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, + export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, ) -> PyResult<()> { crate::generate_bindings( &wit_path, @@ -62,6 +75,14 @@ fn python_generate_bindings( all_features, world_module, &output_dir, + &import_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), + &export_interface_names + .iter() + .map(|(a, b)| (a.as_ref(), b.as_ref())) + .collect(), ) .map_err(|e| PyAssertionError::new_err(format!("{e:?}"))) } diff --git a/src/summary.rs b/src/summary.rs index d10238b..5b6d4e4 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -232,7 +232,12 @@ pub struct Summary<'a> { } impl<'a> Summary<'a> { - pub fn try_new(resolve: &'a Resolve, worlds: &IndexSet) -> Result { + pub fn try_new( + resolve: &'a Resolve, + worlds: &IndexSet, + import_interface_names: &HashMap<&str, &str>, + export_interface_names: &HashMap<&str, &str>, + ) -> Result { let mut me = Self { resolve, functions: Vec::new(), @@ -272,8 +277,14 @@ impl<'a> Summary<'a> { me.types = me.types_sorted(); - me.imported_interface_names = me.interface_names(me.imported_interfaces.keys().copied()); - me.exported_interface_names = me.interface_names(me.exported_interfaces.keys().copied()); + me.imported_interface_names = me.interface_names( + me.imported_interfaces.keys().copied(), + import_interface_names, + ); + me.exported_interface_names = me.interface_names( + me.exported_interfaces.keys().copied(), + export_interface_names, + ); Ok(me) } @@ -1054,6 +1065,7 @@ impl<'a> Summary<'a> { fn interface_names( &self, ids: impl Iterator, + interface_names: &HashMap<&str, &str>, ) -> HashMap { let mut tree = HashMap::<_, HashMap<_, HashMap<_, _>>>::new(); for id in ids { @@ -1083,7 +1095,14 @@ impl<'a> Summary<'a> { .insert( *id, if let Some(version) = version { - if versions.len() == 1 { + if let Some(name) = interface_names.get( + format!( + "{package_namespace}:{package_name}/{name}@{version}" + ) + .as_str(), + ) { + (*name).to_owned() + } else if versions.len() == 1 { if packages.len() == 1 { (*name).to_owned() } else { @@ -1097,6 +1116,10 @@ impl<'a> Summary<'a> { version.to_string().replace('.', "-") ) } + } else if let Some(name) = interface_names.get( + format!("{package_namespace}:{package_name}/{name}").as_str() + ) { + (*name).to_owned() } else if packages.len() == 1 { (*name).to_owned() } else { @@ -1107,7 +1130,10 @@ impl<'a> Summary<'a> { } } else { assert!(names - .insert(*versions.get(&None).unwrap(), (*name).to_owned()) + .insert( + *versions.get(&None).unwrap(), + (*interface_names.get(*name).unwrap_or(name)).to_owned() + ) .is_none()); } } diff --git a/src/test.rs b/src/test.rs index c72bd37..b87abf4 100644 --- a/src/test.rs +++ b/src/test.rs @@ -9,7 +9,7 @@ use { prelude::Strategy, test_runner::{self, TestRng, TestRunner}, }, - std::{env, fs, iter, marker::PhantomData}, + std::{collections::HashMap, env, fs, iter, marker::PhantomData}, tokio::runtime::Runtime, wasmtime::{ component::{Component, InstancePre, Linker, ResourceTable}, @@ -77,6 +77,8 @@ async fn make_component( &tempdir.path().join("app.wasm"), add_to_linker, false, + &HashMap::new(), + &HashMap::new(), ) .await?; From bb528ef6d0b3f616f8081278dc6cc8512811db60 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Wed, 27 Nov 2024 13:35:23 -0700 Subject: [PATCH 2/2] build and cache CPython once in test.yaml i.e. match what we're doing in release.yaml. TODO: Factor out the "Populate cache" job into a reusable action instead of cut-and-pasting it. Signed-off-by: Joel Dice --- .github/workflows/test.yaml | 57 +++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 34ce899..2c9767d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,6 +14,59 @@ env: WASI_SDK_RELEASE: wasi-sockets-alpha-5 jobs: + linux: + name: Populate cache + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install latest Rust nightly toolchain + uses: dtolnay/rust-toolchain@nightly + with: + targets: wasm32-wasip1 wasm32-unknown-unknown + + - name: Install latest Rust stable toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 wasm32-unknown-unknown + components: clippy, rustfmt + + - name: Install Rust std source + shell: bash + run: rustup component add rust-src --toolchain nightly + + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" + cache-on-failure: false + + - name: Install WASI-SDK + shell: bash + run: | + cd /tmp + curl -LO https://github.com/dicej/wasi-sdk/releases/download/${WASI_SDK_RELEASE}/wasi-sdk-${WASI_SDK_VERSION}-linux.tar.gz + tar xf wasi-sdk-${WASI_SDK_VERSION}-linux.tar.gz + mv wasi-sdk-${WASI_SDK_VERSION} /opt/wasi-sdk + + - name: Cache CPython + id: cache-cpython-wasi + uses: actions/cache@v4 + with: + path: cpython/builddir/wasi + key: cpython-wasi + enableCrossOsArchive: true + + - name: Build + shell: bash + run: cargo build --release + test: name: Test strategy: @@ -67,9 +120,9 @@ jobs: shell: bash run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION})" >> ${GITHUB_ENV} - - name: Cache CPython + - name: Restore CPython id: cache-cpython-wasi - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: path: cpython/builddir/wasi key: cpython-wasi