rearrange repo and add spin-python-cli (#8)

* rearrange repo and add `spin-python-cli`

`spin-python-cli` is a stand-alone binary for converting a Python app to a
Spin-compatible Wasm module.  The intention is that we'll publish this as a
`Spin` plugin, analogous to the JS/TS SDK's `js2wasm` utility.

The implementation is a bit more complicated than `js2wasm` because the Python
interpreter loads the core library, the app, and its dependencies from disk, so
we have to:

- Embed the core library in the binary and extract it to a temporary directory prior to pre-initializing with Wizer
- Search for the `pipenv` `site-packages` directory and add it to `PYTHONPATH`
- Set various environment variables so the Python interpreter knows where to find things (and not buffer stdio)
- Set another environment variable so the pre-init function knows which module the app lives in

Signed-off-by: Joel Dice <[email protected]>

* make WASI SDK path configurable and clean up Makefile

Signed-off-by: Joel Dice <[email protected]>

* improve warning message when `site-packages` not found

Signed-off-by: Joel Dice <[email protected]>

* proceed without error if `pipenv` doesn't recognize the CWD

Signed-off-by: Joel Dice <[email protected]>

---------

Signed-off-by: Joel Dice <[email protected]>

Author: dicej

.gitignore

@@ -1,3 +1,4 @@
 /target
 /Cargo.lock
-py/__pycache__
+__pycache__
+*.wasm

Cargo.toml

@@ -1,16 +1,5 @@
-[package]
-name = "python-wasi"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-crate-type = [ "cdylib" ]
-
-[dependencies]
-anyhow = "1"
-bytes = { version = "1.2.1", features = ["serde"] }
-http = "0.2"
-spin-sdk = { git = "https://github.com/fermyon/spin" }
-wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "dde4694aaa6acf9370206527a798ac4ba6a8c5b8" }
-pyo3 = { version = "0.17.3", features = [ "abi3-py310" ] }
-once_cell = "1.16.0"
+[workspace]
+members = [
+  "crates/spin-python-engine",
+  "crates/spin-python-cli"
+]

Makefile

@@ -1,22 +1,23 @@
 WASI_SDK_PATH ?= /opt/wasi-sdk
 
-.PHONY: build
-build: target/config.txt
-	PYO3_CONFIG_FILE=$$(pwd)/target/config.txt cargo build --release --target=wasm32-wasi
-	env -i PYTHONUNBUFFERED=1 \
-		PYTHONHOME=/python \
-		PYTHONPATH=/python:/py:/site-packages \
-		$$(which wizer) \
-		target/wasm32-wasi/release/python_wasi.wasm \
-		--inherit-env true \
-		--wasm-bulk-memory true \
-		--allow-wasi \
-		--dir py \
-		--mapdir python::$$(pwd)/cpython/builddir/wasi/install/lib/python3.11 \
-		--mapdir site-packages::$$(cd py && find $$(pipenv --venv)/lib -name site-packages | head -1) \
-		-o target/wasm32-wasi/release/python-wasi-wizer.wasm
+target/release/spin-python: \
+		target/wasm32-wasi/release/spin_python_engine.wasm \
+		crates/spin-python-cli/build.rs \
+		crates/spin-python-cli/src/main.rs
+	cd crates/spin-python-cli && \
+	SPIN_PYTHON_ENGINE_PATH=../../$< \
+	SPIN_PYTHON_CORE_LIBRARY_PATH=$$(pwd)/../../cpython/builddir/wasi/install/lib/python3.11 \
+	cargo build --release $(BUILD_TARGET)
 
-target/config.txt:
+target/wasm32-wasi/release/spin_python_engine.wasm: \
+		crates/spin-python-engine/src/lib.rs \
+		crates/spin-python-engine/build.rs \
+		target/pyo3-config.txt
+	cd crates/spin-python-engine && \
+	PYO3_CONFIG_FILE=$$(pwd)/../../target/config.txt \
+	cargo build --release --target=wasm32-wasi
+
+target/pyo3-config.txt: crates/spin-python-engine/pyo3-config.txt
 	mkdir -p target
-	cp config.txt target
+	cp $< target
 	echo "lib_dir=$$(pwd)/cpython/builddir/wasi" >> $@

README.md

@@ -4,6 +4,7 @@ This is an experiment to build a Spin Python SDK using CPython, Wizer, and PyO3.
 
 ## Prerequisites
 
+- [WASI SDK](https://github.com/WebAssembly/wasi-sdk) v16 or later, installed in /opt/wasi-sdk
 - [CPython](https://github.com/python/cpython) build prereqs (e.g. Make, Clang, etc.)
 - [Rust](https://rustup.rs/) (including `wasm32-wasi` target)
 - [Wizer](https://github.com/bytecodealliance/wizer) v1.6.0 or later
@@ -30,10 +31,17 @@ make install
 cd ../../..
 ```
 
-Then, build and run this app:
+Then, build the `spin-python-cli`:
 
 ```
-(cd py && pipenv install)
+make
+```
+
+Finally, build and run the example app:
+
+```
+cd examples/hello
+pipenv install
 spin build
 spin up
 ```

crates/spin-python-cli/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "spin-python-cli"
+description = "A Spin plugin to convert Python apps to Spin-compatible WebAssembly modules"
+version = "0.1.0"
+authors = [ "Fermyon Engineering <[email protected]>" ]
+edition = "2021"
+
+[[bin]]
+name = "spin-python"
+path = "src/main.rs"
+
+[dependencies]
+anyhow = "1.0.68"
+clap = { version = "4.1.4", features = [ "derive" ] }
+tar = "0.4.38"
+tempfile = "3.3.0"
+wizer = "1.6.0"
+zstd = "0.10.0"
+
+[build-dependencies]
+anyhow = "1.0.68"
+tar = "0.4.38"
+zstd = "0.10.0"

crates/spin-python-cli/build.rs

@@ -0,0 +1,126 @@
+#![deny(warnings)]
+
+use {
+    anyhow::{bail, Result},
+    std::{
+        env,
+        fs::{self, File},
+        io::{self, Write},
+        path::{Path, PathBuf},
+    },
+    tar::Builder,
+    zstd::Encoder,
+};
+
+const ZSTD_COMPRESSION_LEVEL: i32 = 19;
+
+fn main() -> Result<()> {
+    println!("cargo:rerun-if-changed=build.rs");
+
+    if let Ok("cargo-clippy") = env::var("CARGO_CFG_FEATURE").as_deref() {
+        stubs_for_clippy()
+    } else {
+        package_engine_and_core_library()
+    }
+}
+
+fn stubs_for_clippy() -> Result<()> {
+    println!("cargo:warning=using stubbed engine and core library for static analysis purposes...");
+
+    let engine_path = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("engine.wasm.zstd");
+
+    if !engine_path.exists() {
+        Encoder::new(File::create(engine_path)?, ZSTD_COMPRESSION_LEVEL)?.do_finish()?;
+    }
+
+    let core_library_path =
+        PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("python-lib.tar.zstd");
+
+    if !core_library_path.exists() {
+        Builder::new(Encoder::new(
+            File::create(core_library_path)?,
+            ZSTD_COMPRESSION_LEVEL,
+        )?)
+        .into_inner()?
+        .do_finish()?;
+    }
+
+    Ok(())
+}
+
+fn package_engine_and_core_library() -> Result<()> {
+    let override_engine_path = env::var_os("SPIN_PYTHON_ENGINE_PATH");
+    let engine_path = if let Some(path) = override_engine_path {
+        PathBuf::from(path)
+    } else {
+        let mut path = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
+        path.pop();
+        path.pop();
+        path.join("target/wasm32-wasi/release/spin_python_engine.wasm")
+    };
+
+    println!("cargo:rerun-if-changed={engine_path:?}");
+
+    if engine_path.exists() {
+        let copied_engine_path =
+            PathBuf::from(env::var("OUT_DIR").unwrap()).join("engine.wasm.zstd");
+
+        let mut encoder = Encoder::new(File::create(copied_engine_path)?, ZSTD_COMPRESSION_LEVEL)?;
+        io::copy(&mut File::open(engine_path)?, &mut encoder)?;
+        encoder.do_finish()?;
+    } else {
+        bail!("no such file: {}", engine_path.display())
+    }
+
+    let override_core_library_path = env::var_os("SPIN_PYTHON_CORE_LIBRARY_PATH");
+    let core_library_path = if let Some(path) = override_core_library_path {
+        PathBuf::from(path)
+    } else {
+        let mut path = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
+        path.pop();
+        path.pop();
+        path.join("cpython/builddir/wasi/install/lib/python3.11")
+    };
+
+    println!("cargo:rerun-if-changed={core_library_path:?}");
+
+    if core_library_path.exists() {
+        let copied_core_library_path =
+            PathBuf::from(env::var("OUT_DIR").unwrap()).join("python-lib.tar.zstd");
+
+        let mut builder = Builder::new(Encoder::new(
+            File::create(copied_core_library_path)?,
+            ZSTD_COMPRESSION_LEVEL,
+        )?);
+
+        add(&mut builder, &core_library_path, &core_library_path)?;
+
+        builder.into_inner()?.do_finish()?;
+    } else {
+        bail!("no such directory: {}", core_library_path.display())
+    }
+
+    Ok(())
+}
+
+fn include(path: &Path) -> bool {
+    !(matches!(
+        path.extension().and_then(|e| e.to_str()),
+        Some("a" | "pyc" | "whl")
+    ) || matches!(
+        path.file_name().and_then(|e| e.to_str()),
+        Some("Makefile" | "Changelog" | "NEWS.txt")
+    ))
+}
+
+fn add(builder: &mut Builder<impl Write>, root: &Path, path: &Path) -> Result<()> {
+    if path.is_dir() {
+        for entry in fs::read_dir(path)? {
+            add(builder, root, &entry?.path())?;
+        }
+    } else if include(path) {
+        builder.append_file(path.strip_prefix(root)?, &mut File::open(path)?)?;
+    }
+
+    Ok(())
+}