diff --git a/wasmedge-burn/squeezenet/Dockerfile b/wasmedge-burn/squeezenet/Dockerfile new file mode 100644 index 0000000..5ff47d4 --- /dev/null +++ b/wasmedge-burn/squeezenet/Dockerfile @@ -0,0 +1,4 @@ +# Dockerfile +FROM scratch +COPY wasmedge-wasinn-example-squeezenet.wasm /app.wasm +ENTRYPOINT [ "/app.wasm" ] \ No newline at end of file diff --git a/wasmedge-burn/squeezenet/README.md b/wasmedge-burn/squeezenet/README.md new file mode 100644 index 0000000..b44bcda --- /dev/null +++ b/wasmedge-burn/squeezenet/README.md @@ -0,0 +1,80 @@ +# Squeezenet Example For WASI-NN with Burn Backend + +The current version of the plugin has not been officially released yet, so it needs to be compiled manually. And here we are using two different branches for running in non-container and container environments. + +## Non container + +### Build plugin + +The plugin can ==only support one model type== at a time. + +```bash +git clone https://github.com/WasmEdge/WasmEdge.git +cd WasmEdge + +// For squeezenet model +cmake -GNinja -Bbuild -DCMAKE_BUILD_TYPE=Release -DWASMEDGE_BUILD_AOT_RUNTIME=OFF -DWASMEDGE_PLUGIN_WASI_NN_BURNRS_MODEL=squeezenet +cmake --build build + +// Replace the path to your WasmEdge +export PATH=/build/tools/wasmedge:$PATH +export WASMEDGE_PLUGIN_PATH=/build/plugins/wasi_nn_burnrs +``` +([More about how to build Wasmedge](https://wasmedge.org/docs/contribute/source/os/linux/)) + +### Execute + +```bash +// Make sure you build your plugin with the Squeezenet model enabled then +cd WasmEdge-WASINN-examples/wasmedge-burn/squeezenet + +// Verify with CPU +wasmedge --dir .:. --nn-preload="default:Burn:CPU:squeezenet1.mpk" wasmedge-wasinn-example-squeezenet.wasm samples/bridge.jpg default + +// Verify with GPU +wasmedge --dir .:. --nn-preload="default:Burn:GPU:squeezenet1.mpk" wasmedge-wasinn-example-squeezenet.wasm samples/bridge.jpg default +``` + +## Inside container + +We could use the wasmedge shim as docker wasm runtime to run the example. + +### Build the docker image include the wasm application only + +```bash +cd wasmedge-burn/squeezenet +docker build . --platform wasi/wasm -t squeezenet +``` + +### Execute + +A specific version of Docker Desktop is required to support using WebGPU inside container. +It hasn't been released yet, but you can refer to below link for a related demo. + +https://www.youtube.com/watch?v=ODhJFe4-n6Y + +And our plugin will be packaged as part of the runtime in this experimental version of Docker Desktop. + +```bash +cd WasmEdge-WASINN-examples/wasmedge-burn/squeezenet + +// Verify with CPU +docker run \ + --runtime=io.containerd.wasmedge.v1 \ + --platform=wasi/wasm \ + -v $(pwd):/resource \ + --env WASMEDGE_WASINN_PRELOAD=default:Burn:CPU:/resource/squeezenet1.mpk \ + squeezenet:latest /resource/samples/bridge.jpg default + +// Verify with GPU +docker run \ + --runtime=io.containerd.wasmedge.v1 \ + --platform=wasi/wasm \ + -v $(pwd):/resource \ + --env WASMEDGE_WASINN_PRELOAD=default:Burn:GPU:/resource/squeezenet1.mpk \ + squeezenet:latest /resource/samples/bridge.jpg default +``` + +### Appendix + +Verify CPU-only usage inside the container with the Docker Engine and a self-built Wamsedge shim + plugin. (on going) \ No newline at end of file diff --git a/wasmedge-burn/squeezenet/rust/Cargo.toml b/wasmedge-burn/squeezenet/rust/Cargo.toml new file mode 100644 index 0000000..31ea0de --- /dev/null +++ b/wasmedge-burn/squeezenet/rust/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "wasmedge-wasinn-example-squeezenet" +version = "0.1.0" +authors = ["Second-State"] +readme = "README.md" +edition = "2021" +publish = false + +[dependencies] +wasmedge-wasi-nn = "0.8.0" +image = { version = "0.24.7", features = ["png", "jpeg"] } +squeezenet-burn = { git = "https://github.com/second-state/burn-rs-models.git", branch = "prebuilt-feature", features = [ + "weights_file", +], default-features = false } diff --git a/wasmedge-burn/squeezenet/rust/src/main.rs b/wasmedge-burn/squeezenet/rust/src/main.rs new file mode 100644 index 0000000..8ff1b6f --- /dev/null +++ b/wasmedge-burn/squeezenet/rust/src/main.rs @@ -0,0 +1,86 @@ +use image; +use std::fs::File; +use std::io::Read; +use squeezenet_burn::model::label::LABELS; +use wasmedge_wasi_nn as wasi_nn; + +pub fn main() { + let img_path = std::env::args().nth(1).expect("No image path provided"); + let model_name = std::env::args().nth(2).expect("No model name provided"); + + let graph = + wasi_nn::GraphBuilder::new(wasi_nn::GraphEncoding::Burn, wasi_nn::ExecutionTarget::AUTO) + .build_from_cache(&model_name) + .expect("Failed to build graph"); + + println!("Loaded graph into wasi-nn with ID: {:?}", graph); + + let mut context = graph.init_execution_context().unwrap(); + println!("Created wasi-nn execution context with ID: {:?}", context); + + let tensor_data = image_to_tensor(img_path, 224, 224); + context + .set_input(0, wasi_nn::TensorType::F32, &[1, 3, 224, 224], &tensor_data) + .unwrap(); + + context.compute().unwrap(); + println!("Executed graph inference"); + + let mut output_buffer = vec![0f32; 1000]; + context.get_output(0, &mut output_buffer).unwrap(); + + top_5_classes(output_buffer) +} + +// Take the image located at 'path', open it, resize it to height x width, and then converts +// the pixel precision to FP32. The resulting BGR pixel vector is then returned. +fn image_to_tensor(path: String, height: u32, width: u32) -> Vec { + let mut file_img = File::open(path).unwrap(); + let mut img_buf = Vec::new(); + file_img.read_to_end(&mut img_buf).unwrap(); + let img = image::load_from_memory(&img_buf).unwrap().to_rgb8(); + // Resize the image + let resized = + image::imageops::resize(&img, height, width, ::image::imageops::FilterType::Triangle); + let mut flat_img: Vec = Vec::new(); + // Normize the image and rearrange the tensor data order + for rgb in resized.pixels() { + flat_img.push((rgb[0] as f32 / 255. - 0.485) / 0.229); + } + for rgb in resized.pixels() { + flat_img.push((rgb[1] as f32 / 255. - 0.456) / 0.224); + } + for rgb in resized.pixels() { + flat_img.push((rgb[2] as f32 / 255. - 0.406) / 0.225); + } + return flat_img; +} + +#[derive(Debug)] +pub struct InferenceResult { + index: usize, + probability: f32, + label: String, +} + +fn top_5_classes(probabilities: Vec) { + // Convert the probabilities into a vector of (index, probability) + let mut probabilities: Vec<_> = probabilities.iter().enumerate().collect(); + + // Sort the probabilities in descending order + probabilities.sort_by(|a, b| b.1.partial_cmp(a.1).unwrap()); + + // Take the top 5 probabilities + probabilities.truncate(5); + + // Convert the probabilities into InferenceResult + let result: Vec = probabilities + .into_iter() + .map(|(index, probability)| InferenceResult { + index, + probability: *probability, + label: LABELS[index].to_string(), + }) + .collect(); + println!("Top 5 classes: {:?}", result); +} diff --git a/wasmedge-burn/squeezenet/samples/bridge.jpg b/wasmedge-burn/squeezenet/samples/bridge.jpg new file mode 100644 index 0000000..b438103 Binary files /dev/null and b/wasmedge-burn/squeezenet/samples/bridge.jpg differ diff --git a/wasmedge-burn/squeezenet/samples/cat.jpg b/wasmedge-burn/squeezenet/samples/cat.jpg new file mode 100644 index 0000000..a2287af Binary files /dev/null and b/wasmedge-burn/squeezenet/samples/cat.jpg differ diff --git a/wasmedge-burn/squeezenet/samples/coyote.jpg b/wasmedge-burn/squeezenet/samples/coyote.jpg new file mode 100644 index 0000000..0fadf57 Binary files /dev/null and b/wasmedge-burn/squeezenet/samples/coyote.jpg differ diff --git a/wasmedge-burn/squeezenet/samples/flamingo.jpg b/wasmedge-burn/squeezenet/samples/flamingo.jpg new file mode 100644 index 0000000..c339281 Binary files /dev/null and b/wasmedge-burn/squeezenet/samples/flamingo.jpg differ diff --git a/wasmedge-burn/squeezenet/samples/pelican.jpg b/wasmedge-burn/squeezenet/samples/pelican.jpg new file mode 100644 index 0000000..79a45a8 Binary files /dev/null and b/wasmedge-burn/squeezenet/samples/pelican.jpg differ diff --git a/wasmedge-burn/squeezenet/samples/table-lamp.jpg b/wasmedge-burn/squeezenet/samples/table-lamp.jpg new file mode 100644 index 0000000..9d37e59 Binary files /dev/null and b/wasmedge-burn/squeezenet/samples/table-lamp.jpg differ diff --git a/wasmedge-burn/squeezenet/samples/torch.jpg b/wasmedge-burn/squeezenet/samples/torch.jpg new file mode 100644 index 0000000..4c22d8e Binary files /dev/null and b/wasmedge-burn/squeezenet/samples/torch.jpg differ diff --git a/wasmedge-burn/squeezenet/squeezenet1.mpk b/wasmedge-burn/squeezenet/squeezenet1.mpk new file mode 100644 index 0000000..57fe8e4 Binary files /dev/null and b/wasmedge-burn/squeezenet/squeezenet1.mpk differ diff --git a/wasmedge-burn/squeezenet/wasmedge-wasinn-example-squeezenet.wasm b/wasmedge-burn/squeezenet/wasmedge-wasinn-example-squeezenet.wasm new file mode 100755 index 0000000..fbfdd41 Binary files /dev/null and b/wasmedge-burn/squeezenet/wasmedge-wasinn-example-squeezenet.wasm differ diff --git a/wasmedge-burn/whisper/Dockerfile b/wasmedge-burn/whisper/Dockerfile new file mode 100644 index 0000000..144a5ad --- /dev/null +++ b/wasmedge-burn/whisper/Dockerfile @@ -0,0 +1,4 @@ +# Dockerfile +FROM scratch +COPY wasmedge-wasinn-example-whisper.wasm /app.wasm +ENTRYPOINT [ "/app.wasm" ] \ No newline at end of file diff --git a/wasmedge-burn/whisper/README.md b/wasmedge-burn/whisper/README.md new file mode 100644 index 0000000..61f46c8 --- /dev/null +++ b/wasmedge-burn/whisper/README.md @@ -0,0 +1,110 @@ +# Whisper Example For WASI-NN with Burn Backend + +The current version of the plugin has not been officially released yet, so it needs to be compiled manually. And here we are using two different branches for running in non-container and container environments. + +## Non container + +### Build plugin + +The plugin can ==only support one model type== at a time. + +```bash +git clone https://github.com/WasmEdge/WasmEdge.git +cd WasmEdge + +// For whisper model +cmake -GNinja -Bbuild -DCMAKE_BUILD_TYPE=Release -DWASMEDGE_BUILD_AOT_RUNTIME=OFF -DWASMEDGE_PLUGIN_WASI_NN_BURNRS_MODEL=whisper +cmake --build build + +// Replace the path to your WasmEdge +export PATH=/build/tools/wasmedge:$PATH +export WASMEDGE_PLUGIN_PATH=/build/plugins/wasi_nn_burnrs +``` +([More about how to build Wasmedge](https://wasmedge.org/docs/contribute/source/os/linux/)) + +### Execute + +```bash +// Make sure you build your plugin with the Whisper model enabled then +cd WasmEdge-WASINN-examples/wasmedge-burn/whisper + +// Untar model suite +tar -xvzf model.tar.gz + +// Verify with CPU +wasmedge --dir .:. --nn-preload="default:Burn:CPU:tiny_en.mpk:tiny_en.cfg:tokenizer.json:en" wasmedge-wasinn-example-whisper.wasm audio16k.wav default + +// Verify with GPU +wasmedge --dir .:. --nn-preload="default:Burn:GPU:tiny_en.mpk:tiny_en.cfg:tokenizer.json:en" wasmedge-wasinn-example-whisper.wasm audio16k.wav default +``` + +## Inside container + +We could use the wasmedge shim as docker wasm runtime to run the example. + +### Build the docker image include the wasm application only + +```bash +cd wasmedge-burn/whisper +docker build . --platform wasi/wasm -t whisper +``` + +### Execute + +A specific version of Docker Desktop is required to support using WebGPU inside container. +It hasn't been released yet, but you can refer to below link for a related demo. + +https://www.youtube.com/watch?v=ODhJFe4-n6Y + +And our plugin will be packaged as part of the runtime in this experimental version of Docker Desktop. + +```bash +cd WasmEdge-WASINN-examples/wasmedge-burn/whisper +tar -xvzf model.tar.gz + +// Verify with CPU +docker run \ + --runtime=io.containerd.wasmedge.v1 \ + --platform=wasi/wasm \ + -v $(pwd):/resource \ + --env WASMEDGE_WASINN_PRELOAD=default:Burn:CPU:/resource/tiny_en.mpk:/resource/tiny_en.cfg:/resource/tokenizer.json:en \ + whisper:latest /resource/audio16k.wav default + +// Verify with GPU +docker run \ + --runtime=io.containerd.wasmedge.v1 \ + --platform=wasi/wasm \ + -v $(pwd):/resource \ + --env WASMEDGE_WASINN_PRELOAD=default:Burn:GPU:/resource/tiny_en.mpk:/resource/tiny_en.cfg:/resource/tokenizer.json:en \ + whisper:latest /resource/audio16k.wav default +``` + +or + +You can directly use our prebuilt docker image, which already includes the model weights, configuration file, and other necessary files, so there's no need for additional extraction model.tar.gz. + +```bash +cd WasmEdge-WASINN-examples/wasmedge-burn/whisper + +docker pull secondstate/burn-whisper:latest + +// Verify with CPU +docker run \ + --runtime=io.containerd.wasmedge.v1 \ + --platform=wasi/wasm \ + -v $(pwd):/resource \ + --env WASMEDGE_WASINN_PRELOAD=default:Burn:CPU:/tiny_en.mpk:/tiny_en.cfg:/tokenizer.json:en \ + whisper:latest /resource/audio16k.wav default + +// Verify with GPU +docker run \ + --runtime=io.containerd.wasmedge.v1 \ + --platform=wasi/wasm \ + -v $(pwd):/resource \ + --env WASMEDGE_WASINN_PRELOAD=default:Burn:GPU:/tiny_en.mpk:/tiny_en.cfg:/tokenizer.json:en \ + whisper:latest /resource/audio16k.wav default +``` + +### Appendix + +Verify CPU-only usage inside the container with the Docker Engine and a self-built Wamsedge shim + plugin. (on going) \ No newline at end of file diff --git a/wasmedge-burn/whisper/audio16k.wav b/wasmedge-burn/whisper/audio16k.wav new file mode 100644 index 0000000..8c8c6e0 Binary files /dev/null and b/wasmedge-burn/whisper/audio16k.wav differ diff --git a/wasmedge-burn/whisper/model.tar.gz b/wasmedge-burn/whisper/model.tar.gz new file mode 100644 index 0000000..1abf11e Binary files /dev/null and b/wasmedge-burn/whisper/model.tar.gz differ diff --git a/wasmedge-burn/whisper/rust/Cargo.toml b/wasmedge-burn/whisper/rust/Cargo.toml new file mode 100644 index 0000000..f50a262 --- /dev/null +++ b/wasmedge-burn/whisper/rust/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "wasmedge-wasinn-example-whisper" +version = "0.1.0" +authors = ["Second-State"] +readme = "README.md" +edition = "2021" +publish = false + +[dependencies] +wasmedge-wasi-nn = "0.8.0" +hound = "3.5.0" diff --git a/wasmedge-burn/whisper/rust/src/main.rs b/wasmedge-burn/whisper/rust/src/main.rs new file mode 100644 index 0000000..b47c07f --- /dev/null +++ b/wasmedge-burn/whisper/rust/src/main.rs @@ -0,0 +1,69 @@ +use hound::{self, SampleFormat}; +use std::process; +use wasmedge_wasi_nn as wasi_nn; + +pub fn main() { + let wav_file = std::env::args().nth(1).expect("No wav file name provided"); + let model_name = std::env::args().nth(2).expect("No model name provided"); + + let graph = + wasi_nn::GraphBuilder::new(wasi_nn::GraphEncoding::Burn, wasi_nn::ExecutionTarget::AUTO) + .build_from_cache(&model_name) + .expect("Failed to build graph"); + + println!("Loaded graph into wasi-nn with ID: {:?}", graph); + + let mut context = graph.init_execution_context().unwrap(); + println!("Created wasi-nn execution context with ID: {:?}", context); + + println!("Loading waveform..."); + let (waveform, sample_rate) = match load_audio_waveform(&wav_file) { + Ok((w, sr)) => (w, sr), + Err(e) => { + eprintln!("Failed to load audio file: {}", e); + process::exit(1); + } + }; + assert_eq!(sample_rate, 16000, "The audio sample rate must be 16k."); + + context + .set_input(0, wasi_nn::TensorType::F32, &[1, waveform.len()], &waveform) + .unwrap(); + + context.compute().unwrap(); + println!("Executed audio to text converter."); + + let mut output_buffer = vec![0u8; 100]; + context.get_output(0, &mut output_buffer).unwrap(); + + match String::from_utf8(output_buffer) { + Ok(s) => println!("Text: {}", s), + Err(e) => println!("Error: {}", e), + } +} + +fn load_audio_waveform(filename: &str) -> hound::Result<(Vec, usize)> { + let reader = hound::WavReader::open(filename)?; + let spec = reader.spec(); + + // let duration = reader.duration() as usize; + let channels = spec.channels as usize; + let sample_rate = spec.sample_rate as usize; + // let bits_per_sample = spec.bits_per_sample; + let sample_format = spec.sample_format; + + assert_eq!(sample_rate, 16000, "The audio sample rate must be 16k."); + assert_eq!(channels, 1, "The audio must be single-channel."); + + let max_int_val = 2_u32.pow(spec.bits_per_sample as u32 - 1) - 1; + + let floats = match sample_format { + SampleFormat::Float => reader.into_samples::().collect::>()?, + SampleFormat::Int => reader + .into_samples::() + .map(|s| s.map(|s| s as f32 / max_int_val as f32)) + .collect::>()?, + }; + + return Ok((floats, sample_rate)); +} diff --git a/wasmedge-burn/whisper/wasmedge-wasinn-example-whisper.wasm b/wasmedge-burn/whisper/wasmedge-wasinn-example-whisper.wasm new file mode 100755 index 0000000..d4bd80f Binary files /dev/null and b/wasmedge-burn/whisper/wasmedge-wasinn-example-whisper.wasm differ