Start plumbing the ability to run Pulley in Wasmtime (#9646)

alexcrichton · web-flow · commit 57cd5a9e1e74 · 2024-11-21T19:50:51.000Z
* Start plumbing the ability to run Pulley in Wasmtime

This commit starts setting up some infrastructure to run Pulley modules
in Wasmtime, for example on the CLI. This doesn't actually work just yet
due to other missing pieces of integration but this is enough to get to
a fault and/or other compiler errors at this time. The changes here are:

* An `Engine` allows mismatching targets in the case of Pulley, but the
  pointer widths must match.
* All subcommands of Wasmtime now support `--target`, e.g. `wasmtime
  run`, as previously only the host was allowed.

* Fix compile of bench-api

* Fix miri tests

* Switch from `NativeEndian` to `Endianness`

This switches parsing ELF from always using the native endianness to
instead using a runtime-defined value of endianness. This enables
big-endian platforms such as s390x to load objects produced for pulley,
a little-endian platform.
diff --git a/crates/bench-api/src/lib.rs b/crates/bench-api/src/lib.rs
@@ -141,7 +141,6 @@ use clap::Parser;
 use std::os::raw::{c_int, c_void};
 use std::slice;
 use std::{env, path::PathBuf};
-use target_lexicon::Triple;
 use wasi_common::{sync::WasiCtxBuilder, I32Exit, WasiCtx};
 use wasmtime::{Engine, Instance, Linker, Module, Store};
 use wasmtime_cli_flags::CommonOptions;
@@ -435,7 +434,7 @@ impl BenchState {
         execution_end: extern "C" fn(*mut u8),
         make_wasi_cx: impl FnMut() -> Result<WasiCtx> + 'static,
     ) -> Result<Self> {
-        let mut config = options.config(Some(&Triple::host().to_string()), None)?;
+        let mut config = options.config(None)?;
         // NB: always disable the compilation cache.
         config.disable_cache();
         let engine = Engine::new(&config)?;
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
@@ -480,6 +480,10 @@ pub struct CommonOptions {
     pub wasm: WasmOptions,
     #[arg(skip)]
     pub wasi: WasiOptions,
+
+    /// The target triple; default is the host triple
+    #[arg(long, value_name = "TARGET")]
+    pub target: Option<String>,
 }
 
 macro_rules! match_feature {
@@ -541,11 +545,7 @@ impl CommonOptions {
         Ok(())
     }
 
-    pub fn config(
-        &mut self,
-        target: Option<&str>,
-        pooling_allocator_default: Option<bool>,
-    ) -> Result<Config> {
+    pub fn config(&mut self, pooling_allocator_default: Option<bool>) -> Result<Config> {
         self.configure();
         let mut config = Config::new();
 
@@ -560,7 +560,7 @@ impl CommonOptions {
             _ => err,
         }
         match_feature! {
-            ["cranelift" : target]
+            ["cranelift" : &self.target]
             target => config.target(target)?,
             _ => err,
         }
diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs
@@ -257,11 +257,32 @@ impl Engine {
     fn _check_compatible_with_native_host(&self) -> Result<(), String> {
         #[cfg(any(feature = "cranelift", feature = "winch"))]
         {
+            use target_lexicon::{Architecture, PointerWidth, Triple};
+
             let compiler = self.compiler();
 
-            // Check to see that the config's target matches the host
             let target = compiler.triple();
-            if *target != target_lexicon::Triple::host() {
+            let host = Triple::host();
+            let target_matches_host = || {
+                // If the host target and target triple match, then it's valid
+                // to run results of compilation on this host.
+                if host == *target {
+                    return true;
+                }
+
+                // Otherwise if there's a mismatch the only allowed
+                // configuration at this time is that any target can run Pulley,
+                // Wasmtime's interpreter. This only works though if the
+                // pointer-width of pulley matches the pointer-width of the
+                // host, so check that here.
+                match host.pointer_width() {
+                    Ok(PointerWidth::U32) => target.architecture == Architecture::Pulley32,
+                    Ok(PointerWidth::U64) => target.architecture == Architecture::Pulley64,
+                    _ => false,
+                }
+            };
+
+            if !target_matches_host() {
                 return Err(format!(
                     "target '{target}' specified in the configuration does not match the host"
                 ));
@@ -397,6 +418,22 @@ impl Engine {
             // available.
             FlagValue::Bool(true) => {}
 
+            // Pulley's pointer_width must match the host.
+            FlagValue::Enum("pointer32") => {
+                return if cfg!(target_pointer_width = "32") {
+                    Ok(())
+                } else {
+                    Err("wrong host pointer width".to_string())
+                }
+            }
+            FlagValue::Enum("pointer64") => {
+                return if cfg!(target_pointer_width = "64") {
+                    Ok(())
+                } else {
+                    Err("wrong host pointer width".to_string())
+                }
+            }
+
             // Only `bool` values are supported right now, other settings would
             // need more support here.
             _ => {
diff --git a/crates/wasmtime/src/engine/serialization.rs b/crates/wasmtime/src/engine/serialization.rs
@@ -24,7 +24,7 @@
 use crate::prelude::*;
 use crate::{Engine, ModuleVersionStrategy, Precompiled};
 use core::str::FromStr;
-use object::endian::NativeEndian;
+use object::endian::Endianness;
 #[cfg(any(feature = "cranelift", feature = "winch"))]
 use object::write::{Object, StandardSegment};
 use object::{read::elf::ElfFile64, FileFlags, Object as _, ObjectSection, SectionKind};
@@ -56,7 +56,7 @@ pub fn check_compatible(engine: &Engine, mmap: &[u8], expected: ObjectKind) -> R
     // structured well enough to make this easy and additionally it's not really
     // a perf issue right now so doing that is left for another day's
     // refactoring.
-    let obj = ElfFile64::<NativeEndian>::parse(mmap)
+    let obj = ElfFile64::<Endianness>::parse(mmap)
         .err2anyhow()
         .context("failed to parse precompiled artifact as an ELF")?;
     let expected_e_flags = match expected {
@@ -145,7 +145,7 @@ pub fn append_compiler_info(engine: &Engine, obj: &mut Object<'_>, metadata: &Me
 }
 
 fn detect_precompiled<'data, R: object::ReadRef<'data>>(
-    obj: ElfFile64<'data, NativeEndian, R>,
+    obj: ElfFile64<'data, Endianness, R>,
 ) -> Option<Precompiled> {
     match obj.flags() {
         FileFlags::Elf {
diff --git a/crates/wasmtime/src/runtime/code_memory.rs b/crates/wasmtime/src/runtime/code_memory.rs
@@ -3,7 +3,7 @@
 use crate::prelude::*;
 use crate::runtime::vm::{libcalls, MmapVec, UnwindRegistration};
 use core::ops::Range;
-use object::endian::NativeEndian;
+use object::endian::Endianness;
 use object::read::{elf::ElfFile64, Object, ObjectSection};
 use object::{ObjectSymbol, SectionFlags};
 use wasmtime_environ::{lookup_trap_code, obj, Trap};
@@ -57,7 +57,7 @@ impl CodeMemory {
     /// The returned `CodeMemory` manages the internal `MmapVec` and the
     /// `publish` method is used to actually make the memory executable.
     pub fn new(mmap: MmapVec) -> Result<Self> {
-        let obj = ElfFile64::<NativeEndian>::parse(&mmap[..])
+        let obj = ElfFile64::<Endianness>::parse(&mmap[..])
             .err2anyhow()
             .with_context(|| "failed to parse internal compilation artifact")?;
 
diff --git a/src/commands/compile.rs b/src/commands/compile.rs
@@ -35,10 +35,6 @@ pub struct CompileCommand {
     #[allow(missing_docs)]
     pub common: CommonOptions,
 
-    /// The target triple; default is the host triple
-    #[arg(long, value_name = "TARGET")]
-    pub target: Option<String>,
-
     /// The path of the output compiled module; defaults to `<MODULE>.cwasm`
     #[arg(short = 'o', long, value_name = "OUTPUT")]
     pub output: Option<PathBuf>,
@@ -57,7 +53,7 @@ impl CompileCommand {
     pub fn execute(mut self) -> Result<()> {
         self.common.init_logging()?;
 
-        let mut config = self.common.config(self.target.as_deref(), None)?;
+        let mut config = self.common.config(None)?;
 
         if let Some(path) = self.emit_clif {
             if !path.exists() {
diff --git a/src/commands/explore.rs b/src/commands/explore.rs
@@ -13,10 +13,6 @@ pub struct ExploreCommand {
     #[command(flatten)]
     common: CommonOptions,
 
-    /// The target triple; default is the host triple
-    #[arg(long, value_name = "TARGET")]
-    target: Option<String>,
-
     /// The path of the WebAssembly module to compile
     #[arg(required = true, value_name = "MODULE")]
     module: PathBuf,
@@ -32,7 +28,7 @@ impl ExploreCommand {
     pub fn execute(mut self) -> Result<()> {
         self.common.init_logging()?;
 
-        let mut config = self.common.config(self.target.as_deref(), None)?;
+        let mut config = self.common.config(None)?;
 
         let bytes =
             Cow::Owned(std::fs::read(&self.module).with_context(|| {
@@ -63,7 +59,7 @@ impl ExploreCommand {
 
         wasmtime_explorer::generate(
             &config,
-            self.target.as_deref(),
+            self.common.target.as_deref(),
             clif_dir.as_ref().map(|tmp_dir| tmp_dir.path()),
             &bytes,
             &mut output_file,
diff --git a/src/commands/run.rs b/src/commands/run.rs
@@ -87,7 +87,7 @@ impl RunCommand {
     pub fn execute(mut self) -> Result<()> {
         self.run.common.init_logging()?;
 
-        let mut config = self.run.common.config(None, None)?;
+        let mut config = self.run.common.config(None)?;
         config.async_support(true);
 
         if self.run.common.wasm.timeout.is_some() {
diff --git a/src/commands/serve.rs b/src/commands/serve.rs
@@ -318,7 +318,7 @@ impl ServeCommand {
         let mut config = self
             .run
             .common
-            .config(None, use_pooling_allocator_by_default().unwrap_or(None))?;
+            .config(use_pooling_allocator_by_default().unwrap_or(None))?;
         config.wasm_component_model(true);
         config.async_support(true);
 
diff --git a/src/commands/wast.rs b/src/commands/wast.rs
@@ -23,7 +23,7 @@ impl WastCommand {
     pub fn execute(mut self) -> Result<()> {
         self.common.init_logging()?;
 
-        let config = self.common.config(None, None)?;
+        let config = self.common.config(None)?;
         let store = Store::new(&Engine::new(&config)?, ());
         let mut wast_context = WastContext::new(store);
 
diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs
@@ -58,7 +58,7 @@ fn get_wasmtime_path() -> Result<PathBuf> {
 
 // Run the wasmtime CLI with the provided args and, if it succeeds, return
 // the standard output in a `String`.
-fn run_wasmtime(args: &[&str]) -> Result<String> {
+pub fn run_wasmtime(args: &[&str]) -> Result<String> {
     let output = run_wasmtime_for_output(args, None)?;
     if !output.status.success() {
         bail!(
diff --git a/tests/all/cli_tests/empty-module.wat b/tests/all/cli_tests/empty-module.wat
@@ -0,0 +1 @@
+(module)
diff --git a/tests/all/main.rs b/tests/all/main.rs
@@ -33,6 +33,7 @@ mod name;
 mod noextern;
 mod piped_tests;
 mod pooling_allocator;
+mod pulley;
 mod relocs;
 mod stack_creator;
 mod stack_overflow;
diff --git a/tests/all/pulley.rs b/tests/all/pulley.rs
@@ -0,0 +1,76 @@
+use anyhow::Result;
+use wasmtime::{Config, Engine, Module};
+
+fn pulley_target() -> &'static str {
+    if cfg!(target_pointer_width = "64") {
+        "pulley64"
+    } else {
+        "pulley32"
+    }
+}
+
+fn pulley_config() -> Config {
+    let mut config = Config::new();
+    config.target(pulley_target()).unwrap();
+    config
+}
+
+#[test]
+fn can_compile_pulley_module() -> Result<()> {
+    let engine = Engine::new(&pulley_config())?;
+    Module::new(&engine, "(module)")?;
+    Ok(())
+}
+
+#[test]
+fn can_deserialize_pulley_module() -> Result<()> {
+    let engine = Engine::new(&pulley_config())?;
+    let bytes = engine.precompile_module(b"(module)")?;
+    unsafe {
+        Module::deserialize(&engine, &bytes)?;
+    }
+    Ok(())
+}
+
+#[test]
+fn pulley_wrong_architecture_is_rejected() -> Result<()> {
+    let mut config = Config::new();
+    // Intentionally swap pointer widths here to ensure pulley is the wrong one.
+    if cfg!(target_pointer_width = "64") {
+        config.target("pulley32").unwrap();
+    } else {
+        config.target("pulley64").unwrap();
+    }
+
+    // Creating `Module` should fail as we can't run the wrong architecture.
+    let engine = Engine::new(&config)?;
+    assert!(Module::new(&engine, "(module)").is_err());
+
+    // Precompiling should succeed but deserialization should fail because it's
+    // the wrong pointer width.
+    let engine = Engine::new(&config)?;
+    let bytes = engine.precompile_module(b"(module)")?;
+    unsafe {
+        assert!(Module::deserialize(&engine, &bytes).is_err());
+    }
+    Ok(())
+}
+
+// CLI subcommands should support `--target`
+#[test]
+#[cfg(not(miri))]
+fn can_run_on_cli() -> Result<()> {
+    use crate::cli_tests::run_wasmtime;
+    run_wasmtime(&[
+        "--target",
+        pulley_target(),
+        "tests/all/cli_tests/empty-module.wat",
+    ])?;
+    run_wasmtime(&[
+        "run",
+        "--target",
+        pulley_target(),
+        "tests/all/cli_tests/empty-module.wat",
+    ])?;
+    Ok(())
+}
diff --git a/tests/disas.rs b/tests/disas.rs
@@ -174,7 +174,8 @@ impl Test {
         // Use wasmtime::Config with its `emit_clif` option to get Wasmtime's
         // code generator to jettison CLIF out the back.
         let tempdir = TempDir::new().context("failed to make a tempdir")?;
-        let mut config = self.opts.config(Some(&self.config.target), None)?;
+        let mut config = self.opts.config(None)?;
+        config.target(&self.config.target)?;
         match self.config.test {
             TestKind::Clif | TestKind::Optimize => {
                 config.emit_clif(tempdir.path());
