diff --git a/Cargo.lock b/Cargo.lock index 867d222e5ad..4bfb2c73e44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1715,7 +1715,7 @@ dependencies = [ "spin", "wasmi_collections 0.51.0", "wasmi_core 0.51.0", - "wasmi_ir 0.51.0", + "wasmi_ir2", "wasmparser 0.228.0", "wat", ] diff --git a/Cargo.toml b/Cargo.toml index 360aa9a81d7..53836b57527 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,3 +61,6 @@ codegen-units = 1 [profile.miri] inherits = "dev" debug-assertions = false + +[profile.release] +codegen-units = 1 diff --git a/crates/core/src/trap.rs b/crates/core/src/trap.rs index 7c54f2b0def..11dd7a22d57 100644 --- a/crates/core/src/trap.rs +++ b/crates/core/src/trap.rs @@ -336,6 +336,10 @@ generate_trap_code! { /// desire on the part of the embedder to trap the interpreter rather than /// merely fail the growth operation. GrowthOperationLimited = 10, + + /// This trap is raised when a WebAssembly operation demanded a memory + /// allocation and the host system could not supply the requested amount. + OutOfSystemMemory = 11, } impl TrapCode { @@ -358,6 +362,7 @@ impl TrapCode { Self::BadSignature => "indirect call type mismatch", Self::OutOfFuel => "all fuel consumed by WebAssembly", Self::GrowthOperationLimited => "growth operation limited", + Self::OutOfSystemMemory => "out of system memory", } } } diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index 6df7912fd39..a265888da00 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -95,7 +95,7 @@ pub trait Integer: Sized + Unsigned { /// Signed shift-right `self` by `other`. fn shr_s(lhs: Self, rhs: Self) -> Self; /// Unsigned shift-right `self` by `other`. - fn shr_u(lhs: Self, rhs: Self) -> Self; + fn shr_u(lhs: Self::Uint, rhs: Self::Uint) -> Self::Uint; /// Get left bit rotation result. fn rotl(lhs: Self, rhs: Self) -> Self; /// Get right bit rotation result. @@ -130,25 +130,14 @@ pub trait Integer: Sized + Unsigned { pub trait Unsigned { /// The unsigned type. type Uint; - - /// Converts `self` losslessly to the unsigned type. - fn to_unsigned(self) -> Self::Uint; } impl Unsigned for i32 { type Uint = u32; - #[inline] - fn to_unsigned(self) -> Self::Uint { - self as _ - } } impl Unsigned for i64 { type Uint = u64; - #[inline] - fn to_unsigned(self) -> Self::Uint { - self as _ - } } /// Float-point value. @@ -270,8 +259,8 @@ macro_rules! impl_integer { lhs.wrapping_shr(rhs as u32) } #[inline] - fn shr_u(lhs: Self, rhs: Self) -> Self { - lhs.to_unsigned().wrapping_shr(rhs as u32) as _ + fn shr_u(lhs: Self::Uint, rhs: Self::Uint) -> Self::Uint { + lhs.wrapping_shr(rhs as u32) as _ } #[inline] fn rotl(lhs: Self, rhs: Self) -> Self { diff --git a/crates/core/src/wasm.rs b/crates/core/src/wasm.rs index 61f51f49aec..3b195388e8d 100644 --- a/crates/core/src/wasm.rs +++ b/crates/core/src/wasm.rs @@ -96,8 +96,8 @@ impl_untyped_val! { fn i64_shl(lhs: i64, rhs: i64) -> i64 = Integer::shl; fn i32_shr_s(lhs: i32, rhs: i32) -> i32 = Integer::shr_s; fn i64_shr_s(lhs: i64, rhs: i64) -> i64 = Integer::shr_s; - fn i32_shr_u(lhs: i32, rhs: i32) -> i32 = Integer::shr_u; - fn i64_shr_u(lhs: i64, rhs: i64) -> i64 = Integer::shr_u; + fn i32_shr_u(lhs: u32, rhs: u32) -> u32 = ::shr_u; + fn i64_shr_u(lhs: u64, rhs: u64) -> u64 = ::shr_u; fn i32_rotl(lhs: i32, rhs: i32) -> i32 = Integer::rotl; fn i64_rotl(lhs: i64, rhs: i64) -> i64 = Integer::rotl; fn i32_rotr(lhs: i32, rhs: i32) -> i32 = Integer::rotr; diff --git a/crates/fuzz/Cargo.toml b/crates/fuzz/Cargo.toml index 72fa917d764..d7236e69328 100644 --- a/crates/fuzz/Cargo.toml +++ b/crates/fuzz/Cargo.toml @@ -15,7 +15,7 @@ exclude.workspace = true publish = false [dependencies] -wasmi = { workspace = true, features = ["std", "simd"] } +wasmi = { workspace = true, features = ["std", "simd", "portable-dispatch", "extra-checks"] } wasmtime = { workspace = true, optional = true, features = [ "cranelift", "runtime", diff --git a/crates/fuzz/src/config.rs b/crates/fuzz/src/config.rs index 1e58f34d220..67b68c841a8 100644 --- a/crates/fuzz/src/config.rs +++ b/crates/fuzz/src/config.rs @@ -89,8 +89,8 @@ impl Arbitrary<'_> for FuzzSmithConfig { custom_page_sizes_enabled: true, bulk_memory_enabled: true, reference_types_enabled: false, // TODO: re-enable reference-types for differential fuzzing - simd_enabled: true, - relaxed_simd_enabled: true, + simd_enabled: false, + relaxed_simd_enabled: false, multi_value_enabled: true, memory64_enabled: true, saturating_float_to_int_enabled: true, diff --git a/crates/fuzz/src/oracle/wasmi.rs b/crates/fuzz/src/oracle/wasmi.rs index 1a22b5cd08b..7c4b6961d99 100644 --- a/crates/fuzz/src/oracle/wasmi.rs +++ b/crates/fuzz/src/oracle/wasmi.rs @@ -175,7 +175,9 @@ impl From for FuzzError { | TrapCode::BadConversionToInteger => crate::TrapCode::BadConversionToInteger, | TrapCode::StackOverflow => crate::TrapCode::StackOverflow, | TrapCode::BadSignature => crate::TrapCode::BadSignature, - | TrapCode::OutOfFuel | TrapCode::GrowthOperationLimited => return FuzzError::Other, + | TrapCode::OutOfFuel + | TrapCode::GrowthOperationLimited + | TrapCode::OutOfSystemMemory => return FuzzError::Other, }; FuzzError::Trap(trap_code) } diff --git a/crates/ir2/build/display/op.rs b/crates/ir2/build/display/op.rs index 450dff29f1f..32f58c122ac 100644 --- a/crates/ir2/build/display/op.rs +++ b/crates/ir2/build/display/op.rs @@ -144,7 +144,9 @@ impl Display for DisplayOp<&'_ Isa> { {indent}#[macro_export]\n\ {indent}macro_rules! for_each_op {{\n\ {indent} ($mac:ident) => {{\n\ - {for_each_op_body},\n\ + {indent} $mac! {{\n\ + {for_each_op_body},\n\ + {indent} }}\n\ {indent} }};\n\ {indent}}}\n\ " diff --git a/crates/ir2/build/isa.rs b/crates/ir2/build/isa.rs index 8489358d240..e37800954e5 100644 --- a/crates/ir2/build/isa.rs +++ b/crates/ir2/build/isa.rs @@ -418,15 +418,15 @@ fn add_control_ops(isa: &mut Isa) { Op::from(GenericOp::new( Ident::BranchTable, [ - Field::new(Ident::Index, FieldTy::Slot), Field::new(Ident::LenTargets, FieldTy::U32), + Field::new(Ident::Index, FieldTy::Slot), ], )), Op::from(GenericOp::new( Ident::BranchTableSpan, [ - Field::new(Ident::Index, FieldTy::Slot), Field::new(Ident::LenTargets, FieldTy::U32), + Field::new(Ident::Index, FieldTy::Slot), Field::new(Ident::Values, FieldTy::SlotSpan), Field::new(Ident::LenValues, FieldTy::U16), ], @@ -542,8 +542,8 @@ fn add_global_ops(isa: &mut Isa) { Op::from(GenericOp::new( Ident::GlobalGet, [ - Field::new(Ident::Result, FieldTy::Slot), Field::new(Ident::Global, FieldTy::Global), + Field::new(Ident::Result, FieldTy::Slot), ], )), Op::from(GenericOp::new( @@ -563,8 +563,8 @@ fn add_global_ops(isa: &mut Isa) { Op::from(GenericOp::new( Ident::GlobalSet64, [ - Field::new(Ident::Global, FieldTy::Global), Field::new(Ident::Value, FieldTy::U64), + Field::new(Ident::Global, FieldTy::Global), ], )), ]; diff --git a/crates/wasmi/Cargo.toml b/crates/wasmi/Cargo.toml index 47c5adf5b4f..72b1a18f80f 100644 --- a/crates/wasmi/Cargo.toml +++ b/crates/wasmi/Cargo.toml @@ -21,7 +21,7 @@ exclude = [ [dependencies] wasmi_core = { workspace = true } wasmi_collections = { workspace = true } -wasmi_ir = { workspace = true } +wasmi_ir2 = { workspace = true } wasmparser = { workspace = true, features = ["validate", "features"] } wat = { workspace = true, optional = true } spin = { version = "0.9", default-features = false, features = [ @@ -51,7 +51,14 @@ prefer-btree-collections = [ "wasmparser/prefer-btree-collections", ] wat = ["dep:wat", "std"] -simd = ["wasmi_core/simd", "wasmi_ir/simd", "wasmparser/simd"] +simd = ["wasmi_core/simd", "wasmi_ir2/simd", "wasmparser/simd"] + +# Enables a portable dispatch scheme that avoids tail-call reliance. +# Recommended for targets without stable or guaranteed tail-call support. +portable-dispatch = [] +# Enables an indirect dispatch scheme using op-codes instead of +# embedded function pointers for more compact bytecode. +indirect-dispatch = [] # Enables extra checks performed during Wasmi bytecode execution. # diff --git a/crates/wasmi/src/engine/code_map.rs b/crates/wasmi/src/engine/code_map.rs index 75959cc91a9..51bd9403a94 100644 --- a/crates/wasmi/src/engine/code_map.rs +++ b/crates/wasmi/src/engine/code_map.rs @@ -8,10 +8,10 @@ use super::{FuncTranslationDriver, FuncTranslator, TranslationError, ValidatingFuncTranslator}; use crate::{ collections::arena::{Arena, ArenaIndex}, - core::{Fuel, FuelCostsProvider, UntypedVal}, + core::{Fuel, FuelCostsProvider}, engine::{utils::unreachable_unchecked, ResumableOutOfFuelError}, errors::FuelError, - ir::{index::InternalFunc, Op}, + ir::index::InternalFunc, module::{FuncIdx, ModuleHeader}, Config, Error, @@ -28,6 +28,9 @@ use core::{ use spin::Mutex; use wasmparser::{FuncToValidate, ValidatorResources, WasmFeatures}; +#[cfg(doc)] +use crate::ir::Op; + /// A reference to a compiled function stored in the [`CodeMap`] of an [`Engine`](crate::Engine). #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct EngineFunc(u32); @@ -785,9 +788,7 @@ impl<'a> From<&'a [u8]> for SmallByteSlice { #[derive(Debug)] pub struct CompiledFuncEntity { /// The sequence of [`Op`] of the [`CompiledFuncEntity`]. - instrs: Pin>, - /// The constant values local to the [`EngineFunc`]. - consts: Pin>, + ops: Pin>, /// The number of stack slots used by the [`EngineFunc`] in total. /// /// # Note @@ -802,17 +803,12 @@ impl CompiledFuncEntity { /// /// # Panics /// - /// - If `instrs` is empty. - /// - If `instrs` contains more than `i32::MAX` instructions. - pub fn new(len_stack_slots: u16, instrs: I, consts: C) -> Self - where - I: IntoIterator, - C: IntoIterator, - { - let instrs: Pin> = Pin::new(instrs.into_iter().collect()); - let consts: Pin> = Pin::new(consts.into_iter().collect()); + /// - If `ops` is empty. + /// - If `ops` contains more than `i32::MAX` encoded bytes. + pub fn new(len_stack_slots: u16, ops: &[u8]) -> Self { + let ops: Pin> = Pin::new(ops.into()); assert!( - !instrs.is_empty(), + !ops.is_empty(), "compiled functions must have at least one instruction" ); assert!( @@ -820,13 +816,12 @@ impl CompiledFuncEntity { // However, Wasmi's branch instructions can jump across at most `i32::MAX` // forwards or `i32::MIN` instructions backwards and thus having more than // `i32::MAX` instructions might introduce problems. - instrs.len() <= i32::MAX as usize, + ops.len() <= i32::MAX as usize, "compiled function has too many instructions: {}", - instrs.len(), + ops.len(), ); Self { - instrs, - consts, + ops, len_stack_slots, } } @@ -835,10 +830,8 @@ impl CompiledFuncEntity { /// A shared reference to the data of a [`EngineFunc`]. #[derive(Debug, Copy, Clone)] pub struct CompiledFuncRef<'a> { - /// The sequence of [`Op`] of the [`CompiledFuncEntity`]. - instrs: Pin<&'a [Op]>, - /// The constant values local to the [`EngineFunc`]. - consts: Pin<&'a [UntypedVal]>, + /// The sequence of encoded [`Op`]s of the [`CompiledFuncEntity`]. + ops: Pin<&'a [u8]>, /// The number of stack slots used by the [`EngineFunc`] in total. len_stack_slots: u16, } @@ -847,18 +840,17 @@ impl<'a> From<&'a CompiledFuncEntity> for CompiledFuncRef<'a> { #[inline] fn from(func: &'a CompiledFuncEntity) -> Self { Self { - instrs: func.instrs.as_ref(), - consts: func.consts.as_ref(), + ops: func.ops.as_ref(), len_stack_slots: func.len_stack_slots, } } } impl<'a> CompiledFuncRef<'a> { - /// Returns the sequence of [`Op`] of the [`EngineFunc`]. + /// Returns the sequence of encoded [`Op`]s of the [`EngineFunc`]. #[inline] - pub fn instrs(&self) -> &'a [Op] { - self.instrs.get_ref() + pub fn ops(&self) -> &'a [u8] { + self.ops.get_ref() } /// Returns the number of stack slots used by the [`EngineFunc`]. @@ -866,10 +858,4 @@ impl<'a> CompiledFuncRef<'a> { pub fn len_stack_slots(&self) -> u16 { self.len_stack_slots } - - /// Returns the function local constant values of the [`EngineFunc`]. - #[inline] - pub fn consts(&self) -> &'a [UntypedVal] { - self.consts.get_ref() - } } diff --git a/crates/wasmi/src/engine/executor/cache.rs b/crates/wasmi/src/engine/executor/cache.rs deleted file mode 100644 index 2476f5d3e4a..00000000000 --- a/crates/wasmi/src/engine/executor/cache.rs +++ /dev/null @@ -1,321 +0,0 @@ -use crate::{ - core::UntypedVal, - engine::DedupFuncType, - instance::InstanceEntity, - ir::index, - memory::DataSegment, - module::DEFAULT_MEMORY_INDEX, - store::StoreInner, - table::ElementSegment, - Func, - Global, - Instance, - Memory, - Table, -}; -use core::ptr::{self, NonNull}; - -/// Cached WebAssembly instance. -#[derive(Debug)] -pub struct CachedInstance { - /// The currently used instance. - instance: NonNull, - /// The cached bytes of the default linear memory. - pub memory: CachedMemory, - /// The cached value of the global variable at index 0. - pub global: CachedGlobal, -} - -impl CachedInstance { - /// Creates a new [`CachedInstance`]. - #[inline] - pub fn new(ctx: &mut StoreInner, instance: &Instance) -> Self { - let (instance, memory, global) = Self::load_caches(ctx, instance); - Self { - instance, - memory, - global, - } - } - - /// Loads the [`InstanceEntity`] from the [`StoreInner`]. - #[inline] - fn load_instance<'ctx>(ctx: &'ctx mut StoreInner, instance: &Instance) -> &'ctx InstanceEntity { - ctx.resolve_instance(instance) - } - - /// Loads the cached global and linear memory. - #[inline] - fn load_caches( - ctx: &mut StoreInner, - instance: &Instance, - ) -> (NonNull, CachedMemory, CachedGlobal) { - let entity = Self::load_instance(ctx, instance); - let memory = entity.get_memory(DEFAULT_MEMORY_INDEX); - let global = entity.get_global(0); - let instance = entity.into(); - let memory = memory - .map(|memory| CachedMemory::new(ctx, &memory)) - .unwrap_or_default(); - let global = global - .map(|global| CachedGlobal::new(ctx, &global)) - .unwrap_or_default(); - (instance, memory, global) - } - - /// Update the cached instance, linear memory and global variable. - #[inline] - pub fn update(&mut self, ctx: &mut StoreInner, instance: &Instance) { - (self.instance, self.memory, self.global) = Self::load_caches(ctx, instance); - } - - /// Returns a shared reference to the cached [`InstanceEntity`]. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - unsafe fn as_ref(&self) -> &InstanceEntity { - unsafe { self.instance.as_ref() } - } - - /// Updates the [`CachedMemory`]'s linear memory data pointer. - /// - /// # Note - /// - /// This needs to be called whenever the cached pointer might have changed. - /// - /// The linear memory pointer might change when ... - /// - /// - calling a host function - /// - successfully growing the default linear memory - /// - calling functions defined in other instances via imported or indirect calls - /// - returning from functions that changed the currently used instance - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn update_memory(&mut self, ctx: &mut StoreInner) { - let instance = unsafe { self.as_ref() }; - self.memory = instance - .get_memory(DEFAULT_MEMORY_INDEX) - .map(|memory| CachedMemory::new(ctx, &memory)) - .unwrap_or_default(); - } - - /// Returns the [`Func`] at the `index` if any. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn get_func(&self, index: index::Func) -> Option { - let instance = unsafe { self.as_ref() }; - instance.get_func(u32::from(index)) - } - - /// Returns the [`Memory`] at the `index` if any. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn get_memory(&self, index: index::Memory) -> Option { - let instance = unsafe { self.as_ref() }; - instance.get_memory(u32::from(index)) - } - - /// Returns the [`Table`] at the `index` if any. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn get_table(&self, index: index::Table) -> Option { - let instance = unsafe { self.as_ref() }; - instance.get_table(u32::from(index)) - } - - /// Returns the [`Global`] at the `index` if any. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn get_global(&self, index: index::Global) -> Option { - let instance = unsafe { self.as_ref() }; - instance.get_global(u32::from(index)) - } - - /// Returns the [`DataSegment`] at the `index` if any. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn get_data_segment(&self, index: index::Data) -> Option { - let instance = unsafe { self.as_ref() }; - instance.get_data_segment(u32::from(index)) - } - - /// Returns the [`ElementSegment`] at the `index` if any. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn get_element_segment(&self, index: index::Elem) -> Option { - let instance = unsafe { self.as_ref() }; - instance.get_element_segment(u32::from(index)) - } - - /// Returns the [`DedupFuncType`] at the `index` if any. - /// - /// # Safety - /// - /// It is the callers responsibility to use this method only when the caches are fresh. - #[inline] - pub unsafe fn get_func_type_dedup(&self, index: index::FuncType) -> Option { - let instance = unsafe { self.as_ref() }; - instance.get_signature(u32::from(index)).copied() - } -} - -/// Cached default linear memory bytes. -#[derive(Debug)] -pub struct CachedMemory { - data: NonNull<[u8]>, -} - -impl Default for CachedMemory { - #[inline] - fn default() -> Self { - Self { - data: NonNull::from(&mut []), - } - } -} - -impl CachedMemory { - /// Create a new [`CachedMemory`]. - #[inline] - fn new(ctx: &mut StoreInner, instance: &Memory) -> Self { - let data = Self::load_default_memory(ctx, instance); - Self { data } - } - - /// Loads the default [`Memory`] of the currently used [`Instance`]. - /// - /// # Note - /// - /// Must be called whenever the heap allocation of the [`CachedMemory`] - /// could have been changed and thus the cached pointer invalidated. - /// - /// # Panics - /// - /// If the currently used [`Instance`] does not have a default linear memory. - /// - /// [`Memory`]: crate::Memory - #[inline] - fn load_default_memory(ctx: &mut StoreInner, memory: &Memory) -> NonNull<[u8]> { - ctx.resolve_memory_mut(memory).data_mut().into() - } - - /// Returns a shared slice to the bytes of the cached default linear memory. - /// - /// # Safety - /// - /// The user is required to call [`CachedMemory::load_default_memory`] according to its specification. - #[inline] - pub unsafe fn data(&self) -> &[u8] { - unsafe { self.data.as_ref() } - } - - /// Returns an exclusive slice to the bytes of the cached default linear memory. - /// - /// # Safety - /// - /// The user is required to call [`CachedMemory::load_default_memory`] according to its specification. - #[inline] - pub unsafe fn data_mut(&mut self) -> &mut [u8] { - unsafe { self.data.as_mut() } - } -} - -/// Cached default global variable value. -#[derive(Debug)] -pub struct CachedGlobal { - // Dev. Note: we cannot use `NonNull` here, yet. - // - // The advantage is that we could safely use a static fallback value - // which would be safer than using a null pointer since it would - // only read or overwrite the fallback value instead of reading or - // writing a null pointer which is UB. - // - // We cannot use `NonNull` because it requires pointers - // to mutable statics which have just been allowed in Rust 1.78 but - // not in Rust 1.77 which is Wasmi's MSRV. - // - // We can and should use `NonNull` here once we bump the MSRV. - data: *mut UntypedVal, -} - -impl Default for CachedGlobal { - #[inline] - fn default() -> Self { - Self { - data: ptr::null_mut(), - } - } -} - -impl CachedGlobal { - /// Create a new [`CachedGlobal`]. - #[inline] - fn new(ctx: &mut StoreInner, global: &Global) -> Self { - let data = Self::load_global(ctx, global); - Self { data } - } - - /// Loads the default [`Global`] of the currently used [`Instance`]. - /// - /// # Note - /// - /// Must be called whenever the heap allocation of the [`CachedGlobal`] - /// could have been changed and thus the cached pointer invalidated. - /// - /// # Panics - /// - /// If the currently used [`Instance`] does not have a default linear memory. - /// - /// [`Global`]: crate::Global - #[inline] - fn load_global(ctx: &mut StoreInner, global: &Global) -> *mut UntypedVal { - ctx.resolve_global_mut(global).get_untyped_ptr().as_ptr() - } - - /// Returns the value of the cached global variable. - /// - /// # Safety - /// - /// The user is required to call [`CachedGlobal::load_global`] according to its specification. - #[inline] - pub unsafe fn get(&self) -> UntypedVal { - // SAFETY: This API guarantees to always write to a valid pointer - // as long as `update` is called when needed by the user. - unsafe { self.data.read() } - } - - /// Sets the value of the cached global variable to `new_value`. - /// - /// # Safety - /// - /// The user is required to call [`CachedGlobal::load_global`] according to its specification. - #[inline] - pub unsafe fn set(&mut self, new_value: UntypedVal) { - // SAFETY: This API guarantees to always write to a valid pointer - // as long as `update` is called when needed by the user. - unsafe { self.data.write(new_value) }; - } -} diff --git a/crates/wasmi/src/engine/executor/handler/dispatch/backend/loop.rs b/crates/wasmi/src/engine/executor/handler/dispatch/backend/loop.rs new file mode 100644 index 00000000000..2aa608af101 --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/dispatch/backend/loop.rs @@ -0,0 +1,66 @@ +use crate::engine::executor::handler::{ + dispatch::{fetch_handler, Control, ExecutionOutcome, Break}, + state::{Inst, Ip, Mem0Len, Mem0Ptr, Sp, VmState}, +}; + +#[derive(Debug, Copy, Clone)] +pub struct NextState { + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +} + +pub type Done = Control; + +#[inline(always)] +pub fn control_continue(ip: Ip, sp: Sp, mem0: Mem0Ptr, mem0_len: Mem0Len, instance: Inst) -> Done { + Done::Continue(NextState { + ip, + sp, + mem0, + mem0_len, + instance, + }) +} + +macro_rules! dispatch { + ($state:expr, $ip:expr, $sp:expr, $mem0:expr, $mem0_len:expr, $instance:expr) => {{ + let _: &mut VmState = $state; + return $crate::engine::executor::handler::dispatch::backend::control_continue( + $ip, $sp, $mem0, $mem0_len, $instance, + ); + }}; +} + +pub fn execute_until_done( + mut state: VmState, + mut ip: Ip, + mut sp: Sp, + mut mem0: Mem0Ptr, + mut mem0_len: Mem0Len, + mut instance: Inst, +) -> Result { + let mut handler = fetch_handler(ip); + 'exec: loop { + match handler(&mut state, ip, sp, mem0, mem0_len, instance) { + Done::Continue(next) => { + handler = fetch_handler(next.ip); + ip = next.ip; + sp = next.sp; + mem0 = next.mem0; + mem0_len = next.mem0_len; + instance = next.instance; + continue 'exec; + } + Done::Break(reason) => { + if let Some(trap_code) = reason.trap_code() { + return Err(ExecutionOutcome::from(trap_code)); + } + break 'exec; + } + } + } + state.into_execution_outcome() +} diff --git a/crates/wasmi/src/engine/executor/handler/dispatch/backend/tail.rs b/crates/wasmi/src/engine/executor/handler/dispatch/backend/tail.rs new file mode 100644 index 00000000000..1be04b7c97b --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/dispatch/backend/tail.rs @@ -0,0 +1,31 @@ +use crate::engine::executor::handler::{ + dispatch::{fetch_handler, Break, Control, ExecutionOutcome}, + state::{Inst, Ip, Mem0Len, Mem0Ptr, Sp, VmState}, +}; + +pub enum Never {} +pub type Done = Control; + +macro_rules! dispatch { + ($state:expr, $ip:expr, $sp:expr, $mem0:expr, $mem0_len:expr, $instance:expr) => {{ + let handler = $crate::engine::executor::handler::dispatch::fetch_handler($ip); + return handler($state, $ip, $sp, $mem0, $mem0_len, $instance); + }}; +} + +pub fn execute_until_done( + state: VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Result { + let mut state = state; + let handler = fetch_handler(ip); + let Control::Break(reason) = handler(&mut state, ip, sp, mem0, mem0_len, instance); + if let Some(trap_code) = reason.trap_code() { + return Err(ExecutionOutcome::from(trap_code)); + } + state.into_execution_outcome() +} diff --git a/crates/wasmi/src/engine/executor/handler/dispatch/mod.rs b/crates/wasmi/src/engine/executor/handler/dispatch/mod.rs new file mode 100644 index 00000000000..1bc7a08bbb8 --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/dispatch/mod.rs @@ -0,0 +1,159 @@ +#[cfg_attr(feature = "portable-dispatch", path = "backend/loop.rs")] +#[cfg_attr(not(feature = "portable-dispatch"), path = "backend/tail.rs")] +#[macro_use] +pub mod backend; + +pub use self::backend::{execute_until_done, Done}; +use super::{ + exec, + state::{Inst, Ip, Mem0Len, Mem0Ptr, Sp, VmState}, +}; +use crate::{ + engine::{ResumableHostTrapError, ResumableOutOfFuelError}, + ir, + ir::OpCode, + Error, + TrapCode, +}; +use core::ops::ControlFlow; + +#[inline(always)] +pub fn control_break() -> Control { + Control::Break(Break::WithReason) +} + +#[inline(always)] +pub fn fetch_handler(ip: Ip) -> Handler { + match cfg!(feature = "indirect-dispatch") { + true => { + let (_, op_code) = unsafe { ip.decode::() }; + op_code_to_handler(op_code) + } + false => { + let (_, addr) = unsafe { ip.decode::() }; + unsafe { + ::core::mem::transmute::<*const (), Handler>(::core::ptr::with_exposed_provenance( + addr, + )) + } + } + } +} + +#[derive(Debug)] +pub enum ExecutionOutcome { + Host(ResumableHostTrapError), + OutOfFuel(ResumableOutOfFuelError), + Error(Error), +} + +impl From for Error { + fn from(error: ExecutionOutcome) -> Self { + match error { + ExecutionOutcome::Host(error) => error.into(), + ExecutionOutcome::OutOfFuel(error) => error.into(), + ExecutionOutcome::Error(error) => error, + } + } +} + +impl From for ExecutionOutcome { + fn from(error: ResumableHostTrapError) -> Self { + Self::Host(error) + } +} + +impl From for ExecutionOutcome { + fn from(error: ResumableOutOfFuelError) -> Self { + Self::OutOfFuel(error) + } +} + +impl From for ExecutionOutcome { + fn from(error: TrapCode) -> Self { + Self::Error(error.into()) + } +} + +impl From for ExecutionOutcome { + fn from(error: Error) -> Self { + Self::Error(error) + } +} + +#[derive(Debug, Copy, Clone)] +pub enum Break { + UnreachableCodeReached = TrapCode::UnreachableCodeReached as _, + MemoryOutOfBounds = TrapCode::MemoryOutOfBounds as _, + TableOutOfBounds = TrapCode::TableOutOfBounds as _, + IndirectCallToNull = TrapCode::IndirectCallToNull as _, + IntegerDivisionByZero = TrapCode::IntegerDivisionByZero as _, + IntegerOverflow = TrapCode::IntegerOverflow as _, + BadConversionToInteger = TrapCode::BadConversionToInteger as _, + StackOverflow = TrapCode::StackOverflow as _, + BadSignature = TrapCode::BadSignature as _, + OutOfFuel = TrapCode::OutOfFuel as _, + GrowthOperationLimited = TrapCode::GrowthOperationLimited as _, + OutOfSystemMemory = TrapCode::OutOfSystemMemory as _, + /// Signals that there must be a reason stored externally supplying the caller with more information. + WithReason, +} + +impl From for Break { + #[inline] + fn from(trap_code: TrapCode) -> Self { + match trap_code { + TrapCode::UnreachableCodeReached => Self::UnreachableCodeReached, + TrapCode::MemoryOutOfBounds => Self::MemoryOutOfBounds, + TrapCode::TableOutOfBounds => Self::TableOutOfBounds, + TrapCode::IndirectCallToNull => Self::IndirectCallToNull, + TrapCode::IntegerDivisionByZero => Self::IntegerDivisionByZero, + TrapCode::IntegerOverflow => Self::IntegerOverflow, + TrapCode::BadConversionToInteger => Self::BadConversionToInteger, + TrapCode::StackOverflow => Self::StackOverflow, + TrapCode::BadSignature => Self::BadSignature, + TrapCode::OutOfFuel => Self::OutOfFuel, + TrapCode::GrowthOperationLimited => Self::GrowthOperationLimited, + TrapCode::OutOfSystemMemory => Self::OutOfSystemMemory, + } + } +} + +impl Break { + #[inline] + pub fn trap_code(self) -> Option { + let trap_code = match self { + Self::UnreachableCodeReached => TrapCode::UnreachableCodeReached, + Self::MemoryOutOfBounds => TrapCode::MemoryOutOfBounds, + Self::TableOutOfBounds => TrapCode::TableOutOfBounds, + Self::IndirectCallToNull => TrapCode::IndirectCallToNull, + Self::IntegerDivisionByZero => TrapCode::IntegerDivisionByZero, + Self::IntegerOverflow => TrapCode::IntegerOverflow, + Self::BadConversionToInteger => TrapCode::BadConversionToInteger, + Self::StackOverflow => TrapCode::StackOverflow, + Self::BadSignature => TrapCode::BadSignature, + Self::OutOfFuel => TrapCode::OutOfFuel, + Self::GrowthOperationLimited => TrapCode::GrowthOperationLimited, + Self::OutOfSystemMemory => TrapCode::OutOfSystemMemory, + _ => return None, + }; + Some(trap_code) + } +} + +pub type Control = ControlFlow; + +type Handler = + fn(&mut VmState, ip: Ip, sp: Sp, mem0: Mem0Ptr, mem0_len: Mem0Len, instance: Inst) -> Done; + +macro_rules! expand_op_code_to_handler { + ( $( $snake_case:ident => $camel_case:ident ),* $(,)? ) => { + #[inline(always)] + pub fn op_code_to_handler(code: OpCode) -> Handler { + match code { + $( OpCode::$camel_case => exec::$snake_case, )* + } + } + }; +} +ir::for_each_op!(expand_op_code_to_handler); diff --git a/crates/wasmi/src/engine/executor/handler/eval.rs b/crates/wasmi/src/engine/executor/handler/eval.rs new file mode 100644 index 00000000000..3e97df51319 --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/eval.rs @@ -0,0 +1,133 @@ +use crate::{core::wasm, ir::Sign, TrapCode}; +use core::{ + num::NonZero, + ops::{Div, Rem}, +}; + +pub fn wasmi_i32_div_ssi(lhs: i32, rhs: NonZero) -> Result { + wasm::i32_div_s(lhs, rhs.get()) +} + +pub fn wasmi_i64_div_ssi(lhs: i64, rhs: NonZero) -> Result { + wasm::i64_div_s(lhs, rhs.get()) +} + +pub fn wasmi_u32_div_ssi(lhs: u32, rhs: NonZero) -> u32 { + >>::div(lhs, rhs) +} + +pub fn wasmi_u64_div_ssi(lhs: u64, rhs: NonZero) -> u64 { + >>::div(lhs, rhs) +} + +pub fn wasmi_i32_rem_ssi(lhs: i32, rhs: NonZero) -> Result { + wasm::i32_rem_s(lhs, rhs.get()) +} + +pub fn wasmi_i64_rem_ssi(lhs: i64, rhs: NonZero) -> Result { + wasm::i64_rem_s(lhs, rhs.get()) +} + +pub fn wasmi_u32_rem_ssi(lhs: u32, rhs: NonZero) -> u32 { + >>::rem(lhs, rhs) +} + +pub fn wasmi_u64_rem_ssi(lhs: u64, rhs: NonZero) -> u64 { + >>::rem(lhs, rhs) +} + +pub fn wasmi_i32_shl_ssi(lhs: i32, rhs: u8) -> i32 { + wasm::i32_shl(lhs, i32::from(rhs)) +} + +pub fn wasmi_i32_shr_ssi(lhs: i32, rhs: u8) -> i32 { + wasm::i32_shr_s(lhs, i32::from(rhs)) +} + +pub fn wasmi_u32_shr_ssi(lhs: u32, rhs: u8) -> u32 { + wasm::i32_shr_u(lhs, u32::from(rhs)) +} + +pub fn wasmi_i32_rotl_ssi(lhs: i32, rhs: u8) -> i32 { + wasm::i32_rotl(lhs, i32::from(rhs)) +} + +pub fn wasmi_i32_rotr_ssi(lhs: i32, rhs: u8) -> i32 { + wasm::i32_rotr(lhs, i32::from(rhs)) +} + +pub fn wasmi_i64_shl_ssi(lhs: i64, rhs: u8) -> i64 { + wasm::i64_shl(lhs, i64::from(rhs)) +} + +pub fn wasmi_i64_shr_ssi(lhs: i64, rhs: u8) -> i64 { + wasm::i64_shr_s(lhs, i64::from(rhs)) +} + +pub fn wasmi_u64_shr_ssi(lhs: u64, rhs: u8) -> u64 { + wasm::i64_shr_u(lhs, u64::from(rhs)) +} + +pub fn wasmi_i64_rotl_ssi(lhs: i64, rhs: u8) -> i64 { + wasm::i64_rotl(lhs, i64::from(rhs)) +} + +pub fn wasmi_i64_rotr_ssi(lhs: i64, rhs: u8) -> i64 { + wasm::i64_rotr(lhs, i64::from(rhs)) +} + +pub fn wasmi_i32_and(lhs: i32, rhs: i32) -> bool { + (lhs & rhs) != 0 +} + +pub fn wasmi_i32_not_and(lhs: i32, rhs: i32) -> bool { + !wasmi_i32_and(lhs, rhs) +} + +pub fn wasmi_i32_or(lhs: i32, rhs: i32) -> bool { + (rhs != 0) || (lhs != 0) +} + +pub fn wasmi_i32_not_or(lhs: i32, rhs: i32) -> bool { + !wasmi_i32_or(lhs, rhs) +} + +pub fn wasmi_i64_and(lhs: i64, rhs: i64) -> bool { + (lhs & rhs) != 0 +} + +pub fn wasmi_i64_not_and(lhs: i64, rhs: i64) -> bool { + !wasmi_i64_and(lhs, rhs) +} + +pub fn wasmi_i64_or(lhs: i64, rhs: i64) -> bool { + (rhs != 0) || (lhs != 0) +} + +pub fn wasmi_i64_not_or(lhs: i64, rhs: i64) -> bool { + !wasmi_i64_or(lhs, rhs) +} + +pub fn wasmi_f32_copysign_ssi(lhs: f32, rhs: Sign) -> f32 { + wasm::f32_copysign(lhs, f32::from(rhs)) +} + +pub fn wasmi_f64_copysign_ssi(lhs: f64, rhs: Sign) -> f64 { + wasm::f64_copysign(lhs, f64::from(rhs)) +} + +pub fn wasmi_f32_not_le(lhs: f32, rhs: f32) -> bool { + !wasm::f32_le(lhs, rhs) +} + +pub fn wasmi_f64_not_le(lhs: f64, rhs: f64) -> bool { + !wasm::f64_le(lhs, rhs) +} + +pub fn wasmi_f32_not_lt(lhs: f32, rhs: f32) -> bool { + !wasm::f32_lt(lhs, rhs) +} + +pub fn wasmi_f64_not_lt(lhs: f64, rhs: f64) -> bool { + !wasm::f64_lt(lhs, rhs) +} diff --git a/crates/wasmi/src/engine/executor/handler/exec.rs b/crates/wasmi/src/engine/executor/handler/exec.rs new file mode 100644 index 00000000000..bc73b9d1c97 --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/exec.rs @@ -0,0 +1,1842 @@ +#[cfg(feature = "simd")] +mod simd; + +#[cfg(feature = "simd")] +pub use self::simd::*; + +use super::{ + dispatch::Done, + eval, + state::{mem0_bytes, Inst, Ip, Mem0Len, Mem0Ptr, Sp, VmState}, + utils::{fetch_func, get_value, memory_bytes, offset_ip, set_value}, +}; +use crate::{ + core::{wasm, CoreTable, UntypedVal}, + engine::{ + executor::handler::{ + dispatch::Break, + state::DoneReason, + utils::{ + call_wasm, + call_wasm_or_host, + exec_copy_span, + exec_copy_span_asc, + exec_copy_span_des, + exec_return, + extract_mem0, + fetch_data, + fetch_elem, + fetch_global, + fetch_memory, + fetch_table, + memory_slice, + memory_slice_mut, + resolve_data_mut, + resolve_elem_mut, + resolve_func, + resolve_global, + resolve_indirect_func, + resolve_instance, + resolve_memory, + resolve_table, + resolve_table_mut, + return_call_host, + return_call_wasm, + set_global, + update_instance, + IntoControl as _, + }, + Control, + }, + EngineFunc, + }, + errors::{FuelError, MemoryError, TableError}, + func::FuncEntity, + ir::{self, index, Slot, SlotSpan}, + store::StoreError, + Func, + Ref, + TrapCode, +}; +use core::cmp; + +unsafe fn decode_op(ip: Ip) -> (Ip, Op) { + let ip = match cfg!(feature = "indirect-dispatch") { + true => unsafe { ip.skip::() }, + false => unsafe { ip.skip::<::core::primitive::usize>() }, + }; + unsafe { ip.decode() } +} + +fn identity(value: T) -> T { + value +} + +pub fn trap( + _state: &mut VmState, + ip: Ip, + _sp: Sp, + _mem0: Mem0Ptr, + _mem0_len: Mem0Len, + _instance: Inst, +) -> Done { + let (_ip, crate::ir::decode::Trap { trap_code }) = unsafe { decode_op(ip) }; + trap!(trap_code) +} + +pub fn consume_fuel( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (next_ip, crate::ir::decode::ConsumeFuel { fuel }) = unsafe { decode_op(ip) }; + let consumption_result = state + .store + .inner_mut() + .fuel_mut() + .consume_fuel_unchecked(u64::from(fuel)); + if let Err(FuelError::OutOfFuel { required_fuel }) = consumption_result { + out_of_fuel!(state, ip, required_fuel) + } + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +pub fn copy_span_asc( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + ip, + crate::ir::decode::CopySpanAsc { + results, + values, + len, + }, + ) = unsafe { decode_op(ip) }; + exec_copy_span_asc(sp, results, values, len); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn copy_span_des( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + ip, + crate::ir::decode::CopySpanDes { + results, + values, + len, + }, + ) = unsafe { decode_op(ip) }; + exec_copy_span_des(sp, results, values, len); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn branch( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (_new_ip, crate::ir::decode::Branch { offset }) = unsafe { decode_op(ip) }; + let ip = offset_ip(ip, offset); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn global_get( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::GlobalGet { result, global }) = unsafe { decode_op(ip) }; + let global = fetch_global(instance, global); + let global = resolve_global(state.store, &global); + let value = *global.get_untyped(); + set_value(sp, result, value); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn global_set( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::GlobalSet { global, value }) = unsafe { decode_op(ip) }; + let value: UntypedVal = get_value(value, sp); + set_global(global, value, state, instance); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn global_set32( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::GlobalSet32 { global, value }) = unsafe { decode_op(ip) }; + let value: UntypedVal = get_value(value, sp).into(); + set_global(global, value, state, instance); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn global_set64( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::GlobalSet64 { global, value }) = unsafe { decode_op(ip) }; + let value: UntypedVal = get_value(value, sp).into(); + set_global(global, value, state, instance); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn call_internal( + state: &mut VmState, + ip: Ip, + _sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (caller_ip, crate::ir::decode::CallInternal { params, func }) = unsafe { decode_op(ip) }; + let func = EngineFunc::from(func); + let (callee_ip, callee_sp) = call_wasm(state, caller_ip, params, func, None)?; + dispatch!(state, callee_ip, callee_sp, mem0, mem0_len, instance) +} + +pub fn call_imported( + state: &mut VmState, + ip: Ip, + _sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (caller_ip, crate::ir::decode::CallImported { params, func }) = unsafe { decode_op(ip) }; + let func = fetch_func(instance, func); + let (ip, sp, mem0, mem0_len, instance) = + call_wasm_or_host(state, caller_ip, func, params, mem0, mem0_len, instance)?; + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn call_indirect( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + caller_ip, + crate::ir::decode::CallIndirect { + params, + index, + func_type, + table, + }, + ) = unsafe { decode_op(ip) }; + let func = + resolve_indirect_func(index, table, func_type, state, sp, instance).into_control()?; + let (callee_ip, sp, mem0, mem0_len, instance) = + call_wasm_or_host(state, caller_ip, func, params, mem0, mem0_len, instance)?; + dispatch!(state, callee_ip, sp, mem0, mem0_len, instance) +} + +pub fn return_call_internal( + state: &mut VmState, + ip: Ip, + _sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (_, crate::ir::decode::ReturnCallInternal { params, func }) = unsafe { decode_op(ip) }; + let func = EngineFunc::from(func); + let (callee_ip, callee_sp) = return_call_wasm(state, params, func, None)?; + dispatch!(state, callee_ip, callee_sp, mem0, mem0_len, instance) +} + +pub fn return_call_imported( + state: &mut VmState, + ip: Ip, + _sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (_, crate::ir::decode::ReturnCallImported { params, func }) = unsafe { decode_op(ip) }; + let func = fetch_func(instance, func); + let func_entity = resolve_func(state.store, &func); + let (callee_ip, sp, new_instance) = match func_entity { + FuncEntity::Wasm(func) => { + let wasm_func = func.func_body(); + let callee_instance = *func.instance(); + let callee_instance = resolve_instance(state.store, &callee_instance).into(); + let (callee_ip, callee_sp) = + return_call_wasm(state, params, wasm_func, Some(instance))?; + (callee_ip, callee_sp, callee_instance) + } + FuncEntity::Host(host_func) => { + let host_func = *host_func; + return_call_host(state, func, host_func, params, instance)? + } + }; + let (instance, mem0, mem0_len) = + update_instance(state.store, instance, new_instance, mem0, mem0_len); + dispatch!(state, callee_ip, sp, mem0, mem0_len, instance) +} + +pub fn return_call_indirect( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + _, + crate::ir::decode::ReturnCallIndirect { + params, + index, + func_type, + table, + }, + ) = unsafe { decode_op(ip) }; + let func = + resolve_indirect_func(index, table, func_type, state, sp, instance).into_control()?; + let func_entity = resolve_func(state.store, &func); + let (callee_ip, sp, callee_instance) = match func_entity { + FuncEntity::Wasm(func) => { + let wasm_func = func.func_body(); + let callee_instance = *func.instance(); + let callee_instance: Inst = resolve_instance(state.store, &callee_instance).into(); + let (callee_ip, callee_sp) = + return_call_wasm(state, params, wasm_func, Some(instance))?; + (callee_ip, callee_sp, callee_instance) + } + FuncEntity::Host(host_func) => { + let host_func = *host_func; + return_call_host(state, func, host_func, params, instance)? + } + }; + let (instance, mem0, mem0_len) = + update_instance(state.store, instance, callee_instance, mem0, mem0_len); + dispatch!(state, callee_ip, sp, mem0, mem0_len, instance) +} + +pub fn r#return( + state: &mut VmState, + _ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + exec_return(state, sp, mem0, mem0_len, instance) +} + +pub fn return_span( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (_ip, crate::ir::decode::ReturnSpan { values }) = unsafe { decode_op(ip) }; + let dst = SlotSpan::new(Slot::from(0)); + let src = values.span(); + let len = values.len(); + exec_copy_span_asc(sp, dst, src, len); + exec_return(state, sp, mem0, mem0_len, instance) +} + +macro_rules! handler_return { + ( $( fn $handler:ident($op:ident) = $eval:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (_ip, crate::ir::decode::$op { value }) = unsafe { decode_op(ip) }; + let value = get_value(value, sp); + set_value(sp, Slot::from(0), $eval(value)); + exec_return(state, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_return! { + fn return_slot(ReturnSlot) = identity::; + fn return32(Return32) = identity::; + fn return64(Return64) = identity::; +} + +pub fn memory_size( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::MemorySize { memory, result }) = unsafe { decode_op(ip) }; + let memory = fetch_memory(instance, memory); + let size = resolve_memory(state.store, &memory).size(); + set_value(sp, result, size); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn memory_grow( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + next_ip, + crate::ir::decode::MemoryGrow { + memory, + result, + delta, + }, + ) = unsafe { decode_op(ip) }; + let delta: u64 = get_value(delta, sp); + let memref = fetch_memory(instance, memory); + let mut mem0 = mem0; + let mut mem0_len = mem0_len; + let return_value = match state.store.grow_memory(&memref, delta) { + Ok(return_value) => { + // The `memory.grow` operation might have invalidated the cached + // linear memory so we need to reset it in order for the cache to + // reload in case it is used again. + if memory.is_default() { + (mem0, mem0_len) = extract_mem0(state.store, instance); + } + return_value + } + Err(StoreError::External( + MemoryError::OutOfBoundsGrowth | MemoryError::OutOfSystemMemory, + )) => { + let memory_ty = resolve_memory(state.store, &memref).ty(); + match memory_ty.is_64() { + true => u64::MAX, + false => u64::from(u32::MAX), + } + } + Err(StoreError::External(MemoryError::OutOfFuel { required_fuel })) => { + out_of_fuel!(state, ip, required_fuel) + } + Err(StoreError::External(MemoryError::ResourceLimiterDeniedAllocation)) => { + trap!(TrapCode::GrowthOperationLimited); + } + Err(StoreError::Internal(_error)) => { + // TODO: we do not want to panic in the executor handlers so we somehow + // want to establish a way to signal to the executor that a panic + // occurred, instead. + todo!() + } + Err(error) => { + // TODO: see above + panic!("`memory.grow`: internal interpreter error: {error}") + } + }; + set_value(sp, result, return_value); + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +pub fn memory_copy( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + next_ip, + crate::ir::decode::MemoryCopy { + dst_memory, + src_memory, + dst, + src, + len, + }, + ) = unsafe { decode_op(ip) }; + let dst: u64 = get_value(dst, sp); + let src: u64 = get_value(src, sp); + let len: u64 = get_value(len, sp); + let Ok(dst_index) = usize::try_from(dst) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + let Ok(src_index) = usize::try_from(src) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + let Ok(len) = usize::try_from(len) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + if dst_memory == src_memory { + memory_copy_within(state, ip, instance, dst_memory, dst_index, src_index, len)?; + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) + } + let dst_memory = fetch_memory(instance, dst_memory); + let src_memory = fetch_memory(instance, src_memory); + let (src_memory, dst_memory, fuel) = state + .store + .inner_mut() + .resolve_memory_pair_and_fuel(&src_memory, &dst_memory); + // These accesses just perform the bounds checks required by the Wasm spec. + let src_bytes = memory_slice(src_memory, src_index, len).into_control()?; + let dst_bytes = memory_slice_mut(dst_memory, dst_index, len).into_control()?; + consume_fuel!(state, ip, fuel, |costs| costs + .fuel_for_copying_bytes(len as u64)); + dst_bytes.copy_from_slice(src_bytes); + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +fn memory_copy_within( + state: &mut VmState<'_>, + ip: Ip, + instance: Inst, + dst_memory: index::Memory, + dst_index: usize, + src_index: usize, + len: usize, +) -> Control<(), Break> { + let memory = fetch_memory(instance, dst_memory); + let (memory, fuel) = state.store.inner_mut().resolve_memory_and_fuel_mut(&memory); + // These accesses just perform the bounds checks required by the Wasm spec. + memory_slice(memory, src_index, len).into_control()?; + memory_slice(memory, dst_index, len).into_control()?; + consume_fuel!(state, ip, fuel, |costs| costs + .fuel_for_copying_bytes(len as u64)); + memory + .data_mut() + .copy_within(src_index..src_index.wrapping_add(len), dst_index); + Control::Continue(()) +} + +pub fn memory_fill( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + next_ip, + crate::ir::decode::MemoryFill { + memory, + dst, + len, + value, + }, + ) = unsafe { decode_op(ip) }; + let dst: u64 = get_value(dst, sp); + let len: u64 = get_value(len, sp); + let value: u8 = get_value(value, sp); + let Ok(dst) = usize::try_from(dst) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + let Ok(len) = usize::try_from(len) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + let memory = fetch_memory(instance, memory); + let (memory, fuel) = state.store.inner_mut().resolve_memory_and_fuel_mut(&memory); + let slice = memory_slice_mut(memory, dst, len).into_control()?; + consume_fuel!(state, ip, fuel, |costs| costs + .fuel_for_copying_bytes(len as u64)); + slice.fill(value); + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +pub fn memory_init( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + next_ip, + crate::ir::decode::MemoryInit { + memory, + data, + dst, + src, + len, + }, + ) = unsafe { decode_op(ip) }; + let dst: u64 = get_value(dst, sp); + let src: u32 = get_value(src, sp); + let len: u32 = get_value(len, sp); + let Ok(dst_index) = usize::try_from(dst) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + let Ok(src_index) = usize::try_from(src) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + let Ok(len) = usize::try_from(len) else { + trap!(TrapCode::MemoryOutOfBounds) + }; + let (memory, data, fuel) = state + .store + .inner_mut() + .resolve_memory_init_params(&fetch_memory(instance, memory), &fetch_data(instance, data)); + let memory = memory_slice_mut(memory, dst_index, len).into_control()?; + let Some(data) = data + .bytes() + .get(src_index..) + .and_then(|data| data.get(..len)) + else { + trap!(TrapCode::MemoryOutOfBounds) + }; + consume_fuel!(state, ip, fuel, |costs| costs + .fuel_for_copying_bytes(len as u64)); + memory.copy_from_slice(data); + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +pub fn data_drop( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::DataDrop { data }) = unsafe { decode_op(ip) }; + let data = fetch_data(instance, data); + resolve_data_mut(state.store, &data).drop_bytes(); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn table_size( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::TableSize { table, result }) = unsafe { decode_op(ip) }; + let table = fetch_table(instance, table); + let size = resolve_table(state.store, &table).size(); + set_value(sp, result, size); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn table_grow( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + ip, + crate::ir::decode::TableGrow { + table, + result, + delta, + value, + }, + ) = unsafe { decode_op(ip) }; + let table = fetch_table(instance, table); + let delta = get_value(delta, sp); + let value = get_value(value, sp); + let return_value = match state.store.grow_table(&table, delta, value) { + Ok(return_value) => return_value, + Err(StoreError::External(TableError::GrowOutOfBounds | TableError::OutOfSystemMemory)) => { + let table = resolve_table(state.store, &table); + match table.ty().is_64() { + true => u64::MAX, + false => u64::from(u32::MAX), + } + } + Err(StoreError::External(TableError::OutOfFuel { required_fuel })) => { + done!(state, DoneReason::out_of_fuel(required_fuel)); + } + Err(StoreError::External(TableError::ResourceLimiterDeniedAllocation)) => { + trap!(TrapCode::GrowthOperationLimited); + } + Err(StoreError::Internal(_error)) => { + // TODO: we do not want to panic in the executor handlers so we somehow + // want to establish a way to signal to the executor that a panic + // occurred, instead. + todo!() + } + Err(error) => { + panic!("`table.grow`: internal interpreter error: {error}") + } + }; + set_value(sp, result, return_value); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn table_copy( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + next_ip, + crate::ir::decode::TableCopy { + dst_table, + src_table, + dst, + src, + len, + }, + ) = unsafe { decode_op(ip) }; + let dst: u64 = get_value(dst, sp); + let src: u64 = get_value(src, sp); + let len: u64 = get_value(len, sp); + if dst_table == src_table { + // Case: copy within the same table + let table = fetch_table(instance, dst_table); + let (table, fuel) = state.store.inner_mut().resolve_table_and_fuel_mut(&table); + if let Err(error) = table.copy_within(dst, src, len, Some(fuel)) { + let trap_code = match error { + TableError::CopyOutOfBounds => TrapCode::TableOutOfBounds, + TableError::OutOfSystemMemory => TrapCode::OutOfSystemMemory, + TableError::OutOfFuel { required_fuel } => { + out_of_fuel!(state, ip, required_fuel) + } + _ => panic!("table.copy: unexpected error: {error:?}"), + }; + trap!(trap_code) + } + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) + } + // Case: copy between two different tables + let dst_table = fetch_table(instance, dst_table); + let src_table = fetch_table(instance, src_table); + let (dst_table, src_table, fuel) = state + .store + .inner_mut() + .resolve_table_pair_and_fuel(&dst_table, &src_table); + if let Err(error) = CoreTable::copy(dst_table, dst, src_table, src, len, Some(fuel)) { + let trap_code = match error { + TableError::CopyOutOfBounds => TrapCode::TableOutOfBounds, + TableError::OutOfSystemMemory => TrapCode::OutOfSystemMemory, + TableError::OutOfFuel { required_fuel } => { + out_of_fuel!(state, ip, required_fuel) + } + _ => panic!("table.copy: unexpected error: {error:?}"), + }; + trap!(trap_code) + } + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +pub fn table_fill( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + next_ip, + crate::ir::decode::TableFill { + table, + dst, + len, + value, + }, + ) = unsafe { decode_op(ip) }; + let dst: u64 = get_value(dst, sp); + let len: u64 = get_value(len, sp); + let value: UntypedVal = get_value(value, sp); + let table = fetch_table(instance, table); + let (table, fuel) = state.store.inner_mut().resolve_table_and_fuel_mut(&table); + if let Err(error) = table.fill_untyped(dst, value, len, Some(fuel)) { + let trap_code = match error { + TableError::OutOfSystemMemory => TrapCode::OutOfSystemMemory, + TableError::FillOutOfBounds => TrapCode::TableOutOfBounds, + TableError::OutOfFuel { required_fuel } => { + out_of_fuel!(state, ip, required_fuel) + } + _ => panic!("table.fill: unexpected error: {error:?}"), + }; + trap!(trap_code) + } + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +pub fn table_init( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + next_ip, + crate::ir::decode::TableInit { + table, + elem, + dst, + src, + len, + }, + ) = unsafe { decode_op(ip) }; + let dst: u64 = get_value(dst, sp); + let src: u32 = get_value(src, sp); + let len: u32 = get_value(len, sp); + let table = fetch_table(instance, table); + let elem = fetch_elem(instance, elem); + let (table, element, fuel) = state + .store + .inner_mut() + .resolve_table_init_params(&table, &elem); + if let Err(error) = table.init(element.as_ref(), dst, src, len, Some(fuel)) { + let trap_code = match error { + TableError::OutOfSystemMemory => TrapCode::OutOfSystemMemory, + TableError::InitOutOfBounds => TrapCode::TableOutOfBounds, + TableError::OutOfFuel { required_fuel } => { + out_of_fuel!(state, ip, required_fuel) + } + _ => panic!("table.init: unexpected error: {error:?}"), + }; + trap!(trap_code) + } + dispatch!(state, next_ip, sp, mem0, mem0_len, instance) +} + +pub fn elem_drop( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::ElemDrop { elem }) = unsafe { decode_op(ip) }; + let elem = fetch_elem(instance, elem); + resolve_elem_mut(state.store, &elem).drop_items(); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +macro_rules! impl_table_get { + ( $( fn $handler:ident($op:ident) = $ext:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (ip, crate::ir::decode::$op { table, result, index }) = unsafe { decode_op(ip) }; + let table = fetch_table(instance, table); + let table = resolve_table(state.store, &table); + let index = $ext(get_value(index, sp)); + let value = match table.get_untyped(index) { + Some(value) => value, + None => trap!(TrapCode::TableOutOfBounds) + }; + set_value(sp, result, value); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +impl_table_get! { + fn table_get_ss(TableGet_Ss) = identity; + fn table_get_si(TableGet_Si) = u64::from; +} + +macro_rules! impl_table_set { + ( $( fn $handler:ident($op:ident) = $ext:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (ip, crate::ir::decode::$op { table, index, value }) = unsafe { decode_op(ip) }; + let table = fetch_table(instance, table); + let table = resolve_table_mut(state.store, &table); + let index = $ext(get_value(index, sp)); + let value: u64 = get_value(value, sp); + if let Err(TableError::SetOutOfBounds) = table.set_untyped(index, UntypedVal::from(value)) { + trap!(TrapCode::TableOutOfBounds) + }; + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +impl_table_set! { + fn table_set_ss(TableSet_Ss) = identity; + fn table_set_si(TableSet_Si) = identity; + fn table_set_is(TableSet_Is) = u64::from; + fn table_set_ii(TableSet_Ii) = u64::from; +} + +pub fn ref_func( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::RefFunc { func, result }) = unsafe { decode_op(ip) }; + let func = fetch_func(instance, func); + let funcref = >::from(func); + set_value(sp, result, funcref); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +macro_rules! impl_i64_binop128 { + ( + $( fn $handler:ident($op:ident) = $eval:expr );* $(;)? + ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (ip, crate::ir::decode::$op { results, lhs_lo, lhs_hi, rhs_lo, rhs_hi }) = unsafe { decode_op(ip) }; + let lhs_lo: i64 = get_value(lhs_lo, sp); + let lhs_hi: i64 = get_value(lhs_hi, sp); + let rhs_lo: i64 = get_value(rhs_lo, sp); + let rhs_hi: i64 = get_value(rhs_hi, sp); + let results = results.to_array(); + let (result_lo, result_hi) = $eval(lhs_lo, lhs_hi, rhs_lo, rhs_hi); + set_value(sp, results[0], result_lo); + set_value(sp, results[1], result_hi); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +impl_i64_binop128! { + fn i64_add128(I64Add128) = wasm::i64_add128; + fn i64_sub128(I64Sub128) = wasm::i64_sub128; +} + +macro_rules! impl_i64_mul_wide { + ( + $( fn $handler:ident($op:ident) = $eval:expr );* $(;)? + ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (ip, crate::ir::decode::$op { results, lhs, rhs }) = unsafe { decode_op(ip) }; + let lhs: i64 = get_value(lhs, sp); + let rhs: i64 = get_value(rhs, sp); + let (result_lo, result_hi) = $eval(lhs, rhs); + let results = results.to_array(); + set_value(sp, results[0], result_lo); + set_value(sp, results[1], result_hi); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +impl_i64_mul_wide! { + fn i64_mul_wide(I64MulWide) = wasm::i64_mul_wide_s; + fn u64_mul_wide(U64MulWide) = wasm::i64_mul_wide_u; +} + +/// Fetches the branch table index value and normalizes it to clamp between `0..len_targets`. +fn fetch_branch_table_target(sp: Sp, index: Slot, len_targets: u32) -> usize { + let index: u32 = get_value(index, sp); + let max_index = len_targets - 1; + cmp::min(index, max_index) as usize +} + +pub fn branch_table( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let (ip, crate::ir::decode::BranchTable { len_targets, index }) = unsafe { decode_op(ip) }; + let chosen_target = fetch_branch_table_target(sp, index, len_targets); + let target_offset = 4 * chosen_target; + let ip = unsafe { ip.add(target_offset) }; + let (_, offset) = unsafe { ip.decode::() }; + let ip = offset_ip(ip, offset); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn branch_table_span( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let ( + ip, + crate::ir::decode::BranchTableSpan { + len_targets, + index, + values, + len_values, + }, + ) = unsafe { decode_op(ip) }; + let chosen_target = fetch_branch_table_target(sp, index, len_targets); + let target_offset = 6 * chosen_target; + let ip = unsafe { ip.add(target_offset) }; + let (_, ir::BranchTableTarget { results, offset }) = + unsafe { ip.decode::() }; + // TODO: maybe provide 2 `br_table_span` operation variants if possible: `br_table_span_{asc,des}` + exec_copy_span(sp, results, values, len_values); + let ip = offset_ip(ip, offset); + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +macro_rules! handler_unary { + ( $( fn $handler:ident($op:ident) = $eval:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (ip, $crate::ir::decode::$op { result, value }) = unsafe { decode_op(ip) }; + let value = get_value(value, sp); + let value = $eval(value).into_control()?; + set_value(sp, result, value); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_unary! { + // copy + fn copy(Copy) = identity::; + fn copy32(Copy32) = identity::; + fn copy64(Copy64) = identity::; + // i32 + fn i32_popcnt_ss(I32Popcnt_Ss) = wasm::i32_popcnt; + fn i32_ctz_ss(I32Ctz_Ss) = wasm::i32_ctz; + fn i32_clz_ss(I32Clz_Ss) = wasm::i32_clz; + fn i32_sext8_ss(I32Sext8_Ss) = wasm::i32_extend8_s; + fn i32_sext16_ss(I32Sext16_Ss) = wasm::i32_extend16_s; + fn i32_wrap_i64_ss(I32WrapI64_Ss) = wasm::i32_wrap_i64; + // i64 + fn i64_popcnt_ss(I64Popcnt_Ss) = wasm::i64_popcnt; + fn i64_ctz_ss(I64Ctz_Ss) = wasm::i64_ctz; + fn i64_clz_ss(I64Clz_Ss) = wasm::i64_clz; + fn i64_sext8_ss(I64Sext8_Ss) = wasm::i64_extend8_s; + fn i64_sext16_ss(I64Sext16_Ss) = wasm::i64_extend16_s; + fn i64_sext32_ss(I64Sext32_Ss) = wasm::i64_extend32_s; + // f32 + fn f32_abs_ss(F32Abs_Ss) = wasm::f32_abs; + fn f32_neg_ss(F32Neg_Ss) = wasm::f32_neg; + fn f32_ceil_ss(F32Ceil_Ss) = wasm::f32_ceil; + fn f32_floor_ss(F32Floor_Ss) = wasm::f32_floor; + fn f32_trunc_ss(F32Trunc_Ss) = wasm::f32_trunc; + fn f32_nearest_ss(F32Nearest_Ss) = wasm::f32_nearest; + fn f32_sqrt_ss(F32Sqrt_Ss) = wasm::f32_sqrt; + fn f32_convert_i32_ss(F32ConvertI32_Ss) = wasm::f32_convert_i32_s; + fn f32_convert_u32_ss(F32ConvertU32_Ss) = wasm::f32_convert_i32_u; + fn f32_convert_i64_ss(F32ConvertI64_Ss) = wasm::f32_convert_i64_s; + fn f32_convert_u64_ss(F32ConvertU64_Ss) = wasm::f32_convert_i64_u; + fn f32_demote_f64_ss(F32DemoteF64_Ss) = wasm::f32_demote_f64; + // f64 + fn f64_abs_ss(F64Abs_Ss) = wasm::f64_abs; + fn f64_neg_ss(F64Neg_Ss) = wasm::f64_neg; + fn f64_ceil_ss(F64Ceil_Ss) = wasm::f64_ceil; + fn f64_floor_ss(F64Floor_Ss) = wasm::f64_floor; + fn f64_trunc_ss(F64Trunc_Ss) = wasm::f64_trunc; + fn f64_nearest_ss(F64Nearest_Ss) = wasm::f64_nearest; + fn f64_sqrt_ss(F64Sqrt_Ss) = wasm::f64_sqrt; + fn f64_convert_i32_ss(F64ConvertI32_Ss) = wasm::f64_convert_i32_s; + fn f64_convert_u32_ss(F64ConvertU32_Ss) = wasm::f64_convert_i32_u; + fn f64_convert_i64_ss(F64ConvertI64_Ss) = wasm::f64_convert_i64_s; + fn f64_convert_u64_ss(F64ConvertU64_Ss) = wasm::f64_convert_i64_u; + fn f64_promote_f32_ss(F64PromoteF32_Ss) = wasm::f64_promote_f32; + // f2i conversions + fn i32_trunc_f32_ss(I32TruncF32_Ss) = wasm::i32_trunc_f32_s; + fn u32_trunc_f32_ss(U32TruncF32_Ss) = wasm::i32_trunc_f32_u; + fn i32_trunc_f64_ss(I32TruncF64_Ss) = wasm::i32_trunc_f64_s; + fn u32_trunc_f64_ss(U32TruncF64_Ss) = wasm::i32_trunc_f64_u; + fn i64_trunc_f32_ss(I64TruncF32_Ss) = wasm::i64_trunc_f32_s; + fn u64_trunc_f32_ss(U64TruncF32_Ss) = wasm::i64_trunc_f32_u; + fn i64_trunc_f64_ss(I64TruncF64_Ss) = wasm::i64_trunc_f64_s; + fn u64_trunc_f64_ss(U64TruncF64_Ss) = wasm::i64_trunc_f64_u; + fn i32_trunc_sat_f32_ss(I32TruncSatF32_Ss) = wasm::i32_trunc_sat_f32_s; + fn u32_trunc_sat_f32_ss(U32TruncSatF32_Ss) = wasm::i32_trunc_sat_f32_u; + fn i32_trunc_sat_f64_ss(I32TruncSatF64_Ss) = wasm::i32_trunc_sat_f64_s; + fn u32_trunc_sat_f64_ss(U32TruncSatF64_Ss) = wasm::i32_trunc_sat_f64_u; + fn i64_trunc_sat_f32_ss(I64TruncSatF32_Ss) = wasm::i64_trunc_sat_f32_s; + fn u64_trunc_sat_f32_ss(U64TruncSatF32_Ss) = wasm::i64_trunc_sat_f32_u; + fn i64_trunc_sat_f64_ss(I64TruncSatF64_Ss) = wasm::i64_trunc_sat_f64_s; + fn u64_trunc_sat_f64_ss(U64TruncSatF64_Ss) = wasm::i64_trunc_sat_f64_u; +} + +macro_rules! handler_binary { + ( $( fn $handler:ident($decode:ident) = $eval:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (ip, $crate::ir::decode::$decode { result, lhs, rhs }) = unsafe { decode_op(ip) }; + let lhs = get_value(lhs, sp); + let rhs = get_value(rhs, sp); + let value = $eval(lhs, rhs).into_control()?; + set_value(sp, result, value); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_binary! { + // i32 + // i32: commutative + fn i32_eq_sss(I32Eq_Sss) = wasm::i32_eq; + fn i32_eq_ssi(I32Eq_Ssi) = wasm::i32_eq; + fn i32_and_sss(I32And_Sss) = eval::wasmi_i32_and; + fn i32_and_ssi(I32And_Ssi) = eval::wasmi_i32_and; + fn i32_or_sss(I32Or_Sss) = eval::wasmi_i32_or; + fn i32_or_ssi(I32Or_Ssi) = eval::wasmi_i32_or; + fn i32_not_eq_sss(I32NotEq_Sss) = wasm::i32_ne; + fn i32_not_eq_ssi(I32NotEq_Ssi) = wasm::i32_ne; + fn i32_not_and_sss(I32NotAnd_Sss) = eval::wasmi_i32_not_and; + fn i32_not_and_ssi(I32NotAnd_Ssi) = eval::wasmi_i32_not_and; + fn i32_not_or_sss(I32NotOr_Sss) = eval::wasmi_i32_not_or; + fn i32_not_or_ssi(I32NotOr_Ssi) = eval::wasmi_i32_not_or; + fn i32_add_sss(I32Add_Sss) = wasm::i32_add; + fn i32_add_ssi(I32Add_Ssi) = wasm::i32_add; + fn i32_mul_sss(I32Mul_Sss) = wasm::i32_mul; + fn i32_mul_ssi(I32Mul_Ssi) = wasm::i32_mul; + fn i32_bitand_sss(I32BitAnd_Sss) = wasm::i32_bitand; + fn i32_bitand_ssi(I32BitAnd_Ssi) = wasm::i32_bitand; + fn i32_bitor_sss(I32BitOr_Sss) = wasm::i32_bitor; + fn i32_bitor_ssi(I32BitOr_Ssi) = wasm::i32_bitor; + fn i32_bitxor_sss(I32BitXor_Sss) = wasm::i32_bitxor; + fn i32_bitxor_ssi(I32BitXor_Ssi) = wasm::i32_bitxor; + // i32: non-commutative + fn i32_sub_sss(I32Sub_Sss) = wasm::i32_sub; + fn i32_sub_ssi(I32Sub_Ssi) = wasm::i32_sub; + fn i32_sub_sis(I32Sub_Sis) = wasm::i32_sub; + fn i32_div_sss(I32Div_Sss) = wasm::i32_div_s; + fn i32_div_ssi(I32Div_Ssi) = eval::wasmi_i32_div_ssi; + fn i32_div_sis(I32Div_Sis) = wasm::i32_div_s; + fn u32_div_sss(U32Div_Sss) = wasm::i32_div_u; + fn u32_div_ssi(U32Div_Ssi) = eval::wasmi_u32_div_ssi; + fn u32_div_sis(U32Div_Sis) = wasm::i32_div_u; + fn i32_rem_sss(I32Rem_Sss) = wasm::i32_rem_s; + fn i32_rem_ssi(I32Rem_Ssi) = eval::wasmi_i32_rem_ssi; + fn i32_rem_sis(I32Rem_Sis) = wasm::i32_rem_s; + fn u32_rem_sss(U32Rem_Sss) = wasm::i32_rem_u; + fn u32_rem_ssi(U32Rem_Ssi) = eval::wasmi_u32_rem_ssi; + fn u32_rem_sis(U32Rem_Sis) = wasm::i32_rem_u; + // i32: comparisons + fn i32_le_sss(I32Le_Sss) = wasm::i32_le_s; + fn i32_le_ssi(I32Le_Ssi) = wasm::i32_le_s; + fn i32_le_sis(I32Le_Sis) = wasm::i32_le_s; + fn i32_lt_sss(I32Lt_Sss) = wasm::i32_lt_s; + fn i32_lt_ssi(I32Lt_Ssi) = wasm::i32_lt_s; + fn i32_lt_sis(I32Lt_Sis) = wasm::i32_lt_s; + fn u32_le_sss(U32Le_Sss) = wasm::i32_le_u; + fn u32_le_ssi(U32Le_Ssi) = wasm::i32_le_u; + fn u32_le_sis(U32Le_Sis) = wasm::i32_le_u; + fn u32_lt_sss(U32Lt_Sss) = wasm::i32_lt_u; + fn u32_lt_ssi(U32Lt_Ssi) = wasm::i32_lt_u; + fn u32_lt_sis(U32Lt_Sis) = wasm::i32_lt_u; + // i32: shift + rotate + fn i32_shl_sss(I32Shl_Sss) = wasm::i32_shl; + fn i32_shl_ssi(I32Shl_Ssi) = eval::wasmi_i32_shl_ssi; + fn i32_shl_sis(I32Shl_Sis) = wasm::i32_shl; + fn i32_shr_sss(I32Shr_Sss) = wasm::i32_shr_s; + fn i32_shr_ssi(I32Shr_Ssi) = eval::wasmi_i32_shr_ssi; + fn i32_shr_sis(I32Shr_Sis) = wasm::i32_shr_s; + fn u32_shr_sss(U32Shr_Sss) = wasm::i32_shr_u; + fn u32_shr_ssi(U32Shr_Ssi) = eval::wasmi_u32_shr_ssi; + fn u32_shr_sis(U32Shr_Sis) = wasm::i32_shr_u; + fn i32_rotl_sss(I32Rotl_Sss) = wasm::i32_rotl; + fn i32_rotl_ssi(I32Rotl_Ssi) = eval::wasmi_i32_rotl_ssi; + fn i32_rotl_sis(I32Rotl_Sis) = wasm::i32_rotl; + fn i32_rotr_sss(I32Rotr_Sss) = wasm::i32_rotr; + fn i32_rotr_ssi(I32Rotr_Ssi) = eval::wasmi_i32_rotr_ssi; + fn i32_rotr_sis(I32Rotr_Sis) = wasm::i32_rotr; + // i64 + // i64: commutative + fn i64_eq_sss(I64Eq_Sss) = wasm::i64_eq; + fn i64_eq_ssi(I64Eq_Ssi) = wasm::i64_eq; + fn i64_and_sss(I64And_Sss) = eval::wasmi_i64_and; + fn i64_and_ssi(I64And_Ssi) = eval::wasmi_i64_and; + fn i64_or_sss(I64Or_Sss) = eval::wasmi_i64_or; + fn i64_or_ssi(I64Or_Ssi) = eval::wasmi_i64_or; + fn i64_not_and_sss(I64NotAnd_Sss) = eval::wasmi_i64_not_and; + fn i64_not_and_ssi(I64NotAnd_Ssi) = eval::wasmi_i64_not_and; + fn i64_not_or_sss(I64NotOr_Sss) = eval::wasmi_i64_not_or; + fn i64_not_or_ssi(I64NotOr_Ssi) = eval::wasmi_i64_not_or; + fn i64_not_eq_sss(I64NotEq_Sss) = wasm::i64_ne; + fn i64_not_eq_ssi(I64NotEq_Ssi) = wasm::i64_ne; + fn i64_add_sss(I64Add_Sss) = wasm::i64_add; + fn i64_add_ssi(I64Add_Ssi) = wasm::i64_add; + fn i64_mul_sss(I64Mul_Sss) = wasm::i64_mul; + fn i64_mul_ssi(I64Mul_Ssi) = wasm::i64_mul; + fn i64_bitand_sss(I64BitAnd_Sss) = wasm::i64_bitand; + fn i64_bitand_ssi(I64BitAnd_Ssi) = wasm::i64_bitand; + fn i64_bitor_sss(I64BitOr_Sss) = wasm::i64_bitor; + fn i64_bitor_ssi(I64BitOr_Ssi) = wasm::i64_bitor; + fn i64_bitxor_sss(I64BitXor_Sss) = wasm::i64_bitxor; + fn i64_bitxor_ssi(I64BitXor_Ssi) = wasm::i64_bitxor; + // i64: non-commutative + fn i64_sub_sss(I64Sub_Sss) = wasm::i64_sub; + fn i64_sub_ssi(I64Sub_Ssi) = wasm::i64_sub; + fn i64_sub_sis(I64Sub_Sis) = wasm::i64_sub; + fn i64_div_sss(I64Div_Sss) = wasm::i64_div_s; + fn i64_div_ssi(I64Div_Ssi) = eval::wasmi_i64_div_ssi; + fn i64_div_sis(I64Div_Sis) = wasm::i64_div_s; + fn u64_div_sss(U64Div_Sss) = wasm::i64_div_u; + fn u64_div_ssi(U64Div_Ssi) = eval::wasmi_u64_div_ssi; + fn u64_div_sis(U64Div_Sis) = wasm::i64_div_u; + fn i64_rem_sss(I64Rem_Sss) = wasm::i64_rem_s; + fn i64_rem_ssi(I64Rem_Ssi) = eval::wasmi_i64_rem_ssi; + fn i64_rem_sis(I64Rem_Sis) = wasm::i64_rem_s; + fn u64_rem_sss(U64Rem_Sss) = wasm::i64_rem_u; + fn u64_rem_ssi(U64Rem_Ssi) = eval::wasmi_u64_rem_ssi; + fn u64_rem_sis(U64Rem_Sis) = wasm::i64_rem_u; + // i64: comparisons + fn i64_le_sss(I64Le_Sss) = wasm::i64_le_s; + fn i64_le_ssi(I64Le_Ssi) = wasm::i64_le_s; + fn i64_le_sis(I64Le_Sis) = wasm::i64_le_s; + fn i64_lt_sss(I64Lt_Sss) = wasm::i64_lt_s; + fn i64_lt_ssi(I64Lt_Ssi) = wasm::i64_lt_s; + fn i64_lt_sis(I64Lt_Sis) = wasm::i64_lt_s; + fn u64_le_sss(U64Le_Sss) = wasm::i64_le_u; + fn u64_le_ssi(U64Le_Ssi) = wasm::i64_le_u; + fn u64_le_sis(U64Le_Sis) = wasm::i64_le_u; + fn u64_lt_sss(U64Lt_Sss) = wasm::i64_lt_u; + fn u64_lt_ssi(U64Lt_Ssi) = wasm::i64_lt_u; + fn u64_lt_sis(U64Lt_Sis) = wasm::i64_lt_u; + // i64: shift + rotate + fn i64_shl_sss(I64Shl_Sss) = wasm::i64_shl; + fn i64_shl_ssi(I64Shl_Ssi) = eval::wasmi_i64_shl_ssi; + fn i64_shl_sis(I64Shl_Sis) = wasm::i64_shl; + fn i64_shr_sss(I64Shr_Sss) = wasm::i64_shr_s; + fn i64_shr_ssi(I64Shr_Ssi) = eval::wasmi_i64_shr_ssi; + fn i64_shr_sis(I64Shr_Sis) = wasm::i64_shr_s; + fn u64_shr_sss(U64Shr_Sss) = wasm::i64_shr_u; + fn u64_shr_ssi(U64Shr_Ssi) = eval::wasmi_u64_shr_ssi; + fn u64_shr_sis(U64Shr_Sis) = wasm::i64_shr_u; + fn i64_rotl_sss(I64Rotl_Sss) = wasm::i64_rotl; + fn i64_rotl_ssi(I64Rotl_Ssi) = eval::wasmi_i64_rotl_ssi; + fn i64_rotl_sis(I64Rotl_Sis) = wasm::i64_rotl; + fn i64_rotr_sss(I64Rotr_Sss) = wasm::i64_rotr; + fn i64_rotr_ssi(I64Rotr_Ssi) = eval::wasmi_i64_rotr_ssi; + fn i64_rotr_sis(I64Rotr_Sis) = wasm::i64_rotr; + // f32 + // f32: binary + fn f32_add_sss(F32Add_Sss) = wasm::f32_add; + fn f32_add_ssi(F32Add_Ssi) = wasm::f32_add; + fn f32_add_sis(F32Add_Sis) = wasm::f32_add; + fn f32_sub_sss(F32Sub_Sss) = wasm::f32_sub; + fn f32_sub_ssi(F32Sub_Ssi) = wasm::f32_sub; + fn f32_sub_sis(F32Sub_Sis) = wasm::f32_sub; + fn f32_mul_sss(F32Mul_Sss) = wasm::f32_mul; + fn f32_mul_ssi(F32Mul_Ssi) = wasm::f32_mul; + fn f32_mul_sis(F32Mul_Sis) = wasm::f32_mul; + fn f32_div_sss(F32Div_Sss) = wasm::f32_div; + fn f32_div_ssi(F32Div_Ssi) = wasm::f32_div; + fn f32_div_sis(F32Div_Sis) = wasm::f32_div; + fn f32_min_sss(F32Min_Sss) = wasm::f32_min; + fn f32_min_ssi(F32Min_Ssi) = wasm::f32_min; + fn f32_min_sis(F32Min_Sis) = wasm::f32_min; + fn f32_max_sss(F32Max_Sss) = wasm::f32_max; + fn f32_max_ssi(F32Max_Ssi) = wasm::f32_max; + fn f32_max_sis(F32Max_Sis) = wasm::f32_max; + fn f32_copysign_sss(F32Copysign_Sss) = wasm::f32_copysign; + fn f32_copysign_ssi(F32Copysign_Ssi) = eval::wasmi_f32_copysign_ssi; + fn f32_copysign_sis(F32Copysign_Sis) = wasm::f32_copysign; + // f32: comparisons + fn f32_eq_sss(F32Eq_Sss) = wasm::f32_eq; + fn f32_eq_ssi(F32Eq_Ssi) = wasm::f32_eq; + fn f32_lt_sss(F32Lt_Sss) = wasm::f32_lt; + fn f32_lt_ssi(F32Lt_Ssi) = wasm::f32_lt; + fn f32_lt_sis(F32Lt_Sis) = wasm::f32_lt; + fn f32_le_sss(F32Le_Sss) = wasm::f32_le; + fn f32_le_ssi(F32Le_Ssi) = wasm::f32_le; + fn f32_le_sis(F32Le_Sis) = wasm::f32_le; + fn f32_not_eq_sss(F32NotEq_Sss) = wasm::f32_ne; + fn f32_not_eq_ssi(F32NotEq_Ssi) = wasm::f32_ne; + fn f32_not_lt_sss(F32NotLt_Sss) = eval::wasmi_f32_not_lt; + fn f32_not_lt_ssi(F32NotLt_Ssi) = eval::wasmi_f32_not_lt; + fn f32_not_lt_sis(F32NotLt_Sis) = eval::wasmi_f32_not_lt; + fn f32_not_le_sss(F32NotLe_Sss) = eval::wasmi_f32_not_le; + fn f32_not_le_ssi(F32NotLe_Ssi) = eval::wasmi_f32_not_le; + fn f32_not_le_sis(F32NotLe_Sis) = eval::wasmi_f32_not_le; + // f64 + // f64: binary + fn f64_add_sss(F64Add_Sss) = wasm::f64_add; + fn f64_add_ssi(F64Add_Ssi) = wasm::f64_add; + fn f64_add_sis(F64Add_Sis) = wasm::f64_add; + fn f64_sub_sss(F64Sub_Sss) = wasm::f64_sub; + fn f64_sub_ssi(F64Sub_Ssi) = wasm::f64_sub; + fn f64_sub_sis(F64Sub_Sis) = wasm::f64_sub; + fn f64_mul_sss(F64Mul_Sss) = wasm::f64_mul; + fn f64_mul_ssi(F64Mul_Ssi) = wasm::f64_mul; + fn f64_mul_sis(F64Mul_Sis) = wasm::f64_mul; + fn f64_div_sss(F64Div_Sss) = wasm::f64_div; + fn f64_div_ssi(F64Div_Ssi) = wasm::f64_div; + fn f64_div_sis(F64Div_Sis) = wasm::f64_div; + fn f64_min_sss(F64Min_Sss) = wasm::f64_min; + fn f64_min_ssi(F64Min_Ssi) = wasm::f64_min; + fn f64_min_sis(F64Min_Sis) = wasm::f64_min; + fn f64_max_sss(F64Max_Sss) = wasm::f64_max; + fn f64_max_ssi(F64Max_Ssi) = wasm::f64_max; + fn f64_max_sis(F64Max_Sis) = wasm::f64_max; + fn f64_copysign_sss(F64Copysign_Sss) = wasm::f64_copysign; + fn f64_copysign_ssi(F64Copysign_Ssi) = eval::wasmi_f64_copysign_ssi; + fn f64_copysign_sis(F64Copysign_Sis) = wasm::f64_copysign; + // f64: comparisons + fn f64_eq_sss(F64Eq_Sss) = wasm::f64_eq; + fn f64_eq_ssi(F64Eq_Ssi) = wasm::f64_eq; + fn f64_lt_sss(F64Lt_Sss) = wasm::f64_lt; + fn f64_lt_ssi(F64Lt_Ssi) = wasm::f64_lt; + fn f64_lt_sis(F64Lt_Sis) = wasm::f64_lt; + fn f64_le_sss(F64Le_Sss) = wasm::f64_le; + fn f64_le_ssi(F64Le_Ssi) = wasm::f64_le; + fn f64_le_sis(F64Le_Sis) = wasm::f64_le; + fn f64_not_eq_sss(F64NotEq_Sss) = wasm::f64_ne; + fn f64_not_eq_ssi(F64NotEq_Ssi) = wasm::f64_ne; + fn f64_not_lt_sss(F64NotLt_Sss) = eval::wasmi_f64_not_lt; + fn f64_not_lt_ssi(F64NotLt_Ssi) = eval::wasmi_f64_not_lt; + fn f64_not_lt_sis(F64NotLt_Sis) = eval::wasmi_f64_not_lt; + fn f64_not_le_sss(F64NotLe_Sss) = eval::wasmi_f64_not_le; + fn f64_not_le_ssi(F64NotLe_Ssi) = eval::wasmi_f64_not_le; + fn f64_not_le_sis(F64NotLe_Sis) = eval::wasmi_f64_not_le; +} + +macro_rules! handler_cmp_branch { + ( $( fn $handler:ident($decode:ident) = $eval:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let (next_ip, $crate::ir::decode::$decode { offset, lhs, rhs }) = unsafe { decode_op(ip) }; + let lhs = get_value(lhs, sp); + let rhs = get_value(rhs, sp); + let ip = match $eval(lhs, rhs) { + true => offset_ip(ip, offset), + false => next_ip, + }; + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_cmp_branch! { + // i32 + fn branch_i32_eq_ss(BranchI32Eq_Ss) = wasm::i32_eq; + fn branch_i32_eq_si(BranchI32Eq_Si) = wasm::i32_eq; + fn branch_i32_and_ss(BranchI32And_Ss) = eval::wasmi_i32_and; + fn branch_i32_and_si(BranchI32And_Si) = eval::wasmi_i32_and; + fn branch_i32_or_ss(BranchI32Or_Ss) = eval::wasmi_i32_or; + fn branch_i32_or_si(BranchI32Or_Si) = eval::wasmi_i32_or; + fn branch_i32_not_eq_ss(BranchI32NotEq_Ss) = wasm::i32_ne; + fn branch_i32_not_eq_si(BranchI32NotEq_Si) = wasm::i32_ne; + fn branch_i32_not_and_ss(BranchI32NotAnd_Ss) = eval::wasmi_i32_not_and; + fn branch_i32_not_and_si(BranchI32NotAnd_Si) = eval::wasmi_i32_not_and; + fn branch_i32_not_or_ss(BranchI32NotOr_Ss) = eval::wasmi_i32_not_or; + fn branch_i32_not_or_si(BranchI32NotOr_Si) = eval::wasmi_i32_not_or; + fn branch_i32_le_ss(BranchI32Le_Ss) = wasm::i32_le_s; + fn branch_i32_le_si(BranchI32Le_Si) = wasm::i32_le_s; + fn branch_i32_le_is(BranchI32Le_Is) = wasm::i32_le_s; + fn branch_i32_lt_ss(BranchI32Lt_Ss) = wasm::i32_lt_s; + fn branch_i32_lt_si(BranchI32Lt_Si) = wasm::i32_lt_s; + fn branch_i32_lt_is(BranchI32Lt_Is) = wasm::i32_lt_s; + fn branch_u32_le_ss(BranchU32Le_Ss) = wasm::i32_le_u; + fn branch_u32_le_si(BranchU32Le_Si) = wasm::i32_le_u; + fn branch_u32_le_is(BranchU32Le_Is) = wasm::i32_le_u; + fn branch_u32_lt_ss(BranchU32Lt_Ss) = wasm::i32_lt_u; + fn branch_u32_lt_si(BranchU32Lt_Si) = wasm::i32_lt_u; + fn branch_u32_lt_is(BranchU32Lt_Is) = wasm::i32_lt_u; + // i64 + fn branch_i64_eq_ss(BranchI64Eq_Ss) = wasm::i64_eq; + fn branch_i64_eq_si(BranchI64Eq_Si) = wasm::i64_eq; + fn branch_i64_and_ss(BranchI64And_Ss) = eval::wasmi_i64_and; + fn branch_i64_and_si(BranchI64And_Si) = eval::wasmi_i64_and; + fn branch_i64_or_ss(BranchI64Or_Ss) = eval::wasmi_i64_or; + fn branch_i64_or_si(BranchI64Or_Si) = eval::wasmi_i64_or; + fn branch_i64_not_eq_ss(BranchI64NotEq_Ss) = wasm::i64_ne; + fn branch_i64_not_eq_si(BranchI64NotEq_Si) = wasm::i64_ne; + fn branch_i64_not_and_ss(BranchI64NotAnd_Ss) = eval::wasmi_i64_not_and; + fn branch_i64_not_and_si(BranchI64NotAnd_Si) = eval::wasmi_i64_not_and; + fn branch_i64_not_or_ss(BranchI64NotOr_Ss) = eval::wasmi_i64_not_or; + fn branch_i64_not_or_si(BranchI64NotOr_Si) = eval::wasmi_i64_not_or; + fn branch_i64_le_ss(BranchI64Le_Ss) = wasm::i64_le_s; + fn branch_i64_le_si(BranchI64Le_Si) = wasm::i64_le_s; + fn branch_i64_le_is(BranchI64Le_Is) = wasm::i64_le_s; + fn branch_i64_lt_ss(BranchI64Lt_Ss) = wasm::i64_lt_s; + fn branch_i64_lt_si(BranchI64Lt_Si) = wasm::i64_lt_s; + fn branch_i64_lt_is(BranchI64Lt_Is) = wasm::i64_lt_s; + fn branch_u64_le_ss(BranchU64Le_Ss) = wasm::i64_le_u; + fn branch_u64_le_si(BranchU64Le_Si) = wasm::i64_le_u; + fn branch_u64_le_is(BranchU64Le_Is) = wasm::i64_le_u; + fn branch_u64_lt_ss(BranchU64Lt_Ss) = wasm::i64_lt_u; + fn branch_u64_lt_si(BranchU64Lt_Si) = wasm::i64_lt_u; + fn branch_u64_lt_is(BranchU64Lt_Is) = wasm::i64_lt_u; + // f32 + fn branch_f32_eq_ss(BranchF32Eq_Ss) = wasm::f32_eq; + fn branch_f32_eq_si(BranchF32Eq_Si) = wasm::f32_eq; + fn branch_f32_le_ss(BranchF32Le_Ss) = wasm::f32_le; + fn branch_f32_le_si(BranchF32Le_Si) = wasm::f32_le; + fn branch_f32_le_is(BranchF32Le_Is) = wasm::f32_le; + fn branch_f32_lt_ss(BranchF32Lt_Ss) = wasm::f32_lt; + fn branch_f32_lt_si(BranchF32Lt_Si) = wasm::f32_lt; + fn branch_f32_lt_is(BranchF32Lt_Is) = wasm::f32_lt; + fn branch_f32_not_eq_ss(BranchF32NotEq_Ss) = wasm::f32_ne; + fn branch_f32_not_eq_si(BranchF32NotEq_Si) = wasm::f32_ne; + fn branch_f32_not_le_ss(BranchF32NotLe_Ss) = eval::wasmi_f32_not_le; + fn branch_f32_not_le_si(BranchF32NotLe_Si) = eval::wasmi_f32_not_le; + fn branch_f32_not_le_is(BranchF32NotLe_Is) = eval::wasmi_f32_not_le; + fn branch_f32_not_lt_ss(BranchF32NotLt_Ss) = eval::wasmi_f32_not_lt; + fn branch_f32_not_lt_si(BranchF32NotLt_Si) = eval::wasmi_f32_not_lt; + fn branch_f32_not_lt_is(BranchF32NotLt_Is) = eval::wasmi_f32_not_lt; + // f64 + fn branch_f64_eq_ss(BranchF64Eq_Ss) = wasm::f64_eq; + fn branch_f64_eq_si(BranchF64Eq_Si) = wasm::f64_eq; + fn branch_f64_le_ss(BranchF64Le_Ss) = wasm::f64_le; + fn branch_f64_le_si(BranchF64Le_Si) = wasm::f64_le; + fn branch_f64_le_is(BranchF64Le_Is) = wasm::f64_le; + fn branch_f64_lt_ss(BranchF64Lt_Ss) = wasm::f64_lt; + fn branch_f64_lt_si(BranchF64Lt_Si) = wasm::f64_lt; + fn branch_f64_lt_is(BranchF64Lt_Is) = wasm::f64_lt; + fn branch_f64_not_eq_ss(BranchF64NotEq_Ss) = wasm::f64_ne; + fn branch_f64_not_eq_si(BranchF64NotEq_Si) = wasm::f64_ne; + fn branch_f64_not_le_ss(BranchF64NotLe_Ss) = eval::wasmi_f64_not_le; + fn branch_f64_not_le_si(BranchF64NotLe_Si) = eval::wasmi_f64_not_le; + fn branch_f64_not_le_is(BranchF64NotLe_Is) = eval::wasmi_f64_not_le; + fn branch_f64_not_lt_ss(BranchF64NotLt_Ss) = eval::wasmi_f64_not_lt; + fn branch_f64_not_lt_si(BranchF64NotLt_Si) = eval::wasmi_f64_not_lt; + fn branch_f64_not_lt_is(BranchF64NotLt_Is) = eval::wasmi_f64_not_lt; +} + +macro_rules! handler_select { + ( $( fn $handler:ident($decode:ident) = $eval:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let ( + ip, + $crate::ir::decode::$decode { + result, + val_true, + val_false, + lhs, + rhs, + }, + ) = unsafe { decode_op(ip) }; + let lhs = get_value(lhs, sp); + let rhs = get_value(rhs, sp); + let src = match $eval(lhs, rhs) { + true => val_true, + false => val_false, + }; + let src: UntypedVal = get_value(src, sp); + set_value(sp, result, src); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_select! { + // i32 + fn select_i32_eq_sss(SelectI32Eq_Sss) = wasm::i32_eq; + fn select_i32_eq_ssi(SelectI32Eq_Ssi) = wasm::i32_eq; + fn select_i32_and_sss(SelectI32And_Sss) = eval::wasmi_i32_and; + fn select_i32_and_ssi(SelectI32And_Ssi) = eval::wasmi_i32_and; + fn select_i32_or_sss(SelectI32Or_Sss) = eval::wasmi_i32_or; + fn select_i32_or_ssi(SelectI32Or_Ssi) = eval::wasmi_i32_or; + fn select_i32_le_sss(SelectI32Le_Sss) = wasm::i32_le_s; + fn select_i32_le_ssi(SelectI32Le_Ssi) = wasm::i32_le_s; + fn select_i32_lt_sss(SelectI32Lt_Sss) = wasm::i32_lt_s; + fn select_i32_lt_ssi(SelectI32Lt_Ssi) = wasm::i32_lt_s; + fn select_u32_le_sss(SelectU32Le_Sss) = wasm::i32_le_u; + fn select_u32_le_ssi(SelectU32Le_Ssi) = wasm::i32_le_u; + fn select_u32_lt_sss(SelectU32Lt_Sss) = wasm::i32_lt_u; + fn select_u32_lt_ssi(SelectU32Lt_Ssi) = wasm::i32_lt_u; + // i64 + fn select_i64_eq_sss(SelectI64Eq_Sss) = wasm::i64_eq; + fn select_i64_eq_ssi(SelectI64Eq_Ssi) = wasm::i64_eq; + fn select_i64_and_sss(SelectI64And_Sss) = eval::wasmi_i64_and; + fn select_i64_and_ssi(SelectI64And_Ssi) = eval::wasmi_i64_and; + fn select_i64_or_sss(SelectI64Or_Sss) = eval::wasmi_i64_or; + fn select_i64_or_ssi(SelectI64Or_Ssi) = eval::wasmi_i64_or; + fn select_i64_le_sss(SelectI64Le_Sss) = wasm::i64_le_s; + fn select_i64_le_ssi(SelectI64Le_Ssi) = wasm::i64_le_s; + fn select_i64_lt_sss(SelectI64Lt_Sss) = wasm::i64_lt_s; + fn select_i64_lt_ssi(SelectI64Lt_Ssi) = wasm::i64_lt_s; + fn select_u64_le_sss(SelectU64Le_Sss) = wasm::i64_le_u; + fn select_u64_le_ssi(SelectU64Le_Ssi) = wasm::i64_le_u; + fn select_u64_lt_sss(SelectU64Lt_Sss) = wasm::i64_lt_u; + fn select_u64_lt_ssi(SelectU64Lt_Ssi) = wasm::i64_lt_u; + // f32 + fn select_f32_eq_sss(SelectF32Eq_Sss) = wasm::f32_eq; + fn select_f32_eq_ssi(SelectF32Eq_Ssi) = wasm::f32_eq; + fn select_f32_le_sss(SelectF32Le_Sss) = wasm::f32_le; + fn select_f32_le_ssi(SelectF32Le_Ssi) = wasm::f32_le; + fn select_f32_le_sis(SelectF32Le_Sis) = wasm::f32_le; + fn select_f32_lt_sss(SelectF32Lt_Sss) = wasm::f32_lt; + fn select_f32_lt_ssi(SelectF32Lt_Ssi) = wasm::f32_lt; + fn select_f32_lt_sis(SelectF32Lt_Sis) = wasm::f32_lt; + // f64 + fn select_f64_eq_sss(SelectF64Eq_Sss) = wasm::f64_eq; + fn select_f64_eq_ssi(SelectF64Eq_Ssi) = wasm::f64_eq; + fn select_f64_le_sss(SelectF64Le_Sss) = wasm::f64_le; + fn select_f64_le_ssi(SelectF64Le_Ssi) = wasm::f64_le; + fn select_f64_le_sis(SelectF64Le_Sis) = wasm::f64_le; + fn select_f64_lt_sss(SelectF64Lt_Sss) = wasm::f64_lt; + fn select_f64_lt_ssi(SelectF64Lt_Ssi) = wasm::f64_lt; + fn select_f64_lt_sis(SelectF64Lt_Sis) = wasm::f64_lt; +} + +macro_rules! handler_load_ss { + ( $( fn $handler:ident($decode:ident) = $load:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let ( + ip, + crate::ir::decode::$decode { + result, + ptr, + offset, + memory, + }, + ) = unsafe { decode_op(ip) }; + let ptr: u64 = get_value(ptr, sp); + let offset: u64 = get_value(offset, sp); + let mem_bytes = memory_bytes(memory, mem0, mem0_len, instance, state); + let loaded = $load(mem_bytes, ptr, offset).into_control()?; + set_value(sp, result, loaded); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_load_ss! { + fn load32_ss(Load32_Ss) = wasm::load32; + fn load64_ss(Load64_Ss) = wasm::load64; + fn i32_load8_ss(I32Load8_Ss) = wasm::i32_load8_s; + fn u32_load8_ss(U32Load8_Ss) = wasm::i32_load8_u; + fn i32_load16_ss(I32Load16_Ss) = wasm::i32_load16_s; + fn u32_load16_ss(U32Load16_Ss) = wasm::i32_load16_u; + fn i64_load8_ss(I64Load8_Ss) = wasm::i64_load8_s; + fn u64_load8_ss(U64Load8_Ss) = wasm::i64_load8_u; + fn i64_load16_ss(I64Load16_Ss) = wasm::i64_load16_s; + fn u64_load16_ss(U64Load16_Ss) = wasm::i64_load16_u; + fn i64_load32_ss(I64Load32_Ss) = wasm::i64_load32_s; + fn u64_load32_ss(U64Load32_Ss) = wasm::i64_load32_u; +} + +macro_rules! handler_load_si { + ( $( fn $handler:ident($decode:ident) = $load:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let ( + ip, + crate::ir::decode::$decode { + result, + address, + memory, + }, + ) = unsafe { decode_op(ip) }; + let address = get_value(address, sp); + let mem_bytes = memory_bytes(memory, mem0, mem0_len, instance, state); + let loaded = $load(mem_bytes, usize::from(address)).into_control()?; + set_value(sp, result, loaded); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_load_si! { + fn load32_si(Load32_Si) = wasm::load32_at; + fn load64_si(Load64_Si) = wasm::load64_at; + fn i32_load8_si(I32Load8_Si) = wasm::i32_load8_s_at; + fn u32_load8_si(U32Load8_Si) = wasm::i32_load8_u_at; + fn i32_load16_si(I32Load16_Si) = wasm::i32_load16_s_at; + fn u32_load16_si(U32Load16_Si) = wasm::i32_load16_u_at; + fn i64_load8_si(I64Load8_Si) = wasm::i64_load8_s_at; + fn u64_load8_si(U64Load8_Si) = wasm::i64_load8_u_at; + fn i64_load16_si(I64Load16_Si) = wasm::i64_load16_s_at; + fn u64_load16_si(U64Load16_Si) = wasm::i64_load16_u_at; + fn i64_load32_si(I64Load32_Si) = wasm::i64_load32_s_at; + fn u64_load32_si(U64Load32_Si) = wasm::i64_load32_u_at; +} + +macro_rules! handler_load_mem0_offset16_ss { + ( $( fn $handler:ident($decode:ident) = $load:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let ( + ip, + crate::ir::decode::$decode { + result, + ptr, + offset, + }, + ) = unsafe { decode_op(ip) }; + let ptr = get_value(ptr, sp); + let offset = get_value(offset, sp); + let mem_bytes = mem0_bytes(mem0, mem0_len); + let loaded = $load(mem_bytes, ptr, u64::from(u16::from(offset))).into_control()?; + set_value(sp, result, loaded); + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_load_mem0_offset16_ss! { + fn load32_mem0_offset16_ss(Load32Mem0Offset16_Ss) = wasm::load32; + fn load64_mem0_offset16_ss(Load64Mem0Offset16_Ss) = wasm::load64; + fn i32_load8_mem0_offset16_ss(I32Load8Mem0Offset16_Ss) = wasm::i32_load8_s; + fn u32_load8_mem0_offset16_ss(U32Load8Mem0Offset16_Ss) = wasm::i32_load8_u; + fn i32_load16_mem0_offset16_ss(I32Load16Mem0Offset16_Ss) = wasm::i32_load16_s; + fn u32_load16_mem0_offset16_ss(U32Load16Mem0Offset16_Ss) = wasm::i32_load16_u; + fn i64_load8_mem0_offset16_ss(I64Load8Mem0Offset16_Ss) = wasm::i64_load8_s; + fn u64_load8_mem0_offset16_ss(U64Load8Mem0Offset16_Ss) = wasm::i64_load8_u; + fn i64_load16_mem0_offset16_ss(I64Load16Mem0Offset16_Ss) = wasm::i64_load16_s; + fn u64_load16_mem0_offset16_ss(U64Load16Mem0Offset16_Ss) = wasm::i64_load16_u; + fn i64_load32_mem0_offset16_ss(I64Load32Mem0Offset16_Ss) = wasm::i64_load32_s; + fn u64_load32_mem0_offset16_ss(U64Load32Mem0Offset16_Ss) = wasm::i64_load32_u; +} + +macro_rules! handler_store_sx { + ( $( fn $handler:ident($decode:ident, $hint:ty) = $store:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let ( + ip, + crate::ir::decode::$decode { + ptr, + offset, + value, + memory, + }, + ) = unsafe { decode_op(ip) }; + let ptr = get_value(ptr, sp); + let offset = get_value(offset, sp); + let value: $hint = get_value(value, sp); + let mem_bytes = memory_bytes(memory, mem0, mem0_len, instance, state); + $store(mem_bytes, ptr, offset, value.into()).into_control()?; + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_store_sx! { + fn store32_ss(Store32_Ss, u32) = wasm::store32; + fn store32_si(Store32_Si, u32) = wasm::store32; + fn store64_ss(Store64_Ss, u64) = wasm::store64; + fn store64_si(Store64_Si, u64) = wasm::store64; + fn i32_store8_ss(I32Store8_Ss, i8) = wasm::i32_store8; + fn i32_store8_si(I32Store8_Si, i8) = wasm::i32_store8; + fn i32_store16_ss(I32Store16_Ss, i16) = wasm::i32_store16; + fn i32_store16_si(I32Store16_Si, i16) = wasm::i32_store16; + fn i64_store8_ss(I64Store8_Ss, i8) = wasm::i64_store8; + fn i64_store8_si(I64Store8_Si, i8) = wasm::i64_store8; + fn i64_store16_ss(I64Store16_Ss, i16) = wasm::i64_store16; + fn i64_store16_si(I64Store16_Si, i16) = wasm::i64_store16; + fn i64_store32_ss(I64Store32_Ss, i32) = wasm::i64_store32; + fn i64_store32_si(I64Store32_Si, i32) = wasm::i64_store32; +} + +macro_rules! handler_store_ix { + ( $( fn $handler:ident($decode:ident, $hint:ty) = $store:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let ( + ip, + crate::ir::decode::$decode { + address, + value, + memory, + }, + ) = unsafe { decode_op(ip) }; + let address = get_value(address, sp); + let value: $hint = get_value(value, sp); + let mem_bytes = memory_bytes(memory, mem0, mem0_len, instance, state); + $store(mem_bytes, usize::from(address), value.into()).into_control()?; + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_store_ix! { + fn store32_is(Store32_Is, u32) = wasm::store32_at; + fn store32_ii(Store32_Ii, u32) = wasm::store32_at; + fn store64_is(Store64_Is, u64) = wasm::store64_at; + fn store64_ii(Store64_Ii, u64) = wasm::store64_at; + fn i32_store8_is(I32Store8_Is, i8) = wasm::i32_store8_at; + fn i32_store8_ii(I32Store8_Ii, i8) = wasm::i32_store8_at; + fn i32_store16_is(I32Store16_Is, i16) = wasm::i32_store16_at; + fn i32_store16_ii(I32Store16_Ii, i16) = wasm::i32_store16_at; + fn i64_store8_is(I64Store8_Is, i8) = wasm::i64_store8_at; + fn i64_store8_ii(I64Store8_Ii, i8) = wasm::i64_store8_at; + fn i64_store16_is(I64Store16_Is, i16) = wasm::i64_store16_at; + fn i64_store16_ii(I64Store16_Ii, i16) = wasm::i64_store16_at; + fn i64_store32_is(I64Store32_Is, i32) = wasm::i64_store32_at; + fn i64_store32_ii(I64Store32_Ii, i32) = wasm::i64_store32_at; +} + +macro_rules! handler_store_mem0_offset16_sx { + ( $( fn $handler:ident($decode:ident, $hint:ty) = $store:expr );* $(;)? ) => { + $( + pub fn $handler( + state: &mut VmState, + ip: Ip, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Done { + let ( + ip, + crate::ir::decode::$decode { + ptr, + offset, + value, + }, + ) = unsafe { decode_op(ip) }; + let ptr = get_value(ptr, sp); + let offset = get_value(offset, sp); + let value: $hint = get_value(value, sp); + let mem_bytes = mem0_bytes(mem0, mem0_len); + $store(mem_bytes, ptr, u64::from(u16::from(offset)), value.into()).into_control()?; + dispatch!(state, ip, sp, mem0, mem0_len, instance) + } + )* + }; +} +handler_store_mem0_offset16_sx! { + fn store32_mem0_offset16_ss(Store32Mem0Offset16_Ss, u32) = wasm::store32; + fn store32_mem0_offset16_si(Store32Mem0Offset16_Si, u32) = wasm::store32; + fn store64_mem0_offset16_ss(Store64Mem0Offset16_Ss, u64) = wasm::store64; + fn store64_mem0_offset16_si(Store64Mem0Offset16_Si, u64) = wasm::store64; + fn i32_store8_mem0_offset16_ss(I32Store8Mem0Offset16_Ss, i8) = wasm::i32_store8; + fn i32_store8_mem0_offset16_si(I32Store8Mem0Offset16_Si, i8) = wasm::i32_store8; + fn i32_store16_mem0_offset16_ss(I32Store16Mem0Offset16_Ss, i16) = wasm::i32_store16; + fn i32_store16_mem0_offset16_si(I32Store16Mem0Offset16_Si, i16) = wasm::i32_store16; + fn i64_store8_mem0_offset16_ss(I64Store8Mem0Offset16_Ss, i8) = wasm::i64_store8; + fn i64_store8_mem0_offset16_si(I64Store8Mem0Offset16_Si, i8) = wasm::i64_store8; + fn i64_store16_mem0_offset16_ss(I64Store16Mem0Offset16_Ss, i16) = wasm::i64_store16; + fn i64_store16_mem0_offset16_si(I64Store16Mem0Offset16_Si, i16) = wasm::i64_store16; + fn i64_store32_mem0_offset16_ss(I64Store32Mem0Offset16_Ss, i32) = wasm::i64_store32; + fn i64_store32_mem0_offset16_si(I64Store32Mem0Offset16_Si, i32) = wasm::i64_store32; +} diff --git a/crates/wasmi/src/engine/executor/handler/exec/simd.rs b/crates/wasmi/src/engine/executor/handler/exec/simd.rs new file mode 100644 index 00000000000..ef28cd54444 --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/exec/simd.rs @@ -0,0 +1,274 @@ +use crate::engine::executor::handler::{ + dispatch::Done, + state::{Inst, Ip, Mem0Len, Mem0Ptr, Sp, VmState}, +}; + +macro_rules! gen_execution_handler_stubs { + ( $($name:ident),* $(,)? ) => { + $( + pub fn $name(_state: &mut VmState, _ip: Ip, _sp: Sp, _mem0: Mem0Ptr, _mem0_len: Mem0Len, _instance: Inst) -> Done { todo!() } + )* + }; +} +gen_execution_handler_stubs! { + copy128, + i8x16_shuffle, + v128_splat8_ss, + v128_splat8_si, + v128_splat16_ss, + v128_splat16_si, + v128_splat32_ss, + v128_splat32_si, + v128_splat64_ss, + v128_splat64_si, + s8x16_extract_lane, + u8x16_extract_lane, + s16x8_extract_lane, + u16x8_extract_lane, + u32x4_extract_lane, + u64x2_extract_lane, + v128_replace_lane8x16_sss, + v128_replace_lane8x16_ssi, + v128_replace_lane16x8_sss, + v128_replace_lane16x8_ssi, + v128_replace_lane32x4_sss, + v128_replace_lane32x4_ssi, + v128_replace_lane64x2_sss, + v128_replace_lane64x2_ssi, + i8x16_swizzle_sss, + i8x16_eq_sss, + i8x16_not_eq_sss, + i16x8_eq_sss, + i16x8_not_eq_sss, + i32x4_eq_sss, + i32x4_not_eq_sss, + i64x2_eq_sss, + i64x2_not_eq_sss, + i8x16_lt_sss, + i8x16_le_sss, + i16x8_lt_sss, + i16x8_le_sss, + i32x4_lt_sss, + i32x4_le_sss, + i64x2_lt_sss, + i64x2_le_sss, + u8x16_lt_sss, + u8x16_le_sss, + u16x8_lt_sss, + u16x8_le_sss, + u32x4_lt_sss, + u32x4_le_sss, + f32x4_eq_sss, + f32x4_not_eq_sss, + f32x4_lt_sss, + f32x4_le_sss, + f64x2_eq_sss, + f64x2_not_eq_sss, + f64x2_lt_sss, + f64x2_le_sss, + v128_and_sss, + v128_and_not_sss, + v128_or_sss, + v128_xor_sss, + i8x16_narrow_i16x8_sss, + u8x16_narrow_i16x8_sss, + i8x16_add_sss, + i8x16_add_sat_sss, + u8x16_add_sat_sss, + i8x16_sub_sss, + i8x16_sub_sat_sss, + u8x16_sub_sat_sss, + i8x16_min_sss, + u8x16_min_sss, + i8x16_max_sss, + u8x16_max_sss, + u8x16_avgr_sss, + i16x8_relaxed_dot_i8x16_i7x16_sss, + i16x8_q15_mulr_sat_sss, + i16x8_narrow_i32x4_sss, + u16x8_narrow_i32x4_sss, + i16x8_extmul_low_i8x16_sss, + u16x8_extmul_low_i8x16_sss, + i16x8_extmul_high_i8x16_sss, + u16x8_extmul_high_i8x16_sss, + i16x8_add_sss, + i16x8_add_sat_sss, + u16x8_add_sat_sss, + i16x8_sub_sss, + i16x8_sub_sat_sss, + u16x8_sub_sat_sss, + i16x8_mul_sss, + i16x8_min_sss, + u16x8_min_sss, + i16x8_max_sss, + u16x8_max_sss, + u16x8_avgr_sss, + i32x4_add_sss, + i32x4_sub_sss, + i32x4_mul_sss, + i32x4_min_sss, + u32x4_min_sss, + i32x4_max_sss, + u32x4_max_sss, + i32x4_dot_i16x8_sss, + i32x4_extmul_low_i16x8_sss, + u32x4_extmul_low_i16x8_sss, + i32x4_extmul_high_i16x8_sss, + u32x4_extmul_high_i16x8_sss, + i64x2_add_sss, + i64x2_sub_sss, + i64x2_mul_sss, + i64x2_extmul_low_i32x4_sss, + u64x2_extmul_low_i32x4_sss, + i64x2_extmul_high_i32x4_sss, + u64x2_extmul_high_i32x4_sss, + f32x4_add_sss, + f32x4_sub_sss, + f32x4_mul_sss, + f32x4_div_sss, + f32x4_min_sss, + f32x4_max_sss, + f32x4_pmin_sss, + f32x4_pmax_sss, + f64x2_add_sss, + f64x2_sub_sss, + f64x2_mul_sss, + f64x2_div_sss, + f64x2_min_sss, + f64x2_max_sss, + f64x2_pmin_sss, + f64x2_pmax_sss, + i8x16_shl_sss, + i8x16_shl_ssi, + i8x16_shr_sss, + i8x16_shr_ssi, + u8x16_shr_sss, + u8x16_shr_ssi, + i16x8_shl_sss, + i16x8_shl_ssi, + i16x8_shr_sss, + i16x8_shr_ssi, + u16x8_shr_sss, + u16x8_shr_ssi, + i32x4_shl_sss, + i32x4_shl_ssi, + i32x4_shr_sss, + i32x4_shr_ssi, + u32x4_shr_sss, + u32x4_shr_ssi, + i64x2_shl_sss, + i64x2_shl_ssi, + i64x2_shr_sss, + i64x2_shr_ssi, + u64x2_shr_sss, + u64x2_shr_ssi, + v128_not_ss, + v128_any_true_ss, + i8x16_abs_ss, + i8x16_neg_ss, + i8x16_popcnt_ss, + i8x16_all_true_ss, + i8x16_bitmask_ss, + i16x8_abs_ss, + i16x8_neg_ss, + i16x8_all_true_ss, + i16x8_bitmask_ss, + i16x8_extadd_pairwise_i8x16_ss, + u16x8_extadd_pairwise_i8x16_ss, + i16x8_extend_low_i8x16_ss, + u16x8_extend_low_i8x16_ss, + i16x8_extend_high_i8x16_ss, + u16x8_extend_high_i8x16_ss, + i32x4_abs_ss, + i32x4_neg_ss, + i32x4_all_true_ss, + i32x4_bitmask_ss, + i32x4_extadd_pairwise_i16x8_ss, + u32x4_extadd_pairwise_i16x8_ss, + i32x4_extend_low_i16x8_ss, + u32x4_extend_low_i16x8_ss, + i32x4_extend_high_i16x8_ss, + u32x4_extend_high_i16x8_ss, + i64x2_abs_ss, + i64x2_neg_ss, + i64x2_all_true_ss, + i64x2_bitmask_ss, + i64x2_extend_low_i32x4_ss, + u64x2_extend_low_i32x4_ss, + i64x2_extend_high_i32x4_ss, + u64x2_extend_high_i32x4_ss, + f32x4_demote_zero_f64x2_ss, + f32x4_ceil_ss, + f32x4_floor_ss, + f32x4_trunc_ss, + f32x4_nearest_ss, + f32x4_abs_ss, + f32x4_neg_ss, + f32x4_sqrt_ss, + f64x2_promote_low_f32x4_ss, + f64x2_ceil_ss, + f64x2_floor_ss, + f64x2_trunc_ss, + f64x2_nearest_ss, + f64x2_abs_ss, + f64x2_neg_ss, + f64x2_sqrt_ss, + i32x4_trunc_sat_f32x4_ss, + u32x4_trunc_sat_f32x4_ss, + i32x4_trunc_sat_zero_f64x2_ss, + u32x4_trunc_sat_zero_f64x2_ss, + f32x4_convert_i32x4_ss, + f32x4_convert_u32x4_ss, + f64x2_convert_low_i32x4_ss, + f64x2_convert_low_u32x4_ss, + v128_load_ss, + v128_load_mem0_offset16_ss, + i16x8_load8x8_ss, + i16x8_load8x8_mem0_offset16_ss, + u16x8_load8x8_ss, + u16x8_load8x8_mem0_offset16_ss, + i32x4_load16x4_ss, + i32x4_load16x4_mem0_offset16_ss, + u32x4_load16x4_ss, + u32x4_load16x4_mem0_offset16_ss, + i64x2_load32x2_ss, + i64x2_load32x2_mem0_offset16_ss, + u64x2_load32x2_ss, + u64x2_load32x2_mem0_offset16_ss, + v128_load8_splat_ss, + v128_load8_splat_mem0_offset16_ss, + v128_load16_splat_ss, + v128_load16_splat_mem0_offset16_ss, + v128_load32_splat_ss, + v128_load32_splat_mem0_offset16_ss, + v128_load64_splat_ss, + v128_load64_splat_mem0_offset16_ss, + v128_load32_zero_ss, + v128_load32_zero_mem0_offset16_ss, + v128_load64_zero_ss, + v128_load64_zero_mem0_offset16_ss, + v128_load_lane8_sss, + v128_load_lane8_mem0_offset16_sss, + v128_load_lane16_sss, + v128_load_lane16_mem0_offset16_sss, + v128_load_lane32_sss, + v128_load_lane32_mem0_offset16_sss, + v128_load_lane64_sss, + v128_load_lane64_mem0_offset16_sss, + store128_ss, + store128_mem0_offset16_ss, + v128_store8_lane_ss, + v128_store8_lane_mem0_offset16_ss, + v128_store16_lane_ss, + v128_store16_lane_mem0_offset16_ss, + v128_store32_lane_ss, + v128_store32_lane_mem0_offset16_ss, + v128_store64_lane_ss, + v128_store64_lane_mem0_offset16_ss, + i32x4_relaxed_dot_i8x16_i7x16_add_ssss, + f32x4_relaxed_madd_ssss, + f32x4_relaxed_nmadd_ssss, + f64x2_relaxed_madd_ssss, + f64x2_relaxed_nmadd_ssss, + v128_bitselect_ssss, +} diff --git a/crates/wasmi/src/engine/executor/handler/func.rs b/crates/wasmi/src/engine/executor/handler/func.rs new file mode 100644 index 00000000000..9aa1f989923 --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/func.rs @@ -0,0 +1,268 @@ +use crate::{ + engine::{ + executor::handler::{ + dispatch::{execute_until_done, ExecutionOutcome}, + state::{Inst, Ip, Sp, Stack, VmState}, + utils::{self, resolve_instance, set_value}, + }, + CallParams, + CallResults, + CodeMap, + EngineFunc, + }, + func::HostFuncEntity, + ir::{BoundedSlotSpan, Slot, SlotSpan}, + store::{CallHooks, StoreError}, + CallHook, + Error, + Instance, + Store, +}; +use core::marker::PhantomData; + +pub struct WasmFuncCall<'a, T, State> { + store: &'a mut Store, + stack: &'a mut Stack, + code: &'a CodeMap, + callee_ip: Ip, + callee_sp: Sp, + instance: Inst, + state: State, +} + +impl<'a, T, State> WasmFuncCall<'a, T, State> { + fn new_state(self, state: NewState) -> WasmFuncCall<'a, T, NewState> { + WasmFuncCall { + store: self.store, + stack: self.stack, + code: self.code, + callee_ip: self.callee_ip, + callee_sp: self.callee_sp, + instance: self.instance, + state, + } + } +} + +mod state { + use super::Sp; + use crate::func::{FuncInOut, Trampoline}; + use core::marker::PhantomData; + + pub type Uninit = PhantomData; + pub type Init = PhantomData; + pub type Resumed = PhantomData; + + mod marker { + pub enum Uninit {} + pub enum Init {} + pub enum Resumed {} + } + + pub struct UninitHost<'a> { + pub sp: Sp, + pub inout: FuncInOut<'a>, + pub trampoline: Trampoline, + } + + pub struct InitHost<'a> { + pub sp: Sp, + pub inout: FuncInOut<'a>, + pub trampoline: Trampoline, + } + + pub trait Execute {} + impl Execute for Init {} + impl Execute for Resumed {} + pub struct Done { + pub sp: Sp, + } +} + +impl<'a, T> WasmFuncCall<'a, T, state::Uninit> { + pub fn write_params(self, params: impl CallParams) -> WasmFuncCall<'a, T, state::Init> { + let mut param_slot = Slot::from(0); + for param_value in params.call_params() { + set_value(self.callee_sp, param_slot, param_value); + param_slot = param_slot.next(); + } + self.new_state(PhantomData) + } +} + +impl<'a, T, State: state::Execute> WasmFuncCall<'a, T, State> { + pub fn execute(mut self) -> Result, ExecutionOutcome> { + self.store.invoke_call_hook(CallHook::CallingWasm)?; + let outcome = self.execute_until_done(); + self.store.invoke_call_hook(CallHook::ReturningFromWasm)?; + let sp = outcome?; + Ok(self.new_state(state::Done { sp })) + } + + fn execute_until_done(&mut self) -> Result { + let store = self.store.prune(); + let (mem0, mem0_len) = utils::extract_mem0(store, self.instance); + let state = VmState::new(store, self.stack, self.code); + execute_until_done( + state, + self.callee_ip, + self.callee_sp, + mem0, + mem0_len, + self.instance, + ) + } +} + +impl<'a, T> WasmFuncCall<'a, T, state::Resumed> { + pub fn provide_host_results( + self, + params: impl CallParams, + slots: SlotSpan, + ) -> WasmFuncCall<'a, T, state::Init> { + let mut param_slot = slots.head(); + for param_value in params.call_params() { + set_value(self.callee_sp, param_slot, param_value); + param_slot = param_slot.next(); + } + self.new_state(PhantomData) + } +} + +impl<'a, T> WasmFuncCall<'a, T, state::Done> { + pub fn write_results(self, results: R) -> ::Results { + let len_results = results.len_results(); + let sp = self.state.sp; + let slice = unsafe { sp.as_slice(len_results) }; + results.call_results(slice) + } +} + +pub fn init_wasm_func_call<'a, T>( + store: &'a mut Store, + code: &'a CodeMap, + stack: &'a mut Stack, + engine_func: EngineFunc, + instance: Instance, +) -> Result, Error> { + let compiled_func = code.get(Some(store.inner.fuel_mut()), engine_func)?; + let callee_ip = Ip::from(compiled_func.ops()); + let frame_size = compiled_func.len_stack_slots(); + let callee_params = BoundedSlotSpan::new(SlotSpan::new(Slot::from(0)), 0); + let instance = resolve_instance(store.prune(), &instance).into(); + let callee_sp = stack.push_frame( + None, + callee_ip, + callee_params, + usize::from(frame_size), + Some(instance), + )?; + Ok(WasmFuncCall { + store, + stack, + code, + callee_ip, + callee_sp, + instance, + state: PhantomData, + }) +} + +pub fn resume_wasm_func_call<'a, T>( + store: &'a mut Store, + code: &'a CodeMap, + stack: &'a mut Stack, +) -> Result, Error> { + let (callee_ip, callee_sp, instance) = stack.restore_frame(); + Ok(WasmFuncCall { + store, + stack, + code, + callee_ip, + callee_sp, + instance, + state: PhantomData, + }) +} + +pub fn init_host_func_call<'a, T>( + store: &'a mut Store, + stack: &'a mut Stack, + func: HostFuncEntity, +) -> Result>, Error> { + let len_params = func.len_params(); + let len_results = func.len_results(); + let trampoline = *func.trampoline(); + let callee_params = BoundedSlotSpan::new(SlotSpan::new(Slot::from(0)), len_params); + let (sp, inout) = stack.prepare_host_frame(None, callee_params, len_results)?; + Ok(HostFuncCall { + store, + state: state::UninitHost { + sp, + inout, + trampoline, + }, + }) +} + +#[derive(Debug)] +pub struct HostFuncCall<'a, T, State> { + store: &'a mut Store, + state: State, +} + +impl<'a, T> HostFuncCall<'a, T, state::UninitHost<'a>> { + pub fn write_params(self, params: impl CallParams) -> HostFuncCall<'a, T, state::InitHost<'a>> { + let state::UninitHost { + sp, + inout, + trampoline, + } = self.state; + let mut param_slot = Slot::from(0); + for param_value in params.call_params() { + set_value(sp, param_slot, param_value); + param_slot = param_slot.next(); + } + HostFuncCall { + store: self.store, + state: state::InitHost { + sp, + inout, + trampoline, + }, + } + } +} + +impl<'a, T> HostFuncCall<'a, T, state::InitHost<'a>> { + pub fn execute(self) -> Result, Error> { + let state::InitHost { + sp, + inout, + trampoline, + } = self.state; + let outcome = self + .store + .prune() + .call_host_func(trampoline, None, inout, CallHooks::Ignore); + if let Err(error) = outcome { + match error { + StoreError::External(error) => return Err(error), + StoreError::Internal(error) => panic!("internal interpreter error: {error}"), + } + } + Ok(HostFuncCall { + store: self.store, + state: state::Done { sp }, + }) + } +} + +impl<'a, T> HostFuncCall<'a, T, state::Done> { + pub fn write_results(self, results: R) -> ::Results { + let len_results = results.len_results(); + let sp = self.state.sp; + let slice = unsafe { sp.as_slice(len_results) }; + results.call_results(slice) + } +} diff --git a/crates/wasmi/src/engine/executor/handler/mod.rs b/crates/wasmi/src/engine/executor/handler/mod.rs new file mode 100644 index 00000000000..a27cd040875 --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/mod.rs @@ -0,0 +1,18 @@ +#[macro_use] +mod dispatch; +#[macro_use] +mod utils; +mod eval; +mod exec; +mod func; +mod state; + +pub use self::{ + dispatch::{op_code_to_handler, ExecutionOutcome}, + func::{init_host_func_call, init_wasm_func_call, resume_wasm_func_call}, + state::{Inst, Stack}, +}; +use self::{ + dispatch::{Break, Control, Done}, + state::DoneReason, +}; diff --git a/crates/wasmi/src/engine/executor/handler/state.rs b/crates/wasmi/src/engine/executor/handler/state.rs new file mode 100644 index 00000000000..ebb767c32bd --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/state.rs @@ -0,0 +1,723 @@ +use crate::{ + core::{ReadAs, UntypedVal, WriteAs}, + engine::{ + executor::{ + handler::{ + dispatch::{Control, ExecutionOutcome}, + utils::extract_mem0, + }, + CodeMap, + }, + utils::unreachable_unchecked, + ResumableHostTrapError, + ResumableOutOfFuelError, + StackConfig, + }, + func::FuncInOut, + instance::InstanceEntity, + ir::{self, BoundedSlotSpan, Slot, SlotSpan}, + store::PrunedStore, + Error, + Func, + TrapCode, +}; +use alloc::vec::Vec; +use core::{ + cmp, + mem, + ptr::{self, NonNull}, + slice, +}; + +pub struct VmState<'vm> { + pub store: &'vm mut PrunedStore, + pub stack: &'vm mut Stack, + pub code: &'vm CodeMap, + done_reason: Option, +} + +impl<'vm> VmState<'vm> { + pub fn new(store: &'vm mut PrunedStore, stack: &'vm mut Stack, code: &'vm CodeMap) -> Self { + Self { + store, + stack, + code, + done_reason: None, + } + } + + pub fn done_with(&mut self, reason: impl FnOnce() -> DoneReason) { + #[cold] + #[inline(never)] + fn err(prev: &DoneReason, reason: impl FnOnce() -> DoneReason) -> ! { + panic!( + "\ + tried to done with reason while reason already exists:\n\ + \t- new reason: {:?},\n\ + \t- old reason: {:?},\ + ", + reason(), + prev, + ) + } + + if let Some(prev) = &self.done_reason { + err(prev, reason) + } + self.done_reason = Some(reason()); + } + + pub fn into_execution_outcome(self) -> Result { + let Some(reason) = self.done_reason else { + panic!("missing break reason") + }; + let outcome = match reason { + DoneReason::Return(sp) => return Ok(sp), + DoneReason::Host(error) => error.into(), + DoneReason::OutOfFuel(error) => error.into(), + DoneReason::Error(error) => error.into(), + }; + Err(outcome) + } +} + +#[derive(Debug)] +pub enum DoneReason { + /// The execution finished successfully with a result found at the [`Sp`]. + Return(Sp), + /// A resumable error indicating an error returned by a called host function. + Host(ResumableHostTrapError), + /// A resumable error indicating that the execution ran out of fuel. + OutOfFuel(ResumableOutOfFuelError), + /// A non-resumable error. + Error(Error), +} + +impl DoneReason { + #[cold] + #[inline] + pub fn error(error: Error) -> Self { + Self::Error(error) + } + + #[cold] + #[inline] + pub fn host_error(error: Error, func: Func, results: SlotSpan) -> Self { + Self::Host(ResumableHostTrapError::new(error, func, results)) + } + + #[cold] + #[inline] + pub fn out_of_fuel(required_fuel: u64) -> Self { + Self::OutOfFuel(ResumableOutOfFuelError::new(required_fuel)) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct Inst(NonNull); + +impl From<&'_ InstanceEntity> for Inst { + fn from(entity: &'_ InstanceEntity) -> Self { + Self(entity.into()) + } +} + +impl PartialEq for Inst { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} +impl Eq for Inst {} + +impl Inst { + pub unsafe fn as_ref(&self) -> &InstanceEntity { + unsafe { self.0.as_ref() } + } +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub struct Mem0Ptr(*mut u8); + +impl From<*mut u8> for Mem0Ptr { + fn from(value: *mut u8) -> Self { + Self(value) + } +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub struct Mem0Len(usize); + +impl From for Mem0Len { + fn from(value: usize) -> Self { + Self(value) + } +} + +pub fn mem0_bytes<'a>(mem0: Mem0Ptr, mem0_len: Mem0Len) -> &'a mut [u8] { + unsafe { slice::from_raw_parts_mut(mem0.0, mem0_len.0) } +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub struct Ip { + value: *const u8, +} + +impl<'a> From<&'a [u8]> for Ip { + fn from(ops: &'a [u8]) -> Self { + Self { + value: ops.as_ptr(), + } + } +} + +struct IpDecoder(Ip); +impl ir::Decoder for IpDecoder { + fn read_bytes(&mut self, buffer: &mut [u8]) -> Result<(), ir::DecodeError> { + unsafe { ptr::copy_nonoverlapping(self.0.value, buffer.as_mut_ptr(), buffer.len()) }; + self.0 = unsafe { self.0.add(buffer.len()) }; + Ok(()) + } +} + +impl Ip { + #[inline] + pub unsafe fn decode(self) -> (Ip, T) { + let mut ip = IpDecoder(self); + let decoded = match ::decode(&mut ip) { + Ok(decoded) => decoded, + Err(error) => unsafe { + crate::engine::utils::unreachable_unchecked!( + "failed to decode `OpCode` or op-handler: {error}" + ) + }, + }; + (ip.0, decoded) + } + + pub unsafe fn skip(self) -> Ip { + let (ip, _) = unsafe { self.decode::() }; + ip + } + + pub unsafe fn offset(self, delta: isize) -> Self { + let value = unsafe { self.value.byte_offset(delta) }; + Self { value } + } + + pub unsafe fn add(self, delta: usize) -> Self { + let value = unsafe { self.value.byte_add(delta) }; + Self { value } + } +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub struct Sp { + value: *mut UntypedVal, +} + +impl Sp { + pub fn new(cells: &mut Vec, start: usize) -> Self { + debug_assert!( + // Note: it is fine to use <= here because for zero sized frames + // we sometimes end up with `start == cells.len()` which isn't + // bad since in those cases `Sp` is never used. + start <= cells.len(), + "start = {}, cells.len() = {}", + start, + cells.len() + ); + let value = unsafe { cells.as_mut_ptr().add(start) }; + Self { value } + } + + pub fn dangling() -> Self { + Self { + value: ptr::dangling_mut(), + } + } + + pub fn get(self, slot: Slot) -> T + where + UntypedVal: ReadAs, + { + let index = usize::from(u16::from(slot)); + let value = unsafe { &*self.value.add(index) }; + >::read_as(value) + } + + pub fn set(self, slot: Slot, value: T) + where + UntypedVal: WriteAs, + { + let index = usize::from(u16::from(slot)); + let cell = unsafe { &mut *self.value.add(index) }; + >::write_as(cell, value); + } + + pub unsafe fn as_slice<'a>(self, len: usize) -> &'a [UntypedVal] { + unsafe { core::slice::from_raw_parts(self.value, len) } + } +} + +#[derive(Debug)] +pub struct Stack { + values: ValueStack, + frames: CallStack, +} + +type ReturnCallHost = Control<(Ip, Sp, Inst), Sp>; + +impl Stack { + pub fn new(config: &StackConfig) -> Self { + Self { + values: ValueStack::new(config.min_stack_height(), config.max_stack_height()), + frames: CallStack::new(config.max_recursion_depth()), + } + } + + pub fn empty() -> Self { + Self { + values: ValueStack::empty(), + frames: CallStack::empty(), + } + } + + pub fn reset(&mut self) { + self.values.reset(); + self.frames.reset(); + } + + pub fn capacity(&self) -> usize { + self.values.capacity() + } + + pub fn sync_ip(&mut self, ip: Ip) { + self.frames.sync_ip(ip); + } + + pub fn restore_frame(&mut self) -> (Ip, Sp, Inst) { + let Some((ip, start, instance)) = self.frames.restore_frame() else { + panic!("restore_frame: missing top-frame") + }; + let sp = self.values.sp_or_dangling(start); + (ip, sp, instance) + } + + pub fn return_prepare_host_frame<'a>( + &'a mut self, + callee_params: BoundedSlotSpan, + results_len: u16, + caller_instance: Inst, + ) -> Result<(ReturnCallHost, FuncInOut<'a>), TrapCode> { + let (callee_start, caller) = self.frames.return_prepare_host_frame(caller_instance); + self.values + .return_prepare_host_frame(caller, callee_start, callee_params, results_len) + } + + pub fn prepare_host_frame<'a>( + &'a mut self, + caller_ip: Option, + callee_params: BoundedSlotSpan, + results_len: u16, + ) -> Result<(Sp, FuncInOut<'a>), TrapCode> { + let caller_start = self.frames.prepare_host_frame(caller_ip); + self.values + .prepare_host_frame(caller_start, callee_params, results_len) + } + + #[inline(always)] + pub fn push_frame( + &mut self, + caller_ip: Option, + callee_ip: Ip, + callee_params: BoundedSlotSpan, + callee_size: usize, + callee_instance: Option, + ) -> Result { + let start = self + .frames + .push(caller_ip, callee_ip, callee_params, callee_instance)?; + self.values.push(start, callee_size, callee_params.len()) + } + + pub fn pop_frame( + &mut self, + store: &mut PrunedStore, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + ) -> Option<(Ip, Sp, Mem0Ptr, Mem0Len, Inst)> { + let (ip, start, changed_instance) = self.frames.pop()?; + let sp = self.values.sp_or_dangling(start); + let (mem0, mem0_len, instance) = match changed_instance { + Some(instance) => { + let (mem0, mem0_len) = extract_mem0(store, instance); + (mem0, mem0_len, instance) + } + None => (mem0, mem0_len, instance), + }; + Some((ip, sp, mem0, mem0_len, instance)) + } + + #[inline(always)] + pub fn replace_frame( + &mut self, + callee_ip: Ip, + callee_params: BoundedSlotSpan, + callee_size: usize, + callee_instance: Option, + ) -> Result { + let start = self.frames.replace(callee_ip, callee_instance)?; + self.values.replace(start, callee_size, callee_params) + } +} + +#[derive(Debug)] +pub struct ValueStack { + cells: Vec, + max_height: usize, +} + +impl ValueStack { + fn new(min_height: usize, max_height: usize) -> Self { + debug_assert!(min_height <= max_height); + // We need to convert from `size_of`` to `size_of`: + let sizeof_cell = mem::size_of::(); + let min_height = min_height / sizeof_cell; + let max_height = max_height / sizeof_cell; + let cells = Vec::with_capacity(min_height); + Self { cells, max_height } + } + + fn empty() -> Self { + Self { + cells: Vec::new(), + max_height: 0, + } + } + + fn reset(&mut self) { + self.cells.clear(); + } + + fn capacity(&self) -> usize { + self.cells.capacity() + } + + fn sp(&mut self, start: usize) -> Sp { + Sp::new(&mut self.cells, start) + } + + fn sp_or_dangling(&mut self, start: usize) -> Sp { + match self.cells.is_empty() { + true => { + debug_assert_eq!(start, 0); + Sp::dangling() + } + false => self.sp(start), + } + } + + /// Grows the number of cells to `new_len` if the current number is less than `new_len`. + /// + /// Does nothing if the number of cells is already at least `new_len`. + /// + /// # Errors + /// + /// - Returns [`TrapCode::OutOfSystemMemory`] if the machine ran out of memory. + /// - Returns [`TrapCode::StackOverflow`] if this exceeds the stack's predefined limits. + fn grow_if_needed(&mut self, new_len: usize) -> Result<(), TrapCode> { + if new_len > self.max_height { + return Err(TrapCode::StackOverflow); + } + let capacity = self.cells.capacity(); + let len = self.cells.len(); + if new_len > capacity { + debug_assert!( + self.cells.len() <= self.cells.capacity(), + "capacity must always be larger or equal to the actual number of the cells" + ); + let additional = new_len - len; + self.cells + .try_reserve(additional) + .map_err(|_| TrapCode::OutOfSystemMemory)?; + debug_assert!( + self.cells.capacity() >= new_len, + "capacity must now be at least as large as `new_len` ({new_len}) but found {}", + self.cells.capacity() + ); + } + let max_len = cmp::max(new_len, len); + // Safety: there is no need to initialize the cells since we are operating + // on `UntypedVal` which only has valid bit patterns. + // Note: non-security related initialization of function parameters + // and zero-initialization of function locals happens elsewhere. + unsafe { self.cells.set_len(max_len) }; + Ok(()) + } + + /// # Note + /// + /// In the following code, `callee` represents the called host function frame + /// and `caller` represents the caller of the caller of the host function, a.k.a. + /// the caller's caller. + fn return_prepare_host_frame<'a>( + &'a mut self, + caller: Option<(Ip, usize, Inst)>, + callee_start: usize, + callee_params: BoundedSlotSpan, + results_len: u16, + ) -> Result<(ReturnCallHost, FuncInOut<'a>), TrapCode> { + let caller_start = caller.map(|(_, start, _)| start).unwrap_or(0); + let params_offset = usize::from(u16::from(callee_params.span().head())); + let params_len = usize::from(callee_params.len()); + let results_len = usize::from(results_len); + let callee_size = params_len.max(results_len); + if callee_size == 0 { + let sp = match caller { + Some(_) if caller_start != callee_start => self.sp(caller_start), + _ => Sp::dangling(), + }; + let inout = FuncInOut::new(&mut [], 0, 0); + let control = match caller { + Some((ip, _, instance)) => ReturnCallHost::Continue((ip, sp, instance)), + None => ReturnCallHost::Break(sp), + }; + return Ok((control, inout)); + } + let Some(params_start) = callee_start.checked_add(params_offset) else { + return Err(TrapCode::StackOverflow); + }; + let Some(params_end) = params_start.checked_add(params_len) else { + return Err(TrapCode::StackOverflow); + }; + self.cells + .copy_within(params_start..params_end, callee_start); + let Some(callee_end) = callee_start.checked_add(callee_size) else { + return Err(TrapCode::StackOverflow); + }; + self.grow_if_needed(callee_end)?; + let caller_sp = self.sp(caller_start); + let cells = &mut self.cells[callee_start..callee_end]; + let inout = FuncInOut::new(cells, params_len, results_len); + let control = match caller { + Some((ip, _, instance)) => ReturnCallHost::Continue((ip, caller_sp, instance)), + None => ReturnCallHost::Break(caller_sp), + }; + Ok((control, inout)) + } + + fn prepare_host_frame<'a>( + &'a mut self, + caller_start: usize, + callee_params: BoundedSlotSpan, + results_len: u16, + ) -> Result<(Sp, FuncInOut<'a>), TrapCode> { + let params_offset = usize::from(u16::from(callee_params.span().head())); + let params_len = usize::from(callee_params.len()); + let results_len = usize::from(results_len); + let callee_size = params_len.max(results_len); + let Some(callee_start) = caller_start.checked_add(params_offset) else { + return Err(TrapCode::StackOverflow); + }; + let Some(callee_end) = callee_start.checked_add(callee_size) else { + return Err(TrapCode::StackOverflow); + }; + self.grow_if_needed(callee_end)?; + let sp = self.sp(caller_start); + let cells = &mut self.cells[callee_start..callee_end]; + let inout = FuncInOut::new(cells, params_len, results_len); + Ok((sp, inout)) + } + + #[inline(always)] + fn push(&mut self, start: usize, len_slots: usize, len_params: u16) -> Result { + let len_params = usize::from(len_params); + debug_assert!(len_params <= len_slots); + if len_slots == 0 { + return Ok(Sp::dangling()); + } + let Some(end) = start.checked_add(len_slots) else { + return Err(TrapCode::StackOverflow); + }; + self.grow_if_needed(end)?; + let start_locals = start.wrapping_add(len_params); + self.cells[start_locals..end].fill_with(UntypedVal::default); + let sp = self.sp(start); + Ok(sp) + } + + #[inline(always)] + fn replace( + &mut self, + callee_start: usize, + callee_size: usize, + callee_params: BoundedSlotSpan, + ) -> Result { + let params_len = usize::from(callee_params.len()); + let params_start = usize::from(u16::from(callee_params.span().head())); + let params_end = params_start.wrapping_add(params_len); + if callee_size == 0 { + return Ok(Sp::dangling()); + } + let Some(callee_end) = callee_start.checked_add(callee_size) else { + return Err(TrapCode::StackOverflow); + }; + self.grow_if_needed(callee_end)?; + let Some(callee_cells) = self.cells.get_mut(callee_start..) else { + unsafe { unreachable_unchecked!("ValueStack::replace: out of bounds callee cells") } + }; + callee_cells.copy_within(params_start..params_end, 0); + callee_cells[params_len..callee_size].fill_with(UntypedVal::default); + let sp = self.sp(callee_start); + Ok(sp) + } +} + +#[derive(Debug)] +pub struct CallStack { + frames: Vec, + instance: Option, + max_height: usize, +} + +impl CallStack { + fn new(max_height: usize) -> Self { + Self { + frames: Vec::new(), + instance: None, + max_height, + } + } + + fn empty() -> Self { + Self::new(0) + } + + fn reset(&mut self) { + self.frames.clear(); + self.instance = None; + } + + fn top_start(&self) -> usize { + let Some(top) = self.top() else { return 0 }; + top.start + } + + fn top(&self) -> Option<&Frame> { + self.frames.last() + } + + fn sync_ip(&mut self, ip: Ip) { + let Some(top) = self.frames.last_mut() else { + panic!("must have top call frame") + }; + top.ip = ip; + } + + fn restore_frame(&self) -> Option<(Ip, usize, Inst)> { + let instance = self.instance?; + let top = self.top()?; + Some((top.ip, top.start, instance)) + } + + fn prepare_host_frame(&mut self, caller_ip: Option) -> usize { + if let Some(caller_ip) = caller_ip { + self.sync_ip(caller_ip); + } + self.top_start() + } + + /// # Note + /// + /// In the following code, `callee` represents the called host function frame + /// and `caller` represents the caller of the caller of the host function, a.k.a. + /// the caller's caller. + pub fn return_prepare_host_frame( + &mut self, + callee_instance: Inst, + ) -> (usize, Option<(Ip, usize, Inst)>) { + let callee_start = self.top_start(); + let caller = match self.pop() { + Some((ip, start, instance)) => { + let instance = instance.unwrap_or(callee_instance); + Some((ip, start, instance)) + } + None => None, + }; + (callee_start, caller) + } + + #[inline(always)] + fn push( + &mut self, + caller_ip: Option, + callee_ip: Ip, + callee_params: BoundedSlotSpan, + instance: Option, + ) -> Result { + if self.frames.len() == self.max_height { + return Err(TrapCode::StackOverflow); + } + match caller_ip { + Some(caller_ip) => self.sync_ip(caller_ip), + None => debug_assert!(self.frames.is_empty()), + } + let prev_instance = match instance { + Some(instance) => self.instance.replace(instance), + None => self.instance, + }; + let params_offset = usize::from(u16::from(callee_params.span().head())); + let Some(start) = self.top_start().checked_add(params_offset) else { + return Err(TrapCode::StackOverflow); + }; + self.frames.push(Frame { + ip: callee_ip, + start, + instance: prev_instance, + }); + Ok(start) + } + + fn pop(&mut self) -> Option<(Ip, usize, Option)> { + let Some(popped) = self.frames.pop() else { + unsafe { unreachable_unchecked!("call stack must not be empty") } + }; + let top = self.top()?; + let ip = top.ip; + let start = top.start; + if let Some(instance) = popped.instance { + self.instance = Some(instance); + } + Some((ip, start, popped.instance)) + } + + #[inline(always)] + fn replace(&mut self, callee_ip: Ip, instance: Option) -> Result { + let Some(caller_frame) = self.frames.last_mut() else { + unsafe { unreachable_unchecked!("missing caller frame on the call stack") } + }; + let prev_instance = match instance { + Some(instance) => self.instance.replace(instance), + None => self.instance, + }; + let start = caller_frame.start; + *caller_frame = Frame { + start, + ip: callee_ip, + instance: prev_instance, + }; + Ok(start) + } +} + +#[derive(Debug)] +pub struct Frame { + pub ip: Ip, + start: usize, + instance: Option, +} diff --git a/crates/wasmi/src/engine/executor/handler/utils.rs b/crates/wasmi/src/engine/executor/handler/utils.rs new file mode 100644 index 00000000000..ba82e80c2fe --- /dev/null +++ b/crates/wasmi/src/engine/executor/handler/utils.rs @@ -0,0 +1,537 @@ +use super::state::{mem0_bytes, Inst, Ip, Mem0Len, Mem0Ptr, Sp, VmState}; +use crate::{ + core::{CoreElementSegment, CoreGlobal, CoreMemory, CoreTable, ReadAs, UntypedVal, WriteAs}, + engine::{ + executor::handler::{Break, Control, Done, DoneReason}, + utils::unreachable_unchecked, + DedupFuncType, + EngineFunc, + }, + func::{FuncEntity, HostFuncEntity}, + instance::InstanceEntity, + ir::{index, Address, BoundedSlotSpan, BranchOffset, Offset16, Sign, Slot, SlotSpan}, + memory::{DataSegment, DataSegmentEntity}, + store::{CallHooks, PrunedStore, StoreError, StoreInner}, + table::ElementSegment, + Error, + Func, + Global, + Instance, + Memory, + Ref, + Table, + TrapCode, +}; +use core::num::NonZero; + +macro_rules! consume_fuel { + ($state:expr, $ip:expr, $fuel:expr, $eval:expr) => {{ + if let ::core::result::Result::Err($crate::errors::FuelError::OutOfFuel { required_fuel }) = + $fuel.consume_fuel_if($eval) + { + out_of_fuel!($state, $ip, required_fuel) + } + }}; +} + +macro_rules! out_of_fuel { + ($state:expr, $ip:expr, $required_fuel:expr) => {{ + $state.stack.sync_ip($ip); + done!( + $state, + $crate::engine::executor::handler::DoneReason::out_of_fuel($required_fuel), + ) + }}; +} + +pub fn compile_or_get_func(state: &mut VmState, func: EngineFunc) -> Result<(Ip, usize), Error> { + let fuel_mut = state.store.inner_mut().fuel_mut(); + let compiled_func = state.code.get(Some(fuel_mut), func)?; + let ip = Ip::from(compiled_func.ops()); + let size = usize::from(compiled_func.len_stack_slots()); + Ok((ip, size)) +} + +macro_rules! compile_or_get_func { + ($state:expr, $func:expr) => {{ + match $crate::engine::executor::handler::utils::compile_or_get_func($state, $func) { + Ok((ip, size)) => (ip, size), + Err(error) => done!($state, DoneReason::error(error)), + } + }}; +} + +macro_rules! trap { + ($trap_code:expr) => {{ + return $crate::engine::executor::handler::Control::Break( + $crate::engine::executor::handler::Break::from($trap_code), + ); + }}; +} + +macro_rules! done { + ($state:expr, $reason:expr $(,)? ) => {{ + $state.done_with(move || { + <_ as ::core::convert::Into<$crate::engine::executor::handler::DoneReason>>::into( + $reason, + ) + }); + return $crate::engine::executor::handler::dispatch::control_break(); + }}; +} + +pub trait IntoControl { + type Value; + + fn into_control(self) -> Control; +} + +impl IntoControl for Result { + type Value = T; + + fn into_control(self) -> Control { + match self { + Ok(value) => Control::Continue(value), + Err(trap_code) => Control::Break(Break::from(trap_code)), + } + } +} + +macro_rules! impl_into_control { + ( $($ty:ty),* $(,)? ) => { + $( + impl IntoControl for $ty { + type Value = Self; + + fn into_control(self) -> Control { + Control::Continue(self) + } + } + )* + }; +} +impl_into_control! { + bool, + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + f32, f64, +} + +pub trait GetValue { + fn get_value(src: Self, sp: Sp) -> T; +} + +macro_rules! impl_get_value { + ( $($ty:ty),* $(,)? ) => { + $( + impl GetValue<$ty> for $ty { + #[inline(always)] + fn get_value(src: Self, _sp: Sp) -> $ty { + src + } + } + )* + }; +} +impl_get_value!( + u8, + u16, + u32, + u64, + i8, + i16, + i32, + i64, + f32, + f64, + NonZero, + NonZero, + NonZero, + NonZero, + Sign, + Sign, + Address, + Offset16, +); + +impl GetValue for Slot +where + T: Copy, + UntypedVal: ReadAs, +{ + fn get_value(src: Self, sp: Sp) -> T { + sp.get::(src) + } +} + +pub fn get_value(src: T, sp: Sp) -> L +where + T: GetValue, +{ + >::get_value(src, sp) +} + +pub trait SetValue { + fn set_value(src: Self, value: T, sp: Sp); +} + +impl SetValue for Slot +where + UntypedVal: WriteAs, +{ + fn set_value(src: Self, value: T, sp: Sp) { + sp.set::(src, value) + } +} + +pub fn set_value(sp: Sp, src: T, value: V) +where + T: SetValue, +{ + >::set_value(src, value, sp) +} + +pub fn exec_return( + state: &mut VmState, + sp: Sp, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Done { + let Some((ip, sp, mem0, mem0_len, instance)) = + state.stack.pop_frame(state.store, mem0, mem0_len, instance) + else { + // No more frames on the call stack -> break out of execution! + done!(state, DoneReason::Return(sp)) + }; + dispatch!(state, ip, sp, mem0, mem0_len, instance) +} + +pub fn exec_copy_span(sp: Sp, dst: SlotSpan, src: SlotSpan, len: u16) { + let op = match dst.head() <= src.head() { + true => exec_copy_span_asc, + false => exec_copy_span_des, + }; + op(sp, dst, src, len) +} + +pub fn exec_copy_span_asc(sp: Sp, dst: SlotSpan, src: SlotSpan, len: u16) { + debug_assert!(dst.head() <= src.head()); + let dst = dst.iter(len); + let src = src.iter(len); + for (dst, src) in dst.into_iter().zip(src.into_iter()) { + let value: u64 = get_value(src, sp); + set_value(sp, dst, value); + } +} + +pub fn exec_copy_span_des(sp: Sp, dst: SlotSpan, src: SlotSpan, len: u16) { + debug_assert!(dst.head() >= src.head()); + let dst = dst.iter(len); + let src = src.iter(len); + for (dst, src) in dst.into_iter().zip(src.into_iter()).rev() { + let value: u64 = get_value(src, sp); + set_value(sp, dst, value); + } +} + +pub fn extract_mem0(store: &mut PrunedStore, instance: Inst) -> (Mem0Ptr, Mem0Len) { + let instance = unsafe { instance.as_ref() }; + let Some(memory) = instance.get_memory(0) else { + return (Mem0Ptr::from([].as_mut_ptr()), Mem0Len::from(0)); + }; + let mem0 = resolve_memory_mut(store, &memory).data_mut(); + let mem0_ptr = mem0.as_mut_ptr(); + let mem0_len = mem0.len(); + (Mem0Ptr::from(mem0_ptr), Mem0Len::from(mem0_len)) +} + +pub fn memory_bytes<'a>( + memory: index::Memory, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, + state: &'a mut VmState, +) -> &'a mut [u8] { + if memory.is_default() { + return mem0_bytes::<'a>(mem0, mem0_len); + } + let memory = fetch_memory(instance, memory); + resolve_memory_mut(state.store, &memory).data_mut() +} + +pub fn memory_slice(memory: &CoreMemory, pos: usize, len: usize) -> Result<&[u8], TrapCode> { + memory + .data() + .get(pos..) + .and_then(|memory| memory.get(..len)) + .ok_or(TrapCode::MemoryOutOfBounds) +} + +pub fn memory_slice_mut( + memory: &mut CoreMemory, + pos: usize, + len: usize, +) -> Result<&mut [u8], TrapCode> { + memory + .data_mut() + .get_mut(pos..) + .and_then(|memory| memory.get_mut(..len)) + .ok_or(TrapCode::MemoryOutOfBounds) +} + +pub fn offset_ip(ip: Ip, offset: BranchOffset) -> Ip { + unsafe { ip.offset(i32::from(offset) as isize) } +} + +macro_rules! impl_fetch_from_instance { + ( + $( fn $fn:ident($param:ident: $ty:ty) -> $ret:ty = $getter:expr );* $(;)? + ) => { + $( + pub fn $fn(instance: Inst, $param: $ty) -> $ret { + let instance = unsafe { instance.as_ref() }; + let index = ::core::primitive::u32::from($param); + let Some($param) = $getter(instance, index) else { + unsafe { + $crate::engine::utils::unreachable_unchecked!( + ::core::concat!("missing ", ::core::stringify!($param), " at: {:?}"), + index, + ) + } + }; + $param + } + )* + }; +} +impl_fetch_from_instance! { + fn fetch_data(func: index::Data) -> DataSegment = InstanceEntity::get_data_segment; + fn fetch_elem(func: index::Elem) -> ElementSegment = InstanceEntity::get_element_segment; + fn fetch_func(func: index::Func) -> Func = InstanceEntity::get_func; + fn fetch_global(global: index::Global) -> Global = InstanceEntity::get_global; + fn fetch_memory(memory: index::Memory) -> Memory = InstanceEntity::get_memory; + fn fetch_table(table: index::Table) -> Table = InstanceEntity::get_table; + fn fetch_func_type(func_type: index::FuncType) -> DedupFuncType = { + |instance: &InstanceEntity, index: u32| instance.get_signature(index).copied() + }; +} + +macro_rules! impl_resolve_from_store { + ( + $( fn $fn:ident($param:ident: $ty:ty) -> $ret:ty = $getter:expr );* $(;)? + ) => { + $( + pub fn $fn<'a>(store: &'a mut PrunedStore, $param: $ty) -> $ret { + match $getter(store.inner_mut(), $param) { + ::core::result::Result::Ok($param) => $param, + ::core::result::Result::Err(error) => unsafe { + $crate::engine::utils::unreachable_unchecked!( + ::core::concat!("could not resolve stored ", ::core::stringify!($param), ": {:?}"), + error, + ) + }, + } + } + )* + }; +} +impl_resolve_from_store! { + // fn resolve_elem(elem: &ElementSegment) -> &'a CoreElementSegment = StoreInner::try_resolve_element; + fn resolve_func(func: &Func) -> &'a FuncEntity = StoreInner::try_resolve_func; + fn resolve_global(global: &Global) -> &'a CoreGlobal = StoreInner::try_resolve_global; + fn resolve_memory(memory: &Memory) -> &'a CoreMemory = StoreInner::try_resolve_memory; + fn resolve_table(table: &Table) -> &'a CoreTable = StoreInner::try_resolve_table; + fn resolve_instance(func: &Instance) -> &'a InstanceEntity = StoreInner::try_resolve_instance; + // fn resolve_func_type(func_type: DedupFuncType) -> DedupFuncType = StoreInner::resolve_func_type; + + fn resolve_elem_mut(elem: &ElementSegment) -> &'a mut CoreElementSegment = StoreInner::try_resolve_element_mut; + fn resolve_data_mut(data: &DataSegment) -> &'a mut DataSegmentEntity = StoreInner::try_resolve_data_mut; + fn resolve_global_mut(global: &Global) -> &'a mut CoreGlobal = StoreInner::try_resolve_global_mut; + fn resolve_memory_mut(memory: &Memory) -> &'a mut CoreMemory = StoreInner::try_resolve_memory_mut; + fn resolve_table_mut(table: &Table) -> &'a mut CoreTable = StoreInner::try_resolve_table_mut; +} + +pub fn resolve_indirect_func( + index: Slot, + table: index::Table, + func_type: index::FuncType, + state: &mut VmState<'_>, + sp: Sp, + instance: Inst, +) -> Result { + let index = get_value(index, sp); + let table = fetch_table(instance, table); + let table = resolve_table(state.store, &table); + let funcref = table + .get_untyped(index) + .map(>::from) + .ok_or(TrapCode::TableOutOfBounds)?; + let func = funcref.val().ok_or(TrapCode::IndirectCallToNull)?; + let actual_fnty = resolve_func(state.store, func).ty_dedup(); + let expected_fnty = fetch_func_type(instance, func_type); + if expected_fnty.ne(actual_fnty) { + return Err(TrapCode::BadSignature); + } + Ok(*func) +} + +pub fn set_global(global: index::Global, value: UntypedVal, state: &mut VmState, instance: Inst) { + let global = fetch_global(instance, global); + let global = resolve_global_mut(state.store, &global); + let mut value_ptr = global.get_untyped_ptr(); + let global_ref = unsafe { value_ptr.as_mut() }; + *global_ref = value; +} + +pub fn update_instance( + store: &mut PrunedStore, + instance: Inst, + new_instance: Inst, + mem0: Mem0Ptr, + mem0_len: Mem0Len, +) -> (Inst, Mem0Ptr, Mem0Len) { + if new_instance == instance { + return (instance, mem0, mem0_len); + } + let (mem0, mem0_len) = extract_mem0(store, new_instance); + (new_instance, mem0, mem0_len) +} + +pub fn call_wasm( + state: &mut VmState, + caller_ip: Ip, + params: BoundedSlotSpan, + func: EngineFunc, + instance: Option, +) -> Control<(Ip, Sp), Break> { + let (callee_ip, size) = compile_or_get_func!(state, func); + let callee_sp = state + .stack + .push_frame(Some(caller_ip), callee_ip, params, size, instance) + .into_control()?; + Control::Continue((callee_ip, callee_sp)) +} + +pub fn return_call_wasm( + state: &mut VmState, + params: BoundedSlotSpan, + func: EngineFunc, + instance: Option, +) -> Control<(Ip, Sp), Break> { + let (callee_ip, size) = compile_or_get_func!(state, func); + let callee_sp = state + .stack + .replace_frame(callee_ip, params, size, instance) + .into_control()?; + Control::Continue((callee_ip, callee_sp)) +} + +pub fn call_host( + state: &mut VmState, + func: Func, + caller_ip: Option, + host_func: HostFuncEntity, + params: BoundedSlotSpan, + instance: Option, + call_hooks: CallHooks, +) -> Control { + debug_assert_eq!(params.len(), host_func.len_params()); + let trampoline = *host_func.trampoline(); + let (sp, params_results) = state + .stack + .prepare_host_frame(caller_ip, params, host_func.len_results()) + .into_control()?; + match state + .store + .call_host_func(trampoline, instance, params_results, call_hooks) + { + Ok(()) => {} + Err(StoreError::External(error)) => { + done!(state, DoneReason::host_error(error, func, params.span())) + } + Err(StoreError::Internal(error)) => unsafe { + unreachable_unchecked!( + "internal interpreter error while executing host function: {error}" + ) + }, + } + Control::Continue(sp) +} + +pub fn return_call_host( + state: &mut VmState, + func: Func, + host_func: HostFuncEntity, + params: BoundedSlotSpan, + instance: Inst, +) -> Control<(Ip, Sp, Inst), Break> { + debug_assert_eq!(params.len(), host_func.len_params()); + let trampoline = *host_func.trampoline(); + let (control, params_results) = state + .stack + .return_prepare_host_frame(params, host_func.len_results(), instance) + .into_control()?; + match state + .store + .call_host_func(trampoline, Some(instance), params_results, CallHooks::Call) + { + Ok(()) => {} + Err(StoreError::External(error)) => { + // Note: we won't allow resumption in case the execution would + // have returned with this the host function tail call. + let reason = match control { + Control::Continue(_) => DoneReason::host_error(error, func, params.span()), + Control::Break(_) => DoneReason::error(error), + }; + done!(state, reason) + } + Err(StoreError::Internal(error)) => unsafe { + unreachable_unchecked!( + "internal interpreter error while executing host function: {error}" + ) + }, + } + match control { + Control::Continue((ip, sp, instance)) => Control::Continue((ip, sp, instance)), + Control::Break(sp) => done!(state, DoneReason::Return(sp)), + } +} + +pub fn call_wasm_or_host( + state: &mut VmState, + caller_ip: Ip, + func: Func, + params: BoundedSlotSpan, + mem0: Mem0Ptr, + mem0_len: Mem0Len, + instance: Inst, +) -> Control<(Ip, Sp, Mem0Ptr, Mem0Len, Inst), Break> { + let func_entity = resolve_func(state.store, &func); + let next_state = match func_entity { + FuncEntity::Wasm(wasm_func) => { + let func = wasm_func.func_body(); + let callee_instance = *wasm_func.instance(); + let callee_instance: Inst = resolve_instance(state.store, &callee_instance).into(); + let (callee_ip, callee_sp) = + call_wasm(state, caller_ip, params, func, Some(callee_instance))?; + let (instance, mem0, mem0_len) = + update_instance(state.store, instance, callee_instance, mem0, mem0_len); + (callee_ip, callee_sp, mem0, mem0_len, instance) + } + FuncEntity::Host(host_func) => { + let host_func = *host_func; + let sp = call_host( + state, + func, + Some(caller_ip), + host_func, + params, + Some(instance), + CallHooks::Call, + )?; + (caller_ip, sp, mem0, mem0_len, instance) + } + }; + Control::Continue(next_state) +} diff --git a/crates/wasmi/src/engine/executor/instr_ptr.rs b/crates/wasmi/src/engine/executor/instr_ptr.rs deleted file mode 100644 index e83b87d54e8..00000000000 --- a/crates/wasmi/src/engine/executor/instr_ptr.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::ir::Op; - -/// The instruction pointer to the instruction of a function on the call stack. -#[derive(Debug, Copy, Clone)] -pub struct InstructionPtr { - /// The pointer to the instruction. - ptr: *const Op, -} - -/// It is safe to send an [`InstructionPtr`] to another thread. -/// -/// The access to the pointed-to [`Op`] is read-only and -/// [`Op`] itself is [`Send`]. -/// -/// However, it is not safe to share an [`InstructionPtr`] between threads -/// due to their [`InstructionPtr::offset`] method which relinks the -/// internal pointer and is not synchronized. -unsafe impl Send for InstructionPtr {} - -impl InstructionPtr { - /// Creates a new [`InstructionPtr`] for `instr`. - #[inline] - pub fn new(ptr: *const Op) -> Self { - Self { ptr } - } - - /// Offset the [`InstructionPtr`] by the given value. - /// - /// # Safety - /// - /// The caller is responsible for calling this method only with valid - /// offset values so that the [`InstructionPtr`] never points out of valid - /// bounds of the instructions of the same compiled Wasm function. - #[inline(always)] - pub fn offset(&mut self, by: isize) { - // SAFETY: Within Wasm bytecode execution we are guaranteed by - // Wasm validation and Wasmi codegen to never run out - // of valid bounds using this method. - self.ptr = unsafe { self.ptr.offset(by) }; - } - - #[inline(always)] - pub fn add(&mut self, delta: usize) { - // SAFETY: Within Wasm bytecode execution we are guaranteed by - // Wasm validation and Wasmi codegen to never run out - // of valid bounds using this method. - self.ptr = unsafe { self.ptr.add(delta) }; - } - - /// Returns a shared reference to the currently pointed at [`Op`]. - /// - /// # Safety - /// - /// The caller is responsible for calling this method only when it is - /// guaranteed that the [`InstructionPtr`] is validly pointing inside - /// the boundaries of its associated compiled Wasm function. - #[inline(always)] - pub fn get(&self) -> &Op { - // SAFETY: Within Wasm bytecode execution we are guaranteed by - // Wasm validation and Wasmi codegen to never run out - // of valid bounds using this method. - unsafe { &*self.ptr } - } -} diff --git a/crates/wasmi/src/engine/executor/instrs.rs b/crates/wasmi/src/engine/executor/instrs.rs deleted file mode 100644 index a3eeb74dc27..00000000000 --- a/crates/wasmi/src/engine/executor/instrs.rs +++ /dev/null @@ -1,2717 +0,0 @@ -pub use self::call::dispatch_host_func; -use super::{cache::CachedInstance, InstructionPtr, Stack}; -use crate::{ - core::{hint, wasm, ReadAs, UntypedVal, WriteAs}, - engine::{ - code_map::CodeMap, - executor::stack::{CallFrame, FrameSlots, ValueStack}, - utils::unreachable_unchecked, - DedupFuncType, - EngineFunc, - }, - ir::{index, BlockFuel, Const16, Offset64Hi, Op, ShiftAmount, Slot}, - memory::DataSegment, - store::{PrunedStore, StoreInner}, - table::ElementSegment, - Error, - Func, - Global, - Memory, - Ref, - Table, - TrapCode, -}; - -#[cfg(doc)] -use crate::Instance; - -#[macro_use] -mod utils; - -#[cfg(feature = "simd")] -mod simd; - -mod binary; -mod branch; -mod call; -mod comparison; -mod conversion; -mod copy; -mod global; -mod load; -mod memory; -mod return_; -mod select; -mod store; -mod table; -mod unary; -mod wide_arithmetic; - -macro_rules! forward_return { - ($expr:expr) => {{ - if hint::unlikely($expr.is_break()) { - return Ok(()); - } - }}; -} - -/// Tells if execution loop shall continue or break (return) to the execution's caller. -type ControlFlow = ::core::ops::ControlFlow<(), ()>; - -/// Executes compiled function instructions until execution returns from the root function. -/// -/// # Errors -/// -/// If the execution encounters a trap. -#[inline(never)] -pub fn execute_instrs<'engine>( - store: &mut PrunedStore, - stack: &'engine mut Stack, - code_map: &'engine CodeMap, -) -> Result<(), Error> { - let instance = stack.calls.instance_expect(); - let cache = CachedInstance::new(store.inner_mut(), instance); - let mut executor = Executor::new(stack, code_map, cache); - if let Err(error) = executor.execute(store) { - if error.is_out_of_fuel() { - if let Some(frame) = executor.stack.calls.peek_mut() { - // Note: we need to update the instruction pointer to make it possible to - // resume execution at the current instruction after running out of fuel. - frame.update_instr_ptr(executor.ip); - } - } - return Err(error); - } - Ok(()) -} - -/// An execution context for executing a Wasmi function frame. -#[derive(Debug)] -struct Executor<'engine> { - /// Stores the value stack of live values on the Wasm stack. - sp: FrameSlots, - /// The pointer to the currently executed instruction. - ip: InstructionPtr, - /// The cached instance and instance related data. - cache: CachedInstance, - /// The value and call stacks. - stack: &'engine mut Stack, - /// The static resources of an [`Engine`]. - /// - /// [`Engine`]: crate::Engine - code_map: &'engine CodeMap, -} - -impl<'engine> Executor<'engine> { - /// Creates a new [`Executor`] for executing a Wasmi function frame. - #[inline(always)] - pub fn new( - stack: &'engine mut Stack, - code_map: &'engine CodeMap, - cache: CachedInstance, - ) -> Self { - let frame = stack - .calls - .peek() - .expect("must have call frame on the call stack"); - // Safety: We are using the frame's own base offset as input because it is - // guaranteed by the Wasm validation and translation phase to be - // valid for all register indices used by the associated function body. - let sp = unsafe { stack.values.stack_ptr_at(frame.base_offset()) }; - let ip = frame.instr_ptr(); - Self { - sp, - ip, - cache, - stack, - code_map, - } - } - - /// Executes the function frame until it returns or traps. - #[inline(always)] - fn execute(&mut self, store: &mut PrunedStore) -> Result<(), Error> { - use Op as Instr; - loop { - match *self.ip.get() { - Instr::Trap { trap_code } => self.execute_trap(trap_code)?, - Instr::ConsumeFuel { block_fuel } => { - self.execute_consume_fuel(store.inner_mut(), block_fuel)? - } - Instr::Return => { - forward_return!(self.execute_return(store.inner_mut())) - } - Instr::ReturnSlot { value } => { - forward_return!(self.execute_return_reg(store.inner_mut(), value)) - } - Instr::ReturnSlot2 { values } => { - forward_return!(self.execute_return_reg2(store.inner_mut(), values)) - } - Instr::ReturnSlot3 { values } => { - forward_return!(self.execute_return_reg3(store.inner_mut(), values)) - } - Instr::ReturnImm32 { value } => { - forward_return!(self.execute_return_imm32(store.inner_mut(), value)) - } - Instr::ReturnI64Imm32 { value } => { - forward_return!(self.execute_return_i64imm32(store.inner_mut(), value)) - } - Instr::ReturnF64Imm32 { value } => { - forward_return!(self.execute_return_f64imm32(store.inner_mut(), value)) - } - Instr::ReturnSpan { values } => { - forward_return!(self.execute_return_span(store.inner_mut(), values)) - } - Instr::ReturnMany { values } => { - forward_return!(self.execute_return_many(store.inner_mut(), values)) - } - Instr::Branch { offset } => self.execute_branch(offset), - Instr::BranchTable0 { index, len_targets } => { - self.execute_branch_table_0(index, len_targets) - } - Instr::BranchTableSpan { index, len_targets } => { - self.execute_branch_table_span(index, len_targets) - } - Instr::BranchCmpFallback { lhs, rhs, params } => { - self.execute_branch_cmp_fallback(lhs, rhs, params) - } - Instr::BranchI32And { lhs, rhs, offset } => { - self.execute_branch_i32_and(lhs, rhs, offset) - } - Instr::BranchI32AndImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_and_imm16(lhs, rhs, offset) - } - Instr::BranchI32Or { lhs, rhs, offset } => { - self.execute_branch_i32_or(lhs, rhs, offset) - } - Instr::BranchI32OrImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_or_imm16(lhs, rhs, offset) - } - Instr::BranchI32Nand { lhs, rhs, offset } => { - self.execute_branch_i32_nand(lhs, rhs, offset) - } - Instr::BranchI32NandImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_nand_imm16(lhs, rhs, offset) - } - Instr::BranchI32Nor { lhs, rhs, offset } => { - self.execute_branch_i32_nor(lhs, rhs, offset) - } - Instr::BranchI32NorImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_nor_imm16(lhs, rhs, offset) - } - Instr::BranchI32Eq { lhs, rhs, offset } => { - self.execute_branch_i32_eq(lhs, rhs, offset) - } - Instr::BranchI32EqImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_eq_imm16(lhs, rhs, offset) - } - Instr::BranchI32Ne { lhs, rhs, offset } => { - self.execute_branch_i32_ne(lhs, rhs, offset) - } - Instr::BranchI32NeImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_ne_imm16(lhs, rhs, offset) - } - Instr::BranchI32LtS { lhs, rhs, offset } => { - self.execute_branch_i32_lt_s(lhs, rhs, offset) - } - Instr::BranchI32LtSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LtSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI32LtU { lhs, rhs, offset } => { - self.execute_branch_i32_lt_u(lhs, rhs, offset) - } - Instr::BranchI32LtUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LtUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI32LeS { lhs, rhs, offset } => { - self.execute_branch_i32_le_s(lhs, rhs, offset) - } - Instr::BranchI32LeSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LeSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI32LeU { lhs, rhs, offset } => { - self.execute_branch_i32_le_u(lhs, rhs, offset) - } - Instr::BranchI32LeUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LeUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64And { lhs, rhs, offset } => { - self.execute_branch_i64_and(lhs, rhs, offset) - } - Instr::BranchI64AndImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_and_imm16(lhs, rhs, offset) - } - Instr::BranchI64Or { lhs, rhs, offset } => { - self.execute_branch_i64_or(lhs, rhs, offset) - } - Instr::BranchI64OrImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_or_imm16(lhs, rhs, offset) - } - Instr::BranchI64Nand { lhs, rhs, offset } => { - self.execute_branch_i64_nand(lhs, rhs, offset) - } - Instr::BranchI64NandImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_nand_imm16(lhs, rhs, offset) - } - Instr::BranchI64Nor { lhs, rhs, offset } => { - self.execute_branch_i64_nor(lhs, rhs, offset) - } - Instr::BranchI64NorImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_nor_imm16(lhs, rhs, offset) - } - Instr::BranchI64Eq { lhs, rhs, offset } => { - self.execute_branch_i64_eq(lhs, rhs, offset) - } - Instr::BranchI64EqImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_eq_imm16(lhs, rhs, offset) - } - Instr::BranchI64Ne { lhs, rhs, offset } => { - self.execute_branch_i64_ne(lhs, rhs, offset) - } - Instr::BranchI64NeImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_ne_imm16(lhs, rhs, offset) - } - Instr::BranchI64LtS { lhs, rhs, offset } => { - self.execute_branch_i64_lt_s(lhs, rhs, offset) - } - Instr::BranchI64LtSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LtSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64LtU { lhs, rhs, offset } => { - self.execute_branch_i64_lt_u(lhs, rhs, offset) - } - Instr::BranchI64LtUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LtUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64LeS { lhs, rhs, offset } => { - self.execute_branch_i64_le_s(lhs, rhs, offset) - } - Instr::BranchI64LeSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LeSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64LeU { lhs, rhs, offset } => { - self.execute_branch_i64_le_u(lhs, rhs, offset) - } - Instr::BranchI64LeUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LeUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchF32Eq { lhs, rhs, offset } => { - self.execute_branch_f32_eq(lhs, rhs, offset) - } - Instr::BranchF32Ne { lhs, rhs, offset } => { - self.execute_branch_f32_ne(lhs, rhs, offset) - } - Instr::BranchF32Lt { lhs, rhs, offset } => { - self.execute_branch_f32_lt(lhs, rhs, offset) - } - Instr::BranchF32Le { lhs, rhs, offset } => { - self.execute_branch_f32_le(lhs, rhs, offset) - } - Instr::BranchF32NotLt { lhs, rhs, offset } => { - self.execute_branch_f32_not_lt(lhs, rhs, offset) - } - Instr::BranchF32NotLe { lhs, rhs, offset } => { - self.execute_branch_f32_not_le(lhs, rhs, offset) - } - Instr::BranchF64Eq { lhs, rhs, offset } => { - self.execute_branch_f64_eq(lhs, rhs, offset) - } - Instr::BranchF64Ne { lhs, rhs, offset } => { - self.execute_branch_f64_ne(lhs, rhs, offset) - } - Instr::BranchF64Lt { lhs, rhs, offset } => { - self.execute_branch_f64_lt(lhs, rhs, offset) - } - Instr::BranchF64Le { lhs, rhs, offset } => { - self.execute_branch_f64_le(lhs, rhs, offset) - } - Instr::BranchF64NotLt { lhs, rhs, offset } => { - self.execute_branch_f64_not_lt(lhs, rhs, offset) - } - Instr::BranchF64NotLe { lhs, rhs, offset } => { - self.execute_branch_f64_not_le(lhs, rhs, offset) - } - Instr::Copy { result, value } => self.execute_copy(result, value), - Instr::Copy2 { results, values } => self.execute_copy_2(results, values), - Instr::CopyImm32 { result, value } => self.execute_copy_imm32(result, value), - Instr::CopyI64Imm32 { result, value } => self.execute_copy_i64imm32(result, value), - Instr::CopyF64Imm32 { result, value } => self.execute_copy_f64imm32(result, value), - Instr::CopySpan { - results, - values, - len, - } => self.execute_copy_span(results, values, len), - Instr::CopyMany { results, values } => self.execute_copy_many(results, values), - Instr::ReturnCallInternal0 { func } => { - self.execute_return_call_internal_0(store.inner_mut(), EngineFunc::from(func))? - } - Instr::ReturnCallInternal { func } => { - self.execute_return_call_internal(store.inner_mut(), EngineFunc::from(func))? - } - Instr::ReturnCallImported0 { func } => { - forward_return!(self.execute_return_call_imported_0(store, func)?) - } - Instr::ReturnCallImported { func } => { - forward_return!(self.execute_return_call_imported(store, func)?) - } - Instr::ReturnCallIndirect0 { func_type } => { - forward_return!(self.execute_return_call_indirect_0(store, func_type)?) - } - Instr::ReturnCallIndirect0Imm16 { func_type } => { - forward_return!(self.execute_return_call_indirect_0_imm16(store, func_type)?) - } - Instr::ReturnCallIndirect { func_type } => { - forward_return!(self.execute_return_call_indirect(store, func_type)?) - } - Instr::ReturnCallIndirectImm16 { func_type } => { - forward_return!(self.execute_return_call_indirect_imm16(store, func_type)?) - } - Instr::CallInternal0 { results, func } => self.execute_call_internal_0( - store.inner_mut(), - results, - EngineFunc::from(func), - )?, - Instr::CallInternal { results, func } => { - self.execute_call_internal(store.inner_mut(), results, EngineFunc::from(func))? - } - Instr::CallImported0 { results, func } => { - self.execute_call_imported_0(store, results, func)? - } - Instr::CallImported { results, func } => { - self.execute_call_imported(store, results, func)? - } - Instr::CallIndirect0 { results, func_type } => { - self.execute_call_indirect_0(store, results, func_type)? - } - Instr::CallIndirect0Imm16 { results, func_type } => { - self.execute_call_indirect_0_imm16(store, results, func_type)? - } - Instr::CallIndirect { results, func_type } => { - self.execute_call_indirect(store, results, func_type)? - } - Instr::CallIndirectImm16 { results, func_type } => { - self.execute_call_indirect_imm16(store, results, func_type)? - } - Instr::SelectI32Eq { result, lhs, rhs } => { - self.execute_select_i32_eq(result, lhs, rhs) - } - Instr::SelectI32EqImm16 { result, lhs, rhs } => { - self.execute_select_i32_eq_imm16(result, lhs, rhs) - } - Instr::SelectI32LtS { result, lhs, rhs } => { - self.execute_select_i32_lt_s(result, lhs, rhs) - } - Instr::SelectI32LtSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32LtU { result, lhs, rhs } => { - self.execute_select_i32_lt_u(result, lhs, rhs) - } - Instr::SelectI32LtUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32LeS { result, lhs, rhs } => { - self.execute_select_i32_le_s(result, lhs, rhs) - } - Instr::SelectI32LeSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32LeU { result, lhs, rhs } => { - self.execute_select_i32_le_u(result, lhs, rhs) - } - Instr::SelectI32LeUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32And { result, lhs, rhs } => { - self.execute_select_i32_and(result, lhs, rhs) - } - Instr::SelectI32AndImm16 { result, lhs, rhs } => { - self.execute_select_i32_and_imm16(result, lhs, rhs) - } - Instr::SelectI32Or { result, lhs, rhs } => { - self.execute_select_i32_or(result, lhs, rhs) - } - Instr::SelectI32OrImm16 { result, lhs, rhs } => { - self.execute_select_i32_or_imm16(result, lhs, rhs) - } - Instr::SelectI64Eq { result, lhs, rhs } => { - self.execute_select_i64_eq(result, lhs, rhs) - } - Instr::SelectI64EqImm16 { result, lhs, rhs } => { - self.execute_select_i64_eq_imm16(result, lhs, rhs) - } - Instr::SelectI64LtS { result, lhs, rhs } => { - self.execute_select_i64_lt_s(result, lhs, rhs) - } - Instr::SelectI64LtSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64LtU { result, lhs, rhs } => { - self.execute_select_i64_lt_u(result, lhs, rhs) - } - Instr::SelectI64LtUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64LeS { result, lhs, rhs } => { - self.execute_select_i64_le_s(result, lhs, rhs) - } - Instr::SelectI64LeSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64LeU { result, lhs, rhs } => { - self.execute_select_i64_le_u(result, lhs, rhs) - } - Instr::SelectI64LeUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64And { result, lhs, rhs } => { - self.execute_select_i64_and(result, lhs, rhs) - } - Instr::SelectI64AndImm16 { result, lhs, rhs } => { - self.execute_select_i64_and_imm16(result, lhs, rhs) - } - Instr::SelectI64Or { result, lhs, rhs } => { - self.execute_select_i64_or(result, lhs, rhs) - } - Instr::SelectI64OrImm16 { result, lhs, rhs } => { - self.execute_select_i64_or_imm16(result, lhs, rhs) - } - Instr::SelectF32Eq { result, lhs, rhs } => { - self.execute_select_f32_eq(result, lhs, rhs) - } - Instr::SelectF32Lt { result, lhs, rhs } => { - self.execute_select_f32_lt(result, lhs, rhs) - } - Instr::SelectF32Le { result, lhs, rhs } => { - self.execute_select_f32_le(result, lhs, rhs) - } - Instr::SelectF64Eq { result, lhs, rhs } => { - self.execute_select_f64_eq(result, lhs, rhs) - } - Instr::SelectF64Lt { result, lhs, rhs } => { - self.execute_select_f64_lt(result, lhs, rhs) - } - Instr::SelectF64Le { result, lhs, rhs } => { - self.execute_select_f64_le(result, lhs, rhs) - } - Instr::RefFunc { result, func } => self.execute_ref_func(result, func), - Instr::GlobalGet { result, global } => { - self.execute_global_get(store.inner(), result, global) - } - Instr::GlobalSet { global, input } => { - self.execute_global_set(store.inner_mut(), global, input) - } - Instr::GlobalSetI32Imm16 { global, input } => { - self.execute_global_set_i32imm16(store.inner_mut(), global, input) - } - Instr::GlobalSetI64Imm16 { global, input } => { - self.execute_global_set_i64imm16(store.inner_mut(), global, input) - } - Instr::Load32 { result, offset_lo } => { - self.execute_load32(store.inner(), result, offset_lo)? - } - Instr::Load32At { result, address } => { - self.execute_load32_at(store.inner(), result, address)? - } - Instr::Load32Offset16 { - result, - ptr, - offset, - } => self.execute_load32_offset16(result, ptr, offset)?, - Instr::Load64 { result, offset_lo } => { - self.execute_load64(store.inner(), result, offset_lo)? - } - Instr::Load64At { result, address } => { - self.execute_load64_at(store.inner(), result, address)? - } - Instr::Load64Offset16 { - result, - ptr, - offset, - } => self.execute_load64_offset16(result, ptr, offset)?, - Instr::I32Load8s { result, offset_lo } => { - self.execute_i32_load8_s(store.inner(), result, offset_lo)? - } - Instr::I32Load8sAt { result, address } => { - self.execute_i32_load8_s_at(store.inner(), result, address)? - } - Instr::I32Load8sOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load8_s_offset16(result, ptr, offset)?, - Instr::I32Load8u { result, offset_lo } => { - self.execute_i32_load8_u(store.inner(), result, offset_lo)? - } - Instr::I32Load8uAt { result, address } => { - self.execute_i32_load8_u_at(store.inner(), result, address)? - } - Instr::I32Load8uOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load8_u_offset16(result, ptr, offset)?, - Instr::I32Load16s { result, offset_lo } => { - self.execute_i32_load16_s(store.inner(), result, offset_lo)? - } - Instr::I32Load16sAt { result, address } => { - self.execute_i32_load16_s_at(store.inner(), result, address)? - } - Instr::I32Load16sOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load16_s_offset16(result, ptr, offset)?, - Instr::I32Load16u { result, offset_lo } => { - self.execute_i32_load16_u(store.inner(), result, offset_lo)? - } - Instr::I32Load16uAt { result, address } => { - self.execute_i32_load16_u_at(store.inner(), result, address)? - } - Instr::I32Load16uOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load16_u_offset16(result, ptr, offset)?, - Instr::I64Load8s { result, offset_lo } => { - self.execute_i64_load8_s(store.inner(), result, offset_lo)? - } - Instr::I64Load8sAt { result, address } => { - self.execute_i64_load8_s_at(store.inner(), result, address)? - } - Instr::I64Load8sOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load8_s_offset16(result, ptr, offset)?, - Instr::I64Load8u { result, offset_lo } => { - self.execute_i64_load8_u(store.inner(), result, offset_lo)? - } - Instr::I64Load8uAt { result, address } => { - self.execute_i64_load8_u_at(store.inner(), result, address)? - } - Instr::I64Load8uOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load8_u_offset16(result, ptr, offset)?, - Instr::I64Load16s { result, offset_lo } => { - self.execute_i64_load16_s(store.inner(), result, offset_lo)? - } - Instr::I64Load16sAt { result, address } => { - self.execute_i64_load16_s_at(store.inner(), result, address)? - } - Instr::I64Load16sOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load16_s_offset16(result, ptr, offset)?, - Instr::I64Load16u { result, offset_lo } => { - self.execute_i64_load16_u(store.inner(), result, offset_lo)? - } - Instr::I64Load16uAt { result, address } => { - self.execute_i64_load16_u_at(store.inner(), result, address)? - } - Instr::I64Load16uOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load16_u_offset16(result, ptr, offset)?, - Instr::I64Load32s { result, offset_lo } => { - self.execute_i64_load32_s(store.inner(), result, offset_lo)? - } - Instr::I64Load32sAt { result, address } => { - self.execute_i64_load32_s_at(store.inner(), result, address)? - } - Instr::I64Load32sOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load32_s_offset16(result, ptr, offset)?, - Instr::I64Load32u { result, offset_lo } => { - self.execute_i64_load32_u(store.inner(), result, offset_lo)? - } - Instr::I64Load32uAt { result, address } => { - self.execute_i64_load32_u_at(store.inner(), result, address)? - } - Instr::I64Load32uOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load32_u_offset16(result, ptr, offset)?, - Instr::Store32 { ptr, offset_lo } => { - self.execute_store32(store.inner_mut(), ptr, offset_lo)? - } - Instr::Store32Offset16 { ptr, offset, value } => { - self.execute_store32_offset16(ptr, offset, value)? - } - Instr::Store32At { address, value } => { - self.execute_store32_at(store.inner_mut(), address, value)? - } - Instr::Store64 { ptr, offset_lo } => { - self.execute_store64(store.inner_mut(), ptr, offset_lo)? - } - Instr::Store64Offset16 { ptr, offset, value } => { - self.execute_store64_offset16(ptr, offset, value)? - } - Instr::Store64At { address, value } => { - self.execute_store64_at(store.inner_mut(), address, value)? - } - Instr::I32StoreImm16 { ptr, offset_lo } => { - self.execute_i32_store_imm16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32StoreOffset16Imm16 { ptr, offset, value } => { - self.execute_i32_store_offset16_imm16(ptr, offset, value)? - } - Instr::I32StoreAtImm16 { address, value } => { - self.execute_i32_store_at_imm16(store.inner_mut(), address, value)? - } - Instr::I32Store8 { ptr, offset_lo } => { - self.execute_i32_store8(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store8Imm { ptr, offset_lo } => { - self.execute_i32_store8_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store8Offset16 { ptr, offset, value } => { - self.execute_i32_store8_offset16(ptr, offset, value)? - } - Instr::I32Store8Offset16Imm { ptr, offset, value } => { - self.execute_i32_store8_offset16_imm(ptr, offset, value)? - } - Instr::I32Store8At { address, value } => { - self.execute_i32_store8_at(store.inner_mut(), address, value)? - } - Instr::I32Store8AtImm { address, value } => { - self.execute_i32_store8_at_imm(store.inner_mut(), address, value)? - } - Instr::I32Store16 { ptr, offset_lo } => { - self.execute_i32_store16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store16Imm { ptr, offset_lo } => { - self.execute_i32_store16_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store16Offset16 { ptr, offset, value } => { - self.execute_i32_store16_offset16(ptr, offset, value)? - } - Instr::I32Store16Offset16Imm { ptr, offset, value } => { - self.execute_i32_store16_offset16_imm(ptr, offset, value)? - } - Instr::I32Store16At { address, value } => { - self.execute_i32_store16_at(store.inner_mut(), address, value)? - } - Instr::I32Store16AtImm { address, value } => { - self.execute_i32_store16_at_imm(store.inner_mut(), address, value)? - } - Instr::I64StoreImm16 { ptr, offset_lo } => { - self.execute_i64_store_imm16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64StoreOffset16Imm16 { ptr, offset, value } => { - self.execute_i64_store_offset16_imm16(ptr, offset, value)? - } - Instr::I64StoreAtImm16 { address, value } => { - self.execute_i64_store_at_imm16(store.inner_mut(), address, value)? - } - Instr::I64Store8 { ptr, offset_lo } => { - self.execute_i64_store8(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store8Imm { ptr, offset_lo } => { - self.execute_i64_store8_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store8Offset16 { ptr, offset, value } => { - self.execute_i64_store8_offset16(ptr, offset, value)? - } - Instr::I64Store8Offset16Imm { ptr, offset, value } => { - self.execute_i64_store8_offset16_imm(ptr, offset, value)? - } - Instr::I64Store8At { address, value } => { - self.execute_i64_store8_at(store.inner_mut(), address, value)? - } - Instr::I64Store8AtImm { address, value } => { - self.execute_i64_store8_at_imm(store.inner_mut(), address, value)? - } - Instr::I64Store16 { ptr, offset_lo } => { - self.execute_i64_store16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store16Imm { ptr, offset_lo } => { - self.execute_i64_store16_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store16Offset16 { ptr, offset, value } => { - self.execute_i64_store16_offset16(ptr, offset, value)? - } - Instr::I64Store16Offset16Imm { ptr, offset, value } => { - self.execute_i64_store16_offset16_imm(ptr, offset, value)? - } - Instr::I64Store16At { address, value } => { - self.execute_i64_store16_at(store.inner_mut(), address, value)? - } - Instr::I64Store16AtImm { address, value } => { - self.execute_i64_store16_at_imm(store.inner_mut(), address, value)? - } - Instr::I64Store32 { ptr, offset_lo } => { - self.execute_i64_store32(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store32Imm16 { ptr, offset_lo } => { - self.execute_i64_store32_imm16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store32Offset16 { ptr, offset, value } => { - self.execute_i64_store32_offset16(ptr, offset, value)? - } - Instr::I64Store32Offset16Imm16 { ptr, offset, value } => { - self.execute_i64_store32_offset16_imm16(ptr, offset, value)? - } - Instr::I64Store32At { address, value } => { - self.execute_i64_store32_at(store.inner_mut(), address, value)? - } - Instr::I64Store32AtImm16 { address, value } => { - self.execute_i64_store32_at_imm16(store.inner_mut(), address, value)? - } - Instr::I32Eq { result, lhs, rhs } => self.execute_i32_eq(result, lhs, rhs), - Instr::I32EqImm16 { result, lhs, rhs } => { - self.execute_i32_eq_imm16(result, lhs, rhs) - } - Instr::I32Ne { result, lhs, rhs } => self.execute_i32_ne(result, lhs, rhs), - Instr::I32NeImm16 { result, lhs, rhs } => { - self.execute_i32_ne_imm16(result, lhs, rhs) - } - Instr::I32LtS { result, lhs, rhs } => self.execute_i32_lt_s(result, lhs, rhs), - Instr::I32LtSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_lt_s_imm16_lhs(result, lhs, rhs) - } - Instr::I32LtSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::I32LtU { result, lhs, rhs } => self.execute_i32_lt_u(result, lhs, rhs), - Instr::I32LtUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_lt_u_imm16_lhs(result, lhs, rhs) - } - Instr::I32LtUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::I32LeS { result, lhs, rhs } => self.execute_i32_le_s(result, lhs, rhs), - Instr::I32LeSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_le_s_imm16_lhs(result, lhs, rhs) - } - Instr::I32LeSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::I32LeU { result, lhs, rhs } => self.execute_i32_le_u(result, lhs, rhs), - Instr::I32LeUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_le_u_imm16_lhs(result, lhs, rhs) - } - Instr::I32LeUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64Eq { result, lhs, rhs } => self.execute_i64_eq(result, lhs, rhs), - Instr::I64EqImm16 { result, lhs, rhs } => { - self.execute_i64_eq_imm16(result, lhs, rhs) - } - Instr::I64Ne { result, lhs, rhs } => self.execute_i64_ne(result, lhs, rhs), - Instr::I64NeImm16 { result, lhs, rhs } => { - self.execute_i64_ne_imm16(result, lhs, rhs) - } - Instr::I64LtS { result, lhs, rhs } => self.execute_i64_lt_s(result, lhs, rhs), - Instr::I64LtSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_lt_s_imm16_lhs(result, lhs, rhs) - } - Instr::I64LtSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::I64LtU { result, lhs, rhs } => self.execute_i64_lt_u(result, lhs, rhs), - Instr::I64LtUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_lt_u_imm16_lhs(result, lhs, rhs) - } - Instr::I64LtUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64LeS { result, lhs, rhs } => self.execute_i64_le_s(result, lhs, rhs), - Instr::I64LeSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_le_s_imm16_lhs(result, lhs, rhs) - } - Instr::I64LeSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::I64LeU { result, lhs, rhs } => self.execute_i64_le_u(result, lhs, rhs), - Instr::I64LeUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_le_u_imm16_lhs(result, lhs, rhs) - } - Instr::I64LeUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::F32Eq { result, lhs, rhs } => self.execute_f32_eq(result, lhs, rhs), - Instr::F32Ne { result, lhs, rhs } => self.execute_f32_ne(result, lhs, rhs), - Instr::F32Lt { result, lhs, rhs } => self.execute_f32_lt(result, lhs, rhs), - Instr::F32Le { result, lhs, rhs } => self.execute_f32_le(result, lhs, rhs), - Instr::F32NotLt { result, lhs, rhs } => self.execute_f32_not_lt(result, lhs, rhs), - Instr::F32NotLe { result, lhs, rhs } => self.execute_f32_not_le(result, lhs, rhs), - Instr::F64Eq { result, lhs, rhs } => self.execute_f64_eq(result, lhs, rhs), - Instr::F64Ne { result, lhs, rhs } => self.execute_f64_ne(result, lhs, rhs), - Instr::F64Lt { result, lhs, rhs } => self.execute_f64_lt(result, lhs, rhs), - Instr::F64Le { result, lhs, rhs } => self.execute_f64_le(result, lhs, rhs), - Instr::F64NotLt { result, lhs, rhs } => self.execute_f64_not_lt(result, lhs, rhs), - Instr::F64NotLe { result, lhs, rhs } => self.execute_f64_not_le(result, lhs, rhs), - Instr::I32Clz { result, input } => self.execute_i32_clz(result, input), - Instr::I32Ctz { result, input } => self.execute_i32_ctz(result, input), - Instr::I32Popcnt { result, input } => self.execute_i32_popcnt(result, input), - Instr::I32Add { result, lhs, rhs } => self.execute_i32_add(result, lhs, rhs), - Instr::I32AddImm16 { result, lhs, rhs } => { - self.execute_i32_add_imm16(result, lhs, rhs) - } - Instr::I32Sub { result, lhs, rhs } => self.execute_i32_sub(result, lhs, rhs), - Instr::I32SubImm16Lhs { result, lhs, rhs } => { - self.execute_i32_sub_imm16_lhs(result, lhs, rhs) - } - Instr::I32Mul { result, lhs, rhs } => self.execute_i32_mul(result, lhs, rhs), - Instr::I32MulImm16 { result, lhs, rhs } => { - self.execute_i32_mul_imm16(result, lhs, rhs) - } - Instr::I32DivS { result, lhs, rhs } => self.execute_i32_div_s(result, lhs, rhs)?, - Instr::I32DivSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_div_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I32DivSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_div_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I32DivU { result, lhs, rhs } => self.execute_i32_div_u(result, lhs, rhs)?, - Instr::I32DivUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_div_u_imm16_rhs(result, lhs, rhs) - } - Instr::I32DivUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_div_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I32RemS { result, lhs, rhs } => self.execute_i32_rem_s(result, lhs, rhs)?, - Instr::I32RemSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_rem_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I32RemSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_rem_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I32RemU { result, lhs, rhs } => self.execute_i32_rem_u(result, lhs, rhs)?, - Instr::I32RemUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_rem_u_imm16_rhs(result, lhs, rhs) - } - Instr::I32RemUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_rem_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I32BitAnd { result, lhs, rhs } => self.execute_i32_bitand(result, lhs, rhs), - Instr::I32BitAndImm16 { result, lhs, rhs } => { - self.execute_i32_bitand_imm16(result, lhs, rhs) - } - Instr::I32BitOr { result, lhs, rhs } => self.execute_i32_bitor(result, lhs, rhs), - Instr::I32BitOrImm16 { result, lhs, rhs } => { - self.execute_i32_bitor_imm16(result, lhs, rhs) - } - Instr::I32BitXor { result, lhs, rhs } => self.execute_i32_bitxor(result, lhs, rhs), - Instr::I32BitXorImm16 { result, lhs, rhs } => { - self.execute_i32_bitxor_imm16(result, lhs, rhs) - } - Instr::I32And { result, lhs, rhs } => self.execute_i32_and(result, lhs, rhs), - Instr::I32AndImm16 { result, lhs, rhs } => { - self.execute_i32_and_imm16(result, lhs, rhs) - } - Instr::I32Or { result, lhs, rhs } => self.execute_i32_or(result, lhs, rhs), - Instr::I32OrImm16 { result, lhs, rhs } => { - self.execute_i32_or_imm16(result, lhs, rhs) - } - Instr::I32Nand { result, lhs, rhs } => self.execute_i32_nand(result, lhs, rhs), - Instr::I32NandImm16 { result, lhs, rhs } => { - self.execute_i32_nand_imm16(result, lhs, rhs) - } - Instr::I32Nor { result, lhs, rhs } => self.execute_i32_nor(result, lhs, rhs), - Instr::I32NorImm16 { result, lhs, rhs } => { - self.execute_i32_nor_imm16(result, lhs, rhs) - } - Instr::I32Shl { result, lhs, rhs } => self.execute_i32_shl(result, lhs, rhs), - Instr::I32ShlBy { result, lhs, rhs } => self.execute_i32_shl_by(result, lhs, rhs), - Instr::I32ShlImm16 { result, lhs, rhs } => { - self.execute_i32_shl_imm16(result, lhs, rhs) - } - Instr::I32ShrU { result, lhs, rhs } => self.execute_i32_shr_u(result, lhs, rhs), - Instr::I32ShrUBy { result, lhs, rhs } => { - self.execute_i32_shr_u_by(result, lhs, rhs) - } - Instr::I32ShrUImm16 { result, lhs, rhs } => { - self.execute_i32_shr_u_imm16(result, lhs, rhs) - } - Instr::I32ShrS { result, lhs, rhs } => self.execute_i32_shr_s(result, lhs, rhs), - Instr::I32ShrSBy { result, lhs, rhs } => { - self.execute_i32_shr_s_by(result, lhs, rhs) - } - Instr::I32ShrSImm16 { result, lhs, rhs } => { - self.execute_i32_shr_s_imm16(result, lhs, rhs) - } - Instr::I32Rotl { result, lhs, rhs } => self.execute_i32_rotl(result, lhs, rhs), - Instr::I32RotlBy { result, lhs, rhs } => self.execute_i32_rotl_by(result, lhs, rhs), - Instr::I32RotlImm16 { result, lhs, rhs } => { - self.execute_i32_rotl_imm16(result, lhs, rhs) - } - Instr::I32Rotr { result, lhs, rhs } => self.execute_i32_rotr(result, lhs, rhs), - Instr::I32RotrBy { result, lhs, rhs } => self.execute_i32_rotr_by(result, lhs, rhs), - Instr::I32RotrImm16 { result, lhs, rhs } => { - self.execute_i32_rotr_imm16(result, lhs, rhs) - } - Instr::I64Clz { result, input } => self.execute_i64_clz(result, input), - Instr::I64Ctz { result, input } => self.execute_i64_ctz(result, input), - Instr::I64Popcnt { result, input } => self.execute_i64_popcnt(result, input), - Instr::I64Add { result, lhs, rhs } => self.execute_i64_add(result, lhs, rhs), - Instr::I64AddImm16 { result, lhs, rhs } => { - self.execute_i64_add_imm16(result, lhs, rhs) - } - Instr::I64Sub { result, lhs, rhs } => self.execute_i64_sub(result, lhs, rhs), - Instr::I64SubImm16Lhs { result, lhs, rhs } => { - self.execute_i64_sub_imm16_lhs(result, lhs, rhs) - } - Instr::I64Mul { result, lhs, rhs } => self.execute_i64_mul(result, lhs, rhs), - Instr::I64MulImm16 { result, lhs, rhs } => { - self.execute_i64_mul_imm16(result, lhs, rhs) - } - Instr::I64DivS { result, lhs, rhs } => self.execute_i64_div_s(result, lhs, rhs)?, - Instr::I64DivSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_div_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I64DivSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_div_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I64DivU { result, lhs, rhs } => self.execute_i64_div_u(result, lhs, rhs)?, - Instr::I64DivUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_div_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64DivUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_div_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I64RemS { result, lhs, rhs } => self.execute_i64_rem_s(result, lhs, rhs)?, - Instr::I64RemSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_rem_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I64RemSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_rem_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I64RemU { result, lhs, rhs } => self.execute_i64_rem_u(result, lhs, rhs)?, - Instr::I64RemUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_rem_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64RemUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_rem_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I64BitAnd { result, lhs, rhs } => self.execute_i64_bitand(result, lhs, rhs), - Instr::I64BitAndImm16 { result, lhs, rhs } => { - self.execute_i64_bitand_imm16(result, lhs, rhs) - } - Instr::I64BitOr { result, lhs, rhs } => self.execute_i64_bitor(result, lhs, rhs), - Instr::I64BitOrImm16 { result, lhs, rhs } => { - self.execute_i64_bitor_imm16(result, lhs, rhs) - } - Instr::I64BitXor { result, lhs, rhs } => self.execute_i64_bitxor(result, lhs, rhs), - Instr::I64BitXorImm16 { result, lhs, rhs } => { - self.execute_i64_bitxor_imm16(result, lhs, rhs) - } - Instr::I64And { result, lhs, rhs } => self.execute_i64_and(result, lhs, rhs), - Instr::I64AndImm16 { result, lhs, rhs } => { - self.execute_i64_and_imm16(result, lhs, rhs) - } - Instr::I64Or { result, lhs, rhs } => self.execute_i64_or(result, lhs, rhs), - Instr::I64OrImm16 { result, lhs, rhs } => { - self.execute_i64_or_imm16(result, lhs, rhs) - } - Instr::I64Nand { result, lhs, rhs } => self.execute_i64_nand(result, lhs, rhs), - Instr::I64NandImm16 { result, lhs, rhs } => { - self.execute_i64_nand_imm16(result, lhs, rhs) - } - Instr::I64Nor { result, lhs, rhs } => self.execute_i64_nor(result, lhs, rhs), - Instr::I64NorImm16 { result, lhs, rhs } => { - self.execute_i64_nor_imm16(result, lhs, rhs) - } - Instr::I64Shl { result, lhs, rhs } => self.execute_i64_shl(result, lhs, rhs), - Instr::I64ShlBy { result, lhs, rhs } => self.execute_i64_shl_by(result, lhs, rhs), - Instr::I64ShlImm16 { result, lhs, rhs } => { - self.execute_i64_shl_imm16(result, lhs, rhs) - } - Instr::I64ShrU { result, lhs, rhs } => self.execute_i64_shr_u(result, lhs, rhs), - Instr::I64ShrUBy { result, lhs, rhs } => { - self.execute_i64_shr_u_by(result, lhs, rhs) - } - Instr::I64ShrUImm16 { result, lhs, rhs } => { - self.execute_i64_shr_u_imm16(result, lhs, rhs) - } - Instr::I64ShrS { result, lhs, rhs } => self.execute_i64_shr_s(result, lhs, rhs), - Instr::I64ShrSBy { result, lhs, rhs } => { - self.execute_i64_shr_s_by(result, lhs, rhs) - } - Instr::I64ShrSImm16 { result, lhs, rhs } => { - self.execute_i64_shr_s_imm16(result, lhs, rhs) - } - Instr::I64Rotl { result, lhs, rhs } => self.execute_i64_rotl(result, lhs, rhs), - Instr::I64RotlBy { result, lhs, rhs } => self.execute_i64_rotl_by(result, lhs, rhs), - Instr::I64RotlImm16 { result, lhs, rhs } => { - self.execute_i64_rotl_imm16(result, lhs, rhs) - } - Instr::I64Rotr { result, lhs, rhs } => self.execute_i64_rotr(result, lhs, rhs), - Instr::I64RotrBy { result, lhs, rhs } => self.execute_i64_rotr_by(result, lhs, rhs), - Instr::I64RotrImm16 { result, lhs, rhs } => { - self.execute_i64_rotr_imm16(result, lhs, rhs) - } - Instr::I64Add128 { results, lhs_lo } => self.execute_i64_add128(results, lhs_lo), - Instr::I64Sub128 { results, lhs_lo } => self.execute_i64_sub128(results, lhs_lo), - Instr::I64MulWideS { results, lhs, rhs } => { - self.execute_i64_mul_wide_s(results, lhs, rhs) - } - Instr::I64MulWideU { results, lhs, rhs } => { - self.execute_i64_mul_wide_u(results, lhs, rhs) - } - Instr::I32WrapI64 { result, input } => self.execute_i32_wrap_i64(result, input), - Instr::I32Extend8S { result, input } => self.execute_i32_extend8_s(result, input), - Instr::I32Extend16S { result, input } => self.execute_i32_extend16_s(result, input), - Instr::I64Extend8S { result, input } => self.execute_i64_extend8_s(result, input), - Instr::I64Extend16S { result, input } => self.execute_i64_extend16_s(result, input), - Instr::I64Extend32S { result, input } => self.execute_i64_extend32_s(result, input), - Instr::F32Abs { result, input } => self.execute_f32_abs(result, input), - Instr::F32Neg { result, input } => self.execute_f32_neg(result, input), - Instr::F32Ceil { result, input } => self.execute_f32_ceil(result, input), - Instr::F32Floor { result, input } => self.execute_f32_floor(result, input), - Instr::F32Trunc { result, input } => self.execute_f32_trunc(result, input), - Instr::F32Nearest { result, input } => self.execute_f32_nearest(result, input), - Instr::F32Sqrt { result, input } => self.execute_f32_sqrt(result, input), - Instr::F32Add { result, lhs, rhs } => self.execute_f32_add(result, lhs, rhs), - Instr::F32Sub { result, lhs, rhs } => self.execute_f32_sub(result, lhs, rhs), - Instr::F32Mul { result, lhs, rhs } => self.execute_f32_mul(result, lhs, rhs), - Instr::F32Div { result, lhs, rhs } => self.execute_f32_div(result, lhs, rhs), - Instr::F32Min { result, lhs, rhs } => self.execute_f32_min(result, lhs, rhs), - Instr::F32Max { result, lhs, rhs } => self.execute_f32_max(result, lhs, rhs), - Instr::F32Copysign { result, lhs, rhs } => { - self.execute_f32_copysign(result, lhs, rhs) - } - Instr::F32CopysignImm { result, lhs, rhs } => { - self.execute_f32_copysign_imm(result, lhs, rhs) - } - Instr::F64Abs { result, input } => self.execute_f64_abs(result, input), - Instr::F64Neg { result, input } => self.execute_f64_neg(result, input), - Instr::F64Ceil { result, input } => self.execute_f64_ceil(result, input), - Instr::F64Floor { result, input } => self.execute_f64_floor(result, input), - Instr::F64Trunc { result, input } => self.execute_f64_trunc(result, input), - Instr::F64Nearest { result, input } => self.execute_f64_nearest(result, input), - Instr::F64Sqrt { result, input } => self.execute_f64_sqrt(result, input), - Instr::F64Add { result, lhs, rhs } => self.execute_f64_add(result, lhs, rhs), - Instr::F64Sub { result, lhs, rhs } => self.execute_f64_sub(result, lhs, rhs), - Instr::F64Mul { result, lhs, rhs } => self.execute_f64_mul(result, lhs, rhs), - Instr::F64Div { result, lhs, rhs } => self.execute_f64_div(result, lhs, rhs), - Instr::F64Min { result, lhs, rhs } => self.execute_f64_min(result, lhs, rhs), - Instr::F64Max { result, lhs, rhs } => self.execute_f64_max(result, lhs, rhs), - Instr::F64Copysign { result, lhs, rhs } => { - self.execute_f64_copysign(result, lhs, rhs) - } - Instr::F64CopysignImm { result, lhs, rhs } => { - self.execute_f64_copysign_imm(result, lhs, rhs) - } - Instr::I32TruncF32S { result, input } => { - self.execute_i32_trunc_f32_s(result, input)? - } - Instr::I32TruncF32U { result, input } => { - self.execute_i32_trunc_f32_u(result, input)? - } - Instr::I32TruncF64S { result, input } => { - self.execute_i32_trunc_f64_s(result, input)? - } - Instr::I32TruncF64U { result, input } => { - self.execute_i32_trunc_f64_u(result, input)? - } - Instr::I64TruncF32S { result, input } => { - self.execute_i64_trunc_f32_s(result, input)? - } - Instr::I64TruncF32U { result, input } => { - self.execute_i64_trunc_f32_u(result, input)? - } - Instr::I64TruncF64S { result, input } => { - self.execute_i64_trunc_f64_s(result, input)? - } - Instr::I64TruncF64U { result, input } => { - self.execute_i64_trunc_f64_u(result, input)? - } - Instr::I32TruncSatF32S { result, input } => { - self.execute_i32_trunc_sat_f32_s(result, input) - } - Instr::I32TruncSatF32U { result, input } => { - self.execute_i32_trunc_sat_f32_u(result, input) - } - Instr::I32TruncSatF64S { result, input } => { - self.execute_i32_trunc_sat_f64_s(result, input) - } - Instr::I32TruncSatF64U { result, input } => { - self.execute_i32_trunc_sat_f64_u(result, input) - } - Instr::I64TruncSatF32S { result, input } => { - self.execute_i64_trunc_sat_f32_s(result, input) - } - Instr::I64TruncSatF32U { result, input } => { - self.execute_i64_trunc_sat_f32_u(result, input) - } - Instr::I64TruncSatF64S { result, input } => { - self.execute_i64_trunc_sat_f64_s(result, input) - } - Instr::I64TruncSatF64U { result, input } => { - self.execute_i64_trunc_sat_f64_u(result, input) - } - Instr::F32DemoteF64 { result, input } => self.execute_f32_demote_f64(result, input), - Instr::F64PromoteF32 { result, input } => { - self.execute_f64_promote_f32(result, input) - } - Instr::F32ConvertI32S { result, input } => { - self.execute_f32_convert_i32_s(result, input) - } - Instr::F32ConvertI32U { result, input } => { - self.execute_f32_convert_i32_u(result, input) - } - Instr::F32ConvertI64S { result, input } => { - self.execute_f32_convert_i64_s(result, input) - } - Instr::F32ConvertI64U { result, input } => { - self.execute_f32_convert_i64_u(result, input) - } - Instr::F64ConvertI32S { result, input } => { - self.execute_f64_convert_i32_s(result, input) - } - Instr::F64ConvertI32U { result, input } => { - self.execute_f64_convert_i32_u(result, input) - } - Instr::F64ConvertI64S { result, input } => { - self.execute_f64_convert_i64_s(result, input) - } - Instr::F64ConvertI64U { result, input } => { - self.execute_f64_convert_i64_u(result, input) - } - Instr::TableGet { result, index } => { - self.execute_table_get(store.inner(), result, index)? - } - Instr::TableGetImm { result, index } => { - self.execute_table_get_imm(store.inner(), result, index)? - } - Instr::TableSize { result, table } => { - self.execute_table_size(store.inner(), result, table) - } - Instr::TableSet { index, value } => { - self.execute_table_set(store.inner_mut(), index, value)? - } - Instr::TableSetAt { index, value } => { - self.execute_table_set_at(store.inner_mut(), index, value)? - } - Instr::TableCopy { dst, src, len } => { - self.execute_table_copy(store.inner_mut(), dst, src, len)? - } - Instr::TableInit { dst, src, len } => { - self.execute_table_init(store.inner_mut(), dst, src, len)? - } - Instr::TableFill { dst, len, value } => { - self.execute_table_fill(store.inner_mut(), dst, len, value)? - } - Instr::TableGrow { - result, - delta, - value, - } => self.execute_table_grow(store, result, delta, value)?, - Instr::ElemDrop { index } => self.execute_element_drop(store.inner_mut(), index), - Instr::DataDrop { index } => self.execute_data_drop(store.inner_mut(), index), - Instr::MemorySize { result, memory } => { - self.execute_memory_size(store.inner(), result, memory) - } - Instr::MemoryGrow { result, delta } => { - self.execute_memory_grow(store, result, delta)? - } - Instr::MemoryCopy { dst, src, len } => { - self.execute_memory_copy(store.inner_mut(), dst, src, len)? - } - Instr::MemoryFill { dst, value, len } => { - self.execute_memory_fill(store.inner_mut(), dst, value, len)? - } - Instr::MemoryFillImm { dst, value, len } => { - self.execute_memory_fill_imm(store.inner_mut(), dst, value, len)? - } - Instr::MemoryInit { dst, src, len } => { - self.execute_memory_init(store.inner_mut(), dst, src, len)? - } - Instr::TableIndex { .. } - | Instr::MemoryIndex { .. } - | Instr::DataIndex { .. } - | Instr::ElemIndex { .. } - | Instr::Const32 { .. } - | Instr::I64Const32 { .. } - | Instr::F64Const32 { .. } - | Instr::BranchTableTarget { .. } - | Instr::Slot { .. } - | Instr::Slot2 { .. } - | Instr::Slot3 { .. } - | Instr::SlotAndImm32 { .. } - | Instr::Imm16AndImm32 { .. } - | Instr::SlotSpan { .. } - | Instr::SlotList { .. } - | Instr::CallIndirectParams { .. } - | Instr::CallIndirectParamsImm16 { .. } => self.invalid_instruction_word()?, - #[cfg(feature = "simd")] - Instr::I8x16Splat { result, value } => self.execute_i8x16_splat(result, value), - #[cfg(feature = "simd")] - Instr::I16x8Splat { result, value } => self.execute_i16x8_splat(result, value), - #[cfg(feature = "simd")] - Instr::I32x4Splat { result, value } => self.execute_i32x4_splat(result, value), - #[cfg(feature = "simd")] - Instr::I64x2Splat { result, value } => self.execute_i64x2_splat(result, value), - #[cfg(feature = "simd")] - Instr::F32x4Splat { result, value } => self.execute_f32x4_splat(result, value), - #[cfg(feature = "simd")] - Instr::F64x2Splat { result, value } => self.execute_f64x2_splat(result, value), - #[cfg(feature = "simd")] - Instr::I8x16ExtractLaneS { - result, - value, - lane, - } => self.i8x16_extract_lane_s(result, value, lane), - #[cfg(feature = "simd")] - Instr::I8x16ExtractLaneU { - result, - value, - lane, - } => self.i8x16_extract_lane_u(result, value, lane), - #[cfg(feature = "simd")] - Instr::I16x8ExtractLaneS { - result, - value, - lane, - } => self.i16x8_extract_lane_s(result, value, lane), - #[cfg(feature = "simd")] - Instr::I16x8ExtractLaneU { - result, - value, - lane, - } => self.i16x8_extract_lane_u(result, value, lane), - #[cfg(feature = "simd")] - Instr::I32x4ExtractLane { - result, - value, - lane, - } => self.i32x4_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::I64x2ExtractLane { - result, - value, - lane, - } => self.i64x2_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::F32x4ExtractLane { - result, - value, - lane, - } => self.f32x4_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::F64x2ExtractLane { - result, - value, - lane, - } => self.f64x2_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::I8x16ReplaceLane { - result, - input, - lane, - } => self.execute_i8x16_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I8x16ReplaceLaneImm { - result, - input, - lane, - value, - } => self.execute_i8x16_replace_lane_imm(result, input, lane, value), - #[cfg(feature = "simd")] - Instr::I16x8ReplaceLane { - result, - input, - lane, - } => self.execute_i16x8_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I16x8ReplaceLaneImm { - result, - input, - lane, - } => self.execute_i16x8_replace_lane_imm(result, input, lane), - #[cfg(feature = "simd")] - Instr::I32x4ReplaceLane { - result, - input, - lane, - } => self.execute_i32x4_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I32x4ReplaceLaneImm { - result, - input, - lane, - } => self.execute_i32x4_replace_lane_imm(result, input, lane), - #[cfg(feature = "simd")] - Instr::I64x2ReplaceLane { - result, - input, - lane, - } => self.execute_i64x2_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I64x2ReplaceLaneImm32 { - result, - input, - lane, - } => self.execute_i64x2_replace_lane_imm32(result, input, lane), - #[cfg(feature = "simd")] - Instr::F32x4ReplaceLane { - result, - input, - lane, - } => self.execute_f32x4_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::F32x4ReplaceLaneImm { - result, - input, - lane, - } => self.execute_f32x4_replace_lane_imm(result, input, lane), - #[cfg(feature = "simd")] - Instr::F64x2ReplaceLane { - result, - input, - lane, - } => self.execute_f64x2_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::F64x2ReplaceLaneImm32 { - result, - input, - lane, - } => self.execute_f64x2_replace_lane_imm32(result, input, lane), - #[cfg(feature = "simd")] - Instr::I8x16Shuffle { result, lhs, rhs } => { - self.execute_i8x16_shuffle(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Swizzle { - result, - input, - selector, - } => self.execute_i8x16_swizzle(result, input, selector), - #[cfg(feature = "simd")] - Instr::I8x16Add { result, lhs, rhs } => self.execute_i8x16_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Add { result, lhs, rhs } => self.execute_i16x8_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Add { result, lhs, rhs } => self.execute_i32x4_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Add { result, lhs, rhs } => self.execute_i64x2_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16Sub { result, lhs, rhs } => self.execute_i8x16_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Sub { result, lhs, rhs } => self.execute_i16x8_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Sub { result, lhs, rhs } => self.execute_i32x4_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Sub { result, lhs, rhs } => self.execute_i64x2_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Mul { result, lhs, rhs } => self.execute_i16x8_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Mul { result, lhs, rhs } => self.execute_i32x4_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Mul { result, lhs, rhs } => self.execute_i64x2_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4DotI16x8S { result, lhs, rhs } => { - self.execute_i32x4_dot_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Neg { result, input } => self.execute_i8x16_neg(result, input), - #[cfg(feature = "simd")] - Instr::I16x8Neg { result, input } => self.execute_i16x8_neg(result, input), - #[cfg(feature = "simd")] - Instr::I32x4Neg { result, input } => self.execute_i32x4_neg(result, input), - #[cfg(feature = "simd")] - Instr::I64x2Neg { result, input } => self.execute_i64x2_neg(result, input), - #[cfg(feature = "simd")] - Instr::I16x8ExtmulLowI8x16S { result, lhs, rhs } => { - self.execute_i16x8_extmul_low_i8x16_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtmulHighI8x16S { result, lhs, rhs } => { - self.execute_i16x8_extmul_high_i8x16_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtmulLowI8x16U { result, lhs, rhs } => { - self.execute_i16x8_extmul_low_i8x16_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtmulHighI8x16U { result, lhs, rhs } => { - self.execute_i16x8_extmul_high_i8x16_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulLowI16x8S { result, lhs, rhs } => { - self.execute_i32x4_extmul_low_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulHighI16x8S { result, lhs, rhs } => { - self.execute_i32x4_extmul_high_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulLowI16x8U { result, lhs, rhs } => { - self.execute_i32x4_extmul_low_i16x8_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulHighI16x8U { result, lhs, rhs } => { - self.execute_i32x4_extmul_high_i16x8_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulLowI32x4S { result, lhs, rhs } => { - self.execute_i64x2_extmul_low_i32x4_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulHighI32x4S { result, lhs, rhs } => { - self.execute_i64x2_extmul_high_i32x4_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulLowI32x4U { result, lhs, rhs } => { - self.execute_i64x2_extmul_low_i32x4_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulHighI32x4U { result, lhs, rhs } => { - self.execute_i64x2_extmul_high_i32x4_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtaddPairwiseI8x16S { result, input } => { - self.execute_i16x8_extadd_pairwise_i8x16_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtaddPairwiseI8x16U { result, input } => { - self.execute_i16x8_extadd_pairwise_i8x16_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtaddPairwiseI16x8S { result, input } => { - self.execute_i32x4_extadd_pairwise_i16x8_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtaddPairwiseI16x8U { result, input } => { - self.execute_i32x4_extadd_pairwise_i16x8_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I8x16AddSatS { result, lhs, rhs } => { - self.execute_i8x16_add_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16AddSatU { result, lhs, rhs } => { - self.execute_i8x16_add_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8AddSatS { result, lhs, rhs } => { - self.execute_i16x8_add_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8AddSatU { result, lhs, rhs } => { - self.execute_i16x8_add_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16SubSatS { result, lhs, rhs } => { - self.execute_i8x16_sub_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16SubSatU { result, lhs, rhs } => { - self.execute_i8x16_sub_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8SubSatS { result, lhs, rhs } => { - self.execute_i16x8_sub_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8SubSatU { result, lhs, rhs } => { - self.execute_i16x8_sub_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8Q15MulrSatS { result, lhs, rhs } => { - self.execute_i16x8_q15mulr_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16MinS { result, lhs, rhs } => self.execute_i8x16_min_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16MinU { result, lhs, rhs } => self.execute_i8x16_min_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MinS { result, lhs, rhs } => self.execute_i16x8_min_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MinU { result, lhs, rhs } => self.execute_i16x8_min_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MinS { result, lhs, rhs } => self.execute_i32x4_min_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MinU { result, lhs, rhs } => self.execute_i32x4_min_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16MaxS { result, lhs, rhs } => self.execute_i8x16_max_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16MaxU { result, lhs, rhs } => self.execute_i8x16_max_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MaxS { result, lhs, rhs } => self.execute_i16x8_max_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MaxU { result, lhs, rhs } => self.execute_i16x8_max_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MaxS { result, lhs, rhs } => self.execute_i32x4_max_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MaxU { result, lhs, rhs } => self.execute_i32x4_max_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16AvgrU { result, lhs, rhs } => { - self.execute_i8x16_avgr_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8AvgrU { result, lhs, rhs } => { - self.execute_i16x8_avgr_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Abs { result, input } => self.execute_i8x16_abs(result, input), - #[cfg(feature = "simd")] - Instr::I16x8Abs { result, input } => self.execute_i16x8_abs(result, input), - #[cfg(feature = "simd")] - Instr::I32x4Abs { result, input } => self.execute_i32x4_abs(result, input), - #[cfg(feature = "simd")] - Instr::I64x2Abs { result, input } => self.execute_i64x2_abs(result, input), - #[cfg(feature = "simd")] - Instr::I8x16Shl { result, lhs, rhs } => self.execute_i8x16_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16ShlBy { result, lhs, rhs } => { - self.execute_i8x16_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8Shl { result, lhs, rhs } => self.execute_i16x8_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8ShlBy { result, lhs, rhs } => { - self.execute_i16x8_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4Shl { result, lhs, rhs } => self.execute_i32x4_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4ShlBy { result, lhs, rhs } => { - self.execute_i32x4_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2Shl { result, lhs, rhs } => self.execute_i64x2_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2ShlBy { result, lhs, rhs } => { - self.execute_i64x2_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16ShrS { result, lhs, rhs } => self.execute_i8x16_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16ShrSBy { result, lhs, rhs } => { - self.execute_i8x16_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16ShrU { result, lhs, rhs } => self.execute_i8x16_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16ShrUBy { result, lhs, rhs } => { - self.execute_i8x16_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ShrS { result, lhs, rhs } => self.execute_i16x8_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8ShrSBy { result, lhs, rhs } => { - self.execute_i16x8_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ShrU { result, lhs, rhs } => self.execute_i16x8_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8ShrUBy { result, lhs, rhs } => { - self.execute_i16x8_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ShrS { result, lhs, rhs } => self.execute_i32x4_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4ShrSBy { result, lhs, rhs } => { - self.execute_i32x4_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ShrU { result, lhs, rhs } => self.execute_i32x4_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4ShrUBy { result, lhs, rhs } => { - self.execute_i32x4_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ShrS { result, lhs, rhs } => self.execute_i64x2_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2ShrSBy { result, lhs, rhs } => { - self.execute_i64x2_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ShrU { result, lhs, rhs } => self.execute_i64x2_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2ShrUBy { result, lhs, rhs } => { - self.execute_i64x2_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::V128And { result, lhs, rhs } => self.execute_v128_and(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::V128Or { result, lhs, rhs } => self.execute_v128_or(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::V128Xor { result, lhs, rhs } => self.execute_v128_xor(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::V128Andnot { result, lhs, rhs } => { - self.execute_v128_andnot(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::V128Not { result, input } => self.execute_v128_not(result, input), - #[cfg(feature = "simd")] - Instr::V128Bitselect { result, lhs, rhs } => { - self.execute_v128_bitselect(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Popcnt { result, input } => self.execute_i8x16_popcnt(result, input), - #[cfg(feature = "simd")] - Instr::V128AnyTrue { result, input } => self.execute_v128_any_true(result, input), - #[cfg(feature = "simd")] - Instr::I8x16AllTrue { result, input } => self.execute_i8x16_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I16x8AllTrue { result, input } => self.execute_i16x8_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I32x4AllTrue { result, input } => self.execute_i32x4_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I64x2AllTrue { result, input } => self.execute_i64x2_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I8x16Bitmask { result, input } => self.execute_i8x16_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I16x8Bitmask { result, input } => self.execute_i16x8_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I32x4Bitmask { result, input } => self.execute_i32x4_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I64x2Bitmask { result, input } => self.execute_i64x2_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I8x16Eq { result, lhs, rhs } => self.execute_i8x16_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Eq { result, lhs, rhs } => self.execute_i16x8_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Eq { result, lhs, rhs } => self.execute_i32x4_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Eq { result, lhs, rhs } => self.execute_i64x2_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Eq { result, lhs, rhs } => self.execute_f32x4_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Eq { result, lhs, rhs } => self.execute_f64x2_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16Ne { result, lhs, rhs } => self.execute_i8x16_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Ne { result, lhs, rhs } => self.execute_i16x8_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Ne { result, lhs, rhs } => self.execute_i32x4_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Ne { result, lhs, rhs } => self.execute_i64x2_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Ne { result, lhs, rhs } => self.execute_f32x4_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Ne { result, lhs, rhs } => self.execute_f64x2_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LtS { result, lhs, rhs } => self.execute_i8x16_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LtU { result, lhs, rhs } => self.execute_i8x16_lt_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LtS { result, lhs, rhs } => self.execute_i16x8_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LtU { result, lhs, rhs } => self.execute_i16x8_lt_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LtS { result, lhs, rhs } => self.execute_i32x4_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LtU { result, lhs, rhs } => self.execute_i32x4_lt_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2LtS { result, lhs, rhs } => self.execute_i64x2_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Lt { result, lhs, rhs } => self.execute_f32x4_lt(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Lt { result, lhs, rhs } => self.execute_f64x2_lt(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LeS { result, lhs, rhs } => self.execute_i8x16_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LeU { result, lhs, rhs } => self.execute_i8x16_le_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LeS { result, lhs, rhs } => self.execute_i16x8_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LeU { result, lhs, rhs } => self.execute_i16x8_le_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LeS { result, lhs, rhs } => self.execute_i32x4_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LeU { result, lhs, rhs } => self.execute_i32x4_le_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2LeS { result, lhs, rhs } => self.execute_i64x2_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Le { result, lhs, rhs } => self.execute_f32x4_le(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Le { result, lhs, rhs } => self.execute_f64x2_le(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Neg { result, input } => self.execute_f32x4_neg(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Neg { result, input } => self.execute_f64x2_neg(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Abs { result, input } => self.execute_f32x4_abs(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Abs { result, input } => self.execute_f64x2_abs(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Min { result, lhs, rhs } => self.execute_f32x4_min(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Min { result, lhs, rhs } => self.execute_f64x2_min(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Max { result, lhs, rhs } => self.execute_f32x4_max(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Max { result, lhs, rhs } => self.execute_f64x2_max(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Pmin { result, lhs, rhs } => self.execute_f32x4_pmin(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Pmin { result, lhs, rhs } => self.execute_f64x2_pmin(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Pmax { result, lhs, rhs } => self.execute_f32x4_pmax(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Pmax { result, lhs, rhs } => self.execute_f64x2_pmax(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Add { result, lhs, rhs } => self.execute_f32x4_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Add { result, lhs, rhs } => self.execute_f64x2_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Sub { result, lhs, rhs } => self.execute_f32x4_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Sub { result, lhs, rhs } => self.execute_f64x2_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Div { result, lhs, rhs } => self.execute_f32x4_div(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Div { result, lhs, rhs } => self.execute_f64x2_div(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Mul { result, lhs, rhs } => self.execute_f32x4_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Mul { result, lhs, rhs } => self.execute_f64x2_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Sqrt { result, input } => self.execute_f32x4_sqrt(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Sqrt { result, input } => self.execute_f64x2_sqrt(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Ceil { result, input } => self.execute_f32x4_ceil(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Ceil { result, input } => self.execute_f64x2_ceil(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Floor { result, input } => self.execute_f32x4_floor(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Floor { result, input } => self.execute_f64x2_floor(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Trunc { result, input } => self.execute_f32x4_trunc(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Trunc { result, input } => self.execute_f64x2_trunc(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Nearest { result, input } => self.execute_f32x4_nearest(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Nearest { result, input } => self.execute_f64x2_nearest(result, input), - #[cfg(feature = "simd")] - Instr::F32x4ConvertI32x4S { result, input } => { - self.execute_f32x4_convert_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::F32x4ConvertI32x4U { result, input } => { - self.execute_f32x4_convert_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::F64x2ConvertLowI32x4S { result, input } => { - self.execute_f64x2_convert_low_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::F64x2ConvertLowI32x4U { result, input } => { - self.execute_f64x2_convert_low_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF32x4S { result, input } => { - self.execute_i32x4_trunc_sat_f32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF32x4U { result, input } => { - self.execute_i32x4_trunc_sat_f32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF64x2SZero { result, input } => { - self.execute_i32x4_trunc_sat_f64x2_s_zero(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF64x2UZero { result, input } => { - self.execute_i32x4_trunc_sat_f64x2_u_zero(result, input) - } - #[cfg(feature = "simd")] - Instr::F32x4DemoteF64x2Zero { result, input } => { - self.execute_f32x4_demote_f64x2_zero(result, input) - } - #[cfg(feature = "simd")] - Instr::F64x2PromoteLowF32x4 { result, input } => { - self.execute_f64x2_promote_low_f32x4(result, input) - } - #[cfg(feature = "simd")] - Instr::I8x16NarrowI16x8S { result, lhs, rhs } => { - self.execute_i8x16_narrow_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16NarrowI16x8U { result, lhs, rhs } => { - self.execute_i8x16_narrow_i16x8_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8NarrowI32x4S { result, lhs, rhs } => { - self.execute_i16x8_narrow_i32x4_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8NarrowI32x4U { result, lhs, rhs } => { - self.execute_i16x8_narrow_i32x4_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendLowI8x16S { result, input } => { - self.execute_i16x8_extend_low_i8x16_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendHighI8x16S { result, input } => { - self.execute_i16x8_extend_high_i8x16_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendLowI8x16U { result, input } => { - self.execute_i16x8_extend_low_i8x16_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendHighI8x16U { result, input } => { - self.execute_i16x8_extend_high_i8x16_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendLowI16x8S { result, input } => { - self.execute_i32x4_extend_low_i16x8_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendHighI16x8S { result, input } => { - self.execute_i32x4_extend_high_i16x8_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendLowI16x8U { result, input } => { - self.execute_i32x4_extend_low_i16x8_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendHighI16x8U { result, input } => { - self.execute_i32x4_extend_high_i16x8_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendLowI32x4S { result, input } => { - self.execute_i64x2_extend_low_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendHighI32x4S { result, input } => { - self.execute_i64x2_extend_high_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendLowI32x4U { result, input } => { - self.execute_i64x2_extend_low_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendHighI32x4U { result, input } => { - self.execute_i64x2_extend_high_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::V128Store { ptr, offset_lo } => { - self.execute_v128_store(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128StoreOffset16 { ptr, value, offset } => { - self.execute_v128_store_offset16(ptr, offset, value)? - } - #[cfg(feature = "simd")] - Instr::V128StoreAt { value, address } => { - self.execute_v128_store_at(store.inner_mut(), address, value)? - } - #[cfg(feature = "simd")] - Instr::V128Store8Lane { ptr, offset_lo } => { - self.execute_v128_store8_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store8LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store8_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store8LaneAt { value, address } => { - self.execute_v128_store8_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Store16Lane { ptr, offset_lo } => { - self.execute_v128_store16_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store16LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store16_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store16LaneAt { value, address } => { - self.execute_v128_store16_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Store32Lane { ptr, offset_lo } => { - self.execute_v128_store32_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store32LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store32_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store32LaneAt { value, address } => { - self.execute_v128_store32_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Store64Lane { ptr, offset_lo } => { - self.execute_v128_store64_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store64LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store64_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store64LaneAt { value, address } => { - self.execute_v128_store64_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load { result, offset_lo } => { - self.execute_v128_load(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128LoadAt { result, address } => { - self.execute_v128_load_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128LoadOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32Zero { result, offset_lo } => { - self.execute_v128_load32_zero(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32ZeroAt { result, address } => { - self.execute_v128_load32_zero_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32ZeroOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32_zero_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load64Zero { result, offset_lo } => { - self.execute_v128_load64_zero(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load64ZeroAt { result, address } => { - self.execute_v128_load64_zero_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load64ZeroOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load64_zero_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8Splat { result, offset_lo } => { - self.execute_v128_load8_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8SplatAt { result, address } => { - self.execute_v128_load8_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load8SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load8_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load16Splat { result, offset_lo } => { - self.execute_v128_load16_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16SplatAt { result, address } => { - self.execute_v128_load16_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load16_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32Splat { result, offset_lo } => { - self.execute_v128_load32_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32SplatAt { result, address } => { - self.execute_v128_load32_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load64Splat { result, offset_lo } => { - self.execute_v128_load64_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load64SplatAt { result, address } => { - self.execute_v128_load64_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load64SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load64_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8x8S { result, offset_lo } => { - self.execute_v128_load8x8_s(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8SAt { result, address } => { - self.execute_v128_load8x8_s_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8SOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load8x8_s_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8x8U { result, offset_lo } => { - self.execute_v128_load8x8_u(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8UAt { result, address } => { - self.execute_v128_load8x8_u_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8UOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load8x8_u_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load16x4S { result, offset_lo } => { - self.execute_v128_load16x4_s(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4SAt { result, address } => { - self.execute_v128_load16x4_s_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4SOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load16x4_s_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load16x4U { result, offset_lo } => { - self.execute_v128_load16x4_u(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4UAt { result, address } => { - self.execute_v128_load16x4_u_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4UOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load16x4_u_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32x2S { result, offset_lo } => { - self.execute_v128_load32x2_s(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2SAt { result, address } => { - self.execute_v128_load32x2_s_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2SOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32x2_s_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32x2U { result, offset_lo } => { - self.execute_v128_load32x2_u(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2UAt { result, address } => { - self.execute_v128_load32x2_u_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2UOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32x2_u_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8Lane { result, offset_lo } => { - self.execute_v128_load8_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8LaneAt { result, address } => { - self.execute_v128_load8_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16Lane { result, offset_lo } => { - self.execute_v128_load16_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16LaneAt { result, address } => { - self.execute_v128_load16_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32Lane { result, offset_lo } => { - self.execute_v128_load32_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32LaneAt { result, address } => { - self.execute_v128_load32_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load64Lane { result, offset_lo } => { - self.execute_v128_load64_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load64LaneAt { result, address } => { - self.execute_v128_load64_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::I16x8RelaxedDotI8x16I7x16S { result, lhs, rhs } => { - self.execute_i16x8_relaxed_dot_i8x16_i7x16_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4RelaxedDotI8x16I7x16AddS { result, lhs, rhs } => { - self.execute_i32x4_relaxed_dot_i8x16_i7x16_add_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::F32x4RelaxedMadd { result, a, b } => { - self.execute_f32x4_relaxed_madd(result, a, b) - } - #[cfg(feature = "simd")] - Instr::F32x4RelaxedNmadd { result, a, b } => { - self.execute_f32x4_relaxed_nmadd(result, a, b) - } - #[cfg(feature = "simd")] - Instr::F64x2RelaxedMadd { result, a, b } => { - self.execute_f64x2_relaxed_madd(result, a, b) - } - #[cfg(feature = "simd")] - Instr::F64x2RelaxedNmadd { result, a, b } => { - self.execute_f64x2_relaxed_nmadd(result, a, b) - } - unsupported => panic!("encountered unsupported Wasmi instruction: {unsupported:?}"), - } - } - } -} - -macro_rules! get_entity { - ( - $( - fn $name:ident(&self, index: $index_ty:ty) -> $id_ty:ty; - )* - ) => { - $( - #[doc = ::core::concat!( - "Returns the [`", - ::core::stringify!($id_ty), - "`] at `index` for the currently used [`Instance`].\n\n", - "# Panics\n\n", - "- If there is no [`", - ::core::stringify!($id_ty), - "`] at `index` for the currently used [`Instance`] in `store`." - )] - #[inline] - fn $name(&self, index: $index_ty) -> $id_ty { - unsafe { self.cache.$name(index) } - .unwrap_or_else(|| { - const ENTITY_NAME: &'static str = ::core::stringify!($id_ty); - // Safety: within the Wasmi executor it is assumed that store entity - // indices within the Wasmi bytecode are always valid for the - // store. This is an invariant of the Wasmi translation. - unsafe { - unreachable_unchecked!( - "missing {ENTITY_NAME} at index {index:?} for the currently used instance", - ) - } - }) - } - )* - } -} - -impl Executor<'_> { - get_entity! { - fn get_func(&self, index: index::Func) -> Func; - fn get_func_type_dedup(&self, index: index::FuncType) -> DedupFuncType; - fn get_memory(&self, index: index::Memory) -> Memory; - fn get_table(&self, index: index::Table) -> Table; - fn get_global(&self, index: index::Global) -> Global; - fn get_data_segment(&self, index: index::Data) -> DataSegment; - fn get_element_segment(&self, index: index::Elem) -> ElementSegment; - } - - /// Returns the [`Slot`] value. - fn get_stack_slot(&self, slot: Slot) -> UntypedVal { - // Safety: - It is the responsibility of the `Executor` - // implementation to keep the `sp` pointer valid - // whenever this method is accessed. - // - This is done by updating the `sp` pointer whenever - // the heap underlying the value stack is changed. - unsafe { self.sp.get(slot) } - } - - /// Returns the [`Slot`] value as type `T`. - fn get_stack_slot_as(&self, slot: Slot) -> T - where - UntypedVal: ReadAs, - { - // Safety: - It is the responsibility of the `Executor` - // implementation to keep the `sp` pointer valid - // whenever this method is accessed. - // - This is done by updating the `sp` pointer whenever - // the heap underlying the value stack is changed. - unsafe { self.sp.read_as::(slot) } - } - - /// Sets the [`Slot`] value to `value`. - fn set_stack_slot(&mut self, slot: Slot, value: impl Into) { - // Safety: - It is the responsibility of the `Executor` - // implementation to keep the `sp` pointer valid - // whenever this method is accessed. - // - This is done by updating the `sp` pointer whenever - // the heap underlying the value stack is changed. - unsafe { self.sp.set(slot, value.into()) }; - } - - /// Sets the [`Slot`] value to `value` of type `T`. - fn set_stack_slot_as(&mut self, slot: Slot, value: T) - where - UntypedVal: WriteAs, - { - // Safety: - It is the responsibility of the `Executor` - // implementation to keep the `sp` pointer valid - // whenever this method is accessed. - // - This is done by updating the `sp` pointer whenever - // the heap underlying the value stack is changed. - unsafe { self.sp.write_as::(slot, value) }; - } - - /// Shifts the instruction pointer to the next instruction. - #[inline(always)] - fn next_instr(&mut self) { - self.next_instr_at(1) - } - - /// Shifts the instruction pointer to the next instruction. - /// - /// Has a parameter `skip` to denote how many instruction words - /// to skip to reach the next actual instruction. - /// - /// # Note - /// - /// This is used by Wasmi instructions that have a fixed - /// encoding size of two instruction words such as [`Op::Branch`]. - #[inline(always)] - fn next_instr_at(&mut self, skip: usize) { - self.ip.add(skip) - } - - /// Shifts the instruction pointer to the next instruction and returns `Ok(())`. - /// - /// # Note - /// - /// This is a convenience function for fallible instructions. - #[inline(always)] - fn try_next_instr(&mut self) -> Result<(), Error> { - self.try_next_instr_at(1) - } - - /// Shifts the instruction pointer to the next instruction and returns `Ok(())`. - /// - /// Has a parameter `skip` to denote how many instruction words - /// to skip to reach the next actual instruction. - /// - /// # Note - /// - /// This is a convenience function for fallible instructions. - #[inline(always)] - fn try_next_instr_at(&mut self, skip: usize) -> Result<(), Error> { - self.next_instr_at(skip); - Ok(()) - } - - /// Returns the [`FrameSlots`] of the [`CallFrame`]. - fn frame_stack_ptr_impl(value_stack: &mut ValueStack, frame: &CallFrame) -> FrameSlots { - // Safety: We are using the frame's own base offset as input because it is - // guaranteed by the Wasm validation and translation phase to be - // valid for all register indices used by the associated function body. - unsafe { value_stack.stack_ptr_at(frame.base_offset()) } - } - - /// Initializes the [`Executor`] state for the [`CallFrame`]. - /// - /// # Note - /// - /// The initialization of the [`Executor`] allows for efficient execution. - fn init_call_frame(&mut self, frame: &CallFrame) { - Self::init_call_frame_impl(&mut self.stack.values, &mut self.sp, &mut self.ip, frame) - } - - /// Initializes the [`Executor`] state for the [`CallFrame`]. - /// - /// # Note - /// - /// The initialization of the [`Executor`] allows for efficient execution. - fn init_call_frame_impl( - value_stack: &mut ValueStack, - sp: &mut FrameSlots, - ip: &mut InstructionPtr, - frame: &CallFrame, - ) { - *sp = Self::frame_stack_ptr_impl(value_stack, frame); - *ip = frame.instr_ptr(); - } - - /// Executes a generic unary [`Op`]. - #[inline(always)] - fn execute_unary(&mut self, result: Slot, input: Slot, op: fn(P) -> R) - where - UntypedVal: ReadAs

+ WriteAs, - { - let value = self.get_stack_slot_as::

(input); - self.set_stack_slot_as::(result, op(value)); - self.next_instr(); - } - - /// Executes a fallible generic unary [`Op`]. - #[inline(always)] - fn try_execute_unary( - &mut self, - result: Slot, - input: Slot, - op: fn(P) -> Result, - ) -> Result<(), Error> - where - UntypedVal: ReadAs

+ WriteAs, - { - let value = self.get_stack_slot_as::

(input); - self.set_stack_slot_as::(result, op(value)?); - self.try_next_instr() - } - - /// Executes a generic binary [`Op`]. - #[inline(always)] - fn execute_binary( - &mut self, - result: Slot, - lhs: Slot, - rhs: Slot, - op: fn(Lhs, Rhs) -> Result, - ) where - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a generic binary [`Op`]. - #[inline(always)] - fn execute_binary_imm16_rhs( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - op: fn(Lhs, Rhs) -> T, - ) where - Rhs: From>, - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = Rhs::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a generic binary [`Op`] with reversed operands. - #[inline(always)] - fn execute_binary_imm16_lhs( - &mut self, - result: Slot, - lhs: Const16, - rhs: Slot, - op: fn(Lhs, Rhs) -> T, - ) where - Lhs: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = Lhs::from(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a generic shift or rotate [`Op`]. - #[inline(always)] - fn execute_shift_by( - &mut self, - result: Slot, - lhs: Slot, - rhs: ShiftAmount, - op: fn(Lhs, Rhs) -> T, - ) where - Rhs: From>, - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = Rhs::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a fallible generic binary [`Op`]. - #[inline(always)] - fn try_execute_binary( - &mut self, - result: Slot, - lhs: Slot, - rhs: Slot, - op: fn(Lhs, Rhs) -> Result, - ) -> Result<(), Error> - where - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)?); - self.try_next_instr() - } - - /// Executes a fallible generic binary [`Op`]. - #[inline(always)] - fn try_execute_divrem_imm16_rhs( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - op: fn(Lhs, Rhs) -> Result, - ) -> Result<(), Error> - where - Rhs: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = Rhs::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)?); - self.try_next_instr() - } - - /// Executes a fallible generic binary [`Op`]. - #[inline(always)] - fn execute_divrem_imm16_rhs( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - op: fn(Lhs, NonZeroT) -> T, - ) where - NonZeroT: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = ::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr() - } - - /// Executes a fallible generic binary [`Op`] with reversed operands. - #[inline(always)] - fn try_execute_binary_imm16_lhs( - &mut self, - result: Slot, - lhs: Const16, - rhs: Slot, - op: fn(Lhs, Rhs) -> Result, - ) -> Result<(), Error> - where - Lhs: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = Lhs::from(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)?); - self.try_next_instr() - } - - /// Returns the optional `memory` parameter for a `load_at` [`Op`]. - /// - /// # Note - /// - /// - Returns the default [`index::Memory`] if the parameter is missing. - /// - Bumps `self.ip` if a [`Op::MemoryIndex`] parameter was found. - #[inline(always)] - fn fetch_optional_memory(&mut self, delta: usize) -> index::Memory { - let mut addr: InstructionPtr = self.ip; - addr.add(delta); - match *addr.get() { - Op::MemoryIndex { index } => { - hint::cold(); - self.ip.add(1); - index - } - _ => index::Memory::from(0), - } - } - - /// Fetches the [`Slot`] and [`Offset64Hi`] parameters for a load or store [`Op`]. - unsafe fn fetch_reg_and_offset_hi(&self) -> (Slot, Offset64Hi) { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match addr.get().filter_register_and_offset_hi() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::SlotAndImm32` but found: {instr:?}") - }, - } - } -} - -impl Executor<'_> { - /// Used for all [`Op`] words that are not meant for execution. - /// - /// # Note - /// - /// This includes [`Op`] variants such as [`Op::TableIndex`] - /// that primarily carry parameters for actually executable [`Op`]. - fn invalid_instruction_word(&mut self) -> Result<(), Error> { - // Safety: Wasmi translation guarantees that branches are never taken to instruction parameters directly. - unsafe { - unreachable_unchecked!( - "expected instruction but found instruction parameter: {:?}", - *self.ip.get() - ) - } - } - - /// Executes a Wasm `unreachable` instruction. - fn execute_trap(&mut self, trap_code: TrapCode) -> Result<(), Error> { - Err(Error::from(trap_code)) - } - - /// Executes an [`Op::ConsumeFuel`]. - fn execute_consume_fuel( - &mut self, - store: &mut StoreInner, - block_fuel: BlockFuel, - ) -> Result<(), Error> { - // We do not have to check if fuel metering is enabled since - // [`Op::ConsumeFuel`] are only generated if fuel metering - // is enabled to begin with. - store - .fuel_mut() - .consume_fuel_unchecked(block_fuel.to_u64())?; - self.try_next_instr() - } - - /// Executes an [`Op::RefFunc`]. - fn execute_ref_func(&mut self, result: Slot, func_index: index::Func) { - let func = self.get_func(func_index); - let funcref = >::from(func); - self.set_stack_slot(result, funcref); - self.next_instr(); - } -} - -/// Extension method for [`UntypedVal`] required by the [`Executor`]. -trait UntypedValueExt: Sized { - /// Executes a logical `i{32,64}.and` instruction. - fn and(x: Self, y: Self) -> bool; - - /// Executes a logical `i{32,64}.or` instruction. - fn or(x: Self, y: Self) -> bool; - - /// Executes a fused `i{32,64}.and` + `i{32,64}.eqz` instruction. - fn nand(x: Self, y: Self) -> bool { - !Self::and(x, y) - } - - /// Executes a fused `i{32,64}.or` + `i{32,64}.eqz` instruction. - fn nor(x: Self, y: Self) -> bool { - !Self::or(x, y) - } -} - -impl UntypedValueExt for i32 { - fn and(x: Self, y: Self) -> bool { - wasm::i32_bitand(x, y) != 0 - } - - fn or(x: Self, y: Self) -> bool { - wasm::i32_bitor(x, y) != 0 - } -} - -impl UntypedValueExt for i64 { - fn and(x: Self, y: Self) -> bool { - wasm::i64_bitand(x, y) != 0 - } - - fn or(x: Self, y: Self) -> bool { - wasm::i64_bitor(x, y) != 0 - } -} - -/// Extension method for [`UntypedVal`] required by the [`Executor`]. -trait UntypedValueCmpExt: Sized { - fn not_le(lhs: Self, rhs: Self) -> bool; - fn not_lt(lhs: Self, rhs: Self) -> bool; -} - -impl UntypedValueCmpExt for f32 { - fn not_le(x: Self, y: Self) -> bool { - !wasm::f32_le(x, y) - } - - fn not_lt(x: Self, y: Self) -> bool { - !wasm::f32_lt(x, y) - } -} - -impl UntypedValueCmpExt for f64 { - fn not_le(x: Self, y: Self) -> bool { - !wasm::f64_le(x, y) - } - - fn not_lt(x: Self, y: Self) -> bool { - !wasm::f64_lt(x, y) - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/binary.rs b/crates/wasmi/src/engine/executor/instrs/binary.rs deleted file mode 100644 index c3d55165ba8..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/binary.rs +++ /dev/null @@ -1,323 +0,0 @@ -use super::{Executor, UntypedValueExt}; -use crate::{ - core::wasm, - ir::{Const16, ShiftAmount, Sign, Slot}, - Error, - TrapCode, -}; -use core::num::{NonZeroI32, NonZeroI64, NonZeroU32, NonZeroU64}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - impl_binary_executors! { - (Op::I32Add, execute_i32_add, wasm::i32_add), - (Op::I32Sub, execute_i32_sub, wasm::i32_sub), - (Op::I32Mul, execute_i32_mul, wasm::i32_mul), - (Op::I32BitAnd, execute_i32_bitand, wasm::i32_bitand), - (Op::I32BitOr, execute_i32_bitor, wasm::i32_bitor), - (Op::I32BitXor, execute_i32_bitxor, wasm::i32_bitxor), - (Op::I32And, execute_i32_and, ::and), - (Op::I32Or, execute_i32_or, ::or), - (Op::I32Nand, execute_i32_nand, ::nand), - (Op::I32Nor, execute_i32_nor, ::nor), - - (Op::I64Add, execute_i64_add, wasm::i64_add), - (Op::I64Sub, execute_i64_sub, wasm::i64_sub), - (Op::I64Mul, execute_i64_mul, wasm::i64_mul), - (Op::I64BitAnd, execute_i64_bitand, wasm::i64_bitand), - (Op::I64BitOr, execute_i64_bitor, wasm::i64_bitor), - (Op::I64BitXor, execute_i64_bitxor, wasm::i64_bitxor), - (Op::I64And, execute_i64_and, ::and), - (Op::I64Or, execute_i64_or, ::or), - (Op::I64Nand, execute_i64_nand, ::nand), - (Op::I64Nor, execute_i64_nor, ::nor), - - (Op::I32Shl, execute_i32_shl, wasm::i32_shl), - (Op::I32ShrU, execute_i32_shr_u, wasm::i32_shr_u), - (Op::I32ShrS, execute_i32_shr_s, wasm::i32_shr_s), - (Op::I32Rotl, execute_i32_rotl, wasm::i32_rotl), - (Op::I32Rotr, execute_i32_rotr, wasm::i32_rotr), - - (Op::I64Shl, execute_i64_shl, wasm::i64_shl), - (Op::I64ShrU, execute_i64_shr_u, wasm::i64_shr_u), - (Op::I64ShrS, execute_i64_shr_s, wasm::i64_shr_s), - (Op::I64Rotl, execute_i64_rotl, wasm::i64_rotl), - (Op::I64Rotr, execute_i64_rotr, wasm::i64_rotr), - - (Op::F32Add, execute_f32_add, wasm::f32_add), - (Op::F32Sub, execute_f32_sub, wasm::f32_sub), - (Op::F32Mul, execute_f32_mul, wasm::f32_mul), - (Op::F32Div, execute_f32_div, wasm::f32_div), - (Op::F32Min, execute_f32_min, wasm::f32_min), - (Op::F32Max, execute_f32_max, wasm::f32_max), - (Op::F32Copysign, execute_f32_copysign, wasm::f32_copysign), - - (Op::F64Add, execute_f64_add, wasm::f64_add), - (Op::F64Sub, execute_f64_sub, wasm::f64_sub), - (Op::F64Mul, execute_f64_mul, wasm::f64_mul), - (Op::F64Div, execute_f64_div, wasm::f64_div), - (Op::F64Min, execute_f64_min, wasm::f64_min), - (Op::F64Max, execute_f64_max, wasm::f64_max), - (Op::F64Copysign, execute_f64_copysign, wasm::f64_copysign), - } -} - -macro_rules! impl_binary_imm16 { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { - self.execute_binary_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_binary_imm16! { - (i32, Op::I32AddImm16, execute_i32_add_imm16, wasm::i32_add), - (i32, Op::I32MulImm16, execute_i32_mul_imm16, wasm::i32_mul), - (i32, Op::I32BitAndImm16, execute_i32_bitand_imm16, wasm::i32_bitand), - (i32, Op::I32BitOrImm16, execute_i32_bitor_imm16, wasm::i32_bitor), - (i32, Op::I32BitXorImm16, execute_i32_bitxor_imm16, wasm::i32_bitxor), - (i32, Op::I32AndImm16, execute_i32_and_imm16, ::and), - (i32, Op::I32OrImm16, execute_i32_or_imm16, ::or), - (i32, Op::I32NandImm16, execute_i32_nand_imm16, ::nand), - (i32, Op::I32NorImm16, execute_i32_nor_imm16, ::nor), - - (i64, Op::I64AddImm16, execute_i64_add_imm16, wasm::i64_add), - (i64, Op::I64MulImm16, execute_i64_mul_imm16, wasm::i64_mul), - (i64, Op::I64BitAndImm16, execute_i64_bitand_imm16, wasm::i64_bitand), - (i64, Op::I64BitOrImm16, execute_i64_bitor_imm16, wasm::i64_bitor), - (i64, Op::I64BitXorImm16, execute_i64_bitxor_imm16, wasm::i64_bitxor), - (i64, Op::I64AndImm16, execute_i64_and_imm16, ::and), - (i64, Op::I64OrImm16, execute_i64_or_imm16, ::or), - (i64, Op::I64NandImm16, execute_i64_nand_imm16, ::nand), - (i64, Op::I64NorImm16, execute_i64_nor_imm16, ::nor), - } -} - -macro_rules! impl_shift_by { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: ShiftAmount<$ty>) { - self.execute_shift_by(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_shift_by! { - (i32, Op::I32ShlBy, execute_i32_shl_by, wasm::i32_shl), - (i32, Op::I32ShrUBy, execute_i32_shr_u_by, wasm::i32_shr_u), - (i32, Op::I32ShrSBy, execute_i32_shr_s_by, wasm::i32_shr_s), - (i32, Op::I32RotlBy, execute_i32_rotl_by, wasm::i32_rotl), - (i32, Op::I32RotrBy, execute_i32_rotr_by, wasm::i32_rotr), - - (i64, Op::I64ShlBy, execute_i64_shl_by, wasm::i64_shl), - (i64, Op::I64ShrUBy, execute_i64_shr_u_by, wasm::i64_shr_u), - (i64, Op::I64ShrSBy, execute_i64_shr_s_by, wasm::i64_shr_s), - (i64, Op::I64RotlBy, execute_i64_rotl_by, wasm::i64_rotl), - (i64, Op::I64RotrBy, execute_i64_rotr_by, wasm::i64_rotr), - } -} - -macro_rules! impl_binary_imm16_lhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Const16<$ty>, rhs: Slot) { - self.execute_binary_imm16_lhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_binary_imm16_lhs! { - (i32, Op::I32SubImm16Lhs, execute_i32_sub_imm16_lhs, wasm::i32_sub), - (i64, Op::I64SubImm16Lhs, execute_i64_sub_imm16_lhs, wasm::i64_sub), - - (i32, Op::I32ShlImm16, execute_i32_shl_imm16, wasm::i32_shl), - (i32, Op::I32ShrUImm16, execute_i32_shr_u_imm16, wasm::i32_shr_u), - (i32, Op::I32ShrSImm16, execute_i32_shr_s_imm16, wasm::i32_shr_s), - (i32, Op::I32RotlImm16, execute_i32_rotl_imm16, wasm::i32_rotl), - (i32, Op::I32RotrImm16, execute_i32_rotr_imm16, wasm::i32_rotr), - - (i64, Op::I64ShlImm16, execute_i64_shl_imm16, wasm::i64_shl), - (i64, Op::I64ShrUImm16, execute_i64_shr_u_imm16, wasm::i64_shr_u), - (i64, Op::I64ShrSImm16, execute_i64_shr_s_imm16, wasm::i64_shr_s), - (i64, Op::I64RotlImm16, execute_i64_rotl_imm16, wasm::i64_rotl), - (i64, Op::I64RotrImm16, execute_i64_rotr_imm16, wasm::i64_rotr), - } -} - -macro_rules! impl_fallible_binary { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Slot) -> Result<(), Error> { - self.try_execute_binary(result, lhs, rhs, $op).map_err(Error::from) - } - )* - }; -} -impl Executor<'_> { - impl_fallible_binary! { - (Op::I32DivS, execute_i32_div_s, wasm::i32_div_s), - (Op::I32DivU, execute_i32_div_u, wasm::i32_div_u), - (Op::I32RemS, execute_i32_rem_s, wasm::i32_rem_s), - (Op::I32RemU, execute_i32_rem_u, wasm::i32_rem_u), - - (Op::I64DivS, execute_i64_div_s, wasm::i64_div_s), - (Op::I64DivU, execute_i64_div_u, wasm::i64_div_u), - (Op::I64RemS, execute_i64_rem_s, wasm::i64_rem_s), - (Op::I64RemU, execute_i64_rem_u, wasm::i64_rem_u), - } -} - -/// Extension trait to provide more optimized divide and remainder implementations. -pub trait DivRemExt: Sized { - /// Signed non-zero value type. - type NonZeroS; - /// Unsigned non-zero value type. - type NonZeroU; - - /// Optimized variant of Wasm `i{32,64}.div_s` for immutable non-zero `rhs` values. - fn div_s(self, rhs: Self::NonZeroS) -> Result; - /// Optimized variant of Wasm `i{32,64}.div_u` for immutable non-zero `rhs` values. - fn div_u(self, rhs: Self::NonZeroU) -> Self; - /// Optimized variant of Wasm `i{32,64}.rem_s` for immutable non-zero `rhs` values. - fn rem_s(self, rhs: Self::NonZeroS) -> Result; - /// Optimized variant of Wasm `i{32,64}.rem_u` for immutable non-zero `rhs` values. - fn rem_u(self, rhs: Self::NonZeroU) -> Self; -} - -impl DivRemExt for i32 { - type NonZeroS = NonZeroI32; - type NonZeroU = NonZeroU32; - - fn div_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_div(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn div_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u32) / rhs) as Self - } - - fn rem_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_rem(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn rem_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u32) % rhs) as Self - } -} - -impl DivRemExt for i64 { - type NonZeroS = NonZeroI64; - type NonZeroU = NonZeroU64; - - fn div_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_div(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn div_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u64) / rhs) as Self - } - - fn rem_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_rem(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn rem_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u64) % rhs) as Self - } -} - -macro_rules! impl_divrem_s_imm16_rhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) -> Result<(), Error> { - self.try_execute_divrem_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_divrem_s_imm16_rhs! { - (NonZeroI32, Op::I32DivSImm16Rhs, execute_i32_div_s_imm16_rhs, ::div_s), - (NonZeroI32, Op::I32RemSImm16Rhs, execute_i32_rem_s_imm16_rhs, ::rem_s), - - (NonZeroI64, Op::I64DivSImm16Rhs, execute_i64_div_s_imm16_rhs, ::div_s), - (NonZeroI64, Op::I64RemSImm16Rhs, execute_i64_rem_s_imm16_rhs, ::rem_s), - } -} - -macro_rules! impl_divrem_u_imm16_rhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { - self.execute_divrem_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_divrem_u_imm16_rhs! { - (NonZeroU32, Op::I32DivUImm16Rhs, execute_i32_div_u_imm16_rhs, ::div_u), - (NonZeroU32, Op::I32RemUImm16Rhs, execute_i32_rem_u_imm16_rhs, ::rem_u), - - (NonZeroU64, Op::I64DivUImm16Rhs, execute_i64_div_u_imm16_rhs, ::div_u), - (NonZeroU64, Op::I64RemUImm16Rhs, execute_i64_rem_u_imm16_rhs, ::rem_u), - } -} - -macro_rules! impl_fallible_binary_imm16_lhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Const16<$ty>, rhs: Slot) -> Result<(), Error> { - self.try_execute_binary_imm16_lhs(result, lhs, rhs, $op).map_err(Error::from) - } - )* - }; -} -impl Executor<'_> { - impl_fallible_binary_imm16_lhs! { - (i32, Op::I32DivSImm16Lhs, execute_i32_div_s_imm16_lhs, wasm::i32_div_s), - (u32, Op::I32DivUImm16Lhs, execute_i32_div_u_imm16_lhs, wasm::i32_div_u), - (i32, Op::I32RemSImm16Lhs, execute_i32_rem_s_imm16_lhs, wasm::i32_rem_s), - (u32, Op::I32RemUImm16Lhs, execute_i32_rem_u_imm16_lhs, wasm::i32_rem_u), - - (i64, Op::I64DivSImm16Lhs, execute_i64_div_s_imm16_lhs, wasm::i64_div_s), - (u64, Op::I64DivUImm16Lhs, execute_i64_div_u_imm16_lhs, wasm::i64_div_u), - (i64, Op::I64RemSImm16Lhs, execute_i64_rem_s_imm16_lhs, wasm::i64_rem_s), - (u64, Op::I64RemUImm16Lhs, execute_i64_rem_u_imm16_lhs, wasm::i64_rem_u), - } -} - -impl Executor<'_> { - /// Executes an [`Op::F32CopysignImm`]. - pub fn execute_f32_copysign_imm(&mut self, result: Slot, lhs: Slot, rhs: Sign) { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = f32::from(rhs); - self.set_stack_slot_as::(result, wasm::f32_copysign(lhs, rhs)); - self.next_instr() - } - - /// Executes an [`Op::F64CopysignImm`]. - pub fn execute_f64_copysign_imm(&mut self, result: Slot, lhs: Slot, rhs: Sign) { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = f64::from(rhs); - self.set_stack_slot_as::(result, wasm::f64_copysign(lhs, rhs)); - self.next_instr() - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/branch.rs b/crates/wasmi/src/engine/executor/instrs/branch.rs deleted file mode 100644 index 7cb509c2543..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/branch.rs +++ /dev/null @@ -1,328 +0,0 @@ -use super::{Executor, UntypedValueCmpExt, UntypedValueExt}; -use crate::{ - core::{ReadAs, UntypedVal}, - engine::utils::unreachable_unchecked, - ir::{BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset, Const16, Op, Slot}, -}; -use core::cmp; - -impl Executor<'_> { - /// Branches and adjusts the value stack. - /// - /// # Note - /// - /// Offsets the instruction pointer using the given [`BranchOffset`]. - fn branch_to(&mut self, offset: BranchOffset) { - self.ip.offset(offset.to_i32() as isize) - } - - /// Branches and adjusts the value stack. - /// - /// # Note - /// - /// Offsets the instruction pointer using the given [`BranchOffset`]. - fn branch_to16(&mut self, offset: BranchOffset16) { - self.ip.offset(offset.to_i16() as isize) - } - - pub fn execute_branch(&mut self, offset: BranchOffset) { - self.branch_to(offset) - } - - /// Fetches the branch table index value and normalizes it to clamp between `0..len_targets`. - fn fetch_branch_table_offset(&self, index: Slot, len_targets: u32) -> usize { - let index: u32 = self.get_stack_slot_as::(index); - // The index of the default target which is the last target of the slice. - let max_index = len_targets - 1; - // A normalized index will always yield a target without panicking. - cmp::min(index, max_index) as usize + 1 - } - - pub fn execute_branch_table_0(&mut self, index: Slot, len_targets: u32) { - let offset = self.fetch_branch_table_offset(index, len_targets); - self.ip.add(offset); - } - - pub fn execute_branch_table_span(&mut self, index: Slot, len_targets: u32) { - let offset = self.fetch_branch_table_offset(index, len_targets); - self.ip.add(1); - let values = match *self.ip.get() { - Op::SlotSpan { span } => span, - unexpected => { - // Safety: Wasmi translation guarantees that `Op::SlotSpan` follows. - unsafe { - unreachable_unchecked!("expected `Op::SlotSpan` but found: {unexpected:?}") - } - } - }; - let len = values.len(); - let values = values.span(); - self.ip.add(offset); - match *self.ip.get() { - // Note: we explicitly do _not_ handle branch table returns here for technical reasons. - // They are executed as the next conventional instruction in the pipeline, no special treatment required. - Op::BranchTableTarget { results, offset } => { - self.execute_copy_span_impl(results, values, len); - self.execute_branch(offset) - } - unexpected => { - // Safety: Wasmi translator guarantees that one of the above `Op` variants exists. - unsafe { - unreachable_unchecked!( - "expected target for `Op::BranchTableSpan` but found: {unexpected:?}" - ) - } - } - } - } - - /// Executes a generic fused compare and branch instruction with raw inputs. - #[inline(always)] - fn execute_branch_binop( - &mut self, - lhs: Slot, - rhs: Slot, - offset: impl Into, - f: fn(T, T) -> bool, - ) where - UntypedVal: ReadAs, - { - let lhs: T = self.get_stack_slot_as(lhs); - let rhs: T = self.get_stack_slot_as(rhs); - if f(lhs, rhs) { - return self.branch_to(offset.into()); - } - self.next_instr() - } - - /// Executes a generic fused compare and branch instruction with immediate `rhs` operand. - fn execute_branch_binop_imm16_rhs( - &mut self, - lhs: Slot, - rhs: Const16, - offset: BranchOffset16, - f: fn(T, T) -> bool, - ) where - T: From>, - UntypedVal: ReadAs, - { - let lhs: T = self.get_stack_slot_as(lhs); - let rhs = T::from(rhs); - if f(lhs, rhs) { - return self.branch_to16(offset); - } - self.next_instr() - } - - /// Executes a generic fused compare and branch instruction with immediate `rhs` operand. - fn execute_branch_binop_imm16_lhs( - &mut self, - lhs: Const16, - rhs: Slot, - offset: BranchOffset16, - f: fn(T, T) -> bool, - ) where - T: From>, - UntypedVal: ReadAs, - { - let lhs = T::from(lhs); - let rhs: T = self.get_stack_slot_as(rhs); - if f(lhs, rhs) { - return self.branch_to16(offset); - } - self.next_instr() - } -} - -fn cmp_eq(a: T, b: T) -> bool -where - T: PartialEq, -{ - a == b -} - -fn cmp_ne(a: T, b: T) -> bool -where - T: PartialEq, -{ - a != b -} - -fn cmp_lt(a: T, b: T) -> bool -where - T: PartialOrd, -{ - a < b -} - -fn cmp_le(a: T, b: T) -> bool -where - T: PartialOrd, -{ - a <= b -} - -macro_rules! impl_execute_branch_binop { - ( $( ($ty:ty, Op::$op_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - impl<'engine> Executor<'engine> { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op_name), "`].")] - #[inline(always)] - pub fn $fn_name(&mut self, lhs: Slot, rhs: Slot, offset: BranchOffset16) { - self.execute_branch_binop::<$ty>(lhs, rhs, offset, $op) - } - )* - } - } -} -impl_execute_branch_binop! { - (i32, Op::BranchI32And, execute_branch_i32_and, UntypedValueExt::and), - (i32, Op::BranchI32Or, execute_branch_i32_or, UntypedValueExt::or), - (i32, Op::BranchI32Nand, execute_branch_i32_nand, UntypedValueExt::nand), - (i32, Op::BranchI32Nor, execute_branch_i32_nor, UntypedValueExt::nor), - (i32, Op::BranchI32Eq, execute_branch_i32_eq, cmp_eq), - (i32, Op::BranchI32Ne, execute_branch_i32_ne, cmp_ne), - (i32, Op::BranchI32LtS, execute_branch_i32_lt_s, cmp_lt), - (u32, Op::BranchI32LtU, execute_branch_i32_lt_u, cmp_lt), - (i32, Op::BranchI32LeS, execute_branch_i32_le_s, cmp_le), - (u32, Op::BranchI32LeU, execute_branch_i32_le_u, cmp_le), - - (i64, Op::BranchI64And, execute_branch_i64_and, UntypedValueExt::and), - (i64, Op::BranchI64Or, execute_branch_i64_or, UntypedValueExt::or), - (i64, Op::BranchI64Nand, execute_branch_i64_nand, UntypedValueExt::nand), - (i64, Op::BranchI64Nor, execute_branch_i64_nor, UntypedValueExt::nor), - (i64, Op::BranchI64Eq, execute_branch_i64_eq, cmp_eq), - (i64, Op::BranchI64Ne, execute_branch_i64_ne, cmp_ne), - (i64, Op::BranchI64LtS, execute_branch_i64_lt_s, cmp_lt), - (u64, Op::BranchI64LtU, execute_branch_i64_lt_u, cmp_lt), - (i64, Op::BranchI64LeS, execute_branch_i64_le_s, cmp_le), - (u64, Op::BranchI64LeU, execute_branch_i64_le_u, cmp_le), - - (f32, Op::BranchF32Eq, execute_branch_f32_eq, cmp_eq), - (f32, Op::BranchF32Ne, execute_branch_f32_ne, cmp_ne), - (f32, Op::BranchF32Lt, execute_branch_f32_lt, cmp_lt), - (f32, Op::BranchF32Le, execute_branch_f32_le, cmp_le), - (f32, Op::BranchF32NotLt, execute_branch_f32_not_lt, UntypedValueCmpExt::not_lt), - (f32, Op::BranchF32NotLe, execute_branch_f32_not_le, UntypedValueCmpExt::not_le), - - (f64, Op::BranchF64Eq, execute_branch_f64_eq, cmp_eq), - (f64, Op::BranchF64Ne, execute_branch_f64_ne, cmp_ne), - (f64, Op::BranchF64Lt, execute_branch_f64_lt, cmp_lt), - (f64, Op::BranchF64Le, execute_branch_f64_le, cmp_le), - (f64, Op::BranchF64NotLt, execute_branch_f64_not_lt, UntypedValueCmpExt::not_lt), - (f64, Op::BranchF64NotLe, execute_branch_f64_not_le, UntypedValueCmpExt::not_le), -} - -macro_rules! impl_execute_branch_binop_imm16_rhs { - ( $( ($ty:ty, Op::$op_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - impl<'engine> Executor<'engine> { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op_name), "`].")] - pub fn $fn_name(&mut self, lhs: Slot, rhs: Const16<$ty>, offset: BranchOffset16) { - self.execute_branch_binop_imm16_rhs::<$ty>(lhs, rhs, offset, $op) - } - )* - } - } -} -impl_execute_branch_binop_imm16_rhs! { - (i32, Op::BranchI32AndImm16, execute_branch_i32_and_imm16, UntypedValueExt::and), - (i32, Op::BranchI32OrImm16, execute_branch_i32_or_imm16, UntypedValueExt::or), - (i32, Op::BranchI32NandImm16, execute_branch_i32_nand_imm16, UntypedValueExt::nand), - (i32, Op::BranchI32NorImm16, execute_branch_i32_nor_imm16, UntypedValueExt::nor), - (i32, Op::BranchI32EqImm16, execute_branch_i32_eq_imm16, cmp_eq), - (i32, Op::BranchI32NeImm16, execute_branch_i32_ne_imm16, cmp_ne), - (i32, Op::BranchI32LtSImm16Rhs, execute_branch_i32_lt_s_imm16_rhs, cmp_lt), - (u32, Op::BranchI32LtUImm16Rhs, execute_branch_i32_lt_u_imm16_rhs, cmp_lt), - (i32, Op::BranchI32LeSImm16Rhs, execute_branch_i32_le_s_imm16_rhs, cmp_le), - (u32, Op::BranchI32LeUImm16Rhs, execute_branch_i32_le_u_imm16_rhs, cmp_le), - - (i64, Op::BranchI64AndImm16, execute_branch_i64_and_imm16, UntypedValueExt::and), - (i64, Op::BranchI64OrImm16, execute_branch_i64_or_imm16, UntypedValueExt::or), - (i64, Op::BranchI64NandImm16, execute_branch_i64_nand_imm16, UntypedValueExt::nand), - (i64, Op::BranchI64NorImm16, execute_branch_i64_nor_imm16, UntypedValueExt::nor), - (i64, Op::BranchI64EqImm16, execute_branch_i64_eq_imm16, cmp_eq), - (i64, Op::BranchI64NeImm16, execute_branch_i64_ne_imm16, cmp_ne), - (i64, Op::BranchI64LtSImm16Rhs, execute_branch_i64_lt_s_imm16_rhs, cmp_lt), - (u64, Op::BranchI64LtUImm16Rhs, execute_branch_i64_lt_u_imm16_rhs, cmp_lt), - (i64, Op::BranchI64LeSImm16Rhs, execute_branch_i64_le_s_imm16_rhs, cmp_le), - (u64, Op::BranchI64LeUImm16Rhs, execute_branch_i64_le_u_imm16_rhs, cmp_le), -} - -macro_rules! impl_execute_branch_binop_imm16_lhs { - ( $( ($ty:ty, Op::$op_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - impl<'engine> Executor<'engine> { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op_name), "`].")] - pub fn $fn_name(&mut self, lhs: Const16<$ty>, rhs: Slot, offset: BranchOffset16) { - self.execute_branch_binop_imm16_lhs::<$ty>(lhs, rhs, offset, $op) - } - )* - } - } -} -impl_execute_branch_binop_imm16_lhs! { - (i32, Op::BranchI32LtSImm16Lhs, execute_branch_i32_lt_s_imm16_lhs, cmp_lt), - (u32, Op::BranchI32LtUImm16Lhs, execute_branch_i32_lt_u_imm16_lhs, cmp_lt), - (i32, Op::BranchI32LeSImm16Lhs, execute_branch_i32_le_s_imm16_lhs, cmp_le), - (u32, Op::BranchI32LeUImm16Lhs, execute_branch_i32_le_u_imm16_lhs, cmp_le), - - (i64, Op::BranchI64LtSImm16Lhs, execute_branch_i64_lt_s_imm16_lhs, cmp_lt), - (u64, Op::BranchI64LtUImm16Lhs, execute_branch_i64_lt_u_imm16_lhs, cmp_lt), - (i64, Op::BranchI64LeSImm16Lhs, execute_branch_i64_le_s_imm16_lhs, cmp_le), - (u64, Op::BranchI64LeUImm16Lhs, execute_branch_i64_le_u_imm16_lhs, cmp_le), -} - -impl Executor<'_> { - /// Executes an [`Op::BranchCmpFallback`]. - pub fn execute_branch_cmp_fallback(&mut self, lhs: Slot, rhs: Slot, params: Slot) { - use Comparator as C; - let params: u64 = self.get_stack_slot_as(params); - let Some(params) = ComparatorAndOffset::from_u64(params) else { - panic!("encountered invalidaly encoded ComparatorOffsetParam: {params:?}") - }; - let offset = params.offset; - match params.cmp { - C::I32Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::I32Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::I32LtS => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I32LtU => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I32LeS => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I32LeU => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I32And => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::and), - C::I32Or => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::or), - C::I32Nand => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nand), - C::I32Nor => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nor), - C::I64Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::I64Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::I64LtS => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I64LtU => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I64LeS => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I64LeU => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I64And => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::and), - C::I64Or => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::or), - C::I64Nand => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nand), - C::I64Nor => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nor), - C::F32Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::F32Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::F32Lt => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::F32Le => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::F32NotLt => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_lt) - } - C::F32NotLe => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_le) - } - C::F64Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::F64Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::F64Lt => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::F64Le => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::F64NotLt => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_lt) - } - C::F64NotLe => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_le) - } - }; - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/call.rs b/crates/wasmi/src/engine/executor/instrs/call.rs deleted file mode 100644 index 4fae2d8372d..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/call.rs +++ /dev/null @@ -1,713 +0,0 @@ -use super::{ControlFlow, Executor, InstructionPtr}; -use crate::{ - engine::{ - code_map::CompiledFuncRef, - executor::stack::{CallFrame, FrameParams, ValueStack}, - utils::unreachable_unchecked, - EngineFunc, - FuncInOut, - ResumableHostTrapError, - }, - func::{FuncEntity, HostFuncEntity}, - ir::{index, Op, Slot, SlotSpan}, - store::{CallHooks, PrunedStore, StoreError, StoreInner}, - Error, - Func, - Instance, - Ref, - TrapCode, -}; -use core::array; - -/// Dispatches and executes the host function. -/// -/// Returns the number of parameters and results of the called host function. -/// -/// # Errors -/// -/// Returns the error of the host function if an error occurred. -pub fn dispatch_host_func( - store: &mut PrunedStore, - value_stack: &mut ValueStack, - host_func: HostFuncEntity, - instance: Option<&Instance>, - call_hooks: CallHooks, -) -> Result<(u16, u16), Error> { - let len_params = host_func.len_params(); - let len_results = host_func.len_results(); - let max_inout = len_params.max(len_results); - let values = value_stack.as_slice_mut(); - let params_results = FuncInOut::new( - values.split_at_mut(values.len() - usize::from(max_inout)).1, - usize::from(len_params), - usize::from(len_results), - ); - match store.call_host_func(&host_func, instance, params_results, call_hooks) { - Err(StoreError::Internal(error)) => { - panic!("`call.host`: internal interpreter error: {error}") - } - Err(StoreError::External(error)) => { - // Note: We drop the values that have been temporarily added to - // the stack to act as parameter and result buffer for the - // called host function. Since the host function failed we - // need to clean up the temporary buffer values here. - // This is required for resumable calls to work properly. - value_stack.drop(usize::from(max_inout)); - return Err(error); - } - _ => {} - } - Ok((len_params, len_results)) -} - -/// The kind of a function call. -#[derive(Debug, Copy, Clone)] -pub enum CallKind { - /// A nested function call. - Nested, - /// A tailing function call. - Tail, -} - -trait CallContext { - const KIND: CallKind; - const HAS_PARAMS: bool; -} -trait ReturnCallContext: CallContext {} - -mod marker { - use super::{CallContext, CallKind, ReturnCallContext}; - - pub enum ReturnCall0 {} - impl CallContext for ReturnCall0 { - const KIND: CallKind = CallKind::Tail; - const HAS_PARAMS: bool = false; - } - impl ReturnCallContext for ReturnCall0 {} - - pub enum ReturnCall {} - impl CallContext for ReturnCall { - const KIND: CallKind = CallKind::Tail; - const HAS_PARAMS: bool = true; - } - impl ReturnCallContext for ReturnCall {} - - pub enum NestedCall0 {} - impl CallContext for NestedCall0 { - const KIND: CallKind = CallKind::Nested; - const HAS_PARAMS: bool = false; - } - - pub enum NestedCall {} - impl CallContext for NestedCall { - const KIND: CallKind = CallKind::Nested; - const HAS_PARAMS: bool = true; - } -} - -impl Executor<'_> { - /// Updates the [`InstructionPtr`] of the caller [`CallFrame`] before dispatching a call. - /// - /// # Note - /// - /// The `offset` denotes how many [`Op`] words make up the call instruction. - #[inline(always)] - fn update_instr_ptr_at(&mut self, offset: usize) { - // Note: we explicitly do not mutate `self.ip` since that would make - // other parts of the code more fragile with respect to instruction ordering. - self.ip.add(offset); - let caller = self - .stack - .calls - .peek_mut() - .expect("caller call frame must be on the stack"); - caller.update_instr_ptr(self.ip); - } - - /// Fetches the [`Op::CallIndirectParams`] parameter for a call [`Op`]. - /// - /// # Note - /// - /// - This advances the [`InstructionPtr`] to the next [`Op`]. - /// - This is done by encoding an [`Op::TableGet`] instruction - /// word following the actual instruction where the [`index::Table`] - /// parameter belongs to. - /// - This is required for some instructions that do not fit into - /// a single instruction word and store a [`index::Table`] value in - /// another instruction word. - fn pull_call_indirect_params(&mut self) -> (u64, index::Table) { - self.ip.add(1); - match *self.ip.get() { - Op::CallIndirectParams { index, table } => { - let index: u64 = self.get_stack_slot_as(index); - (index, table) - } - unexpected => { - // Safety: Wasmi translation guarantees that correct instruction parameter follows. - unsafe { - unreachable_unchecked!( - "expected `Op::CallIndirectParams` but found {unexpected:?}" - ) - } - } - } - } - - /// Fetches the [`Op::CallIndirectParamsImm16`] parameter for a call [`Op`]. - /// - /// # Note - /// - /// - This advances the [`InstructionPtr`] to the next [`Op`]. - /// - This is done by encoding an [`Op::TableGet`] instruction - /// word following the actual instruction where the [`index::Table`] - /// parameter belongs to. - /// - This is required for some instructions that do not fit into - /// a single instruction word and store a [`index::Table`] value in - /// another instruction word. - fn pull_call_indirect_params_imm16(&mut self) -> (u64, index::Table) { - self.ip.add(1); - match *self.ip.get() { - Op::CallIndirectParamsImm16 { index, table } => { - let index: u64 = index.into(); - (index, table) - } - unexpected => { - // Safety: Wasmi translation guarantees that correct instruction parameter follows. - unsafe { - unreachable_unchecked!( - "expected `Op::CallIndirectParamsImm16` but found {unexpected:?}" - ) - } - } - } - } - - /// Creates a [`CallFrame`] for calling the [`EngineFunc`]. - #[inline(always)] - fn dispatch_compiled_func( - &mut self, - results: SlotSpan, - func: CompiledFuncRef, - ) -> Result { - // We have to reinstantiate the `self.sp` [`FrameSlots`] since we just called - // [`ValueStack::alloc_call_frame`] which might invalidate all live [`FrameSlots`]. - let caller = self - .stack - .calls - .peek() - .expect("need to have a caller on the call stack"); - let (mut uninit_params, offsets) = self.stack.values.alloc_call_frame(func, |this| { - // Safety: We use the base offset of a live call frame on the call stack. - self.sp = unsafe { this.stack_ptr_at(caller.base_offset()) }; - })?; - let instr_ptr = InstructionPtr::new(func.instrs().as_ptr()); - let frame = CallFrame::new(instr_ptr, offsets, results); - if ::HAS_PARAMS { - self.copy_call_params(&mut uninit_params); - } - uninit_params.init_zeroes(); - Ok(frame) - } - - /// Copies the parameters from caller for the callee [`CallFrame`]. - /// - /// This will also adjust the instruction pointer to point to the - /// last call parameter [`Op`] if any. - fn copy_call_params(&mut self, uninit_params: &mut FrameParams) { - self.ip.add(1); - if let Op::SlotList { .. } = self.ip.get() { - self.copy_call_params_list(uninit_params); - } - match self.ip.get() { - Op::Slot { slot } => { - self.copy_regs(uninit_params, array::from_ref(slot)); - } - Op::Slot2 { slots } => { - self.copy_regs(uninit_params, slots); - } - Op::Slot3 { slots } => { - self.copy_regs(uninit_params, slots); - } - unexpected => { - // Safety: Wasmi translation guarantees that register list finalizer exists. - unsafe { - unreachable_unchecked!( - "expected register-list finalizer but found: {unexpected:?}" - ) - } - } - } - } - - /// Copies an array of [`Slot`] to the `dst` [`Slot`] span. - fn copy_regs(&self, uninit_params: &mut FrameParams, regs: &[Slot; N]) { - for value in regs { - let value = self.get_stack_slot(*value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { uninit_params.init_next(value) } - } - } - - /// Copies a list of [`Op::SlotList`] to the `dst` [`Slot`] span. - /// Copies the parameters from `src` for the called [`CallFrame`]. - /// - /// This will make the [`InstructionPtr`] point to the [`Op`] following the - /// last [`Op::SlotList`] if any. - #[cold] - fn copy_call_params_list(&mut self, uninit_params: &mut FrameParams) { - while let Op::SlotList { regs } = self.ip.get() { - self.copy_regs(uninit_params, regs); - self.ip.add(1); - } - } - - /// Prepares a [`EngineFunc`] call with optional call parameters. - #[inline(always)] - fn prepare_compiled_func_call( - &mut self, - store: &mut StoreInner, - results: SlotSpan, - func: EngineFunc, - mut instance: Option, - ) -> Result<(), Error> { - let func = self.code_map.get(Some(store.fuel_mut()), func)?; - let mut called = self.dispatch_compiled_func::(results, func)?; - match ::KIND { - CallKind::Nested => { - // We need to update the instruction pointer of the caller call frame. - self.update_instr_ptr_at(1); - } - CallKind::Tail => { - // In case of a tail call we have to remove the caller call frame after - // allocating the callee call frame. This moves all cells of the callee frame - // and may invalidate pointers to it. - // - // Safety: - // - // We provide `merge_call_frames` properly with `frame` that has just been allocated - // on the value stack which is what the function expects. After this operation we ensure - // that `self.sp` is adjusted via a call to `init_call_frame` since it may have been - // invalidated by this method. - let caller_instance = unsafe { self.stack.merge_call_frames(&mut called) }; - if let Some(caller_instance) = caller_instance { - instance.get_or_insert(caller_instance); - } - } - } - self.init_call_frame(&called); - self.stack.calls.push(called, instance)?; - Ok(()) - } - - /// Executes an [`Op::ReturnCallInternal0`]. - #[inline(always)] - pub fn execute_return_call_internal_0( - &mut self, - store: &mut StoreInner, - func: EngineFunc, - ) -> Result<(), Error> { - self.execute_return_call_internal_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallInternal`]. - #[inline(always)] - pub fn execute_return_call_internal( - &mut self, - store: &mut StoreInner, - func: EngineFunc, - ) -> Result<(), Error> { - self.execute_return_call_internal_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallInternal`] or [`Op::ReturnCallInternal0`]. - #[inline(always)] - fn execute_return_call_internal_impl( - &mut self, - store: &mut StoreInner, - func: EngineFunc, - ) -> Result<(), Error> { - let results = self.caller_results(); - self.prepare_compiled_func_call::(store, results, func, None) - } - - /// Returns the `results` [`SlotSpan`] of the top-most [`CallFrame`] on the [`CallStack`]. - /// - /// # Note - /// - /// We refer to the top-most [`CallFrame`] as the `caller` since this method is used for - /// tail call instructions for which the top-most [`CallFrame`] is the caller. - /// - /// [`CallStack`]: crate::engine::executor::stack::CallStack - #[inline(always)] - fn caller_results(&self) -> SlotSpan { - self.stack - .calls - .peek() - .expect("must have caller on the stack") - .results() - } - - /// Executes an [`Op::CallInternal0`]. - #[inline(always)] - pub fn execute_call_internal_0( - &mut self, - store: &mut StoreInner, - results: SlotSpan, - func: EngineFunc, - ) -> Result<(), Error> { - self.prepare_compiled_func_call::(store, results, func, None) - } - - /// Executes an [`Op::CallInternal`]. - #[inline(always)] - pub fn execute_call_internal( - &mut self, - store: &mut StoreInner, - results: SlotSpan, - func: EngineFunc, - ) -> Result<(), Error> { - self.prepare_compiled_func_call::(store, results, func, None) - } - - /// Executes an [`Op::ReturnCallImported0`]. - pub fn execute_return_call_imported_0( - &mut self, - store: &mut PrunedStore, - func: index::Func, - ) -> Result { - self.execute_return_call_imported_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallImported`]. - pub fn execute_return_call_imported( - &mut self, - store: &mut PrunedStore, - func: index::Func, - ) -> Result { - self.execute_return_call_imported_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallImported`] or [`Op::ReturnCallImported0`]. - fn execute_return_call_imported_impl( - &mut self, - store: &mut PrunedStore, - func: index::Func, - ) -> Result { - let func = self.get_func(func); - self.execute_call_imported_impl::(store, None, &func) - } - - /// Executes an [`Op::CallImported0`]. - pub fn execute_call_imported_0( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func: index::Func, - ) -> Result<(), Error> { - let func = self.get_func(func); - _ = self.execute_call_imported_impl::(store, Some(results), &func)?; - Ok(()) - } - - /// Executes an [`Op::CallImported`]. - pub fn execute_call_imported( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func: index::Func, - ) -> Result<(), Error> { - let func = self.get_func(func); - _ = self.execute_call_imported_impl::(store, Some(results), &func)?; - Ok(()) - } - - /// Executes an imported or indirect (tail) call instruction. - fn execute_call_imported_impl( - &mut self, - store: &mut PrunedStore, - results: Option, - func: &Func, - ) -> Result { - match store.inner().resolve_func(func) { - FuncEntity::Wasm(func) => { - let instance = *func.instance(); - let func_body = func.func_body(); - let results = results.unwrap_or_else(|| self.caller_results()); - self.prepare_compiled_func_call::( - store.inner_mut(), - results, - func_body, - Some(instance), - )?; - self.cache.update(store.inner_mut(), &instance); - Ok(ControlFlow::Continue(())) - } - FuncEntity::Host(host_func) => { - let host_func = *host_func; - self.execute_host_func::(store, results, func, host_func) - } - } - } - - /// Executes a host function. - /// - /// # Note - /// - /// This uses the value stack to store parameters and results of the host function call. - /// Returns an [`ErrorKind::ResumableHostTrap`] variant if the host function returned an error - /// and there are still call frames on the call stack making it possible to resume the - /// execution at a later point in time. - /// - /// [`ErrorKind::ResumableHostTrap`]: crate::error::ErrorKind::ResumableHostTrap - fn execute_host_func( - &mut self, - store: &mut PrunedStore, - results: Option, - func: &Func, - host_func: HostFuncEntity, - ) -> Result { - let len_params = host_func.len_params(); - let len_results = host_func.len_results(); - let max_inout = usize::from(len_params.max(len_results)); - let instance = *self.stack.calls.instance_expect(); - // We have to reinstantiate the `self.sp` [`FrameSlots`] since we just called - // [`ValueStack::reserve`] which might invalidate all live [`FrameSlots`]. - let (caller, popped_instance) = match ::KIND { - CallKind::Nested => self.stack.calls.peek().copied().map(|frame| (frame, None)), - CallKind::Tail => self.stack.calls.pop(), - } - .expect("need to have a caller on the call stack"); - let buffer = self.stack.values.extend_by(max_inout, |this| { - // Safety: we use the base offset of a live call frame on the call stack. - self.sp = unsafe { this.stack_ptr_at(caller.base_offset()) }; - })?; - if ::HAS_PARAMS { - let mut uninit_params = FrameParams::new(buffer); - self.copy_call_params(&mut uninit_params); - } - if matches!(::KIND, CallKind::Nested) { - self.update_instr_ptr_at(1); - } - let results = results.unwrap_or_else(|| caller.results()); - self.dispatch_host_func(store, host_func, &instance) - .map_err(|error| match self.stack.calls.is_empty() { - true => error, - false => ResumableHostTrapError::new(error, *func, results).into(), - })?; - self.cache.update(store.inner_mut(), &instance); - let results = results.iter(len_results); - match ::KIND { - CallKind::Nested => { - let returned = self.stack.values.drop_return(max_inout); - for (result, value) in results.zip(returned) { - // # Safety (1) - // - // We can safely acquire the stack pointer to the caller's and callee's (host) - // call frames because we just allocated the host call frame and can be sure that - // they are different. - // In the following we make sure to not access registers out of bounds of each - // call frame since we rely on Wasm validation and proper Wasm translation to - // provide us with valid result registers. - unsafe { self.sp.set(result, *value) }; - } - Ok(ControlFlow::Continue(())) - } - CallKind::Tail => { - let (mut regs, cf) = match self.stack.calls.peek() { - Some(frame) => { - // Case: return the caller's caller frame registers. - let sp = unsafe { self.stack.values.stack_ptr_at(frame.base_offset()) }; - (sp, ControlFlow::Continue(())) - } - None => { - // Case: call stack is empty -> return the root frame registers. - let sp = self.stack.values.root_stack_ptr(); - (sp, ControlFlow::Break(())) - } - }; - let returned = self.stack.values.drop_return(max_inout); - for (result, value) in results.zip(returned) { - // # Safety (1) - // - // We can safely acquire the stack pointer to the caller's and callee's (host) - // call frames because we just allocated the host call frame and can be sure that - // they are different. - // In the following we make sure to not access registers out of bounds of each - // call frame since we rely on Wasm validation and proper Wasm translation to - // provide us with valid result registers. - unsafe { regs.set(result, *value) }; - } - self.stack.values.truncate(caller.frame_offset()); - let new_instance = popped_instance.and_then(|_| self.stack.calls.instance()); - if let Some(new_instance) = new_instance { - self.cache.update(store.inner_mut(), new_instance); - } - if let Some(caller) = self.stack.calls.peek() { - Self::init_call_frame_impl( - &mut self.stack.values, - &mut self.sp, - &mut self.ip, - caller, - ); - } - Ok(cf) - } - } - } - - /// Convenience forwarder to [`dispatch_host_func`]. - fn dispatch_host_func( - &mut self, - store: &mut PrunedStore, - host_func: HostFuncEntity, - instance: &Instance, - ) -> Result<(u16, u16), Error> { - dispatch_host_func( - store, - &mut self.stack.values, - host_func, - Some(instance), - CallHooks::Call, - ) - } - - /// Executes an [`Op::CallIndirect0`]. - pub fn execute_return_call_indirect_0( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0Imm16`]. - pub fn execute_return_call_indirect_0_imm16( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params_imm16(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0`]. - pub fn execute_return_call_indirect( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0Imm16`]. - pub fn execute_return_call_indirect_imm16( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params_imm16(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0`]. - pub fn execute_call_indirect_0( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirect0Imm16`]. - pub fn execute_call_indirect_0_imm16( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params_imm16(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirect`]. - pub fn execute_call_indirect( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirectImm16`]. - pub fn execute_call_indirect_imm16( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params_imm16(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirect`] and [`Op::CallIndirect0`]. - fn execute_call_indirect_impl( - &mut self, - store: &mut PrunedStore, - results: Option, - func_type: index::FuncType, - index: u64, - table: index::Table, - ) -> Result { - let table = self.get_table(table); - let funcref = store - .inner() - .resolve_table(&table) - .get_untyped(index) - .map(>::from) - .ok_or(TrapCode::TableOutOfBounds)?; - let func = funcref.val().ok_or(TrapCode::IndirectCallToNull)?; - let actual_signature = store.inner().resolve_func(func).ty_dedup(); - let expected_signature = &self.get_func_type_dedup(func_type); - if actual_signature != expected_signature { - return Err(Error::from(TrapCode::BadSignature)); - } - self.execute_call_imported_impl::(store, results, func) - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/comparison.rs b/crates/wasmi/src/engine/executor/instrs/comparison.rs deleted file mode 100644 index c01625a35f7..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/comparison.rs +++ /dev/null @@ -1,94 +0,0 @@ -use super::{Executor, UntypedValueCmpExt}; -use crate::{ - core::wasm, - ir::{Const16, Slot}, -}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - impl_binary_executors! { - (Op::I32Eq, execute_i32_eq, wasm::i32_eq), - (Op::I32Ne, execute_i32_ne, wasm::i32_ne), - (Op::I32LtS, execute_i32_lt_s, wasm::i32_lt_s), - (Op::I32LtU, execute_i32_lt_u, wasm::i32_lt_u), - (Op::I32LeS, execute_i32_le_s, wasm::i32_le_s), - (Op::I32LeU, execute_i32_le_u, wasm::i32_le_u), - - (Op::I64Eq, execute_i64_eq, wasm::i64_eq), - (Op::I64Ne, execute_i64_ne, wasm::i64_ne), - (Op::I64LtS, execute_i64_lt_s, wasm::i64_lt_s), - (Op::I64LtU, execute_i64_lt_u, wasm::i64_lt_u), - (Op::I64LeS, execute_i64_le_s, wasm::i64_le_s), - (Op::I64LeU, execute_i64_le_u, wasm::i64_le_u), - - (Op::F32Eq, execute_f32_eq, wasm::f32_eq), - (Op::F32Ne, execute_f32_ne, wasm::f32_ne), - (Op::F32Lt, execute_f32_lt, wasm::f32_lt), - (Op::F32Le, execute_f32_le, wasm::f32_le), - (Op::F32NotLt, execute_f32_not_lt, ::not_lt), - (Op::F32NotLe, execute_f32_not_le, ::not_le), - - (Op::F64Eq, execute_f64_eq, wasm::f64_eq), - (Op::F64Ne, execute_f64_ne, wasm::f64_ne), - (Op::F64Lt, execute_f64_lt, wasm::f64_lt), - (Op::F64Le, execute_f64_le, wasm::f64_le), - (Op::F64NotLt, execute_f64_not_lt, ::not_lt), - (Op::F64NotLe, execute_f64_not_le, ::not_le), - } -} - -macro_rules! impl_comparison_imm16_rhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { - self.execute_binary_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_comparison_imm16_rhs! { - (i32, Op::I32EqImm16, execute_i32_eq_imm16, wasm::i32_eq), - (i32, Op::I32NeImm16, execute_i32_ne_imm16, wasm::i32_ne), - (i32, Op::I32LtSImm16Rhs, execute_i32_lt_s_imm16_rhs, wasm::i32_lt_s), - (u32, Op::I32LtUImm16Rhs, execute_i32_lt_u_imm16_rhs, wasm::i32_lt_u), - (i32, Op::I32LeSImm16Rhs, execute_i32_le_s_imm16_rhs, wasm::i32_le_s), - (u32, Op::I32LeUImm16Rhs, execute_i32_le_u_imm16_rhs, wasm::i32_le_u), - - (i64, Op::I64EqImm16, execute_i64_eq_imm16, wasm::i64_eq), - (i64, Op::I64NeImm16, execute_i64_ne_imm16, wasm::i64_ne), - (i64, Op::I64LtSImm16Rhs, execute_i64_lt_s_imm16_rhs, wasm::i64_lt_s), - (u64, Op::I64LtUImm16Rhs, execute_i64_lt_u_imm16_rhs, wasm::i64_lt_u), - (i64, Op::I64LeSImm16Rhs, execute_i64_le_s_imm16_rhs, wasm::i64_le_s), - (u64, Op::I64LeUImm16Rhs, execute_i64_le_u_imm16_rhs, wasm::i64_le_u), - } -} - -macro_rules! impl_comparison_imm16_lhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Const16<$ty>, rhs: Slot) { - self.execute_binary_imm16_lhs(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_comparison_imm16_lhs! { - (i32, Op::I32LtSImm16Lhs, execute_i32_lt_s_imm16_lhs, wasm::i32_lt_s), - (u32, Op::I32LtUImm16Lhs, execute_i32_lt_u_imm16_lhs, wasm::i32_lt_u), - (i32, Op::I32LeSImm16Lhs, execute_i32_le_s_imm16_lhs, wasm::i32_le_s), - (u32, Op::I32LeUImm16Lhs, execute_i32_le_u_imm16_lhs, wasm::i32_le_u), - - (i64, Op::I64LtSImm16Lhs, execute_i64_lt_s_imm16_lhs, wasm::i64_lt_s), - (u64, Op::I64LtUImm16Lhs, execute_i64_lt_u_imm16_lhs, wasm::i64_lt_u), - (i64, Op::I64LeSImm16Lhs, execute_i64_le_s_imm16_lhs, wasm::i64_le_s), - (u64, Op::I64LeUImm16Lhs, execute_i64_le_u_imm16_lhs, wasm::i64_le_u), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/conversion.rs b/crates/wasmi/src/engine/executor/instrs/conversion.rs deleted file mode 100644 index e291aeb2085..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/conversion.rs +++ /dev/null @@ -1,60 +0,0 @@ -use super::Executor; -use crate::{core::wasm, ir::Slot, Error}; - -#[cfg(doc)] -use crate::ir::Op; - -macro_rules! impl_fallible_conversion_impls { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, input: Slot) -> Result<(), Error> { - self.try_execute_unary(result, input, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_unary_executors! { - (Op::I32WrapI64, execute_i32_wrap_i64, wasm::i32_wrap_i64), - - (Op::I32TruncSatF32S, execute_i32_trunc_sat_f32_s, wasm::i32_trunc_sat_f32_s), - (Op::I32TruncSatF32U, execute_i32_trunc_sat_f32_u, wasm::i32_trunc_sat_f32_u), - (Op::I32TruncSatF64S, execute_i32_trunc_sat_f64_s, wasm::i32_trunc_sat_f64_s), - (Op::I32TruncSatF64U, execute_i32_trunc_sat_f64_u, wasm::i32_trunc_sat_f64_u), - (Op::I64TruncSatF32S, execute_i64_trunc_sat_f32_s, wasm::i64_trunc_sat_f32_s), - (Op::I64TruncSatF32U, execute_i64_trunc_sat_f32_u, wasm::i64_trunc_sat_f32_u), - (Op::I64TruncSatF64S, execute_i64_trunc_sat_f64_s, wasm::i64_trunc_sat_f64_s), - (Op::I64TruncSatF64U, execute_i64_trunc_sat_f64_u, wasm::i64_trunc_sat_f64_u), - - (Op::I32Extend8S, execute_i32_extend8_s, wasm::i32_extend8_s), - (Op::I32Extend16S, execute_i32_extend16_s, wasm::i32_extend16_s), - (Op::I64Extend8S, execute_i64_extend8_s, wasm::i64_extend8_s), - (Op::I64Extend16S, execute_i64_extend16_s, wasm::i64_extend16_s), - (Op::I64Extend32S, execute_i64_extend32_s, wasm::i64_extend32_s), - - (Op::F32DemoteF64, execute_f32_demote_f64, wasm::f32_demote_f64), - (Op::F64PromoteF32, execute_f64_promote_f32, wasm::f64_promote_f32), - - (Op::F32ConvertI32S, execute_f32_convert_i32_s, wasm::f32_convert_i32_s), - (Op::F32ConvertI32U, execute_f32_convert_i32_u, wasm::f32_convert_i32_u), - (Op::F32ConvertI64S, execute_f32_convert_i64_s, wasm::f32_convert_i64_s), - (Op::F32ConvertI64U, execute_f32_convert_i64_u, wasm::f32_convert_i64_u), - (Op::F64ConvertI32S, execute_f64_convert_i32_s, wasm::f64_convert_i32_s), - (Op::F64ConvertI32U, execute_f64_convert_i32_u, wasm::f64_convert_i32_u), - (Op::F64ConvertI64S, execute_f64_convert_i64_s, wasm::f64_convert_i64_s), - (Op::F64ConvertI64U, execute_f64_convert_i64_u, wasm::f64_convert_i64_u), - } - - impl_fallible_conversion_impls! { - (Op::I32TruncF32S, execute_i32_trunc_f32_s, wasm::i32_trunc_f32_s), - (Op::I32TruncF32U, execute_i32_trunc_f32_u, wasm::i32_trunc_f32_u), - (Op::I32TruncF64S, execute_i32_trunc_f64_s, wasm::i32_trunc_f64_s), - (Op::I32TruncF64U, execute_i32_trunc_f64_u, wasm::i32_trunc_f64_u), - (Op::I64TruncF32S, execute_i64_trunc_f32_s, wasm::i64_trunc_f32_s), - (Op::I64TruncF32U, execute_i64_trunc_f32_u, wasm::i64_trunc_f32_u), - (Op::I64TruncF64S, execute_i64_trunc_f64_s, wasm::i64_trunc_f64_s), - (Op::I64TruncF64U, execute_i64_trunc_f64_u, wasm::i64_trunc_f64_u), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/copy.rs b/crates/wasmi/src/engine/executor/instrs/copy.rs deleted file mode 100644 index 12e9cdd9f43..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/copy.rs +++ /dev/null @@ -1,117 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::UntypedVal, - engine::utils::unreachable_unchecked, - ir::{AnyConst32, Const32, FixedSlotSpan, Op, Slot, SlotSpan}, -}; -use core::slice; - -impl Executor<'_> { - /// Executes a generic `copy` [`Op`]. - fn execute_copy_impl(&mut self, result: Slot, value: T, f: fn(&mut Self, T) -> UntypedVal) { - let value = f(self, value); - self.set_stack_slot(result, value); - self.next_instr() - } - - /// Executes an [`Op::Copy`]. - pub fn execute_copy(&mut self, result: Slot, value: Slot) { - self.execute_copy_impl(result, value, |this, value| this.get_stack_slot(value)) - } - - /// Executes an [`Op::Copy2`]. - pub fn execute_copy_2(&mut self, results: FixedSlotSpan<2>, values: [Slot; 2]) { - self.execute_copy_2_impl(results, values); - self.next_instr() - } - - /// Internal implementation of [`Op::Copy2`] execution. - fn execute_copy_2_impl(&mut self, results: FixedSlotSpan<2>, values: [Slot; 2]) { - let result0 = results.span().head(); - let result1 = result0.next(); - // We need `tmp` in case `results[0] == values[1]` to avoid overwriting `values[1]` before reading it. - let tmp = self.get_stack_slot(values[1]); - self.set_stack_slot(result0, self.get_stack_slot(values[0])); - self.set_stack_slot(result1, tmp); - } - - /// Executes an [`Op::CopyImm32`]. - pub fn execute_copy_imm32(&mut self, result: Slot, value: AnyConst32) { - self.execute_copy_impl(result, value, |_, value| UntypedVal::from(u32::from(value))) - } - - /// Executes an [`Op::CopyI64Imm32`]. - pub fn execute_copy_i64imm32(&mut self, result: Slot, value: Const32) { - self.execute_copy_impl(result, value, |_, value| UntypedVal::from(i64::from(value))) - } - - /// Executes an [`Op::CopyF64Imm32`]. - pub fn execute_copy_f64imm32(&mut self, result: Slot, value: Const32) { - self.execute_copy_impl(result, value, |_, value| UntypedVal::from(f64::from(value))) - } - - /// Executes an [`Op::CopySpan`]. - /// - /// # Note - /// - /// - This instruction assumes that `results` and `values` do _not_ overlap - /// and thus can copy all the elements without a costly temporary buffer. - /// - If `results` and `values` _do_ overlap [`Op::CopySpan`] is used. - pub fn execute_copy_span(&mut self, results: SlotSpan, values: SlotSpan, len: u16) { - self.execute_copy_span_impl(results, values, len); - self.next_instr(); - } - - /// Internal implementation of [`Op::CopySpan`] execution. - pub fn execute_copy_span_impl(&mut self, results: SlotSpan, values: SlotSpan, len: u16) { - let results = results.iter(len); - let values = values.iter(len); - for (result, value) in results.into_iter().zip(values.into_iter()) { - let value = self.get_stack_slot(value); - self.set_stack_slot(result, value); - } - } - - /// Executes an [`Op::CopyMany`]. - pub fn execute_copy_many(&mut self, results: SlotSpan, values: [Slot; 2]) { - self.ip.add(1); - self.ip = self.execute_copy_many_impl(self.ip, results, &values); - self.next_instr() - } - - /// Internal implementation of [`Op::CopyMany`] execution. - pub fn execute_copy_many_impl( - &mut self, - ip: InstructionPtr, - results: SlotSpan, - values: &[Slot], - ) -> InstructionPtr { - let mut ip = ip; - let mut result = results.head(); - let mut copy_values = |values: &[Slot]| { - for &value in values { - let value = self.get_stack_slot(value); - self.set_stack_slot(result, value); - result = result.next(); - } - }; - copy_values(values); - while let Op::SlotList { regs } = ip.get() { - copy_values(regs); - ip.add(1); - } - let values = match ip.get() { - Op::Slot { slot } => slice::from_ref(slot), - Op::Slot2 { slots } => slots, - Op::Slot3 { slots } => slots, - unexpected => { - // Safety: Wasmi translator guarantees that slot-list finalizer exists. - unsafe { - unreachable_unchecked!("expected slot-list finalizer but found: {unexpected:?}") - } - } - }; - copy_values(values); - ip - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/global.rs b/crates/wasmi/src/engine/executor/instrs/global.rs deleted file mode 100644 index 1f6484cfa2a..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/global.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::Executor; -use crate::{ - core::{hint, UntypedVal}, - ir::{index, Const16, Slot}, - store::StoreInner, -}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - /// Executes an [`Op::GlobalGet`]. - pub fn execute_global_get(&mut self, store: &StoreInner, result: Slot, global: index::Global) { - let value = match u32::from(global) { - 0 => unsafe { self.cache.global.get() }, - _ => { - hint::cold(); - let global = self.get_global(global); - *store.resolve_global(&global).get_untyped() - } - }; - self.set_stack_slot(result, value); - self.next_instr() - } - - /// Executes an [`Op::GlobalSet`]. - pub fn execute_global_set( - &mut self, - store: &mut StoreInner, - global: index::Global, - input: Slot, - ) { - let input = self.get_stack_slot(input); - self.execute_global_set_impl(store, global, input) - } - - /// Executes an [`Op::GlobalSetI32Imm16`]. - pub fn execute_global_set_i32imm16( - &mut self, - store: &mut StoreInner, - global: index::Global, - input: Const16, - ) { - let input = i32::from(input).into(); - self.execute_global_set_impl(store, global, input) - } - - /// Executes an [`Op::GlobalSetI64Imm16`]. - pub fn execute_global_set_i64imm16( - &mut self, - store: &mut StoreInner, - global: index::Global, - input: Const16, - ) { - let input = i64::from(input).into(); - self.execute_global_set_impl(store, global, input) - } - - /// Executes a generic `global.set` instruction. - fn execute_global_set_impl( - &mut self, - store: &mut StoreInner, - global: index::Global, - new_value: UntypedVal, - ) { - match u32::from(global) { - 0 => unsafe { self.cache.global.set(new_value) }, - _ => { - hint::cold(); - let global = self.get_global(global); - let mut ptr = store.resolve_global_mut(&global).get_untyped_ptr(); - // Safety: - // - Wasmi translation won't create `global.set` instructions for immutable globals. - // - Wasm validation ensures that values with matching types are written to globals. - unsafe { - *ptr.as_mut() = new_value; - } - } - }; - self.next_instr() - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/load.rs b/crates/wasmi/src/engine/executor/instrs/load.rs deleted file mode 100644 index 7577c461d8d..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/load.rs +++ /dev/null @@ -1,411 +0,0 @@ -use super::Executor; -use crate::{ - core::{wasm, UntypedVal, WriteAs}, - ir::{index::Memory, Address32, Offset16, Offset64, Offset64Hi, Offset64Lo, Slot}, - store::StoreInner, - Error, - TrapCode, -}; - -#[cfg(feature = "simd")] -use crate::{core::simd, V128}; - -#[cfg(doc)] -use crate::ir::Op; - -/// The function signature of Wasm load operations. -type WasmLoadOp = fn(memory: &[u8], ptr: u64, offset: u64) -> Result; - -/// The function signature of Wasm load operations. -type WasmLoadAtOp = fn(memory: &[u8], address: usize) -> Result; - -impl Executor<'_> { - /// Returns the register `value` and `offset` parameters for a `load` [`Op`]. - fn fetch_ptr_and_offset_hi(&self) -> (Slot, Offset64Hi) { - // Safety: Wasmi translation guarantees that `Op::SlotAndImm32` exists. - unsafe { self.fetch_reg_and_offset_hi() } - } - - /// Executes a generic Wasm `load[N_{s|u}]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.load` - /// - `{i32, i64}.load8_s` - /// - `{i32, i64}.load8_u` - /// - `{i32, i64}.load16_s` - /// - `{i32, i64}.load16_u` - /// - `i64.load32_s` - /// - `i64.load32_u` - fn execute_load_extend( - &mut self, - store: &StoreInner, - memory: Memory, - result: Slot, - address: u64, - offset: Offset64, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_memory_bytes(memory, store); - let loaded_value = load_extend(memory, address, u64::from(offset))?; - self.set_stack_slot_as::(result, loaded_value); - Ok(()) - } - - /// Executes a generic Wasm `load[N_{s|u}]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.load` - /// - `{i32, i64}.load8_s` - /// - `{i32, i64}.load8_u` - /// - `{i32, i64}.load16_s` - /// - `{i32, i64}.load16_u` - /// - `i64.load32_s` - /// - `i64.load32_u` - fn execute_load_extend_at( - &mut self, - store: &StoreInner, - memory: Memory, - result: Slot, - address: Address32, - load_extend_at: WasmLoadAtOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_memory_bytes(memory, store); - let loaded_value = load_extend_at(memory, usize::from(address))?; - self.set_stack_slot_as::(result, loaded_value); - Ok(()) - } - - /// Executes a generic Wasm `store[N_{s|u}]` operation on the default memory. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.load` - /// - `{i32, i64}.load8_s` - /// - `{i32, i64}.load8_u` - /// - `{i32, i64}.load16_s` - /// - `{i32, i64}.load16_u` - /// - `i64.load32_s` - /// - `i64.load32_u` - fn execute_load_extend_mem0( - &mut self, - result: Slot, - address: u64, - offset: Offset64, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_default_memory_bytes(); - let loaded_value = load_extend(memory, address, u64::from(offset))?; - self.set_stack_slot_as::(result, loaded_value); - Ok(()) - } - - /// Executes a generic `load` [`Op`]. - fn execute_load_impl( - &mut self, - store: &StoreInner, - result: Slot, - offset_lo: Offset64Lo, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let (ptr, offset_hi) = self.fetch_ptr_and_offset_hi(); - let memory = self.fetch_optional_memory(2); - let address = self.get_stack_slot_as::(ptr); - let offset = Offset64::combine(offset_hi, offset_lo); - self.execute_load_extend::(store, memory, result, address, offset, load_extend)?; - self.try_next_instr_at(2) - } - - /// Executes a generic `load_at` [`Op`]. - fn execute_load_at_impl( - &mut self, - store: &StoreInner, - result: Slot, - address: Address32, - load_extend_at: WasmLoadAtOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_optional_memory(1); - self.execute_load_extend_at::(store, memory, result, address, load_extend_at)?; - self.try_next_instr() - } - - /// Executes a generic `load_offset16` [`Op`]. - fn execute_load_offset16_impl( - &mut self, - result: Slot, - ptr: Slot, - offset: Offset16, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let address = self.get_stack_slot_as::(ptr); - let offset = Offset64::from(offset); - self.execute_load_extend_mem0::(result, address, offset, load_extend)?; - self.try_next_instr() - } -} - -macro_rules! impl_execute_load { - ( $( - ( - $ty:ty, - (Op::$var_load:expr, $fn_load:ident), - (Op::$var_load_at:expr, $fn_load_at:ident), - (Op::$var_load_off16:expr, $fn_load_off16:ident), - $load_fn:expr, - $load_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_load), "`].")] - pub fn $fn_load(&mut self, store: &StoreInner, result: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - self.execute_load_impl(store, result, offset_lo, $load_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_load_at), "`].")] - pub fn $fn_load_at(&mut self, store: &StoreInner, result: Slot, address: Address32) -> Result<(), Error> { - self.execute_load_at_impl(store, result, address, $load_at_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_load_off16), "`].")] - pub fn $fn_load_off16(&mut self, result: Slot, ptr: Slot, offset: Offset16) -> Result<(), Error> { - self.execute_load_offset16_impl::<$ty>(result, ptr, offset, $load_fn) - } - )* - } -} - -impl Executor<'_> { - #[cfg(feature = "simd")] - impl_execute_load! { - ( - V128, - (Op::V128Load, execute_v128_load), - (Op::V128LoadAt, execute_v128_load_at), - (Op::V128LoadOffset16, execute_v128_load_offset16), - simd::v128_load, - simd::v128_load_at, - ), - ( - V128, - (Op::V128Load8x8S, execute_v128_load8x8_s), - (Op::V128Load8x8SAt, execute_v128_load8x8_s_at), - (Op::V128Load8x8SOffset16, execute_v128_load8x8_s_offset16), - simd::v128_load8x8_s, - simd::v128_load8x8_s_at, - ), - ( - V128, - (Op::V128Load8x8U, execute_v128_load8x8_u), - (Op::V128Load8x8UAt, execute_v128_load8x8_u_at), - (Op::V128Load8x8UOffset16, execute_v128_load8x8_u_offset16), - simd::v128_load8x8_u, - simd::v128_load8x8_u_at, - ), - ( - V128, - (Op::V128Load16x4S, execute_v128_load16x4_s), - (Op::V128Load16x4SAt, execute_v128_load16x4_s_at), - (Op::V128Load16x4SOffset16, execute_v128_load16x4_s_offset16), - simd::v128_load16x4_s, - simd::v128_load16x4_s_at, - ), - ( - V128, - (Op::V128Load16x4U, execute_v128_load16x4_u), - (Op::V128Load16x4UAt, execute_v128_load16x4_u_at), - (Op::V128Load16x4UOffset16, execute_v128_load16x4_u_offset16), - simd::v128_load16x4_u, - simd::v128_load16x4_u_at, - ), - ( - V128, - (Op::V128Load32x2S, execute_v128_load32x2_s), - (Op::V128Load32x2SAt, execute_v128_load32x2_s_at), - (Op::V128Load32x2SOffset16, execute_v128_load32x2_s_offset16), - simd::v128_load32x2_s, - simd::v128_load32x2_s_at, - ), - ( - V128, - (Op::V128Load32x2U, execute_v128_load32x2_u), - (Op::V128Load32x2UAt, execute_v128_load32x2_u_at), - (Op::V128Load32x2UOffset16, execute_v128_load32x2_u_offset16), - simd::v128_load32x2_u, - simd::v128_load32x2_u_at, - ), - ( - V128, - (Op::V128Load8Splat, execute_v128_load8_splat), - (Op::V128Load8SplatAt, execute_v128_load8_splat_at), - (Op::V128Load8SplatOffset16, execute_v128_load8_splat_offset16), - simd::v128_load8_splat, - simd::v128_load8_splat_at, - ), - ( - V128, - (Op::V128Load16Splat, execute_v128_load16_splat), - (Op::V128Load16SplatAt, execute_v128_load16_splat_at), - (Op::V128Load16SplatOffset16, execute_v128_load16_splat_offset16), - simd::v128_load16_splat, - simd::v128_load16_splat_at, - ), - ( - V128, - (Op::V128Load32Splat, execute_v128_load32_splat), - (Op::V128Load32SplatAt, execute_v128_load32_splat_at), - (Op::V128Load32SplatOffset16, execute_v128_load32_splat_offset16), - simd::v128_load32_splat, - simd::v128_load32_splat_at, - ), - ( - V128, - (Op::V128Load64Splat, execute_v128_load64_splat), - (Op::V128Load64SplatAt, execute_v128_load64_splat_at), - (Op::V128Load64SplatOffset16, execute_v128_load64_splat_offset16), - simd::v128_load64_splat, - simd::v128_load64_splat_at, - ), - ( - V128, - (Op::V128Load32Zero, execute_v128_load32_zero), - (Op::V128Load32ZeroAt, execute_v128_load32_zero_at), - (Op::V128Load32ZeroOffset16, execute_v128_load32_zero_offset16), - simd::v128_load32_zero, - simd::v128_load32_zero_at, - ), - ( - V128, - (Op::V128Load64Zero, execute_v128_load64_zero), - (Op::V128Load64ZeroAt, execute_v128_load64_zero_at), - (Op::V128Load64ZeroOffset16, execute_v128_load64_zero_offset16), - simd::v128_load64_zero, - simd::v128_load64_zero_at, - ), - } - - impl_execute_load! { - ( - u32, - (Op::Load32, execute_load32), - (Op::Load32At, execute_load32_at), - (Op::Load32Offset16, execute_load32_offset16), - wasm::load32, - wasm::load32_at, - ), - ( - u64, - (Op::Load64, execute_load64), - (Op::Load64At, execute_load64_at), - (Op::Load64Offset16, execute_load64_offset16), - wasm::load64, - wasm::load64_at, - ), - - ( - i32, - (Op::I32Load8s, execute_i32_load8_s), - (Op::I32Load8sAt, execute_i32_load8_s_at), - (Op::I32Load8sOffset16, execute_i32_load8_s_offset16), - wasm::i32_load8_s, - wasm::i32_load8_s_at, - ), - ( - i32, - (Op::I32Load8u, execute_i32_load8_u), - (Op::I32Load8uAt, execute_i32_load8_u_at), - (Op::I32Load8uOffset16, execute_i32_load8_u_offset16), - wasm::i32_load8_u, - wasm::i32_load8_u_at, - ), - ( - i32, - (Op::I32Load16s, execute_i32_load16_s), - (Op::I32Load16sAt, execute_i32_load16_s_at), - (Op::I32Load16sOffset16, execute_i32_load16_s_offset16), - wasm::i32_load16_s, - wasm::i32_load16_s_at, - ), - ( - i32, - (Op::I32Load16u, execute_i32_load16_u), - (Op::I32Load16uAt, execute_i32_load16_u_at), - (Op::I32Load16uOffset16, execute_i32_load16_u_offset16), - wasm::i32_load16_u, - wasm::i32_load16_u_at, - ), - - ( - i64, - (Op::I64Load8s, execute_i64_load8_s), - (Op::I64Load8sAt, execute_i64_load8_s_at), - (Op::I64Load8sOffset16, execute_i64_load8_s_offset16), - wasm::i64_load8_s, - wasm::i64_load8_s_at, - ), - ( - i64, - (Op::I64Load8u, execute_i64_load8_u), - (Op::I64Load8uAt, execute_i64_load8_u_at), - (Op::I64Load8uOffset16, execute_i64_load8_u_offset16), - wasm::i64_load8_u, - wasm::i64_load8_u_at, - ), - ( - i64, - (Op::I64Load16s, execute_i64_load16_s), - (Op::I64Load16sAt, execute_i64_load16_s_at), - (Op::I64Load16sOffset16, execute_i64_load16_s_offset16), - wasm::i64_load16_s, - wasm::i64_load16_s_at, - ), - ( - i64, - (Op::I64Load16u, execute_i64_load16_u), - (Op::I64Load16uAt, execute_i64_load16_u_at), - (Op::I64Load16uOffset16, execute_i64_load16_u_offset16), - wasm::i64_load16_u, - wasm::i64_load16_u_at, - ), - ( - i64, - (Op::I64Load32s, execute_i64_load32_s), - (Op::I64Load32sAt, execute_i64_load32_s_at), - (Op::I64Load32sOffset16, execute_i64_load32_s_offset16), - wasm::i64_load32_s, - wasm::i64_load32_s_at, - ), - ( - i64, - (Op::I64Load32u, execute_i64_load32_u), - (Op::I64Load32uAt, execute_i64_load32_u_at), - (Op::I64Load32uOffset16, execute_i64_load32_u_offset16), - wasm::i64_load32_u, - wasm::i64_load32_u_at, - ), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/memory.rs b/crates/wasmi/src/engine/executor/instrs/memory.rs deleted file mode 100644 index 5cbf1b7b7d6..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/memory.rs +++ /dev/null @@ -1,281 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - engine::{utils::unreachable_unchecked, ResumableOutOfFuelError}, - errors::MemoryError, - ir::{ - index::{Data, Memory}, - Op, - Slot, - }, - store::{PrunedStore, StoreError, StoreInner}, - Error, - TrapCode, -}; - -impl Executor<'_> { - /// Returns the [`Op::MemoryIndex`] parameter for an [`Op`]. - fn fetch_memory_index(&self, offset: usize) -> Memory { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::MemoryIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::MemoryIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::MemoryIndex` but found: {unexpected:?}") - } - } - } - } - - /// Returns the [`Op::DataIndex`] parameter for an [`Op`]. - fn fetch_data_segment_index(&self, offset: usize) -> Data { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::DataIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::DataIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::DataIndex` but found: {unexpected:?}") - } - } - } - } - - /// Executes an [`Op::DataDrop`]. - pub fn execute_data_drop(&mut self, store: &mut StoreInner, segment_index: Data) { - let segment = self.get_data_segment(segment_index); - store.resolve_data_mut(&segment).drop_bytes(); - self.next_instr(); - } - - /// Executes an [`Op::MemorySize`]. - pub fn execute_memory_size(&mut self, store: &StoreInner, result: Slot, memory: Memory) { - self.execute_memory_size_impl(store, result, memory); - self.next_instr() - } - - /// Underlying implementation of [`Op::MemorySize`]. - fn execute_memory_size_impl(&mut self, store: &StoreInner, result: Slot, memory: Memory) { - let memory = self.get_memory(memory); - let size = store.resolve_memory(&memory).size(); - self.set_stack_slot(result, size); - } - - /// Executes an [`Op::MemoryGrow`]. - pub fn execute_memory_grow( - &mut self, - store: &mut PrunedStore, - result: Slot, - delta: Slot, - ) -> Result<(), Error> { - let delta: u64 = self.get_stack_slot_as(delta); - let memory = self.fetch_memory_index(1); - if delta == 0 { - // Case: growing by 0 pages means there is nothing to do - self.execute_memory_size_impl(store.inner(), result, memory); - return self.try_next_instr_at(2); - } - let memory = self.get_memory(memory); - let return_value = match store.grow_memory(&memory, delta) { - Ok(return_value) => { - // The `memory.grow` operation might have invalidated the cached - // linear memory so we need to reset it in order for the cache to - // reload in case it is used again. - // - // Safety: the instance has not changed thus calling this is valid. - unsafe { self.cache.update_memory(store.inner_mut()) }; - return_value - } - Err(StoreError::External( - MemoryError::OutOfBoundsGrowth | MemoryError::OutOfSystemMemory, - )) => { - let memory_ty = store.inner().resolve_memory(&memory).ty(); - match memory_ty.is_64() { - true => u64::MAX, - false => u64::from(u32::MAX), - } - } - Err(StoreError::External(MemoryError::OutOfFuel { required_fuel })) => { - return Err(Error::from(ResumableOutOfFuelError::new(required_fuel))) - } - Err(StoreError::External(MemoryError::ResourceLimiterDeniedAllocation)) => { - return Err(Error::from(TrapCode::GrowthOperationLimited)) - } - Err(error) => { - panic!("`table.grow`: internal interpreter error: {error}") - } - }; - self.set_stack_slot(result, return_value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::MemoryCopy`]. - pub fn execute_memory_copy( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u64 = self.get_stack_slot_as(src); - let len: u64 = self.get_stack_slot_as(len); - let Ok(dst_index) = usize::try_from(dst) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(src_index) = usize::try_from(src) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(len) = usize::try_from(len) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let dst_memory = self.fetch_memory_index(1); - let src_memory = self.fetch_memory_index(2); - if src_memory == dst_memory { - return self - .execute_memory_copy_within_impl(store, src_memory, dst_index, src_index, len); - } - let (src_memory, dst_memory, fuel) = store.resolve_memory_pair_and_fuel( - &self.get_memory(src_memory), - &self.get_memory(dst_memory), - ); - // These accesses just perform the bounds checks required by the Wasm spec. - let src_bytes = src_memory - .data() - .get(src_index..) - .and_then(|memory| memory.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - let dst_bytes = dst_memory - .data_mut() - .get_mut(dst_index..) - .and_then(|memory| memory.get_mut(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - dst_bytes.copy_from_slice(src_bytes); - self.try_next_instr_at(3) - } - - /// Executes a generic `memory.copy` instruction. - fn execute_memory_copy_within_impl( - &mut self, - store: &mut StoreInner, - memory: Memory, - dst_index: usize, - src_index: usize, - len: usize, - ) -> Result<(), Error> { - let memory = self.get_memory(memory); - let (memory, fuel) = store.resolve_memory_and_fuel_mut(&memory); - let bytes = memory.data_mut(); - // These accesses just perform the bounds checks required by the Wasm spec. - bytes - .get(src_index..) - .and_then(|memory| memory.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - bytes - .get(dst_index..) - .and_then(|memory| memory.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - bytes.copy_within(src_index..src_index.wrapping_add(len), dst_index); - self.try_next_instr_at(3) - } - - /// Executes an [`Op::MemoryFill`]. - pub fn execute_memory_fill( - &mut self, - store: &mut StoreInner, - dst: Slot, - value: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let value: u8 = self.get_stack_slot_as(value); - let len: u64 = self.get_stack_slot_as(len); - self.execute_memory_fill_impl(store, dst, value, len) - } - - /// Executes an [`Op::MemoryFillImm`]. - pub fn execute_memory_fill_imm( - &mut self, - store: &mut StoreInner, - dst: Slot, - value: u8, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let len: u64 = self.get_stack_slot_as(len); - self.execute_memory_fill_impl(store, dst, value, len) - } - - /// Executes a generic `memory.fill` instruction. - #[inline(never)] - fn execute_memory_fill_impl( - &mut self, - store: &mut StoreInner, - dst: u64, - value: u8, - len: u64, - ) -> Result<(), Error> { - let Ok(dst) = usize::try_from(dst) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(len) = usize::try_from(len) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let memory = self.fetch_memory_index(1); - let memory = self.get_memory(memory); - let (memory, fuel) = store.resolve_memory_and_fuel_mut(&memory); - let slice = memory - .data_mut() - .get_mut(dst..) - .and_then(|memory| memory.get_mut(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - slice.fill(value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::MemoryInit`]. - pub fn execute_memory_init( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u32 = self.get_stack_slot_as(src); - let len: u32 = self.get_stack_slot_as(len); - let Ok(dst_index) = usize::try_from(dst) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(src_index) = usize::try_from(src) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(len) = usize::try_from(len) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let memory_index: Memory = self.fetch_memory_index(1); - let data_index: Data = self.fetch_data_segment_index(2); - let (memory, data, fuel) = store.resolve_memory_init_params( - &self.get_memory(memory_index), - &self.get_data_segment(data_index), - ); - let memory = memory - .data_mut() - .get_mut(dst_index..) - .and_then(|memory| memory.get_mut(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - let data = data - .bytes() - .get(src_index..) - .and_then(|data| data.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - memory.copy_from_slice(data); - self.try_next_instr_at(3) - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/return_.rs b/crates/wasmi/src/engine/executor/instrs/return_.rs deleted file mode 100644 index 10f5bdc7f16..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/return_.rs +++ /dev/null @@ -1,237 +0,0 @@ -use super::{ControlFlow, Executor, InstructionPtr}; -use crate::{ - core::UntypedVal, - engine::{executor::stack::FrameSlots, utils::unreachable_unchecked}, - ir::{AnyConst32, BoundedSlotSpan, Const32, Op, Slot, SlotSpan}, - store::StoreInner, -}; -use core::slice; - -impl Executor<'_> { - /// Returns the execution to the caller. - /// - /// Any return values are expected to already have been transferred - /// from the returning callee to the caller. - fn return_impl(&mut self, store: &mut StoreInner) -> ControlFlow { - let (returned, popped_instance) = self - .stack - .calls - .pop() - .expect("the executing call frame is always on the stack"); - self.stack.values.truncate(returned.frame_offset()); - let new_instance = popped_instance.and_then(|_| self.stack.calls.instance()); - if let Some(new_instance) = new_instance { - self.cache.update(store, new_instance); - } - match self.stack.calls.peek() { - Some(caller) => { - Self::init_call_frame_impl( - &mut self.stack.values, - &mut self.sp, - &mut self.ip, - caller, - ); - ControlFlow::Continue(()) - } - None => ControlFlow::Break(()), - } - } - - /// Execute an [`Op::Return`]. - pub fn execute_return(&mut self, store: &mut StoreInner) -> ControlFlow { - self.return_impl(store) - } - - /// Returns the [`FrameSlots`] of the caller and the [`SlotSpan`] of the results. - /// - /// The returned [`FrameSlots`] is valid for all [`Slot`] in the returned [`SlotSpan`]. - fn return_caller_results(&mut self) -> (FrameSlots, SlotSpan) { - let (callee, caller) = self - .stack - .calls - .peek_2() - .expect("the callee must exist on the call stack"); - match caller { - Some(caller) => { - // Case: we need to return the `value` back to the caller frame. - // - // In this case we transfer the single return `value` to the `results` - // register span of the caller's call frame. - // - // Safety: The caller call frame is still live on the value stack - // and therefore it is safe to acquire its value stack pointer. - let caller_sp = unsafe { self.stack.values.stack_ptr_at(caller.base_offset()) }; - let results = callee.results(); - (caller_sp, results) - } - None => { - // Case: the root call frame is returning. - // - // In this case we transfer the single return `value` to the root - // register span of the entire value stack which is simply its zero index. - let dst_sp = self.stack.values.root_stack_ptr(); - let results = SlotSpan::new(Slot::from(0)); - (dst_sp, results) - } - } - } - - /// Execute a generic return [`Op`] returning a single value. - fn execute_return_value( - &mut self, - store: &mut StoreInner, - value: T, - f: fn(&Self, T) -> UntypedVal, - ) -> ControlFlow { - let (mut caller_sp, results) = self.return_caller_results(); - let value = f(self, value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { caller_sp.set(results.head(), value) } - self.return_impl(store) - } - - /// Execute an [`Op::ReturnSlot`] returning a single [`Slot`] value. - pub fn execute_return_reg(&mut self, store: &mut StoreInner, value: Slot) -> ControlFlow { - self.execute_return_value(store, value, Self::get_stack_slot) - } - - /// Execute an [`Op::ReturnSlot2`] returning two [`Slot`] values. - pub fn execute_return_reg2( - &mut self, - store: &mut StoreInner, - values: [Slot; 2], - ) -> ControlFlow { - self.execute_return_reg_n_impl::<2>(store, values) - } - - /// Execute an [`Op::ReturnSlot3`] returning three [`Slot`] values. - pub fn execute_return_reg3( - &mut self, - store: &mut StoreInner, - values: [Slot; 3], - ) -> ControlFlow { - self.execute_return_reg_n_impl::<3>(store, values) - } - - /// Executes an [`Op::ReturnSlot2`] or [`Op::ReturnSlot3`] generically. - fn execute_return_reg_n_impl( - &mut self, - store: &mut StoreInner, - values: [Slot; N], - ) -> ControlFlow { - let (mut caller_sp, results) = self.return_caller_results(); - debug_assert!(u16::try_from(N).is_ok()); - for (result, value) in results.iter(N as u16).zip(values) { - let value = self.get_stack_slot(value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { caller_sp.set(result, value) } - } - self.return_impl(store) - } - - /// Execute an [`Op::ReturnImm32`] returning a single 32-bit value. - pub fn execute_return_imm32( - &mut self, - store: &mut StoreInner, - value: AnyConst32, - ) -> ControlFlow { - self.execute_return_value(store, value, |_, value| u32::from(value).into()) - } - - /// Execute an [`Op::ReturnI64Imm32`] returning a single 32-bit encoded `i64` value. - pub fn execute_return_i64imm32( - &mut self, - store: &mut StoreInner, - value: Const32, - ) -> ControlFlow { - self.execute_return_value(store, value, |_, value| i64::from(value).into()) - } - - /// Execute an [`Op::ReturnF64Imm32`] returning a single 32-bit encoded `f64` value. - pub fn execute_return_f64imm32( - &mut self, - store: &mut StoreInner, - value: Const32, - ) -> ControlFlow { - self.execute_return_value(store, value, |_, value| f64::from(value).into()) - } - - /// Execute an [`Op::ReturnSpan`] returning many values. - pub fn execute_return_span( - &mut self, - store: &mut StoreInner, - values: BoundedSlotSpan, - ) -> ControlFlow { - let (mut caller_sp, results) = self.return_caller_results(); - let results = results.iter(values.len()); - for (result, value) in results.zip(values) { - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - let value = self.get_stack_slot(value); - unsafe { caller_sp.set(result, value) } - } - self.return_impl(store) - } - - /// Execute an [`Op::ReturnMany`] returning many values. - pub fn execute_return_many( - &mut self, - store: &mut StoreInner, - values: [Slot; 3], - ) -> ControlFlow { - self.ip.add(1); - self.copy_many_return_values(self.ip, &values); - self.return_impl(store) - } - - /// Copies many return values to the caller frame. - /// - /// # Note - /// - /// Used by the execution logic for - /// - /// - [`Op::ReturnMany`] - pub fn copy_many_return_values(&mut self, ip: InstructionPtr, values: &[Slot]) { - let (mut caller_sp, results) = self.return_caller_results(); - let mut result = results.head(); - let mut copy_results = |values: &[Slot]| { - for value in values { - let value = self.get_stack_slot(*value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { caller_sp.set(result, value) } - result = result.next(); - } - }; - copy_results(values); - let mut ip = ip; - while let Op::SlotList { regs } = ip.get() { - copy_results(regs); - ip.add(1); - } - let values = match ip.get() { - Op::Slot { slot } => slice::from_ref(slot), - Op::Slot2 { slots } => slots, - Op::Slot3 { slots } => slots, - unexpected => { - // Safety: Wasmi translation guarantees that a slot-list finalizer exists. - unsafe { - unreachable_unchecked!( - "unexpected slot-list finalizer but found: {unexpected:?}" - ) - } - } - }; - copy_results(values); - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/select.rs b/crates/wasmi/src/engine/executor/instrs/select.rs deleted file mode 100644 index 1aa23d2ab3f..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/select.rs +++ /dev/null @@ -1,140 +0,0 @@ -use super::{Executor, InstructionPtr, UntypedValueExt}; -use crate::{ - core::{wasm, ReadAs, UntypedVal}, - engine::utils::unreachable_unchecked, - ir::{Const16, Op, Slot}, -}; - -impl Executor<'_> { - /// Fetches two [`Slot`]s. - fn fetch_register_2(&self) -> (Slot, Slot) { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Slot2 { - slots: [slot0, slot1], - } => (slot0, slot1), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::Slot2`] exists. - unsafe { unreachable_unchecked!("expected `Op::Slot2` but found {unexpected:?}") } - } - } - } - - /// Executes a fused `cmp`+`select` instruction. - #[inline(always)] - fn execute_cmp_select_impl( - &mut self, - result: Slot, - lhs: Slot, - rhs: Slot, - f: fn(T, T) -> bool, - ) where - UntypedVal: ReadAs, - { - let (true_val, false_val) = self.fetch_register_2(); - let lhs: T = self.get_stack_slot_as(lhs); - let rhs: T = self.get_stack_slot_as(rhs); - let selected = self.get_stack_slot(match f(lhs, rhs) { - true => true_val, - false => false_val, - }); - self.set_stack_slot(result, selected); - self.next_instr_at(2); - } - - /// Executes a fused `cmp`+`select` instruction with immediate `rhs` parameter. - #[inline(always)] - fn execute_cmp_select_imm_rhs_impl( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - f: fn(T, T) -> bool, - ) where - UntypedVal: ReadAs, - T: From>, - { - let (true_val, false_val) = self.fetch_register_2(); - let lhs: T = self.get_stack_slot_as(lhs); - let rhs: T = rhs.into(); - let selected = self.get_stack_slot(match f(lhs, rhs) { - true => true_val, - false => false_val, - }); - self.set_stack_slot(result, selected); - self.next_instr_at(2); - } -} - -macro_rules! impl_cmp_select_for { - ( - $( - (Op::$doc_name:ident, $fn_name:ident, $op:expr) - ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($doc_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Slot) { - self.execute_cmp_select_impl(result, lhs, rhs, $op) - } - )* - }; -} - -macro_rules! impl_cmp_select_imm_rhs_for { - ( - $( - ($ty:ty, Op::$doc_name:ident, $fn_name:ident, $op:expr) - ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($doc_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { - self.execute_cmp_select_imm_rhs_impl::<$ty>(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_cmp_select_for! { - (Op::SelectI32Eq, execute_select_i32_eq, wasm::i32_eq), - (Op::SelectI32LtS, execute_select_i32_lt_s, wasm::i32_lt_s), - (Op::SelectI32LtU, execute_select_i32_lt_u, wasm::i32_lt_u), - (Op::SelectI32LeS, execute_select_i32_le_s, wasm::i32_le_s), - (Op::SelectI32LeU, execute_select_i32_le_u, wasm::i32_le_u), - (Op::SelectI32And, execute_select_i32_and, ::and), - (Op::SelectI32Or, execute_select_i32_or, ::or), - (Op::SelectI64Eq, execute_select_i64_eq, wasm::i64_eq), - (Op::SelectI64LtS, execute_select_i64_lt_s, wasm::i64_lt_s), - (Op::SelectI64LtU, execute_select_i64_lt_u, wasm::i64_lt_u), - (Op::SelectI64LeS, execute_select_i64_le_s, wasm::i64_le_s), - (Op::SelectI64LeU, execute_select_i64_le_u, wasm::i64_le_u), - (Op::SelectI64And, execute_select_i64_and, ::and), - (Op::SelectI64Or, execute_select_i64_or, ::or), - (Op::SelectF32Eq, execute_select_f32_eq, wasm::f32_eq), - (Op::SelectF32Lt, execute_select_f32_lt, wasm::f32_lt), - (Op::SelectF32Le, execute_select_f32_le, wasm::f32_le), - (Op::SelectF64Eq, execute_select_f64_eq, wasm::f64_eq), - (Op::SelectF64Lt, execute_select_f64_lt, wasm::f64_lt), - (Op::SelectF64Le, execute_select_f64_le, wasm::f64_le), - } - - impl_cmp_select_imm_rhs_for! { - (i32, Op::SelectI32EqImm16, execute_select_i32_eq_imm16, wasm::i32_eq), - (i32, Op::SelectI32LtSImm16Rhs, execute_select_i32_lt_s_imm16_rhs, wasm::i32_lt_s), - (u32, Op::SelectI32LtUImm16Rhs, execute_select_i32_lt_u_imm16_rhs, wasm::i32_lt_u), - (i32, Op::SelectI32LeSImm16Rhs, execute_select_i32_le_s_imm16_rhs, wasm::i32_le_s), - (u32, Op::SelectI32LeUImm16Rhs, execute_select_i32_le_u_imm16_rhs, wasm::i32_le_u), - (i32, Op::SelectI32AndImm16, execute_select_i32_and_imm16, UntypedValueExt::and), - (i32, Op::SelectI32OrImm16, execute_select_i32_or_imm16, UntypedValueExt::or), - (i64, Op::SelectI64EqImm16, execute_select_i64_eq_imm16, wasm::i64_eq), - (i64, Op::SelectI64LtSImm16Rhs, execute_select_i64_lt_s_imm16_rhs, wasm::i64_lt_s), - (u64, Op::SelectI64LtUImm16Rhs, execute_select_i64_lt_u_imm16_rhs, wasm::i64_lt_u), - (i64, Op::SelectI64LeSImm16Rhs, execute_select_i64_le_s_imm16_rhs, wasm::i64_le_s), - (u64, Op::SelectI64LeUImm16Rhs, execute_select_i64_le_u_imm16_rhs, wasm::i64_le_u), - (i64, Op::SelectI64AndImm16, execute_select_i64_and_imm16, UntypedValueExt::and), - (i64, Op::SelectI64OrImm16, execute_select_i64_or_imm16, UntypedValueExt::or), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/simd.rs b/crates/wasmi/src/engine/executor/instrs/simd.rs deleted file mode 100644 index ee66ddf1257..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/simd.rs +++ /dev/null @@ -1,814 +0,0 @@ -use super::Executor; -use crate::{ - core::{ - simd::{ - self, - ImmLaneIdx16, - ImmLaneIdx2, - ImmLaneIdx32, - ImmLaneIdx4, - ImmLaneIdx8, - IntoLaneIdx, - }, - UntypedVal, - WriteAs, - }, - engine::{executor::InstructionPtr, utils::unreachable_unchecked}, - ir::{index, Address32, AnyConst32, Offset64, Offset64Lo, Offset8, Op, ShiftAmount, Slot}, - store::StoreInner, - Error, - TrapCode, - V128, -}; - -#[cfg(doc)] -use crate::ir::Offset64Hi; - -impl Executor<'_> { - /// Fetches a [`Slot`] from an [`Op::Slot`] instruction parameter. - fn fetch_register(&self) -> Slot { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Slot { slot } => slot, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::Slot`] exists. - unsafe { unreachable_unchecked!("expected `Op::Slot` but found {unexpected:?}") } - } - } - } - - /// Fetches the [`Slot`] and [`Offset64Hi`] parameters for a load or store [`Op`]. - unsafe fn fetch_reg_and_lane(&self, delta: usize) -> (Slot, LaneType) - where - LaneType: TryFrom, - { - let mut addr: InstructionPtr = self.ip; - addr.add(delta); - match addr.get().filter_register_and_lane::() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::SlotAndImm32` but found: {instr:?}") - }, - } - } - - /// Returns the register `value` and `lane` parameters for a `load` [`Op`]. - pub fn fetch_value_and_lane(&self, delta: usize) -> (Slot, LaneType) - where - LaneType: TryFrom, - { - // Safety: Wasmi translation guarantees that `Op::SlotAndImm32` exists. - unsafe { self.fetch_reg_and_lane::(delta) } - } - - /// Fetches a [`Slot`] from an [`Op::Const32`] instruction parameter. - fn fetch_const32_as(&self) -> T - where - T: From, - { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Const32 { value } => value.into(), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::Const32`] exists. - unsafe { unreachable_unchecked!("expected `Op::Const32` but found {unexpected:?}") } - } - } - } - - /// Fetches a [`Slot`] from an [`Op::I64Const32`] instruction parameter. - fn fetch_i64const32(&self) -> i64 { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::I64Const32 { value } => value.into(), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::I64Const32`] exists. - unsafe { - unreachable_unchecked!("expected `Op::I64Const32` but found {unexpected:?}") - } - } - } - } - - /// Fetches a [`Slot`] from an [`Op::F64Const32`] instruction parameter. - fn fetch_f64const32(&self) -> f64 { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::F64Const32 { value } => value.into(), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::F64Const32`] exists. - unsafe { - unreachable_unchecked!("expected `Op::F64Const32` but found {unexpected:?}") - } - } - } - } - - /// Executes an [`Op::I8x16Shuffle`] instruction. - pub fn execute_i8x16_shuffle(&mut self, result: Slot, lhs: Slot, rhs: Slot) { - let selector = self.fetch_register(); - let lhs = self.get_stack_slot_as::(lhs); - let rhs = self.get_stack_slot_as::(rhs); - let selector = self - .get_stack_slot_as::(selector) - .as_u128() - .to_ne_bytes() - .map(|lane| { - match ImmLaneIdx32::try_from(lane) { - Ok(lane) => lane, - Err(_) => { - // Safety: Wasmi translation guarantees that the indices are within bounds. - unsafe { unreachable_unchecked!("unexpected out of bounds index: {lane}") } - } - } - }); - self.set_stack_slot_as::(result, simd::i8x16_shuffle(lhs, rhs, selector)); - self.next_instr_at(2); - } -} - -macro_rules! impl_ternary_simd_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr $(,)?) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, a: Slot, b: Slot) { - let c = self.fetch_register(); - let a = self.get_stack_slot_as::(a); - let b = self.get_stack_slot_as::(b); - let c = self.get_stack_slot_as::(c); - self.set_stack_slot_as::(result, $op(a, b, c)); - self.next_instr_at(2); - } - )* - }; -} - -impl Executor<'_> { - impl_ternary_simd_executors! { - (Op::V128Bitselect, execute_v128_bitselect, simd::v128_bitselect), - ( - Op::I32x4RelaxedDotI8x16I7x16AddS, - execute_i32x4_relaxed_dot_i8x16_i7x16_add_s, - simd::i32x4_relaxed_dot_i8x16_i7x16_add_s, - ), - (Op::F32x4RelaxedMadd, execute_f32x4_relaxed_madd, simd::f32x4_relaxed_madd), - (Op::F32x4RelaxedNmadd, execute_f32x4_relaxed_nmadd, simd::f32x4_relaxed_nmadd), - (Op::F64x2RelaxedMadd, execute_f64x2_relaxed_madd, simd::f64x2_relaxed_madd), - (Op::F64x2RelaxedNmadd, execute_f64x2_relaxed_nmadd, simd::f64x2_relaxed_nmadd), - } -} - -macro_rules! impl_replace_lane_ops { - ( - $( - ($ty:ty, Op::$instr_name:ident, $exec_name:ident, $execute:expr) - ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($instr_name), "`].")] - pub fn $exec_name(&mut self, result: Slot, input: Slot, lane: <$ty as IntoLaneIdx>::LaneIdx) { - let value = self.fetch_register(); - let input = self.get_stack_slot_as::(input); - let value = self.get_stack_slot_as::<$ty>(value); - self.set_stack_slot_as::(result, $execute(input, lane, value)); - self.next_instr_at(2); - } - )* - }; -} - -impl Executor<'_> { - impl_replace_lane_ops! { - (i8, Op::I8x16ReplaceLane, execute_i8x16_replace_lane, simd::i8x16_replace_lane), - (i16, Op::I16x8ReplaceLane, execute_i16x8_replace_lane, simd::i16x8_replace_lane), - (i32, Op::I32x4ReplaceLane, execute_i32x4_replace_lane, simd::i32x4_replace_lane), - (i64, Op::I64x2ReplaceLane, execute_i64x2_replace_lane, simd::i64x2_replace_lane), - (f32, Op::F32x4ReplaceLane, execute_f32x4_replace_lane, simd::f32x4_replace_lane), - (f64, Op::F64x2ReplaceLane, execute_f64x2_replace_lane, simd::f64x2_replace_lane), - } - - /// Executes an [`Op::I8x16ReplaceLaneImm`] instruction. - pub fn execute_i8x16_replace_lane_imm( - &mut self, - result: Slot, - input: Slot, - lane: ImmLaneIdx16, - value: i8, - ) { - self.execute_replace_lane_impl(result, input, lane, value, 1, simd::i8x16_replace_lane) - } - - /// Executes an [`Op::I16x8ReplaceLaneImm`] instruction. - pub fn execute_i16x8_replace_lane_imm(&mut self, result: Slot, input: Slot, lane: ImmLaneIdx8) { - let value = self.fetch_const32_as::() as i16; - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::i16x8_replace_lane) - } - - /// Executes an [`Op::I32x4ReplaceLaneImm`] instruction. - pub fn execute_i32x4_replace_lane_imm(&mut self, result: Slot, input: Slot, lane: ImmLaneIdx4) { - let value = self.fetch_const32_as::(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::i32x4_replace_lane) - } - - /// Executes an [`Op::I64x2ReplaceLaneImm32`] instruction. - pub fn execute_i64x2_replace_lane_imm32( - &mut self, - result: Slot, - input: Slot, - lane: ImmLaneIdx2, - ) { - let value = self.fetch_i64const32(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::i64x2_replace_lane) - } - - /// Executes an [`Op::F32x4ReplaceLaneImm`] instruction. - pub fn execute_f32x4_replace_lane_imm(&mut self, result: Slot, input: Slot, lane: ImmLaneIdx4) { - let value = self.fetch_const32_as::(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::f32x4_replace_lane) - } - - /// Executes an [`Op::F64x2ReplaceLaneImm32`] instruction. - pub fn execute_f64x2_replace_lane_imm32( - &mut self, - result: Slot, - input: Slot, - lane: ImmLaneIdx2, - ) { - let value = self.fetch_f64const32(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::f64x2_replace_lane) - } - - /// Generically execute a SIMD replace lane instruction. - fn execute_replace_lane_impl( - &mut self, - result: Slot, - input: Slot, - lane: LaneType, - value: T, - delta: usize, - eval: fn(V128, LaneType, T) -> V128, - ) { - let input = self.get_stack_slot_as::(input); - self.set_stack_slot_as::(result, eval(input, lane, value)); - self.next_instr_at(delta); - } - - impl_unary_executors! { - (Op::V128AnyTrue, execute_v128_any_true, simd::v128_any_true), - (Op::I8x16AllTrue, execute_i8x16_all_true, simd::i8x16_all_true), - (Op::I8x16Bitmask, execute_i8x16_bitmask, simd::i8x16_bitmask), - (Op::I16x8AllTrue, execute_i16x8_all_true, simd::i16x8_all_true), - (Op::I16x8Bitmask, execute_i16x8_bitmask, simd::i16x8_bitmask), - (Op::I32x4AllTrue, execute_i32x4_all_true, simd::i32x4_all_true), - (Op::I32x4Bitmask, execute_i32x4_bitmask, simd::i32x4_bitmask), - (Op::I64x2AllTrue, execute_i64x2_all_true, simd::i64x2_all_true), - (Op::I64x2Bitmask, execute_i64x2_bitmask, simd::i64x2_bitmask), - - (Op::I8x16Neg, execute_i8x16_neg, simd::i8x16_neg), - (Op::I16x8Neg, execute_i16x8_neg, simd::i16x8_neg), - (Op::I16x8Neg, execute_i32x4_neg, simd::i32x4_neg), - (Op::I16x8Neg, execute_i64x2_neg, simd::i64x2_neg), - (Op::I16x8Neg, execute_f32x4_neg, simd::f32x4_neg), - (Op::I16x8Neg, execute_f64x2_neg, simd::f64x2_neg), - - (Op::I8x16Abs, execute_i8x16_abs, simd::i8x16_abs), - (Op::I16x8Abs, execute_i16x8_abs, simd::i16x8_abs), - (Op::I16x8Abs, execute_i32x4_abs, simd::i32x4_abs), - (Op::I16x8Abs, execute_i64x2_abs, simd::i64x2_abs), - (Op::I16x8Abs, execute_f32x4_abs, simd::f32x4_abs), - (Op::I16x8Abs, execute_f64x2_abs, simd::f64x2_abs), - - (Op::I8x16Splat, execute_i8x16_splat, simd::i8x16_splat), - (Op::I16x8Splat, execute_i16x8_splat, simd::i16x8_splat), - (Op::I32x4Splat, execute_i32x4_splat, simd::i32x4_splat), - (Op::I64x2Splat, execute_i64x2_splat, simd::i64x2_splat), - (Op::F32x4Splat, execute_f32x4_splat, simd::f32x4_splat), - (Op::F64x2Splat, execute_f64x2_splat, simd::f64x2_splat), - - (Op::I16x8ExtaddPairwiseI8x16S, execute_i16x8_extadd_pairwise_i8x16_s, simd::i16x8_extadd_pairwise_i8x16_s), - (Op::I16x8ExtaddPairwiseI8x16U, execute_i16x8_extadd_pairwise_i8x16_u, simd::i16x8_extadd_pairwise_i8x16_u), - (Op::I32x4ExtaddPairwiseI16x8S, execute_i32x4_extadd_pairwise_i16x8_s, simd::i32x4_extadd_pairwise_i16x8_s), - (Op::I32x4ExtaddPairwiseI16x8U, execute_i32x4_extadd_pairwise_i16x8_u, simd::i32x4_extadd_pairwise_i16x8_u), - - (Op::F32x4Ceil, execute_f32x4_ceil, simd::f32x4_ceil), - (Op::F32x4Floor, execute_f32x4_floor, simd::f32x4_floor), - (Op::F32x4Trunc, execute_f32x4_trunc, simd::f32x4_trunc), - (Op::F32x4Nearest, execute_f32x4_nearest, simd::f32x4_nearest), - (Op::F32x4Sqrt, execute_f32x4_sqrt, simd::f32x4_sqrt), - (Op::F64x2Ceil, execute_f64x2_ceil, simd::f64x2_ceil), - (Op::F64x2Floor, execute_f64x2_floor, simd::f64x2_floor), - (Op::F64x2Trunc, execute_f64x2_trunc, simd::f64x2_trunc), - (Op::F64x2Nearest, execute_f64x2_nearest, simd::f64x2_nearest), - (Op::F64x2Sqrt, execute_f64x2_sqrt, simd::f64x2_sqrt), - - (Op::V128Not, execute_v128_not, simd::v128_not), - (Op::I8x16Popcnt, execute_i8x16_popcnt, simd::i8x16_popcnt), - - (Op::i16x8_extend_low_i8x16_s, execute_i16x8_extend_low_i8x16_s, simd::i16x8_extend_low_i8x16_s), - (Op::i16x8_extend_high_i8x16_s, execute_i16x8_extend_high_i8x16_s, simd::i16x8_extend_high_i8x16_s), - (Op::i16x8_extend_low_i8x16_u, execute_i16x8_extend_low_i8x16_u, simd::i16x8_extend_low_i8x16_u), - (Op::i16x8_extend_high_i8x16_u, execute_i16x8_extend_high_i8x16_u, simd::i16x8_extend_high_i8x16_u), - (Op::i32x4_extend_low_i16x8_s, execute_i32x4_extend_low_i16x8_s, simd::i32x4_extend_low_i16x8_s), - (Op::i32x4_extend_high_i16x8_s, execute_i32x4_extend_high_i16x8_s, simd::i32x4_extend_high_i16x8_s), - (Op::i32x4_extend_low_i16x8_u, execute_i32x4_extend_low_i16x8_u, simd::i32x4_extend_low_i16x8_u), - (Op::i32x4_extend_high_i16x8_u, execute_i32x4_extend_high_i16x8_u, simd::i32x4_extend_high_i16x8_u), - (Op::i64x2_extend_low_i32x4_s, execute_i64x2_extend_low_i32x4_s, simd::i64x2_extend_low_i32x4_s), - (Op::i64x2_extend_high_i32x4_s, execute_i64x2_extend_high_i32x4_s, simd::i64x2_extend_high_i32x4_s), - (Op::i64x2_extend_low_i32x4_u, execute_i64x2_extend_low_i32x4_u, simd::i64x2_extend_low_i32x4_u), - (Op::i64x2_extend_high_i32x4_u, execute_i64x2_extend_high_i32x4_u, simd::i64x2_extend_high_i32x4_u), - - (Op::I32x4TruncSatF32x4S, execute_i32x4_trunc_sat_f32x4_s, simd::i32x4_trunc_sat_f32x4_s), - (Op::I32x4TruncSatF32x4U, execute_i32x4_trunc_sat_f32x4_u, simd::i32x4_trunc_sat_f32x4_u), - (Op::F32x4ConvertI32x4S, execute_f32x4_convert_i32x4_s, simd::f32x4_convert_i32x4_s), - (Op::F32x4ConvertI32x4U, execute_f32x4_convert_i32x4_u, simd::f32x4_convert_i32x4_u), - (Op::I32x4TruncSatF64x2SZero, execute_i32x4_trunc_sat_f64x2_s_zero, simd::i32x4_trunc_sat_f64x2_s_zero), - (Op::I32x4TruncSatF64x2UZero, execute_i32x4_trunc_sat_f64x2_u_zero, simd::i32x4_trunc_sat_f64x2_u_zero), - (Op::F64x2ConvertLowI32x4S, execute_f64x2_convert_low_i32x4_s, simd::f64x2_convert_low_i32x4_s), - (Op::F64x2ConvertLowI32x4U, execute_f64x2_convert_low_i32x4_u, simd::f64x2_convert_low_i32x4_u), - (Op::F32x4DemoteF64x2Zero, execute_f32x4_demote_f64x2_zero, simd::f32x4_demote_f64x2_zero), - (Op::F64x2PromoteLowF32x4, execute_f64x2_promote_low_f32x4, simd::f64x2_promote_low_f32x4), - } - - impl_binary_executors! { - (Op::I8x16Swizzle, execute_i8x16_swizzle, simd::i8x16_swizzle), - - (Op::I16x8Q15MulrSatS, execute_i16x8_q15mulr_sat_s, simd::i16x8_q15mulr_sat_s), - (Op::I32x4DotI16x8S, execute_i32x4_dot_i16x8_s, simd::i32x4_dot_i16x8_s), - (Op::I16x8RelaxedDotI8x16I7x16S, execute_i16x8_relaxed_dot_i8x16_i7x16_s, simd::i16x8_relaxed_dot_i8x16_i7x16_s), - - (Op::I16x8ExtmulLowI8x16S, execute_i16x8_extmul_low_i8x16_s, simd::i16x8_extmul_low_i8x16_s), - (Op::I16x8ExtmulHighI8x16S, execute_i16x8_extmul_high_i8x16_s, simd::i16x8_extmul_high_i8x16_s), - (Op::I16x8ExtmulLowI8x16U, execute_i16x8_extmul_low_i8x16_u, simd::i16x8_extmul_low_i8x16_u), - (Op::I16x8ExtmulHighI8x16U, execute_i16x8_extmul_high_i8x16_u, simd::i16x8_extmul_high_i8x16_u), - (Op::I32x4ExtmulLowI16x8S, execute_i32x4_extmul_low_i16x8_s, simd::i32x4_extmul_low_i16x8_s), - (Op::I32x4ExtmulHighI16x8S, execute_i32x4_extmul_high_i16x8_s, simd::i32x4_extmul_high_i16x8_s), - (Op::I32x4ExtmulLowI16x8U, execute_i32x4_extmul_low_i16x8_u, simd::i32x4_extmul_low_i16x8_u), - (Op::I32x4ExtmulHighI16x8U, execute_i32x4_extmul_high_i16x8_u, simd::i32x4_extmul_high_i16x8_u), - (Op::I64x2ExtmulLowI32x4S, execute_i64x2_extmul_low_i32x4_s, simd::i64x2_extmul_low_i32x4_s), - (Op::I64x2ExtmulHighI32x4S, execute_i64x2_extmul_high_i32x4_s, simd::i64x2_extmul_high_i32x4_s), - (Op::I64x2ExtmulLowI32x4U, execute_i64x2_extmul_low_i32x4_u, simd::i64x2_extmul_low_i32x4_u), - (Op::I64x2ExtmulHighI32x4U, execute_i64x2_extmul_high_i32x4_u, simd::i64x2_extmul_high_i32x4_u), - - (Op::I32x4Add, execute_i32x4_add, simd::i32x4_add), - (Op::I32x4Sub, execute_i32x4_sub, simd::i32x4_sub), - (Op::I32x4Mul, execute_i32x4_mul, simd::i32x4_mul), - - (Op::I64x2Add, execute_i64x2_add, simd::i64x2_add), - (Op::I64x2Sub, execute_i64x2_sub, simd::i64x2_sub), - (Op::I64x2Mul, execute_i64x2_mul, simd::i64x2_mul), - - (Op::I8x16Eq, execute_i8x16_eq, simd::i8x16_eq), - (Op::I8x16Ne, execute_i8x16_ne, simd::i8x16_ne), - (Op::I8x16LtS, execute_i8x16_lt_s, simd::i8x16_lt_s), - (Op::I8x16LtU, execute_i8x16_lt_u, simd::i8x16_lt_u), - (Op::I8x16LeS, execute_i8x16_le_s, simd::i8x16_le_s), - (Op::I8x16LeU, execute_i8x16_le_u, simd::i8x16_le_u), - (Op::I16x8Eq, execute_i16x8_eq, simd::i16x8_eq), - (Op::I16x8Ne, execute_i16x8_ne, simd::i16x8_ne), - (Op::I16x8LtS, execute_i16x8_lt_s, simd::i16x8_lt_s), - (Op::I16x8LtU, execute_i16x8_lt_u, simd::i16x8_lt_u), - (Op::I16x8LeS, execute_i16x8_le_s, simd::i16x8_le_s), - (Op::I16x8LeU, execute_i16x8_le_u, simd::i16x8_le_u), - (Op::I32x4Eq, execute_i32x4_eq, simd::i32x4_eq), - (Op::I32x4Ne, execute_i32x4_ne, simd::i32x4_ne), - (Op::I32x4LtS, execute_i32x4_lt_s, simd::i32x4_lt_s), - (Op::I32x4LtU, execute_i32x4_lt_u, simd::i32x4_lt_u), - (Op::I32x4LeS, execute_i32x4_le_s, simd::i32x4_le_s), - (Op::I32x4LeU, execute_i32x4_le_u, simd::i32x4_le_u), - (Op::I64x2Eq, execute_i64x2_eq, simd::i64x2_eq), - (Op::I64x2Ne, execute_i64x2_ne, simd::i64x2_ne), - (Op::I64x2LtS, execute_i64x2_lt_s, simd::i64x2_lt_s), - (Op::I64x2LeS, execute_i64x2_le_s, simd::i64x2_le_s), - (Op::F32x4Eq, execute_f32x4_eq, simd::f32x4_eq), - (Op::F32x4Ne, execute_f32x4_ne, simd::f32x4_ne), - (Op::F32x4Lt, execute_f32x4_lt, simd::f32x4_lt), - (Op::F32x4Le, execute_f32x4_le, simd::f32x4_le), - (Op::F64x2Eq, execute_f64x2_eq, simd::f64x2_eq), - (Op::F64x2Ne, execute_f64x2_ne, simd::f64x2_ne), - (Op::F64x2Lt, execute_f64x2_lt, simd::f64x2_lt), - (Op::F64x2Le, execute_f64x2_le, simd::f64x2_le), - - (Op::I8x16MinS, execute_i8x16_min_s, simd::i8x16_min_s), - (Op::I8x16MinU, execute_i8x16_min_u, simd::i8x16_min_u), - (Op::I8x16MaxS, execute_i8x16_max_s, simd::i8x16_max_s), - (Op::I8x16MaxU, execute_i8x16_max_u, simd::i8x16_max_u), - (Op::I8x16AvgrU, execute_i8x16_avgr_u, simd::i8x16_avgr_u), - (Op::I16x8MinS, execute_i16x8_min_s, simd::i16x8_min_s), - (Op::I16x8MinU, execute_i16x8_min_u, simd::i16x8_min_u), - (Op::I16x8MaxS, execute_i16x8_max_s, simd::i16x8_max_s), - (Op::I16x8MaxU, execute_i16x8_max_u, simd::i16x8_max_u), - (Op::I16x8AvgrU, execute_i16x8_avgr_u, simd::i16x8_avgr_u), - (Op::I32x4MinS, execute_i32x4_min_s, simd::i32x4_min_s), - (Op::I32x4MinU, execute_i32x4_min_u, simd::i32x4_min_u), - (Op::I32x4MaxS, execute_i32x4_max_s, simd::i32x4_max_s), - (Op::I32x4MaxU, execute_i32x4_max_u, simd::i32x4_max_u), - - (Op::I8x16Shl, execute_i8x16_shl, simd::i8x16_shl), - (Op::I8x16ShrS, execute_i8x16_shr_s, simd::i8x16_shr_s), - (Op::I8x16ShrU, execute_i8x16_shr_u, simd::i8x16_shr_u), - (Op::I16x8Shl, execute_i16x8_shl, simd::i16x8_shl), - (Op::I16x8ShrS, execute_i16x8_shr_s, simd::i16x8_shr_s), - (Op::I16x8ShrU, execute_i16x8_shr_u, simd::i16x8_shr_u), - (Op::I32x4Shl, execute_i32x4_shl, simd::i32x4_shl), - (Op::I32x4ShrS, execute_i32x4_shr_s, simd::i32x4_shr_s), - (Op::I32x4ShrU, execute_i32x4_shr_u, simd::i32x4_shr_u), - (Op::I64x2Shl, execute_i64x2_shl, simd::i64x2_shl), - (Op::I64x2ShrS, execute_i64x2_shr_s, simd::i64x2_shr_s), - (Op::I64x2ShrU, execute_i64x2_shr_u, simd::i64x2_shr_u), - - (Op::I8x16Add, execute_i8x16_add, simd::i8x16_add), - (Op::I8x16AddSatS, execute_i8x16_add_sat_s, simd::i8x16_add_sat_s), - (Op::I8x16AddSatU, execute_i8x16_add_sat_u, simd::i8x16_add_sat_u), - (Op::I8x16Sub, execute_i8x16_sub, simd::i8x16_sub), - (Op::I8x16SubSatS, execute_i8x16_sub_sat_s, simd::i8x16_sub_sat_s), - (Op::I8x16SubSatU, execute_i8x16_sub_sat_u, simd::i8x16_sub_sat_u), - - (Op::I16x8Add, execute_i16x8_add, simd::i16x8_add), - (Op::I16x8AddSatS, execute_i16x8_add_sat_s, simd::i16x8_add_sat_s), - (Op::I16x8AddSatU, execute_i16x8_add_sat_u, simd::i16x8_add_sat_u), - (Op::I16x8Sub, execute_i16x8_sub, simd::i16x8_sub), - (Op::I16x8SubSatS, execute_i16x8_sub_sat_s, simd::i16x8_sub_sat_s), - (Op::I16x8SubSatU, execute_i16x8_sub_sat_u, simd::i16x8_sub_sat_u), - (Op::I16x8Sub, execute_i16x8_mul, simd::i16x8_mul), - - (Op::V128And, execute_v128_and, simd::v128_and), - (Op::V128Andnot, execute_v128_andnot, simd::v128_andnot), - (Op::V128Or, execute_v128_or, simd::v128_or), - (Op::V128Xor, execute_v128_xor, simd::v128_xor), - - (Op::F32x4Add, execute_f32x4_add, simd::f32x4_add), - (Op::F32x4Sub, execute_f32x4_sub, simd::f32x4_sub), - (Op::F32x4Mul, execute_f32x4_mul, simd::f32x4_mul), - (Op::F32x4Div, execute_f32x4_div, simd::f32x4_div), - (Op::F32x4Min, execute_f32x4_min, simd::f32x4_min), - (Op::F32x4Max, execute_f32x4_max, simd::f32x4_max), - (Op::F32x4Pmin, execute_f32x4_pmin, simd::f32x4_pmin), - (Op::F32x4Pmax, execute_f32x4_pmax, simd::f32x4_pmax), - - (Op::F64x2Add, execute_f64x2_add, simd::f64x2_add), - (Op::F64x2Sub, execute_f64x2_sub, simd::f64x2_sub), - (Op::F64x2Mul, execute_f64x2_mul, simd::f64x2_mul), - (Op::F64x2Div, execute_f64x2_div, simd::f64x2_div), - (Op::F64x2Min, execute_f64x2_min, simd::f64x2_min), - (Op::F64x2Max, execute_f64x2_max, simd::f64x2_max), - (Op::F64x2Pmin, execute_f64x2_pmin, simd::f64x2_pmin), - (Op::F64x2Pmax, execute_f64x2_pmax, simd::f64x2_pmax), - - (Op::I8x16NarrowI16x8S, execute_i8x16_narrow_i16x8_s, simd::i8x16_narrow_i16x8_s), - (Op::I8x16NarrowI16x8U, execute_i8x16_narrow_i16x8_u, simd::i8x16_narrow_i16x8_u), - (Op::I16x8NarrowI32x4S, execute_i16x8_narrow_i32x4_s, simd::i16x8_narrow_i32x4_s), - (Op::I16x8NarrowI32x4U, execute_i16x8_narrow_i32x4_u, simd::i16x8_narrow_i32x4_u), - } -} - -impl Executor<'_> { - /// Executes a generic SIMD extract-lane [`Op`]. - #[inline(always)] - fn execute_extract_lane( - &mut self, - result: Slot, - input: Slot, - lane: Lane, - op: fn(V128, Lane) -> T, - ) where - UntypedVal: WriteAs, - { - let input = self.get_stack_slot_as::(input); - self.set_stack_slot_as::(result, op(input, lane)); - self.next_instr(); - } -} - -macro_rules! impl_extract_lane_executors { - ( - $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, input: Slot, lane: <$ty as IntoLaneIdx>::LaneIdx) { - self.execute_extract_lane(result, input, lane, $op) - } - )* - }; -} -impl Executor<'_> { - impl_extract_lane_executors! { - (i8, Op::I8x16ExtractLaneS, i8x16_extract_lane_s, simd::i8x16_extract_lane_s), - (u8, Op::I8x16ExtractLaneU, i8x16_extract_lane_u, simd::i8x16_extract_lane_u), - (i16, Op::I16x8ExtractLaneS, i16x8_extract_lane_s, simd::i16x8_extract_lane_s), - (u16, Op::I16x8ExtractLaneU, i16x8_extract_lane_u, simd::i16x8_extract_lane_u), - (i32, Op::I32x4ExtractLane, i32x4_extract_lane, simd::i32x4_extract_lane), - (u32, Op::F32x4ExtractLane, f32x4_extract_lane, simd::f32x4_extract_lane), - (i64, Op::I64x2ExtractLane, i64x2_extract_lane, simd::i64x2_extract_lane), - (u64, Op::F64x2ExtractLane, f64x2_extract_lane, simd::f64x2_extract_lane), - } -} - -impl Executor<'_> { - /// Generically execute a SIMD shift operation with immediate shift amount. - #[inline(always)] - fn execute_simd_shift_by( - &mut self, - result: Slot, - lhs: Slot, - rhs: ShiftAmount, - op: fn(V128, u32) -> V128, - ) { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = rhs.into(); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } -} - -macro_rules! impl_simd_shift_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: ShiftAmount) { - self.execute_simd_shift_by(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_simd_shift_executors! { - (Op::I8x16ShlBy, execute_i8x16_shl_by, simd::i8x16_shl), - (Op::I8x16ShrSBy, execute_i8x16_shr_s_by, simd::i8x16_shr_s), - (Op::I8x16ShrUBy, execute_i8x16_shr_u_by, simd::i8x16_shr_u), - (Op::I16x8ShlBy, execute_i16x8_shl_by, simd::i16x8_shl), - (Op::I16x8ShrSBy, execute_i16x8_shr_s_by, simd::i16x8_shr_s), - (Op::I16x8ShrUBy, execute_i16x8_shr_u_by, simd::i16x8_shr_u), - (Op::I32x4ShlBy, execute_i32x4_shl_by, simd::i32x4_shl), - (Op::I32x4ShrSBy, execute_i32x4_shr_s_by, simd::i32x4_shr_s), - (Op::I32x4ShrUBy, execute_i32x4_shr_u_by, simd::i32x4_shr_u), - (Op::I64x2ShlBy, execute_i64x2_shl_by, simd::i64x2_shl), - (Op::I64x2ShrSBy, execute_i64x2_shr_s_by, simd::i64x2_shr_s), - (Op::I64x2ShrUBy, execute_i64x2_shr_u_by, simd::i64x2_shr_u), - } -} - -impl Executor<'_> { - /// Returns the optional `memory` parameter for a `load_at` [`Op`]. - /// - /// # Note - /// - /// - Returns the default [`index::Memory`] if the parameter is missing. - /// - Bumps `self.ip` if a [`Op::MemoryIndex`] parameter was found. - #[inline(always)] - fn fetch_lane_and_memory(&mut self, delta: usize) -> (LaneType, index::Memory) - where - LaneType: TryFrom, - { - let mut addr: InstructionPtr = self.ip; - addr.add(delta); - match addr.get().filter_lane_and_memory() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::Imm16AndImm32` but found: {instr:?}") - }, - } - } -} - -type V128LoadLane = - fn(memory: &[u8], ptr: u64, offset: u64, x: V128, lane: LaneType) -> Result; - -type V128LoadLaneAt = - fn(memory: &[u8], address: usize, x: V128, lane: LaneType) -> Result; - -macro_rules! impl_execute_v128_load_lane { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &StoreInner, - result: Slot, - offset_lo: Offset64Lo, - ) -> Result<(), Error> { - self.execute_v128_load_lane_impl::<<$ty as IntoLaneIdx>::LaneIdx>(store, result, offset_lo, $eval) - } - )* - }; -} - -macro_rules! impl_execute_v128_load_lane_at { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &StoreInner, - result: Slot, - address: Address32, - ) -> Result<(), Error> { - self.execute_v128_load_lane_at_impl::<<$ty as IntoLaneIdx>::LaneIdx>(store, result, address, $eval) - } - )* - }; -} - -impl Executor<'_> { - fn execute_v128_load_lane_impl( - &mut self, - store: &StoreInner, - result: Slot, - offset_lo: Offset64Lo, - load: V128LoadLane, - ) -> Result<(), Error> - where - LaneType: TryFrom + Into + Copy, - { - let (ptr, offset_hi) = self.fetch_value_and_offset_hi(); - let (v128, lane) = self.fetch_value_and_lane::(2); - let memory = self.fetch_optional_memory(3); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - let v128 = self.get_stack_slot_as::(v128); - let memory = self.fetch_memory_bytes(memory, store); - let loaded = load(memory, ptr, u64::from(offset), v128, lane)?; - self.set_stack_slot_as::(result, loaded); - self.try_next_instr_at(3) - } - - impl_execute_v128_load_lane! { - (u8, Op::V128Load8Lane, execute_v128_load8_lane, simd::v128_load8_lane), - (u16, Op::V128Load16Lane, execute_v128_load16_lane, simd::v128_load16_lane), - (u32, Op::V128Load32Lane, execute_v128_load32_lane, simd::v128_load32_lane), - (u64, Op::V128Load64Lane, execute_v128_load64_lane, simd::v128_load64_lane), - } - - fn execute_v128_load_lane_at_impl( - &mut self, - store: &StoreInner, - result: Slot, - address: Address32, - load_at: V128LoadLaneAt, - ) -> Result<(), Error> - where - LaneType: TryFrom + Into + Copy, - { - let (v128, lane) = self.fetch_value_and_lane::(1); - let memory = self.fetch_optional_memory(2); - let v128 = self.get_stack_slot_as::(v128); - let memory = self.fetch_memory_bytes(memory, store); - let loaded = load_at(memory, usize::from(address), v128, lane)?; - self.set_stack_slot_as::(result, loaded); - self.try_next_instr_at(2) - } - - impl_execute_v128_load_lane_at! { - (u8, Op::V128Load8LaneAt, execute_v128_load8_lane_at, simd::v128_load8_lane_at), - (u16, Op::V128Load16LaneAt, execute_v128_load16_lane_at, simd::v128_load16_lane_at), - (u32, Op::V128Load32LaneAt, execute_v128_load32_lane_at, simd::v128_load32_lane_at), - (u64, Op::V128Load64LaneAt, execute_v128_load64_lane_at, simd::v128_load64_lane_at), - } -} - -macro_rules! impl_execute_v128_store_lane { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - ) -> Result<(), Error> { - self.execute_v128_store_lane::<$ty>(store, ptr, offset_lo, $eval) - } - )* - }; -} - -macro_rules! impl_execute_v128_store_lane_offset16 { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - ptr: Slot, - value: Slot, - offset: Offset8, - lane: <$ty as IntoLaneIdx>::LaneIdx, - ) -> Result<(), Error> { - self.execute_v128_store_lane_offset8::<$ty>(ptr, value, offset, lane, $eval) - } - )* - }; -} - -macro_rules! impl_execute_v128_store_lane_at { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &mut StoreInner, - value: Slot, - address: Address32, - ) -> Result<(), Error> { - self.execute_v128_store_lane_at::<$ty>(store, value, address, $eval) - } - )* - }; -} - -type V128StoreLane = fn( - memory: &mut [u8], - ptr: u64, - offset: u64, - value: V128, - lane: LaneType, -) -> Result<(), TrapCode>; - -type V128StoreLaneAt = - fn(memory: &mut [u8], address: usize, value: V128, lane: LaneType) -> Result<(), TrapCode>; - -impl Executor<'_> { - fn execute_v128_store_lane( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - eval: V128StoreLane, - ) -> Result<(), Error> { - let (value, offset_hi) = self.fetch_value_and_offset_hi(); - let (lane, memory) = self.fetch_lane_and_memory(2); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - let v128 = self.get_stack_slot_as::(value); - let memory = self.fetch_memory_bytes_mut(memory, store); - eval(memory, ptr, u64::from(offset), v128, lane)?; - self.try_next_instr_at(3) - } - - impl_execute_v128_store_lane! { - (u8, Op::V128Store8Lane, execute_v128_store8_lane, simd::v128_store8_lane), - (u16, Op::V128Store16Lane, execute_v128_store16_lane, simd::v128_store16_lane), - (u32, Op::V128Store32Lane, execute_v128_store32_lane, simd::v128_store32_lane), - (u64, Op::V128Store64Lane, execute_v128_store64_lane, simd::v128_store64_lane), - } - - fn execute_v128_store_lane_offset8( - &mut self, - ptr: Slot, - value: Slot, - offset: Offset8, - lane: T::LaneIdx, - eval: V128StoreLane, - ) -> Result<(), Error> { - let ptr = self.get_stack_slot_as::(ptr); - let offset = u64::from(Offset64::from(offset)); - let v128 = self.get_stack_slot_as::(value); - let memory = self.fetch_default_memory_bytes_mut(); - eval(memory, ptr, offset, v128, lane)?; - self.try_next_instr() - } - - impl_execute_v128_store_lane_offset16! { - (u8, Op::V128Store8LaneOffset8, execute_v128_store8_lane_offset8, simd::v128_store8_lane), - (u16, Op::V128Store16LaneOffset8, execute_v128_store16_lane_offset8, simd::v128_store16_lane), - (u32, Op::V128Store32LaneOffset8, execute_v128_store32_lane_offset8, simd::v128_store32_lane), - (u64, Op::V128Store64LaneOffset8, execute_v128_store64_lane_offset8, simd::v128_store64_lane), - } - - fn execute_v128_store_lane_at( - &mut self, - store: &mut StoreInner, - value: Slot, - address: Address32, - eval: V128StoreLaneAt, - ) -> Result<(), Error> - where - T::LaneIdx: TryFrom + Into, - { - let (lane, memory) = self.fetch_lane_and_memory::(1); - let v128 = self.get_stack_slot_as::(value); - let memory = self.fetch_memory_bytes_mut(memory, store); - eval(memory, usize::from(address), v128, lane)?; - self.try_next_instr_at(2) - } - - impl_execute_v128_store_lane_at! { - (u8, Op::V128Store8LaneAt, execute_v128_store8_lane_at, simd::v128_store8_lane_at), - (u16, Op::V128Store16LaneAt, execute_v128_store16_lane_at, simd::v128_store16_lane_at), - (u32, Op::V128Store32LaneAt, execute_v128_store32_lane_at, simd::v128_store32_lane_at), - (u64, Op::V128Store64LaneAt, execute_v128_store64_lane_at, simd::v128_store64_lane_at), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/store.rs b/crates/wasmi/src/engine/executor/instrs/store.rs deleted file mode 100644 index 0774df438b4..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/store.rs +++ /dev/null @@ -1,480 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::{wasm, ReadAs, UntypedVal}, - engine::utils::unreachable_unchecked, - ir::{ - index::Memory, - Address32, - AnyConst16, - Const16, - Offset16, - Offset64, - Offset64Hi, - Offset64Lo, - Slot, - }, - store::StoreInner, - Error, - TrapCode, -}; - -#[cfg(feature = "simd")] -use crate::{core::simd, V128}; - -#[cfg(doc)] -use crate::ir::Op; - -/// The function signature of Wasm store operations. -type WasmStoreOp = - fn(memory: &mut [u8], address: u64, offset: u64, value: T) -> Result<(), TrapCode>; - -/// The function signature of Wasm store operations. -type WasmStoreAtOp = fn(memory: &mut [u8], address: usize, value: T) -> Result<(), TrapCode>; - -impl Executor<'_> { - /// Returns the immediate `value` and `offset_hi` parameters for a `load` [`Op`]. - fn fetch_value_and_offset_imm(&self) -> (T, Offset64Hi) - where - T: From, - { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match addr.get().filter_imm16_and_offset_hi::() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::SlotAndImm32` but found: {instr:?}") - }, - } - } - - /// Executes a generic Wasm `store[N]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.store` - /// - `{i32, i64}.store8` - /// - `{i32, i64}.store16` - /// - `i64.store32` - pub(super) fn execute_store_wrap( - &mut self, - store: &mut StoreInner, - memory: Memory, - address: u64, - offset: Offset64, - value: T, - store_wrap: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_memory_bytes_mut(memory, store); - store_wrap(memory, address, u64::from(offset), value)?; - Ok(()) - } - - /// Executes a generic Wasm `store[N]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.store` - /// - `{i32, i64}.store8` - /// - `{i32, i64}.store16` - /// - `i64.store32` - fn execute_store_wrap_at( - &mut self, - store: &mut StoreInner, - memory: Memory, - address: Address32, - value: T, - store_wrap_at: WasmStoreAtOp, - ) -> Result<(), Error> { - let memory = self.fetch_memory_bytes_mut(memory, store); - store_wrap_at(memory, usize::from(address), value)?; - Ok(()) - } - - /// Executes a generic Wasm `store[N]` operation for the default memory. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.store` - /// - `{i32, i64}.store8` - /// - `{i32, i64}.store16` - /// - `i64.store32` - fn execute_store_wrap_mem0( - &mut self, - address: u64, - offset: Offset64, - value: T, - store_wrap: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_default_memory_bytes_mut(); - store_wrap(memory, address, u64::from(offset), value)?; - Ok(()) - } - - fn execute_store( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let (value, offset_hi) = self.fetch_value_and_offset_hi(); - let memory = self.fetch_optional_memory(2); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - let value = self.get_stack_slot_as::(value); - self.execute_store_wrap::(store, memory, ptr, offset, value, store_op)?; - self.try_next_instr_at(2) - } - - fn execute_store_imm( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - offset_hi: Offset64Hi, - value: T, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_optional_memory(2); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - self.execute_store_wrap::(store, memory, ptr, offset, value, store_op)?; - self.try_next_instr_at(2) - } - - fn execute_store_offset16( - &mut self, - ptr: Slot, - offset: Offset16, - value: Slot, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let ptr = self.get_stack_slot_as::(ptr); - let value = self.get_stack_slot_as::(value); - self.execute_store_wrap_mem0::(ptr, Offset64::from(offset), value, store_op)?; - self.try_next_instr() - } - - fn execute_store_offset16_imm16( - &mut self, - ptr: Slot, - offset: Offset16, - value: T, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let ptr = self.get_stack_slot_as::(ptr); - self.execute_store_wrap_mem0::(ptr, Offset64::from(offset), value, store_op)?; - self.try_next_instr() - } - - fn execute_store_at( - &mut self, - store: &mut StoreInner, - address: Address32, - value: Slot, - store_at_op: WasmStoreAtOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_optional_memory(1); - self.execute_store_wrap_at::( - store, - memory, - address, - self.get_stack_slot_as::(value), - store_at_op, - )?; - self.try_next_instr() - } - - fn execute_store_at_imm16( - &mut self, - store: &mut StoreInner, - address: Address32, - value: T, - store_at_op: WasmStoreAtOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_optional_memory(1); - self.execute_store_wrap_at::(store, memory, address, value, store_at_op)?; - self.try_next_instr() - } -} - -macro_rules! impl_execute_istore { - ( $( - ( - $ty:ty, - ($from_ty:ty => $to_ty:ty), - (Op::$var_store_imm:ident, $fn_store_imm:ident), - (Op::$var_store_off16_imm16:ident, $fn_store_off16_imm16:ident), - (Op::$var_store_at_imm16:ident, $fn_store_at_imm16:ident), - $store_fn:expr, - $store_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_store_imm), "`].")] - #[allow(clippy::cast_lossless)] - pub fn $fn_store_imm(&mut self, store: &mut StoreInner, ptr: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - let (value, offset_hi) = self.fetch_value_and_offset_imm::<$to_ty>(); - self.execute_store_imm::<$ty>(store, ptr, offset_lo, offset_hi, value as $ty, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_off16_imm16), "`].")] - #[allow(clippy::cast_lossless)] - pub fn $fn_store_off16_imm16( - &mut self, - ptr: Slot, - offset: Offset16, - value: $from_ty, - ) -> Result<(), Error> { - self.execute_store_offset16_imm16::<$ty>(ptr, offset, <$to_ty>::from(value) as _, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_at_imm16), "`].")] - #[allow(clippy::cast_lossless)] - pub fn $fn_store_at_imm16( - &mut self, - store: &mut StoreInner, - address: Address32, - value: $from_ty, - ) -> Result<(), Error> { - #[allow(clippy::cast_lossless)] - self.execute_store_at_imm16::<$ty>(store, address, <$to_ty>::from(value) as _, $store_at_fn) - } - )* - }; -} -impl Executor<'_> { - impl_execute_istore! { - ( - u32, - (Const16 => i32), - (Op::I32StoreImm16, execute_i32_store_imm16), - (Op::I32StoreOffset16Imm16, execute_i32_store_offset16_imm16), - (Op::I32StoreAtImm16, execute_i32_store_at_imm16), - wasm::store32, - wasm::store32_at, - ), - ( - u64, - (Const16 => i64), - (Op::I64StoreImm16, execute_i64_store_imm16), - (Op::I64StoreOffset16Imm16, execute_i64_store_offset16_imm16), - (Op::I64StoreAtImm16, execute_i64_store_at_imm16), - wasm::store64, - wasm::store64_at, - ), - } -} - -macro_rules! impl_execute_istore_trunc { - ( $( - ( - $ty:ty, - ($from_ty:ty => $to_ty:ty), - (Op::$var_store:ident, $fn_store:ident), - (Op::$var_store_imm:ident, $fn_store_imm:ident), - (Op::$var_store_off16:ident, $fn_store_off16:ident), - (Op::$var_store_off16_imm16:ident, $fn_store_off16_imm16:ident), - (Op::$var_store_at:ident, $fn_store_at:ident), - (Op::$var_store_at_imm16:ident, $fn_store_at_imm16:ident), - $store_fn:expr, - $store_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - impl_execute_istore! { - ( - $ty, - ($from_ty => $to_ty), - (Op::$var_store_imm, $fn_store_imm), - (Op::$var_store_off16_imm16, $fn_store_off16_imm16), - (Op::$var_store_at_imm16, $fn_store_at_imm16), - $store_fn, - $store_at_fn, - ) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store), "`].")] - pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - self.execute_store::<$ty>(store, ptr, offset_lo, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_off16), "`].")] - pub fn $fn_store_off16( - &mut self, - ptr: Slot, - offset: Offset16, - value: Slot, - ) -> Result<(), Error> { - self.execute_store_offset16::<$ty>(ptr, offset, value, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_at), "`].")] - pub fn $fn_store_at(&mut self, store: &mut StoreInner, address: Address32, value: Slot) -> Result<(), Error> { - self.execute_store_at::<$ty>(store, address, value, $store_at_fn) - } - )* - }; -} -impl Executor<'_> { - impl_execute_istore_trunc! { - ( - i32, - (i8 => i8), - (Op::I32Store8, execute_i32_store8), - (Op::I32Store8Imm, execute_i32_store8_imm), - (Op::I32Store8Offset16, execute_i32_store8_offset16), - (Op::I32Store8Offset16Imm, execute_i32_store8_offset16_imm), - (Op::I32Store8At, execute_i32_store8_at), - (Op::I32Store8AtImm, execute_i32_store8_at_imm), - wasm::i32_store8, - wasm::i32_store8_at, - ), - ( - i32, - (i16 => i16), - (Op::I32Store16, execute_i32_store16), - (Op::I32Store16Imm, execute_i32_store16_imm), - (Op::I32Store16Offset16, execute_i32_store16_offset16), - (Op::I32Store16Offset16Imm, execute_i32_store16_offset16_imm), - (Op::I32Store16At, execute_i32_store16_at), - (Op::I32Store16AtImm, execute_i32_store16_at_imm), - wasm::i32_store16, - wasm::i32_store16_at, - ), - ( - i64, - (i8 => i8), - (Op::I64Store8, execute_i64_store8), - (Op::I64Store8Imm, execute_i64_store8_imm), - (Op::I64Store8Offset16, execute_i64_store8_offset16), - (Op::I64Store8Offset16Imm, execute_i64_store8_offset16_imm), - (Op::I64Store8At, execute_i64_store8_at), - (Op::I64Store8AtImm, execute_i64_store8_at_imm), - wasm::i64_store8, - wasm::i64_store8_at, - ), - ( - i64, - (i16 => i16), - (Op::I64Store16, execute_i64_store16), - (Op::I64Store16Imm, execute_i64_store16_imm), - (Op::I64Store16Offset16, execute_i64_store16_offset16), - (Op::I64Store16Offset16Imm, execute_i64_store16_offset16_imm), - (Op::I64Store16At, execute_i64_store16_at), - (Op::I64Store16AtImm, execute_i64_store16_at_imm), - wasm::i64_store16, - wasm::i64_store16_at, - ), - ( - i64, - (Const16 => i32), - (Op::I64Store32, execute_i64_store32), - (Op::I64Store32Imm16, execute_i64_store32_imm16), - (Op::I64Store32Offset16, execute_i64_store32_offset16), - (Op::I64Store32Offset16Imm16, execute_i64_store32_offset16_imm16), - (Op::I64Store32At, execute_i64_store32_at), - (Op::I64Store32AtImm16, execute_i64_store32_at_imm16), - wasm::i64_store32, - wasm::i64_store32_at, - ), - } -} - -macro_rules! impl_execute_store { - ( $( - ( - $ty:ty, - (Op::$var_store:ident, $fn_store:ident), - (Op::$var_store_off16:ident, $fn_store_off16:ident), - (Op::$var_store_at:ident, $fn_store_at:ident), - $store_fn:expr, - $store_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_store), "`].")] - pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - self.execute_store::<$ty>(store, ptr, offset_lo, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_off16), "`].")] - pub fn $fn_store_off16( - &mut self, - ptr: Slot, - offset: Offset16, - value: Slot, - ) -> Result<(), Error> { - self.execute_store_offset16::<$ty>(ptr, offset, value, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_at), "`].")] - pub fn $fn_store_at(&mut self, store: &mut StoreInner, address: Address32, value: Slot) -> Result<(), Error> { - self.execute_store_at::<$ty>(store, address, value, $store_at_fn) - } - )* - } -} - -impl Executor<'_> { - #[cfg(feature = "simd")] - impl_execute_store! { - ( - V128, - (Op::V128Store, execute_v128_store), - (Op::V128StoreOffset16, execute_v128_store_offset16), - (Op::V128StoreAt, execute_v128_store_at), - simd::v128_store, - simd::v128_store_at, - ), - } - - impl_execute_store! { - ( - u32, - (Op::Store32, execute_store32), - (Op::Store32Offset16, execute_store32_offset16), - (Op::Store32At, execute_store32_at), - wasm::store32, - wasm::store32_at, - ), - ( - u64, - (Op::Store64, execute_store64), - (Op::Store64Offset16, execute_store64_offset16), - (Op::Store64At, execute_store64_at), - wasm::store64, - wasm::store64_at, - ), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/table.rs b/crates/wasmi/src/engine/executor/instrs/table.rs deleted file mode 100644 index a4a2c8f9806..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/table.rs +++ /dev/null @@ -1,256 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::CoreTable, - engine::{utils::unreachable_unchecked, ResumableOutOfFuelError}, - errors::TableError, - ir::{ - index::{Elem, Table}, - Const32, - Op, - Slot, - }, - store::{PrunedStore, StoreError, StoreInner}, - Error, - TrapCode, -}; - -impl Executor<'_> { - /// Returns the [`Op::TableIndex`] parameter for an [`Op`]. - fn fetch_table_index(&self, offset: usize) -> Table { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::TableIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::TableIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::TableIndex` but found: {unexpected:?}") - } - } - } - } - - /// Returns the [`Op::ElemIndex`] parameter for an [`Op`]. - fn fetch_element_segment_index(&self, offset: usize) -> Elem { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::ElemIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::ElemIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::ElemIndex` but found: {unexpected:?}") - } - } - } - } - - /// Executes an [`Op::TableGet`]. - pub fn execute_table_get( - &mut self, - store: &StoreInner, - result: Slot, - index: Slot, - ) -> Result<(), Error> { - let index: u64 = self.get_stack_slot_as(index); - self.execute_table_get_impl(store, result, index) - } - - /// Executes an [`Op::TableGetImm`]. - pub fn execute_table_get_imm( - &mut self, - store: &StoreInner, - result: Slot, - index: Const32, - ) -> Result<(), Error> { - let index: u64 = index.into(); - self.execute_table_get_impl(store, result, index) - } - - /// Executes a `table.get` instruction generically. - fn execute_table_get_impl( - &mut self, - store: &StoreInner, - result: Slot, - index: u64, - ) -> Result<(), Error> { - let table_index = self.fetch_table_index(1); - let table = self.get_table(table_index); - let value = store - .resolve_table(&table) - .get_untyped(index) - .ok_or(TrapCode::TableOutOfBounds)?; - self.set_stack_slot(result, value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::TableSize`]. - pub fn execute_table_size(&mut self, store: &StoreInner, result: Slot, table_index: Table) { - self.execute_table_size_impl(store, result, table_index); - self.next_instr(); - } - - /// Executes a generic `table.size` instruction. - fn execute_table_size_impl(&mut self, store: &StoreInner, result: Slot, table_index: Table) { - let table = self.get_table(table_index); - let size = store.resolve_table(&table).size(); - self.set_stack_slot(result, size); - } - - /// Executes an [`Op::TableSet`]. - pub fn execute_table_set( - &mut self, - store: &mut StoreInner, - index: Slot, - value: Slot, - ) -> Result<(), Error> { - let index: u64 = self.get_stack_slot_as(index); - self.execute_table_set_impl(store, index, value) - } - - /// Executes an [`Op::TableSetAt`]. - pub fn execute_table_set_at( - &mut self, - store: &mut StoreInner, - index: Const32, - value: Slot, - ) -> Result<(), Error> { - let index: u64 = index.into(); - self.execute_table_set_impl(store, index, value) - } - - /// Executes a generic `table.set` instruction. - fn execute_table_set_impl( - &mut self, - store: &mut StoreInner, - index: u64, - value: Slot, - ) -> Result<(), Error> { - let table_index = self.fetch_table_index(1); - let table = self.get_table(table_index); - let value = self.get_stack_slot(value); - store - .resolve_table_mut(&table) - .set_untyped(index, value) - .map_err(|_| TrapCode::TableOutOfBounds)?; - self.try_next_instr_at(2) - } - - /// Executes an [`Op::TableCopy`]. - pub fn execute_table_copy( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u64 = self.get_stack_slot_as(src); - let len: u64 = self.get_stack_slot_as(len); - let dst_table_index = self.fetch_table_index(1); - let src_table_index = self.fetch_table_index(2); - if dst_table_index == src_table_index { - // Case: copy within the same table - let table = self.get_table(dst_table_index); - let (table, fuel) = store.resolve_table_and_fuel_mut(&table); - table.copy_within(dst, src, len, Some(fuel))?; - } else { - // Case: copy between two different tables - let dst_table = self.get_table(dst_table_index); - let src_table = self.get_table(src_table_index); - // Copy from one table to another table: - let (dst_table, src_table, fuel) = - store.resolve_table_pair_and_fuel(&dst_table, &src_table); - CoreTable::copy(dst_table, dst, src_table, src, len, Some(fuel))?; - } - self.try_next_instr_at(3) - } - - /// Executes an [`Op::TableInit`]. - pub fn execute_table_init( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u32 = self.get_stack_slot_as(src); - let len: u32 = self.get_stack_slot_as(len); - let table_index = self.fetch_table_index(1); - let element_index = self.fetch_element_segment_index(2); - let (table, element, fuel) = store.resolve_table_init_params( - &self.get_table(table_index), - &self.get_element_segment(element_index), - ); - table.init(element.as_ref(), dst, src, len, Some(fuel))?; - self.try_next_instr_at(3) - } - - /// Executes an [`Op::TableFill`]. - pub fn execute_table_fill( - &mut self, - store: &mut StoreInner, - dst: Slot, - len: Slot, - value: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let len: u64 = self.get_stack_slot_as(len); - let table_index = self.fetch_table_index(1); - let value = self.get_stack_slot(value); - let table = self.get_table(table_index); - let (table, fuel) = store.resolve_table_and_fuel_mut(&table); - table.fill_untyped(dst, value, len, Some(fuel))?; - self.try_next_instr_at(2) - } - - /// Executes an [`Op::TableGrow`]. - pub fn execute_table_grow( - &mut self, - store: &mut PrunedStore, - result: Slot, - delta: Slot, - value: Slot, - ) -> Result<(), Error> { - let delta: u64 = self.get_stack_slot_as(delta); - let table_index = self.fetch_table_index(1); - if delta == 0 { - // Case: growing by 0 elements means there is nothing to do - self.execute_table_size_impl(store.inner(), result, table_index); - return self.try_next_instr_at(2); - } - let table = self.get_table(table_index); - let value = self.get_stack_slot(value); - let return_value = match store.grow_table(&table, delta, value) { - Ok(return_value) => return_value, - Err(StoreError::External( - TableError::GrowOutOfBounds | TableError::OutOfSystemMemory, - )) => { - let table_ty = store.inner().resolve_table(&table).ty(); - match table_ty.is_64() { - true => u64::MAX, - false => u64::from(u32::MAX), - } - } - Err(StoreError::External(TableError::OutOfFuel { required_fuel })) => { - return Err(Error::from(ResumableOutOfFuelError::new(required_fuel))) - } - Err(StoreError::External(TableError::ResourceLimiterDeniedAllocation)) => { - return Err(Error::from(TrapCode::GrowthOperationLimited)) - } - Err(error) => { - panic!("`memory.grow`: internal interpreter error: {error}") - } - }; - self.set_stack_slot(result, return_value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::ElemDrop`]. - pub fn execute_element_drop(&mut self, store: &mut StoreInner, segment_index: Elem) { - let segment = self.get_element_segment(segment_index); - store.resolve_element_mut(&segment).drop_items(); - self.next_instr(); - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/unary.rs b/crates/wasmi/src/engine/executor/instrs/unary.rs deleted file mode 100644 index 063db5142e3..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/unary.rs +++ /dev/null @@ -1,33 +0,0 @@ -use super::Executor; -use crate::{core::wasm, ir::Slot}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - impl_unary_executors! { - (Op::I32Clz, execute_i32_clz, wasm::i32_clz), - (Op::I32Ctz, execute_i32_ctz, wasm::i32_ctz), - (Op::I32Popcnt, execute_i32_popcnt, wasm::i32_popcnt), - - (Op::I64Clz, execute_i64_clz, wasm::i64_clz), - (Op::I64Ctz, execute_i64_ctz, wasm::i64_ctz), - (Op::I64Popcnt, execute_i64_popcnt, wasm::i64_popcnt), - - (Op::F32Abs, execute_f32_abs, wasm::f32_abs), - (Op::F32Neg, execute_f32_neg, wasm::f32_neg), - (Op::F32Ceil, execute_f32_ceil, wasm::f32_ceil), - (Op::F32Floor, execute_f32_floor, wasm::f32_floor), - (Op::F32Trunc, execute_f32_trunc, wasm::f32_trunc), - (Op::F32Nearest, execute_f32_nearest, wasm::f32_nearest), - (Op::F32Sqrt, execute_f32_sqrt, wasm::f32_sqrt), - - (Op::F64Abs, execute_f64_abs, wasm::f64_abs), - (Op::F64Neg, execute_f64_neg, wasm::f64_neg), - (Op::F64Ceil, execute_f64_ceil, wasm::f64_ceil), - (Op::F64Floor, execute_f64_floor, wasm::f64_floor), - (Op::F64Trunc, execute_f64_trunc, wasm::f64_trunc), - (Op::F64Nearest, execute_f64_nearest, wasm::f64_nearest), - (Op::F64Sqrt, execute_f64_sqrt, wasm::f64_sqrt), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/utils.rs b/crates/wasmi/src/engine/executor/instrs/utils.rs deleted file mode 100644 index fd93cd2e1ed..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/utils.rs +++ /dev/null @@ -1,117 +0,0 @@ -use super::Executor; -use crate::{ - ir::{index::Memory, Offset64Hi, Slot}, - store::StoreInner, -}; - -#[cfg(doc)] -use crate::ir::Op; - -macro_rules! impl_unary_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, input: Slot) { - self.execute_unary(result, input, $op) - } - )* - }; -} - -macro_rules! impl_binary_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Slot) { - self.execute_binary(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - /// Returns the register `value` and `offset` parameters for a `load` [`Op`]. - pub fn fetch_value_and_offset_hi(&self) -> (Slot, Offset64Hi) { - // Safety: Wasmi translation guarantees that `Op::SlotAndImm32` exists. - unsafe { self.fetch_reg_and_offset_hi() } - } - - /// Fetches the bytes of the default memory at index 0. - pub fn fetch_default_memory_bytes(&self) -> &[u8] { - // Safety: the `self.cache.memory` pointer is always synchronized - // conservatively whenever it could have been invalidated. - unsafe { self.cache.memory.data() } - } - - /// Fetches the bytes of the given `memory`. - pub fn fetch_memory_bytes<'exec, 'store, 'bytes>( - &'exec self, - memory: Memory, - store: &'store StoreInner, - ) -> &'bytes [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - match memory.is_default() { - true => self.fetch_default_memory_bytes(), - false => self.fetch_non_default_memory_bytes(memory, store), - } - } - - /// Fetches the bytes of the given non-default `memory`. - #[cold] - pub fn fetch_non_default_memory_bytes<'exec, 'store, 'bytes>( - &'exec self, - memory: Memory, - store: &'store StoreInner, - ) -> &'bytes [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - let memory = self.get_memory(memory); - store.resolve_memory(&memory).data() - } - - /// Fetches the bytes of the default memory at index 0. - #[inline] - pub fn fetch_default_memory_bytes_mut(&mut self) -> &mut [u8] { - // Safety: the `self.cache.memory` pointer is always synchronized - // conservatively whenever it could have been invalidated. - unsafe { self.cache.memory.data_mut() } - } - - /// Fetches the bytes of the given `memory`. - #[inline] - pub fn fetch_memory_bytes_mut<'exec, 'store, 'bytes>( - &'exec mut self, - memory: Memory, - store: &'store mut StoreInner, - ) -> &'bytes mut [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - match memory.is_default() { - true => self.fetch_default_memory_bytes_mut(), - false => self.fetch_non_default_memory_bytes_mut(memory, store), - } - } - - /// Fetches the bytes of the given non-default `memory`. - #[cold] - #[inline] - pub fn fetch_non_default_memory_bytes_mut<'exec, 'store, 'bytes>( - &'exec mut self, - memory: Memory, - store: &'store mut StoreInner, - ) -> &'bytes mut [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - let memory = self.get_memory(memory); - store.resolve_memory_mut(&memory).data_mut() - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/wide_arithmetic.rs b/crates/wasmi/src/engine/executor/instrs/wide_arithmetic.rs deleted file mode 100644 index 6d39b556baf..00000000000 --- a/crates/wasmi/src/engine/executor/instrs/wide_arithmetic.rs +++ /dev/null @@ -1,95 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::wasm, - engine::utils::unreachable_unchecked, - ir::{FixedSlotSpan, Op, Slot}, -}; - -/// Parameters for the `i64.add128` and `i64.sub128` instructions. -struct Params128 { - /// The register storing the high 64-bit part of the `lhs` parameter value. - lhs_hi: Slot, - /// The register storing the low 64-bit part of the `rhs` parameter value. - rhs_lo: Slot, - /// The register storing the low 64-bit part of the `rhs` parameter value. - rhs_hi: Slot, -} - -/// Function signature for `i64.binop128` handlers. -type BinOp128Fn = fn(lhs_lo: i64, lhs_hi: i64, rhs_lo: i64, rhs_hi: i64) -> (i64, i64); - -/// Function signature for `i64.mul_wide_sx` handlers. -type I64MulWideFn = fn(lhs: i64, rhs: i64) -> (i64, i64); - -impl Executor<'_> { - /// Fetches the parameters required by the `i64.add128` and `i64.sub128` instructions. - fn fetch_params128(&self) -> Params128 { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Slot3 { slots } => Params128 { - lhs_hi: slots[0], - rhs_lo: slots[1], - rhs_hi: slots[2], - }, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::MemoryIndex`] exists. - unsafe { unreachable_unchecked!("expected `Op::Slot3` but found: {unexpected:?}") } - } - } - } - - /// Executes a generic Wasm `i64.binop128` instruction. - fn execute_i64_binop128(&mut self, results: [Slot; 2], lhs_lo: Slot, binop: BinOp128Fn) { - let Params128 { - lhs_hi, - rhs_lo, - rhs_hi, - } = self.fetch_params128(); - let lhs_lo: i64 = self.get_stack_slot_as(lhs_lo); - let lhs_hi: i64 = self.get_stack_slot_as(lhs_hi); - let rhs_lo: i64 = self.get_stack_slot_as(rhs_lo); - let rhs_hi: i64 = self.get_stack_slot_as(rhs_hi); - let (result_lo, result_hi) = binop(lhs_lo, lhs_hi, rhs_lo, rhs_hi); - self.set_stack_slot(results[0], result_lo); - self.set_stack_slot(results[1], result_hi); - self.next_instr_at(2) - } - - /// Executes an [`Op::I64Add128`]. - pub fn execute_i64_add128(&mut self, results: [Slot; 2], lhs_lo: Slot) { - self.execute_i64_binop128(results, lhs_lo, wasm::i64_add128) - } - - /// Executes an [`Op::I64Sub128`]. - pub fn execute_i64_sub128(&mut self, results: [Slot; 2], lhs_lo: Slot) { - self.execute_i64_binop128(results, lhs_lo, wasm::i64_sub128) - } - - /// Executes a generic Wasm `i64.mul_wide_sx` instruction. - fn execute_i64_mul_wide_sx( - &mut self, - results: FixedSlotSpan<2>, - lhs: Slot, - rhs: Slot, - mul_wide: I64MulWideFn, - ) { - let lhs: i64 = self.get_stack_slot_as(lhs); - let rhs: i64 = self.get_stack_slot_as(rhs); - let (result_lo, result_hi) = mul_wide(lhs, rhs); - let results = results.to_array(); - self.set_stack_slot(results[0], result_lo); - self.set_stack_slot(results[1], result_hi); - self.next_instr() - } - - /// Executes an [`Op::I64MulWideS`]. - pub fn execute_i64_mul_wide_s(&mut self, results: FixedSlotSpan<2>, lhs: Slot, rhs: Slot) { - self.execute_i64_mul_wide_sx(results, lhs, rhs, wasm::i64_mul_wide_s) - } - - /// Executes an [`Op::I64MulWideU`]. - pub fn execute_i64_mul_wide_u(&mut self, results: FixedSlotSpan<2>, lhs: Slot, rhs: Slot) { - self.execute_i64_mul_wide_sx(results, lhs, rhs, wasm::i64_mul_wide_u) - } -} diff --git a/crates/wasmi/src/engine/executor/mod.rs b/crates/wasmi/src/engine/executor/mod.rs index 257cf04f456..dd0ab848cfe 100644 --- a/crates/wasmi/src/engine/executor/mod.rs +++ b/crates/wasmi/src/engine/executor/mod.rs @@ -1,11 +1,13 @@ -pub(crate) use self::stack::Stack; -use self::{ - instr_ptr::InstructionPtr, - instrs::{dispatch_host_func, execute_instrs}, - stack::CallFrame, -}; +pub use self::handler::{op_code_to_handler, Inst, Stack}; +use super::code_map::CodeMap; use crate::{ engine::{ + executor::handler::{ + init_host_func_call, + init_wasm_func_call, + resume_wasm_func_call, + ExecutionOutcome, + }, CallParams, CallResults, EngineInner, @@ -13,10 +15,7 @@ use crate::{ ResumableCallHostTrap, ResumableCallOutOfFuel, }, - func::HostFuncEntity, - ir::{Slot, SlotSpan}, - store::CallHooks, - CallHook, + ir::SlotSpan, Error, Func, FuncEntity, @@ -24,12 +23,7 @@ use crate::{ StoreContextMut, }; -use super::{code_map::CodeMap, ResumableError}; - -mod cache; -mod instr_ptr; -mod instrs; -mod stack; +mod handler; impl EngineInner { /// Executes the given [`Func`] with the given `params` and returns the `results`. @@ -51,13 +45,9 @@ impl EngineInner { { let mut stack = self.stacks.lock().reuse_or_new(); let results = EngineExecutor::new(&self.code_map, &mut stack) - .execute_root_func(ctx.store, func, params, results) - .map_err(|error| match error.into_resumable() { - Ok(error) => error.into_error(), - Err(error) => error, - }); + .execute_root_func(ctx.store, func, params, results)?; self.stacks.lock().recycle(stack); - results + Ok(results) } /// Executes the given [`Func`] resumably with the given `params` and returns the `results`. @@ -79,42 +69,39 @@ impl EngineInner { { let store = ctx.store; let mut stack = self.stacks.lock().reuse_or_new(); - let results = EngineExecutor::new(&self.code_map, &mut stack) + let outcome = EngineExecutor::new(&self.code_map, &mut stack) .execute_root_func(store, func, params, results); - match results { - Ok(results) => { + let results = match outcome { + Ok(results) => results, + Err(ExecutionOutcome::Host(error)) => { + let host_func = *error.host_func(); + let caller_results = *error.caller_results(); + let host_error = error.into_error(); + return Ok(ResumableCallBase::HostTrap(ResumableCallHostTrap::new( + store.engine().clone(), + stack, + *func, + host_func, + host_error, + caller_results, + ))); + } + Err(ExecutionOutcome::OutOfFuel(error)) => { + let required_fuel = error.required_fuel(); + return Ok(ResumableCallBase::OutOfFuel(ResumableCallOutOfFuel::new( + store.engine().clone(), + stack, + *func, + required_fuel, + ))); + } + Err(ExecutionOutcome::Error(error)) => { self.stacks.lock().recycle(stack); - Ok(ResumableCallBase::Finished(results)) + return Err(error); } - Err(error) => match error.into_resumable() { - Ok(ResumableError::HostTrap(error)) => { - let host_func = *error.host_func(); - let caller_results = *error.caller_results(); - let host_error = error.into_error(); - Ok(ResumableCallBase::HostTrap(ResumableCallHostTrap::new( - store.engine().clone(), - *func, - host_func, - host_error, - caller_results, - stack, - ))) - } - Ok(ResumableError::OutOfFuel(error)) => { - let required_fuel = error.required_fuel(); - Ok(ResumableCallBase::OutOfFuel(ResumableCallOutOfFuel::new( - store.engine().clone(), - *func, - stack, - required_fuel, - ))) - } - Err(error) => { - self.stacks.lock().recycle(stack); - Err(error) - } - }, - } + }; + self.stacks.lock().recycle(stack); + Ok(ResumableCallBase::Finished(results)) } /// Resumes the given [`Func`] with the given `params` and returns the `results`. @@ -136,30 +123,27 @@ impl EngineInner { { let caller_results = invocation.caller_results(); let mut executor = EngineExecutor::new(&self.code_map, invocation.common.stack_mut()); - let results = executor.resume_func_host_trap(ctx.store, params, caller_results, results); - match results { - Ok(results) => { + let outcome = executor.resume_func_host_trap(ctx.store, params, caller_results, results); + let results = match outcome { + Ok(results) => results, + Err(ExecutionOutcome::Host(error)) => { + let host_func = *error.host_func(); + let caller_results = *error.caller_results(); + invocation.update(host_func, error.into_error(), caller_results); + return Ok(ResumableCallBase::HostTrap(invocation)); + } + Err(ExecutionOutcome::OutOfFuel(error)) => { + let required_fuel = error.required_fuel(); + let invocation = invocation.update_to_out_of_fuel(required_fuel); + return Ok(ResumableCallBase::OutOfFuel(invocation)); + } + Err(ExecutionOutcome::Error(error)) => { self.stacks.lock().recycle(invocation.common.take_stack()); - Ok(ResumableCallBase::Finished(results)) + return Err(error); } - Err(error) => match error.into_resumable() { - Ok(ResumableError::HostTrap(error)) => { - let host_func = *error.host_func(); - let caller_results = *error.caller_results(); - invocation.update(host_func, error.into_error(), caller_results); - Ok(ResumableCallBase::HostTrap(invocation)) - } - Ok(ResumableError::OutOfFuel(error)) => { - let required_fuel = error.required_fuel(); - let invocation = invocation.update_to_out_of_fuel(required_fuel); - Ok(ResumableCallBase::OutOfFuel(invocation)) - } - Err(error) => { - self.stacks.lock().recycle(invocation.common.take_stack()); - Err(error) - } - }, - } + }; + self.stacks.lock().recycle(invocation.common.take_stack()); + Ok(ResumableCallBase::Finished(results)) } /// Resumes the given [`Func`] after running out of fuel and returns the `results`. @@ -179,33 +163,27 @@ impl EngineInner { Results: CallResults, { let mut executor = EngineExecutor::new(&self.code_map, invocation.common.stack_mut()); - let results = executor.resume_func_out_of_fuel(ctx.store, results); - match results { - Ok(results) => { + let outcome = executor.resume_func_out_of_fuel(ctx.store, results); + let results = match outcome { + Ok(results) => results, + Err(ExecutionOutcome::Host(error)) => { + let host_func = *error.host_func(); + let caller_results = *error.caller_results(); + let invocation = + invocation.update_to_host_trap(host_func, error.into_error(), caller_results); + return Ok(ResumableCallBase::HostTrap(invocation)); + } + Err(ExecutionOutcome::OutOfFuel(error)) => { + invocation.update(error.required_fuel()); + return Ok(ResumableCallBase::OutOfFuel(invocation)); + } + Err(ExecutionOutcome::Error(error)) => { self.stacks.lock().recycle(invocation.common.take_stack()); - Ok(ResumableCallBase::Finished(results)) + return Err(error); } - Err(error) => match error.into_resumable() { - Ok(ResumableError::HostTrap(error)) => { - let host_func = *error.host_func(); - let caller_results = *error.caller_results(); - let invocation = invocation.update_to_host_trap( - host_func, - error.into_error(), - caller_results, - ); - Ok(ResumableCallBase::HostTrap(invocation)) - } - Ok(ResumableError::OutOfFuel(error)) => { - invocation.update(error.required_fuel()); - Ok(ResumableCallBase::OutOfFuel(invocation)) - } - Err(error) => { - self.stacks.lock().recycle(invocation.common.take_stack()); - Err(error) - } - }, - } + }; + self.stacks.lock().recycle(invocation.common.take_stack()); + Ok(ResumableCallBase::Finished(results)) } } @@ -218,10 +196,6 @@ pub struct EngineExecutor<'engine> { stack: &'engine mut Stack, } -/// Convenience function that does nothing to its `&mut` parameter. -#[inline] -fn do_nothing(_: &mut T) {} - impl<'engine> EngineExecutor<'engine> { /// Creates a new [`EngineExecutor`] for the given [`Stack`]. fn new(code_map: &'engine CodeMap, stack: &'engine mut Stack) -> Self { @@ -243,61 +217,30 @@ impl<'engine> EngineExecutor<'engine> { func: &Func, params: impl CallParams, results: Results, - ) -> Result<::Results, Error> + ) -> Result<::Results, ExecutionOutcome> where Results: CallResults, { self.stack.reset(); - match store.inner.resolve_func(func) { + let results = match store.inner.resolve_func(func) { FuncEntity::Wasm(wasm_func) => { // We reserve space on the stack to write the results of the root function execution. - let len_results = results.len_results(); - self.stack.values.extend_by(len_results, do_nothing)?; let instance = *wasm_func.instance(); let engine_func = wasm_func.func_body(); - let compiled_func = self - .code_map - .get(Some(store.inner.fuel_mut()), engine_func)?; - let (mut uninit_params, offsets) = self - .stack - .values - .alloc_call_frame(compiled_func, do_nothing)?; - for value in params.call_params() { - unsafe { uninit_params.init_next(value) }; - } - uninit_params.init_zeroes(); - self.stack.calls.push( - CallFrame::new( - InstructionPtr::new(compiled_func.instrs().as_ptr()), - offsets, - SlotSpan::new(Slot::from(0)), - ), - Some(instance), - )?; - store.invoke_call_hook(CallHook::CallingWasm)?; - self.execute_func(store)?; - store.invoke_call_hook(CallHook::ReturningFromWasm)?; + let call = + init_wasm_func_call(store, self.code_map, self.stack, engine_func, instance)?; + call.write_params(params).execute()?.write_results(results) } FuncEntity::Host(host_func) => { // The host function signature is required for properly // adjusting, inspecting and manipulating the value stack. // In case the host function returns more values than it takes // we are required to extend the value stack. - let len_params = host_func.len_params(); - let len_results = host_func.len_results(); - let max_inout = len_params.max(len_results); - let uninit = self - .stack - .values - .extend_by(usize::from(max_inout), do_nothing)?; - for (uninit, param) in uninit.iter_mut().zip(params.call_params()) { - uninit.write(param); - } let host_func = *host_func; - self.dispatch_host_func(store, host_func)?; + let call = init_host_func_call(store, self.stack, host_func)?; + call.write_params(params).execute()?.write_results(results) } }; - let results = self.write_results_back(results); Ok(results) } @@ -314,25 +257,16 @@ impl<'engine> EngineExecutor<'engine> { &mut self, store: &mut Store, params: impl CallParams, - caller_results: SlotSpan, + params_slots: SlotSpan, results: Results, - ) -> Result<::Results, Error> + ) -> Result<::Results, ExecutionOutcome> where Results: CallResults, { - let caller = self - .stack - .calls - .peek() - .expect("must have caller call frame on stack upon function resumption"); - let mut caller_sp = unsafe { self.stack.values.stack_ptr_at(caller.base_offset()) }; - let call_params = params.call_params(); - let len_params = call_params.len(); - for (result, param) in caller_results.iter_sized(len_params).zip(call_params) { - unsafe { caller_sp.set(result, param) }; - } - self.execute_func(store)?; - let results = self.write_results_back(results); + let results = resume_wasm_func_call(store, self.code_map, self.stack)? + .provide_host_results(params, params_slots) + .execute()? + .write_results(results); Ok(results) } @@ -348,57 +282,13 @@ impl<'engine> EngineExecutor<'engine> { &mut self, store: &mut Store, results: Results, - ) -> Result<::Results, Error> + ) -> Result<::Results, ExecutionOutcome> where Results: CallResults, { - self.execute_func(store)?; - let results = self.write_results_back(results); + let results = resume_wasm_func_call(store, self.code_map, self.stack)? + .execute()? + .write_results(results); Ok(results) } - - /// Executes the top most Wasm function on the [`Stack`] until the [`Stack`] is empty. - /// - /// # Errors - /// - /// When encountering a Wasm or host trap during execution. - #[inline(always)] - fn execute_func(&mut self, store: &mut Store) -> Result<(), Error> { - execute_instrs(store.prune(), self.stack, self.code_map) - } - - /// Convenience forwarder to [`dispatch_host_func`]. - #[inline(always)] - fn dispatch_host_func( - &mut self, - store: &mut Store, - host_func: HostFuncEntity, - ) -> Result<(), Error> { - dispatch_host_func( - store.prune(), - &mut self.stack.values, - host_func, - None, - CallHooks::Ignore, - )?; - Ok(()) - } - - /// Writes the results of the function execution back into the `results` buffer. - /// - /// # Note - /// - /// The value stack is empty after this operation. - /// - /// # Panics - /// - /// - If the `results` buffer length does not match the remaining amount of stack values. - #[inline(always)] - fn write_results_back(&mut self, results: Results) -> ::Results - where - Results: CallResults, - { - let len_results = results.len_results(); - results.call_results(&self.stack.values.as_slice()[..len_results]) - } } diff --git a/crates/wasmi/src/engine/executor/stack/calls.rs b/crates/wasmi/src/engine/executor/stack/calls.rs deleted file mode 100644 index f91cf686867..00000000000 --- a/crates/wasmi/src/engine/executor/stack/calls.rs +++ /dev/null @@ -1,266 +0,0 @@ -use super::{err_stack_overflow, BaseValueStackOffset, FrameValueStackOffset}; -use crate::{ - collections::HeadVec, - engine::executor::InstructionPtr, - ir::SlotSpan, - Instance, - TrapCode, -}; -use alloc::vec::Vec; - -#[cfg(doc)] -use crate::{ - engine::executor::stack::ValueStack, - engine::EngineFunc, - ir::Op, - ir::Slot, - Global, - Memory, - Table, -}; - -/// The stack of nested function calls. -#[derive(Debug, Default)] -pub struct CallStack { - /// The stack of nested function call frames. - frames: Vec, - /// The [`Instance`] used at certain frame stack heights. - instances: HeadVec, - /// The maximum allowed recursion depth. - /// - /// # Note - /// - /// A [`TrapCode::StackOverflow`] is raised if the recursion limit is exceeded. - recursion_limit: usize, -} - -impl CallStack { - /// Creates a new [`CallStack`] using the given recursion limit. - pub fn new(recursion_limit: usize) -> Self { - Self { - frames: Vec::new(), - instances: HeadVec::default(), - recursion_limit, - } - } - - /// Clears the [`CallStack`] entirely. - /// - /// # Note - /// - /// The [`CallStack`] can sometimes be left in a non-empty state upon - /// executing a function, for example when a trap is encountered. We - /// reset the [`CallStack`] before executing the next function to - /// provide a clean slate for all executions. - #[inline(always)] - pub fn reset(&mut self) { - self.frames.clear(); - self.instances.clear(); - } - - /// Returns the number of [`CallFrame`]s on the [`CallStack`]. - #[inline(always)] - fn len(&self) -> usize { - self.frames.len() - } - - /// Returns `true` if the [`CallStack`] is empty. - #[inline(always)] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the currently used [`Instance`]. - #[inline(always)] - pub fn instance(&self) -> Option<&Instance> { - self.instances.last() - } - - /// Returns the currently used [`Instance`]. - /// - /// # Panics - /// - /// If there is no currently used [`Instance`]. - /// This happens if the [`CallStack`] is empty. - #[inline(always)] - #[track_caller] - pub fn instance_expect(&self) -> &Instance { - self.instance() - .expect("the currently used instance must be present") - } - - /// Pushes a [`CallFrame`] onto the [`CallStack`]. - /// - /// # Errors - /// - /// If the recursion limit has been reached. - #[inline(always)] - pub fn push( - &mut self, - mut call: CallFrame, - instance: Option, - ) -> Result<(), TrapCode> { - if self.len() == self.recursion_limit { - return Err(err_stack_overflow()); - } - if let Some(instance) = instance { - call.changed_instance = self.push_instance(instance); - } - self.frames.push(call); - Ok(()) - } - - /// Pushes the `instance` onto the internal instances stack. - /// - /// Returns `true` if the [`Instance`] stack has been adjusted. - #[inline(always)] - fn push_instance(&mut self, instance: Instance) -> bool { - if let Some(last) = self.instances.last() { - if instance.eq(last) { - return false; - } - } - self.instances.push(instance); - true - } - - /// Pops the last [`CallFrame`] from the [`CallStack`] if any. - /// - /// Returns the popped [`Instance`] in case the popped [`CallFrame`] - /// introduced a new [`Instance`] on the [`CallStack`]. - #[inline(always)] - pub fn pop(&mut self) -> Option<(CallFrame, Option)> { - let frame = self.frames.pop()?; - let instance = match frame.changed_instance { - true => self.instances.pop(), - false => None, - }; - Some((frame, instance)) - } - - /// Peeks the last [`CallFrame`] of the [`CallStack`] if any. - #[inline(always)] - pub fn peek(&self) -> Option<&CallFrame> { - self.frames.last() - } - - /// Peeks the last [`CallFrame`] of the [`CallStack`] if any. - #[inline(always)] - pub fn peek_mut(&mut self) -> Option<&mut CallFrame> { - self.frames.last_mut() - } - - /// Peeks the two top-most [`CallFrame`] on the [`CallStack`] if any. - /// - /// # Note - /// - /// - The top-most [`CallFrame`] on the [`CallStack`] is referred to as the `callee`. - /// - The second top-most [`CallFrame`] on the [`CallStack`] is referred to as the `caller`. - /// - /// So this function returns a pair of `(callee, caller?)`. - pub fn peek_2(&self) -> Option<(&CallFrame, Option<&CallFrame>)> { - let (callee, remaining) = self.frames.split_last()?; - let caller = remaining.last(); - Some((callee, caller)) - } -} - -/// Offsets for a [`CallFrame`] into the [`ValueStack`]. -#[derive(Debug, Copy, Clone)] -pub struct StackOffsets { - /// Offset to the first mutable cell of a [`CallFrame`]. - pub base: BaseValueStackOffset, - /// Offset to the first cell of a [`CallFrame`]. - pub frame: FrameValueStackOffset, -} - -impl StackOffsets { - /// Moves the [`StackOffsets`] values down by `delta`. - /// - /// # Note - /// - /// This is used for the implementation of tail calls. - #[inline(always)] - fn move_down(&mut self, delta: usize) { - let base = usize::from(self.base); - let frame = usize::from(self.frame); - debug_assert!(delta <= base); - debug_assert!(delta <= frame); - self.base = BaseValueStackOffset::new(base - delta); - self.frame = FrameValueStackOffset::new(frame - delta); - } -} - -/// A single frame of a called [`EngineFunc`]. -#[derive(Debug, Copy, Clone)] -pub struct CallFrame { - /// The pointer to the [`Op`] that is executed next. - instr_ptr: InstructionPtr, - /// Offsets of the [`CallFrame`] into the [`ValueStack`]. - offsets: StackOffsets, - /// Span of registers were the caller expects them in its [`CallFrame`]. - results: SlotSpan, - /// Is `true` if this [`CallFrame`] changed the currently used [`Instance`]. - /// - /// - This flag is an optimization to reduce the amount of accesses on the - /// instance stack of the [`CallStack`] for the common case where this is - /// not needed. - /// - This flag is private to the [`CallStack`] and shall not be observable - /// from the outside. - changed_instance: bool, -} - -impl CallFrame { - /// Creates a new [`CallFrame`]. - pub fn new(instr_ptr: InstructionPtr, offsets: StackOffsets, results: SlotSpan) -> Self { - Self { - instr_ptr, - offsets, - results, - changed_instance: false, - } - } - - /// Moves the [`ValueStack`] offsets of the [`CallFrame`] down by `delta`. - /// - /// # Note - /// - /// This is used for the implementation of tail calls. - pub fn move_down(&mut self, delta: usize) { - self.offsets.move_down(delta); - } - - /// Updates the [`InstructionPtr`] of the [`CallFrame`]. - /// - /// This is required before dispatching a nested function call to update - /// the instruction pointer of the caller so that it can continue at that - /// position when the called function returns. - pub fn update_instr_ptr(&mut self, new_instr_ptr: InstructionPtr) { - self.instr_ptr = new_instr_ptr; - } - - /// Returns the [`InstructionPtr`] of the [`CallFrame`]. - pub fn instr_ptr(&self) -> InstructionPtr { - self.instr_ptr - } - - /// Returns the [`FrameValueStackOffset`] of the [`CallFrame`]. - pub fn frame_offset(&self) -> FrameValueStackOffset { - self.offsets.frame - } - - /// Returns the [`BaseValueStackOffset`] of the [`CallFrame`]. - pub fn base_offset(&self) -> BaseValueStackOffset { - self.offsets.base - } - - /// Returns the [`SlotSpan`] of the [`CallFrame`]. - /// - /// # Note - /// - /// The registers yielded by the returned [`SlotSpan`] - /// refer to the [`CallFrame`] of the caller of this [`CallFrame`]. - pub fn results(&self) -> SlotSpan { - self.results - } -} diff --git a/crates/wasmi/src/engine/executor/stack/mod.rs b/crates/wasmi/src/engine/executor/stack/mod.rs deleted file mode 100644 index d74a268f2f7..00000000000 --- a/crates/wasmi/src/engine/executor/stack/mod.rs +++ /dev/null @@ -1,88 +0,0 @@ -mod calls; -mod values; - -pub use self::{ - calls::{CallFrame, CallStack, StackOffsets}, - values::{BaseValueStackOffset, FrameParams, FrameSlots, FrameValueStackOffset, ValueStack}, -}; -use crate::{engine::StackConfig, Instance, TrapCode}; - -/// Returns a [`TrapCode`] signalling a stack overflow. -#[cold] -fn err_stack_overflow() -> TrapCode { - TrapCode::StackOverflow -} - -/// Data structure that combines both value stack and call stack. -#[derive(Debug, Default)] -pub struct Stack { - /// The call stack. - pub calls: CallStack, - /// The value stack. - pub values: ValueStack, -} - -impl Stack { - /// Creates a new [`Stack`] given the [`Config`]. - /// - /// [`Config`]: [`crate::Config`] - pub fn new(config: &StackConfig) -> Self { - let calls = CallStack::new(config.max_recursion_depth()); - let values = ValueStack::new(config.min_stack_height(), config.max_stack_height()); - Self { calls, values } - } - - /// Resets the [`Stack`] for clean reuse. - pub fn reset(&mut self) { - self.calls.reset(); - self.values.reset(); - } - - /// Create an empty [`Stack`]. - /// - /// # Note - /// - /// Empty stacks require no heap allocations and are cheap to construct. - pub fn empty() -> Self { - Self { - values: ValueStack::empty(), - calls: CallStack::default(), - } - } - - /// Returns the capacity of the [`Stack`]. - pub fn capacity(&self) -> usize { - self.values.capacity() - } - - /// Merge the two top-most [`CallFrame`] with respect to a tail call. - /// - /// # Panics (Debug) - /// - /// - If the two top-most [`CallFrame`] do not have matching `results`. - /// - If there are not at least two [`CallFrame`] on the [`CallStack`]. - /// - /// # Safety - /// - /// Any [`FrameSlots`] allocated within the range `from..to` on the [`ValueStack`] - /// may be invalidated by this operation. It is the caller's responsibility to reinstantiate - /// all [`FrameSlots`] affected by this. - #[inline] - #[must_use] - pub unsafe fn merge_call_frames(&mut self, callee: &mut CallFrame) -> Option { - let (caller, instance) = self.calls.pop().expect("caller call frame must exist"); - debug_assert_eq!(callee.results(), caller.results()); - debug_assert!(caller.base_offset() <= callee.base_offset()); - // Safety: - // - // We only drain cells of the second top-most call frame on the value stack. - // Therefore only value stack offsets of the top-most call frame on the - // value stack are going to be invalidated which we ensure to adjust and - // reinstantiate after this operation. - let len_drained = self - .values - .drain(caller.frame_offset(), callee.frame_offset()); - callee.move_down(len_drained); - instance - } -} diff --git a/crates/wasmi/src/engine/executor/stack/values.rs b/crates/wasmi/src/engine/executor/stack/values.rs deleted file mode 100644 index 499218e7bff..00000000000 --- a/crates/wasmi/src/engine/executor/stack/values.rs +++ /dev/null @@ -1,456 +0,0 @@ -use super::{err_stack_overflow, StackOffsets}; -use crate::{ - core::{ReadAs, UntypedVal, WriteAs}, - engine::code_map::CompiledFuncRef, - ir::Slot, - TrapCode, -}; -use alloc::vec::Vec; -use core::{ - fmt::{self, Debug}, - mem::{self, MaybeUninit}, - ops::Range, - ptr, - slice, -}; - -#[cfg(doc)] -use super::calls::CallFrame; -#[cfg(doc)] -use crate::engine::EngineFunc; - -pub struct ValueStack { - /// The values on the [`ValueStack`]. - values: Vec, - /// Maximal possible `sp` value. - max_len: usize, -} - -impl ValueStack { - /// Default value for initial value stack height in bytes. - pub const DEFAULT_MIN_HEIGHT: usize = 1024; - - /// Default value for maximum value stack height in bytes. - pub const DEFAULT_MAX_HEIGHT: usize = 1024 * Self::DEFAULT_MIN_HEIGHT; -} - -impl Debug for ValueStack { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ValueStack") - .field("max_len", &self.max_len) - .field("entries", &&self.values[..]) - .finish() - } -} - -#[cfg(test)] -impl PartialEq for ValueStack { - fn eq(&self, other: &Self) -> bool { - self.values == other.values - } -} - -#[cfg(test)] -impl Eq for ValueStack {} - -impl Default for ValueStack { - fn default() -> Self { - const REGISTER_SIZE: usize = mem::size_of::(); - Self::new( - Self::DEFAULT_MIN_HEIGHT / REGISTER_SIZE, - Self::DEFAULT_MAX_HEIGHT / REGISTER_SIZE, - ) - } -} - -impl ValueStack { - /// Creates a new empty [`ValueStack`]. - /// - /// # Panics - /// - /// - If the `initial_len` is zero. - /// - If the `initial_len` is greater than `maximum_len`. - pub fn new(initial_len: usize, maximum_len: usize) -> Self { - assert!( - initial_len > 0, - "cannot initialize the value stack with zero length", - ); - assert!( - initial_len <= maximum_len, - "initial value stack length is greater than maximum value stack length", - ); - Self { - values: Vec::with_capacity(initial_len), - max_len: maximum_len, - } - } - - /// Creates an empty [`ValueStack`] that does not allocate heap memory. - /// - /// # Note - /// - /// This is required for resumable functions in order to replace their - /// proper stack with a cheap dummy one. - pub fn empty() -> Self { - Self { - values: Vec::new(), - max_len: 0, - } - } - - /// Resets the [`ValueStack`] for reuse. - /// - /// # Note - /// - /// The [`ValueStack`] can sometimes be left in a non-empty state upon - /// executing a function, for example when a trap is encountered. We - /// reset the [`ValueStack`] before executing the next function to - /// provide a clean slate for all executions. - pub fn reset(&mut self) { - self.values.clear(); - } - - /// Returns the root [`FrameSlots`] pointing to the first value on the [`ValueStack`]. - pub fn root_stack_ptr(&mut self) -> FrameSlots { - FrameSlots::new(self.values.as_mut_ptr()) - } - - /// Returns the [`FrameSlots`] at the given `offset`. - pub unsafe fn stack_ptr_at(&mut self, offset: impl Into) -> FrameSlots { - let ptr = self.values.as_mut_ptr().add(offset.into().0); - FrameSlots::new(ptr) - } - - /// Returns the capacity of the [`ValueStack`]. - pub fn capacity(&self) -> usize { - debug_assert!(self.values.len() <= self.values.capacity()); - self.values.capacity() - } - - /// Reserves enough space for `additional` cells on the [`ValueStack`]. - /// - /// This may heap allocate in case the [`ValueStack`] ran out of preallocated memory. - /// - /// # Errors - /// - /// When trying to grow the [`ValueStack`] over its maximum size limit. - #[inline(always)] - pub fn extend_by( - &mut self, - additional: usize, - on_resize: impl FnOnce(&mut Self), - ) -> Result<&mut [MaybeUninit], TrapCode> { - if additional >= self.max_len() - self.len() { - return Err(err_stack_overflow()); - } - let prev_capacity = self.capacity(); - self.values.reserve(additional); - if prev_capacity != self.capacity() { - on_resize(self); - } - let spare = self.values.spare_capacity_mut().as_mut_ptr(); - unsafe { self.values.set_len(self.values.len() + additional) }; - Ok(unsafe { slice::from_raw_parts_mut(spare, additional) }) - } - - /// Returns the current length of the [`ValueStack`]. - #[inline(always)] - fn len(&self) -> usize { - debug_assert!(self.values.len() <= self.max_len); - self.values.len() - } - - /// Returns the maximum length of the [`ValueStack`]. - #[inline(always)] - fn max_len(&self) -> usize { - debug_assert!(self.values.len() <= self.max_len); - self.max_len - } - - /// Drop the last `amount` cells of the [`ValueStack`]. - /// - /// # Panics (Debug) - /// - /// If `amount` is greater than the [`ValueStack`] height. - #[inline(always)] - pub fn drop(&mut self, amount: usize) { - assert!(self.len() >= amount); - // Safety: we just asserted that the current length is large enough to not underflow. - unsafe { self.values.set_len(self.len() - amount) }; - } - - /// Drop the last `amount` cells of the [`ValueStack`] and returns a slice to them. - /// - /// # Panics (Debug) - /// - /// If `amount` is greater than the [`ValueStack`] height. - #[inline(always)] - pub fn drop_return(&mut self, amount: usize) -> &[UntypedVal] { - let len = self.len(); - let dropped = unsafe { self.values.get_unchecked(len - amount..) }.as_ptr(); - self.drop(amount); - unsafe { slice::from_raw_parts(dropped, amount) } - } - - /// Shrink the [`ValueStack`] to the [`ValueStackOffset`]. - /// - /// # Panics (Debug) - /// - /// If `new_sp` is greater than the current [`ValueStack`] pointer. - #[inline(always)] - pub fn truncate(&mut self, new_len: impl Into) { - let new_len = new_len.into().0; - assert!(new_len <= self.len()); - // Safety: we just asserted that the new length is valid. - unsafe { self.values.set_len(new_len) }; - } - - /// Allocates a new [`EngineFunc`] on the [`ValueStack`]. - /// - /// Returns the [`BaseValueStackOffset`] and [`FrameValueStackOffset`] of the allocated [`EngineFunc`]. - /// - /// # Note - /// - /// - All live [`FrameSlots`] might be invalidated and need to be reinstantiated. - /// - The parameters of the allocated [`EngineFunc`] are set to zero - /// and require proper initialization after this call. - /// - /// # Errors - /// - /// When trying to grow the [`ValueStack`] over its maximum size limit. - pub fn alloc_call_frame( - &mut self, - func: CompiledFuncRef, - on_resize: impl FnMut(&mut Self), - ) -> Result<(FrameParams, StackOffsets), TrapCode> { - let len_stack_slots = func.len_stack_slots(); - let len_consts = func.consts().len(); - let len = self.len(); - let mut spare = self - .extend_by(len_stack_slots as usize, on_resize)? - .iter_mut(); - (&mut spare) - .zip(func.consts()) - .for_each(|(uninit, const_value)| { - uninit.write(*const_value); - }); - let params = FrameParams::new(spare.into_slice()); - let frame = ValueStackOffset(len); - let base = ValueStackOffset(len + len_consts); - Ok(( - params, - StackOffsets { - base: BaseValueStackOffset(base), - frame: FrameValueStackOffset(frame), - }, - )) - } - - /// Returns a shared slice over the values of the [`ValueStack`]. - #[inline(always)] - pub fn as_slice(&self) -> &[UntypedVal] { - self.values.as_slice() - } - - /// Returns an exclusive slice over the values of the [`ValueStack`]. - #[inline(always)] - pub fn as_slice_mut(&mut self) -> &mut [UntypedVal] { - self.values.as_mut_slice() - } - - /// Removes the slice `from..to` of [`UntypedVal`] cells from the [`ValueStack`]. - /// - /// Returns the number of drained [`ValueStack`] cells. - /// - /// # Safety - /// - /// - This invalidates all [`FrameSlots`] within the range `from..` and the caller has to - /// make sure to properly reinstantiate all those pointers after this operation. - /// - This also invalidates all [`FrameValueStackOffset`] and [`BaseValueStackOffset`] indices - /// within the range `from..`. - #[inline(always)] - pub fn drain(&mut self, from: FrameValueStackOffset, to: FrameValueStackOffset) -> usize { - debug_assert!(from <= to); - let from = from.0 .0; - let to = to.0 .0; - debug_assert!(from <= self.len()); - debug_assert!(to <= self.len()); - let len_drained = to - from; - self.values.drain(from..to); - len_drained - } -} - -/// The offset of the [`FrameSlots`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct ValueStackOffset(usize); - -impl From for ValueStackOffset { - fn from(offset: FrameValueStackOffset) -> Self { - offset.0 - } -} - -impl From for ValueStackOffset { - fn from(offset: BaseValueStackOffset) -> Self { - offset.0 - } -} - -/// Returned when allocating a new [`CallFrame`] on the [`ValueStack`]. -/// -/// # Note -/// -/// This points to the first cell of the allocated [`CallFrame`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct FrameValueStackOffset(ValueStackOffset); - -impl FrameValueStackOffset { - /// Creates a new [`FrameValueStackOffset`] at the `index`. - pub(super) fn new(index: usize) -> Self { - Self(ValueStackOffset(index)) - } -} - -impl From for usize { - fn from(offset: FrameValueStackOffset) -> usize { - offset.0 .0 - } -} - -impl From for FrameValueStackOffset { - fn from(offset: ValueStackOffset) -> Self { - Self(offset) - } -} - -/// Returned when allocating a new [`CallFrame`] on the [`ValueStack`]. -/// -/// # Note -/// -/// This points to the first mutable cell of the allocated [`CallFrame`]. -/// The first mutable cell of a [`CallFrame`] is accessed by [`Slot(0)`]. -/// -/// [`Slot(0)`]: [`Slot`] -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct BaseValueStackOffset(ValueStackOffset); - -impl BaseValueStackOffset { - /// Creates a new [`BaseValueStackOffset`] at the `index`. - pub(super) fn new(index: usize) -> Self { - Self(ValueStackOffset(index)) - } -} - -impl From for usize { - fn from(offset: BaseValueStackOffset) -> usize { - offset.0 .0 - } -} - -/// Uninitialized parameters of a [`CallFrame`]. -pub struct FrameParams { - range: Range<*mut MaybeUninit>, -} - -impl FrameParams { - /// Creates a new [`FrameSlots`]. - pub fn new(ptr: &mut [MaybeUninit]) -> Self { - Self { - range: ptr.as_mut_ptr_range(), - } - } - - /// Sets the value of the `register` to `value`.` - /// - /// # Safety - /// - /// It is the callers responsibility to provide a [`Slot`] that - /// does not access the underlying [`ValueStack`] out of bounds. - pub unsafe fn init_next(&mut self, value: UntypedVal) { - self.range.start.write(MaybeUninit::new(value)); - self.range.start = self.range.start.add(1); - } - - /// Zero-initialize the remaining locals and parameters. - pub fn init_zeroes(mut self) { - debug_assert!(self.range.start <= self.range.end); - while !core::ptr::eq(self.range.start, self.range.end) { - // Safety: We do not write out-of-buffer due to the above condition. - unsafe { self.init_next(UntypedVal::from(0_u64)) } - } - } -} - -/// Accessor to the [`Slot`] values of a [`CallFrame`] on the [`CallStack`]. -/// -/// [`CallStack`]: [`super::CallStack`] -pub struct FrameSlots { - /// The underlying raw pointer to a [`CallFrame`] on the [`ValueStack`]. - ptr: *mut UntypedVal, -} - -impl Debug for FrameSlots { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", &self.ptr) - } -} - -impl FrameSlots { - /// Creates a new [`FrameSlots`]. - fn new(ptr: *mut UntypedVal) -> Self { - Self { ptr } - } - - /// Returns the [`UntypedVal`] at the given [`Slot`]. - /// - /// # Safety - /// - /// It is the callers responsibility to provide a [`Slot`] that - /// does not access the underlying [`ValueStack`] out of bounds. - pub unsafe fn get(&self, slot: Slot) -> UntypedVal { - ptr::read(self.register_offset(slot)) - } - - /// Returns the [`UntypedVal`] at the given [`Slot`]. - /// - /// # Safety - /// - /// It is the callers responsibility to provide a [`Slot`] that - /// does not access the underlying [`ValueStack`] out of bounds. - pub unsafe fn read_as(&self, slot: Slot) -> T - where - UntypedVal: ReadAs, - { - UntypedVal::read_as(&*self.register_offset(slot)) - } - - /// Sets the value of the `register` to `value`.` - /// - /// # Safety - /// - /// It is the callers responsibility to provide a [`Slot`] that - /// does not access the underlying [`ValueStack`] out of bounds. - pub unsafe fn set(&mut self, slot: Slot, value: UntypedVal) { - ptr::write(self.register_offset(slot), value) - } - - /// Sets the value of the `register` to `value`.` - /// - /// # Safety - /// - /// It is the callers responsibility to provide a [`Slot`] that - /// does not access the underlying [`ValueStack`] out of bounds. - pub unsafe fn write_as(&mut self, slot: Slot, value: T) - where - UntypedVal: WriteAs, - { - let val: &mut UntypedVal = &mut *self.register_offset(slot); - val.write_as(value); - } - - /// Returns the underlying pointer offset by the [`Slot`] index. - unsafe fn register_offset(&self, slot: Slot) -> *mut UntypedVal { - unsafe { self.ptr.offset(isize::from(i16::from(slot))) } - } -} diff --git a/crates/wasmi/src/engine/mod.rs b/crates/wasmi/src/engine/mod.rs index 4149a47c65b..fc350bdff99 100644 --- a/crates/wasmi/src/engine/mod.rs +++ b/crates/wasmi/src/engine/mod.rs @@ -13,7 +13,7 @@ mod utils; pub(crate) use self::{ block_type::BlockType, - executor::Stack, + executor::{Inst, Stack}, func_types::DedupFuncType, translator::{ FuncTranslationDriver, @@ -37,7 +37,6 @@ pub use self::{ ResumableCall, ResumableCallHostTrap, ResumableCallOutOfFuel, - ResumableError, ResumableHostTrapError, ResumableOutOfFuelError, TypedResumableCall, @@ -49,7 +48,6 @@ pub use self::{ }; use crate::{ collections::arena::{ArenaIndex, GuardedEntity}, - func::FuncInOut, module::{FuncIdx, ModuleHeader}, Error, Func, @@ -518,6 +516,9 @@ pub struct EngineStacks { config: StackConfig, } +unsafe impl Send for EngineStacks {} // TODO: write safety docs +unsafe impl Sync for EngineStacks {} // TODO: write safety docs + impl EngineStacks { /// Creates new [`EngineStacks`] with the given [`StackConfig`]. pub fn new(config: &StackConfig) -> Self { diff --git a/crates/wasmi/src/engine/resumable.rs b/crates/wasmi/src/engine/resumable.rs index 642e8d468e5..14a96aa7869 100644 --- a/crates/wasmi/src/engine/resumable.rs +++ b/crates/wasmi/src/engine/resumable.rs @@ -7,7 +7,6 @@ use crate::{ AsContextMut, Engine, Error, - TrapCode, Val, WasmResults, }; @@ -32,23 +31,6 @@ pub(crate) enum ResumableCallBase { OutOfFuel(ResumableCallOutOfFuel), } -/// Any resumable error. -#[derive(Debug)] -pub enum ResumableError { - HostTrap(ResumableHostTrapError), - OutOfFuel(ResumableOutOfFuelError), -} - -impl ResumableError { - /// Consumes `self` to return the underlying [`Error`]. - pub fn into_error(self) -> Error { - match self { - ResumableError::HostTrap(error) => error.into_error(), - ResumableError::OutOfFuel(error) => error.into_error(), - } - } -} - /// Error returned from a called host function in a resumable state. #[derive(Debug)] pub struct ResumableHostTrapError { @@ -125,11 +107,6 @@ impl ResumableOutOfFuelError { pub(crate) fn required_fuel(self) -> u64 { self.required_fuel } - - /// Consumes `self` to return the underlying [`Error`]. - pub(crate) fn into_error(self) -> Error { - Error::from(TrapCode::OutOfFuel) - } } /// Returned by calling a [`Func`] in a resumable way. @@ -283,11 +260,11 @@ impl ResumableCallHostTrap { /// Creates a new [`ResumableCallHostTrap`]. pub(super) fn new( engine: Engine, + stack: Stack, func: Func, host_func: Func, host_error: Error, caller_results: SlotSpan, - stack: Stack, ) -> Self { Self { common: ResumableCallCommon::new(engine, func, stack), @@ -414,7 +391,7 @@ pub struct ResumableCallOutOfFuel { impl ResumableCallOutOfFuel { /// Creates a new [`ResumableCallOutOfFuel`]. - pub(super) fn new(engine: Engine, func: Func, stack: Stack, required_fuel: u64) -> Self { + pub(super) fn new(engine: Engine, stack: Stack, func: Func, required_fuel: u64) -> Self { Self { common: ResumableCallCommon::new(engine, func, stack), required_fuel, diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index 98d97c9b7b8..89396459ae1 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -1,27 +1,8 @@ use crate::{ - core::UntypedVal, - ir::{BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset, Op, Slot}, + ir::{self, BranchOffset, Op, Slot}, Error, }; -/// Types able to allocate function local constant values. -/// -/// # Note -/// -/// This allows to cheaply convert immediate values to [`Slot`]s. -/// -/// # Errors -/// -/// If the function local constant allocation from immediate value to [`Slot`] failed. -pub trait AllocConst { - /// Allocates a new function local constant value and returns its [`Slot`]. - /// - /// # Note - /// - /// Constant values allocated this way are deduplicated and return shared [`Slot`]. - fn alloc_const>(&mut self, value: T) -> Result; -} - /// Extension trait to return [`Slot`] result of compare [`Op`]s. pub trait CompareResult { /// Returns the result [`Slot`] of the compare [`Op`]. @@ -38,173 +19,108 @@ pub trait CompareResult { impl CompareResult for Op { fn compare_result(&self) -> Option { let result = match *self { - | Op::I32BitAnd { result, .. } - | Op::I32BitAndImm16 { result, .. } - | Op::I32BitOr { result, .. } - | Op::I32BitOrImm16 { result, .. } - | Op::I32BitXor { result, .. } - | Op::I32BitXorImm16 { result, .. } - | Op::I32And { result, .. } - | Op::I32AndImm16 { result, .. } - | Op::I32Or { result, .. } - | Op::I32OrImm16 { result, .. } - | Op::I32Nand { result, .. } - | Op::I32NandImm16 { result, .. } - | Op::I32Nor { result, .. } - | Op::I32NorImm16 { result, .. } - | Op::I32Eq { result, .. } - | Op::I32EqImm16 { result, .. } - | Op::I32Ne { result, .. } - | Op::I32NeImm16 { result, .. } - | Op::I32LtS { result, .. } - | Op::I32LtSImm16Lhs { result, .. } - | Op::I32LtSImm16Rhs { result, .. } - | Op::I32LtU { result, .. } - | Op::I32LtUImm16Lhs { result, .. } - | Op::I32LtUImm16Rhs { result, .. } - | Op::I32LeS { result, .. } - | Op::I32LeSImm16Lhs { result, .. } - | Op::I32LeSImm16Rhs { result, .. } - | Op::I32LeU { result, .. } - | Op::I32LeUImm16Lhs { result, .. } - | Op::I32LeUImm16Rhs { result, .. } - | Op::I64BitAnd { result, .. } - | Op::I64BitAndImm16 { result, .. } - | Op::I64BitOr { result, .. } - | Op::I64BitOrImm16 { result, .. } - | Op::I64BitXor { result, .. } - | Op::I64BitXorImm16 { result, .. } - | Op::I64And { result, .. } - | Op::I64AndImm16 { result, .. } - | Op::I64Or { result, .. } - | Op::I64OrImm16 { result, .. } - | Op::I64Nand { result, .. } - | Op::I64NandImm16 { result, .. } - | Op::I64Nor { result, .. } - | Op::I64NorImm16 { result, .. } - | Op::I64Eq { result, .. } - | Op::I64EqImm16 { result, .. } - | Op::I64Ne { result, .. } - | Op::I64NeImm16 { result, .. } - | Op::I64LtS { result, .. } - | Op::I64LtSImm16Lhs { result, .. } - | Op::I64LtSImm16Rhs { result, .. } - | Op::I64LtU { result, .. } - | Op::I64LtUImm16Lhs { result, .. } - | Op::I64LtUImm16Rhs { result, .. } - | Op::I64LeS { result, .. } - | Op::I64LeSImm16Lhs { result, .. } - | Op::I64LeSImm16Rhs { result, .. } - | Op::I64LeU { result, .. } - | Op::I64LeUImm16Lhs { result, .. } - | Op::I64LeUImm16Rhs { result, .. } - | Op::F32Eq { result, .. } - | Op::F32Ne { result, .. } - | Op::F32Lt { result, .. } - | Op::F32Le { result, .. } - | Op::F32NotLt { result, .. } - | Op::F32NotLe { result, .. } - | Op::F64Eq { result, .. } - | Op::F64Ne { result, .. } - | Op::F64Lt { result, .. } - | Op::F64Le { result, .. } - | Op::F64NotLt { result, .. } - | Op::F64NotLe { result, .. } => result, + // i32 + | Op::I32BitAnd_Sss { result, .. } + | Op::I32BitAnd_Ssi { result, .. } + | Op::I32BitOr_Sss { result, .. } + | Op::I32BitOr_Ssi { result, .. } + | Op::I32BitXor_Sss { result, .. } + | Op::I32BitXor_Ssi { result, .. } + | Op::I32Eq_Sss { result, .. } + | Op::I32Eq_Ssi { result, .. } + | Op::I32And_Sss { result, .. } + | Op::I32And_Ssi { result, .. } + | Op::I32Or_Sss { result, .. } + | Op::I32Or_Ssi { result, .. } + | Op::I32NotEq_Sss { result, .. } + | Op::I32NotEq_Ssi { result, .. } + | Op::I32NotAnd_Sss { result, .. } + | Op::I32NotAnd_Ssi { result, .. } + | Op::I32NotOr_Sss { result, .. } + | Op::I32NotOr_Ssi { result, .. } + | Op::I32Lt_Sss { result, .. } + | Op::I32Lt_Ssi { result, .. } + | Op::I32Lt_Sis { result, .. } + | Op::U32Lt_Sss { result, .. } + | Op::U32Lt_Ssi { result, .. } + | Op::U32Lt_Sis { result, .. } + | Op::I32Le_Sss { result, .. } + | Op::I32Le_Ssi { result, .. } + | Op::I32Le_Sis { result, .. } + | Op::U32Le_Sss { result, .. } + | Op::U32Le_Ssi { result, .. } + | Op::U32Le_Sis { result, .. } + // i64 + | Op::I64BitAnd_Sss { result, .. } + | Op::I64BitAnd_Ssi { result, .. } + | Op::I64BitOr_Sss { result, .. } + | Op::I64BitOr_Ssi { result, .. } + | Op::I64BitXor_Sss { result, .. } + | Op::I64BitXor_Ssi { result, .. } + | Op::I64Eq_Sss { result, .. } + | Op::I64Eq_Ssi { result, .. } + | Op::I64And_Sss { result, .. } + | Op::I64And_Ssi { result, .. } + | Op::I64Or_Sss { result, .. } + | Op::I64Or_Ssi { result, .. } + | Op::I64NotEq_Sss { result, .. } + | Op::I64NotEq_Ssi { result, .. } + | Op::I64NotAnd_Sss { result, .. } + | Op::I64NotAnd_Ssi { result, .. } + | Op::I64NotOr_Sss { result, .. } + | Op::I64NotOr_Ssi { result, .. } + | Op::I64Lt_Sss { result, .. } + | Op::I64Lt_Ssi { result, .. } + | Op::I64Lt_Sis { result, .. } + | Op::U64Lt_Sss { result, .. } + | Op::U64Lt_Ssi { result, .. } + | Op::U64Lt_Sis { result, .. } + | Op::I64Le_Sss { result, .. } + | Op::I64Le_Ssi { result, .. } + | Op::I64Le_Sis { result, .. } + | Op::U64Le_Sss { result, .. } + | Op::U64Le_Ssi { result, .. } + | Op::U64Le_Sis { result, .. } + // f32 + | Op::F32Eq_Sss { result, .. } + | Op::F32Eq_Ssi { result, .. } + | Op::F32Lt_Sss { result, .. } + | Op::F32Lt_Ssi { result, .. } + | Op::F32Lt_Sis { result, .. } + | Op::F32Le_Sss { result, .. } + | Op::F32Le_Ssi { result, .. } + | Op::F32Le_Sis { result, .. } + | Op::F32NotEq_Sss { result, .. } + | Op::F32NotEq_Ssi { result, .. } + | Op::F32NotLt_Sss { result, .. } + | Op::F32NotLt_Ssi { result, .. } + | Op::F32NotLt_Sis { result, .. } + | Op::F32NotLe_Sss { result, .. } + | Op::F32NotLe_Ssi { result, .. } + | Op::F32NotLe_Sis { result, .. } + // f64 + | Op::F64Eq_Sss { result, .. } + | Op::F64Eq_Ssi { result, .. } + | Op::F64Lt_Sss { result, .. } + | Op::F64Lt_Ssi { result, .. } + | Op::F64Lt_Sis { result, .. } + | Op::F64Le_Sss { result, .. } + | Op::F64Le_Ssi { result, .. } + | Op::F64Le_Sis { result, .. } + | Op::F64NotEq_Sss { result, .. } + | Op::F64NotEq_Ssi { result, .. } + | Op::F64NotLt_Sss { result, .. } + | Op::F64NotLt_Ssi { result, .. } + | Op::F64NotLt_Sis { result, .. } + | Op::F64NotLe_Sss { result, .. } + | Op::F64NotLe_Ssi { result, .. } + | Op::F64NotLe_Sis { result, .. } => result, _ => return None, }; Some(result) } } -pub trait ReplaceCmpResult: Sized { - /// Returns `self` `cmp` instruction with the `new_result`. - /// - /// Returns `None` if `self` is not a `cmp` instruction. - fn replace_cmp_result(&self, new_result: Slot) -> Option; -} - -impl ReplaceCmpResult for Op { - fn replace_cmp_result(&self, new_result: Slot) -> Option { - let mut copy = *self; - match &mut copy { - | Op::I32BitAnd { result, .. } - | Op::I32BitAndImm16 { result, .. } - | Op::I32BitOr { result, .. } - | Op::I32BitOrImm16 { result, .. } - | Op::I32BitXor { result, .. } - | Op::I32BitXorImm16 { result, .. } - | Op::I32And { result, .. } - | Op::I32AndImm16 { result, .. } - | Op::I32Or { result, .. } - | Op::I32OrImm16 { result, .. } - | Op::I32Nand { result, .. } - | Op::I32NandImm16 { result, .. } - | Op::I32Nor { result, .. } - | Op::I32NorImm16 { result, .. } - | Op::I32Eq { result, .. } - | Op::I32EqImm16 { result, .. } - | Op::I32Ne { result, .. } - | Op::I32NeImm16 { result, .. } - | Op::I32LtS { result, .. } - | Op::I32LtSImm16Lhs { result, .. } - | Op::I32LtSImm16Rhs { result, .. } - | Op::I32LtU { result, .. } - | Op::I32LtUImm16Lhs { result, .. } - | Op::I32LtUImm16Rhs { result, .. } - | Op::I32LeS { result, .. } - | Op::I32LeSImm16Lhs { result, .. } - | Op::I32LeSImm16Rhs { result, .. } - | Op::I32LeU { result, .. } - | Op::I32LeUImm16Lhs { result, .. } - | Op::I32LeUImm16Rhs { result, .. } - | Op::I64BitAnd { result, .. } - | Op::I64BitAndImm16 { result, .. } - | Op::I64BitOr { result, .. } - | Op::I64BitOrImm16 { result, .. } - | Op::I64BitXor { result, .. } - | Op::I64BitXorImm16 { result, .. } - | Op::I64And { result, .. } - | Op::I64AndImm16 { result, .. } - | Op::I64Or { result, .. } - | Op::I64OrImm16 { result, .. } - | Op::I64Nand { result, .. } - | Op::I64NandImm16 { result, .. } - | Op::I64Nor { result, .. } - | Op::I64NorImm16 { result, .. } - | Op::I64Eq { result, .. } - | Op::I64EqImm16 { result, .. } - | Op::I64Ne { result, .. } - | Op::I64NeImm16 { result, .. } - | Op::I64LtS { result, .. } - | Op::I64LtSImm16Lhs { result, .. } - | Op::I64LtSImm16Rhs { result, .. } - | Op::I64LtU { result, .. } - | Op::I64LtUImm16Lhs { result, .. } - | Op::I64LtUImm16Rhs { result, .. } - | Op::I64LeS { result, .. } - | Op::I64LeSImm16Lhs { result, .. } - | Op::I64LeSImm16Rhs { result, .. } - | Op::I64LeU { result, .. } - | Op::I64LeUImm16Lhs { result, .. } - | Op::I64LeUImm16Rhs { result, .. } - | Op::F32Eq { result, .. } - | Op::F32Ne { result, .. } - | Op::F32Lt { result, .. } - | Op::F32Le { result, .. } - | Op::F32NotLt { result, .. } - | Op::F32NotLe { result, .. } - | Op::F64Eq { result, .. } - | Op::F64Ne { result, .. } - | Op::F64Lt { result, .. } - | Op::F64Le { result, .. } - | Op::F64NotLt { result, .. } - | Op::F64NotLe { result, .. } => *result = new_result, - _ => return None, - }; - Some(copy) - } -} - pub trait NegateCmpInstr: Sized { /// Negates the compare (`cmp`) [`Op`]. fn negate_cmp_instr(&self) -> Option; @@ -215,83 +131,101 @@ impl NegateCmpInstr for Op { #[rustfmt::skip] let negated = match *self { // i32 - Op::I32Eq { result, lhs, rhs } => Op::i32_ne(result, lhs, rhs), - Op::I32Ne { result, lhs, rhs } => Op::i32_eq(result, lhs, rhs), - Op::I32LeS { result, lhs, rhs } => Op::i32_lt_s(result, rhs, lhs), - Op::I32LeU { result, lhs, rhs } => Op::i32_lt_u(result, rhs, lhs), - Op::I32LtS { result, lhs, rhs } => Op::i32_le_s(result, rhs, lhs), - Op::I32LtU { result, lhs, rhs } => Op::i32_le_u(result, rhs, lhs), - Op::I32EqImm16 { result, lhs, rhs } => Op::i32_ne_imm16(result, lhs, rhs), - Op::I32NeImm16 { result, lhs, rhs } => Op::i32_eq_imm16(result, lhs, rhs), - Op::I32LeSImm16Rhs { result, lhs, rhs } => Op::i32_lt_s_imm16_lhs(result, rhs, lhs), - Op::I32LeUImm16Rhs { result, lhs, rhs } => Op::i32_lt_u_imm16_lhs(result, rhs, lhs), - Op::I32LtSImm16Rhs { result, lhs, rhs } => Op::i32_le_s_imm16_lhs(result, rhs, lhs), - Op::I32LtUImm16Rhs { result, lhs, rhs } => Op::i32_le_u_imm16_lhs(result, rhs, lhs), - Op::I32LeSImm16Lhs { result, lhs, rhs } => Op::i32_lt_s_imm16_rhs(result, rhs, lhs), - Op::I32LeUImm16Lhs { result, lhs, rhs } => Op::i32_lt_u_imm16_rhs(result, rhs, lhs), - Op::I32LtSImm16Lhs { result, lhs, rhs } => Op::i32_le_s_imm16_rhs(result, rhs, lhs), - Op::I32LtUImm16Lhs { result, lhs, rhs } => Op::i32_le_u_imm16_rhs(result, rhs, lhs), - // i32 (and, or, xor) - Op::I32BitAnd { result, lhs, rhs } => Op::i32_nand(result, lhs, rhs), - Op::I32BitOr { result, lhs, rhs } => Op::i32_nor(result, lhs, rhs), - Op::I32BitXor { result, lhs, rhs } => Op::i32_eq(result, lhs, rhs), - Op::I32BitAndImm16 { result, lhs, rhs } => Op::i32_nand_imm16(result, lhs, rhs), - Op::I32BitOrImm16 { result, lhs, rhs } => Op::i32_nor_imm16(result, lhs, rhs), - Op::I32BitXorImm16 { result, lhs, rhs } => Op::i32_eq_imm16(result, lhs, rhs), - Op::I32And { result, lhs, rhs } => Op::i32_nand(result, lhs, rhs), - Op::I32Or { result, lhs, rhs } => Op::i32_nor(result, lhs, rhs), - Op::I32AndImm16 { result, lhs, rhs } => Op::i32_nand_imm16(result, lhs, rhs), - Op::I32OrImm16 { result, lhs, rhs } => Op::i32_nor_imm16(result, lhs, rhs), - Op::I32Nand { result, lhs, rhs } => Op::i32_and(result, lhs, rhs), - Op::I32Nor { result, lhs, rhs } => Op::i32_or(result, lhs, rhs), - Op::I32NandImm16 { result, lhs, rhs } => Op::i32_and_imm16(result, lhs, rhs), - Op::I32NorImm16 { result, lhs, rhs } => Op::i32_or_imm16(result, lhs, rhs), + | Op::I32Eq_Sss { result, lhs, rhs } => Op::i32_not_eq_sss(result, lhs, rhs), + | Op::I32Eq_Ssi { result, lhs, rhs } => Op::i32_not_eq_ssi(result, lhs, rhs), + | Op::I32And_Sss { result, lhs, rhs } + | Op::I32BitAnd_Sss { result, lhs, rhs } => Op::i32_not_and_sss(result, lhs, rhs), + | Op::I32And_Ssi { result, lhs, rhs } + | Op::I32BitAnd_Ssi { result, lhs, rhs } => Op::i32_not_and_ssi(result, lhs, rhs), + | Op::I32Or_Sss { result, lhs, rhs } + | Op::I32BitOr_Sss { result, lhs, rhs } => Op::i32_not_or_sss(result, lhs, rhs), + | Op::I32Or_Ssi { result, lhs, rhs } + | Op::I32BitOr_Ssi { result, lhs, rhs } => Op::i32_not_or_ssi(result, lhs, rhs), + | Op::I32NotEq_Sss { result, lhs, rhs } + | Op::I32BitXor_Sss { result, lhs, rhs } => Op::i32_eq_sss(result, lhs, rhs), + | Op::I32NotEq_Ssi { result, lhs, rhs } + | Op::I32BitXor_Ssi { result, lhs, rhs } => Op::i32_eq_ssi(result, lhs, rhs), + | Op::I32NotAnd_Sss { result, lhs, rhs } => Op::i32_and_sss(result, lhs, rhs), + | Op::I32NotAnd_Ssi { result, lhs, rhs } => Op::i32_and_ssi(result, lhs, rhs), + | Op::I32NotOr_Sss { result, lhs, rhs } => Op::i32_or_sss(result, lhs, rhs), + | Op::I32NotOr_Ssi { result, lhs, rhs } => Op::i32_or_ssi(result, lhs, rhs), + | Op::I32Lt_Sss { result, lhs, rhs } => Op::i32_le_sss(result, rhs, lhs), + | Op::I32Lt_Ssi { result, lhs, rhs } => Op::i32_le_sis(result, rhs, lhs), + | Op::I32Lt_Sis { result, lhs, rhs } => Op::i32_le_ssi(result, rhs, lhs), + | Op::U32Lt_Sss { result, lhs, rhs } => Op::u32_le_sss(result, rhs, lhs), + | Op::U32Lt_Ssi { result, lhs, rhs } => Op::u32_le_sis(result, rhs, lhs), + | Op::U32Lt_Sis { result, lhs, rhs } => Op::u32_le_ssi(result, rhs, lhs), + | Op::I32Le_Sss { result, lhs, rhs } => Op::i32_lt_sss(result, rhs, lhs), + | Op::I32Le_Ssi { result, lhs, rhs } => Op::i32_lt_sis(result, rhs, lhs), + | Op::I32Le_Sis { result, lhs, rhs } => Op::i32_lt_ssi(result, rhs, lhs), + | Op::U32Le_Sss { result, lhs, rhs } => Op::u32_lt_sss(result, rhs, lhs), + | Op::U32Le_Ssi { result, lhs, rhs } => Op::u32_lt_sis(result, rhs, lhs), + | Op::U32Le_Sis { result, lhs, rhs } => Op::u32_lt_ssi(result, rhs, lhs), // i64 - Op::I64Eq { result, lhs, rhs } => Op::i64_ne(result, lhs, rhs), - Op::I64Ne { result, lhs, rhs } => Op::i64_eq(result, lhs, rhs), - Op::I64LeS { result, lhs, rhs } => Op::i64_lt_s(result, rhs, lhs), - Op::I64LeU { result, lhs, rhs } => Op::i64_lt_u(result, rhs, lhs), - Op::I64LtS { result, lhs, rhs } => Op::i64_le_s(result, rhs, lhs), - Op::I64LtU { result, lhs, rhs } => Op::i64_le_u(result, rhs, lhs), - Op::I64EqImm16 { result, lhs, rhs } => Op::i64_ne_imm16(result, lhs, rhs), - Op::I64NeImm16 { result, lhs, rhs } => Op::i64_eq_imm16(result, lhs, rhs), - Op::I64LeSImm16Rhs { result, lhs, rhs } => Op::i64_lt_s_imm16_lhs(result, rhs, lhs), - Op::I64LeUImm16Rhs { result, lhs, rhs } => Op::i64_lt_u_imm16_lhs(result, rhs, lhs), - Op::I64LtSImm16Rhs { result, lhs, rhs } => Op::i64_le_s_imm16_lhs(result, rhs, lhs), - Op::I64LtUImm16Rhs { result, lhs, rhs } => Op::i64_le_u_imm16_lhs(result, rhs, lhs), - Op::I64LeSImm16Lhs { result, lhs, rhs } => Op::i64_lt_s_imm16_rhs(result, rhs, lhs), - Op::I64LeUImm16Lhs { result, lhs, rhs } => Op::i64_lt_u_imm16_rhs(result, rhs, lhs), - Op::I64LtSImm16Lhs { result, lhs, rhs } => Op::i64_le_s_imm16_rhs(result, rhs, lhs), - Op::I64LtUImm16Lhs { result, lhs, rhs } => Op::i64_le_u_imm16_rhs(result, rhs, lhs), - // i64 (and, or, xor) - Op::I64BitAnd { result, lhs, rhs } => Op::i64_nand(result, lhs, rhs), - Op::I64BitOr { result, lhs, rhs } => Op::i64_nor(result, lhs, rhs), - Op::I64BitXor { result, lhs, rhs } => Op::i64_eq(result, lhs, rhs), - Op::I64BitAndImm16 { result, lhs, rhs } => Op::i64_nand_imm16(result, lhs, rhs), - Op::I64BitOrImm16 { result, lhs, rhs } => Op::i64_nor_imm16(result, lhs, rhs), - Op::I64BitXorImm16 { result, lhs, rhs } => Op::i64_eq_imm16(result, lhs, rhs), - Op::I64And { result, lhs, rhs } => Op::i64_nand(result, lhs, rhs), - Op::I64Or { result, lhs, rhs } => Op::i64_nor(result, lhs, rhs), - Op::I64AndImm16 { result, lhs, rhs } => Op::i64_nand_imm16(result, lhs, rhs), - Op::I64OrImm16 { result, lhs, rhs } => Op::i64_nor_imm16(result, lhs, rhs), - Op::I64Nand { result, lhs, rhs } => Op::i64_and(result, lhs, rhs), - Op::I64Nor { result, lhs, rhs } => Op::i64_or(result, lhs, rhs), - Op::I64NandImm16 { result, lhs, rhs } => Op::i64_and_imm16(result, lhs, rhs), - Op::I64NorImm16 { result, lhs, rhs } => Op::i64_or_imm16(result, lhs, rhs), + | Op::I64Eq_Sss { result, lhs, rhs } => Op::i64_not_eq_sss(result, lhs, rhs), + | Op::I64Eq_Ssi { result, lhs, rhs } => Op::i64_not_eq_ssi(result, lhs, rhs), + | Op::I64And_Sss { result, lhs, rhs } + | Op::I64BitAnd_Sss { result, lhs, rhs } => Op::i64_not_and_sss(result, lhs, rhs), + | Op::I64And_Ssi { result, lhs, rhs } + | Op::I64BitAnd_Ssi { result, lhs, rhs } => Op::i64_not_and_ssi(result, lhs, rhs), + | Op::I64Or_Sss { result, lhs, rhs } + | Op::I64BitOr_Sss { result, lhs, rhs } => Op::i64_not_or_sss(result, lhs, rhs), + | Op::I64Or_Ssi { result, lhs, rhs } + | Op::I64BitOr_Ssi { result, lhs, rhs } => Op::i64_not_or_ssi(result, lhs, rhs), + | Op::I64NotEq_Sss { result, lhs, rhs } + | Op::I64BitXor_Sss { result, lhs, rhs } => Op::i64_eq_sss(result, lhs, rhs), + | Op::I64NotEq_Ssi { result, lhs, rhs } + | Op::I64BitXor_Ssi { result, lhs, rhs } => Op::i64_eq_ssi(result, lhs, rhs), + | Op::I64NotAnd_Sss { result, lhs, rhs } => Op::i64_and_sss(result, lhs, rhs), + | Op::I64NotAnd_Ssi { result, lhs, rhs } => Op::i64_and_ssi(result, lhs, rhs), + | Op::I64NotOr_Sss { result, lhs, rhs } => Op::i64_or_sss(result, lhs, rhs), + | Op::I64NotOr_Ssi { result, lhs, rhs } => Op::i64_or_ssi(result, lhs, rhs), + | Op::I64Lt_Sss { result, lhs, rhs } => Op::i64_le_sss(result, rhs, lhs), + | Op::I64Lt_Ssi { result, lhs, rhs } => Op::i64_le_sis(result, rhs, lhs), + | Op::I64Lt_Sis { result, lhs, rhs } => Op::i64_le_ssi(result, rhs, lhs), + | Op::U64Lt_Sss { result, lhs, rhs } => Op::u64_le_sss(result, rhs, lhs), + | Op::U64Lt_Ssi { result, lhs, rhs } => Op::u64_le_sis(result, rhs, lhs), + | Op::U64Lt_Sis { result, lhs, rhs } => Op::u64_le_ssi(result, rhs, lhs), + | Op::I64Le_Sss { result, lhs, rhs } => Op::i64_lt_sss(result, rhs, lhs), + | Op::I64Le_Ssi { result, lhs, rhs } => Op::i64_lt_sis(result, rhs, lhs), + | Op::I64Le_Sis { result, lhs, rhs } => Op::i64_lt_ssi(result, rhs, lhs), + | Op::U64Le_Sss { result, lhs, rhs } => Op::u64_lt_sss(result, rhs, lhs), + | Op::U64Le_Ssi { result, lhs, rhs } => Op::u64_lt_sis(result, rhs, lhs), + | Op::U64Le_Sis { result, lhs, rhs } => Op::u64_lt_ssi(result, rhs, lhs), // f32 - Op::F32Eq { result, lhs, rhs } => Op::f32_ne(result, lhs, rhs), - Op::F32Ne { result, lhs, rhs } => Op::f32_eq(result, lhs, rhs), - Op::F32Le { result, lhs, rhs } => Op::f32_not_le(result, lhs, rhs), - Op::F32Lt { result, lhs, rhs } => Op::f32_not_lt(result, lhs, rhs), - Op::F32NotLe { result, lhs, rhs } => Op::f32_le(result, lhs, rhs), - Op::F32NotLt { result, lhs, rhs } => Op::f32_lt(result, lhs, rhs), + Op::F32Eq_Sss { result, lhs, rhs } => Op::f32_not_eq_sss(result, lhs, rhs), + Op::F32Eq_Ssi { result, lhs, rhs } => Op::f32_not_eq_ssi(result, lhs, rhs), + Op::F32Le_Sss { result, lhs, rhs } => Op::f32_not_le_sss(result, lhs, rhs), + Op::F32Le_Ssi { result, lhs, rhs } => Op::f32_not_le_ssi(result, lhs, rhs), + Op::F32Le_Sis { result, lhs, rhs } => Op::f32_not_le_sis(result, lhs, rhs), + Op::F32Lt_Sss { result, lhs, rhs } => Op::f32_not_lt_sss(result, lhs, rhs), + Op::F32Lt_Ssi { result, lhs, rhs } => Op::f32_not_lt_ssi(result, lhs, rhs), + Op::F32Lt_Sis { result, lhs, rhs } => Op::f32_not_lt_sis(result, lhs, rhs), + Op::F32NotEq_Sss { result, lhs, rhs } => Op::f32_eq_sss(result, lhs, rhs), + Op::F32NotEq_Ssi { result, lhs, rhs } => Op::f32_eq_ssi(result, lhs, rhs), + Op::F32NotLe_Sss { result, lhs, rhs } => Op::f32_le_sss(result, lhs, rhs), + Op::F32NotLe_Ssi { result, lhs, rhs } => Op::f32_le_ssi(result, lhs, rhs), + Op::F32NotLe_Sis { result, lhs, rhs } => Op::f32_le_sis(result, lhs, rhs), + Op::F32NotLt_Sss { result, lhs, rhs } => Op::f32_lt_sss(result, lhs, rhs), + Op::F32NotLt_Ssi { result, lhs, rhs } => Op::f32_lt_ssi(result, lhs, rhs), + Op::F32NotLt_Sis { result, lhs, rhs } => Op::f32_lt_sis(result, lhs, rhs), // f64 - Op::F64Eq { result, lhs, rhs } => Op::f64_ne(result, lhs, rhs), - Op::F64Ne { result, lhs, rhs } => Op::f64_eq(result, lhs, rhs), - Op::F64Le { result, lhs, rhs } => Op::f64_not_le(result, lhs, rhs), - Op::F64Lt { result, lhs, rhs } => Op::f64_not_lt(result, lhs, rhs), - Op::F64NotLe { result, lhs, rhs } => Op::f64_le(result, lhs, rhs), - Op::F64NotLt { result, lhs, rhs } => Op::f64_lt(result, lhs, rhs), + Op::F64Eq_Sss { result, lhs, rhs } => Op::f64_not_eq_sss(result, lhs, rhs), + Op::F64Eq_Ssi { result, lhs, rhs } => Op::f64_not_eq_ssi(result, lhs, rhs), + Op::F64Le_Sss { result, lhs, rhs } => Op::f64_not_le_sss(result, lhs, rhs), + Op::F64Le_Ssi { result, lhs, rhs } => Op::f64_not_le_ssi(result, lhs, rhs), + Op::F64Le_Sis { result, lhs, rhs } => Op::f64_not_le_sis(result, lhs, rhs), + Op::F64Lt_Sss { result, lhs, rhs } => Op::f64_not_lt_sss(result, lhs, rhs), + Op::F64Lt_Ssi { result, lhs, rhs } => Op::f64_not_lt_ssi(result, lhs, rhs), + Op::F64Lt_Sis { result, lhs, rhs } => Op::f64_not_lt_sis(result, lhs, rhs), + Op::F64NotEq_Sss { result, lhs, rhs } => Op::f64_eq_sss(result, lhs, rhs), + Op::F64NotEq_Ssi { result, lhs, rhs } => Op::f64_eq_ssi(result, lhs, rhs), + Op::F64NotLe_Sss { result, lhs, rhs } => Op::f64_le_sss(result, lhs, rhs), + Op::F64NotLe_Ssi { result, lhs, rhs } => Op::f64_le_ssi(result, lhs, rhs), + Op::F64NotLe_Sis { result, lhs, rhs } => Op::f64_le_sis(result, lhs, rhs), + Op::F64NotLt_Sss { result, lhs, rhs } => Op::f64_lt_sss(result, lhs, rhs), + Op::F64NotLt_Ssi { result, lhs, rhs } => Op::f64_lt_ssi(result, lhs, rhs), + Op::F64NotLt_Sis { result, lhs, rhs } => Op::f64_lt_sis(result, lhs, rhs), _ => return None, }; Some(negated) @@ -311,80 +245,104 @@ impl LogicalizeCmpInstr for Op { #[rustfmt::skip] let logicalized = match *self { // Bitwise -> Logical: i32 - Op::I32BitAnd { result, lhs, rhs } => Op::i32_and(result, lhs, rhs), - Op::I32BitOr { result, lhs, rhs } => Op::i32_or(result, lhs, rhs), - Op::I32BitXor { result, lhs, rhs } => Op::i32_ne(result, lhs, rhs), - Op::I32BitAndImm16 { result, lhs, rhs } => Op::i32_and_imm16(result, lhs, rhs), - Op::I32BitOrImm16 { result, lhs, rhs } => Op::i32_or_imm16(result, lhs, rhs), - Op::I32BitXorImm16 { result, lhs, rhs } => Op::i32_ne_imm16(result, lhs, rhs), + | Op::I32BitAnd_Sss { result, lhs, rhs } => Op::i32_and_sss(result, lhs, rhs), + | Op::I32BitOr_Sss { result, lhs, rhs } => Op::i32_or_sss(result, lhs, rhs), + | Op::I32BitXor_Sss { result, lhs, rhs } => Op::i32_not_eq_sss(result, lhs, rhs), + | Op::I32BitAnd_Ssi { result, lhs, rhs } => Op::i32_and_ssi(result, lhs, rhs), + | Op::I32BitOr_Ssi { result, lhs, rhs } => Op::i32_or_ssi(result, lhs, rhs), + | Op::I32BitXor_Ssi { result, lhs, rhs } => Op::i32_not_eq_ssi(result, lhs, rhs), // Bitwise -> Logical: i64 - Op::I64BitAnd { result, lhs, rhs } => Op::i64_and(result, lhs, rhs), - Op::I64BitOr { result, lhs, rhs } => Op::i64_or(result, lhs, rhs), - Op::I64BitXor { result, lhs, rhs } => Op::i64_ne(result, lhs, rhs), - Op::I64BitAndImm16 { result, lhs, rhs } => Op::i64_and_imm16(result, lhs, rhs), - Op::I64BitOrImm16 { result, lhs, rhs } => Op::i64_or_imm16(result, lhs, rhs), - Op::I64BitXorImm16 { result, lhs, rhs } => Op::i64_ne_imm16(result, lhs, rhs), + | Op::I64BitAnd_Sss { result, lhs, rhs } => Op::i64_and_sss(result, lhs, rhs), + | Op::I64BitOr_Sss { result, lhs, rhs } => Op::i64_or_sss(result, lhs, rhs), + | Op::I64BitXor_Sss { result, lhs, rhs } => Op::i64_not_eq_sss(result, lhs, rhs), + | Op::I64BitAnd_Ssi { result, lhs, rhs } => Op::i64_and_ssi(result, lhs, rhs), + | Op::I64BitOr_Ssi { result, lhs, rhs } => Op::i64_or_ssi(result, lhs, rhs), + | Op::I64BitXor_Ssi { result, lhs, rhs } => Op::i64_not_eq_ssi(result, lhs, rhs), // Logical -> Logical - Op::I32Eq { .. } | - Op::I32Ne { .. } | - Op::I32LeS { .. } | - Op::I32LeU { .. } | - Op::I32LtS { .. } | - Op::I32LtU { .. } | - Op::I32EqImm16 { .. } | - Op::I32NeImm16 { .. } | - Op::I32LeSImm16Rhs { .. } | - Op::I32LeUImm16Rhs { .. } | - Op::I32LtSImm16Rhs { .. } | - Op::I32LtUImm16Rhs { .. } | - Op::I32LeSImm16Lhs { .. } | - Op::I32LeUImm16Lhs { .. } | - Op::I32LtSImm16Lhs { .. } | - Op::I32LtUImm16Lhs { .. } | - Op::I32And { .. } | - Op::I32Or { .. } | - Op::I32AndImm16 { .. } | - Op::I32OrImm16 { .. } | - Op::I32Nand { .. } | - Op::I32Nor { .. } | - Op::I32NandImm16 { .. } | - Op::I32NorImm16 { .. } | - Op::I64Eq { .. } | - Op::I64Ne { .. } | - Op::I64LeS { .. } | - Op::I64LeU { .. } | - Op::I64LtS { .. } | - Op::I64LtU { .. } | - Op::I64EqImm16 { .. } | - Op::I64NeImm16 { .. } | - Op::I64LeSImm16Rhs { .. } | - Op::I64LeUImm16Rhs { .. } | - Op::I64LtSImm16Rhs { .. } | - Op::I64LtUImm16Rhs { .. } | - Op::I64LeSImm16Lhs { .. } | - Op::I64LeUImm16Lhs { .. } | - Op::I64LtSImm16Lhs { .. } | - Op::I64LtUImm16Lhs { .. } | - Op::I64And { .. } | - Op::I64Or { .. } | - Op::I64AndImm16 { .. } | - Op::I64OrImm16 { .. } | - Op::I64Nand { .. } | - Op::I64Nor { .. } | - Op::I64NandImm16 { .. } | - Op::I64NorImm16 { .. } | - Op::F32Eq { .. } | - Op::F32Ne { .. } | - Op::F32Lt { .. } | - Op::F32Le { .. } | - Op::F32NotLt { .. } | - Op::F32NotLe { .. } | - Op::F64Eq { .. } | - Op::F64Ne { .. } | - Op::F64Lt { .. } | - Op::F64Le { .. } | - Op::F64NotLt { .. } | - Op::F64NotLe { .. } => *self, + // i32 + | Op::I32Eq_Sss { .. } + | Op::I32Eq_Ssi { .. } + | Op::I32And_Sss { .. } + | Op::I32And_Ssi { .. } + | Op::I32Or_Sss { .. } + | Op::I32Or_Ssi { .. } + | Op::I32NotEq_Sss { .. } + | Op::I32NotEq_Ssi { .. } + | Op::I32NotAnd_Sss { .. } + | Op::I32NotAnd_Ssi { .. } + | Op::I32NotOr_Sss { .. } + | Op::I32NotOr_Ssi { .. } + | Op::I32Lt_Sss { .. } + | Op::I32Lt_Ssi { .. } + | Op::I32Lt_Sis { .. } + | Op::U32Lt_Sss { .. } + | Op::U32Lt_Ssi { .. } + | Op::U32Lt_Sis { .. } + | Op::I32Le_Sss { .. } + | Op::I32Le_Ssi { .. } + | Op::I32Le_Sis { .. } + | Op::U32Le_Sss { .. } + | Op::U32Le_Ssi { .. } + | Op::U32Le_Sis { .. } + // i64 + | Op::I64Eq_Sss { .. } + | Op::I64Eq_Ssi { .. } + | Op::I64And_Sss { .. } + | Op::I64And_Ssi { .. } + | Op::I64Or_Sss { .. } + | Op::I64Or_Ssi { .. } + | Op::I64NotEq_Sss { .. } + | Op::I64NotEq_Ssi { .. } + | Op::I64NotAnd_Sss { .. } + | Op::I64NotAnd_Ssi { .. } + | Op::I64NotOr_Sss { .. } + | Op::I64NotOr_Ssi { .. } + | Op::I64Lt_Sss { .. } + | Op::I64Lt_Ssi { .. } + | Op::I64Lt_Sis { .. } + | Op::U64Lt_Sss { .. } + | Op::U64Lt_Ssi { .. } + | Op::U64Lt_Sis { .. } + | Op::I64Le_Sss { .. } + | Op::I64Le_Ssi { .. } + | Op::I64Le_Sis { .. } + | Op::U64Le_Sss { .. } + | Op::U64Le_Ssi { .. } + | Op::U64Le_Sis { .. } + // f32 + | Op::F32Eq_Sss { .. } + | Op::F32Eq_Ssi { .. } + | Op::F32Le_Sss { .. } + | Op::F32Le_Ssi { .. } + | Op::F32Le_Sis { .. } + | Op::F32Lt_Sss { .. } + | Op::F32Lt_Ssi { .. } + | Op::F32Lt_Sis { .. } + | Op::F32NotEq_Sss { .. } + | Op::F32NotEq_Ssi { .. } + | Op::F32NotLe_Sss { .. } + | Op::F32NotLe_Ssi { .. } + | Op::F32NotLe_Sis { .. } + | Op::F32NotLt_Sss { .. } + | Op::F32NotLt_Ssi { .. } + | Op::F32NotLt_Sis { .. } + // f64 + | Op::F64Eq_Sss { .. } + | Op::F64Eq_Ssi { .. } + | Op::F64Le_Sss { .. } + | Op::F64Le_Ssi { .. } + | Op::F64Le_Sis { .. } + | Op::F64Lt_Sss { .. } + | Op::F64Lt_Ssi { .. } + | Op::F64Lt_Sis { .. } + | Op::F64NotEq_Sss { .. } + | Op::F64NotEq_Ssi { .. } + | Op::F64NotLe_Sss { .. } + | Op::F64NotLe_Ssi { .. } + | Op::F64NotLe_Sis { .. } + | Op::F64NotLt_Sss { .. } + | Op::F64NotLt_Ssi { .. } + | Op::F64NotLt_Sis { .. } => *self, _ => return None, }; Some(logicalized) @@ -394,6 +352,8 @@ impl LogicalizeCmpInstr for Op { pub trait TryIntoCmpSelectInstr: Sized { fn try_into_cmp_select_instr( &self, + val_true: Slot, + val_false: Slot, get_result: impl FnOnce() -> Result, ) -> Result; } @@ -401,520 +361,360 @@ pub trait TryIntoCmpSelectInstr: Sized { /// The outcome of `cmp`+`select` op-code fusion. pub enum CmpSelectFusion { /// The `cmp`+`select` fusion was applied and may require swapping operands. - Applied { fused: Op, swap_operands: bool }, + Applied(Op), /// The `cmp`+`select` fusion was _not_ applied. Unapplied, } -/// Returns `true` if a `cmp`+`select` fused instruction required to swap its operands. -#[rustfmt::skip] -fn cmp_select_swap_operands(instr: &Op) -> bool { - matches!(instr, - | Op::I32Ne { .. } - | Op::I32NeImm16 { .. } - | Op::I32LeSImm16Lhs { .. } - | Op::I32LeUImm16Lhs { .. } - | Op::I32LtSImm16Lhs { .. } - | Op::I32LtUImm16Lhs { .. } - | Op::I32BitXor { .. } - | Op::I32BitXorImm16 { .. } - | Op::I64BitXor { .. } - | Op::I64BitXorImm16 { .. } - | Op::I32Nand { .. } - | Op::I32Nor { .. } - | Op::I32NandImm16 { .. } - | Op::I32NorImm16 { .. } - | Op::I64Ne { .. } - | Op::I64NeImm16 { .. } - | Op::I64LeSImm16Lhs { .. } - | Op::I64LeUImm16Lhs { .. } - | Op::I64LtSImm16Lhs { .. } - | Op::I64LtUImm16Lhs { .. } - | Op::I64Nand { .. } - | Op::I64Nor { .. } - | Op::I64NandImm16 { .. } - | Op::I64NorImm16 { .. } - | Op::F32Ne { .. } - | Op::F64Ne { .. } - | Op::F32NotLt { .. } - | Op::F32NotLe { .. } - | Op::F64NotLt { .. } - | Op::F64NotLe { .. } - ) -} - impl TryIntoCmpSelectInstr for Op { fn try_into_cmp_select_instr( &self, + val_true: Slot, + val_false: Slot, get_result: impl FnOnce() -> Result, ) -> Result { if !self.is_compare_instr() { return Ok(CmpSelectFusion::Unapplied); } - let swap_operands = cmp_select_swap_operands(self); let result = get_result()?; #[rustfmt::skip] let fused = match *self { // i32 - Op::I32Eq { lhs, rhs, .. } => Op::select_i32_eq(result, lhs, rhs), - Op::I32Ne { lhs, rhs, .. } => Op::select_i32_eq(result, lhs, rhs), - Op::I32LeS { lhs, rhs, .. } => Op::select_i32_le_s(result, lhs, rhs), - Op::I32LeU { lhs, rhs, .. } => Op::select_i32_le_u(result, lhs, rhs), - Op::I32LtS { lhs, rhs, .. } => Op::select_i32_lt_s(result, lhs, rhs), - Op::I32LtU { lhs, rhs, .. } => Op::select_i32_lt_u(result, lhs, rhs), - Op::I32EqImm16 { lhs, rhs, .. } => Op::select_i32_eq_imm16(result, lhs, rhs), - Op::I32NeImm16 { lhs, rhs, .. } => Op::select_i32_eq_imm16(result, lhs, rhs), - Op::I32LeSImm16Lhs { lhs, rhs, .. } => Op::select_i32_lt_s_imm16_rhs(result, rhs, lhs), - Op::I32LeUImm16Lhs { lhs, rhs, .. } => Op::select_i32_lt_u_imm16_rhs(result, rhs, lhs), - Op::I32LtSImm16Lhs { lhs, rhs, .. } => Op::select_i32_le_s_imm16_rhs(result, rhs, lhs), - Op::I32LtUImm16Lhs { lhs, rhs, .. } => Op::select_i32_le_u_imm16_rhs(result, rhs, lhs), - Op::I32LeSImm16Rhs { lhs, rhs, .. } => Op::select_i32_le_s_imm16_rhs(result, lhs, rhs), - Op::I32LeUImm16Rhs { lhs, rhs, .. } => Op::select_i32_le_u_imm16_rhs(result, lhs, rhs), - Op::I32LtSImm16Rhs { lhs, rhs, .. } => Op::select_i32_lt_s_imm16_rhs(result, lhs, rhs), - Op::I32LtUImm16Rhs { lhs, rhs, .. } => Op::select_i32_lt_u_imm16_rhs(result, lhs, rhs), - // i32 (and, or, xor) - Op::I32BitAnd { lhs, rhs, .. } => Op::select_i32_and(result, lhs, rhs), - Op::I32BitOr { lhs, rhs, .. } => Op::select_i32_or(result, lhs, rhs), - Op::I32BitXor { lhs, rhs, .. } => Op::select_i32_eq(result, lhs, rhs), - Op::I32And { lhs, rhs, .. } => Op::select_i32_and(result, lhs, rhs), - Op::I32Or { lhs, rhs, .. } => Op::select_i32_or(result, lhs, rhs), - Op::I32Nand { lhs, rhs, .. } => Op::select_i32_and(result, lhs, rhs), - Op::I32Nor { lhs, rhs, .. } => Op::select_i32_or(result, lhs, rhs), - Op::I32BitAndImm16 { lhs, rhs, .. } => Op::select_i32_and_imm16(result, lhs, rhs), - Op::I32BitOrImm16 { lhs, rhs, .. } => Op::select_i32_or_imm16(result, lhs, rhs), - Op::I32BitXorImm16 { lhs, rhs, .. } => Op::select_i32_eq_imm16(result, lhs, rhs), - Op::I32AndImm16 { lhs, rhs, .. } => Op::select_i32_and_imm16(result, lhs, rhs), - Op::I32OrImm16 { lhs, rhs, .. } => Op::select_i32_or_imm16(result, lhs, rhs), - Op::I32NandImm16 { lhs, rhs, .. } => Op::select_i32_and_imm16(result, lhs, rhs), - Op::I32NorImm16 { lhs, rhs, .. } => Op::select_i32_or_imm16(result, lhs, rhs), + | Op::I32And_Sss { lhs, rhs, .. } + | Op::I32BitAnd_Sss { lhs, rhs, .. } => Op::select_i32_and_sss(result, val_true, val_false, lhs, rhs), + | Op::I32And_Ssi { lhs, rhs, .. } + | Op::I32BitAnd_Ssi { lhs, rhs, .. } => Op::select_i32_and_ssi(result, val_true, val_false, lhs, rhs), + | Op::I32Or_Sss { lhs, rhs, .. } + | Op::I32BitOr_Sss { lhs, rhs, .. } => Op::select_i32_or_sss(result, val_true, val_false, lhs, rhs), + | Op::I32Or_Ssi { lhs, rhs, .. } + | Op::I32BitOr_Ssi { lhs, rhs, .. } => Op::select_i32_or_ssi(result, val_true, val_false, lhs, rhs), + | Op::I32NotEq_Sss { lhs, rhs, .. } + | Op::I32BitXor_Sss { lhs, rhs, .. } => Op::select_i32_eq_sss(result, val_false, val_true, lhs, rhs), + | Op::I32NotEq_Ssi { lhs, rhs, .. } + | Op::I32BitXor_Ssi { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, val_false, val_true, lhs, rhs), + | Op::I32Eq_Sss { lhs, rhs, .. } => Op::select_i32_eq_sss(result, val_true, val_false, lhs, rhs), + | Op::I32Eq_Ssi { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, val_true, val_false, lhs, rhs), + | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::select_i32_and_sss(result, val_false, val_true, lhs, rhs), + | Op::I32NotAnd_Ssi { lhs, rhs, .. } => Op::select_i32_and_ssi(result, val_false, val_true, lhs, rhs), + | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::select_i32_or_sss(result, val_false, val_true, lhs, rhs), + | Op::I32NotOr_Ssi { lhs, rhs, .. } => Op::select_i32_or_ssi(result, val_false, val_true, lhs, rhs), + | Op::I32Lt_Sss { lhs, rhs, .. } => Op::select_i32_lt_sss(result, val_true, val_false, lhs, rhs), + | Op::I32Lt_Ssi { lhs, rhs, .. } => Op::select_i32_lt_ssi(result, val_true, val_false, lhs, rhs), + | Op::I32Lt_Sis { lhs, rhs, .. } => Op::select_i32_le_ssi(result, val_false, val_true, rhs, lhs), + | Op::U32Lt_Sss { lhs, rhs, .. } => Op::select_u32_lt_sss(result, val_true, val_false, lhs, rhs), + | Op::U32Lt_Ssi { lhs, rhs, .. } => Op::select_u32_lt_ssi(result, val_true, val_false, lhs, rhs), + | Op::U32Lt_Sis { lhs, rhs, .. } => Op::select_u32_le_ssi(result, val_false, val_true, rhs, lhs), + | Op::I32Le_Sss { lhs, rhs, .. } => Op::select_i32_le_sss(result, val_true, val_false, lhs, rhs), + | Op::I32Le_Ssi { lhs, rhs, .. } => Op::select_i32_le_ssi(result, val_true, val_false, lhs, rhs), + | Op::I32Le_Sis { lhs, rhs, .. } => Op::select_i32_lt_ssi(result, val_false, val_true, rhs, lhs), + | Op::U32Le_Sss { lhs, rhs, .. } => Op::select_u32_le_sss(result, val_true, val_false, lhs, rhs), + | Op::U32Le_Ssi { lhs, rhs, .. } => Op::select_u32_le_ssi(result, val_true, val_false, lhs, rhs), + | Op::U32Le_Sis { lhs, rhs, .. } => Op::select_u32_lt_ssi(result, val_false, val_true, rhs, lhs), // i64 - Op::I64Eq { lhs, rhs, .. } => Op::select_i64_eq(result, lhs, rhs), - Op::I64Ne { lhs, rhs, .. } => Op::select_i64_eq(result, lhs, rhs), - Op::I64LeS { lhs, rhs, .. } => Op::select_i64_le_s(result, lhs, rhs), - Op::I64LeU { lhs, rhs, .. } => Op::select_i64_le_u(result, lhs, rhs), - Op::I64LtS { lhs, rhs, .. } => Op::select_i64_lt_s(result, lhs, rhs), - Op::I64LtU { lhs, rhs, .. } => Op::select_i64_lt_u(result, lhs, rhs), - Op::I64EqImm16 { lhs, rhs, .. } => Op::select_i64_eq_imm16(result, lhs, rhs), - Op::I64NeImm16 { lhs, rhs, .. } => Op::select_i64_eq_imm16(result, lhs, rhs), - Op::I64LeSImm16Lhs { lhs, rhs, .. } => Op::select_i64_lt_s_imm16_rhs(result, rhs, lhs), - Op::I64LeUImm16Lhs { lhs, rhs, .. } => Op::select_i64_lt_u_imm16_rhs(result, rhs, lhs), - Op::I64LtSImm16Lhs { lhs, rhs, .. } => Op::select_i64_le_s_imm16_rhs(result, rhs, lhs), - Op::I64LtUImm16Lhs { lhs, rhs, .. } => Op::select_i64_le_u_imm16_rhs(result, rhs, lhs), - Op::I64LeSImm16Rhs { lhs, rhs, .. } => Op::select_i64_le_s_imm16_rhs(result, lhs, rhs), - Op::I64LeUImm16Rhs { lhs, rhs, .. } => Op::select_i64_le_u_imm16_rhs(result, lhs, rhs), - Op::I64LtSImm16Rhs { lhs, rhs, .. } => Op::select_i64_lt_s_imm16_rhs(result, lhs, rhs), - Op::I64LtUImm16Rhs { lhs, rhs, .. } => Op::select_i64_lt_u_imm16_rhs(result, lhs, rhs), - // i64 (and, or, xor) - Op::I64BitAnd { lhs, rhs, .. } => Op::select_i64_and(result, lhs, rhs), - Op::I64BitOr { lhs, rhs, .. } => Op::select_i64_or(result, lhs, rhs), - Op::I64BitXor { lhs, rhs, .. } => Op::select_i64_eq(result, lhs, rhs), - Op::I64And { lhs, rhs, .. } => Op::select_i64_and(result, lhs, rhs), - Op::I64Or { lhs, rhs, .. } => Op::select_i64_or(result, lhs, rhs), - Op::I64Nand { lhs, rhs, .. } => Op::select_i64_and(result, lhs, rhs), - Op::I64Nor { lhs, rhs, .. } => Op::select_i64_or(result, lhs, rhs), - Op::I64BitAndImm16 { lhs, rhs, .. } => Op::select_i64_and_imm16(result, lhs, rhs), - Op::I64BitOrImm16 { lhs, rhs, .. } => Op::select_i64_or_imm16(result, lhs, rhs), - Op::I64BitXorImm16 { lhs, rhs, .. } => Op::select_i64_eq_imm16(result, lhs, rhs), - Op::I64AndImm16 { lhs, rhs, .. } => Op::select_i64_and_imm16(result, lhs, rhs), - Op::I64OrImm16 { lhs, rhs, .. } => Op::select_i64_or_imm16(result, lhs, rhs), - Op::I64NandImm16 { lhs, rhs, .. } => Op::select_i64_and_imm16(result, lhs, rhs), - Op::I64NorImm16 { lhs, rhs, .. } => Op::select_i64_or_imm16(result, lhs, rhs), + | Op::I64And_Sss { lhs, rhs, .. } + | Op::I64BitAnd_Sss { lhs, rhs, .. } => Op::select_i64_and_sss(result, val_true, val_false, lhs, rhs), + | Op::I64And_Ssi { lhs, rhs, .. } + | Op::I64BitAnd_Ssi { lhs, rhs, .. } => Op::select_i64_and_ssi(result, val_true, val_false, lhs, rhs), + | Op::I64Or_Sss { lhs, rhs, .. } + | Op::I64BitOr_Sss { lhs, rhs, .. } => Op::select_i64_or_sss(result, val_true, val_false, lhs, rhs), + | Op::I64Or_Ssi { lhs, rhs, .. } + | Op::I64BitOr_Ssi { lhs, rhs, .. } => Op::select_i64_or_ssi(result, val_true, val_false, lhs, rhs), + | Op::I64NotEq_Sss { lhs, rhs, .. } + | Op::I64BitXor_Sss { lhs, rhs, .. } => Op::select_i64_eq_sss(result, val_false, val_true, lhs, rhs), + | Op::I64NotEq_Ssi { lhs, rhs, .. } + | Op::I64BitXor_Ssi { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, val_false, val_true, lhs, rhs), + | Op::I64Eq_Sss { lhs, rhs, .. } => Op::select_i64_eq_sss(result, val_true, val_false, lhs, rhs), + | Op::I64Eq_Ssi { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, val_true, val_false, lhs, rhs), + | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::select_i64_and_sss(result, val_false, val_true, lhs, rhs), + | Op::I64NotAnd_Ssi { lhs, rhs, .. } => Op::select_i64_and_ssi(result, val_false, val_true, lhs, rhs), + | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::select_i64_or_sss(result, val_false, val_true, lhs, rhs), + | Op::I64NotOr_Ssi { lhs, rhs, .. } => Op::select_i64_or_ssi(result, val_false, val_true, lhs, rhs), + | Op::I64Lt_Sss { lhs, rhs, .. } => Op::select_i64_lt_sss(result, val_true, val_false, lhs, rhs), + | Op::I64Lt_Ssi { lhs, rhs, .. } => Op::select_i64_lt_ssi(result, val_true, val_false, lhs, rhs), + | Op::I64Lt_Sis { lhs, rhs, .. } => Op::select_i64_le_ssi(result, val_false, val_true, rhs, lhs), + | Op::U64Lt_Sss { lhs, rhs, .. } => Op::select_u64_lt_sss(result, val_true, val_false, lhs, rhs), + | Op::U64Lt_Ssi { lhs, rhs, .. } => Op::select_u64_lt_ssi(result, val_true, val_false, lhs, rhs), + | Op::U64Lt_Sis { lhs, rhs, .. } => Op::select_u64_le_ssi(result, val_false, val_true, rhs, lhs), + | Op::I64Le_Sss { lhs, rhs, .. } => Op::select_i64_le_sss(result, val_true, val_false, lhs, rhs), + | Op::I64Le_Ssi { lhs, rhs, .. } => Op::select_i64_le_ssi(result, val_true, val_false, lhs, rhs), + | Op::I64Le_Sis { lhs, rhs, .. } => Op::select_i64_lt_ssi(result, val_false, val_true, rhs, lhs), + | Op::U64Le_Sss { lhs, rhs, .. } => Op::select_u64_le_sss(result, val_true, val_false, lhs, rhs), + | Op::U64Le_Ssi { lhs, rhs, .. } => Op::select_u64_le_ssi(result, val_true, val_false, lhs, rhs), + | Op::U64Le_Sis { lhs, rhs, .. } => Op::select_u64_lt_ssi(result, val_false, val_true, rhs, lhs), // f32 - Op::F32Eq { lhs, rhs, .. } => Op::select_f32_eq(result, lhs, rhs), - Op::F32Ne { lhs, rhs, .. } => Op::select_f32_eq(result, lhs, rhs), - Op::F32Lt { lhs, rhs, .. } => Op::select_f32_lt(result, lhs, rhs), - Op::F32Le { lhs, rhs, .. } => Op::select_f32_le(result, lhs, rhs), - Op::F32NotLt { lhs, rhs, .. } => Op::select_f32_lt(result, lhs, rhs), - Op::F32NotLe { lhs, rhs, .. } => Op::select_f32_le(result, lhs, rhs), + | Op::F32Eq_Sss { lhs, rhs, .. } => Op::select_f32_eq_sss(result, val_true, val_false, lhs, rhs), + | Op::F32Eq_Ssi { lhs, rhs, .. } => Op::select_f32_eq_ssi(result, val_true, val_false, lhs, rhs), + | Op::F32Lt_Sss { lhs, rhs, .. } => Op::select_f32_lt_sss(result, val_true, val_false, lhs, rhs), + | Op::F32Lt_Ssi { lhs, rhs, .. } => Op::select_f32_lt_ssi(result, val_true, val_false, lhs, rhs), + | Op::F32Lt_Sis { lhs, rhs, .. } => Op::select_f32_lt_sis(result, val_true, val_false, lhs, rhs), + | Op::F32Le_Sss { lhs, rhs, .. } => Op::select_f32_le_sss(result, val_true, val_false, lhs, rhs), + | Op::F32Le_Ssi { lhs, rhs, .. } => Op::select_f32_le_ssi(result, val_true, val_false, lhs, rhs), + | Op::F32Le_Sis { lhs, rhs, .. } => Op::select_f32_le_sis(result, val_true, val_false, lhs, rhs), + | Op::F32NotEq_Sss { lhs, rhs, .. } => Op::select_f32_eq_sss(result, val_false, val_true, lhs, rhs), + | Op::F32NotEq_Ssi { lhs, rhs, .. } => Op::select_f32_eq_ssi(result, val_false, val_true, lhs, rhs), + | Op::F32NotLt_Sss { lhs, rhs, .. } => Op::select_f32_lt_sss(result, val_false, val_true, lhs, rhs), + | Op::F32NotLt_Ssi { lhs, rhs, .. } => Op::select_f32_lt_ssi(result, val_false, val_true, lhs, rhs), + | Op::F32NotLt_Sis { lhs, rhs, .. } => Op::select_f32_lt_sis(result, val_false, val_true, lhs, rhs), + | Op::F32NotLe_Sss { lhs, rhs, .. } => Op::select_f32_le_sss(result, val_false, val_true, lhs, rhs), + | Op::F32NotLe_Ssi { lhs, rhs, .. } => Op::select_f32_le_ssi(result, val_false, val_true, lhs, rhs), + | Op::F32NotLe_Sis { lhs, rhs, .. } => Op::select_f32_le_sis(result, val_false, val_true, lhs, rhs), // f64 - Op::F64Eq { lhs, rhs, .. } => Op::select_f64_eq(result, lhs, rhs), - Op::F64Ne { lhs, rhs, .. } => Op::select_f64_eq(result, lhs, rhs), - Op::F64Lt { lhs, rhs, .. } => Op::select_f64_lt(result, lhs, rhs), - Op::F64Le { lhs, rhs, .. } => Op::select_f64_le(result, lhs, rhs), - Op::F64NotLt { lhs, rhs, .. } => Op::select_f64_lt(result, lhs, rhs), - Op::F64NotLe { lhs, rhs, .. } => Op::select_f64_le(result, lhs, rhs), + | Op::F64Eq_Sss { lhs, rhs, .. } => Op::select_f64_eq_sss(result, val_true, val_false, lhs, rhs), + | Op::F64Eq_Ssi { lhs, rhs, .. } => Op::select_f64_eq_ssi(result, val_true, val_false, lhs, rhs), + | Op::F64Lt_Sss { lhs, rhs, .. } => Op::select_f64_lt_sss(result, val_true, val_false, lhs, rhs), + | Op::F64Lt_Ssi { lhs, rhs, .. } => Op::select_f64_lt_ssi(result, val_true, val_false, lhs, rhs), + | Op::F64Lt_Sis { lhs, rhs, .. } => Op::select_f64_lt_sis(result, val_true, val_false, lhs, rhs), + | Op::F64Le_Sss { lhs, rhs, .. } => Op::select_f64_le_sss(result, val_true, val_false, lhs, rhs), + | Op::F64Le_Ssi { lhs, rhs, .. } => Op::select_f64_le_ssi(result, val_true, val_false, lhs, rhs), + | Op::F64Le_Sis { lhs, rhs, .. } => Op::select_f64_le_sis(result, val_true, val_false, lhs, rhs), + | Op::F64NotEq_Sss { lhs, rhs, .. } => Op::select_f64_eq_sss(result, val_false, val_true, lhs, rhs), + | Op::F64NotEq_Ssi { lhs, rhs, .. } => Op::select_f64_eq_ssi(result, val_false, val_true, lhs, rhs), + | Op::F64NotLt_Sss { lhs, rhs, .. } => Op::select_f64_lt_sss(result, val_false, val_true, lhs, rhs), + | Op::F64NotLt_Ssi { lhs, rhs, .. } => Op::select_f64_lt_ssi(result, val_false, val_true, lhs, rhs), + | Op::F64NotLt_Sis { lhs, rhs, .. } => Op::select_f64_lt_sis(result, val_false, val_true, lhs, rhs), + | Op::F64NotLe_Sss { lhs, rhs, .. } => Op::select_f64_le_sss(result, val_false, val_true, lhs, rhs), + | Op::F64NotLe_Ssi { lhs, rhs, .. } => Op::select_f64_le_ssi(result, val_false, val_true, lhs, rhs), + | Op::F64NotLe_Sis { lhs, rhs, .. } => Op::select_f64_le_sis(result, val_false, val_true, lhs, rhs), _ => unreachable!("expected to successfully fuse cmp+select"), }; - Ok(CmpSelectFusion::Applied { - fused, - swap_operands, - }) + Ok(CmpSelectFusion::Applied(fused)) } } pub trait TryIntoCmpBranchInstr: Sized { - fn try_into_cmp_branch_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error>; + fn try_into_cmp_branch_instr(&self, offset: BranchOffset) -> Option; } impl TryIntoCmpBranchInstr for Op { - fn try_into_cmp_branch_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error> { - let Ok(offset) = BranchOffset16::try_from(offset) else { - return self.try_into_cmp_branch_fallback_instr(offset, stack); - }; + fn try_into_cmp_branch_instr(&self, offset: BranchOffset) -> Option { #[rustfmt::skip] let cmp_branch_instr = match *self { // i32 - Op::I32Eq { lhs, rhs, .. } => Op::branch_i32_eq(lhs, rhs, offset), - Op::I32Ne { lhs, rhs, .. } => Op::branch_i32_ne(lhs, rhs, offset), - Op::I32LeS { lhs, rhs, .. } => Op::branch_i32_le_s(lhs, rhs, offset), - Op::I32LeU { lhs, rhs, .. } => Op::branch_i32_le_u(lhs, rhs, offset), - Op::I32LtS { lhs, rhs, .. } => Op::branch_i32_lt_s(lhs, rhs, offset), - Op::I32LtU { lhs, rhs, .. } => Op::branch_i32_lt_u(lhs, rhs, offset), - Op::I32EqImm16 { lhs, rhs, .. } => Op::branch_i32_eq_imm16(lhs, rhs, offset), - Op::I32NeImm16 { lhs, rhs, .. } => Op::branch_i32_ne_imm16(lhs, rhs, offset), - Op::I32LeSImm16Lhs { lhs, rhs, .. } => Op::branch_i32_le_s_imm16_lhs(lhs, rhs, offset), - Op::I32LeUImm16Lhs { lhs, rhs, .. } => Op::branch_i32_le_u_imm16_lhs(lhs, rhs, offset), - Op::I32LtSImm16Lhs { lhs, rhs, .. } => Op::branch_i32_lt_s_imm16_lhs(lhs, rhs, offset), - Op::I32LtUImm16Lhs { lhs, rhs, .. } => Op::branch_i32_lt_u_imm16_lhs(lhs, rhs, offset), - Op::I32LeSImm16Rhs { lhs, rhs, .. } => Op::branch_i32_le_s_imm16_rhs(lhs, rhs, offset), - Op::I32LeUImm16Rhs { lhs, rhs, .. } => Op::branch_i32_le_u_imm16_rhs(lhs, rhs, offset), - Op::I32LtSImm16Rhs { lhs, rhs, .. } => Op::branch_i32_lt_s_imm16_rhs(lhs, rhs, offset), - Op::I32LtUImm16Rhs { lhs, rhs, .. } => Op::branch_i32_lt_u_imm16_rhs(lhs, rhs, offset), - // i32 (and, or, xor) - Op::I32BitAnd { lhs, rhs, .. } => Op::branch_i32_and(lhs, rhs, offset), - Op::I32BitOr { lhs, rhs, .. } => Op::branch_i32_or(lhs, rhs, offset), - Op::I32BitXor { lhs, rhs, .. } => Op::branch_i32_ne(lhs, rhs, offset), - Op::I32And { lhs, rhs, .. } => Op::branch_i32_and(lhs, rhs, offset), - Op::I32Or { lhs, rhs, .. } => Op::branch_i32_or(lhs, rhs, offset), - Op::I32Nand { lhs, rhs, .. } => Op::branch_i32_nand(lhs, rhs, offset), - Op::I32Nor { lhs, rhs, .. } => Op::branch_i32_nor(lhs, rhs, offset), - Op::I32BitAndImm16 { lhs, rhs, .. } => Op::branch_i32_and_imm16(lhs, rhs, offset), - Op::I32BitOrImm16 { lhs, rhs, .. } => Op::branch_i32_or_imm16(lhs, rhs, offset), - Op::I32BitXorImm16 { lhs, rhs, .. } => Op::branch_i32_ne_imm16(lhs, rhs, offset), - Op::I32AndImm16 { lhs, rhs, .. } => Op::branch_i32_and_imm16(lhs, rhs, offset), - Op::I32OrImm16 { lhs, rhs, .. } => Op::branch_i32_or_imm16(lhs, rhs, offset), - Op::I32NandImm16 { lhs, rhs, .. } => Op::branch_i32_nand_imm16(lhs, rhs, offset), - Op::I32NorImm16 { lhs, rhs, .. } => Op::branch_i32_nor_imm16(lhs, rhs, offset), + | Op::I32Eq_Sss { lhs, rhs, .. } => Op::branch_i32_eq_ss(offset, lhs, rhs), + | Op::I32Eq_Ssi { lhs, rhs, .. } => Op::branch_i32_eq_si(offset, lhs, rhs), + | Op::I32And_Sss { lhs, rhs, .. } + | Op::I32BitAnd_Sss { lhs, rhs, .. } => Op::branch_i32_and_ss(offset, lhs, rhs), + | Op::I32And_Ssi { lhs, rhs, .. } + | Op::I32BitAnd_Ssi { lhs, rhs, .. } => Op::branch_i32_and_si(offset, lhs, rhs), + | Op::I32Or_Sss { lhs, rhs, .. } + | Op::I32BitOr_Sss { lhs, rhs, .. } => Op::branch_i32_or_ss(offset, lhs, rhs), + | Op::I32Or_Ssi { lhs, rhs, .. } + | Op::I32BitOr_Ssi { lhs, rhs, .. } => Op::branch_i32_or_si(offset, lhs, rhs), + | Op::I32NotEq_Sss { lhs, rhs, .. } + | Op::I32BitXor_Sss { lhs, rhs, .. } => Op::branch_i32_not_eq_ss(offset, lhs, rhs), + | Op::I32NotEq_Ssi { lhs, rhs, .. } + | Op::I32BitXor_Ssi { lhs, rhs, .. } => Op::branch_i32_not_eq_si(offset, lhs, rhs), + | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::branch_i32_not_and_ss(offset, lhs, rhs), + | Op::I32NotAnd_Ssi { lhs, rhs, .. } => Op::branch_i32_not_and_si(offset, lhs, rhs), + | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::branch_i32_not_or_ss(offset, lhs, rhs), + | Op::I32NotOr_Ssi { lhs, rhs, .. } => Op::branch_i32_not_or_si(offset, lhs, rhs), + | Op::I32Lt_Sss { lhs, rhs, .. } => Op::branch_i32_lt_ss(offset, lhs, rhs), + | Op::I32Lt_Ssi { lhs, rhs, .. } => Op::branch_i32_lt_si(offset, lhs, rhs), + | Op::I32Lt_Sis { lhs, rhs, .. } => Op::branch_i32_lt_is(offset, lhs, rhs), + | Op::U32Lt_Sss { lhs, rhs, .. } => Op::branch_u32_lt_ss(offset, lhs, rhs), + | Op::U32Lt_Ssi { lhs, rhs, .. } => Op::branch_u32_lt_si(offset, lhs, rhs), + | Op::U32Lt_Sis { lhs, rhs, .. } => Op::branch_u32_lt_is(offset, lhs, rhs), + | Op::I32Le_Sss { lhs, rhs, .. } => Op::branch_i32_le_ss(offset, lhs, rhs), + | Op::I32Le_Ssi { lhs, rhs, .. } => Op::branch_i32_le_si(offset, lhs, rhs), + | Op::I32Le_Sis { lhs, rhs, .. } => Op::branch_i32_le_is(offset, lhs, rhs), + | Op::U32Le_Sss { lhs, rhs, .. } => Op::branch_u32_le_ss(offset, lhs, rhs), + | Op::U32Le_Ssi { lhs, rhs, .. } => Op::branch_u32_le_si(offset, lhs, rhs), + | Op::U32Le_Sis { lhs, rhs, .. } => Op::branch_u32_le_is(offset, lhs, rhs), // i64 - Op::I64Eq { lhs, rhs, .. } => Op::branch_i64_eq(lhs, rhs, offset), - Op::I64Ne { lhs, rhs, .. } => Op::branch_i64_ne(lhs, rhs, offset), - Op::I64LeS { lhs, rhs, .. } => Op::branch_i64_le_s(lhs, rhs, offset), - Op::I64LeU { lhs, rhs, .. } => Op::branch_i64_le_u(lhs, rhs, offset), - Op::I64LtS { lhs, rhs, .. } => Op::branch_i64_lt_s(lhs, rhs, offset), - Op::I64LtU { lhs, rhs, .. } => Op::branch_i64_lt_u(lhs, rhs, offset), - Op::I64EqImm16 { lhs, rhs, .. } => Op::branch_i64_eq_imm16(lhs, rhs, offset), - Op::I64NeImm16 { lhs, rhs, .. } => Op::branch_i64_ne_imm16(lhs, rhs, offset), - Op::I64LeSImm16Lhs { lhs, rhs, .. } => Op::branch_i64_le_s_imm16_lhs(lhs, rhs, offset), - Op::I64LeUImm16Lhs { lhs, rhs, .. } => Op::branch_i64_le_u_imm16_lhs(lhs, rhs, offset), - Op::I64LtSImm16Lhs { lhs, rhs, .. } => Op::branch_i64_lt_s_imm16_lhs(lhs, rhs, offset), - Op::I64LtUImm16Lhs { lhs, rhs, .. } => Op::branch_i64_lt_u_imm16_lhs(lhs, rhs, offset), - Op::I64LeSImm16Rhs { lhs, rhs, .. } => Op::branch_i64_le_s_imm16_rhs(lhs, rhs, offset), - Op::I64LeUImm16Rhs { lhs, rhs, .. } => Op::branch_i64_le_u_imm16_rhs(lhs, rhs, offset), - Op::I64LtSImm16Rhs { lhs, rhs, .. } => Op::branch_i64_lt_s_imm16_rhs(lhs, rhs, offset), - Op::I64LtUImm16Rhs { lhs, rhs, .. } => Op::branch_i64_lt_u_imm16_rhs(lhs, rhs, offset), - // i64 (and, or, xor) - Op::I64BitAnd { lhs, rhs, .. } => Op::branch_i64_and(lhs, rhs, offset), - Op::I64BitOr { lhs, rhs, .. } => Op::branch_i64_or(lhs, rhs, offset), - Op::I64BitXor { lhs, rhs, .. } => Op::branch_i64_ne(lhs, rhs, offset), - Op::I64And { lhs, rhs, .. } => Op::branch_i64_and(lhs, rhs, offset), - Op::I64Or { lhs, rhs, .. } => Op::branch_i64_or(lhs, rhs, offset), - Op::I64Nand { lhs, rhs, .. } => Op::branch_i64_nand(lhs, rhs, offset), - Op::I64Nor { lhs, rhs, .. } => Op::branch_i64_nor(lhs, rhs, offset), - Op::I64BitAndImm16 { lhs, rhs, .. } => Op::branch_i64_and_imm16(lhs, rhs, offset), - Op::I64BitOrImm16 { lhs, rhs, .. } => Op::branch_i64_or_imm16(lhs, rhs, offset), - Op::I64BitXorImm16 { lhs, rhs, .. } => Op::branch_i64_ne_imm16(lhs, rhs, offset), - Op::I64AndImm16 { lhs, rhs, .. } => Op::branch_i64_and_imm16(lhs, rhs, offset), - Op::I64OrImm16 { lhs, rhs, .. } => Op::branch_i64_or_imm16(lhs, rhs, offset), - Op::I64NandImm16 { lhs, rhs, .. } => Op::branch_i64_nand_imm16(lhs, rhs, offset), - Op::I64NorImm16 { lhs, rhs, .. } => Op::branch_i64_nor_imm16(lhs, rhs, offset), + | Op::I64Eq_Sss { lhs, rhs, .. } => Op::branch_i64_eq_ss(offset, lhs, rhs), + | Op::I64Eq_Ssi { lhs, rhs, .. } => Op::branch_i64_eq_si(offset, lhs, rhs), + | Op::I64And_Sss { lhs, rhs, .. } + | Op::I64BitAnd_Sss { lhs, rhs, .. } => Op::branch_i64_and_ss(offset, lhs, rhs), + | Op::I64And_Ssi { lhs, rhs, .. } + | Op::I64BitAnd_Ssi { lhs, rhs, .. } => Op::branch_i64_and_si(offset, lhs, rhs), + | Op::I64Or_Sss { lhs, rhs, .. } + | Op::I64BitOr_Sss { lhs, rhs, .. } => Op::branch_i64_or_ss(offset, lhs, rhs), + | Op::I64Or_Ssi { lhs, rhs, .. } + | Op::I64BitOr_Ssi { lhs, rhs, .. } => Op::branch_i64_or_si(offset, lhs, rhs), + | Op::I64NotEq_Sss { lhs, rhs, .. } + | Op::I64BitXor_Sss { lhs, rhs, .. } => Op::branch_i64_not_eq_ss(offset, lhs, rhs), + | Op::I64NotEq_Ssi { lhs, rhs, .. } + | Op::I64BitXor_Ssi { lhs, rhs, .. } => Op::branch_i64_not_eq_si(offset, lhs, rhs), + | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::branch_i64_not_and_ss(offset, lhs, rhs), + | Op::I64NotAnd_Ssi { lhs, rhs, .. } => Op::branch_i64_not_and_si(offset, lhs, rhs), + | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::branch_i64_not_or_ss(offset, lhs, rhs), + | Op::I64NotOr_Ssi { lhs, rhs, .. } => Op::branch_i64_not_or_si(offset, lhs, rhs), + | Op::I64Lt_Sss { lhs, rhs, .. } => Op::branch_i64_lt_ss(offset, lhs, rhs), + | Op::I64Lt_Ssi { lhs, rhs, .. } => Op::branch_i64_lt_si(offset, lhs, rhs), + | Op::I64Lt_Sis { lhs, rhs, .. } => Op::branch_i64_lt_is(offset, lhs, rhs), + | Op::U64Lt_Sss { lhs, rhs, .. } => Op::branch_u64_lt_ss(offset, lhs, rhs), + | Op::U64Lt_Ssi { lhs, rhs, .. } => Op::branch_u64_lt_si(offset, lhs, rhs), + | Op::U64Lt_Sis { lhs, rhs, .. } => Op::branch_u64_lt_is(offset, lhs, rhs), + | Op::I64Le_Sss { lhs, rhs, .. } => Op::branch_i64_le_ss(offset, lhs, rhs), + | Op::I64Le_Ssi { lhs, rhs, .. } => Op::branch_i64_le_si(offset, lhs, rhs), + | Op::I64Le_Sis { lhs, rhs, .. } => Op::branch_i64_le_is(offset, lhs, rhs), + | Op::U64Le_Sss { lhs, rhs, .. } => Op::branch_u64_le_ss(offset, lhs, rhs), + | Op::U64Le_Ssi { lhs, rhs, .. } => Op::branch_u64_le_si(offset, lhs, rhs), + | Op::U64Le_Sis { lhs, rhs, .. } => Op::branch_u64_le_is(offset, lhs, rhs), // f32 - Op::F32Eq { lhs, rhs, .. } => Op::branch_f32_eq(lhs, rhs, offset), - Op::F32Ne { lhs, rhs, .. } => Op::branch_f32_ne(lhs, rhs, offset), - Op::F32Lt { lhs, rhs, .. } => Op::branch_f32_lt(lhs, rhs, offset), - Op::F32Le { lhs, rhs, .. } => Op::branch_f32_le(lhs, rhs, offset), - Op::F32NotLt { lhs, rhs, .. } => Op::branch_f32_not_lt(lhs, rhs, offset), - Op::F32NotLe { lhs, rhs, .. } => Op::branch_f32_not_le(lhs, rhs, offset), + | Op::F32Eq_Sss { lhs, rhs, .. } => Op::branch_f32_eq_ss(offset, lhs, rhs), + | Op::F32Eq_Ssi { lhs, rhs, .. } => Op::branch_f32_eq_si(offset, lhs, rhs), + | Op::F32Lt_Sss { lhs, rhs, .. } => Op::branch_f32_lt_ss(offset, lhs, rhs), + | Op::F32Lt_Ssi { lhs, rhs, .. } => Op::branch_f32_lt_si(offset, lhs, rhs), + | Op::F32Lt_Sis { lhs, rhs, .. } => Op::branch_f32_lt_is(offset, lhs, rhs), + | Op::F32Le_Sss { lhs, rhs, .. } => Op::branch_f32_le_ss(offset, lhs, rhs), + | Op::F32Le_Ssi { lhs, rhs, .. } => Op::branch_f32_le_si(offset, lhs, rhs), + | Op::F32Le_Sis { lhs, rhs, .. } => Op::branch_f32_le_is(offset, lhs, rhs), + | Op::F32NotEq_Sss { lhs, rhs, .. } => Op::branch_f32_not_eq_ss(offset, lhs, rhs), + | Op::F32NotEq_Ssi { lhs, rhs, .. } => Op::branch_f32_not_eq_si(offset, lhs, rhs), + | Op::F32NotLt_Sss { lhs, rhs, .. } => Op::branch_f32_not_lt_ss(offset, lhs, rhs), + | Op::F32NotLt_Ssi { lhs, rhs, .. } => Op::branch_f32_not_lt_si(offset, lhs, rhs), + | Op::F32NotLt_Sis { lhs, rhs, .. } => Op::branch_f32_not_lt_is(offset, lhs, rhs), + | Op::F32NotLe_Sss { lhs, rhs, .. } => Op::branch_f32_not_le_ss(offset, lhs, rhs), + | Op::F32NotLe_Ssi { lhs, rhs, .. } => Op::branch_f32_not_le_si(offset, lhs, rhs), + | Op::F32NotLe_Sis { lhs, rhs, .. } => Op::branch_f32_not_le_is(offset, lhs, rhs), // f64 - Op::F64Eq { lhs, rhs, .. } => Op::branch_f64_eq(lhs, rhs, offset), - Op::F64Ne { lhs, rhs, .. } => Op::branch_f64_ne(lhs, rhs, offset), - Op::F64Lt { lhs, rhs, .. } => Op::branch_f64_lt(lhs, rhs, offset), - Op::F64Le { lhs, rhs, .. } => Op::branch_f64_le(lhs, rhs, offset), - Op::F64NotLt { lhs, rhs, .. } => Op::branch_f64_not_lt(lhs, rhs, offset), - Op::F64NotLe { lhs, rhs, .. } => Op::branch_f64_not_le(lhs, rhs, offset), - _ => return Ok(None), + | Op::F64Eq_Sss { lhs, rhs, .. } => Op::branch_f64_eq_ss(offset, lhs, rhs), + | Op::F64Eq_Ssi { lhs, rhs, .. } => Op::branch_f64_eq_si(offset, lhs, rhs), + | Op::F64Lt_Sss { lhs, rhs, .. } => Op::branch_f64_lt_ss(offset, lhs, rhs), + | Op::F64Lt_Ssi { lhs, rhs, .. } => Op::branch_f64_lt_si(offset, lhs, rhs), + | Op::F64Lt_Sis { lhs, rhs, .. } => Op::branch_f64_lt_is(offset, lhs, rhs), + | Op::F64Le_Sss { lhs, rhs, .. } => Op::branch_f64_le_ss(offset, lhs, rhs), + | Op::F64Le_Ssi { lhs, rhs, .. } => Op::branch_f64_le_si(offset, lhs, rhs), + | Op::F64Le_Sis { lhs, rhs, .. } => Op::branch_f64_le_is(offset, lhs, rhs), + | Op::F64NotEq_Sss { lhs, rhs, .. } => Op::branch_f64_not_eq_ss(offset, lhs, rhs), + | Op::F64NotEq_Ssi { lhs, rhs, .. } => Op::branch_f64_not_eq_si(offset, lhs, rhs), + | Op::F64NotLt_Sss { lhs, rhs, .. } => Op::branch_f64_not_lt_ss(offset, lhs, rhs), + | Op::F64NotLt_Ssi { lhs, rhs, .. } => Op::branch_f64_not_lt_si(offset, lhs, rhs), + | Op::F64NotLt_Sis { lhs, rhs, .. } => Op::branch_f64_not_lt_is(offset, lhs, rhs), + | Op::F64NotLe_Sss { lhs, rhs, .. } => Op::branch_f64_not_le_ss(offset, lhs, rhs), + | Op::F64NotLe_Ssi { lhs, rhs, .. } => Op::branch_f64_not_le_si(offset, lhs, rhs), + | Op::F64NotLe_Sis { lhs, rhs, .. } => Op::branch_f64_not_le_is(offset, lhs, rhs), + _ => return None, }; - Ok(Some(cmp_branch_instr)) + Some(cmp_branch_instr) } } -pub trait TryIntoCmpBranchFallbackInstr { - fn try_into_cmp_branch_fallback_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error>; -} +/// Extension trait for [`Op`] to update [`BranchOffset`] of branch operators. +pub trait UpdateBranchOffset: Sized { + /// Updates the [`BranchOffset`] of `self` to `new_offset`. + /// + /// # Panics + /// + /// - If `self` does not have a [`BranchOffset`] to udpate. + /// - If the [`BranchOffset`] of `self` is already initialized. (Debug) + fn update_branch_offset(&mut self, new_offset: BranchOffset); -impl TryIntoCmpBranchFallbackInstr for Op { - fn try_into_cmp_branch_fallback_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error> { - debug_assert!(BranchOffset16::try_from(offset).is_err()); - let Some(comparator) = try_into_cmp_br_comparator(self) else { - return Ok(None); - }; - #[rustfmt::skip] - let (lhs, rhs) = match *self { - | Op::BranchI32And { lhs, rhs, .. } - | Op::BranchI32Or { lhs, rhs, .. } - | Op::BranchI32Nand { lhs, rhs, .. } - | Op::BranchI32Nor { lhs, rhs, .. } - | Op::BranchI32Eq { lhs, rhs, .. } - | Op::BranchI32Ne { lhs, rhs, .. } - | Op::BranchI32LtS { lhs, rhs, .. } - | Op::BranchI32LtU { lhs, rhs, .. } - | Op::BranchI32LeS { lhs, rhs, .. } - | Op::BranchI32LeU { lhs, rhs, .. } - | Op::BranchI64And { lhs, rhs, .. } - | Op::BranchI64Or { lhs, rhs, .. } - | Op::BranchI64Nand { lhs, rhs, .. } - | Op::BranchI64Nor { lhs, rhs, .. } - | Op::BranchI64Eq { lhs, rhs, .. } - | Op::BranchI64Ne { lhs, rhs, .. } - | Op::BranchI64LtS { lhs, rhs, .. } - | Op::BranchI64LtU { lhs, rhs, .. } - | Op::BranchI64LeS { lhs, rhs, .. } - | Op::BranchI64LeU { lhs, rhs, .. } - | Op::BranchF32Eq { lhs, rhs, .. } - | Op::BranchF32Ne { lhs, rhs, .. } - | Op::BranchF32Lt { lhs, rhs, .. } - | Op::BranchF32Le { lhs, rhs, .. } - | Op::BranchF32NotLt { lhs, rhs, .. } - | Op::BranchF32NotLe { lhs, rhs, .. } - | Op::BranchF64Eq { lhs, rhs, .. } - | Op::BranchF64Ne { lhs, rhs, .. } - | Op::BranchF64Lt { lhs, rhs, .. } - | Op::BranchF64Le { lhs, rhs, .. } - | Op::BranchF64NotLt { lhs, rhs, .. } - | Op::BranchF64NotLe { lhs, rhs, .. } => (lhs, rhs), - | Op::BranchI32AndImm16 { lhs, rhs, .. } - | Op::BranchI32OrImm16 { lhs, rhs, .. } - | Op::BranchI32NandImm16 { lhs, rhs, .. } - | Op::BranchI32NorImm16 { lhs, rhs, .. } - | Op::BranchI32EqImm16 { lhs, rhs, .. } - | Op::BranchI32NeImm16 { lhs, rhs, .. } - | Op::BranchI32LtSImm16Rhs { lhs, rhs, .. } - | Op::BranchI32LeSImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(i32::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI32LtSImm16Lhs { lhs, rhs, .. } - | Op::BranchI32LeSImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(i32::from(lhs))?; - (lhs, rhs) - } - | Op::BranchI32LtUImm16Rhs { lhs, rhs, .. } - | Op::BranchI32LeUImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(u32::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI32LtUImm16Lhs { lhs, rhs, .. } - | Op::BranchI32LeUImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(u32::from(lhs))?; - (lhs, rhs) - } - | Op::BranchI64AndImm16 { lhs, rhs, .. } - | Op::BranchI64OrImm16 { lhs, rhs, .. } - | Op::BranchI64NandImm16 { lhs, rhs, .. } - | Op::BranchI64NorImm16 { lhs, rhs, .. } - | Op::BranchI64EqImm16 { lhs, rhs, .. } - | Op::BranchI64NeImm16 { lhs, rhs, .. } - | Op::BranchI64LtSImm16Rhs { lhs, rhs, .. } - | Op::BranchI64LeSImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(i64::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI64LtSImm16Lhs { lhs, rhs, .. } - | Op::BranchI64LeSImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(i64::from(lhs))?; - (lhs, rhs) - } - | Op::BranchI64LtUImm16Rhs { lhs, rhs, .. } - | Op::BranchI64LeUImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(u64::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI64LtUImm16Lhs { lhs, rhs, .. } - | Op::BranchI64LeUImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(u64::from(lhs))?; - (lhs, rhs) - } - _ => return Ok(None), - }; - let params = stack.alloc_const(ComparatorAndOffset::new(comparator, offset))?; - Ok(Some(Op::branch_cmp_fallback(lhs, rhs, params))) + /// Consumes `self` and returns it back with its [`BranchOffset`] set to `new_offset`. + fn with_branch_offset(self, new_offset: BranchOffset) -> Self { + let mut this = self; + this.update_branch_offset(new_offset); + this } } -fn try_into_cmp_br_comparator(instr: &Op) -> Option { - #[rustfmt::skip] - let comparator = match *instr { - // i32 - | Op::BranchI32Eq { .. } | Op::BranchI32EqImm16 { .. } => Comparator::I32Eq, - | Op::BranchI32Ne { .. } | Op::BranchI32NeImm16 { .. } => Comparator::I32Ne, - | Op::BranchI32LtS { .. } - | Op::BranchI32LtSImm16Lhs { .. } - | Op::BranchI32LtSImm16Rhs { .. } => Comparator::I32LtS, - | Op::BranchI32LtU { .. } - | Op::BranchI32LtUImm16Lhs { .. } - | Op::BranchI32LtUImm16Rhs { .. } => Comparator::I32LtU, - | Op::BranchI32LeS { .. } - | Op::BranchI32LeSImm16Lhs { .. } - | Op::BranchI32LeSImm16Rhs { .. } => Comparator::I32LeS, - | Op::BranchI32LeU { .. } - | Op::BranchI32LeUImm16Lhs { .. } - | Op::BranchI32LeUImm16Rhs { .. } => Comparator::I32LeU, - // i32 (and,or,xor) - | Op::BranchI32And { .. } => Comparator::I32And, - | Op::BranchI32Or { .. } => Comparator::I32Or, - | Op::BranchI32Nand { .. } => Comparator::I32Nand, - | Op::BranchI32Nor { .. } => Comparator::I32Nor, - // i64 - | Op::BranchI64Eq { .. } | Op::BranchI64EqImm16 { .. } => Comparator::I64Eq, - | Op::BranchI64Ne { .. } | Op::BranchI64NeImm16 { .. } => Comparator::I64Ne, - | Op::BranchI64LtS { .. } - | Op::BranchI64LtSImm16Lhs { .. } - | Op::BranchI64LtSImm16Rhs { .. } => Comparator::I64LtS, - | Op::BranchI64LtU { .. } - | Op::BranchI64LtUImm16Lhs { .. } - | Op::BranchI64LtUImm16Rhs { .. } => Comparator::I64LtU, - | Op::BranchI64LeS { .. } - | Op::BranchI64LeSImm16Lhs { .. } - | Op::BranchI64LeSImm16Rhs { .. } => Comparator::I64LeS, - | Op::BranchI64LeU { .. } - | Op::BranchI64LeUImm16Lhs { .. } - | Op::BranchI64LeUImm16Rhs { .. } => Comparator::I64LeU, - // f32 - | Op::BranchF32Eq { .. } => Comparator::F32Eq, - | Op::BranchF32Ne { .. } => Comparator::F32Ne, - | Op::BranchF32Lt { .. } => Comparator::F32Lt, - | Op::BranchF32Le { .. } => Comparator::F32Le, - | Op::BranchF32NotLt { .. } => Comparator::F32NotLt, - | Op::BranchF32NotLe { .. } => Comparator::F32NotLe, - // f64 - | Op::BranchF64Eq { .. } => Comparator::F64Eq, - | Op::BranchF64Ne { .. } => Comparator::F64Ne, - | Op::BranchF64Lt { .. } => Comparator::F64Lt, - | Op::BranchF64Le { .. } => Comparator::F64Le, - | Op::BranchF64NotLt { .. } => Comparator::F64NotLt, - | Op::BranchF64NotLe { .. } => Comparator::F64NotLe, - _ => return None, - }; - Some(comparator) +impl UpdateBranchOffset for ir::BranchOffset { + fn update_branch_offset(&mut self, new_offset: BranchOffset) { + *self = new_offset; + } } -/// Extension trait to update the branch offset of an [`Op`]. -pub trait UpdateBranchOffset { - /// Updates the [`BranchOffset`] for the branch [`Op`]. - /// - /// # Panics - /// - /// If `self` is not a branch [`Op`]. - fn update_branch_offset( - &mut self, - stack: &mut impl AllocConst, - new_offset: BranchOffset, - ) -> Result<(), Error>; +impl UpdateBranchOffset for ir::BranchTableTarget { + fn update_branch_offset(&mut self, new_offset: BranchOffset) { + self.offset = new_offset; + } } impl UpdateBranchOffset for Op { - #[rustfmt::skip] - fn update_branch_offset( - &mut self, - stack: &mut impl AllocConst, - new_offset: BranchOffset, - ) -> Result<(), Error> { + #[track_caller] + fn update_branch_offset(&mut self, new_offset: BranchOffset) { match self { + // unconditional | Op::Branch { offset } - | Op::BranchTableTarget { offset, .. } => { - offset.init(new_offset); - return Ok(()); - } - _ => {} - }; - let offset = match self { - | Op::BranchI32And { offset, .. } - | Op::BranchI32Or { offset, .. } - | Op::BranchI32Nand { offset, .. } - | Op::BranchI32Nor { offset, .. } - | Op::BranchI32Eq { offset, .. } - | Op::BranchI32Ne { offset, .. } - | Op::BranchI32LtS { offset, .. } - | Op::BranchI32LtU { offset, .. } - | Op::BranchI32LeS { offset, .. } - | Op::BranchI32LeU { offset, .. } - | Op::BranchI64And { offset, .. } - | Op::BranchI64Or { offset, .. } - | Op::BranchI64Nand { offset, .. } - | Op::BranchI64Nor { offset, .. } - | Op::BranchI64Eq { offset, .. } - | Op::BranchI64Ne { offset, .. } - | Op::BranchI64LtS { offset, .. } - | Op::BranchI64LtU { offset, .. } - | Op::BranchI64LeS { offset, .. } - | Op::BranchI64LeU { offset, .. } - | Op::BranchF32Eq { offset, .. } - | Op::BranchF32Ne { offset, .. } - | Op::BranchF32Lt { offset, .. } - | Op::BranchF32Le { offset, .. } - | Op::BranchF32NotLt { offset, .. } - | Op::BranchF32NotLe { offset, .. } - | Op::BranchF64Eq { offset, .. } - | Op::BranchF64Ne { offset, .. } - | Op::BranchF64Lt { offset, .. } - | Op::BranchF64Le { offset, .. } - | Op::BranchF64NotLt { offset, .. } - | Op::BranchF64NotLe { offset, .. } - | Op::BranchI32AndImm16 { offset, .. } - | Op::BranchI32OrImm16 { offset, .. } - | Op::BranchI32NandImm16 { offset, .. } - | Op::BranchI32NorImm16 { offset, .. } - | Op::BranchI32EqImm16 { offset, .. } - | Op::BranchI32NeImm16 { offset, .. } - | Op::BranchI32LtSImm16Lhs { offset, .. } - | Op::BranchI32LtSImm16Rhs { offset, .. } - | Op::BranchI32LeSImm16Lhs { offset, .. } - | Op::BranchI32LeSImm16Rhs { offset, .. } - | Op::BranchI32LtUImm16Lhs { offset, .. } - | Op::BranchI32LtUImm16Rhs { offset, .. } - | Op::BranchI32LeUImm16Lhs { offset, .. } - | Op::BranchI32LeUImm16Rhs { offset, .. } - | Op::BranchI64AndImm16 { offset, .. } - | Op::BranchI64OrImm16 { offset, .. } - | Op::BranchI64NandImm16 { offset, .. } - | Op::BranchI64NorImm16 { offset, .. } - | Op::BranchI64EqImm16 { offset, .. } - | Op::BranchI64NeImm16 { offset, .. } - | Op::BranchI64LtSImm16Lhs { offset, .. } - | Op::BranchI64LtSImm16Rhs { offset, .. } - | Op::BranchI64LeSImm16Lhs { offset, .. } - | Op::BranchI64LeSImm16Rhs { offset, .. } - | Op::BranchI64LtUImm16Lhs { offset, .. } - | Op::BranchI64LtUImm16Rhs { offset, .. } - | Op::BranchI64LeUImm16Lhs { offset, .. } - | Op::BranchI64LeUImm16Rhs { offset, .. } => offset, - unexpected => { - panic!("expected a Wasmi `cmp`+`branch` instruction but found: {unexpected:?}") + // i32 + | Op::BranchI32Eq_Ss { offset, .. } + | Op::BranchI32Eq_Si { offset, .. } + | Op::BranchI32And_Ss { offset, .. } + | Op::BranchI32And_Si { offset, .. } + | Op::BranchI32Or_Ss { offset, .. } + | Op::BranchI32Or_Si { offset, .. } + | Op::BranchI32NotEq_Ss { offset, .. } + | Op::BranchI32NotEq_Si { offset, .. } + | Op::BranchI32NotAnd_Ss { offset, .. } + | Op::BranchI32NotAnd_Si { offset, .. } + | Op::BranchI32NotOr_Ss { offset, .. } + | Op::BranchI32NotOr_Si { offset, .. } + | Op::BranchI32Lt_Ss { offset, .. } + | Op::BranchI32Lt_Si { offset, .. } + | Op::BranchI32Lt_Is { offset, .. } + | Op::BranchU32Lt_Ss { offset, .. } + | Op::BranchU32Lt_Si { offset, .. } + | Op::BranchU32Lt_Is { offset, .. } + | Op::BranchI32Le_Ss { offset, .. } + | Op::BranchI32Le_Si { offset, .. } + | Op::BranchI32Le_Is { offset, .. } + | Op::BranchU32Le_Ss { offset, .. } + | Op::BranchU32Le_Si { offset, .. } + | Op::BranchU32Le_Is { offset, .. } + // i64 + | Op::BranchI64Eq_Ss { offset, .. } + | Op::BranchI64Eq_Si { offset, .. } + | Op::BranchI64And_Ss { offset, .. } + | Op::BranchI64And_Si { offset, .. } + | Op::BranchI64Or_Ss { offset, .. } + | Op::BranchI64Or_Si { offset, .. } + | Op::BranchI64NotEq_Ss { offset, .. } + | Op::BranchI64NotEq_Si { offset, .. } + | Op::BranchI64NotAnd_Ss { offset, .. } + | Op::BranchI64NotAnd_Si { offset, .. } + | Op::BranchI64NotOr_Ss { offset, .. } + | Op::BranchI64NotOr_Si { offset, .. } + | Op::BranchI64Lt_Ss { offset, .. } + | Op::BranchI64Lt_Si { offset, .. } + | Op::BranchI64Lt_Is { offset, .. } + | Op::BranchU64Lt_Ss { offset, .. } + | Op::BranchU64Lt_Si { offset, .. } + | Op::BranchU64Lt_Is { offset, .. } + | Op::BranchI64Le_Ss { offset, .. } + | Op::BranchI64Le_Si { offset, .. } + | Op::BranchI64Le_Is { offset, .. } + | Op::BranchU64Le_Ss { offset, .. } + | Op::BranchU64Le_Si { offset, .. } + | Op::BranchU64Le_Is { offset, .. } + // f32 + | Op::BranchF32Eq_Ss { offset, .. } + | Op::BranchF32Eq_Si { offset, .. } + | Op::BranchF32Lt_Ss { offset, .. } + | Op::BranchF32Lt_Si { offset, .. } + | Op::BranchF32Lt_Is { offset, .. } + | Op::BranchF32Le_Ss { offset, .. } + | Op::BranchF32Le_Si { offset, .. } + | Op::BranchF32Le_Is { offset, .. } + | Op::BranchF32NotEq_Ss { offset, .. } + | Op::BranchF32NotEq_Si { offset, .. } + | Op::BranchF32NotLt_Ss { offset, .. } + | Op::BranchF32NotLt_Si { offset, .. } + | Op::BranchF32NotLt_Is { offset, .. } + | Op::BranchF32NotLe_Ss { offset, .. } + | Op::BranchF32NotLe_Si { offset, .. } + | Op::BranchF32NotLe_Is { offset, .. } + // f64 + | Op::BranchF64Eq_Ss { offset, .. } + | Op::BranchF64Eq_Si { offset, .. } + | Op::BranchF64Lt_Ss { offset, .. } + | Op::BranchF64Lt_Si { offset, .. } + | Op::BranchF64Lt_Is { offset, .. } + | Op::BranchF64Le_Ss { offset, .. } + | Op::BranchF64Le_Si { offset, .. } + | Op::BranchF64Le_Is { offset, .. } + | Op::BranchF64NotEq_Ss { offset, .. } + | Op::BranchF64NotEq_Si { offset, .. } + | Op::BranchF64NotLt_Ss { offset, .. } + | Op::BranchF64NotLt_Si { offset, .. } + | Op::BranchF64NotLt_Is { offset, .. } + | Op::BranchF64NotLe_Ss { offset, .. } + | Op::BranchF64NotLe_Si { offset, .. } + | Op::BranchF64NotLe_Is { offset, .. } => { + debug_assert!(!offset.is_init()); + *offset = new_offset; } - }; - if offset.init(new_offset).is_err() { - // Case: we need to convert `self` into its cmp+branch fallback instruction variant - // since adjusting the 16-bit offset failed. - let Some(fallback) = self.try_into_cmp_branch_fallback_instr(new_offset, stack)? else { - unreachable!("failed to create cmp+branch fallback instruction for: {self:?}"); - }; - *self = fallback; + _ => panic!("expected branch `Op` but found: {:?}", self), } - Ok(()) } } diff --git a/crates/wasmi/src/engine/translator/error.rs b/crates/wasmi/src/engine/translator/error.rs index 93ca88f4f89..c326440338a 100644 --- a/crates/wasmi/src/engine/translator/error.rs +++ b/crates/wasmi/src/engine/translator/error.rs @@ -34,6 +34,8 @@ pub enum TranslationError { TooManyLocalVariables, /// The function failed to compiled lazily. LazyCompilationFailed, + /// Ran out of system memory during translation. + OutOfSystemMemory, } impl TranslationError { @@ -52,64 +54,41 @@ impl Error for TranslationError {} impl Display for TranslationError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { + let message = match self { Self::UnsupportedBlockType(error) => { - write!(f, "encountered unsupported Wasm block type: {error:?}") + return write!(f, "encountered unsupported Wasm block type: {error:?}") } Self::UnsupportedValueType(error) => { - write!(f, "encountered unsupported Wasm value type: {error:?}") + return write!(f, "encountered unsupported Wasm value type: {error:?}") } Self::BranchTableTargetsOutOfBounds => { - write!( - f, - "branch table targets are out of bounds for wasmi bytecode" - ) - } - Self::BranchOffsetOutOfBounds => { - write!(f, "branching offset is out of bounds for wasmi bytecode") + "branch table targets are out of bounds for wasmi bytecode" } + Self::BranchOffsetOutOfBounds => "branching offset is out of bounds for wasmi bytecode", Self::BlockFuelOutOfBounds => { - write!( - f, - "fuel required to execute a block is out of bounds for wasmi bytecode" - ) + "fuel required to execute a block is out of bounds for wasmi bytecode" } Self::AllocatedTooManySlots => { - write!( - f, - "translation requires more registers for a function than available" - ) - } - Self::SlotOutOfBounds => { - write!(f, "tried to access out of bounds register index") + "translation requires more registers for a function than available" } + Self::SlotOutOfBounds => "tried to access out of bounds register index", Self::EmulatedValueStackOverflow => { - write!(f, "function requires value stack with out of bounds depth") + "function requires value stack with out of bounds depth" } Self::ProviderSliceOverflow => { - write!(f, "tried to allocate too many or too large provider slices") + "tried to allocate too many or too large provider slices" } Self::TooManyFuncLocalConstValues => { - write!( - f, - "tried to allocate too many function local constant values" - ) - } - Self::TooManyFunctionResults => { - write!(f, "encountered function with too many function results") - } - Self::TooManyFunctionParams => { - write!(f, "encountered function with too many function parameters") - } - Self::TooManyLocalVariables => { - write!(f, "encountered function with too many local variables") + "tried to allocate too many function local constant values" } + Self::TooManyFunctionResults => "encountered function with too many function results", + Self::TooManyFunctionParams => "encountered function with too many function parameters", + Self::TooManyLocalVariables => "encountered function with too many local variables", Self::LazyCompilationFailed => { - write!( - f, - "lazy function compilation encountered a Wasm validation or translation error" - ) + "lazy function compilation encountered a Wasm validation or translation error" } - } + Self::OutOfSystemMemory => "ran out of system memory during translation", + }; + f.write_str(message) } } diff --git a/crates/wasmi/src/engine/translator/func/encoder.rs b/crates/wasmi/src/engine/translator/func/encoder.rs new file mode 100644 index 00000000000..e4f97b04741 --- /dev/null +++ b/crates/wasmi/src/engine/translator/func/encoder.rs @@ -0,0 +1,789 @@ +use super::{Reset, ReusableAllocations}; +use crate::{ + core::FuelCostsProvider, + engine::{ + executor::op_code_to_handler, + translator::{ + comparator::UpdateBranchOffset, + func::{ + labels::{Label, ResolvedLabelUser}, + LabelRef, + LabelRegistry, + }, + }, + TranslationError, + }, + ir::{self, BlockFuel, BranchOffset, Encode as _, Op, OpCode}, + Engine, + Error, +}; +use alloc::vec::Vec; +use core::{cmp, fmt, marker::PhantomData, mem}; + +/// Fuel amount required by certain operators. +type FuelUsed = u64; + +/// A byte position within the encoded byte buffer. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct BytePos(usize); + +impl From for BytePos { + fn from(index: usize) -> Self { + Self(index) + } +} + +impl From for usize { + fn from(pos: BytePos) -> Self { + pos.0 + } +} + +/// A position within the encoded byte buffer and its known encoded type. +pub struct Pos { + /// The underlying byte position. + value: BytePos, + /// The type marker denoting what value type has been encoded. + marker: PhantomData T>, +} + +impl From for Pos { + fn from(value: BytePos) -> Self { + Self { + value, + marker: PhantomData, + } + } +} +impl From> for BytePos { + fn from(pos: Pos) -> Self { + pos.value + } +} +impl Copy for Pos {} +impl Clone for Pos { + fn clone(&self) -> Self { + *self + } +} +impl PartialEq for Pos { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } +} +impl Eq for Pos {} +impl PartialOrd for Pos { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for Pos { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.value.cmp(&other.value) + } +} +impl fmt::Debug for Pos { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pos") + .field("value", &self.value) + .field("marker", &self.marker) + .finish() + } +} + +#[derive(Debug, Default)] +pub struct EncodedOps { + buffer: Vec, + temp: Option, +} + +/// A [`Pos`] of an encoded item that needs to be reported back. +#[derive(Debug)] +enum ReportingPos { + /// The temporary object is a [`BranchOffset`]. + BranchOffset(Pos), + /// The temporary object is a [`BlockFuel`]. + BlockFuel(Pos), +} + +impl Reset for EncodedOps { + fn reset(&mut self) { + self.buffer.clear(); + } +} + +impl EncodedOps { + /// Returns the next [`BytePos`]. + #[must_use] + fn next_pos(&self) -> BytePos { + BytePos::from(self.buffer.len()) + } + + /// Takes the reporting [`Pos`] if any exists. + #[must_use] + fn take_reporting_pos(&mut self) -> Option { + self.temp.take() + } +} + +impl ir::Encoder for EncodedOps { + type Pos = BytePos; + type Error = TranslationError; + + fn write_bytes(&mut self, bytes: &[u8]) -> Result { + let pos = self.buffer.len(); + if self.buffer.try_reserve(bytes.len()).is_err() { + return Err(TranslationError::OutOfSystemMemory); + } + self.buffer.extend(bytes); + Ok(BytePos::from(pos)) + } + + fn encode_op_code(&mut self, code: OpCode) -> Result { + encode_op_code(self, code) + } + + fn branch_offset( + &mut self, + pos: Self::Pos, + _branch_offset: BranchOffset, + ) -> Result<(), Self::Error> { + debug_assert!(self.temp.is_none()); + self.temp = Some(ReportingPos::BranchOffset(pos.into())); + Ok(()) + } + + fn block_fuel( + &mut self, + pos: Self::Pos, + _block_fuel: ir::BlockFuel, + ) -> Result<(), Self::Error> { + debug_assert!(self.temp.is_none()); + self.temp = Some(ReportingPos::BlockFuel(pos.into())); + Ok(()) + } +} + +/// Creates and encodes the buffer of encoded [`Op`]s for a function. +#[derive(Debug, Default)] +pub struct OpEncoder { + /// The currently staged [`Op`]. + /// + /// # Note + /// + /// - This allows the last [`Op`] to be peeked, inspected and manipulated. + /// - For example, this is useful to perform op-code fusion or adjusting the result slot. + staged: Option, + /// The fuel costs of instructions. + /// + /// This is `Some` if fuel metering is enabled, otherwise `None`. + fuel_costs: Option, + /// The list of constructed instructions and their parameters. + ops: EncodedOps, + /// Labels and label users for control flow and encoded branch operators. + labels: LabelRegistry, +} + +/// The staged [`Op`] and information about its fuel consumption. +#[derive(Debug, Copy, Clone)] +pub struct StagedOp { + /// The staged [`Op`]. + op: Op, + /// Fuel information for the staged [`Op`]. + /// + /// - The [`Op::ConsumeFuel`] operator associated to the staged [`Op`] if any. + /// - The fuel required by the staged [`Op`]. + fuel: Option<(Pos, FuelUsed)>, +} + +impl StagedOp { + /// Creates a new [`StagedOp`] from `op` and `fuel`. + pub fn new(op: Op, fuel: Option<(Pos, FuelUsed)>) -> Self { + Self { op, fuel } + } + + /// Replaces the current [`Op`] with `op`. + /// + /// Returns the [`Op`] being replaced. + pub fn replace(&mut self, op: Op) -> Op { + mem::replace(&mut self.op, op) + } +} + +impl ReusableAllocations for OpEncoder { + type Allocations = OpEncoderAllocations; + + fn into_allocations(self) -> Self::Allocations { + Self::Allocations { + ops: self.ops, + labels: self.labels, + } + } +} + +/// The reusable heap allocations of the [`OpEncoder`]. +#[derive(Debug, Default)] +pub struct OpEncoderAllocations { + /// The list of constructed instructions and their parameters. + ops: EncodedOps, + /// Labels and label users for control flow and encoded branch operators. + labels: LabelRegistry, +} + +impl Reset for OpEncoderAllocations { + fn reset(&mut self) { + self.ops.reset(); + self.labels.reset(); + } +} + +impl OpEncoder { + /// Creates a new [`OpEncoder`]. + pub fn new(engine: &Engine, alloc: OpEncoderAllocations) -> Self { + let config = engine.config(); + let fuel_costs = config + .get_consume_fuel() + .then(|| config.fuel_costs()) + .cloned(); + Self { + staged: None, + fuel_costs, + ops: alloc.ops, + labels: alloc.labels, + } + } + + /// Allocates a new unpinned [`Label`]. + pub fn new_label(&mut self) -> LabelRef { + self.labels.new_label() + } + + /// Pins the [`Label`] at `lref` to the current encoded bytestream position. + /// + /// # Panics + /// + /// If there is a staged [`Op`]. + pub fn pin_label(&mut self, lref: LabelRef) -> Result<(), Error> { + self.try_encode_staged()?; + let next_pos = Pos::from(self.ops.next_pos()); + self.labels.pin_label(lref, next_pos); + Ok(()) + } + + /// Pins the [`Label`] at `lref` to the current encoded bytestream position if unpinned. + /// + /// # Note + /// + /// Does nothing if the label is already pinned. + /// + /// # Panics + /// + /// If there is a staged [`Op`]. + pub fn pin_label_if_unpinned(&mut self, lref: LabelRef) -> Result<(), Error> { + self.try_encode_staged()?; + let next_pos = Pos::from(self.ops.next_pos()); + self.labels.pin_label_if_unpinned(lref, next_pos); + Ok(()) + } + + /// Resolves the [`BranchOffset`] to `lref` from the current encoded bytestream position if `lref` is pinned. + /// + /// + /// # Note + /// + /// Returns an uninitialized [`BranchOffset`] if `lref` refers to an unpinned [`Label`]. + /// + /// # Panics + /// + /// If there is a staged [`Op`]. + fn try_resolve_label(&mut self, lref: LabelRef) -> Result { + assert!(self.staged.is_none()); + let src = self.ops.next_pos(); + let offset = match self.labels.get_label(lref) { + Label::Pinned(dst) => trace_branch_offset(src, dst)?, + Label::Unpinned => BranchOffset::uninit(), + }; + Ok(offset) + } + + /// Returns the staged [`Op`] if any. + pub fn peek_staged(&self) -> Option { + self.staged.map(|staged| staged.op) + } + + /// Sets the staged [`Op`] to `new_staged` and encodes the previously staged [`Op`] if any. + /// + /// Returns the [`Pos`] of the staged [`Op`] if it was encoded. + pub fn stage( + &mut self, + new_staged: Op, + fuel_op: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result, Error> { + let fuel = match (fuel_op, &self.fuel_costs) { + (None, None) => None, + (Some(fuel_op), Some(fuel_costs)) => Some((fuel_op, fuel_selector.select(fuel_costs))), + _ => unreachable!(), + }; + let new_staged = StagedOp::new(new_staged, fuel); + if let Some(old_staged) = self.staged.replace(new_staged) { + self.encode_staged(old_staged)?; + } + Ok(Pos::from(self.ops.next_pos())) + } + + /// Encodes the staged [`Op`] if there is any. + /// + /// # Note + /// + /// - After this operation there will be no more staged [`Op`]. + /// - Does nothing if there is no staged [`Op`]. + pub fn try_encode_staged(&mut self) -> Result<(), Error> { + if let Some(staged) = self.staged.take() { + self.encode_staged(staged)?; + } + debug_assert!(self.staged.is_none()); + Ok(()) + } + + /// Encodes the `staged_op`. + /// + /// - Bumps fuel consumption of the associated [`Op::ConsumeFuel`] operator. + /// - Returns the [`Pos`] of the encoded [`StagedOp`]. + /// + /// # Panics (Debug) + /// + /// If the staged operator unexpectedly issued [`BranchOffset`] or [`BlockFuel`] fields. + /// Those operators may never be staged and must be taken care of directly. + fn encode_staged(&mut self, staged: StagedOp) -> Result, Error> { + if let Some((fuel_pos, fuel_used)) = staged.fuel { + self.bump_fuel_consumption_by(Some(fuel_pos), fuel_used)?; + } + let pos = self.encode_impl(staged.op)?; + debug_assert!(self.ops.temp.is_none()); + Ok(pos) + } + + /// Drops the staged [`Op`] without encoding it. + /// + /// Returns the staged [`Op`]'s fuel information or `None` if fuel metering is disabled. + /// + /// # Panics + /// + /// If there was no staged [`Op`]. + pub fn drop_staged(&mut self) -> (Option>, FuelUsed) { + let Some(staged) = self.staged.take() else { + panic!("could not drop staged `Op` since there was none") + }; + debug_assert!(self.staged.is_none()); + let fuel_pos = staged.fuel.map(|(pos, _)| pos); + let fuel_used = staged.fuel.map(|(_, used)| used).unwrap_or(0); + (fuel_pos, fuel_used) + } + + /// Replaces the staged [`Op`] with `new_staged`. + /// + /// - This does __not__ encode the currently staged [`Op`] but merely replaces it. + /// - Returns the [`Pos`] of the newly staged [`Op`]. + /// + /// # Panics (Debug) + /// + /// If there currently is no staged [`Op`] that can be replaced. + pub fn replace_staged(&mut self, new_staged: Op) -> Result, Error> { + let Some(staged) = self.staged.as_mut() else { + panic!("expected a staged `Op` but found `None`") + }; + staged.replace(new_staged); + Ok(Pos::from(self.ops.next_pos())) + } + + /// Encodes an item of type `T` to the [`OpEncoder`] and returns its [`Pos`]. + /// + /// # Note + /// + /// Bumps the `fuel` of the [`Op::ConsumeFuel`] accordingly. + pub fn encode( + &mut self, + op: T, + fuel_pos: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result, Error> { + self.try_encode_staged()?; + self.bump_fuel_consumption(fuel_pos, fuel_selector)?; + let pos = self.encode_impl(op)?; + debug_assert!(self.ops.take_reporting_pos().is_none()); + debug_assert!(self.staged.is_none()); + Ok(pos) + } + + /// Encodes an [`Op::ConsumeFuel`] operator to `self`. + /// + /// # Note + /// + /// The pushed [`Op::ConsumeFuel`] is initialized with base fuel costs. + pub fn encode_consume_fuel(&mut self) -> Result>, Error> { + let Some(fuel_costs) = &self.fuel_costs else { + return Ok(None); + }; + let consumed_fuel = BlockFuel::from(fuel_costs.base()); + self.try_encode_staged()?; + Op::consume_fuel(consumed_fuel).encode(&mut self.ops)?; + let Some(ReportingPos::BlockFuel(pos)) = self.ops.take_reporting_pos() else { + unreachable!("expected encoded `BlockFuel` entry but found none") + }; + debug_assert!(self.staged.is_none()); + Ok(Some(pos)) + } + + /// Encodes a type with [`BranchOffset`] to the [`OpEncoder`] and returns its [`Pos`] and [`Pos`]. + /// + /// # Note + /// + /// Bumps the `fuel` of the [`Op::ConsumeFuel`] accordingly. + pub fn encode_branch( + &mut self, + dst: LabelRef, + make_branch: impl FnOnce(BranchOffset) -> T, + fuel_pos: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result<(Pos, Pos), Error> + where + T: ir::Encode + UpdateBranchOffset, + { + self.try_encode_staged()?; + self.bump_fuel_consumption(fuel_pos, fuel_selector)?; + let offset = self.try_resolve_label(dst)?; + let item = make_branch(offset); + let pos_item = self.encode_impl(item)?; + let pos_offset = match self.ops.take_reporting_pos() { + Some(ReportingPos::BranchOffset(pos)) => pos, + _ => panic!("missing encoded position for `BranchOffset`"), + }; + if !self.labels.is_pinned(dst) { + self.labels + .new_user(dst, BytePos::from(pos_item), pos_offset); + } + debug_assert!(self.staged.is_none()); + Ok((pos_item, pos_offset)) + } + + /// Encodes an [`Op`] to the [`OpEncoder`] and returns its [`Pos`]. + /// + /// # Note + /// + /// - Encodes `last` [`Op`] prior to `op` if `last` is `Some`. + /// - After this call `last` will yield `None`. + fn encode_impl(&mut self, op: T) -> Result, Error> + where + T: ir::Encode, + { + let pos = self.ops.next_pos(); + op.encode(&mut self.ops)?; + Ok(Pos::from(pos)) + } + + /// Bumps consumed fuel for [`Op::ConsumeFuel`] at `fuel_pos` by `fuel_selector(fuel_costs)`. + /// + /// Does nothing if fuel metering is disabled. + /// + /// # Errors + /// + /// If consumed fuel is out of bounds after this operation. + fn bump_fuel_consumption( + &mut self, + fuel_pos: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result<(), Error> { + debug_assert_eq!(fuel_pos.is_some(), self.fuel_costs.is_some()); + let fuel_used = self + .fuel_costs + .as_ref() + .map(|costs| fuel_selector.select(costs)) + .unwrap_or(0); + if fuel_used == 0 { + return Ok(()); + } + self.bump_fuel_consumption_by(fuel_pos, fuel_used) + } + + /// Bumps consumed fuel for [`Op::ConsumeFuel`] at `fuel_pos` by `delta`. + /// + /// Does nothing if fuel metering is disabled. + /// + /// # Errors + /// + /// If consumed fuel is out of bounds after this operation. + fn bump_fuel_consumption_by( + &mut self, + fuel_pos: Option>, + delta: FuelUsed, + ) -> Result<(), Error> { + debug_assert_eq!(fuel_pos.is_some(), self.fuel_costs.is_some()); + let fuel_pos = match fuel_pos { + None => return Ok(()), + Some(fuel_pos) => fuel_pos, + }; + self.ops + .update_encoded(fuel_pos, |mut fuel| -> Option { + fuel.bump_by(delta).ok()?; + Some(fuel) + }); + Ok(()) + } + + /// Returns an iterator yielding all encoded [`Op`]s of the [`OpEncoder`] as bytes. + pub fn encoded_ops(&mut self) -> &[u8] { + debug_assert!(self.staged.is_none()); + &self.ops.buffer[..] + } + + /// Updates the branch offsets of all branch instructions inplace. + /// + /// # Panics + /// + /// If this is used before all branching labels have been pinned. + pub fn update_branch_offsets(&mut self) -> Result<(), Error> { + for user in self.labels.resolved_users() { + let ResolvedLabelUser { src, dst, pos } = user; + let offset = trace_branch_offset(src, dst)?; + self.ops.update_branch_offset(pos, offset)?; + } + Ok(()) + } +} + +/// Error indicating that in-place updating of encoded items failed. +struct UpdateEncodedError { + /// The underlying kind of error. + kind: UpdateEncodedErrorKind, + /// The type that is decoded, updated and re-encoded. + marker: PhantomData T>, +} + +impl From for UpdateEncodedError { + fn from(kind: UpdateEncodedErrorKind) -> Self { + Self { + kind, + marker: PhantomData, + } + } +} +impl Clone for UpdateEncodedError { + fn clone(&self) -> Self { + *self + } +} +impl Copy for UpdateEncodedError {} +impl fmt::Debug for UpdateEncodedError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("UpdateEncodedError") + .field("kind", &self.kind) + .field("marker", &self.marker) + .finish() + } +} +impl fmt::Display for UpdateEncodedError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self.kind { + UpdateEncodedErrorKind::BufferOutOfBounds => "buffer out of bounds", + UpdateEncodedErrorKind::FailedToDecode => "failed to decode", + UpdateEncodedErrorKind::FailedToEncode => "failed to encode", + UpdateEncodedErrorKind::FailedToUpdateEncoded => "failed to update encoded", + }; + let type_name = core::any::type_name::(); + write!(f, "{message}: {type_name}") + } +} + +/// Kinds of errors indicating that in-place updating of encoded items failed. +#[derive(Debug, Copy, Clone)] +enum UpdateEncodedErrorKind { + /// Buffer is out of bounds for the position of the update. + BufferOutOfBounds, + /// Failed to decode the encoded item. + FailedToDecode, + /// Failed to encode the updated item. + FailedToEncode, + /// Failed to update the encoded item. + FailedToUpdateEncoded, +} + +impl EncodedOps { + /// Updates the encoded [`BranchOffset`] at `pos` to `offset`. + /// + /// # Panics + /// + /// - If `pos` was out of bounds for `self`. + /// - If the [`BranchOffset`] at `pos` failed to be decoded, updated or re-encoded. + pub fn update_branch_offset( + &mut self, + pos: Pos, + offset: BranchOffset, + ) -> Result<(), Error> { + self.update_encoded(pos, |_| Some(offset)); + Ok(()) + } + + /// Updates an encoded value `v` of type `T` at `pos` in-place using the result of `f(v)`. + /// + /// # Panics + /// + /// - If the underlying bytes buffer is out of bounds for `pos`. + /// - If decodiing of `T` at `pos` fails. + /// - If encodiing of `T` at `pos` fails. + fn update_encoded(&mut self, pos: Pos, f: impl FnOnce(T) -> Option) + where + T: ir::Encode + ir::Decode, + { + if let Err(error) = self + .update_encoded_or_err(pos, f) + .map_err(>::from) + { + panic!("`OpEncoder::update_encoded` unexpectedly failed: {error}") + } + } + + /// Updates a value of type `T` at `pos` using `f` in the encoded buffer. + /// + /// # Errors + /// + /// - If the underlying bytes buffer is out of bounds for `pos`. + /// - If decodiing of `T` at `pos` fails. + /// - If encodiing of `T` at `pos` fails. + /// - If `f(value)` returns `None` and thus updating failed. + fn update_encoded_or_err( + &mut self, + pos: Pos, + f: impl FnOnce(T) -> Option, + ) -> Result<(), UpdateEncodedErrorKind> + where + T: ir::Decode + ir::Encode, + { + let at = usize::from(BytePos::from(pos)); + let Some(buffer) = self.buffer.get_mut(at..) else { + return Err(UpdateEncodedErrorKind::BufferOutOfBounds); + }; + let Ok(decoded) = T::decode(&mut &buffer[..]) else { + return Err(UpdateEncodedErrorKind::FailedToDecode); + }; + let Some(updated) = f(decoded) else { + return Err(UpdateEncodedErrorKind::FailedToUpdateEncoded); + }; + if updated.encode(&mut SliceEncoder::from(buffer)).is_err() { + return Err(UpdateEncodedErrorKind::FailedToEncode); + } + Ok(()) + } +} + +/// Utility type to encode items to a slice of bytes. +pub struct SliceEncoder<'a> { + /// The underlying bytes that will store the encoded items. + bytes: &'a mut [u8], +} + +/// An error that may occur upon encoding items to a byte slice. +#[derive(Debug, Copy, Clone)] +pub struct SliceEncoderError; + +impl<'a> From<&'a mut [u8]> for SliceEncoder<'a> { + fn from(bytes: &'a mut [u8]) -> Self { + Self { bytes } + } +} + +impl<'a> ir::Encoder for SliceEncoder<'a> { + type Pos = (); + type Error = SliceEncoderError; + + fn write_bytes(&mut self, bytes: &[u8]) -> Result { + let Some(buffer) = self.bytes.get_mut(..bytes.len()) else { + return Err(SliceEncoderError); + }; + buffer.copy_from_slice(bytes); + Ok(()) + } + + fn encode_op_code(&mut self, code: OpCode) -> Result { + encode_op_code(self, code) + } + + fn branch_offset( + &mut self, + _pos: Self::Pos, + _branch_offset: BranchOffset, + ) -> Result<(), Self::Error> { + Ok(()) + } + + fn block_fuel(&mut self, _pos: Self::Pos, _block_fuel: BlockFuel) -> Result<(), Self::Error> { + Ok(()) + } +} + +/// Convenience trait to wrap type usable as fuel costs selectors. +pub trait FuelCostsSelector { + /// Selects the fuel usage from the [`FuelCostsProvider`]. + fn select(self, costs: &FuelCostsProvider) -> FuelUsed; +} + +impl FuelCostsSelector for T +where + T: FnOnce(&FuelCostsProvider) -> FuelUsed, +{ + fn select(self, costs: &FuelCostsProvider) -> FuelUsed { + self(costs) + } +} + +impl FuelCostsSelector for BlockFuel { + fn select(self, _costs: &FuelCostsProvider) -> FuelUsed { + FuelUsed::from(self) + } +} + +impl FuelCostsSelector for FuelUsed { + fn select(self, _costs: &FuelCostsProvider) -> FuelUsed { + self + } +} + +/// Encodes an [`OpCode`] to a generic [`ir::Encoder`]. +fn encode_op_code(encoder: &mut E, code: OpCode) -> Result { + match cfg!(feature = "indirect-dispatch") { + true => { + // Note: encoding for indirect-threading + // + // The op-codes are not resolved during translation time and must + // be resolved during execution time. This decreases memory footprint + // of the encoded IR at the cost of execution performance. + u16::from(code).encode(encoder) + } + false => { + // Note: encoding for direct-threading + // + // The op-codes are resolved during translation time (now) to their + // underlying function pointers. This increases memory footprint + // of the encoded IR but improves execution performance. + (op_code_to_handler(code) as usize).encode(encoder) + } + } +} + +/// Creates an initialized [`BranchOffset`] from `src` to `dst`. +/// +/// # Errors +/// +/// If the resulting [`BranchOffset`] is out of bounds. +fn trace_branch_offset(src: BytePos, dst: Pos) -> Result { + fn trace_offset_or_none(src: BytePos, dst: BytePos) -> Option { + let src = isize::try_from(usize::from(src)).ok()?; + let dst = isize::try_from(usize::from(dst)).ok()?; + let offset = dst.checked_sub(src)?; + i32::try_from(offset).map(BranchOffset::from).ok() + } + let Some(offset) = trace_offset_or_none(src, BytePos::from(dst)) else { + return Err(Error::from(TranslationError::BranchOffsetOutOfBounds)); + }; + Ok(offset) +} diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs deleted file mode 100644 index 3b28c02847a..00000000000 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ /dev/null @@ -1,403 +0,0 @@ -use super::{Reset, ReusableAllocations}; -use crate::{ - core::FuelCostsProvider, - engine::translator::{ - comparator::{ - CmpSelectFusion, - CompareResult as _, - TryIntoCmpSelectInstr as _, - UpdateBranchOffset as _, - }, - func::{Operand, Stack, StackLayout, StackSpace}, - relink_result::RelinkResult, - utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, - }, - ir::{BranchOffset, Op, Slot}, - module::ModuleHeader, - Engine, - Error, - ValType, -}; -use alloc::vec::{self, Vec}; - -/// Creates and encodes the list of [`Op`]s for a function. -#[derive(Debug, Default)] -pub struct InstrEncoder { - /// The list of constructed instructions and their parameters. - instrs: Vec, - /// The fuel costs of instructions. - /// - /// This is `Some` if fuel metering is enabled, otherwise `None`. - fuel_costs: Option, - /// The last pushed non-parameter [`Op`]. - last_instr: Option, -} - -impl ReusableAllocations for InstrEncoder { - type Allocations = InstrEncoderAllocations; - - fn into_allocations(self) -> Self::Allocations { - Self::Allocations { - instrs: self.instrs, - } - } -} - -/// The reusable heap allocations of the [`InstrEncoder`]. -#[derive(Debug, Default)] -pub struct InstrEncoderAllocations { - /// The list of constructed instructions and their parameters. - instrs: Vec, -} - -impl Reset for InstrEncoderAllocations { - fn reset(&mut self) { - self.instrs.clear(); - } -} - -impl InstrEncoder { - /// Creates a new [`InstrEncoder`]. - pub fn new(engine: &Engine, alloc: InstrEncoderAllocations) -> Self { - let config = engine.config(); - let fuel_costs = config - .get_consume_fuel() - .then(|| config.fuel_costs()) - .cloned(); - Self { - instrs: alloc.instrs, - fuel_costs, - last_instr: None, - } - } - - /// Returns the next [`Instr`]. - #[must_use] - pub fn next_instr(&self) -> Instr { - Instr::from_usize(self.instrs.len()) - } - - /// Pushes an [`Op::ConsumeFuel`] instruction to `self`. - /// - /// # Note - /// - /// The pushes [`Op::ConsumeFuel`] is initialized with base fuel costs. - pub fn push_consume_fuel_instr(&mut self) -> Result, Error> { - let Some(fuel_costs) = &self.fuel_costs else { - return Ok(None); - }; - let base_costs = fuel_costs.base(); - let Ok(base_costs) = u32::try_from(base_costs) else { - panic!("out of bounds base fuel costs: {base_costs}"); - }; - let instr = self.push_instr_impl(Op::consume_fuel(base_costs))?; - Ok(Some(instr)) - } - - /// Pushes a non-parameter [`Op`] to the [`InstrEncoder`]. - /// - /// Returns an [`Instr`] that refers to the pushed [`Op`]. - pub fn push_instr( - &mut self, - instruction: Op, - consume_fuel: Option, - f: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result { - self.bump_fuel_consumption(consume_fuel, f)?; - self.push_instr_impl(instruction) - } - - /// Pushes a non-parameter [`Op`] to the [`InstrEncoder`]. - fn push_instr_impl(&mut self, instruction: Op) -> Result { - debug_assert!( - !instruction.is_instruction_parameter(), - "parameter: {instruction:?}" - ); - let instr = self.next_instr(); - self.instrs.push(instruction); - self.last_instr = Some(instr); - Ok(instr) - } - - /// Replaces `instr` with `new_instr` in `self`. - /// - /// - Returns `Ok(true)` if replacement was successful. - /// - Returns `Ok(false)` if replacement was unsuccessful. - /// - /// # Panics (Debug) - /// - /// If `instr` or `new_instr` are [`Op`] parameters. - pub fn try_replace_instr(&mut self, instr: Instr, new_instr: Op) -> Result { - debug_assert!( - !new_instr.is_instruction_parameter(), - "parameter: {new_instr:?}" - ); - let Some(last_instr) = self.last_instr else { - return Ok(false); - }; - let replace = self.get_mut(instr); - debug_assert!(!replace.is_instruction_parameter(), "parameter: {instr:?}"); - if instr != last_instr { - return Ok(false); - } - *replace = new_instr; - Ok(true) - } - - /// Tries to replace the result of the last instruction with `new_result` if possible. - /// - /// # Note - /// - /// - `old_result`: - /// just required for additional safety to check if the last instruction - /// really is the source of the `local.set` or `local.tee`. - /// - `new_result`: - /// the new result which shall replace the `old_result`. - pub fn try_replace_result( - &mut self, - new_result: Slot, - old_result: Slot, - layout: &StackLayout, - module: &ModuleHeader, - ) -> Result { - if !matches!(layout.stack_space(new_result), StackSpace::Local) { - // Case: cannot replace result if `new_result` isn't a local. - return Ok(false); - } - let Some(last_instr) = self.last_instr else { - // Case: cannot replace result without last instruction. - return Ok(false); - }; - if !self - .get_mut(last_instr) - .relink_result(module, new_result, old_result)? - { - // Case: it was impossible to relink the result of `last_instr. - return Ok(false); - } - Ok(true) - } - - /// Tries to fuse a compare instruction with a Wasm `select` instruction. - /// - /// # Returns - /// - /// - Returns `Some` if fusion was successful. - /// - Returns `None` if fusion could not be applied. - pub fn try_fuse_select( - &mut self, - ty: ValType, - select_condition: Slot, - layout: &StackLayout, - stack: &mut Stack, - ) -> Result, Error> { - let Some(last_instr) = self.last_instr else { - // If there is no last instruction there is no comparison instruction to negate. - return Ok(None); - }; - let last_instruction = self.get(last_instr); - let Some(last_result) = last_instruction.compare_result() else { - // All negatable instructions have a single result register. - return Ok(None); - }; - if matches!(layout.stack_space(last_result), StackSpace::Local) { - // The instruction stores its result into a local variable which - // is an observable side effect which we are not allowed to mutate. - return Ok(None); - } - if last_result != select_condition { - // The result of the last instruction and the select's `condition` - // are not equal thus indicating that we cannot fuse the instructions. - return Ok(None); - } - let CmpSelectFusion::Applied { - fused, - swap_operands, - } = last_instruction.try_into_cmp_select_instr(|| { - let select_result = stack.push_temp(ty, Some(last_instr))?; - let select_result = layout.temp_to_reg(select_result)?; - Ok(select_result) - })? - else { - return Ok(None); - }; - let last_instr = self.get_mut(last_instr); - *last_instr = fused; - Ok(Some(swap_operands)) - } - - /// Pushes an [`Op`] parameter to the [`InstrEncoder`]. - /// - /// The parameter is associated to the last pushed [`Op`]. - pub fn push_param(&mut self, instruction: Op) { - self.instrs.push(instruction); - } - - /// Returns a shared reference to the [`Op`] associated to [`Instr`]. - /// - /// # Panics - /// - /// If `instr` is out of bounds for `self`. - pub fn get(&self, instr: Instr) -> &Op { - &self.instrs[instr.into_usize()] - } - - /// Returns an exclusive reference to the [`Op`] associated to [`Instr`]. - /// - /// # Panics - /// - /// If `instr` is out of bounds for `self`. - fn get_mut(&mut self, instr: Instr) -> &mut Op { - &mut self.instrs[instr.into_usize()] - } - - /// Resets the [`Instr`] last created via [`InstrEncoder::push_instr`]. - /// - /// # Note - /// - /// The `last_instr` info is used for op-code fusion of `local.set` - /// `local.tee`, compare, conditional branch and `select` instructions. - /// - /// Whenever ending a control frame during Wasm translation `last_instr` - /// needs to be reset to `None` to signal that no such optimization is - /// valid across control flow boundaries. - pub fn reset_last_instr(&mut self) { - self.last_instr = None; - } - - /// Updates the branch offset of `instr` to `offset`. - /// - /// # Errors - /// - /// If the branch offset could not be updated for `instr`. - pub fn update_branch_offset( - &mut self, - instr: Instr, - offset: BranchOffset, - layout: &mut StackLayout, - ) -> Result<(), Error> { - self.get_mut(instr).update_branch_offset(layout, offset)?; - Ok(()) - } - - /// Bumps consumed fuel for [`Op::ConsumeFuel`] of `instr` by `delta`. - /// - /// # Errors - /// - /// If consumed fuel is out of bounds after this operation. - pub fn bump_fuel_consumption( - &mut self, - consume_fuel: Option, - f: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result<(), Error> { - let (fuel_costs, consume_fuel) = match (&self.fuel_costs, consume_fuel) { - (None, None) => return Ok(()), - (Some(fuel_costs), Some(consume_fuel)) => (fuel_costs, consume_fuel), - _ => { - panic!( - "fuel metering state mismatch: fuel_costs: {:?}, fuel_instr: {:?}", - self.fuel_costs, consume_fuel, - ); - } - }; - let fuel_consumed = f(fuel_costs); - self.get_mut(consume_fuel) - .bump_fuel_consumption(fuel_consumed)?; - Ok(()) - } - - /// Encode the top-most `len` operands on the stack as register list. - /// - /// # Note - /// - /// This is used for the following n-ary instructions: - /// - /// - [`Op::ReturnMany`] - /// - [`Op::CopyMany`] - /// - [`Op::CallInternal`] - /// - [`Op::CallImported`] - /// - [`Op::CallIndirect`] - /// - [`Op::ReturnCallInternal`] - /// - [`Op::ReturnCallImported`] - /// - [`Op::ReturnCallIndirect`] - pub fn encode_register_list( - &mut self, - operands: &[Operand], - layout: &mut StackLayout, - ) -> Result<(), Error> { - let mut remaining = operands; - let mut operand_to_reg = - |operand: &Operand| -> Result { layout.operand_to_reg(*operand) }; - let instr = loop { - match remaining { - [] => return Ok(()), - [v0] => { - let v0 = operand_to_reg(v0)?; - break Op::slot(v0); - } - [v0, v1] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - break Op::slot2_ext(v0, v1); - } - [v0, v1, v2] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - let v2 = operand_to_reg(v2)?; - break Op::slot3_ext(v0, v1, v2); - } - [v0, v1, v2, rest @ ..] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - let v2 = operand_to_reg(v2)?; - let instr = Op::slot_list_ext(v0, v1, v2); - self.push_param(instr); - remaining = rest; - } - }; - }; - self.push_param(instr); - Ok(()) - } - - /// Returns an iterator yielding all [`Op`]s of the [`InstrEncoder`]. - /// - /// # Note - /// - /// The [`InstrEncoder`] will be empty after this operation. - pub fn drain(&mut self) -> InstrEncoderIter<'_> { - InstrEncoderIter { - iter: self.instrs.drain(..), - } - } - - /// Returns the last instruction of the [`InstrEncoder`] if any. - pub fn last_instr(&self) -> Option { - self.last_instr - } -} - -/// Iterator yielding all [`Op`]s of the [`InstrEncoder`]. -#[derive(Debug)] -pub struct InstrEncoderIter<'a> { - /// The underlying iterator. - iter: vec::Drain<'a, Op>, -} - -impl<'a> Iterator for InstrEncoderIter<'a> { - type Item = Op; - - fn next(&mut self) -> Option { - self.iter.next() - } - - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl ExactSizeIterator for InstrEncoderIter<'_> { - fn len(&self) -> usize { - self.iter.len() - } -} diff --git a/crates/wasmi/src/engine/translator/func/labels.rs b/crates/wasmi/src/engine/translator/func/labels.rs new file mode 100644 index 00000000000..ef18338335b --- /dev/null +++ b/crates/wasmi/src/engine/translator/func/labels.rs @@ -0,0 +1,238 @@ +use crate::{ + engine::translator::func::{ + encoder::{BytePos, Pos}, + utils::Reset, + }, + ir::{BranchOffset, Op}, +}; +use alloc::vec::Vec; +use core::{ + error::Error as CoreError, + fmt::{self, Display}, + slice::Iter as SliceIter, +}; + +/// A reference to an [`Label`]. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct LabelRef(usize); + +impl From for LabelRef { + #[inline] + fn from(value: usize) -> Self { + Self(value) + } +} + +impl From for usize { + #[inline] + fn from(value: LabelRef) -> Self { + value.0 + } +} + +/// The label registry. +/// +/// Allows to allocate new labels pin them and resolve pinned ones. +#[derive(Debug, Default)] +pub struct LabelRegistry { + /// Registered labels and their states: pinned or unpinned. + labels: RegisteredLabels, + /// Label users that could not be immediately resolved. + users: Vec, +} + +/// All registered labels. +#[derive(Debug, Default)] +pub struct RegisteredLabels { + labels: Vec

{ let memory_type = *self .module - .get_type_of_memory(MemoryIdx::from(u32::from(mem))); + .get_type_of_memory(MemoryIdx::from(u32::from(u16::from(mem)))); let ptr = match memory_type.is_64() { true => u64::from(ptr), false => u64::from(u32::from(ptr)), @@ -2848,7 +2313,13 @@ impl FuncTranslator { /// Translates a Wasm `i64.binop128` instruction from the `wide-arithmetic` proposal. fn translate_i64_binop128( &mut self, - make_instr: fn(results: [Slot; 2], lhs_lo: Slot) -> Op, + make_instr: fn( + results: FixedSlotSpan<2>, + lhs_lo: Slot, + lhs_hi: Slot, + rhs_lo: Slot, + rhs_hi: Slot, + ) -> Op, const_eval: fn(lhs_lo: i64, lhs_hi: i64, rhs_lo: i64, rhs_hi: i64) -> (i64, i64), ) -> Result<(), Error> { bail_unreachable!(self); @@ -2871,19 +2342,22 @@ impl FuncTranslator { self.stack.push_immediate(result_hi)?; return Ok(()); } - let rhs_lo = self.layout.operand_to_reg(rhs_lo)?; - let rhs_hi = self.layout.operand_to_reg(rhs_hi)?; - let lhs_lo = self.layout.operand_to_reg(lhs_lo)?; - let lhs_hi = self.layout.operand_to_reg(lhs_hi)?; - let result_lo = self.stack.push_temp(ValType::I64, None)?; - let result_hi = self.stack.push_temp(ValType::I64, None)?; - let result_lo = self.layout.temp_to_reg(result_lo)?; - let result_hi = self.layout.temp_to_reg(result_hi)?; + let rhs_lo = self.copy_if_immediate(rhs_lo)?; + let rhs_hi = self.copy_if_immediate(rhs_hi)?; + let lhs_lo = self.copy_if_immediate(lhs_lo)?; + let lhs_hi = self.copy_if_immediate(lhs_hi)?; + let result_lo = self.stack.push_temp(ValType::I64)?; + let result_hi = self.stack.push_temp(ValType::I64)?; + let result_lo = self.layout.temp_to_slot(result_lo)?; + let result_hi = self.layout.temp_to_slot(result_hi)?; + let Ok(results) = >::new(SlotSpan::new(result_lo)) else { + return Err(Error::from(TranslationError::AllocatedTooManySlots)); + }; + debug_assert_eq!(results.to_array(), [result_lo, result_hi]); self.push_instr( - make_instr([result_lo, result_hi], lhs_lo), + make_instr(results, lhs_lo, lhs_hi, rhs_lo, rhs_hi), FuelCostsProvider::base, )?; - self.push_param(Op::slot3_ext(lhs_hi, rhs_lo, rhs_hi))?; Ok(()) } @@ -2903,33 +2377,33 @@ impl FuncTranslator { self.stack.push_immediate(result_hi)?; return Ok(()); } - (lhs, Operand::Immediate(rhs)) => { - let rhs = rhs.val(); - if self.try_opt_i64_mul_wide_sx(lhs, rhs, signed)? { + (lhs, Operand::Immediate(rhs_imm)) => { + let rhs_val = rhs_imm.val(); + if self.try_opt_i64_mul_wide_sx(lhs, rhs_val, signed)? { return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.const_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.copy_if_immediate(rhs)?; (lhs, rhs) } - (Operand::Immediate(lhs), rhs) => { - let lhs = lhs.val(); - if self.try_opt_i64_mul_wide_sx(rhs, lhs, signed)? { + (Operand::Immediate(lhs_imm), rhs) => { + let lhs_val = lhs_imm.val(); + if self.try_opt_i64_mul_wide_sx(rhs, lhs_val, signed)? { return Ok(()); } - let lhs = self.layout.const_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.copy_if_immediate(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; (lhs, rhs) } (lhs, rhs) => { - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; (lhs, rhs) } }; - let result0 = self.stack.push_temp(ValType::I64, None)?; - let _result1 = self.stack.push_temp(ValType::I64, None)?; - let result0 = self.layout.temp_to_reg(result0)?; + let result0 = self.stack.push_temp(ValType::I64)?; + let _result1 = self.stack.push_temp(ValType::I64)?; + let result0 = self.layout.temp_to_slot(result0)?; let Ok(results) = >::new(SlotSpan::new(result0)) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; @@ -2961,7 +2435,7 @@ impl FuncTranslator { if matches!(lhs, Operand::Temp(_)) { // Case: `lhs` is temporary and thus might need a copy to its new result. let consume_fuel_instr = self.stack.consume_fuel_instr(); - let result = self.layout.temp_to_reg(result)?; + let result = self.layout.temp_to_slot(result)?; self.encode_copy(result, lhs, consume_fuel_instr)?; } self.stack.push_immediate(0_i64)?; // hi-bits diff --git a/crates/wasmi/src/engine/translator/func/op.rs b/crates/wasmi/src/engine/translator/func/op.rs index 24e2ba536c5..eb090f18ee1 100644 --- a/crates/wasmi/src/engine/translator/func/op.rs +++ b/crates/wasmi/src/engine/translator/func/op.rs @@ -1,160 +1,359 @@ -use crate::ir::{Address32, Offset16, Offset64Lo, Op, Slot}; +use crate::{ + core::Typed, + engine::translator::utils::{ToBits, Wrap}, + ir::{index::Memory, Address, Offset16, Op, Slot}, + ValType, +}; /// Trait implemented by all Wasm operators that can be translated as wrapping store instructions. -pub trait StoreWrapOperator { +pub trait StoreOperator { /// The type of the value to the stored. - type Value; - /// The type of the wrapped value. - type Wrapped; - /// The type of the value as (at most) 16-bit encoded instruction parameter. - type Param; - - fn store(ptr: Slot, offset_lo: Offset64Lo) -> Op; - fn store_imm(ptr: Slot, offset_lo: Offset64Lo) -> Op; - fn store_offset16(ptr: Slot, offset: Offset16, value: Slot) -> Op; - fn store_offset16_imm(ptr: Slot, offset: Offset16, value: Self::Param) -> Op; - fn store_at(value: Slot, address: Address32) -> Op; - fn store_at_imm(value: Self::Param, address: Address32) -> Op; + type Value: Typed; + /// The type of immediate values. + type Immediate; + + /// Converts the value into the immediate value type. + /// + /// # Examples + /// + /// - Wrapping for wrapping stores. + /// - Conversion to bits type or identity for normal stores. + fn into_immediate(value: Self::Value) -> Self::Immediate; + + fn store_ss(ptr: Slot, offset: u64, value: Slot, memory: Memory) -> Op; + fn store_si(ptr: Slot, offset: u64, value: Self::Immediate, memory: Memory) -> Op; + fn store_is(address: Address, value: Slot, memory: Memory) -> Op; + fn store_ii(address: Address, value: Self::Immediate, memory: Memory) -> Op; + fn store_mem0_offset16_ss(ptr: Slot, offset: Offset16, value: Slot) -> Op; + fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Immediate) -> Op; } macro_rules! impl_store_wrap { ( $( - impl StoreWrapOperator for $name:ident { + impl StoreOperator for $name:ident { type Value = $value_ty:ty; - type Wrapped = $wrapped_ty:ty; - type Param = $param_ty:ty; - - fn store = $store:expr; - fn store_imm = $store_imm:expr; - fn store_offset16 = $store_offset16:expr; - fn store_offset16_imm = $store_offset16_imm:expr; - fn store_at = $store_at:expr; - fn store_at_imm = $store_at_imm:expr; + type Immediate = $immediate_ty:ty; + + fn into_immediate = $apply:expr; + + fn store_ss = $store_ss:expr; + fn store_si = $store_si:expr; + fn store_is = $store_is:expr; + fn store_ii = $store_ii:expr; + fn store_mem0_offset16_ss = $store_mem0_offset16_ss:expr; + fn store_mem0_offset16_si = $store_mem0_offset16_si:expr; } )* ) => { $( pub enum $name {} - impl StoreWrapOperator for $name { + impl StoreOperator for $name { type Value = $value_ty; - type Wrapped = $wrapped_ty; - type Param = $param_ty; + type Immediate = $immediate_ty; + + fn into_immediate(value: Self::Value) -> Self::Immediate { + $apply(value) + } - fn store(ptr: Slot, offset_lo: Offset64Lo) -> Op { - $store(ptr, offset_lo) + fn store_ss(ptr: Slot, offset: u64, value: Slot, memory: Memory) -> Op { + $store_ss(ptr, offset, value, memory) } - fn store_imm(ptr: Slot, offset_lo: Offset64Lo) -> Op { - $store_imm(ptr, offset_lo) + fn store_si(ptr: Slot, offset: u64, value: Self::Immediate, memory: Memory) -> Op { + $store_si(ptr, offset, value, memory) } - fn store_offset16(ptr: Slot, offset: Offset16, value: Slot) -> Op { - $store_offset16(ptr, offset, value) + fn store_is(address: Address, value: Slot, memory: Memory) -> Op { + $store_is(address, value, memory) } - fn store_offset16_imm(ptr: Slot, offset: Offset16, value: Self::Param) -> Op { - $store_offset16_imm(ptr, offset, value) + fn store_ii(address: Address, value: Self::Immediate, memory: Memory) -> Op { + $store_ii(address, value, memory) } - fn store_at(value: Slot, address: Address32) -> Op { - $store_at(value, address) + fn store_mem0_offset16_ss(ptr: Slot, offset: Offset16, value: Slot) -> Op { + $store_mem0_offset16_ss(ptr, offset, value) } - fn store_at_imm(value: Self::Param, address: Address32) -> Op { - $store_at_imm(value, address) + fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Immediate) -> Op { + $store_mem0_offset16_si(ptr, offset, value) } } )* }; } impl_store_wrap! { - impl StoreWrapOperator for I32Store { + impl StoreOperator for I32Store { type Value = i32; - type Wrapped = i32; - type Param = i16; + type Immediate = u32; - fn store = Op::store32; - fn store_imm = Op::i32_store_imm16; - fn store_offset16 = Op::store32_offset16; - fn store_offset16_imm = Op::i32_store_offset16_imm16; - fn store_at = Op::store32_at; - fn store_at_imm = Op::i32_store_at_imm16; + fn into_immediate = ::to_bits; + fn store_ss = Op::store32_ss; + fn store_si = Op::store32_si; + fn store_is = Op::store32_is; + fn store_ii = Op::store32_ii; + fn store_mem0_offset16_ss = Op::store32_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store32_mem0_offset16_si; } - impl StoreWrapOperator for I64Store { + impl StoreOperator for I64Store { type Value = i64; - type Wrapped = i64; - type Param = i16; + type Immediate = u64; - fn store = Op::store64; - fn store_imm = Op::i64_store_imm16; - fn store_offset16 = Op::store64_offset16; - fn store_offset16_imm = Op::i64_store_offset16_imm16; - fn store_at = Op::store64_at; - fn store_at_imm = Op::i64_store_at_imm16; + fn into_immediate = ::to_bits; + fn store_ss = Op::store64_ss; + fn store_si = Op::store64_si; + fn store_is = Op::store64_is; + fn store_ii = Op::store64_ii; + fn store_mem0_offset16_ss = Op::store64_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store64_mem0_offset16_si; } - impl StoreWrapOperator for I32Store8 { + impl StoreOperator for F32Store { + type Value = f32; + type Immediate = u32; + + fn into_immediate = ::to_bits; + fn store_ss = Op::store32_ss; + fn store_si = Op::store32_si; + fn store_is = Op::store32_is; + fn store_ii = Op::store32_ii; + fn store_mem0_offset16_ss = Op::store32_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store32_mem0_offset16_si; + } + + impl StoreOperator for F64Store { + type Value = f64; + type Immediate = u64; + + fn into_immediate = ::to_bits; + fn store_ss = Op::store64_ss; + fn store_si = Op::store64_si; + fn store_is = Op::store64_is; + fn store_ii = Op::store64_ii; + fn store_mem0_offset16_ss = Op::store64_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store64_mem0_offset16_si; + } + + impl StoreOperator for I32Store8 { type Value = i32; - type Wrapped = i8; - type Param = i8; + type Immediate = i8; - fn store = Op::i32_store8; - fn store_imm = Op::i32_store8_imm; - fn store_offset16 = Op::i32_store8_offset16; - fn store_offset16_imm = Op::i32_store8_offset16_imm; - fn store_at = Op::i32_store8_at; - fn store_at_imm = Op::i32_store8_at_imm; + fn into_immediate = >::wrap; + fn store_ss = Op::i32_store8_ss; + fn store_si = Op::i32_store8_si; + fn store_is = Op::i32_store8_is; + fn store_ii = Op::i32_store8_ii; + fn store_mem0_offset16_ss = Op::i32_store8_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i32_store8_mem0_offset16_si; } - impl StoreWrapOperator for I32Store16 { + impl StoreOperator for I32Store16 { type Value = i32; - type Wrapped = i16; - type Param = i16; + type Immediate = i16; - fn store = Op::i32_store16; - fn store_imm = Op::i32_store16_imm; - fn store_offset16 = Op::i32_store16_offset16; - fn store_offset16_imm = Op::i32_store16_offset16_imm; - fn store_at = Op::i32_store16_at; - fn store_at_imm = Op::i32_store16_at_imm; + fn into_immediate = >::wrap; + fn store_ss = Op::i32_store16_ss; + fn store_si = Op::i32_store16_si; + fn store_is = Op::i32_store16_is; + fn store_ii = Op::i32_store16_ii; + fn store_mem0_offset16_ss = Op::i32_store16_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i32_store16_mem0_offset16_si; } - impl StoreWrapOperator for I64Store8 { + impl StoreOperator for I64Store8 { type Value = i64; - type Wrapped = i8; - type Param = i8; + type Immediate = i8; - fn store = Op::i64_store8; - fn store_imm = Op::i64_store8_imm; - fn store_offset16 = Op::i64_store8_offset16; - fn store_offset16_imm = Op::i64_store8_offset16_imm; - fn store_at = Op::i64_store8_at; - fn store_at_imm = Op::i64_store8_at_imm; + fn into_immediate = >::wrap; + fn store_ss = Op::i64_store8_ss; + fn store_si = Op::i64_store8_si; + fn store_is = Op::i64_store8_is; + fn store_ii = Op::i64_store8_ii; + fn store_mem0_offset16_ss = Op::i64_store8_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i64_store8_mem0_offset16_si; } - impl StoreWrapOperator for I64Store16 { + impl StoreOperator for I64Store16 { type Value = i64; - type Wrapped = i16; - type Param = i16; + type Immediate = i16; - fn store = Op::i64_store16; - fn store_imm = Op::i64_store16_imm; - fn store_offset16 = Op::i64_store16_offset16; - fn store_offset16_imm = Op::i64_store16_offset16_imm; - fn store_at = Op::i64_store16_at; - fn store_at_imm = Op::i64_store16_at_imm; + fn into_immediate = >::wrap; + fn store_ss = Op::i64_store16_ss; + fn store_si = Op::i64_store16_si; + fn store_is = Op::i64_store16_is; + fn store_ii = Op::i64_store16_ii; + fn store_mem0_offset16_ss = Op::i64_store16_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i64_store16_mem0_offset16_si; } - impl StoreWrapOperator for I64Store32 { + impl StoreOperator for I64Store32 { type Value = i64; - type Wrapped = i32; - type Param = i16; - - fn store = Op::i64_store32; - fn store_imm = Op::i64_store32_imm16; - fn store_offset16 = Op::i64_store32_offset16; - fn store_offset16_imm = Op::i64_store32_offset16_imm16; - fn store_at = Op::i64_store32_at; - fn store_at_imm = Op::i64_store32_at_imm16; + type Immediate = i32; + + fn into_immediate = >::wrap; + fn store_ss = Op::i64_store32_ss; + fn store_si = Op::i64_store32_si; + fn store_is = Op::i64_store32_is; + fn store_ii = Op::i64_store32_ii; + fn store_mem0_offset16_ss = Op::i64_store32_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i64_store32_mem0_offset16_si; + } +} + +/// Trait implemented by all Wasm operators that can be translated as load extend instructions. +pub trait LoadOperator { + /// The type of the loaded value. + const LOADED_TY: ValType; + + fn load_ss(result: Slot, ptr: Slot, offset: u64, memory: Memory) -> Op; + fn load_si(_address: Address, _memory: Memory) -> Option Op> { + Op>>::None + } + fn load_mem0_offset16_ss(result: Slot, ptr: Slot, offset: Offset16) -> Op; +} + +macro_rules! impl_load_extend { + ( $( + impl LoadOperator for $name:ident { + const LOADED_TY: ValType = $loaded_ty:expr; + + fn load_ss = $store_ss:expr; + $( fn load_si = $store_si:expr; )? + fn load_mem0_offset16_ss = $store_mem0_offset16_ss:expr; + } + )* ) => { + $( + pub enum $name {} + impl LoadOperator for $name { + const LOADED_TY: ValType = $loaded_ty; + + fn load_ss(result: Slot, ptr: Slot, offset: u64, memory: Memory) -> Op { + $store_ss(result, ptr, offset, memory) + } + + $( + fn load_si(address: Address, memory: Memory) -> Option Op> { + Some(move |result| $store_si(result, address, memory)) + } + )? + + fn load_mem0_offset16_ss(result: Slot, ptr: Slot, offset: Offset16) -> Op { + $store_mem0_offset16_ss(result, ptr, offset) + } + } + )* + }; +} +impl_load_extend! { + impl LoadOperator for I32Load { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::load32_ss; + fn load_si = Op::load32_si; + fn load_mem0_offset16_ss = Op::load32_mem0_offset16_ss; + } + + impl LoadOperator for I32Load8 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::i32_load8_ss; + fn load_si = Op::i32_load8_si; + fn load_mem0_offset16_ss = Op::i32_load8_mem0_offset16_ss; + } + + impl LoadOperator for U32Load8 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::u32_load8_ss; + fn load_si = Op::u32_load8_si; + fn load_mem0_offset16_ss = Op::u32_load8_mem0_offset16_ss; + } + + impl LoadOperator for I32Load16 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::i32_load16_ss; + fn load_si = Op::i32_load16_si; + fn load_mem0_offset16_ss = Op::i32_load16_mem0_offset16_ss; + } + + impl LoadOperator for U32Load16 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::u32_load16_ss; + fn load_si = Op::u32_load16_si; + fn load_mem0_offset16_ss = Op::u32_load16_mem0_offset16_ss; + } + + impl LoadOperator for I64Load { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::load64_ss; + fn load_si = Op::load64_si; + fn load_mem0_offset16_ss = Op::load64_mem0_offset16_ss; + } + + impl LoadOperator for I64Load8 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::i64_load8_ss; + fn load_si = Op::i64_load8_si; + fn load_mem0_offset16_ss = Op::i64_load8_mem0_offset16_ss; + } + + impl LoadOperator for U64Load8 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::u64_load8_ss; + fn load_si = Op::u64_load8_si; + fn load_mem0_offset16_ss = Op::u64_load8_mem0_offset16_ss; + } + + impl LoadOperator for I64Load16 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::i64_load16_ss; + fn load_si = Op::i64_load16_si; + fn load_mem0_offset16_ss = Op::i64_load16_mem0_offset16_ss; + } + + impl LoadOperator for U64Load16 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::u64_load16_ss; + fn load_si = Op::u64_load16_si; + fn load_mem0_offset16_ss = Op::u64_load16_mem0_offset16_ss; + } + + impl LoadOperator for I64Load32 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::i64_load32_ss; + fn load_si = Op::i64_load32_si; + fn load_mem0_offset16_ss = Op::i64_load32_mem0_offset16_ss; + } + + impl LoadOperator for U64Load32 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::u64_load32_ss; + fn load_si = Op::u64_load32_si; + fn load_mem0_offset16_ss = Op::u64_load32_mem0_offset16_ss; + } + + impl LoadOperator for F32Load { + const LOADED_TY: ValType = ValType::F32; + + fn load_ss = Op::load32_ss; + fn load_si = Op::load32_si; + fn load_mem0_offset16_ss = Op::load32_mem0_offset16_ss; + } + + impl LoadOperator for F64Load { + const LOADED_TY: ValType = ValType::F64; + + fn load_ss = Op::load64_ss; + fn load_si = Op::load64_si; + fn load_mem0_offset16_ss = Op::load64_mem0_offset16_ss; } } diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index f914b835c2b..c41adb01092 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -7,20 +7,15 @@ use crate::{ core::{simd::IntoLaneIdx, FuelCostsProvider, Typed, TypedVal}, engine::translator::{ func::{utils::Input, Operand}, - utils::{Instr, Wrap}, + utils::{IntoShiftAmount, ToBits, Wrap}, }, ir::{ index::{self, Memory}, - Address32, - IntoShiftAmount, - Offset64, - Offset64Lo, - Offset8, + Offset16, Op, Slot, }, Error, - TrapCode, ValType, V128, }; @@ -30,24 +25,24 @@ impl FuncTranslator { /// Generically translate any of the Wasm `simd` splat instructions. fn translate_simd_splat( &mut self, - make_instr: fn(result: Slot, value: Slot) -> Op, - const_eval: fn(Wrapped) -> V128, + make_instr_ss: fn(result: Slot, value: Slot) -> Op, + make_instr_si: fn(result: Slot, value: ::Out) -> Op, ) -> Result<(), Error> where T: From + Wrap, + Wrapped: ToBits, { bail_unreachable!(self); let value = self.stack.pop(); - if let Operand::Immediate(value) = value { - let value = T::from(value.val()).wrap(); - let result = const_eval(value); - self.stack.push_immediate(result)?; - return Ok(()); - }; - let value = self.layout.operand_to_reg(value)?; + let value = self.make_input(value, |_this, value| { + Ok(Input::Immediate(T::from(value).wrap().to_bits())) + })?; self.push_instr_with_result( ValType::V128, - |result| make_instr(result, value), + |result| match value { + Input::Slot(value) => make_instr_ss(result, value), + Input::Immediate(value) => make_instr_si(result, value), + }, FuelCostsProvider::simd, )?; Ok(()) @@ -73,7 +68,7 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); }; - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; self.push_instr_with_result( ::TY, |result| make_instr(result, input, lane), @@ -99,32 +94,18 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let input = self.layout.operand_to_reg(input)?; - let value = - self.make_input::(value, |this, value| { - match T::value_to_imm(T::Item::from(value)) { - Some(imm) => Ok(Input::Immediate(imm)), - None => { - let imm = this.layout.const_to_reg(value)?; - Ok(Input::Slot(imm)) - } - } - })?; - let param = match value { - Input::Slot(value) => Some(Op::slot(value)), - Input::Immediate(value) => T::replace_lane_imm_param(value), - }; + let input = self.layout.operand_to_slot(input)?; + let value = self.make_input::(value, |_this, value| { + Ok(Input::Immediate(T::into_immediate(T::Item::from(value)))) + })?; self.push_instr_with_result( ::TY, |result| match value { - Input::Slot(_) => T::replace_lane(result, input, lane), - Input::Immediate(value) => T::replace_lane_imm(result, input, lane, value), + Input::Slot(value) => T::replace_lane_sss(result, input, lane, value), + Input::Immediate(value) => T::replace_lane_ssi(result, input, lane, value), }, FuelCostsProvider::simd, )?; - if let Some(param) = param { - self.push_param(param)?; - } Ok(()) } @@ -145,7 +126,7 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); }; - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; self.push_instr_with_result( ::TY, |result| make_instr(result, input), @@ -168,8 +149,8 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ValType::V128, |result| make_instr(result, lhs, rhs), @@ -181,7 +162,7 @@ impl FuncTranslator { /// Generically translate a Wasm ternary instruction. fn translate_simd_ternary( &mut self, - make_instr: fn(result: Slot, a: Slot, b: Slot) -> Op, + make_instr: fn(result: Slot, a: Slot, b: Slot, c: Slot) -> Op, const_eval: fn(lhas: V128, b: V128, c: V128) -> V128, ) -> Result<(), Error> { bail_unreachable!(self); @@ -193,27 +174,26 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let lhs = self.layout.operand_to_reg(a)?; - let rhs = self.layout.operand_to_reg(b)?; - let selector = self.layout.operand_to_reg(c)?; + let lhs = self.copy_if_immediate(a)?; + let rhs = self.copy_if_immediate(b)?; + let selector = self.copy_if_immediate(c)?; self.push_instr_with_result( ValType::V128, - |result| make_instr(result, lhs, rhs), + |result| make_instr(result, lhs, rhs, selector), FuelCostsProvider::simd, )?; - self.push_param(Op::slot(selector))?; Ok(()) } /// Generically translate a Wasm SIMD shift instruction. fn translate_simd_shift( &mut self, - make_instr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_instr_imm: fn(result: Slot, lhs: Slot, rhs: ::Output) -> Op, + make_instr_sss: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, + make_instr_ssi: fn(result: Slot, lhs: Slot, rhs: ::Value) -> Op, const_eval: fn(lhs: V128, rhs: u32) -> V128, ) -> Result<(), Error> where - T: IntoShiftAmount>, + T: IntoShiftAmount + From, { bail_unreachable!(self); let (lhs, rhs) = self.stack.pop2(); @@ -229,19 +209,19 @@ impl FuncTranslator { self.stack.push_operand(lhs)?; return Ok(()); }; - let lhs = self.layout.operand_to_reg(lhs)?; + let lhs = self.copy_if_immediate(lhs)?; self.push_instr_with_result( ValType::V128, - |result| make_instr_imm(result, lhs, rhs), + |result| make_instr_ssi(result, lhs, rhs), FuelCostsProvider::simd, )?; return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ValType::V128, - |result| make_instr(result, lhs, rhs), + |result| make_instr_sss(result, lhs, rhs), FuelCostsProvider::simd, )?; Ok(()) @@ -251,72 +231,45 @@ impl FuncTranslator { &mut self, memarg: MemArg, lane: u8, - make_instr: fn(result: Slot, offset_lo: Offset64Lo) -> Op, - make_instr_at: fn(result: Slot, address: Address32) -> Op, + load_lane: fn( + result: Slot, + ptr: Slot, + offset: u64, + memory: index::Memory, + v128: Slot, + lane: T::LaneIdx, + ) -> Op, + load_lane_mem0_offset16: fn( + result: Slot, + ptr: Slot, + offset: Offset16, + v128: Slot, + lane: T::LaneIdx, + ) -> Op, ) -> Result<(), Error> { bail_unreachable!(self); - let (memory, offset) = Self::decode_memarg(memarg); + let (memory, offset) = Self::decode_memarg(memarg)?; let Ok(lane) = ::try_from(lane) else { panic!("encountered out of bounds lane: {lane}"); }; - let (ptr, x) = self.stack.pop2(); - let x = self.layout.operand_to_reg(x)?; - let (ptr, offset) = match ptr { - Operand::Immediate(ptr) => { - let Some(address) = self.effective_address(memory, ptr.val(), offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; - if let Ok(address) = Address32::try_from(address) { - return self.translate_v128_load_lane_at::( - memory, - x, - lane, - address, - make_instr_at, - ); - } - let zero_ptr = self.layout.const_to_reg(0_u64)?; - (zero_ptr, u64::from(address)) - } - ptr => { - let ptr = self.layout.operand_to_reg(ptr)?; - (ptr, offset) + let (ptr, v128) = self.stack.pop2(); + let ptr = self.copy_if_immediate(ptr)?; + let v128 = self.copy_if_immediate(v128)?; + if memory.is_default() { + if let Ok(offset) = Offset16::try_from(offset) { + self.push_instr_with_result( + ::TY, + |result| load_lane_mem0_offset16(result, ptr, offset, v128, lane), + FuelCostsProvider::load, + )?; + return Ok(()); } - }; - let (offset_hi, offset_lo) = Offset64::split(offset); - self.push_instr_with_result( - ::TY, - |result| make_instr(result, offset_lo), - FuelCostsProvider::load, - )?; - self.push_param(Op::slot_and_offset_hi(ptr, offset_hi))?; - self.push_param(Op::slot_and_lane(x, lane))?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; } - Ok(()) - } - - fn translate_v128_load_lane_at( - &mut self, - memory: Memory, - x: Slot, - lane: LaneType, - address: Address32, - make_instr_at: fn(result: Slot, address: Address32) -> Op, - ) -> Result<(), Error> - where - LaneType: Into, - { self.push_instr_with_result( ::TY, - |result| make_instr_at(result, address), + |result| load_lane(result, ptr, offset, memory, v128, lane), FuelCostsProvider::load, )?; - self.push_param(Op::slot_and_lane(x, lane))?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } Ok(()) } @@ -325,9 +278,13 @@ impl FuncTranslator { &mut self, memarg: MemArg, lane: u8, - make_instr: fn(ptr: Slot, offset_lo: Offset64Lo) -> Op, - make_instr_offset8: fn(ptr: Slot, value: Slot, offset: Offset8, lane: T::LaneIdx) -> Op, - make_instr_at: fn(value: Slot, address: Address32) -> Op, + make_instr: fn(ptr: Slot, offset: u64, value: Slot, memory: Memory, lane: T::LaneIdx) -> Op, + make_instr_mem0_offset16: fn( + ptr: Slot, + offset: Offset16, + value: Slot, + lane: T::LaneIdx, + ) -> Op, translate_imm: fn( &mut Self, memarg: MemArg, @@ -347,81 +304,24 @@ impl FuncTranslator { // lane value and translate as a more efficient non-SIMD operation. return translate_imm(self, memarg, ptr, lane, V128::from(v128.val())); } - v128 => self.layout.operand_to_reg(v128)?, + Operand::Local(v128) => self.layout.local_to_slot(v128)?, + Operand::Temp(v128) => self.layout.temp_to_slot(v128)?, }; - let (memory, offset) = Self::decode_memarg(memarg); - let (ptr, offset) = match ptr { - Operand::Immediate(ptr) => { - let Some(address) = self.effective_address(memory, ptr.val(), offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; - if let Ok(address) = Address32::try_from(address) { - return self.translate_v128_store_lane_at::( - memory, - address, - v128, - lane, - make_instr_at, - ); - } - // Case: we cannot use specialized encoding and thus have to fall back - // to the general case where `ptr` is zero and `offset` stores the - // `ptr+offset` address value. - let zero_ptr = self.layout.const_to_reg(0_u64)?; - (zero_ptr, u64::from(address)) - } - ptr => { - let ptr = self.layout.operand_to_reg(ptr)?; - (ptr, offset) + let (memory, offset) = Self::decode_memarg(memarg)?; + let ptr = self.copy_if_immediate(ptr)?; + if memory.is_default() { + if let Ok(offset16) = Offset16::try_from(offset) { + self.push_instr( + make_instr_mem0_offset16(ptr, offset16, v128, lane), + FuelCostsProvider::store, + )?; + return Ok(()); } - }; - if let Ok(Some(_)) = - self.translate_v128_store_lane_mem0(memory, ptr, offset, v128, lane, make_instr_offset8) - { - return Ok(()); } - let (offset_hi, offset_lo) = Offset64::split(offset); - let instr = make_instr(ptr, offset_lo); - let param = Op::slot_and_offset_hi(v128, offset_hi); - let param2 = Op::lane_and_memory_index(lane, memory); - self.push_instr(instr, FuelCostsProvider::store)?; - self.push_param(param)?; - self.push_param(param2)?; - Ok(()) - } - - fn translate_v128_store_lane_at( - &mut self, - memory: index::Memory, - address: Address32, - value: Slot, - lane: T::LaneIdx, - make_instr_at: fn(value: Slot, address: Address32) -> Op, - ) -> Result<(), Error> { - self.push_instr(make_instr_at(value, address), FuelCostsProvider::store)?; - self.push_param(Op::lane_and_memory_index(lane, memory))?; - Ok(()) - } - - fn translate_v128_store_lane_mem0( - &mut self, - memory: Memory, - ptr: Slot, - offset: u64, - value: Slot, - lane: LaneType, - make_instr_offset8: fn(Slot, Slot, Offset8, LaneType) -> Op, - ) -> Result, Error> { - if !memory.is_default() { - return Ok(None); - } - let Ok(offset8) = Offset8::try_from(offset) else { - return Ok(None); - }; - let instr = self.push_instr( - make_instr_offset8(ptr, value, offset8, lane), + self.push_instr( + make_instr(ptr, offset, v128, memory, lane), FuelCostsProvider::store, )?; - Ok(Some(instr)) + Ok(()) } } diff --git a/crates/wasmi/src/engine/translator/func/simd/op.rs b/crates/wasmi/src/engine/translator/func/simd/op.rs index 87556503453..21e548d3721 100644 --- a/crates/wasmi/src/engine/translator/func/simd/op.rs +++ b/crates/wasmi/src/engine/translator/func/simd/op.rs @@ -1,7 +1,9 @@ use super::IntoLaneIdx; use crate::{ core::{simd, Typed}, - ir::{Const32, Op, Slot}, + engine::translator::{func::op::LoadOperator, utils::ToBits}, + ir::{index::Memory, Offset16, Op, Slot}, + ValType, V128, }; @@ -9,24 +11,27 @@ pub trait SimdReplaceLane { type Item: Typed + IntoLaneIdx + Copy; type Immediate: Copy; + fn into_immediate(value: Self::Item) -> Self::Immediate; + fn const_eval( input: V128, lane: ::LaneIdx, value: Self::Item, ) -> V128; - fn replace_lane(result: Slot, input: Slot, lane: ::LaneIdx) -> Op; + fn replace_lane_sss( + result: Slot, + input: Slot, + lane: ::LaneIdx, + value: Slot, + ) -> Op; - fn replace_lane_imm( + fn replace_lane_ssi( result: Slot, input: Slot, lane: ::LaneIdx, value: Self::Immediate, ) -> Op; - - fn replace_lane_imm_param(value: Self::Immediate) -> Option; - - fn value_to_imm(value: Self::Item) -> Option; } macro_rules! impl_replace_lane { @@ -37,10 +42,9 @@ macro_rules! impl_replace_lane { type Immediate = $imm_ty:ty; fn const_eval = $const_eval:expr; - fn replace_lane = $replace_lane:expr; - fn replace_lane_imm = $replace_lane_imm:expr; - fn replace_lane_imm_param = $replace_lane_imm_param:expr; - fn value_to_imm = $value_to_imm:expr; + fn into_immediate = $into_immediate:expr; + fn replace_lane_sss = $replace_lane_sss:expr; + fn replace_lane_ssi = $replace_lane_ssi:expr; } )* ) => { @@ -59,105 +63,180 @@ macro_rules! impl_replace_lane { $const_eval(input, lane, value) } - fn replace_lane( + fn into_immediate(value: Self::Item) -> Self::Immediate { + $into_immediate(value) + } + + fn replace_lane_sss( result: Slot, input: Slot, lane: ::LaneIdx, + value: Slot, ) -> Op { - $replace_lane(result, input, lane) + $replace_lane_sss(result, input, value, lane) } - fn replace_lane_imm( + fn replace_lane_ssi( result: Slot, input: Slot, lane: ::LaneIdx, value: Self::Immediate, ) -> Op { - $replace_lane_imm(result, input, lane, value) - } - - fn replace_lane_imm_param(value: Self::Immediate) -> Option { - $replace_lane_imm_param(value) - } - - fn value_to_imm(value: Self::Item) -> Option { - $value_to_imm(value) + $replace_lane_ssi(result, input, value, lane) } } )* }; } -macro_rules! wrap { - ($f:expr) => { - |result, input, lane, _value| $f(result, input, lane) - }; -} - impl_replace_lane! { impl SimdReplaceLane for I8x16ReplaceLane { type Item = i8; - type Immediate = i8; + type Immediate = u8; fn const_eval = simd::i8x16_replace_lane; - fn replace_lane = Op::i8x16_replace_lane; - fn replace_lane_imm = Op::i8x16_replace_lane_imm; - fn replace_lane_imm_param = |_| None; - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::v128_replace_lane8x16_sss; + fn replace_lane_ssi = Op::v128_replace_lane8x16_ssi; } impl SimdReplaceLane for I16x8ReplaceLane { type Item = i16; - type Immediate = i16; + type Immediate = u16; fn const_eval = simd::i16x8_replace_lane; - fn replace_lane = Op::i16x8_replace_lane; - fn replace_lane_imm = wrap!(Op::i16x8_replace_lane_imm); - fn replace_lane_imm_param = |value| Some(Op::const32(i32::from(value))); - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::v128_replace_lane16x8_sss; + fn replace_lane_ssi = Op::v128_replace_lane16x8_ssi; } impl SimdReplaceLane for I32x4ReplaceLane { type Item = i32; - type Immediate = i32; + type Immediate = u32; fn const_eval = simd::i32x4_replace_lane; - fn replace_lane = Op::i32x4_replace_lane; - fn replace_lane_imm = wrap!(Op::i32x4_replace_lane_imm); - fn replace_lane_imm_param = |value| Some(Op::const32(value)); - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::v128_replace_lane32x4_sss; + fn replace_lane_ssi = Op::v128_replace_lane32x4_ssi; } impl SimdReplaceLane for I64x2ReplaceLane { type Item = i64; - type Immediate = Const32; + type Immediate = u64; fn const_eval = simd::i64x2_replace_lane; - fn replace_lane = Op::i64x2_replace_lane; - fn replace_lane_imm = wrap!(Op::i64x2_replace_lane_imm32); - fn replace_lane_imm_param = |value| Some(Op::i64const32(value)); - fn value_to_imm = |value| >::try_from(value).ok(); + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::v128_replace_lane64x2_sss; + fn replace_lane_ssi = Op::v128_replace_lane64x2_ssi; } impl SimdReplaceLane for F32x4ReplaceLane { type Item = f32; - type Immediate = f32; + type Immediate = u32; fn const_eval = simd::f32x4_replace_lane; - fn replace_lane = Op::f32x4_replace_lane; - fn replace_lane_imm = wrap!(Op::f32x4_replace_lane_imm); - fn replace_lane_imm_param = |value| Some(Op::const32(value)); - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::v128_replace_lane32x4_sss; + fn replace_lane_ssi = Op::v128_replace_lane32x4_ssi; } impl SimdReplaceLane for F64x2ReplaceLane { type Item = f64; - type Immediate = Const32; + type Immediate = u64; fn const_eval = simd::f64x2_replace_lane; - fn replace_lane = Op::f64x2_replace_lane; - fn replace_lane_imm = wrap!(Op::f64x2_replace_lane_imm32); - fn replace_lane_imm_param = |value| Some(Op::f64const32(value)); - fn value_to_imm = |value| >::try_from(value).ok(); + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::v128_replace_lane64x2_sss; + fn replace_lane_ssi = Op::v128_replace_lane64x2_ssi; + } +} + +macro_rules! impl_simd_load { + ( $( + impl LoadOperator for $name:ident { + fn load_ss = $store_ss:expr; + fn load_mem0_offset16_ss = $store_mem0_offset16_ss:expr; + } + )* ) => { + $( + pub enum $name {} + impl LoadOperator for $name { + const LOADED_TY: ValType = ValType::V128; + + fn load_ss(result: Slot, ptr: Slot, offset: u64, memory: Memory) -> Op { + $store_ss(result, ptr, offset, memory) + } + + fn load_mem0_offset16_ss(result: Slot, ptr: Slot, offset: Offset16) -> Op { + $store_mem0_offset16_ss(result, ptr, offset) + } + } + )* + }; +} +impl_simd_load! { + impl LoadOperator for V128Load { + fn load_ss = Op::v128_load_ss; + fn load_mem0_offset16_ss = Op::v128_load_mem0_offset16_ss; + } + + impl LoadOperator for I16x8Load8x8 { + fn load_ss = Op::i16x8_load8x8_ss; + fn load_mem0_offset16_ss = Op::i16x8_load8x8_mem0_offset16_ss; + } + + impl LoadOperator for U16x8Load8x8 { + fn load_ss = Op::u16x8_load8x8_ss; + fn load_mem0_offset16_ss = Op::u16x8_load8x8_mem0_offset16_ss; + } + + impl LoadOperator for I32x4Load16x4 { + fn load_ss = Op::i32x4_load16x4_ss; + fn load_mem0_offset16_ss = Op::i32x4_load16x4_mem0_offset16_ss; + } + + impl LoadOperator for U32x4Load16x4 { + fn load_ss = Op::u32x4_load16x4_ss; + fn load_mem0_offset16_ss = Op::u32x4_load16x4_mem0_offset16_ss; + } + + impl LoadOperator for I64x2Load32x2 { + fn load_ss = Op::i64x2_load32x2_ss; + fn load_mem0_offset16_ss = Op::i64x2_load32x2_mem0_offset16_ss; + } + + impl LoadOperator for U64x2Load32x2 { + fn load_ss = Op::u64x2_load32x2_ss; + fn load_mem0_offset16_ss = Op::u64x2_load32x2_mem0_offset16_ss; + } + + impl LoadOperator for V128Load8Splat { + fn load_ss = Op::v128_load8_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load8_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load16Splat { + fn load_ss = Op::v128_load16_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load16_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load32Splat { + fn load_ss = Op::v128_load32_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load32_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load64Splat { + fn load_ss = Op::v128_load64_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load64_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load32Zero { + fn load_ss = Op::v128_load32_zero_ss; + fn load_mem0_offset16_ss = Op::v128_load32_zero_mem0_offset16_ss; + } + + impl LoadOperator for V128Load64Zero { + fn load_ss = Op::v128_load64_zero_ss; + fn load_mem0_offset16_ss = Op::v128_load64_zero_mem0_offset16_ss; } } diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index a1afea04ac0..5fb56d6daea 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -1,7 +1,7 @@ use super::FuncTranslator; use crate::{ core::{ - simd::{self, ImmLaneIdx32}, + simd::{self, ImmLaneIdx}, FuelCostsProvider, TypedVal, }, @@ -31,150 +31,73 @@ macro_rules! swap_ops { impl VisitSimdOperator<'_> for FuncTranslator { fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load, - Op::v128_load_offset16, - Op::v128_load_at, - ) + self.translate_load::(memarg) } fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load8x8_s, - Op::v128_load8x8_s_offset16, - Op::v128_load8x8_s_at, - ) + self.translate_load::(memarg) } fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load8x8_u, - Op::v128_load8x8_u_offset16, - Op::v128_load8x8_u_at, - ) + self.translate_load::(memarg) } fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load16x4_s, - Op::v128_load16x4_s_offset16, - Op::v128_load16x4_s_at, - ) + self.translate_load::(memarg) } fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load16x4_u, - Op::v128_load16x4_u_offset16, - Op::v128_load16x4_u_at, - ) + self.translate_load::(memarg) } fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load32x2_s, - Op::v128_load32x2_s_offset16, - Op::v128_load32x2_s_at, - ) + self.translate_load::(memarg) } fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load32x2_u, - Op::v128_load32x2_u_offset16, - Op::v128_load32x2_u_at, - ) + self.translate_load::(memarg) } fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load8_splat, - Op::v128_load8_splat_offset16, - Op::v128_load8_splat_at, - ) + self.translate_load::(memarg) } fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load16_splat, - Op::v128_load16_splat_offset16, - Op::v128_load16_splat_at, - ) + self.translate_load::(memarg) } fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load32_splat, - Op::v128_load32_splat_offset16, - Op::v128_load32_splat_at, - ) + self.translate_load::(memarg) } fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load64_splat, - Op::v128_load64_splat_offset16, - Op::v128_load64_splat_at, - ) + self.translate_load::(memarg) } fn visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load32_zero, - Op::v128_load32_zero_offset16, - Op::v128_load32_zero_at, - ) + self.translate_load::(memarg) } fn visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load64_zero, - Op::v128_load64_zero_offset16, - Op::v128_load64_zero_at, - ) + self.translate_load::(memarg) } - fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { - self.translate_store( - memarg, - Op::v128_store, - Op::v128_store_offset16, - Op::v128_store_at, - ) + fn visit_v128_store(&mut self, _memarg: MemArg) -> Self::Output { + // self.translate_store( + // memarg, + // Op::v128_store, + // Op::v128_store_offset16, + // Op::v128_store_at, + // ) + todo!() } fn visit_v128_load8_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load8_lane, - Op::v128_load8_lane_at, + Op::v128_load_lane8_sss, + Op::v128_load_lane8_mem0_offset16_sss, ) } @@ -182,8 +105,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load16_lane, - Op::v128_load16_lane_at, + Op::v128_load_lane16_sss, + Op::v128_load_lane16_mem0_offset16_sss, ) } @@ -191,8 +114,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load32_lane, - Op::v128_load32_lane_at, + Op::v128_load_lane32_sss, + Op::v128_load_lane32_mem0_offset16_sss, ) } @@ -200,8 +123,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load64_lane, - Op::v128_load64_lane_at, + Op::v128_load_lane64_sss, + Op::v128_load_lane64_mem0_offset16_sss, ) } @@ -209,13 +132,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store8_lane, - Op::v128_store8_lane_offset8, - Op::v128_store8_lane_at, + Op::v128_store8_lane_ss, + Op::v128_store8_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i8x16_extract_lane_s(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } @@ -224,13 +146,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store16_lane, - Op::v128_store16_lane_offset8, - Op::v128_store16_lane_at, + Op::v128_store16_lane_ss, + Op::v128_store16_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i16x8_extract_lane_s(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } @@ -239,13 +160,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store32_lane, - Op::v128_store32_lane_offset8, - Op::v128_store32_lane_at, + Op::v128_store32_lane_ss, + Op::v128_store32_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i32x4_extract_lane(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } @@ -254,13 +174,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store64_lane, - Op::v128_store64_lane_offset8, - Op::v128_store64_lane_at, + Op::v128_store64_lane_ss, + Op::v128_store64_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i64x2_extract_lane(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } @@ -274,8 +193,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { bail_unreachable!(self); - let selector: [ImmLaneIdx32; 16] = array::from_fn(|i| { - let Ok(lane) = ImmLaneIdx32::try_from(lanes[i]) else { + let selector: [ImmLaneIdx<32>; 16] = array::from_fn(|i| { + let Ok(lane) = >::try_from(lanes[i]) else { panic!("encountered out of bounds lane at index {i}: {}", lanes[i]) }; lane @@ -286,24 +205,20 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; - let selector = self - .layout - .const_to_reg(V128::from(u128::from_ne_bytes(lanes)))?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ValType::V128, - |result| Op::i8x16_shuffle(result, lhs, rhs), + |result| Op::i8x16_shuffle(result, lhs, rhs, selector), FuelCostsProvider::simd, )?; - self.push_param(Op::slot(selector))?; Ok(()) } fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i8x16_extract_lane_s, + Op::s8x16_extract_lane, simd::i8x16_extract_lane_s, ) } @@ -311,7 +226,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i8x16_extract_lane_u, + Op::u8x16_extract_lane, simd::i8x16_extract_lane_u, ) } @@ -319,7 +234,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i16x8_extract_lane_s, + Op::s16x8_extract_lane, simd::i16x8_extract_lane_s, ) } @@ -327,7 +242,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i16x8_extract_lane_u, + Op::u16x8_extract_lane, simd::i16x8_extract_lane_u, ) } @@ -335,7 +250,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i32x4_extract_lane, + Op::u32x4_extract_lane, simd::i32x4_extract_lane, ) } @@ -343,7 +258,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i64x2_extract_lane, + Op::u64x2_extract_lane, simd::i64x2_extract_lane, ) } @@ -351,7 +266,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::f32x4_extract_lane, + Op::u32x4_extract_lane, simd::f32x4_extract_lane, ) } @@ -359,7 +274,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::f64x2_extract_lane, + Op::u64x2_extract_lane, simd::f64x2_extract_lane, ) } @@ -389,855 +304,897 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_swizzle(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_swizzle, simd::i8x16_swizzle) + self.translate_simd_binary(Op::i8x16_swizzle_sss, simd::i8x16_swizzle) } fn visit_i8x16_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i8x16_splat, simd::i8x16_splat) + self.translate_simd_splat::(Op::v128_splat8_ss, Op::v128_splat8_si) } fn visit_i16x8_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i16x8_splat, simd::i16x8_splat) + self.translate_simd_splat::(Op::v128_splat16_ss, Op::v128_splat16_si) } fn visit_i32x4_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i32x4_splat, simd::i32x4_splat) + self.translate_simd_splat::(Op::v128_splat32_ss, Op::v128_splat32_si) } fn visit_i64x2_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i64x2_splat, simd::i64x2_splat) + self.translate_simd_splat::(Op::v128_splat64_ss, Op::v128_splat64_si) } fn visit_f32x4_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::f32x4_splat, simd::f32x4_splat) + self.translate_simd_splat::(Op::v128_splat32_ss, Op::v128_splat32_si) } fn visit_f64x2_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::f64x2_splat, simd::f64x2_splat) + self.translate_simd_splat::(Op::v128_splat64_ss, Op::v128_splat64_si) } fn visit_i8x16_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_eq, simd::i8x16_eq) + self.translate_simd_binary(Op::i8x16_eq_sss, simd::i8x16_eq) } fn visit_i8x16_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_ne, simd::i8x16_ne) + self.translate_simd_binary(Op::i8x16_not_eq_sss, simd::i8x16_ne) } fn visit_i8x16_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_lt_s, simd::i8x16_lt_s) + self.translate_simd_binary(Op::i8x16_lt_sss, simd::i8x16_lt_s) } fn visit_i8x16_lt_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_lt_u, simd::i8x16_lt_u) + self.translate_simd_binary(Op::u8x16_lt_sss, simd::i8x16_lt_u) } fn visit_i8x16_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_lt_s), simd::i8x16_gt_s) + self.translate_simd_binary(swap_ops!(Op::i8x16_lt_sss), simd::i8x16_gt_s) } fn visit_i8x16_gt_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_lt_u), simd::i8x16_gt_u) + self.translate_simd_binary(swap_ops!(Op::u8x16_lt_sss), simd::i8x16_gt_u) } fn visit_i8x16_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_le_s, simd::i8x16_le_s) + self.translate_simd_binary(Op::i8x16_le_sss, simd::i8x16_le_s) } fn visit_i8x16_le_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_le_u, simd::i8x16_le_u) + self.translate_simd_binary(Op::u8x16_le_sss, simd::i8x16_le_u) } fn visit_i8x16_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_le_s), simd::i8x16_ge_s) + self.translate_simd_binary(swap_ops!(Op::i8x16_le_sss), simd::i8x16_ge_s) } fn visit_i8x16_ge_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_le_u), simd::i8x16_ge_u) + self.translate_simd_binary(swap_ops!(Op::u8x16_le_sss), simd::i8x16_ge_u) } fn visit_i16x8_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_eq, simd::i16x8_eq) + self.translate_simd_binary(Op::i16x8_eq_sss, simd::i16x8_eq) } fn visit_i16x8_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_ne, simd::i16x8_ne) + self.translate_simd_binary(Op::i16x8_not_eq_sss, simd::i16x8_ne) } fn visit_i16x8_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_lt_s, simd::i16x8_lt_s) + self.translate_simd_binary(Op::i16x8_lt_sss, simd::i16x8_lt_s) } fn visit_i16x8_lt_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_lt_u, simd::i16x8_lt_u) + self.translate_simd_binary(Op::u16x8_lt_sss, simd::i16x8_lt_u) } fn visit_i16x8_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_lt_s), simd::i16x8_gt_s) + self.translate_simd_binary(swap_ops!(Op::i16x8_lt_sss), simd::i16x8_gt_s) } fn visit_i16x8_gt_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_lt_u), simd::i16x8_gt_u) + self.translate_simd_binary(swap_ops!(Op::u16x8_lt_sss), simd::i16x8_gt_u) } fn visit_i16x8_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_le_s, simd::i16x8_le_s) + self.translate_simd_binary(Op::i16x8_le_sss, simd::i16x8_le_s) } fn visit_i16x8_le_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_le_u, simd::i16x8_le_u) + self.translate_simd_binary(Op::u16x8_le_sss, simd::i16x8_le_u) } fn visit_i16x8_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_le_s), simd::i16x8_ge_s) + self.translate_simd_binary(swap_ops!(Op::i16x8_le_sss), simd::i16x8_ge_s) } fn visit_i16x8_ge_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_le_u), simd::i16x8_ge_u) + self.translate_simd_binary(swap_ops!(Op::u16x8_le_sss), simd::i16x8_ge_u) } fn visit_i32x4_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_eq, simd::i32x4_eq) + self.translate_simd_binary(Op::i32x4_eq_sss, simd::i32x4_eq) } fn visit_i32x4_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_ne, simd::i32x4_ne) + self.translate_simd_binary(Op::i32x4_not_eq_sss, simd::i32x4_ne) } fn visit_i32x4_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_lt_s, simd::i32x4_lt_s) + self.translate_simd_binary(Op::i32x4_lt_sss, simd::i32x4_lt_s) } fn visit_i32x4_lt_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_lt_u, simd::i32x4_lt_u) + self.translate_simd_binary(Op::u32x4_lt_sss, simd::i32x4_lt_u) } fn visit_i32x4_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_lt_s), simd::i32x4_gt_s) + self.translate_simd_binary(swap_ops!(Op::i32x4_lt_sss), simd::i32x4_gt_s) } fn visit_i32x4_gt_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_lt_u), simd::i32x4_gt_u) + self.translate_simd_binary(swap_ops!(Op::u32x4_lt_sss), simd::i32x4_gt_u) } fn visit_i32x4_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_le_s, simd::i32x4_le_s) + self.translate_simd_binary(Op::i32x4_le_sss, simd::i32x4_le_s) } fn visit_i32x4_le_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_le_u, simd::i32x4_le_u) + self.translate_simd_binary(Op::u32x4_le_sss, simd::i32x4_le_u) } fn visit_i32x4_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_le_s), simd::i32x4_ge_s) + self.translate_simd_binary(swap_ops!(Op::i32x4_le_sss), simd::i32x4_ge_s) } fn visit_i32x4_ge_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_le_u), simd::i32x4_ge_u) + self.translate_simd_binary(swap_ops!(Op::u32x4_le_sss), simd::i32x4_ge_u) } fn visit_i64x2_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_eq, simd::i64x2_eq) + self.translate_simd_binary(Op::i64x2_eq_sss, simd::i64x2_eq) } fn visit_i64x2_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_ne, simd::i64x2_ne) + self.translate_simd_binary(Op::i64x2_not_eq_sss, simd::i64x2_ne) } fn visit_i64x2_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_lt_s, simd::i64x2_lt_s) + self.translate_simd_binary(Op::i64x2_lt_sss, simd::i64x2_lt_s) } fn visit_i64x2_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i64x2_lt_s), simd::i64x2_gt_s) + self.translate_simd_binary(swap_ops!(Op::i64x2_lt_sss), simd::i64x2_gt_s) } fn visit_i64x2_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_le_s, simd::i64x2_le_s) + self.translate_simd_binary(Op::i64x2_le_sss, simd::i64x2_le_s) } fn visit_i64x2_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i64x2_le_s), simd::i64x2_ge_s) + self.translate_simd_binary(swap_ops!(Op::i64x2_le_sss), simd::i64x2_ge_s) } fn visit_f32x4_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_eq, simd::f32x4_eq) + self.translate_simd_binary(Op::f32x4_eq_sss, simd::f32x4_eq) } fn visit_f32x4_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_ne, simd::f32x4_ne) + self.translate_simd_binary(Op::f32x4_not_eq_sss, simd::f32x4_ne) } fn visit_f32x4_lt(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_lt, simd::f32x4_lt) + self.translate_simd_binary(Op::f32x4_lt_sss, simd::f32x4_lt) } fn visit_f32x4_gt(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f32x4_lt), simd::f32x4_gt) + self.translate_simd_binary(swap_ops!(Op::f32x4_lt_sss), simd::f32x4_gt) } fn visit_f32x4_le(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_le, simd::f32x4_le) + self.translate_simd_binary(Op::f32x4_le_sss, simd::f32x4_le) } fn visit_f32x4_ge(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f32x4_le), simd::f32x4_ge) + self.translate_simd_binary(swap_ops!(Op::f32x4_le_sss), simd::f32x4_ge) } fn visit_f64x2_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_eq, simd::f64x2_eq) + self.translate_simd_binary(Op::f64x2_eq_sss, simd::f64x2_eq) } fn visit_f64x2_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_ne, simd::f64x2_ne) + self.translate_simd_binary(Op::f64x2_not_eq_sss, simd::f64x2_ne) } fn visit_f64x2_lt(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_lt, simd::f64x2_lt) + self.translate_simd_binary(Op::f64x2_lt_sss, simd::f64x2_lt) } fn visit_f64x2_gt(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f64x2_lt), simd::f64x2_gt) + self.translate_simd_binary(swap_ops!(Op::f64x2_lt_sss), simd::f64x2_gt) } fn visit_f64x2_le(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_le, simd::f64x2_le) + self.translate_simd_binary(Op::f64x2_le_sss, simd::f64x2_le) } fn visit_f64x2_ge(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f64x2_le), simd::f64x2_ge) + self.translate_simd_binary(swap_ops!(Op::f64x2_le_sss), simd::f64x2_ge) } fn visit_v128_not(&mut self) -> Self::Output { - self.translate_simd_unary(Op::v128_not, simd::v128_not) + self.translate_simd_unary(Op::v128_not_ss, simd::v128_not) } fn visit_v128_and(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_and, simd::v128_and) + self.translate_simd_binary(Op::v128_and_sss, simd::v128_and) } fn visit_v128_andnot(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_andnot, simd::v128_andnot) + self.translate_simd_binary(Op::v128_and_not_sss, simd::v128_andnot) } fn visit_v128_or(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_or, simd::v128_or) + self.translate_simd_binary(Op::v128_or_sss, simd::v128_or) } fn visit_v128_xor(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_xor, simd::v128_xor) + self.translate_simd_binary(Op::v128_xor_sss, simd::v128_xor) } fn visit_v128_bitselect(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::v128_bitselect, simd::v128_bitselect) + self.translate_simd_ternary(Op::v128_bitselect_ssss, simd::v128_bitselect) } fn visit_v128_any_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::v128_any_true, simd::v128_any_true) + self.translate_simd_unary(Op::v128_any_true_ss, simd::v128_any_true) } fn visit_i8x16_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_abs, simd::i8x16_abs) + self.translate_simd_unary(Op::i8x16_abs_ss, simd::i8x16_abs) } fn visit_i8x16_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_neg, simd::i8x16_neg) + self.translate_simd_unary(Op::i8x16_neg_ss, simd::i8x16_neg) } fn visit_i8x16_popcnt(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_popcnt, simd::i8x16_popcnt) + self.translate_simd_unary(Op::i8x16_popcnt_ss, simd::i8x16_popcnt) } fn visit_i8x16_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_all_true, simd::i8x16_all_true) + self.translate_simd_unary(Op::i8x16_all_true_ss, simd::i8x16_all_true) } fn visit_i8x16_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_bitmask, simd::i8x16_bitmask) + self.translate_simd_unary(Op::i8x16_bitmask_ss, simd::i8x16_bitmask) } fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_narrow_i16x8_s, simd::i8x16_narrow_i16x8_s) + self.translate_simd_binary(Op::i8x16_narrow_i16x8_sss, simd::i8x16_narrow_i16x8_s) } fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_narrow_i16x8_u, simd::i8x16_narrow_i16x8_u) + self.translate_simd_binary(Op::u8x16_narrow_i16x8_sss, simd::i8x16_narrow_i16x8_u) } fn visit_i8x16_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i8x16_shl, Op::i8x16_shl_by, simd::i8x16_shl) + self.translate_simd_shift::(Op::i8x16_shl_sss, Op::i8x16_shl_ssi, simd::i8x16_shl) } fn visit_i8x16_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i8x16_shr_s, Op::i8x16_shr_s_by, simd::i8x16_shr_s) + self.translate_simd_shift::(Op::i8x16_shr_sss, Op::i8x16_shr_ssi, simd::i8x16_shr_s) } fn visit_i8x16_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i8x16_shr_u, Op::i8x16_shr_u_by, simd::i8x16_shr_u) + self.translate_simd_shift::(Op::u8x16_shr_sss, Op::u8x16_shr_ssi, simd::i8x16_shr_u) } fn visit_i8x16_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_add, simd::i8x16_add) + self.translate_simd_binary(Op::i8x16_add_sss, simd::i8x16_add) } fn visit_i8x16_add_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_add_sat_s, simd::i8x16_add_sat_s) + self.translate_simd_binary(Op::i8x16_add_sat_sss, simd::i8x16_add_sat_s) } fn visit_i8x16_add_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_add_sat_u, simd::i8x16_add_sat_u) + self.translate_simd_binary(Op::u8x16_add_sat_sss, simd::i8x16_add_sat_u) } fn visit_i8x16_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_sub, simd::i8x16_sub) + self.translate_simd_binary(Op::i8x16_sub_sss, simd::i8x16_sub) } fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_sub_sat_s, simd::i8x16_sub_sat_s) + self.translate_simd_binary(Op::i8x16_sub_sat_sss, simd::i8x16_sub_sat_s) } fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_sub_sat_u, simd::i8x16_sub_sat_u) + self.translate_simd_binary(Op::u8x16_sub_sat_sss, simd::i8x16_sub_sat_u) } fn visit_i8x16_min_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_min_s, simd::i8x16_min_s) + self.translate_simd_binary(Op::i8x16_min_sss, simd::i8x16_min_s) } fn visit_i8x16_min_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_min_u, simd::i8x16_min_u) + self.translate_simd_binary(Op::u8x16_min_sss, simd::i8x16_min_u) } fn visit_i8x16_max_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_max_s, simd::i8x16_max_s) + self.translate_simd_binary(Op::i8x16_max_sss, simd::i8x16_max_s) } fn visit_i8x16_max_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_max_u, simd::i8x16_max_u) + self.translate_simd_binary(Op::u8x16_max_sss, simd::i8x16_max_u) } fn visit_i8x16_avgr_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_avgr_u, simd::i8x16_avgr_u) + self.translate_simd_binary(Op::u8x16_avgr_sss, simd::i8x16_avgr_u) } fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extadd_pairwise_i8x16_s, + Op::i16x8_extadd_pairwise_i8x16_ss, simd::i16x8_extadd_pairwise_i8x16_s, ) } fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extadd_pairwise_i8x16_u, + Op::u16x8_extadd_pairwise_i8x16_ss, simd::i16x8_extadd_pairwise_i8x16_u, ) } fn visit_i16x8_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_abs, simd::i16x8_abs) + self.translate_simd_unary(Op::i16x8_abs_ss, simd::i16x8_abs) } fn visit_i16x8_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_neg, simd::i16x8_neg) + self.translate_simd_unary(Op::i16x8_neg_ss, simd::i16x8_neg) } fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_q15mulr_sat_s, simd::i16x8_q15mulr_sat_s) + self.translate_simd_binary(Op::i16x8_q15_mulr_sat_sss, simd::i16x8_q15mulr_sat_s) } fn visit_i16x8_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_all_true, simd::i16x8_all_true) + self.translate_simd_unary(Op::i16x8_all_true_ss, simd::i16x8_all_true) } fn visit_i16x8_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_bitmask, simd::i16x8_bitmask) + self.translate_simd_unary(Op::i16x8_bitmask_ss, simd::i16x8_bitmask) } fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_narrow_i32x4_s, simd::i16x8_narrow_i32x4_s) + self.translate_simd_binary(Op::i16x8_narrow_i32x4_sss, simd::i16x8_narrow_i32x4_s) } fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_narrow_i32x4_u, simd::i16x8_narrow_i32x4_u) + self.translate_simd_binary(Op::u16x8_narrow_i32x4_sss, simd::i16x8_narrow_i32x4_u) } fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_extend_low_i8x16_s, simd::i16x8_extend_low_i8x16_s) + self.translate_simd_unary( + Op::i16x8_extend_low_i8x16_ss, + simd::i16x8_extend_low_i8x16_s, + ) } fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extend_high_i8x16_s, + Op::i16x8_extend_high_i8x16_ss, simd::i16x8_extend_high_i8x16_s, ) } fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_extend_low_i8x16_u, simd::i16x8_extend_low_i8x16_u) + self.translate_simd_unary( + Op::u16x8_extend_low_i8x16_ss, + simd::i16x8_extend_low_i8x16_u, + ) } fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extend_high_i8x16_u, + Op::u16x8_extend_high_i8x16_ss, simd::i16x8_extend_high_i8x16_u, ) } fn visit_i16x8_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i16x8_shl, Op::i16x8_shl_by, simd::i16x8_shl) + self.translate_simd_shift::(Op::i16x8_shl_sss, Op::i16x8_shl_ssi, simd::i16x8_shl) } fn visit_i16x8_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i16x8_shr_s, Op::i16x8_shr_s_by, simd::i16x8_shr_s) + self.translate_simd_shift::(Op::i16x8_shr_sss, Op::i16x8_shr_ssi, simd::i16x8_shr_s) } fn visit_i16x8_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i16x8_shr_u, Op::i16x8_shr_u_by, simd::i16x8_shr_u) + self.translate_simd_shift::(Op::u16x8_shr_sss, Op::u16x8_shr_ssi, simd::i16x8_shr_u) } fn visit_i16x8_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_add, simd::i16x8_add) + self.translate_simd_binary(Op::i16x8_add_sss, simd::i16x8_add) } fn visit_i16x8_add_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_add_sat_s, simd::i16x8_add_sat_s) + self.translate_simd_binary(Op::i16x8_add_sat_sss, simd::i16x8_add_sat_s) } fn visit_i16x8_add_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_add_sat_u, simd::i16x8_add_sat_u) + self.translate_simd_binary(Op::u16x8_add_sat_sss, simd::i16x8_add_sat_u) } fn visit_i16x8_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_sub, simd::i16x8_sub) + self.translate_simd_binary(Op::i16x8_sub_sss, simd::i16x8_sub) } fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_sub_sat_s, simd::i16x8_sub_sat_s) + self.translate_simd_binary(Op::i16x8_sub_sat_sss, simd::i16x8_sub_sat_s) } fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_sub_sat_u, simd::i16x8_sub_sat_u) + self.translate_simd_binary(Op::u16x8_sub_sat_sss, simd::i16x8_sub_sat_u) } fn visit_i16x8_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_mul, simd::i16x8_mul) + self.translate_simd_binary(Op::i16x8_mul_sss, simd::i16x8_mul) } fn visit_i16x8_min_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_min_s, simd::i16x8_min_s) + self.translate_simd_binary(Op::i16x8_min_sss, simd::i16x8_min_s) } fn visit_i16x8_min_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_min_u, simd::i16x8_min_u) + self.translate_simd_binary(Op::u16x8_min_sss, simd::i16x8_min_u) } fn visit_i16x8_max_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_max_s, simd::i16x8_max_s) + self.translate_simd_binary(Op::i16x8_max_sss, simd::i16x8_max_s) } fn visit_i16x8_max_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_max_u, simd::i16x8_max_u) + self.translate_simd_binary(Op::u16x8_max_sss, simd::i16x8_max_u) } fn visit_i16x8_avgr_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_avgr_u, simd::i16x8_avgr_u) + self.translate_simd_binary(Op::u16x8_avgr_sss, simd::i16x8_avgr_u) } fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_extmul_low_i8x16_s, simd::i16x8_extmul_low_i8x16_s) + self.translate_simd_binary( + Op::i16x8_extmul_low_i8x16_sss, + simd::i16x8_extmul_low_i8x16_s, + ) } fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i16x8_extmul_high_i8x16_s, + Op::i16x8_extmul_high_i8x16_sss, simd::i16x8_extmul_high_i8x16_s, ) } fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_extmul_low_i8x16_u, simd::i16x8_extmul_low_i8x16_u) + self.translate_simd_binary( + Op::u16x8_extmul_low_i8x16_sss, + simd::i16x8_extmul_low_i8x16_u, + ) } fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i16x8_extmul_high_i8x16_u, + Op::u16x8_extmul_high_i8x16_sss, simd::i16x8_extmul_high_i8x16_u, ) } fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extadd_pairwise_i16x8_s, + Op::i32x4_extadd_pairwise_i16x8_ss, simd::i32x4_extadd_pairwise_i16x8_s, ) } fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extadd_pairwise_i16x8_u, + Op::u32x4_extadd_pairwise_i16x8_ss, simd::i32x4_extadd_pairwise_i16x8_u, ) } fn visit_i32x4_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_abs, simd::i32x4_abs) + self.translate_simd_unary(Op::i32x4_abs_ss, simd::i32x4_abs) } fn visit_i32x4_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_neg, simd::i32x4_neg) + self.translate_simd_unary(Op::i32x4_neg_ss, simd::i32x4_neg) } fn visit_i32x4_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_all_true, simd::i32x4_all_true) + self.translate_simd_unary(Op::i32x4_all_true_ss, simd::i32x4_all_true) } fn visit_i32x4_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_bitmask, simd::i32x4_bitmask) + self.translate_simd_unary(Op::i32x4_bitmask_ss, simd::i32x4_bitmask) } fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_extend_low_i16x8_s, simd::i32x4_extend_low_i16x8_s) + self.translate_simd_unary( + Op::i32x4_extend_low_i16x8_ss, + simd::i32x4_extend_low_i16x8_s, + ) } fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extend_high_i16x8_s, + Op::i32x4_extend_high_i16x8_ss, simd::i32x4_extend_high_i16x8_s, ) } fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_extend_low_i16x8_u, simd::i32x4_extend_low_i16x8_u) + self.translate_simd_unary( + Op::u32x4_extend_low_i16x8_ss, + simd::i32x4_extend_low_i16x8_u, + ) } fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extend_high_i16x8_u, + Op::u32x4_extend_high_i16x8_ss, simd::i32x4_extend_high_i16x8_u, ) } fn visit_i32x4_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i32x4_shl, Op::i32x4_shl_by, simd::i32x4_shl) + self.translate_simd_shift::(Op::i32x4_shl_sss, Op::i32x4_shl_ssi, simd::i32x4_shl) } fn visit_i32x4_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i32x4_shr_s, Op::i32x4_shr_s_by, simd::i32x4_shr_s) + self.translate_simd_shift::(Op::i32x4_shr_sss, Op::i32x4_shr_ssi, simd::i32x4_shr_s) } fn visit_i32x4_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i32x4_shr_u, Op::i32x4_shr_u_by, simd::i32x4_shr_u) + self.translate_simd_shift::(Op::u32x4_shr_sss, Op::u32x4_shr_ssi, simd::i32x4_shr_u) } fn visit_i32x4_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_add, simd::i32x4_add) + self.translate_simd_binary(Op::i32x4_add_sss, simd::i32x4_add) } fn visit_i32x4_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_sub, simd::i32x4_sub) + self.translate_simd_binary(Op::i32x4_sub_sss, simd::i32x4_sub) } fn visit_i32x4_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_mul, simd::i32x4_mul) + self.translate_simd_binary(Op::i32x4_mul_sss, simd::i32x4_mul) } fn visit_i32x4_min_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_min_s, simd::i32x4_min_s) + self.translate_simd_binary(Op::i32x4_min_sss, simd::i32x4_min_s) } fn visit_i32x4_min_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_min_u, simd::i32x4_min_u) + self.translate_simd_binary(Op::u32x4_min_sss, simd::i32x4_min_u) } fn visit_i32x4_max_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_max_s, simd::i32x4_max_s) + self.translate_simd_binary(Op::i32x4_max_sss, simd::i32x4_max_s) } fn visit_i32x4_max_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_max_u, simd::i32x4_max_u) + self.translate_simd_binary(Op::u32x4_max_sss, simd::i32x4_max_u) } fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_dot_i16x8_s, simd::i32x4_dot_i16x8_s) + self.translate_simd_binary(Op::i32x4_dot_i16x8_sss, simd::i32x4_dot_i16x8_s) } fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_extmul_low_i16x8_s, simd::i32x4_extmul_low_i16x8_s) + self.translate_simd_binary( + Op::i32x4_extmul_low_i16x8_sss, + simd::i32x4_extmul_low_i16x8_s, + ) } fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i32x4_extmul_high_i16x8_s, + Op::i32x4_extmul_high_i16x8_sss, simd::i32x4_extmul_high_i16x8_s, ) } fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_extmul_low_i16x8_u, simd::i32x4_extmul_low_i16x8_u) + self.translate_simd_binary( + Op::u32x4_extmul_low_i16x8_sss, + simd::i32x4_extmul_low_i16x8_u, + ) } fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i32x4_extmul_high_i16x8_u, + Op::u32x4_extmul_high_i16x8_sss, simd::i32x4_extmul_high_i16x8_u, ) } fn visit_i64x2_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_abs, simd::i64x2_abs) + self.translate_simd_unary(Op::i64x2_abs_ss, simd::i64x2_abs) } fn visit_i64x2_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_neg, simd::i64x2_neg) + self.translate_simd_unary(Op::i64x2_neg_ss, simd::i64x2_neg) } fn visit_i64x2_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_all_true, simd::i64x2_all_true) + self.translate_simd_unary(Op::i64x2_all_true_ss, simd::i64x2_all_true) } fn visit_i64x2_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_bitmask, simd::i64x2_bitmask) + self.translate_simd_unary(Op::i64x2_bitmask_ss, simd::i64x2_bitmask) } fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_extend_low_i32x4_s, simd::i64x2_extend_low_i32x4_s) + self.translate_simd_unary( + Op::i64x2_extend_low_i32x4_ss, + simd::i64x2_extend_low_i32x4_s, + ) } fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i64x2_extend_high_i32x4_s, + Op::i64x2_extend_high_i32x4_ss, simd::i64x2_extend_high_i32x4_s, ) } fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_extend_low_i32x4_u, simd::i64x2_extend_low_i32x4_u) + self.translate_simd_unary( + Op::u64x2_extend_low_i32x4_ss, + simd::i64x2_extend_low_i32x4_u, + ) } fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i64x2_extend_high_i32x4_u, + Op::u64x2_extend_high_i32x4_ss, simd::i64x2_extend_high_i32x4_u, ) } fn visit_i64x2_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i64x2_shl, Op::i64x2_shl_by, simd::i64x2_shl) + self.translate_simd_shift::(Op::i64x2_shl_sss, Op::i64x2_shl_ssi, simd::i64x2_shl) } fn visit_i64x2_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i64x2_shr_s, Op::i64x2_shr_s_by, simd::i64x2_shr_s) + self.translate_simd_shift::(Op::i64x2_shr_sss, Op::i64x2_shr_ssi, simd::i64x2_shr_s) } fn visit_i64x2_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i64x2_shr_u, Op::i64x2_shr_u_by, simd::i64x2_shr_u) + self.translate_simd_shift::(Op::u64x2_shr_sss, Op::u64x2_shr_ssi, simd::i64x2_shr_u) } fn visit_i64x2_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_add, simd::i64x2_add) + self.translate_simd_binary(Op::i64x2_add_sss, simd::i64x2_add) } fn visit_i64x2_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_sub, simd::i64x2_sub) + self.translate_simd_binary(Op::i64x2_sub_sss, simd::i64x2_sub) } fn visit_i64x2_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_mul, simd::i64x2_mul) + self.translate_simd_binary(Op::i64x2_mul_sss, simd::i64x2_mul) } fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_extmul_low_i32x4_s, simd::i64x2_extmul_low_i32x4_s) + self.translate_simd_binary( + Op::i64x2_extmul_low_i32x4_sss, + simd::i64x2_extmul_low_i32x4_s, + ) } fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i64x2_extmul_high_i32x4_s, + Op::i64x2_extmul_high_i32x4_sss, simd::i64x2_extmul_high_i32x4_s, ) } fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_extmul_low_i32x4_u, simd::i64x2_extmul_low_i32x4_u) + self.translate_simd_binary( + Op::i64x2_extmul_low_i32x4_sss, + simd::i64x2_extmul_low_i32x4_u, + ) } fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i64x2_extmul_high_i32x4_u, + Op::u64x2_extmul_high_i32x4_sss, simd::i64x2_extmul_high_i32x4_u, ) } fn visit_f32x4_ceil(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_ceil, simd::f32x4_ceil) + self.translate_simd_unary(Op::f32x4_ceil_ss, simd::f32x4_ceil) } fn visit_f32x4_floor(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_floor, simd::f32x4_floor) + self.translate_simd_unary(Op::f32x4_floor_ss, simd::f32x4_floor) } fn visit_f32x4_trunc(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_trunc, simd::f32x4_trunc) + self.translate_simd_unary(Op::f32x4_trunc_ss, simd::f32x4_trunc) } fn visit_f32x4_nearest(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_nearest, simd::f32x4_nearest) + self.translate_simd_unary(Op::f32x4_nearest_ss, simd::f32x4_nearest) } fn visit_f32x4_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_abs, simd::f32x4_abs) + self.translate_simd_unary(Op::f32x4_abs_ss, simd::f32x4_abs) } fn visit_f32x4_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_neg, simd::f32x4_neg) + self.translate_simd_unary(Op::f32x4_neg_ss, simd::f32x4_neg) } fn visit_f32x4_sqrt(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_sqrt, simd::f32x4_sqrt) + self.translate_simd_unary(Op::f32x4_sqrt_ss, simd::f32x4_sqrt) } fn visit_f32x4_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_add, simd::f32x4_add) + self.translate_simd_binary(Op::f32x4_add_sss, simd::f32x4_add) } fn visit_f32x4_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_sub, simd::f32x4_sub) + self.translate_simd_binary(Op::f32x4_sub_sss, simd::f32x4_sub) } fn visit_f32x4_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_mul, simd::f32x4_mul) + self.translate_simd_binary(Op::f32x4_mul_sss, simd::f32x4_mul) } fn visit_f32x4_div(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_div, simd::f32x4_div) + self.translate_simd_binary(Op::f32x4_div_sss, simd::f32x4_div) } fn visit_f32x4_min(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_min, simd::f32x4_min) + self.translate_simd_binary(Op::f32x4_min_sss, simd::f32x4_min) } fn visit_f32x4_max(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_max, simd::f32x4_max) + self.translate_simd_binary(Op::f32x4_max_sss, simd::f32x4_max) } fn visit_f32x4_pmin(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_pmin, simd::f32x4_pmin) + self.translate_simd_binary(Op::f32x4_pmin_sss, simd::f32x4_pmin) } fn visit_f32x4_pmax(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_pmax, simd::f32x4_pmax) + self.translate_simd_binary(Op::f32x4_pmax_sss, simd::f32x4_pmax) } fn visit_f64x2_ceil(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_ceil, simd::f64x2_ceil) + self.translate_simd_unary(Op::f64x2_ceil_ss, simd::f64x2_ceil) } fn visit_f64x2_floor(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_floor, simd::f64x2_floor) + self.translate_simd_unary(Op::f64x2_floor_ss, simd::f64x2_floor) } fn visit_f64x2_trunc(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_trunc, simd::f64x2_trunc) + self.translate_simd_unary(Op::f64x2_trunc_ss, simd::f64x2_trunc) } fn visit_f64x2_nearest(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_nearest, simd::f64x2_nearest) + self.translate_simd_unary(Op::f64x2_nearest_ss, simd::f64x2_nearest) } fn visit_f64x2_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_abs, simd::f64x2_abs) + self.translate_simd_unary(Op::f64x2_abs_ss, simd::f64x2_abs) } fn visit_f64x2_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_neg, simd::f64x2_neg) + self.translate_simd_unary(Op::f64x2_neg_ss, simd::f64x2_neg) } fn visit_f64x2_sqrt(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_sqrt, simd::f64x2_sqrt) + self.translate_simd_unary(Op::f64x2_sqrt_ss, simd::f64x2_sqrt) } fn visit_f64x2_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_add, simd::f64x2_add) + self.translate_simd_binary(Op::f64x2_add_sss, simd::f64x2_add) } fn visit_f64x2_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_sub, simd::f64x2_sub) + self.translate_simd_binary(Op::f64x2_sub_sss, simd::f64x2_sub) } fn visit_f64x2_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_mul, simd::f64x2_mul) + self.translate_simd_binary(Op::f64x2_mul_sss, simd::f64x2_mul) } fn visit_f64x2_div(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_div, simd::f64x2_div) + self.translate_simd_binary(Op::f64x2_div_sss, simd::f64x2_div) } fn visit_f64x2_min(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_min, simd::f64x2_min) + self.translate_simd_binary(Op::f64x2_min_sss, simd::f64x2_min) } fn visit_f64x2_max(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_max, simd::f64x2_max) + self.translate_simd_binary(Op::f64x2_max_sss, simd::f64x2_max) } fn visit_f64x2_pmin(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_pmin, simd::f64x2_pmin) + self.translate_simd_binary(Op::f64x2_pmin_sss, simd::f64x2_pmin) } fn visit_f64x2_pmax(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_pmax, simd::f64x2_pmax) + self.translate_simd_binary(Op::f64x2_pmax_sss, simd::f64x2_pmax) } fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_trunc_sat_f32x4_s, simd::i32x4_trunc_sat_f32x4_s) + self.translate_simd_unary(Op::i32x4_trunc_sat_f32x4_ss, simd::i32x4_trunc_sat_f32x4_s) } fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_trunc_sat_f32x4_u, simd::i32x4_trunc_sat_f32x4_u) + self.translate_simd_unary(Op::u32x4_trunc_sat_f32x4_ss, simd::i32x4_trunc_sat_f32x4_u) } fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_convert_i32x4_s, simd::f32x4_convert_i32x4_s) + self.translate_simd_unary(Op::f32x4_convert_i32x4_ss, simd::f32x4_convert_i32x4_s) } fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_convert_i32x4_u, simd::f32x4_convert_i32x4_u) + self.translate_simd_unary(Op::f32x4_convert_u32x4_ss, simd::f32x4_convert_i32x4_u) } fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_trunc_sat_f64x2_s_zero, + Op::i32x4_trunc_sat_zero_f64x2_ss, simd::i32x4_trunc_sat_f64x2_s_zero, ) } fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_trunc_sat_f64x2_u_zero, + Op::u32x4_trunc_sat_zero_f64x2_ss, simd::i32x4_trunc_sat_f64x2_u_zero, ) } fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::f64x2_convert_low_i32x4_s, + Op::f64x2_convert_low_i32x4_ss, simd::f64x2_convert_low_i32x4_s, ) } fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::f64x2_convert_low_i32x4_u, + Op::f64x2_convert_low_u32x4_ss, simd::f64x2_convert_low_i32x4_u, ) } fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_demote_f64x2_zero, simd::f32x4_demote_f64x2_zero) + self.translate_simd_unary( + Op::f32x4_demote_zero_f64x2_ss, + simd::f32x4_demote_f64x2_zero, + ) } fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_promote_low_f32x4, simd::f64x2_promote_low_f32x4) + self.translate_simd_unary( + Op::f64x2_promote_low_f32x4_ss, + simd::f64x2_promote_low_f32x4, + ) } fn visit_i8x16_relaxed_swizzle(&mut self) -> Self::Output { @@ -1261,19 +1218,19 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_f32x4_relaxed_madd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f32x4_relaxed_madd, simd::f32x4_relaxed_madd) + self.translate_simd_ternary(Op::f32x4_relaxed_madd_ssss, simd::f32x4_relaxed_madd) } fn visit_f32x4_relaxed_nmadd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f32x4_relaxed_nmadd, simd::f32x4_relaxed_nmadd) + self.translate_simd_ternary(Op::f32x4_relaxed_nmadd_ssss, simd::f32x4_relaxed_nmadd) } fn visit_f64x2_relaxed_madd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f64x2_relaxed_madd, simd::f64x2_relaxed_madd) + self.translate_simd_ternary(Op::f64x2_relaxed_madd_ssss, simd::f64x2_relaxed_madd) } fn visit_f64x2_relaxed_nmadd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f64x2_relaxed_nmadd, simd::f64x2_relaxed_nmadd) + self.translate_simd_ternary(Op::f64x2_relaxed_nmadd_ssss, simd::f64x2_relaxed_nmadd) } fn visit_i8x16_relaxed_laneselect(&mut self) -> Self::Output { @@ -1314,14 +1271,14 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i16x8_relaxed_dot_i8x16_i7x16_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i16x8_relaxed_dot_i8x16_i7x16_s, + Op::i16x8_relaxed_dot_i8x16_i7x16_sss, simd::i16x8_relaxed_dot_i8x16_i7x16_s, ) } fn visit_i32x4_relaxed_dot_i8x16_i7x16_add_s(&mut self) -> Self::Output { self.translate_simd_ternary( - Op::i32x4_relaxed_dot_i8x16_i7x16_add_s, + Op::i32x4_relaxed_dot_i8x16_i7x16_add_ssss, simd::i32x4_relaxed_dot_i8x16_i7x16_add_s, ) } diff --git a/crates/wasmi/src/engine/translator/func/stack/control.rs b/crates/wasmi/src/engine/translator/func/stack/control.rs index 399662855fd..d655e794612 100644 --- a/crates/wasmi/src/engine/translator/func/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func/stack/control.rs @@ -1,9 +1,10 @@ use super::{Operand, Reset}; use crate::{ engine::{ - translator::{labels::LabelRef, utils::Instr}, + translator::func::{labels::LabelRef, Pos}, BlockType, }, + ir, Engine, }; use alloc::vec::{Drain, Vec}; @@ -43,7 +44,7 @@ pub struct ControlStack { /// fuel consumption instruction since this information is accessed commonly. /// /// [`Op`]: crate::ir::Op - consume_fuel_instr: Option, + consume_fuel_instr: Option>, /// Special operand stack to memorize operands for `else` control frames. else_operands: ElseOperands, /// This is `true` if an `if` with else providers was just popped from the stack. @@ -101,7 +102,7 @@ impl ControlStack { /// /// Returns `None` otherwise. #[inline] - pub fn consume_fuel_instr(&self) -> Option { + pub fn consume_fuel_instr(&self) -> Option> { debug_assert!(!self.is_empty()); self.consume_fuel_instr } @@ -128,7 +129,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) { debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(BlockControlFrame { @@ -147,7 +148,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) { debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(LoopControlFrame { @@ -166,7 +167,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, reachability: IfReachability, else_operands: impl IntoIterator, ) { @@ -191,7 +192,7 @@ impl ControlStack { pub fn push_else( &mut self, if_frame: IfControlFrame, - consume_fuel: Option, + consume_fuel: Option>, is_end_of_then_reachable: bool, ) { debug_assert!(!self.orphaned_else_operands); @@ -305,7 +306,7 @@ impl<'a> ControlFrameBase for ControlFrameMut<'a> { self.0.len_branch_params(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.0.consume_fuel_instr() } } @@ -418,7 +419,7 @@ pub trait ControlFrameBase { /// Returns a reference to the [`Op::ConsumeFuel`] of `self`. /// /// Returns `None` if fuel metering is disabled. - fn consume_fuel_instr(&self) -> Option; + fn consume_fuel_instr(&self) -> Option>; } impl ControlFrameBase for ControlFrame { @@ -496,7 +497,7 @@ impl ControlFrameBase for ControlFrame { } } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { match self { ControlFrame::Block(frame) => frame.consume_fuel_instr(), ControlFrame::Loop(frame) => frame.consume_fuel_instr(), @@ -523,7 +524,7 @@ pub struct BlockControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`BlockControlFrame`]. label: LabelRef, } @@ -553,7 +554,7 @@ impl ControlFrameBase for BlockControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } @@ -572,7 +573,7 @@ pub struct LoopControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`LoopControlFrame`]. label: LabelRef, } @@ -602,7 +603,7 @@ impl ControlFrameBase for LoopControlFrame { self.ty.len_params(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } @@ -621,7 +622,7 @@ pub struct IfControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`IfControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -672,7 +673,7 @@ impl ControlFrameBase for IfControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } @@ -716,7 +717,7 @@ pub struct ElseControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`ElseControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -807,7 +808,7 @@ impl ControlFrameBase for ElseControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } diff --git a/crates/wasmi/src/engine/translator/func/stack/mod.rs b/crates/wasmi/src/engine/translator/func/stack/mod.rs index d42de743cf5..b64cafda821 100644 --- a/crates/wasmi/src/engine/translator/func/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func/stack/mod.rs @@ -1,3 +1,5 @@ +#![expect(unused)] + mod control; mod locals; mod operand; @@ -21,20 +23,17 @@ pub use self::{ IfReachability, LoopControlFrame, }, - operand::{ImmediateOperand, Operand, TempOperand}, + operand::{ImmediateOperand, LocalOperand, Operand, TempOperand}, operands::{OperandIdx, PreservedAllLocalsIter, PreservedLocalsIter}, }; use super::{Reset, ReusableAllocations}; use crate::{ core::TypedVal, engine::{ - translator::{ - func::{stack::operands::PeekedOperands, LocalIdx}, - labels::LabelRef, - utils::Instr, - }, + translator::func::{labels::LabelRef, stack::operands::PeekedOperands, LocalIdx, Pos}, BlockType, }, + ir, Engine, Error, ValType, @@ -158,7 +157,7 @@ impl Stack { &mut self, ty: BlockType, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -170,7 +169,7 @@ impl Stack { /// /// # Note /// - /// This inherits the `consume_fuel` [`Instr`] from the parent [`ControlFrame`]. + /// This inherits the `consume_fuel` [`Pos`] from the parent [`ControlFrame`]. /// /// # Errors /// @@ -199,7 +198,7 @@ impl Stack { &mut self, ty: BlockType, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -228,7 +227,7 @@ impl Stack { ty: BlockType, label: LabelRef, reachability: IfReachability, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -260,7 +259,7 @@ impl Stack { &mut self, if_frame: IfControlFrame, is_end_of_then_reachable: bool, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); self.push_else_operands(&if_frame)?; @@ -359,8 +358,8 @@ impl Stack { /// /// If too many operands have been pushed onto the [`Stack`]. #[inline] - pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { - self.operands.push_temp(ty, instr) + pub fn push_temp(&mut self, ty: ValType) -> Result { + self.operands.push_temp(ty) } /// Pushes an immediate `value` on the [`Stack`]. @@ -472,6 +471,13 @@ impl Stack { buffer.reverse(); } + /// Drops `len` operands form the stack. + pub fn drop_n(&mut self, len: usize) { + for _ in 0..len { + self.pop(); + } + } + /// Preserve all locals on the [`Stack`] that refer to `local_index`. /// /// This is done by converting those locals to [`Operand::Temp`] and yielding them. @@ -521,7 +527,7 @@ impl Stack { /// /// Returns `None` otherwise. #[inline] - pub fn consume_fuel_instr(&self) -> Option { + pub fn consume_fuel_instr(&self) -> Option> { self.controls.consume_fuel_instr() } } diff --git a/crates/wasmi/src/engine/translator/func/stack/operand.rs b/crates/wasmi/src/engine/translator/func/stack/operand.rs index 1ff14745d85..f2e24de6327 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operand.rs @@ -1,5 +1,5 @@ use super::{LocalIdx, OperandIdx, StackOperand}; -use crate::{core::TypedVal, engine::translator::utils::Instr, ValType}; +use crate::{core::TypedVal, engine::translator::func::encoder::BytePos, ValType}; #[cfg(doc)] use super::Stack; @@ -22,7 +22,7 @@ impl Operand { StackOperand::Local { local_index, ty, .. } => Self::local(index, local_index, ty), - StackOperand::Temp { ty, instr } => Self::temp(index, ty, instr), + StackOperand::Temp { ty } => Self::temp(index, ty), StackOperand::Immediate { val } => Self::immediate(index, val), } } @@ -47,12 +47,8 @@ impl Operand { } /// Creates a temporary [`Operand`]. - pub(super) fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { - Self::Temp(TempOperand { - operand_index, - ty, - instr, - }) + pub(super) fn temp(operand_index: OperandIdx, ty: ValType) -> Self { + Self::Temp(TempOperand { operand_index, ty }) } /// Creates an immediate [`Operand`]. @@ -125,8 +121,6 @@ pub struct TempOperand { operand_index: OperandIdx, /// The type of the temporary. ty: ValType, - /// The instruction which created this [`TempOperand`] as its result if any. - instr: Option, } impl From for Operand { @@ -145,11 +139,6 @@ impl TempOperand { pub fn ty(&self) -> ValType { self.ty } - - /// Returns the instruction which created this [`TempOperand`] as its result if any. - pub fn instr(&self) -> Option { - self.instr - } } /// An immediate value on the [`Stack`]. diff --git a/crates/wasmi/src/engine/translator/func/stack/operands.rs b/crates/wasmi/src/engine/translator/func/stack/operands.rs index f2a7f26c8c2..809ed15bd30 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operands.rs @@ -1,5 +1,5 @@ use super::{LocalIdx, LocalsHead, Operand, Reset}; -use crate::{core::TypedVal, engine::translator::utils::Instr, Error, ValType}; +use crate::{core::TypedVal, engine::translator::func::encoder::BytePos, Error, ValType}; use alloc::vec::Vec; use core::{num::NonZero, slice}; @@ -47,8 +47,6 @@ pub enum StackOperand { Temp { /// The type of the temporary value. ty: ValType, - /// The instruction which has this [`StackOperand`] as result if any. - instr: Option, }, /// An immediate value on the [`OperandStack`]. Immediate { @@ -147,7 +145,7 @@ impl OperandStack { pub fn push_operand(&mut self, operand: Operand) -> Result { match operand { Operand::Local(operand) => self.push_local(operand.local_index(), operand.ty()), - Operand::Temp(operand) => self.push_temp(operand.ty(), operand.instr()), + Operand::Temp(operand) => self.push_temp(operand.ty()), Operand::Immediate(operand) => self.push_immediate(operand.val()), } } @@ -183,9 +181,9 @@ impl OperandStack { /// /// If too many operands have been pushed onto the [`OperandStack`]. #[inline] - pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { + pub fn push_temp(&mut self, ty: ValType) -> Result { let idx = self.next_index(); - self.operands.push(StackOperand::Temp { ty, instr }); + self.operands.push(StackOperand::Temp { ty }); self.update_max_stack_height(); Ok(idx) } @@ -290,7 +288,7 @@ impl OperandStack { let operand = self.get_at(index); let ty = operand.ty(); self.try_unlink_local(operand); - self.operands[usize::from(index)] = StackOperand::Temp { ty, instr: None }; + self.operands[usize::from(index)] = StackOperand::Temp { ty }; operand } diff --git a/crates/wasmi/src/engine/translator/func/utils.rs b/crates/wasmi/src/engine/translator/func/utils.rs index 734e1c14350..3f0a88c18a9 100644 --- a/crates/wasmi/src/engine/translator/func/utils.rs +++ b/crates/wasmi/src/engine/translator/func/utils.rs @@ -1,4 +1,4 @@ -use crate::ir::{Const16, Const32, Slot}; +use crate::ir::{Op, Slot}; /// Bail out early in case the current code is unreachable. /// @@ -49,12 +49,6 @@ pub trait ReusableAllocations { fn into_allocations(self) -> Self::Allocations; } -/// A 16-bit encoded input to Wasmi instruction. -pub type Input16 = Input>; - -/// A 32-bit encoded input to Wasmi instruction. -pub type Input32 = Input>; - /// A concrete input to a Wasmi instruction. pub enum Input { /// A [`Slot`] operand. @@ -62,3 +56,23 @@ pub enum Input { /// A 16-bit encoded immediate value operand. Immediate(T), } + +/// Extension trait to update the result [`Slot`] of an [`Op`]. +pub trait UpdateResultSlot: Sized { + /// Updates the result [`Slot`] of `self` if possible. + /// + /// # Note + /// + /// - Returns `Some` resulting `Self` if the update was successful. + /// - Returns `None` if the result update could not be applied. + fn update_result_slot(&self, new_result: Slot) -> Option; +} + +impl UpdateResultSlot for Op { + fn update_result_slot(&self, new_result: Slot) -> Option { + let mut op = *self; + let result_mut = op.result_mut()?; + *result_mut = new_result; + Some(op) + } +} diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 60204910d4c..3cbb2112ba4 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -11,8 +11,8 @@ use crate::{ }, BlockType, }, - ir::{self, Const16, Op}, - module::{self, FuncIdx, MemoryIdx, TableIdx, WasmiValueType}, + ir::{self, index, Op}, + module::{self, MemoryIdx, TableIdx, WasmiValueType}, Error, ExternRef, Func, @@ -108,7 +108,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } self.preserve_all_locals()?; let block_ty = BlockType::new(block_ty, &self.module); - let end_label = self.labels.new_label(); + let end_label = self.instrs.new_label(); self.stack.push_block(block_ty, end_label)?; Ok(()) } @@ -121,15 +121,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } let block_ty = BlockType::new(block_ty, &self.module); let len_params = block_ty.len_params(&self.engine); - let continue_label = self.labels.new_label(); + let continue_label = self.instrs.new_label(); let consume_fuel = self.stack.consume_fuel_instr(); - self.move_operands_to_temp(usize::from(len_params), consume_fuel)?; - self.pin_label(continue_label); - let consume_fuel = self.instrs.push_consume_fuel_instr()?; + if len_params > 0 { + self.move_operands_to_temp(usize::from(len_params), consume_fuel)?; + } + self.instrs.pin_label(continue_label)?; + let consume_fuel = self.instrs.encode_consume_fuel()?; self.stack .push_loop(block_ty, continue_label, consume_fuel)?; - // Need to reset `last_instr` because a loop header is a control flow boundary. - self.instrs.reset_last_instr(); Ok(()) } @@ -139,7 +139,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.stack.push_unreachable(ControlFrameKind::If)?; return Ok(()); } - let end_label = self.labels.new_label(); + let end_label = self.instrs.new_label(); let condition = self.stack.pop(); self.preserve_all_locals()?; let (reachability, consume_fuel_instr) = match condition { @@ -156,10 +156,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { (reachability, consume_fuel_instr) } _ => { - let else_label = self.labels.new_label(); + let else_label = self.instrs.new_label(); self.encode_br_eqz(condition, else_label)?; let reachability = IfReachability::Both { else_label }; - let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + let consume_fuel_instr = self.instrs.encode_consume_fuel()?; (reachability, consume_fuel_instr) } }; @@ -192,12 +192,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.encode_br(frame.label())?; } // Start of `else` block: - self.labels - .pin_label(else_label, self.instrs.next_instr()) - .unwrap(); - self.instrs.reset_last_instr(); + self.instrs.pin_label(else_label)?; } - let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + let consume_fuel_instr = self.instrs.encode_consume_fuel()?; self.reachable = frame.is_else_reachable(); self.stack .push_else(frame, is_end_of_then_reachable, consume_fuel_instr)?; @@ -270,15 +267,13 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } // Case: fallback to copy branch parameters conditionally let consume_fuel_instr = self.stack.consume_fuel_instr(); - let skip_label = self.labels.new_label(); + let skip_label = self.instrs.new_label(); self.encode_br_eqz(condition, skip_label)?; if let Some(branch_results) = branch_results { self.encode_copies(branch_results, len_branch_params, consume_fuel_instr)?; } self.encode_br(label)?; - self.labels - .pin_label(skip_label, self.instrs.next_instr()) - .unwrap(); + self.instrs.pin_label(skip_label)?; Ok(()) } @@ -317,7 +312,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let Ok(default_target) = usize::try_from(default_target) else { panic!("out of bounds `default_target` does not fit into `usize`: {default_target}"); }; - let index = self.layout.operand_to_reg(index)?; + let index = self.layout.operand_to_slot(index)?; let len_branch_params = self .stack .peek_control(default_target) @@ -345,65 +340,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_call(&mut self, function_index: u32) -> Self::Output { - bail_unreachable!(self); - let func_idx = FuncIdx::from(function_index); - let func_type = self.resolve_func_type(func_idx); - let len_params = usize::from(func_type.len_params()); - let results = self.call_regspan(len_params)?; - let instr = match self.module.get_engine_func(func_idx) { - Some(engine_func) => { - // Case: We are calling an internal function and can optimize - // this case by using the special instruction for it. - match len_params { - 0 => Op::call_internal_0(results, engine_func), - _ => Op::call_internal(results, engine_func), - } - } - None => { - // Case: We are calling an imported function and must use the - // general calling operator for it. - match len_params { - 0 => Op::call_imported_0(results, function_index), - _ => Op::call_imported(results, function_index), - } - } - }; - let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; - if let Some(span) = self.push_results(call_instr, func_type.results())? { - debug_assert_eq!(span, results); - } - Ok(()) + self.translate_call(function_index, Op::call_internal, Op::call_imported) } #[inline(never)] fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { - bail_unreachable!(self); - let func_type = self.resolve_type(type_index); - let index = self.stack.pop(); - let indirect_params = self.call_indirect_params(index, table_index)?; - let len_params = usize::from(func_type.len_params()); - let results = self.call_regspan(len_params)?; - let instr = match (len_params, indirect_params) { - (0, Op::CallIndirectParams { .. }) => Op::call_indirect_0(results, type_index), - (0, Op::CallIndirectParamsImm16 { .. }) => { - Op::call_indirect_0_imm16(results, type_index) - } - (_, Op::CallIndirectParams { .. }) => Op::call_indirect(results, type_index), - (_, Op::CallIndirectParamsImm16 { .. }) => Op::call_indirect_imm16(results, type_index), - _ => unreachable!(), - }; - let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; - self.push_param(indirect_params)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; - if let Some(span) = self.push_results(call_instr, func_type.results())? { - debug_assert_eq!(span, results); - } - Ok(()) + self.translate_call_indirect(type_index, table_index, Op::call_indirect) } #[inline(never)] @@ -461,7 +403,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let global_idx = ir::index::Global::from(global_index); self.push_instr_with_result( content, - |result| Op::global_get(result, global_idx), + |result| Op::global_get(global_idx, result), FuelCostsProvider::instance, )?; Ok(()) @@ -470,13 +412,14 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_global_set(&mut self, global_index: u32) -> Self::Output { bail_unreachable!(self); - let global = ir::index::Global::from(global_index); - let input = match self.stack.pop() { + let global = index::Global::from(global_index); + let input = self.stack.pop(); + let value = match input { Operand::Immediate(input) => input.val(), input => { // Case: `global.set` with simple register input. - let input = self.layout.operand_to_reg(input)?; - self.push_instr(Op::global_set(input, global), FuelCostsProvider::instance)?; + let input = self.layout.operand_to_slot(input)?; + self.push_instr(Op::global_set(global, input), FuelCostsProvider::instance)?; return Ok(()); } }; @@ -484,233 +427,137 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (global_type, _init_value) = self .module .get_global(module::GlobalIdx::from(global_index)); - debug_assert_eq!(global_type.content(), input.ty()); - match global_type.content() { - ValType::I32 => { - if let Ok(value) = Const16::try_from(i32::from(input)) { - // Case: `global.set` with 16-bit encoded `i32` value. - self.push_instr( - Op::global_set_i32imm16(value, global), - FuelCostsProvider::instance, - )?; - return Ok(()); - } + debug_assert_eq!(global_type.content(), value.ty()); + let global_set_instr = match global_type.content() { + ValType::I32 => Op::global_set32(global, u32::from(value)), + ValType::I64 => Op::global_set64(u64::from(value), global), + ValType::F32 => Op::global_set32(global, f32::from(value).to_bits()), + ValType::F64 => Op::global_set64(f64::from(value).to_bits(), global), + ValType::FuncRef | ValType::ExternRef => Op::global_set64(u64::from(value), global), + ValType::V128 => { + let consume_fuel = self.stack.consume_fuel_instr(); + let temp = self.copy_operand_to_temp(input, consume_fuel)?; + Op::global_set(global, temp) } - ValType::I64 => { - if let Ok(value) = Const16::try_from(i64::from(input)) { - // Case: `global.set` with 16-bit encoded `i64` value. - self.push_instr( - Op::global_set_i64imm16(value, global), - FuelCostsProvider::instance, - )?; - return Ok(()); - } - } - _ => {} }; // Note: at this point we have to allocate a function local constant. - let cref = self.layout.const_to_reg(input)?; - self.push_instr(Op::global_set(cref, global), FuelCostsProvider::instance)?; + self.push_instr(global_set_instr, FuelCostsProvider::instance)?; Ok(()) } #[inline(never)] fn visit_i32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::load32, - Op::load32_offset16, - Op::load32_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::load64, - Op::load64_offset16, - Op::load64_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_f32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::F32, - Op::load32, - Op::load32_offset16, - Op::load32_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_f64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::F64, - Op::load64, - Op::load64_offset16, - Op::load64_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::i32_load8_s, - Op::i32_load8_s_offset16, - Op::i32_load8_s_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::i32_load8_u, - Op::i32_load8_u_offset16, - Op::i32_load8_u_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::i32_load16_s, - Op::i32_load16_s_offset16, - Op::i32_load16_s_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::i32_load16_u, - Op::i32_load16_u_offset16, - Op::i32_load16_u_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load8_s, - Op::i64_load8_s_offset16, - Op::i64_load8_s_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load8_u, - Op::i64_load8_u_offset16, - Op::i64_load8_u_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load16_s, - Op::i64_load16_s_offset16, - Op::i64_load16_s_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load16_u, - Op::i64_load16_u_offset16, - Op::i64_load16_u_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load32_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load32_s, - Op::i64_load32_s_offset16, - Op::i64_load32_s_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load32_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load32_u, - Op::i64_load32_u_offset16, - Op::i64_load32_u_at, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_f32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_store(memarg, Op::store32, Op::store32_offset16, Op::store32_at) + self.translate_store::(memarg) } #[inline(never)] fn visit_f64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_store(memarg, Op::store64, Op::store64_offset16, Op::store64_at) + self.translate_store::(memarg) } #[inline(never)] fn visit_i32_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i32_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store32(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] @@ -721,9 +568,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { .get_type_of_memory(MemoryIdx::from(mem)) .index_ty() .ty(); + let memory = index::Memory::try_from(mem)?; self.push_instr_with_result( index_ty, - |result| Op::memory_size(result, mem), + |result| Op::memory_size(result, memory), FuelCostsProvider::instance, )?; Ok(()) @@ -736,6 +584,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { .module .get_type_of_memory(MemoryIdx::from(mem)) .index_ty(); + let memory = index::Memory::try_from(mem)?; let delta = self.stack.pop(); if let Operand::Immediate(delta) = delta { let delta = delta.val(); @@ -751,20 +600,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { // as `memory.size` instruction instead. self.push_instr_with_result( index_ty.ty(), - |result| Op::memory_size(result, mem), + |result| Op::memory_size(result, memory), FuelCostsProvider::instance, )?; return Ok(()); } } // Case: fallback to generic `memory.grow` instruction - let delta = self.immediate_to_reg(delta)?; + let delta = self.copy_if_immediate(delta)?; self.push_instr_with_result( index_ty.ty(), - |result| Op::memory_grow(result, delta), + |result| Op::memory_grow(result, delta, memory), FuelCostsProvider::instance, )?; - self.push_param(Op::memory_index(mem))?; Ok(()) } @@ -808,8 +656,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_eq(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_eq, - Op::i32_eq_imm16, + Op::i32_eq_sss, + Op::i32_eq_ssi, wasm::i32_eq, FuncTranslator::fuse_eqz, ) @@ -818,8 +666,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_ne(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_ne, - Op::i32_ne_imm16, + Op::i32_not_eq_sss, + Op::i32_not_eq_ssi, wasm::i32_ne, FuncTranslator::fuse_nez, ) @@ -828,9 +676,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_lt_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_lt_s, - Op::i32_lt_s_imm16_rhs, - Op::i32_lt_s_imm16_lhs, + Op::i32_lt_sss, + Op::i32_lt_ssi, + Op::i32_lt_sis, wasm::i32_lt_s, ) } @@ -838,9 +686,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_lt_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_lt_u, - Op::i32_lt_u_imm16_rhs, - Op::i32_lt_u_imm16_lhs, + Op::u32_lt_sss, + Op::u32_lt_ssi, + Op::u32_lt_sis, wasm::i32_lt_u, ) } @@ -848,9 +696,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_gt_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_lt_s), - swap_ops!(Op::i32_lt_s_imm16_lhs), - swap_ops!(Op::i32_lt_s_imm16_rhs), + swap_ops!(Op::i32_lt_sss), + swap_ops!(Op::i32_lt_sis), + swap_ops!(Op::i32_lt_ssi), wasm::i32_gt_s, ) } @@ -858,9 +706,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_gt_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_lt_u), - swap_ops!(Op::i32_lt_u_imm16_lhs), - swap_ops!(Op::i32_lt_u_imm16_rhs), + swap_ops!(Op::u32_lt_sss), + swap_ops!(Op::u32_lt_sis), + swap_ops!(Op::u32_lt_ssi), wasm::i32_gt_u, ) } @@ -868,9 +716,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_le_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_le_s, - Op::i32_le_s_imm16_rhs, - Op::i32_le_s_imm16_lhs, + Op::i32_le_sss, + Op::i32_le_ssi, + Op::i32_le_sis, wasm::i32_le_s, ) } @@ -878,9 +726,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_le_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_le_u, - Op::i32_le_u_imm16_rhs, - Op::i32_le_u_imm16_lhs, + Op::u32_le_sss, + Op::u32_le_ssi, + Op::u32_le_sis, wasm::i32_le_u, ) } @@ -888,9 +736,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_ge_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_le_s), - swap_ops!(Op::i32_le_s_imm16_lhs), - swap_ops!(Op::i32_le_s_imm16_rhs), + swap_ops!(Op::i32_le_sss), + swap_ops!(Op::i32_le_sis), + swap_ops!(Op::i32_le_ssi), wasm::i32_ge_s, ) } @@ -898,9 +746,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_ge_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_le_u), - swap_ops!(Op::i32_le_u_imm16_lhs), - swap_ops!(Op::i32_le_u_imm16_rhs), + swap_ops!(Op::u32_le_sss), + swap_ops!(Op::u32_le_sis), + swap_ops!(Op::u32_le_ssi), wasm::i32_ge_u, ) } @@ -915,8 +763,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_eq(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_eq, - Op::i64_eq_imm16, + Op::i64_eq_sss, + Op::i64_eq_ssi, wasm::i64_eq, FuncTranslator::fuse_eqz, ) @@ -925,8 +773,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_ne(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_ne, - Op::i64_ne_imm16, + Op::i64_not_eq_sss, + Op::i64_not_eq_ssi, wasm::i64_ne, FuncTranslator::fuse_nez, ) @@ -935,9 +783,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_lt_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_lt_s, - Op::i64_lt_s_imm16_rhs, - Op::i64_lt_s_imm16_lhs, + Op::i64_lt_sss, + Op::i64_lt_ssi, + Op::i64_lt_sis, wasm::i64_lt_s, ) } @@ -945,9 +793,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_lt_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_lt_u, - Op::i64_lt_u_imm16_rhs, - Op::i64_lt_u_imm16_lhs, + Op::u64_lt_sss, + Op::u64_lt_ssi, + Op::u64_lt_sis, wasm::i64_lt_u, ) } @@ -955,9 +803,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_gt_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_lt_s), - swap_ops!(Op::i64_lt_s_imm16_lhs), - swap_ops!(Op::i64_lt_s_imm16_rhs), + swap_ops!(Op::i64_lt_sss), + swap_ops!(Op::i64_lt_sis), + swap_ops!(Op::i64_lt_ssi), wasm::i64_gt_s, ) } @@ -965,9 +813,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_gt_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_lt_u), - swap_ops!(Op::i64_lt_u_imm16_lhs), - swap_ops!(Op::i64_lt_u_imm16_rhs), + swap_ops!(Op::u64_lt_sss), + swap_ops!(Op::u64_lt_sis), + swap_ops!(Op::u64_lt_ssi), wasm::i64_gt_u, ) } @@ -975,9 +823,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_le_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_le_s, - Op::i64_le_s_imm16_rhs, - Op::i64_le_s_imm16_lhs, + Op::i64_le_sss, + Op::i64_le_ssi, + Op::i64_le_sis, wasm::i64_le_s, ) } @@ -985,9 +833,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_le_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_le_u, - Op::i64_le_u_imm16_rhs, - Op::i64_le_u_imm16_lhs, + Op::u64_le_sss, + Op::u64_le_ssi, + Op::u64_le_sis, wasm::i64_le_u, ) } @@ -995,9 +843,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_ge_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_le_s), - swap_ops!(Op::i64_le_s_imm16_lhs), - swap_ops!(Op::i64_le_s_imm16_rhs), + swap_ops!(Op::i64_le_sss), + swap_ops!(Op::i64_le_sis), + swap_ops!(Op::i64_le_ssi), wasm::i64_ge_s, ) } @@ -1005,93 +853,133 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_ge_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_le_u), - swap_ops!(Op::i64_le_u_imm16_lhs), - swap_ops!(Op::i64_le_u_imm16_rhs), + swap_ops!(Op::u64_le_sss), + swap_ops!(Op::u64_le_sis), + swap_ops!(Op::u64_le_ssi), wasm::i64_ge_u, ) } #[inline(never)] fn visit_f32_eq(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_eq, wasm::f32_eq) + self.translate_binary_commutative( + Op::f32_eq_sss, + Op::f32_eq_ssi, + wasm::f32_eq, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f32_ne(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_ne, wasm::f32_ne) + self.translate_binary_commutative( + Op::f32_not_eq_sss, + Op::f32_not_eq_ssi, + wasm::f32_ne, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f32_lt(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_lt, wasm::f32_lt) + self.translate_binary(Op::f32_lt_sss, Op::f32_lt_ssi, Op::f32_lt_sis, wasm::f32_lt) } #[inline(never)] fn visit_f32_gt(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f32_lt), wasm::f32_gt) + self.translate_binary( + swap_ops!(Op::f32_lt_sss), + swap_ops!(Op::f32_lt_sis), + swap_ops!(Op::f32_lt_ssi), + wasm::f32_gt, + ) } #[inline(never)] fn visit_f32_le(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_le, wasm::f32_le) + self.translate_binary(Op::f32_le_sss, Op::f32_le_ssi, Op::f32_le_sis, wasm::f32_le) } #[inline(never)] fn visit_f32_ge(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f32_le), wasm::f32_ge) + self.translate_binary( + swap_ops!(Op::f32_le_sss), + swap_ops!(Op::f32_le_sis), + swap_ops!(Op::f32_le_ssi), + wasm::f32_ge, + ) } #[inline(never)] fn visit_f64_eq(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_eq, wasm::f64_eq) + self.translate_binary_commutative( + Op::f64_eq_sss, + Op::f64_eq_ssi, + wasm::f64_eq, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f64_ne(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_ne, wasm::f64_ne) + self.translate_binary_commutative( + Op::f64_not_eq_sss, + Op::f64_not_eq_ssi, + wasm::f64_ne, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f64_lt(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_lt, wasm::f64_lt) + self.translate_binary(Op::f64_lt_sss, Op::f64_lt_ssi, Op::f64_lt_sis, wasm::f64_lt) } #[inline(never)] fn visit_f64_gt(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f64_lt), wasm::f64_gt) + self.translate_binary( + swap_ops!(Op::f64_lt_sss), + swap_ops!(Op::f64_lt_sis), + swap_ops!(Op::f64_lt_ssi), + wasm::f64_gt, + ) } #[inline(never)] fn visit_f64_le(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_le, wasm::f64_le) + self.translate_binary(Op::f64_le_sss, Op::f64_le_ssi, Op::f64_le_sis, wasm::f64_le) } #[inline(never)] fn visit_f64_ge(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f64_le), wasm::f64_ge) + self.translate_binary( + swap_ops!(Op::f64_le_sss), + swap_ops!(Op::f64_le_sis), + swap_ops!(Op::f64_le_ssi), + wasm::f64_ge, + ) } #[inline(never)] fn visit_i32_clz(&mut self) -> Self::Output { - self.translate_unary::(Op::i32_clz, wasm::i32_clz) + self.translate_unary::(Op::i32_clz_ss, wasm::i32_clz) } #[inline(never)] fn visit_i32_ctz(&mut self) -> Self::Output { - self.translate_unary::(Op::i32_ctz, wasm::i32_ctz) + self.translate_unary::(Op::i32_ctz_ss, wasm::i32_ctz) } #[inline(never)] fn visit_i32_popcnt(&mut self) -> Self::Output { - self.translate_unary::(Op::i32_popcnt, wasm::i32_popcnt) + self.translate_unary::(Op::i32_popcnt_ss, wasm::i32_popcnt) } #[inline(never)] fn visit_i32_add(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_add, - Op::i32_add_imm16, + Op::i32_add_sss, + Op::i32_add_ssi, wasm::i32_add, FuncTranslator::no_opt_ri, ) @@ -1099,10 +987,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_sub(&mut self) -> Self::Output { - self.translate_isub( - Op::i32_sub, - Op::i32_add_imm16, - Op::i32_sub_imm16_lhs, + self.translate_binary( + Op::i32_sub_sss, + Op::i32_sub_ssi, + Op::i32_sub_sis, wasm::i32_sub, ) } @@ -1110,8 +998,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_mul(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_mul, - Op::i32_mul_imm16, + Op::i32_mul_sss, + Op::i32_mul_ssi, wasm::i32_mul, FuncTranslator::no_opt_ri, ) @@ -1120,9 +1008,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_div_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_div_s, - Op::i32_div_s_imm16_rhs, - Op::i32_div_s_imm16_lhs, + Op::i32_div_sss, + Op::i32_div_ssi, + Op::i32_div_sis, wasm::i32_div_s, ) } @@ -1130,9 +1018,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_div_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_div_u, - Op::i32_div_u_imm16_rhs, - Op::i32_div_u_imm16_lhs, + Op::u32_div_sss, + Op::u32_div_ssi, + Op::u32_div_sis, wasm::i32_div_u, ) } @@ -1140,9 +1028,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rem_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_rem_s, - Op::i32_rem_s_imm16_rhs, - Op::i32_rem_s_imm16_lhs, + Op::i32_rem_sss, + Op::i32_rem_ssi, + Op::i32_rem_sis, wasm::i32_rem_s, ) } @@ -1150,9 +1038,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rem_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_rem_u, - Op::i32_rem_u_imm16_rhs, - Op::i32_rem_u_imm16_lhs, + Op::u32_rem_sss, + Op::u32_rem_ssi, + Op::u32_rem_sis, wasm::i32_rem_u, ) } @@ -1160,8 +1048,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bitand, - Op::i32_bitand_imm16, + Op::i32_bitand_sss, + Op::i32_bitand_ssi, wasm::i32_bitand, FuncTranslator::no_opt_ri, ) @@ -1170,8 +1058,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bitor, - Op::i32_bitor_imm16, + Op::i32_bitor_sss, + Op::i32_bitor_ssi, wasm::i32_bitor, FuncTranslator::no_opt_ri, ) @@ -1180,8 +1068,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bitxor, - Op::i32_bitxor_imm16, + Op::i32_bitxor_sss, + Op::i32_bitxor_ssi, wasm::i32_bitxor, FuncTranslator::no_opt_ri, ) @@ -1190,9 +1078,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_shl(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_shl, - Op::i32_shl_by, - Op::i32_shl_imm16, + Op::i32_shl_sss, + Op::i32_shl_ssi, + Op::i32_shl_sis, wasm::i32_shl, ) } @@ -1200,19 +1088,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_shr_s(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_shr_s, - Op::i32_shr_s_by, - Op::i32_shr_s_imm16, + Op::i32_shr_sss, + Op::i32_shr_ssi, + Op::i32_shr_sis, wasm::i32_shr_s, ) } #[inline(never)] fn visit_i32_shr_u(&mut self) -> Self::Output { - self.translate_shift::( - Op::i32_shr_u, - Op::i32_shr_u_by, - Op::i32_shr_u_imm16, + self.translate_shift::( + Op::u32_shr_sss, + Op::u32_shr_ssi, + Op::u32_shr_sis, wasm::i32_shr_u, ) } @@ -1220,9 +1108,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rotl(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_rotl, - Op::i32_rotl_by, - Op::i32_rotl_imm16, + Op::i32_rotl_sss, + Op::i32_rotl_ssi, + Op::i32_rotl_sis, wasm::i32_rotl, ) } @@ -1230,33 +1118,33 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rotr(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_rotr, - Op::i32_rotr_by, - Op::i32_rotr_imm16, + Op::i32_rotr_sss, + Op::i32_rotr_ssi, + Op::i32_rotr_sis, wasm::i32_rotr, ) } #[inline(never)] fn visit_i64_clz(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_clz, wasm::i64_clz) + self.translate_unary::(Op::i64_clz_ss, wasm::i64_clz) } #[inline(never)] fn visit_i64_ctz(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_ctz, wasm::i64_ctz) + self.translate_unary::(Op::i64_ctz_ss, wasm::i64_ctz) } #[inline(never)] fn visit_i64_popcnt(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_popcnt, wasm::i64_popcnt) + self.translate_unary::(Op::i64_popcnt_ss, wasm::i64_popcnt) } #[inline(never)] fn visit_i64_add(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_add, - Op::i64_add_imm16, + Op::i64_add_sss, + Op::i64_add_ssi, wasm::i64_add, FuncTranslator::no_opt_ri, ) @@ -1264,10 +1152,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_sub(&mut self) -> Self::Output { - self.translate_isub( - Op::i64_sub, - Op::i64_add_imm16, - Op::i64_sub_imm16_lhs, + self.translate_binary( + Op::i64_sub_sss, + Op::i64_sub_ssi, + Op::i64_sub_sis, wasm::i64_sub, ) } @@ -1275,8 +1163,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_mul(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_mul, - Op::i64_mul_imm16, + Op::i64_mul_sss, + Op::i64_mul_ssi, wasm::i64_mul, FuncTranslator::no_opt_ri, ) @@ -1285,9 +1173,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_div_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_div_s, - Op::i64_div_s_imm16_rhs, - Op::i64_div_s_imm16_lhs, + Op::i64_div_sss, + Op::i64_div_ssi, + Op::i64_div_sis, wasm::i64_div_s, ) } @@ -1295,9 +1183,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_div_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_div_u, - Op::i64_div_u_imm16_rhs, - Op::i64_div_u_imm16_lhs, + Op::u64_div_sss, + Op::u64_div_ssi, + Op::u64_div_sis, wasm::i64_div_u, ) } @@ -1305,9 +1193,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rem_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_rem_s, - Op::i64_rem_s_imm16_rhs, - Op::i64_rem_s_imm16_lhs, + Op::i64_rem_sss, + Op::i64_rem_ssi, + Op::i64_rem_sis, wasm::i64_rem_s, ) } @@ -1315,9 +1203,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rem_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_rem_u, - Op::i64_rem_u_imm16_rhs, - Op::i64_rem_u_imm16_lhs, + Op::u64_rem_sss, + Op::u64_rem_ssi, + Op::u64_rem_sis, wasm::i64_rem_u, ) } @@ -1325,8 +1213,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bitand, - Op::i64_bitand_imm16, + Op::i64_bitand_sss, + Op::i64_bitand_ssi, wasm::i64_bitand, FuncTranslator::no_opt_ri, ) @@ -1335,8 +1223,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bitor, - Op::i64_bitor_imm16, + Op::i64_bitor_sss, + Op::i64_bitor_ssi, wasm::i64_bitor, FuncTranslator::no_opt_ri, ) @@ -1345,8 +1233,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bitxor, - Op::i64_bitxor_imm16, + Op::i64_bitxor_sss, + Op::i64_bitxor_ssi, wasm::i64_bitxor, FuncTranslator::no_opt_ri, ) @@ -1355,9 +1243,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_shl(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_shl, - Op::i64_shl_by, - Op::i64_shl_imm16, + Op::i64_shl_sss, + Op::i64_shl_ssi, + Op::i64_shl_sis, wasm::i64_shl, ) } @@ -1365,19 +1253,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_shr_s(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_shr_s, - Op::i64_shr_s_by, - Op::i64_shr_s_imm16, + Op::i64_shr_sss, + Op::i64_shr_ssi, + Op::i64_shr_sis, wasm::i64_shr_s, ) } #[inline(never)] fn visit_i64_shr_u(&mut self) -> Self::Output { - self.translate_shift::( - Op::i64_shr_u, - Op::i64_shr_u_by, - Op::i64_shr_u_imm16, + self.translate_shift::( + Op::u64_shr_sss, + Op::u64_shr_ssi, + Op::u64_shr_sis, wasm::i64_shr_u, ) } @@ -1385,9 +1273,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rotl(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_rotl, - Op::i64_rotl_by, - Op::i64_rotl_imm16, + Op::i64_rotl_sss, + Op::i64_rotl_ssi, + Op::i64_rotl_sis, wasm::i64_rotl, ) } @@ -1395,181 +1283,251 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rotr(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_rotr, - Op::i64_rotr_by, - Op::i64_rotr_imm16, + Op::i64_rotr_sss, + Op::i64_rotr_ssi, + Op::i64_rotr_sis, wasm::i64_rotr, ) } #[inline(never)] fn visit_f32_abs(&mut self) -> Self::Output { - self.translate_unary(Op::f32_abs, wasm::f32_abs) + self.translate_unary(Op::f32_abs_ss, wasm::f32_abs) } #[inline(never)] fn visit_f32_neg(&mut self) -> Self::Output { - self.translate_unary(Op::f32_neg, wasm::f32_neg) + self.translate_unary(Op::f32_neg_ss, wasm::f32_neg) } #[inline(never)] fn visit_f32_ceil(&mut self) -> Self::Output { - self.translate_unary(Op::f32_ceil, wasm::f32_ceil) + self.translate_unary(Op::f32_ceil_ss, wasm::f32_ceil) } #[inline(never)] fn visit_f32_floor(&mut self) -> Self::Output { - self.translate_unary(Op::f32_floor, wasm::f32_floor) + self.translate_unary(Op::f32_floor_ss, wasm::f32_floor) } #[inline(never)] fn visit_f32_trunc(&mut self) -> Self::Output { - self.translate_unary(Op::f32_trunc, wasm::f32_trunc) + self.translate_unary(Op::f32_trunc_ss, wasm::f32_trunc) } #[inline(never)] fn visit_f32_nearest(&mut self) -> Self::Output { - self.translate_unary(Op::f32_nearest, wasm::f32_nearest) + self.translate_unary(Op::f32_nearest_ss, wasm::f32_nearest) } #[inline(never)] fn visit_f32_sqrt(&mut self) -> Self::Output { - self.translate_unary(Op::f32_sqrt, wasm::f32_sqrt) + self.translate_unary(Op::f32_sqrt_ss, wasm::f32_sqrt) } #[inline(never)] fn visit_f32_add(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_add, wasm::f32_add) + self.translate_binary( + Op::f32_add_sss, + Op::f32_add_ssi, + Op::f32_add_sis, + wasm::f32_add, + ) } #[inline(never)] fn visit_f32_sub(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_sub, wasm::f32_sub) + self.translate_binary( + Op::f32_sub_sss, + Op::f32_sub_ssi, + Op::f32_sub_sis, + wasm::f32_sub, + ) } #[inline(never)] fn visit_f32_mul(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_mul, wasm::f32_mul) + self.translate_binary( + Op::f32_mul_sss, + Op::f32_mul_ssi, + Op::f32_mul_sis, + wasm::f32_mul, + ) } #[inline(never)] fn visit_f32_div(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_div, wasm::f32_div) + self.translate_binary( + Op::f32_div_sss, + Op::f32_div_ssi, + Op::f32_div_sis, + wasm::f32_div, + ) } #[inline(never)] fn visit_f32_min(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_min, wasm::f32_min) + self.translate_binary( + Op::f32_min_sss, + Op::f32_min_ssi, + Op::f32_min_sis, + wasm::f32_min, + ) } #[inline(never)] fn visit_f32_max(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_max, wasm::f32_max) + self.translate_binary( + Op::f32_max_sss, + Op::f32_max_ssi, + Op::f32_max_sis, + wasm::f32_max, + ) } #[inline(never)] fn visit_f32_copysign(&mut self) -> Self::Output { - self.translate_fcopysign::(Op::f32_copysign, Op::f32_copysign_imm, wasm::f32_copysign) + self.translate_fcopysign::( + Op::f32_copysign_sss, + Op::f32_copysign_ssi, + Op::f32_copysign_sis, + wasm::f32_copysign, + ) } #[inline(never)] fn visit_f64_abs(&mut self) -> Self::Output { - self.translate_unary(Op::f64_abs, wasm::f64_abs) + self.translate_unary(Op::f64_abs_ss, wasm::f64_abs) } #[inline(never)] fn visit_f64_neg(&mut self) -> Self::Output { - self.translate_unary(Op::f64_neg, wasm::f64_neg) + self.translate_unary(Op::f64_neg_ss, wasm::f64_neg) } #[inline(never)] fn visit_f64_ceil(&mut self) -> Self::Output { - self.translate_unary(Op::f64_ceil, wasm::f64_ceil) + self.translate_unary(Op::f64_ceil_ss, wasm::f64_ceil) } #[inline(never)] fn visit_f64_floor(&mut self) -> Self::Output { - self.translate_unary(Op::f64_floor, wasm::f64_floor) + self.translate_unary(Op::f64_floor_ss, wasm::f64_floor) } #[inline(never)] fn visit_f64_trunc(&mut self) -> Self::Output { - self.translate_unary(Op::f64_trunc, wasm::f64_trunc) + self.translate_unary(Op::f64_trunc_ss, wasm::f64_trunc) } #[inline(never)] fn visit_f64_nearest(&mut self) -> Self::Output { - self.translate_unary(Op::f64_nearest, wasm::f64_nearest) + self.translate_unary(Op::f64_nearest_ss, wasm::f64_nearest) } #[inline(never)] fn visit_f64_sqrt(&mut self) -> Self::Output { - self.translate_unary(Op::f64_sqrt, wasm::f64_sqrt) + self.translate_unary(Op::f64_sqrt_ss, wasm::f64_sqrt) } #[inline(never)] fn visit_f64_add(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_add, wasm::f64_add) + self.translate_binary( + Op::f64_add_sss, + Op::f64_add_ssi, + Op::f64_add_sis, + wasm::f64_add, + ) } #[inline(never)] fn visit_f64_sub(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_sub, wasm::f64_sub) + self.translate_binary( + Op::f64_sub_sss, + Op::f64_sub_ssi, + Op::f64_sub_sis, + wasm::f64_sub, + ) } #[inline(never)] fn visit_f64_mul(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_mul, wasm::f64_mul) + self.translate_binary( + Op::f64_mul_sss, + Op::f64_mul_ssi, + Op::f64_mul_sis, + wasm::f64_mul, + ) } #[inline(never)] fn visit_f64_div(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_div, wasm::f64_div) + self.translate_binary( + Op::f64_div_sss, + Op::f64_div_ssi, + Op::f64_div_sis, + wasm::f64_div, + ) } #[inline(never)] fn visit_f64_min(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_min, wasm::f64_min) + self.translate_binary( + Op::f64_min_sss, + Op::f64_min_ssi, + Op::f64_min_sis, + wasm::f64_min, + ) } #[inline(never)] fn visit_f64_max(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_max, wasm::f64_max) + self.translate_binary( + Op::f64_max_sss, + Op::f64_max_ssi, + Op::f64_max_sis, + wasm::f64_max, + ) } #[inline(never)] fn visit_f64_copysign(&mut self) -> Self::Output { - self.translate_fcopysign::(Op::f64_copysign, Op::f64_copysign_imm, wasm::f64_copysign) + self.translate_fcopysign::( + Op::f64_copysign_sss, + Op::f64_copysign_ssi, + Op::f64_copysign_sis, + wasm::f64_copysign, + ) } #[inline(never)] fn visit_i32_wrap_i64(&mut self) -> Self::Output { - self.translate_unary(Op::i32_wrap_i64, wasm::i32_wrap_i64) + self.translate_unary(Op::i32_wrap_i64_ss, wasm::i32_wrap_i64) } #[inline(never)] fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f32_s, wasm::i32_trunc_f32_s) + self.translate_unary_fallible(Op::i32_trunc_f32_ss, wasm::i32_trunc_f32_s) } #[inline(never)] fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f32_u, wasm::i32_trunc_f32_u) + self.translate_unary_fallible(Op::u32_trunc_f32_ss, wasm::i32_trunc_f32_u) } #[inline(never)] fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f64_s, wasm::i32_trunc_f64_s) + self.translate_unary_fallible(Op::i32_trunc_f64_ss, wasm::i32_trunc_f64_s) } #[inline(never)] fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f64_u, wasm::i32_trunc_f64_u) + self.translate_unary_fallible(Op::u32_trunc_f64_ss, wasm::i32_trunc_f64_u) } #[inline(never)] fn visit_i64_extend_i32_s(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_extend32_s, wasm::i64_extend_i32_s) + self.translate_unary::(Op::i64_sext32_ss, wasm::i64_extend_i32_s) } #[inline(never)] @@ -1579,72 +1537,72 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f32_s, wasm::i64_trunc_f32_s) + self.translate_unary_fallible(Op::i64_trunc_f32_ss, wasm::i64_trunc_f32_s) } #[inline(never)] fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f32_u, wasm::i64_trunc_f32_u) + self.translate_unary_fallible(Op::u64_trunc_f32_ss, wasm::i64_trunc_f32_u) } #[inline(never)] fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f64_s, wasm::i64_trunc_f64_s) + self.translate_unary_fallible(Op::i64_trunc_f64_ss, wasm::i64_trunc_f64_s) } #[inline(never)] fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f64_u, wasm::i64_trunc_f64_u) + self.translate_unary_fallible(Op::u64_trunc_f64_ss, wasm::i64_trunc_f64_u) } #[inline(never)] fn visit_f32_convert_i32_s(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i32_s, wasm::f32_convert_i32_s) + self.translate_unary(Op::f32_convert_i32_ss, wasm::f32_convert_i32_s) } #[inline(never)] fn visit_f32_convert_i32_u(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i32_u, wasm::f32_convert_i32_u) + self.translate_unary(Op::f32_convert_u32_ss, wasm::f32_convert_i32_u) } #[inline(never)] fn visit_f32_convert_i64_s(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i64_s, wasm::f32_convert_i64_s) + self.translate_unary(Op::f32_convert_i64_ss, wasm::f32_convert_i64_s) } #[inline(never)] fn visit_f32_convert_i64_u(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i64_u, wasm::f32_convert_i64_u) + self.translate_unary(Op::f32_convert_u64_ss, wasm::f32_convert_i64_u) } #[inline(never)] fn visit_f32_demote_f64(&mut self) -> Self::Output { - self.translate_unary(Op::f32_demote_f64, wasm::f32_demote_f64) + self.translate_unary(Op::f32_demote_f64_ss, wasm::f32_demote_f64) } #[inline(never)] fn visit_f64_convert_i32_s(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i32_s, wasm::f64_convert_i32_s) + self.translate_unary(Op::f64_convert_i32_ss, wasm::f64_convert_i32_s) } #[inline(never)] fn visit_f64_convert_i32_u(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i32_u, wasm::f64_convert_i32_u) + self.translate_unary(Op::f64_convert_u32_ss, wasm::f64_convert_i32_u) } #[inline(never)] fn visit_f64_convert_i64_s(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i64_s, wasm::f64_convert_i64_s) + self.translate_unary(Op::f64_convert_i64_ss, wasm::f64_convert_i64_s) } #[inline(never)] fn visit_f64_convert_i64_u(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i64_u, wasm::f64_convert_i64_u) + self.translate_unary(Op::f64_convert_u64_ss, wasm::f64_convert_i64_u) } #[inline(never)] fn visit_f64_promote_f32(&mut self) -> Self::Output { - self.translate_unary(Op::f64_promote_f32, wasm::f64_promote_f32) + self.translate_unary(Op::f64_promote_f32_ss, wasm::f64_promote_f32) } #[inline(never)] @@ -1669,86 +1627,92 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_extend8_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_extend8_s, wasm::i32_extend8_s) + self.translate_unary(Op::i32_sext8_ss, wasm::i32_extend8_s) } #[inline(never)] fn visit_i32_extend16_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_extend16_s, wasm::i32_extend16_s) + self.translate_unary(Op::i32_sext16_ss, wasm::i32_extend16_s) } #[inline(never)] fn visit_i64_extend8_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_extend8_s, wasm::i64_extend8_s) + self.translate_unary(Op::i64_sext8_ss, wasm::i64_extend8_s) } #[inline(never)] fn visit_i64_extend16_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_extend16_s, wasm::i64_extend16_s) + self.translate_unary(Op::i64_sext16_ss, wasm::i64_extend16_s) } #[inline(never)] fn visit_i64_extend32_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_extend32_s, wasm::i64_extend32_s) + self.translate_unary(Op::i64_sext32_ss, wasm::i64_extend32_s) } #[inline(never)] fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f32_s, wasm::i32_trunc_sat_f32_s) + self.translate_unary(Op::i32_trunc_sat_f32_ss, wasm::i32_trunc_sat_f32_s) } #[inline(never)] fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f32_u, wasm::i32_trunc_sat_f32_u) + self.translate_unary(Op::u32_trunc_sat_f32_ss, wasm::i32_trunc_sat_f32_u) } #[inline(never)] fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f64_s, wasm::i32_trunc_sat_f64_s) + self.translate_unary(Op::i32_trunc_sat_f64_ss, wasm::i32_trunc_sat_f64_s) } #[inline(never)] fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f64_u, wasm::i32_trunc_sat_f64_u) + self.translate_unary(Op::u32_trunc_sat_f64_ss, wasm::i32_trunc_sat_f64_u) } #[inline(never)] fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f32_s, wasm::i64_trunc_sat_f32_s) + self.translate_unary(Op::i64_trunc_sat_f32_ss, wasm::i64_trunc_sat_f32_s) } #[inline(never)] fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f32_u, wasm::i64_trunc_sat_f32_u) + self.translate_unary(Op::u64_trunc_sat_f32_ss, wasm::i64_trunc_sat_f32_u) } #[inline(never)] fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f64_s, wasm::i64_trunc_sat_f64_s) + self.translate_unary(Op::i64_trunc_sat_f64_ss, wasm::i64_trunc_sat_f64_s) } #[inline(never)] fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f64_u, wasm::i64_trunc_sat_f64_u) + self.translate_unary(Op::u64_trunc_sat_f64_ss, wasm::i64_trunc_sat_f64_u) } #[inline(never)] fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; - self.push_instr(Op::memory_init(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::memory_index(mem))?; - self.push_param(Op::data_index(data_index))?; + let memory = index::Memory::try_from(mem)?; + let data = index::Data::from(data_index); + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; + self.push_instr( + Op::memory_init(memory, data, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } #[inline(never)] fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { bail_unreachable!(self); - self.push_instr(Op::data_drop(data_index), FuelCostsProvider::instance)?; + self.push_instr( + Op::data_drop(index::Data::from(data_index)), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1756,12 +1720,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; - self.push_instr(Op::memory_copy(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::memory_index(dst_mem))?; - self.push_param(Op::memory_index(src_mem))?; + let dst_memory = index::Memory::try_from(dst_mem)?; + let src_memory = index::Memory::try_from(src_mem)?; + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; + self.push_instr( + Op::memory_copy(dst_memory, src_memory, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1769,18 +1736,14 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, value, len) = self.stack.pop3(); - let dst = self.immediate_to_reg(dst)?; - let value = self.make_input(value, |_, value| { - let byte = u32::from(value) as u8; - Ok(Input::Immediate(byte)) - })?; - let len = self.immediate_to_reg(len)?; - let instr: Op = match value { - Input::Slot(value) => Op::memory_fill(dst, value, len), - Input::Immediate(value) => Op::memory_fill_imm(dst, value, len), - }; - self.push_instr(instr, FuelCostsProvider::instance)?; - self.push_param(Op::memory_index(mem))?; + let memory = index::Memory::try_from(mem)?; + let dst = self.copy_if_immediate(dst)?; + let len = self.copy_if_immediate(len)?; + let value = self.copy_if_immediate(value)?; + self.push_instr( + Op::memory_fill(memory, dst, len, value), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1788,19 +1751,25 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_init(&mut self, elem_index: u32, table: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; - self.push_instr(Op::table_init(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::table_index(table))?; - self.push_param(Op::elem_index(elem_index))?; + let table = index::Table::from(table); + let elem = index::Elem::from(elem_index); + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; + self.push_instr( + Op::table_init(table, elem, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } #[inline(never)] fn visit_elem_drop(&mut self, elem_index: u32) -> Self::Output { bail_unreachable!(self); - self.push_instr(Op::elem_drop(elem_index), FuelCostsProvider::instance)?; + self.push_instr( + Op::elem_drop(index::Elem::from(elem_index)), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1808,12 +1777,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; - self.push_instr(Op::table_copy(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::table_index(dst_table))?; - self.push_param(Op::table_index(src_table))?; + let dst_table = index::Table::from(dst_table); + let src_table = index::Table::from(src_table); + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; + self.push_instr( + Op::table_copy(dst_table, src_table, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1847,11 +1819,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.stack.push_local(input.local_index(), ValType::I64)?; self.visit_i64_eqz() } - Operand::Temp(input) => { + Operand::Temp(_) => { // Note: `funcref` and `externref` both serialize to `UntypedValue` // as `u64` so we can use `i64.eqz` translation for `ref.is_null` // via reinterpretation of the value's type. - self.stack.push_temp(ValType::I64, input.instr())?; + self.stack.push_temp(ValType::I64)?; self.visit_i64_eqz() } Operand::Immediate(input) => { @@ -1872,7 +1844,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { bail_unreachable!(self); self.push_instr_with_result( ValType::FuncRef, - |result| Op::ref_func(result, function_index), + |result| Op::ref_func(result, index::Func::from(function_index)), FuelCostsProvider::instance, )?; Ok(()) @@ -1882,11 +1854,14 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_fill(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let (dst, value, len) = self.stack.pop3(); - let dst = self.immediate_to_reg(dst)?; - let value = self.immediate_to_reg(value)?; - let len = self.immediate_to_reg(len)?; - self.push_instr(Op::table_fill(dst, len, value), FuelCostsProvider::instance)?; - self.push_param(Op::table_index(table))?; + let table = index::Table::from(table); + let dst = self.copy_if_immediate(dst)?; + let value = self.copy_if_immediate(value)?; + let len = self.copy_if_immediate(len)?; + self.push_instr( + Op::table_fill(table, dst, len, value), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1894,19 +1869,19 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_get(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index = self.stack.pop(); let item_ty = table_type.element(); let index_ty = table_type.index_ty(); - let index = self.make_index32(index, index_ty)?; + let index = self.make_index32_or_copy(index, index_ty)?; self.push_instr_with_result( item_ty, |result| match index { - Input::Slot(index) => Op::table_get(result, index), - Input::Immediate(index) => Op::table_get_imm(result, index), + Input::Slot(index) => Op::table_get_ss(result, index, table), + Input::Immediate(index) => Op::table_get_si(result, index, table), }, FuelCostsProvider::instance, )?; - self.push_param(Op::table_index(table))?; Ok(()) } @@ -1914,16 +1889,22 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_set(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index_ty = table_type.index_ty(); let (index, value) = self.stack.pop2(); - let index = self.make_index32(index, index_ty)?; - let value = self.layout.operand_to_reg(value)?; - let instr = match index { - Input::Slot(index) => Op::table_set(index, value), - Input::Immediate(index) => Op::table_set_at(value, index), + let index = self.make_index32_or_copy(index, index_ty)?; + let value = self.make_input(value, |_this, value| { + Ok(Input::Immediate(u64::from(value.untyped()))) + })?; + let instr = match (index, value) { + (Input::Slot(index), Input::Slot(value)) => Op::table_set_ss(table, index, value), + (Input::Slot(index), Input::Immediate(value)) => Op::table_set_si(table, index, value), + (Input::Immediate(index), Input::Slot(value)) => Op::table_set_is(table, index, value), + (Input::Immediate(index), Input::Immediate(value)) => { + Op::table_set_ii(table, index, value) + } }; self.push_instr(instr, FuelCostsProvider::instance)?; - self.push_param(Op::table_index(table))?; Ok(()) } @@ -1931,6 +1912,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_grow(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index_ty = table_type.index_ty(); let (value, delta) = self.stack.pop2(); if let Operand::Immediate(delta) = delta { @@ -1953,14 +1935,13 @@ impl<'a> VisitOperator<'a> for FuncTranslator { return Ok(()); } } - let value = self.immediate_to_reg(value)?; - let delta = self.immediate_to_reg(delta)?; + let value = self.copy_if_immediate(value)?; + let delta = self.copy_if_immediate(delta)?; self.push_instr_with_result( index_ty.ty(), - |result| Op::table_grow(result, delta, value), + |result| Op::table_grow(result, delta, value, table), FuelCostsProvider::instance, )?; - self.push_param(Op::table_index(table))?; Ok(()) } @@ -1968,6 +1949,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_size(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index_ty = table_type.index_ty(); self.push_instr_with_result( index_ty.ty(), @@ -1979,55 +1961,18 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_return_call(&mut self, function_index: u32) -> Self::Output { - bail_unreachable!(self); - let func_idx = FuncIdx::from(function_index); - let func_type = self.resolve_func_type(func_idx); - let len_params = usize::from(func_type.len_params()); - let instr = match self.module.get_engine_func(func_idx) { - Some(engine_func) => { - // Case: We are calling an internal function and can optimize - // this case by using the special instruction for it. - match len_params { - 0 => Op::return_call_internal_0(engine_func), - _ => Op::return_call_internal(engine_func), - } - } - None => { - // Case: We are calling an imported function and must use the - // general calling operator for it. - match len_params { - 0 => Op::return_call_imported_0(function_index), - _ => Op::return_call_imported(function_index), - } - } - }; - self.push_instr(instr, FuelCostsProvider::call)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; + self.translate_call( + function_index, + Op::return_call_internal, + Op::return_call_imported, + )?; self.reachable = false; Ok(()) } #[inline(never)] fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { - bail_unreachable!(self); - let func_type = self.resolve_type(type_index); - let index = self.stack.pop(); - let indirect_params = self.call_indirect_params(index, table_index)?; - let len_params = usize::from(func_type.len_params()); - let instr = match (len_params, indirect_params) { - (0, Op::CallIndirectParams { .. }) => Op::return_call_indirect_0(type_index), - (0, Op::CallIndirectParamsImm16 { .. }) => Op::return_call_indirect_0_imm16(type_index), - (_, Op::CallIndirectParams { .. }) => Op::return_call_indirect(type_index), - (_, Op::CallIndirectParamsImm16 { .. }) => Op::return_call_indirect_imm16(type_index), - _ => unreachable!(), - }; - self.push_instr(instr, FuelCostsProvider::call)?; - self.push_param(indirect_params)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; + self.translate_call_indirect(type_index, table_index, Op::return_call_indirect)?; self.reachable = false; Ok(()) } @@ -2044,11 +1989,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_mul_wide_s(&mut self) -> Self::Output { - self.translate_i64_mul_wide_sx(Op::i64_mul_wide_s, wasm::i64_mul_wide_s, true) + self.translate_i64_mul_wide_sx(Op::i64_mul_wide, wasm::i64_mul_wide_s, true) } #[inline(never)] fn visit_i64_mul_wide_u(&mut self) -> Self::Output { - self.translate_i64_mul_wide_sx(Op::i64_mul_wide_u, wasm::i64_mul_wide_u, false) + self.translate_i64_mul_wide_sx(Op::u64_mul_wide, wasm::i64_mul_wide_u, false) } } diff --git a/crates/wasmi/src/engine/translator/labels.rs b/crates/wasmi/src/engine/translator/labels.rs deleted file mode 100644 index eab8c906ac8..00000000000 --- a/crates/wasmi/src/engine/translator/labels.rs +++ /dev/null @@ -1,212 +0,0 @@ -use crate::{engine::translator::utils::Instr, ir::BranchOffset, Error}; -use alloc::vec::Vec; -use core::{ - fmt::{self, Display}, - slice::Iter as SliceIter, -}; - -/// A label during the Wasmi compilation process. -#[derive(Debug, Copy, Clone)] -pub enum Label { - /// The label has already been pinned to a particular [`Instr`]. - Pinned(Instr), - /// The label is still unpinned. - Unpinned, -} - -/// A reference to an [`Label`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct LabelRef(u32); - -impl LabelRef { - /// Returns the `usize` value of the [`LabelRef`]. - #[inline] - fn into_usize(self) -> usize { - self.0 as usize - } -} - -/// The label registry. -/// -/// Allows to allocate new labels pin them and resolve pinned ones. -#[derive(Debug, Default)] -pub struct LabelRegistry { - labels: Vec