From f16afd9f22c73687e3aa2c029f632e44e408e2d5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 3 Sep 2025 12:43:02 +0200 Subject: [PATCH 001/423] add wasmi_ir2 as wasmi dependency --- crates/wasmi/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/Cargo.toml b/crates/wasmi/Cargo.toml index 47c5adf5b4..93953f2fd5 100644 --- a/crates/wasmi/Cargo.toml +++ b/crates/wasmi/Cargo.toml @@ -22,6 +22,7 @@ exclude = [ wasmi_core = { workspace = true } wasmi_collections = { workspace = true } wasmi_ir = { workspace = true } +wasmi_ir2 = { workspace = true } wasmparser = { workspace = true, features = ["validate", "features"] } wat = { workspace = true, optional = true } spin = { version = "0.9", default-features = false, features = [ From a43e2c81cb30ef8564c33f4f4608d553d3f77df0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 3 Sep 2025 12:43:17 +0200 Subject: [PATCH 002/423] re-export wasmi_ir2 as ir2 within wasmi --- crates/wasmi/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/lib.rs b/crates/wasmi/src/lib.rs index 235aa45693..77fd1c1c1f 100644 --- a/crates/wasmi/src/lib.rs +++ b/crates/wasmi/src/lib.rs @@ -140,10 +140,14 @@ mod core { #[doc(inline)] use wasmi_collections as collections; -/// Definitions from the `wasmi_collections` crate. +/// Definitions from the `wasmi_ir` crate. #[doc(inline)] use wasmi_ir as ir; +/// Definitions from the `wasmi_ir2` crate. +#[doc(inline)] +use wasmi_ir2 as ir2; + /// Defines some errors that may occur upon interaction with Wasmi. pub mod errors { pub use super::{ From 22e65365b0b191432bc9d890cbe895967b7bf2f8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 3 Sep 2025 12:43:25 +0200 Subject: [PATCH 003/423] update Cargo.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 867d222e5a..1efe7d603e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1716,6 +1716,7 @@ dependencies = [ "wasmi_collections 0.51.0", "wasmi_core 0.51.0", "wasmi_ir 0.51.0", + "wasmi_ir2", "wasmparser 0.228.0", "wat", ] From d4792148989384b874a5a3ddd57907ad45fdbdf4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:13:29 +0200 Subject: [PATCH 004/423] remove old wasmi_ir dependency from wasmi --- Cargo.lock | 1 - crates/wasmi/Cargo.toml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1efe7d603e..4bfb2c73e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1715,7 +1715,6 @@ dependencies = [ "spin", "wasmi_collections 0.51.0", "wasmi_core 0.51.0", - "wasmi_ir 0.51.0", "wasmi_ir2", "wasmparser 0.228.0", "wat", diff --git a/crates/wasmi/Cargo.toml b/crates/wasmi/Cargo.toml index 93953f2fd5..d3766d9434 100644 --- a/crates/wasmi/Cargo.toml +++ b/crates/wasmi/Cargo.toml @@ -21,7 +21,6 @@ exclude = [ [dependencies] wasmi_core = { workspace = true } wasmi_collections = { workspace = true } -wasmi_ir = { workspace = true } wasmi_ir2 = { workspace = true } wasmparser = { workspace = true, features = ["validate", "features"] } wat = { workspace = true, optional = true } @@ -52,7 +51,7 @@ prefer-btree-collections = [ "wasmparser/prefer-btree-collections", ] wat = ["dep:wat", "std"] -simd = ["wasmi_core/simd", "wasmi_ir/simd", "wasmparser/simd"] +simd = ["wasmi_core/simd", "wasmi_ir2/simd", "wasmparser/simd"] # Enables extra checks performed during Wasmi bytecode execution. # From db48f14420779f256593ea0ab5211aebe385aeaf Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:15:31 +0200 Subject: [PATCH 005/423] remove re-export of old wasmi_ir --- crates/wasmi/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/wasmi/src/lib.rs b/crates/wasmi/src/lib.rs index 77fd1c1c1f..ac29e3897e 100644 --- a/crates/wasmi/src/lib.rs +++ b/crates/wasmi/src/lib.rs @@ -142,11 +142,7 @@ use wasmi_collections as collections; /// Definitions from the `wasmi_ir` crate. #[doc(inline)] -use wasmi_ir as ir; - -/// Definitions from the `wasmi_ir2` crate. -#[doc(inline)] -use wasmi_ir2 as ir2; +use wasmi_ir2 as ir; /// Defines some errors that may occur upon interaction with Wasmi. pub mod errors { From d4ea0fa13064157579a64b12193a2da8bb324ef6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:16:20 +0200 Subject: [PATCH 006/423] rename field of ConsumeFuel operator --- crates/wasmi/src/engine/executor/instrs.rs | 4 ++-- crates/wasmi/src/engine/translator/utils.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/executor/instrs.rs b/crates/wasmi/src/engine/executor/instrs.rs index a3eeb74dc2..3b53e5e749 100644 --- a/crates/wasmi/src/engine/executor/instrs.rs +++ b/crates/wasmi/src/engine/executor/instrs.rs @@ -135,8 +135,8 @@ impl<'engine> Executor<'engine> { loop { match *self.ip.get() { Instr::Trap { trap_code } => self.execute_trap(trap_code)?, - Instr::ConsumeFuel { block_fuel } => { - self.execute_consume_fuel(store.inner_mut(), block_fuel)? + Instr::ConsumeFuel { fuel } => { + self.execute_consume_fuel(store.inner_mut(), fuel)? } Instr::Return => { forward_return!(self.execute_return(store.inner_mut())) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index d8b5564f5b..4ee6d568c9 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -169,7 +169,7 @@ pub trait BumpFuelConsumption { impl BumpFuelConsumption for Op { fn bump_fuel_consumption(&mut self, delta: u64) -> Result<(), Error> { match self { - Self::ConsumeFuel { block_fuel } => block_fuel.bump_by(delta).map_err(Error::from), + Self::ConsumeFuel { fuel } => fuel.bump_by(delta).map_err(Error::from), instr => panic!("expected `Op::ConsumeFuel` but found: {instr:?}"), } } From fea37ac871b5fdbc6950582df33f02ef8c31b1e6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:16:48 +0200 Subject: [PATCH 007/423] add ToBits utility trait --- crates/wasmi/src/engine/translator/utils.rs | 51 +++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 4ee6d568c9..f18448f830 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -258,3 +258,54 @@ impl Instr { self.0.abs_diff(other.0) } } + +/// Types that can be converted into bits. +pub trait ToBits { + /// The output bits type of [`ToBits`]. + type Out: Copy; + + /// Converts `self` into a 32-bit `u32` value. + fn to_bits(self) -> Self::Out; +} + +impl ToBits for u32 { + type Out = u32; + fn to_bits(self) -> Self::Out { + self + } +} + +impl ToBits for i32 { + type Out = u32; + fn to_bits(self) -> Self::Out { + u32::from_ne_bytes(self.to_ne_bytes()) + } +} + +impl ToBits for f32 { + type Out = u32; + fn to_bits(self) -> Self::Out { + self.to_bits() + } +} + +impl ToBits for u64 { + type Out = u64; + fn to_bits(self) -> Self::Out { + self + } +} + +impl ToBits for i64 { + type Out = u64; + fn to_bits(self) -> Self::Out { + u64::from_ne_bytes(self.to_ne_bytes()) + } +} + +impl ToBits for f64 { + type Out = u64; + fn to_bits(self) -> Self::Out { + self.to_bits() + } +} From 2bb3b5ee0af60cf0aa2f537208e95ec47dc0ad8a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:16:59 +0200 Subject: [PATCH 008/423] add IntoShiftAmount utility trait --- .../wasmi/src/engine/translator/func/utils.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func/utils.rs b/crates/wasmi/src/engine/translator/func/utils.rs index 734e1c1435..65ba25446e 100644 --- a/crates/wasmi/src/engine/translator/func/utils.rs +++ b/crates/wasmi/src/engine/translator/func/utils.rs @@ -62,3 +62,31 @@ pub enum Input { /// A 16-bit encoded immediate value operand. Immediate(T), } + +pub trait IntoShiftAmount { + /// The type denoting the shift amount. + /// + /// This is an unsigned integer ranging from `1..N` where `N` is the number of bits in `Self`. + type Value: Copy; + + /// Returns `self` wrapped into a proper shift amount for `Self`. + /// + /// Returns `None` if the resulting shift amount is 0, a.k.a. a no-op. + fn into_shift_amount(self) -> Option; +} + +macro_rules! impl_into_shift_amount { + ( $($ty:ty),* $(,)? ) => { + $( + impl IntoShiftAmount for $ty { + type Value = u8; + + fn into_shift_amount(self) -> Option { + let len_bits = (::core::mem::size_of::() * 8) as Self; + self.checked_rem_euclid(len_bits) + } + } + )* + }; +} +impl_into_shift_amount!(i32, u32, i64, u64, i128, u128); From 50ad14ea5e192abb5f87ab17039343a8f8dfc597 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:18:20 +0200 Subject: [PATCH 009/423] remove no longer valid imports --- crates/wasmi/src/engine/executor/instrs.rs | 2 +- .../src/engine/executor/instrs/binary.rs | 2 +- .../src/engine/executor/instrs/branch.rs | 2 +- .../src/engine/executor/instrs/comparison.rs | 5 +---- .../wasmi/src/engine/executor/instrs/copy.rs | 2 +- .../src/engine/executor/instrs/global.rs | 2 +- .../wasmi/src/engine/executor/instrs/load.rs | 2 +- .../src/engine/executor/instrs/return_.rs | 2 +- .../src/engine/executor/instrs/select.rs | 2 +- .../wasmi/src/engine/executor/instrs/simd.rs | 2 +- .../wasmi/src/engine/executor/instrs/store.rs | 12 +----------- .../wasmi/src/engine/executor/instrs/table.rs | 1 - .../wasmi/src/engine/executor/instrs/utils.rs | 2 +- .../wasmi/src/engine/translator/comparator.rs | 3 +-- .../src/engine/translator/func/layout/mod.rs | 7 +------ .../wasmi/src/engine/translator/func/mod.rs | 19 ++++--------------- crates/wasmi/src/engine/translator/func/op.rs | 5 ++++- .../src/engine/translator/func/simd/mod.rs | 5 ----- .../src/engine/translator/func/simd/op.rs | 2 +- .../wasmi/src/engine/translator/func/utils.rs | 2 +- .../src/engine/translator/relink_result.rs | 2 +- crates/wasmi/src/engine/translator/utils.rs | 2 +- 22 files changed, 26 insertions(+), 59 deletions(-) diff --git a/crates/wasmi/src/engine/executor/instrs.rs b/crates/wasmi/src/engine/executor/instrs.rs index 3b53e5e749..b588b51e54 100644 --- a/crates/wasmi/src/engine/executor/instrs.rs +++ b/crates/wasmi/src/engine/executor/instrs.rs @@ -9,7 +9,7 @@ use crate::{ DedupFuncType, EngineFunc, }, - ir::{index, BlockFuel, Const16, Offset64Hi, Op, ShiftAmount, Slot}, + ir::{index, BlockFuel, Op, Slot}, memory::DataSegment, store::{PrunedStore, StoreInner}, table::ElementSegment, diff --git a/crates/wasmi/src/engine/executor/instrs/binary.rs b/crates/wasmi/src/engine/executor/instrs/binary.rs index c3d55165ba..d6b92ed4a8 100644 --- a/crates/wasmi/src/engine/executor/instrs/binary.rs +++ b/crates/wasmi/src/engine/executor/instrs/binary.rs @@ -1,7 +1,7 @@ use super::{Executor, UntypedValueExt}; use crate::{ core::wasm, - ir::{Const16, ShiftAmount, Sign, Slot}, + ir::{Sign, Slot}, Error, TrapCode, }; diff --git a/crates/wasmi/src/engine/executor/instrs/branch.rs b/crates/wasmi/src/engine/executor/instrs/branch.rs index 7cb509c254..84e0adcfeb 100644 --- a/crates/wasmi/src/engine/executor/instrs/branch.rs +++ b/crates/wasmi/src/engine/executor/instrs/branch.rs @@ -2,7 +2,7 @@ use super::{Executor, UntypedValueCmpExt, UntypedValueExt}; use crate::{ core::{ReadAs, UntypedVal}, engine::utils::unreachable_unchecked, - ir::{BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset, Const16, Op, Slot}, + ir::{BranchOffset, Op, Slot}, }; use core::cmp; diff --git a/crates/wasmi/src/engine/executor/instrs/comparison.rs b/crates/wasmi/src/engine/executor/instrs/comparison.rs index c01625a35f..9d6f0b44d9 100644 --- a/crates/wasmi/src/engine/executor/instrs/comparison.rs +++ b/crates/wasmi/src/engine/executor/instrs/comparison.rs @@ -1,8 +1,5 @@ use super::{Executor, UntypedValueCmpExt}; -use crate::{ - core::wasm, - ir::{Const16, Slot}, -}; +use crate::{core::wasm, ir::Slot}; #[cfg(doc)] use crate::ir::Op; diff --git a/crates/wasmi/src/engine/executor/instrs/copy.rs b/crates/wasmi/src/engine/executor/instrs/copy.rs index 12e9cdd9f4..7447ac6727 100644 --- a/crates/wasmi/src/engine/executor/instrs/copy.rs +++ b/crates/wasmi/src/engine/executor/instrs/copy.rs @@ -2,7 +2,7 @@ use super::{Executor, InstructionPtr}; use crate::{ core::UntypedVal, engine::utils::unreachable_unchecked, - ir::{AnyConst32, Const32, FixedSlotSpan, Op, Slot, SlotSpan}, + ir::{FixedSlotSpan, Op, Slot, SlotSpan}, }; use core::slice; diff --git a/crates/wasmi/src/engine/executor/instrs/global.rs b/crates/wasmi/src/engine/executor/instrs/global.rs index 1f6484cfa2..9ab3882cda 100644 --- a/crates/wasmi/src/engine/executor/instrs/global.rs +++ b/crates/wasmi/src/engine/executor/instrs/global.rs @@ -1,7 +1,7 @@ use super::Executor; use crate::{ core::{hint, UntypedVal}, - ir::{index, Const16, Slot}, + ir::{index, Slot}, store::StoreInner, }; diff --git a/crates/wasmi/src/engine/executor/instrs/load.rs b/crates/wasmi/src/engine/executor/instrs/load.rs index 7577c461d8..01e7139521 100644 --- a/crates/wasmi/src/engine/executor/instrs/load.rs +++ b/crates/wasmi/src/engine/executor/instrs/load.rs @@ -1,7 +1,7 @@ use super::Executor; use crate::{ core::{wasm, UntypedVal, WriteAs}, - ir::{index::Memory, Address32, Offset16, Offset64, Offset64Hi, Offset64Lo, Slot}, + ir::{index::Memory, Offset16, Slot}, store::StoreInner, Error, TrapCode, diff --git a/crates/wasmi/src/engine/executor/instrs/return_.rs b/crates/wasmi/src/engine/executor/instrs/return_.rs index 10f5bdc7f1..f29139c589 100644 --- a/crates/wasmi/src/engine/executor/instrs/return_.rs +++ b/crates/wasmi/src/engine/executor/instrs/return_.rs @@ -2,7 +2,7 @@ use super::{ControlFlow, Executor, InstructionPtr}; use crate::{ core::UntypedVal, engine::{executor::stack::FrameSlots, utils::unreachable_unchecked}, - ir::{AnyConst32, BoundedSlotSpan, Const32, Op, Slot, SlotSpan}, + ir::{BoundedSlotSpan, Op, Slot, SlotSpan}, store::StoreInner, }; use core::slice; diff --git a/crates/wasmi/src/engine/executor/instrs/select.rs b/crates/wasmi/src/engine/executor/instrs/select.rs index 1aa23d2ab3..a578f640d1 100644 --- a/crates/wasmi/src/engine/executor/instrs/select.rs +++ b/crates/wasmi/src/engine/executor/instrs/select.rs @@ -2,7 +2,7 @@ use super::{Executor, InstructionPtr, UntypedValueExt}; use crate::{ core::{wasm, ReadAs, UntypedVal}, engine::utils::unreachable_unchecked, - ir::{Const16, Op, Slot}, + ir::{Op, Slot}, }; impl Executor<'_> { diff --git a/crates/wasmi/src/engine/executor/instrs/simd.rs b/crates/wasmi/src/engine/executor/instrs/simd.rs index ee66ddf125..1ab02d8c27 100644 --- a/crates/wasmi/src/engine/executor/instrs/simd.rs +++ b/crates/wasmi/src/engine/executor/instrs/simd.rs @@ -14,7 +14,7 @@ use crate::{ WriteAs, }, engine::{executor::InstructionPtr, utils::unreachable_unchecked}, - ir::{index, Address32, AnyConst32, Offset64, Offset64Lo, Offset8, Op, ShiftAmount, Slot}, + ir::{index, Op, Slot}, store::StoreInner, Error, TrapCode, diff --git a/crates/wasmi/src/engine/executor/instrs/store.rs b/crates/wasmi/src/engine/executor/instrs/store.rs index 0774df438b..3eb51684d7 100644 --- a/crates/wasmi/src/engine/executor/instrs/store.rs +++ b/crates/wasmi/src/engine/executor/instrs/store.rs @@ -2,17 +2,7 @@ use super::{Executor, InstructionPtr}; use crate::{ core::{wasm, ReadAs, UntypedVal}, engine::utils::unreachable_unchecked, - ir::{ - index::Memory, - Address32, - AnyConst16, - Const16, - Offset16, - Offset64, - Offset64Hi, - Offset64Lo, - Slot, - }, + ir::{index::Memory, Offset16, Slot}, store::StoreInner, Error, TrapCode, diff --git a/crates/wasmi/src/engine/executor/instrs/table.rs b/crates/wasmi/src/engine/executor/instrs/table.rs index a4a2c8f980..923d246043 100644 --- a/crates/wasmi/src/engine/executor/instrs/table.rs +++ b/crates/wasmi/src/engine/executor/instrs/table.rs @@ -5,7 +5,6 @@ use crate::{ errors::TableError, ir::{ index::{Elem, Table}, - Const32, Op, Slot, }, diff --git a/crates/wasmi/src/engine/executor/instrs/utils.rs b/crates/wasmi/src/engine/executor/instrs/utils.rs index fd93cd2e1e..9c81432186 100644 --- a/crates/wasmi/src/engine/executor/instrs/utils.rs +++ b/crates/wasmi/src/engine/executor/instrs/utils.rs @@ -1,6 +1,6 @@ use super::Executor; use crate::{ - ir::{index::Memory, Offset64Hi, Slot}, + ir::{index::Memory, Slot}, store::StoreInner, }; diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index 98d97c9b7b..7a66e983d4 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -1,6 +1,5 @@ use crate::{ - core::UntypedVal, - ir::{BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset, Op, Slot}, + ir::{BranchOffset, Op, Slot}, Error, }; diff --git a/crates/wasmi/src/engine/translator/func/layout/mod.rs b/crates/wasmi/src/engine/translator/func/layout/mod.rs index b101e04723..7e999d924e 100644 --- a/crates/wasmi/src/engine/translator/func/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func/layout/mod.rs @@ -2,12 +2,7 @@ mod consts; use self::consts::{ConstRegistry, ConstRegistryIter}; use super::{LocalIdx, Operand, OperandIdx, Reset}; -use crate::{ - core::UntypedVal, - engine::{translator::comparator::AllocConst, TranslationError}, - ir::Slot, - Error, -}; +use crate::{core::UntypedVal, engine::TranslationError, ir::Slot, Error}; #[cfg(doc)] use super::Stack; diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index c819d0e2b0..095026e700 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -30,7 +30,7 @@ use self::{ StackAllocations, TempOperand, }, - utils::{Input, Input16, Input32, Reset, ReusableAllocations}, + utils::{Input, IntoShiftAmount, Reset, ReusableAllocations}, }; use crate::{ core::{FuelCostsProvider, IndexType, Typed, TypedVal, UntypedVal}, @@ -40,11 +40,10 @@ use crate::{ CompareResult as _, LogicalizeCmpInstr, NegateCmpInstr, - ReplaceCmpResult, TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, - utils::{Instr, WasmFloat, WasmInteger, Wrap}, + utils::{Instr, WasmFloat, WasmInteger}, WasmTranslator, }, BlockType, @@ -54,31 +53,21 @@ use crate::{ ir::{ index, Address, - Address32, - AnyConst16, BoundedSlotSpan, - BranchOffset, - BranchOffset16, - Comparator, - ComparatorAndOffset, - Const16, - Const32, FixedSlotSpan, - IntoShiftAmount, Offset16, - Offset64, - Offset64Lo, Op, Sign, Slot, SlotSpan, }, - module::{FuncIdx, FuncTypeIdx, MemoryIdx, ModuleHeader, TableIdx, WasmiValueType}, + module::{FuncIdx, FuncTypeIdx, MemoryIdx, ModuleHeader, WasmiValueType}, Engine, Error, FuncType, TrapCode, ValType, + V128, }; use alloc::vec::Vec; use core::mem; diff --git a/crates/wasmi/src/engine/translator/func/op.rs b/crates/wasmi/src/engine/translator/func/op.rs index 24e2ba536c..0f7e0906e3 100644 --- a/crates/wasmi/src/engine/translator/func/op.rs +++ b/crates/wasmi/src/engine/translator/func/op.rs @@ -1,4 +1,7 @@ -use crate::ir::{Address32, Offset16, Offset64Lo, Op, Slot}; +use crate::{ + engine::translator::utils::{ToBits, Wrap}, + ir::{index::Memory, Address, Offset16, Op, Slot}, +}; /// Trait implemented by all Wasm operators that can be translated as wrapping store instructions. pub trait StoreWrapOperator { diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index f914b835c2..fa482d8a50 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -11,11 +11,6 @@ use crate::{ }, ir::{ index::{self, Memory}, - Address32, - IntoShiftAmount, - Offset64, - Offset64Lo, - Offset8, Op, Slot, }, diff --git a/crates/wasmi/src/engine/translator/func/simd/op.rs b/crates/wasmi/src/engine/translator/func/simd/op.rs index 8755650345..10702532a1 100644 --- a/crates/wasmi/src/engine/translator/func/simd/op.rs +++ b/crates/wasmi/src/engine/translator/func/simd/op.rs @@ -1,7 +1,7 @@ use super::IntoLaneIdx; use crate::{ core::{simd, Typed}, - ir::{Const32, Op, Slot}, + ir::{Op, Slot}, V128, }; diff --git a/crates/wasmi/src/engine/translator/func/utils.rs b/crates/wasmi/src/engine/translator/func/utils.rs index 65ba25446e..d294b82e1c 100644 --- a/crates/wasmi/src/engine/translator/func/utils.rs +++ b/crates/wasmi/src/engine/translator/func/utils.rs @@ -1,4 +1,4 @@ -use crate::ir::{Const16, Const32, Slot}; +use crate::ir::Slot; /// Bail out early in case the current code is unreachable. /// diff --git a/crates/wasmi/src/engine/translator/relink_result.rs b/crates/wasmi/src/engine/translator/relink_result.rs index 2d0a7d807f..b66160a9d0 100644 --- a/crates/wasmi/src/engine/translator/relink_result.rs +++ b/crates/wasmi/src/engine/translator/relink_result.rs @@ -1,6 +1,6 @@ use crate::{ engine::EngineFunc, - ir::{index, Op, Slot, SlotSpan, VisitResults}, + ir::{index, Op, Slot, SlotSpan}, module::ModuleHeader, Engine, Error, diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index f18448f830..47fdb62fcd 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -1,6 +1,6 @@ use crate::{ core::{Typed, TypedVal, UntypedVal}, - ir::{Const16, Op, Sign}, + ir::{Op, Sign}, Error, ExternRef, Func, From c7a152e76774e254dc7f5c1f6b1ca61cfa4f4587 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:18:52 +0200 Subject: [PATCH 010/423] remove no longer needed AllocConst utility trait --- .../wasmi/src/engine/translator/comparator.rs | 18 ------------------ .../src/engine/translator/func/layout/mod.rs | 6 ------ 2 files changed, 24 deletions(-) diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index 7a66e983d4..841f30ffda 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -3,24 +3,6 @@ use crate::{ Error, }; -/// Types able to allocate function local constant values. -/// -/// # Note -/// -/// This allows to cheaply convert immediate values to [`Slot`]s. -/// -/// # Errors -/// -/// If the function local constant allocation from immediate value to [`Slot`] failed. -pub trait AllocConst { - /// Allocates a new function local constant value and returns its [`Slot`]. - /// - /// # Note - /// - /// Constant values allocated this way are deduplicated and return shared [`Slot`]. - fn alloc_const>(&mut self, value: T) -> Result; -} - /// Extension trait to return [`Slot`] result of compare [`Op`]s. pub trait CompareResult { /// Returns the result [`Slot`] of the compare [`Op`]. diff --git a/crates/wasmi/src/engine/translator/func/layout/mod.rs b/crates/wasmi/src/engine/translator/func/layout/mod.rs index 7e999d924e..bcf7330b00 100644 --- a/crates/wasmi/src/engine/translator/func/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func/layout/mod.rs @@ -125,12 +125,6 @@ impl StackLayout { } } -impl AllocConst for StackLayout { - fn alloc_const>(&mut self, value: T) -> Result { - self.const_to_reg(value) - } -} - /// The [`StackSpace`] of a [`Slot`]. #[derive(Debug, Copy, Clone)] pub enum StackSpace { From 2b2042fd3e7c4cf2e20b1fed8c36f62b6e3fd230 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:19:06 +0200 Subject: [PATCH 011/423] simplify RelinkResult impls --- .../src/engine/translator/relink_result.rs | 44 ++----------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/crates/wasmi/src/engine/translator/relink_result.rs b/crates/wasmi/src/engine/translator/relink_result.rs index b66160a9d0..7ec948b759 100644 --- a/crates/wasmi/src/engine/translator/relink_result.rs +++ b/crates/wasmi/src/engine/translator/relink_result.rs @@ -47,19 +47,6 @@ impl Visitor { } } -impl VisitResults for Visitor { - #[inline] - fn visit_result_reg(&mut self, slot: &mut Slot) { - if self.replaced.is_err() { - return; - } - self.replaced = relink_simple(slot, self.new_result, self.old_result); - } - - #[inline(always)] - fn visit_result_regs(&mut self, _slots: &mut SlotSpan, _len: Option) {} -} - impl RelinkResult for Op { fn relink_result( &mut self, @@ -67,34 +54,9 @@ impl RelinkResult for Op { new_result: Slot, old_result: Slot, ) -> Result { - // Note: for call instructions we have to infer with special handling if they return - // a single value which allows us to relink the single result register. - match self { - Self::CallInternal0 { results, func } | Self::CallInternal { results, func } => { - relink_call_internal( - results, - EngineFunc::from(*func), - module, - new_result, - old_result, - ) - } - Self::CallImported0 { results, func } | Self::CallImported { results, func } => { - relink_call_imported(results, *func, module, new_result, old_result) - } - Self::CallIndirect0 { results, func_type } - | Self::CallIndirect0Imm16 { results, func_type } - | Self::CallIndirect { results, func_type } - | Self::CallIndirectImm16 { results, func_type } => { - relink_call_indirect(results, *func_type, module, new_result, old_result) - } - instr => { - // Fallback: only relink results of instructions with statically known single results. - let mut visitor = Visitor::new(new_result, old_result); - instr.visit_results(&mut visitor); - visitor.replaced - } - } + let mut visitor = Visitor::new(new_result, old_result); + self.visit_results(&mut visitor); + visitor.replaced } } From dbc0aad113d1e2b24ddafc85b6ee610eea7c1d0f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:19:21 +0200 Subject: [PATCH 012/423] remove no longer needed Input16 and Input32 types --- crates/wasmi/src/engine/translator/func/utils.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/utils.rs b/crates/wasmi/src/engine/translator/func/utils.rs index d294b82e1c..bb198fe3be 100644 --- a/crates/wasmi/src/engine/translator/func/utils.rs +++ b/crates/wasmi/src/engine/translator/func/utils.rs @@ -49,12 +49,6 @@ pub trait ReusableAllocations { fn into_allocations(self) -> Self::Allocations; } -/// A 16-bit encoded input to Wasmi instruction. -pub type Input16 = Input>; - -/// A 32-bit encoded input to Wasmi instruction. -pub type Input32 = Input>; - /// A concrete input to a Wasmi instruction. pub enum Input { /// A [`Slot`] operand. From d3f91a175a54a608de5d101d2f1e4a6bb329fa7e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:19:39 +0200 Subject: [PATCH 013/423] update comparator.rs contents --- .../wasmi/src/engine/translator/comparator.rs | 1328 +++++++---------- 1 file changed, 565 insertions(+), 763 deletions(-) diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index 841f30ffda..ee9063bc5b 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -19,173 +19,108 @@ pub trait CompareResult { impl CompareResult for Op { fn compare_result(&self) -> Option { let result = match *self { - | Op::I32BitAnd { result, .. } - | Op::I32BitAndImm16 { result, .. } - | Op::I32BitOr { result, .. } - | Op::I32BitOrImm16 { result, .. } - | Op::I32BitXor { result, .. } - | Op::I32BitXorImm16 { result, .. } - | Op::I32And { result, .. } - | Op::I32AndImm16 { result, .. } - | Op::I32Or { result, .. } - | Op::I32OrImm16 { result, .. } - | Op::I32Nand { result, .. } - | Op::I32NandImm16 { result, .. } - | Op::I32Nor { result, .. } - | Op::I32NorImm16 { result, .. } - | Op::I32Eq { result, .. } - | Op::I32EqImm16 { result, .. } - | Op::I32Ne { result, .. } - | Op::I32NeImm16 { result, .. } - | Op::I32LtS { result, .. } - | Op::I32LtSImm16Lhs { result, .. } - | Op::I32LtSImm16Rhs { result, .. } - | Op::I32LtU { result, .. } - | Op::I32LtUImm16Lhs { result, .. } - | Op::I32LtUImm16Rhs { result, .. } - | Op::I32LeS { result, .. } - | Op::I32LeSImm16Lhs { result, .. } - | Op::I32LeSImm16Rhs { result, .. } - | Op::I32LeU { result, .. } - | Op::I32LeUImm16Lhs { result, .. } - | Op::I32LeUImm16Rhs { result, .. } - | Op::I64BitAnd { result, .. } - | Op::I64BitAndImm16 { result, .. } - | Op::I64BitOr { result, .. } - | Op::I64BitOrImm16 { result, .. } - | Op::I64BitXor { result, .. } - | Op::I64BitXorImm16 { result, .. } - | Op::I64And { result, .. } - | Op::I64AndImm16 { result, .. } - | Op::I64Or { result, .. } - | Op::I64OrImm16 { result, .. } - | Op::I64Nand { result, .. } - | Op::I64NandImm16 { result, .. } - | Op::I64Nor { result, .. } - | Op::I64NorImm16 { result, .. } - | Op::I64Eq { result, .. } - | Op::I64EqImm16 { result, .. } - | Op::I64Ne { result, .. } - | Op::I64NeImm16 { result, .. } - | Op::I64LtS { result, .. } - | Op::I64LtSImm16Lhs { result, .. } - | Op::I64LtSImm16Rhs { result, .. } - | Op::I64LtU { result, .. } - | Op::I64LtUImm16Lhs { result, .. } - | Op::I64LtUImm16Rhs { result, .. } - | Op::I64LeS { result, .. } - | Op::I64LeSImm16Lhs { result, .. } - | Op::I64LeSImm16Rhs { result, .. } - | Op::I64LeU { result, .. } - | Op::I64LeUImm16Lhs { result, .. } - | Op::I64LeUImm16Rhs { result, .. } - | Op::F32Eq { result, .. } - | Op::F32Ne { result, .. } - | Op::F32Lt { result, .. } - | Op::F32Le { result, .. } - | Op::F32NotLt { result, .. } - | Op::F32NotLe { result, .. } - | Op::F64Eq { result, .. } - | Op::F64Ne { result, .. } - | Op::F64Lt { result, .. } - | Op::F64Le { result, .. } - | Op::F64NotLt { result, .. } - | Op::F64NotLe { result, .. } => result, + // i32 + | Op::I32BitAnd_Sss { result, .. } + | Op::I32BitAnd_Ssi { result, .. } + | Op::I32BitOr_Sss { result, .. } + | Op::I32BitOr_Ssi { result, .. } + | Op::I32BitXor_Sss { result, .. } + | Op::I32BitXor_Ssi { result, .. } + | Op::I32Eq_Sss { result, .. } + | Op::I32Eq_Ssi { result, .. } + | Op::I32And_Sss { result, .. } + | Op::I32And_Ssi { result, .. } + | Op::I32Or_Sss { result, .. } + | Op::I32Or_Ssi { result, .. } + | Op::I32NotEq_Sss { result, .. } + | Op::I32NotEq_Ssi { result, .. } + | Op::I32NotAnd_Sss { result, .. } + | Op::I32NotAnd_Ssi { result, .. } + | Op::I32NotOr_Sss { result, .. } + | Op::I32NotOr_Ssi { result, .. } + | Op::I32Lt_Sss { result, .. } + | Op::I32Lt_Ssi { result, .. } + | Op::I32Lt_Sis { result, .. } + | Op::U32Lt_Sss { result, .. } + | Op::U32Lt_Ssi { result, .. } + | Op::U32Lt_Sis { result, .. } + | Op::I32Le_Sss { result, .. } + | Op::I32Le_Ssi { result, .. } + | Op::I32Le_Sis { result, .. } + | Op::U32Le_Sss { result, .. } + | Op::U32Le_Ssi { result, .. } + | Op::U32Le_Sis { result, .. } + // i64 + | Op::I64BitAnd_Sss { result, .. } + | Op::I64BitAnd_Ssi { result, .. } + | Op::I64BitOr_Sss { result, .. } + | Op::I64BitOr_Ssi { result, .. } + | Op::I64BitXor_Sss { result, .. } + | Op::I64BitXor_Ssi { result, .. } + | Op::I64Eq_Sss { result, .. } + | Op::I64Eq_Ssi { result, .. } + | Op::I64And_Sss { result, .. } + | Op::I64And_Ssi { result, .. } + | Op::I64Or_Sss { result, .. } + | Op::I64Or_Ssi { result, .. } + | Op::I64NotEq_Sss { result, .. } + | Op::I64NotEq_Ssi { result, .. } + | Op::I64NotAnd_Sss { result, .. } + | Op::I64NotAnd_Ssi { result, .. } + | Op::I64NotOr_Sss { result, .. } + | Op::I64NotOr_Ssi { result, .. } + | Op::I64Lt_Sss { result, .. } + | Op::I64Lt_Ssi { result, .. } + | Op::I64Lt_Sis { result, .. } + | Op::U64Lt_Sss { result, .. } + | Op::U64Lt_Ssi { result, .. } + | Op::U64Lt_Sis { result, .. } + | Op::I64Le_Sss { result, .. } + | Op::I64Le_Ssi { result, .. } + | Op::I64Le_Sis { result, .. } + | Op::U64Le_Sss { result, .. } + | Op::U64Le_Ssi { result, .. } + | Op::U64Le_Sis { result, .. } + // f32 + | Op::F32Eq_Sss { result, .. } + | Op::F32Eq_Ssi { result, .. } + | Op::F32Lt_Sss { result, .. } + | Op::F32Lt_Ssi { result, .. } + | Op::F32Lt_Sis { result, .. } + | Op::F32Le_Sss { result, .. } + | Op::F32Le_Ssi { result, .. } + | Op::F32Le_Sis { result, .. } + | Op::F32NotEq_Sss { result, .. } + | Op::F32NotEq_Ssi { result, .. } + | Op::F32NotLt_Sss { result, .. } + | Op::F32NotLt_Ssi { result, .. } + | Op::F32NotLt_Sis { result, .. } + | Op::F32NotLe_Sss { result, .. } + | Op::F32NotLe_Ssi { result, .. } + | Op::F32NotLe_Sis { result, .. } + // f64 + | Op::F64Eq_Sss { result, .. } + | Op::F64Eq_Ssi { result, .. } + | Op::F64Lt_Sss { result, .. } + | Op::F64Lt_Ssi { result, .. } + | Op::F64Lt_Sis { result, .. } + | Op::F64Le_Sss { result, .. } + | Op::F64Le_Ssi { result, .. } + | Op::F64Le_Sis { result, .. } + | Op::F64NotEq_Sss { result, .. } + | Op::F64NotEq_Ssi { result, .. } + | Op::F64NotLt_Sss { result, .. } + | Op::F64NotLt_Ssi { result, .. } + | Op::F64NotLt_Sis { result, .. } + | Op::F64NotLe_Sss { result, .. } + | Op::F64NotLe_Ssi { result, .. } + | Op::F64NotLe_Sis { result, .. } => result, _ => return None, }; Some(result) } } -pub trait ReplaceCmpResult: Sized { - /// Returns `self` `cmp` instruction with the `new_result`. - /// - /// Returns `None` if `self` is not a `cmp` instruction. - fn replace_cmp_result(&self, new_result: Slot) -> Option; -} - -impl ReplaceCmpResult for Op { - fn replace_cmp_result(&self, new_result: Slot) -> Option { - let mut copy = *self; - match &mut copy { - | Op::I32BitAnd { result, .. } - | Op::I32BitAndImm16 { result, .. } - | Op::I32BitOr { result, .. } - | Op::I32BitOrImm16 { result, .. } - | Op::I32BitXor { result, .. } - | Op::I32BitXorImm16 { result, .. } - | Op::I32And { result, .. } - | Op::I32AndImm16 { result, .. } - | Op::I32Or { result, .. } - | Op::I32OrImm16 { result, .. } - | Op::I32Nand { result, .. } - | Op::I32NandImm16 { result, .. } - | Op::I32Nor { result, .. } - | Op::I32NorImm16 { result, .. } - | Op::I32Eq { result, .. } - | Op::I32EqImm16 { result, .. } - | Op::I32Ne { result, .. } - | Op::I32NeImm16 { result, .. } - | Op::I32LtS { result, .. } - | Op::I32LtSImm16Lhs { result, .. } - | Op::I32LtSImm16Rhs { result, .. } - | Op::I32LtU { result, .. } - | Op::I32LtUImm16Lhs { result, .. } - | Op::I32LtUImm16Rhs { result, .. } - | Op::I32LeS { result, .. } - | Op::I32LeSImm16Lhs { result, .. } - | Op::I32LeSImm16Rhs { result, .. } - | Op::I32LeU { result, .. } - | Op::I32LeUImm16Lhs { result, .. } - | Op::I32LeUImm16Rhs { result, .. } - | Op::I64BitAnd { result, .. } - | Op::I64BitAndImm16 { result, .. } - | Op::I64BitOr { result, .. } - | Op::I64BitOrImm16 { result, .. } - | Op::I64BitXor { result, .. } - | Op::I64BitXorImm16 { result, .. } - | Op::I64And { result, .. } - | Op::I64AndImm16 { result, .. } - | Op::I64Or { result, .. } - | Op::I64OrImm16 { result, .. } - | Op::I64Nand { result, .. } - | Op::I64NandImm16 { result, .. } - | Op::I64Nor { result, .. } - | Op::I64NorImm16 { result, .. } - | Op::I64Eq { result, .. } - | Op::I64EqImm16 { result, .. } - | Op::I64Ne { result, .. } - | Op::I64NeImm16 { result, .. } - | Op::I64LtS { result, .. } - | Op::I64LtSImm16Lhs { result, .. } - | Op::I64LtSImm16Rhs { result, .. } - | Op::I64LtU { result, .. } - | Op::I64LtUImm16Lhs { result, .. } - | Op::I64LtUImm16Rhs { result, .. } - | Op::I64LeS { result, .. } - | Op::I64LeSImm16Lhs { result, .. } - | Op::I64LeSImm16Rhs { result, .. } - | Op::I64LeU { result, .. } - | Op::I64LeUImm16Lhs { result, .. } - | Op::I64LeUImm16Rhs { result, .. } - | Op::F32Eq { result, .. } - | Op::F32Ne { result, .. } - | Op::F32Lt { result, .. } - | Op::F32Le { result, .. } - | Op::F32NotLt { result, .. } - | Op::F32NotLe { result, .. } - | Op::F64Eq { result, .. } - | Op::F64Ne { result, .. } - | Op::F64Lt { result, .. } - | Op::F64Le { result, .. } - | Op::F64NotLt { result, .. } - | Op::F64NotLe { result, .. } => *result = new_result, - _ => return None, - }; - Some(copy) - } -} - pub trait NegateCmpInstr: Sized { /// Negates the compare (`cmp`) [`Op`]. fn negate_cmp_instr(&self) -> Option; @@ -196,83 +131,101 @@ impl NegateCmpInstr for Op { #[rustfmt::skip] let negated = match *self { // i32 - Op::I32Eq { result, lhs, rhs } => Op::i32_ne(result, lhs, rhs), - Op::I32Ne { result, lhs, rhs } => Op::i32_eq(result, lhs, rhs), - Op::I32LeS { result, lhs, rhs } => Op::i32_lt_s(result, rhs, lhs), - Op::I32LeU { result, lhs, rhs } => Op::i32_lt_u(result, rhs, lhs), - Op::I32LtS { result, lhs, rhs } => Op::i32_le_s(result, rhs, lhs), - Op::I32LtU { result, lhs, rhs } => Op::i32_le_u(result, rhs, lhs), - Op::I32EqImm16 { result, lhs, rhs } => Op::i32_ne_imm16(result, lhs, rhs), - Op::I32NeImm16 { result, lhs, rhs } => Op::i32_eq_imm16(result, lhs, rhs), - Op::I32LeSImm16Rhs { result, lhs, rhs } => Op::i32_lt_s_imm16_lhs(result, rhs, lhs), - Op::I32LeUImm16Rhs { result, lhs, rhs } => Op::i32_lt_u_imm16_lhs(result, rhs, lhs), - Op::I32LtSImm16Rhs { result, lhs, rhs } => Op::i32_le_s_imm16_lhs(result, rhs, lhs), - Op::I32LtUImm16Rhs { result, lhs, rhs } => Op::i32_le_u_imm16_lhs(result, rhs, lhs), - Op::I32LeSImm16Lhs { result, lhs, rhs } => Op::i32_lt_s_imm16_rhs(result, rhs, lhs), - Op::I32LeUImm16Lhs { result, lhs, rhs } => Op::i32_lt_u_imm16_rhs(result, rhs, lhs), - Op::I32LtSImm16Lhs { result, lhs, rhs } => Op::i32_le_s_imm16_rhs(result, rhs, lhs), - Op::I32LtUImm16Lhs { result, lhs, rhs } => Op::i32_le_u_imm16_rhs(result, rhs, lhs), - // i32 (and, or, xor) - Op::I32BitAnd { result, lhs, rhs } => Op::i32_nand(result, lhs, rhs), - Op::I32BitOr { result, lhs, rhs } => Op::i32_nor(result, lhs, rhs), - Op::I32BitXor { result, lhs, rhs } => Op::i32_eq(result, lhs, rhs), - Op::I32BitAndImm16 { result, lhs, rhs } => Op::i32_nand_imm16(result, lhs, rhs), - Op::I32BitOrImm16 { result, lhs, rhs } => Op::i32_nor_imm16(result, lhs, rhs), - Op::I32BitXorImm16 { result, lhs, rhs } => Op::i32_eq_imm16(result, lhs, rhs), - Op::I32And { result, lhs, rhs } => Op::i32_nand(result, lhs, rhs), - Op::I32Or { result, lhs, rhs } => Op::i32_nor(result, lhs, rhs), - Op::I32AndImm16 { result, lhs, rhs } => Op::i32_nand_imm16(result, lhs, rhs), - Op::I32OrImm16 { result, lhs, rhs } => Op::i32_nor_imm16(result, lhs, rhs), - Op::I32Nand { result, lhs, rhs } => Op::i32_and(result, lhs, rhs), - Op::I32Nor { result, lhs, rhs } => Op::i32_or(result, lhs, rhs), - Op::I32NandImm16 { result, lhs, rhs } => Op::i32_and_imm16(result, lhs, rhs), - Op::I32NorImm16 { result, lhs, rhs } => Op::i32_or_imm16(result, lhs, rhs), + | Op::I32Eq_Sss { result, lhs, rhs } => Op::i32_not_eq_sss(result, lhs, rhs), + | Op::I32Eq_Ssi { result, lhs, rhs } => Op::i32_not_eq_ssi(result, lhs, rhs), + | Op::I32And_Sss { result, lhs, rhs } + | Op::I32BitAnd_Sss { result, lhs, rhs } => Op::i32_not_and_sss(result, lhs, rhs), + | Op::I32And_Ssi { result, lhs, rhs } + | Op::I32BitAnd_Ssi { result, lhs, rhs } => Op::i32_not_and_ssi(result, lhs, rhs), + | Op::I32Or_Sss { result, lhs, rhs } + | Op::I32BitOr_Sss { result, lhs, rhs } => Op::i32_not_or_sss(result, lhs, rhs), + | Op::I32Or_Ssi { result, lhs, rhs } + | Op::I32BitOr_Ssi { result, lhs, rhs } => Op::i32_not_or_ssi(result, lhs, rhs), + | Op::I32NotEq_Sss { result, lhs, rhs } + | Op::I32BitXor_Sss { result, lhs, rhs } => Op::i32_eq_sss(result, lhs, rhs), + | Op::I32NotEq_Ssi { result, lhs, rhs } + | Op::I32BitXor_Ssi { result, lhs, rhs } => Op::i32_eq_ssi(result, lhs, rhs), + | Op::I32NotAnd_Sss { result, lhs, rhs } => Op::i32_and_sss(result, lhs, rhs), + | Op::I32NotAnd_Ssi { result, lhs, rhs } => Op::i32_and_ssi(result, lhs, rhs), + | Op::I32NotOr_Sss { result, lhs, rhs } => Op::i32_or_sss(result, lhs, rhs), + | Op::I32NotOr_Ssi { result, lhs, rhs } => Op::i32_or_ssi(result, lhs, rhs), + | Op::I32Lt_Sss { result, lhs, rhs } => Op::i32_le_sss(result, rhs, lhs), + | Op::I32Lt_Ssi { result, lhs, rhs } => Op::i32_le_sis(result, rhs, lhs), + | Op::I32Lt_Sis { result, lhs, rhs } => Op::i32_le_ssi(result, rhs, lhs), + | Op::U32Lt_Sss { result, lhs, rhs } => Op::u32_le_sss(result, rhs, lhs), + | Op::U32Lt_Ssi { result, lhs, rhs } => Op::u32_le_sis(result, rhs, lhs), + | Op::U32Lt_Sis { result, lhs, rhs } => Op::u32_le_ssi(result, rhs, lhs), + | Op::I32Le_Sss { result, lhs, rhs } => Op::i32_lt_sss(result, rhs, lhs), + | Op::I32Le_Ssi { result, lhs, rhs } => Op::i32_lt_sis(result, rhs, lhs), + | Op::I32Le_Sis { result, lhs, rhs } => Op::i32_lt_ssi(result, rhs, lhs), + | Op::U32Le_Sss { result, lhs, rhs } => Op::u32_lt_sss(result, rhs, lhs), + | Op::U32Le_Ssi { result, lhs, rhs } => Op::u32_lt_sis(result, rhs, lhs), + | Op::U32Le_Sis { result, lhs, rhs } => Op::u32_lt_ssi(result, rhs, lhs), // i64 - Op::I64Eq { result, lhs, rhs } => Op::i64_ne(result, lhs, rhs), - Op::I64Ne { result, lhs, rhs } => Op::i64_eq(result, lhs, rhs), - Op::I64LeS { result, lhs, rhs } => Op::i64_lt_s(result, rhs, lhs), - Op::I64LeU { result, lhs, rhs } => Op::i64_lt_u(result, rhs, lhs), - Op::I64LtS { result, lhs, rhs } => Op::i64_le_s(result, rhs, lhs), - Op::I64LtU { result, lhs, rhs } => Op::i64_le_u(result, rhs, lhs), - Op::I64EqImm16 { result, lhs, rhs } => Op::i64_ne_imm16(result, lhs, rhs), - Op::I64NeImm16 { result, lhs, rhs } => Op::i64_eq_imm16(result, lhs, rhs), - Op::I64LeSImm16Rhs { result, lhs, rhs } => Op::i64_lt_s_imm16_lhs(result, rhs, lhs), - Op::I64LeUImm16Rhs { result, lhs, rhs } => Op::i64_lt_u_imm16_lhs(result, rhs, lhs), - Op::I64LtSImm16Rhs { result, lhs, rhs } => Op::i64_le_s_imm16_lhs(result, rhs, lhs), - Op::I64LtUImm16Rhs { result, lhs, rhs } => Op::i64_le_u_imm16_lhs(result, rhs, lhs), - Op::I64LeSImm16Lhs { result, lhs, rhs } => Op::i64_lt_s_imm16_rhs(result, rhs, lhs), - Op::I64LeUImm16Lhs { result, lhs, rhs } => Op::i64_lt_u_imm16_rhs(result, rhs, lhs), - Op::I64LtSImm16Lhs { result, lhs, rhs } => Op::i64_le_s_imm16_rhs(result, rhs, lhs), - Op::I64LtUImm16Lhs { result, lhs, rhs } => Op::i64_le_u_imm16_rhs(result, rhs, lhs), - // i64 (and, or, xor) - Op::I64BitAnd { result, lhs, rhs } => Op::i64_nand(result, lhs, rhs), - Op::I64BitOr { result, lhs, rhs } => Op::i64_nor(result, lhs, rhs), - Op::I64BitXor { result, lhs, rhs } => Op::i64_eq(result, lhs, rhs), - Op::I64BitAndImm16 { result, lhs, rhs } => Op::i64_nand_imm16(result, lhs, rhs), - Op::I64BitOrImm16 { result, lhs, rhs } => Op::i64_nor_imm16(result, lhs, rhs), - Op::I64BitXorImm16 { result, lhs, rhs } => Op::i64_eq_imm16(result, lhs, rhs), - Op::I64And { result, lhs, rhs } => Op::i64_nand(result, lhs, rhs), - Op::I64Or { result, lhs, rhs } => Op::i64_nor(result, lhs, rhs), - Op::I64AndImm16 { result, lhs, rhs } => Op::i64_nand_imm16(result, lhs, rhs), - Op::I64OrImm16 { result, lhs, rhs } => Op::i64_nor_imm16(result, lhs, rhs), - Op::I64Nand { result, lhs, rhs } => Op::i64_and(result, lhs, rhs), - Op::I64Nor { result, lhs, rhs } => Op::i64_or(result, lhs, rhs), - Op::I64NandImm16 { result, lhs, rhs } => Op::i64_and_imm16(result, lhs, rhs), - Op::I64NorImm16 { result, lhs, rhs } => Op::i64_or_imm16(result, lhs, rhs), + | Op::I64Eq_Sss { result, lhs, rhs } => Op::i64_not_eq_sss(result, lhs, rhs), + | Op::I64Eq_Ssi { result, lhs, rhs } => Op::i64_not_eq_ssi(result, lhs, rhs), + | Op::I64And_Sss { result, lhs, rhs } + | Op::I64BitAnd_Sss { result, lhs, rhs } => Op::i64_not_and_sss(result, lhs, rhs), + | Op::I64And_Ssi { result, lhs, rhs } + | Op::I64BitAnd_Ssi { result, lhs, rhs } => Op::i64_not_and_ssi(result, lhs, rhs), + | Op::I64Or_Sss { result, lhs, rhs } + | Op::I64BitOr_Sss { result, lhs, rhs } => Op::i64_not_or_sss(result, lhs, rhs), + | Op::I64Or_Ssi { result, lhs, rhs } + | Op::I64BitOr_Ssi { result, lhs, rhs } => Op::i64_not_or_ssi(result, lhs, rhs), + | Op::I64NotEq_Sss { result, lhs, rhs } + | Op::I64BitXor_Sss { result, lhs, rhs } => Op::i64_eq_sss(result, lhs, rhs), + | Op::I64NotEq_Ssi { result, lhs, rhs } + | Op::I64BitXor_Ssi { result, lhs, rhs } => Op::i64_eq_ssi(result, lhs, rhs), + | Op::I64NotAnd_Sss { result, lhs, rhs } => Op::i64_and_sss(result, lhs, rhs), + | Op::I64NotAnd_Ssi { result, lhs, rhs } => Op::i64_and_ssi(result, lhs, rhs), + | Op::I64NotOr_Sss { result, lhs, rhs } => Op::i64_or_sss(result, lhs, rhs), + | Op::I64NotOr_Ssi { result, lhs, rhs } => Op::i64_or_ssi(result, lhs, rhs), + | Op::I64Lt_Sss { result, lhs, rhs } => Op::i64_le_sss(result, rhs, lhs), + | Op::I64Lt_Ssi { result, lhs, rhs } => Op::i64_le_sis(result, rhs, lhs), + | Op::I64Lt_Sis { result, lhs, rhs } => Op::i64_le_ssi(result, rhs, lhs), + | Op::U64Lt_Sss { result, lhs, rhs } => Op::u64_le_sss(result, rhs, lhs), + | Op::U64Lt_Ssi { result, lhs, rhs } => Op::u64_le_sis(result, rhs, lhs), + | Op::U64Lt_Sis { result, lhs, rhs } => Op::u64_le_ssi(result, rhs, lhs), + | Op::I64Le_Sss { result, lhs, rhs } => Op::i64_lt_sss(result, rhs, lhs), + | Op::I64Le_Ssi { result, lhs, rhs } => Op::i64_lt_sis(result, rhs, lhs), + | Op::I64Le_Sis { result, lhs, rhs } => Op::i64_lt_ssi(result, rhs, lhs), + | Op::U64Le_Sss { result, lhs, rhs } => Op::u64_lt_sss(result, rhs, lhs), + | Op::U64Le_Ssi { result, lhs, rhs } => Op::u64_lt_sis(result, rhs, lhs), + | Op::U64Le_Sis { result, lhs, rhs } => Op::u64_lt_ssi(result, rhs, lhs), // f32 - Op::F32Eq { result, lhs, rhs } => Op::f32_ne(result, lhs, rhs), - Op::F32Ne { result, lhs, rhs } => Op::f32_eq(result, lhs, rhs), - Op::F32Le { result, lhs, rhs } => Op::f32_not_le(result, lhs, rhs), - Op::F32Lt { result, lhs, rhs } => Op::f32_not_lt(result, lhs, rhs), - Op::F32NotLe { result, lhs, rhs } => Op::f32_le(result, lhs, rhs), - Op::F32NotLt { result, lhs, rhs } => Op::f32_lt(result, lhs, rhs), + Op::F32Eq_Sss { result, lhs, rhs } => Op::f32_not_eq_sss(result, lhs, rhs), + Op::F32Eq_Ssi { result, lhs, rhs } => Op::f32_not_eq_ssi(result, lhs, rhs), + Op::F32Le_Sss { result, lhs, rhs } => Op::f32_not_le_sss(result, lhs, rhs), + Op::F32Le_Ssi { result, lhs, rhs } => Op::f32_not_le_ssi(result, lhs, rhs), + Op::F32Le_Sis { result, lhs, rhs } => Op::f32_not_le_sis(result, lhs, rhs), + Op::F32Lt_Sss { result, lhs, rhs } => Op::f32_not_lt_sss(result, lhs, rhs), + Op::F32Lt_Ssi { result, lhs, rhs } => Op::f32_not_lt_ssi(result, lhs, rhs), + Op::F32Lt_Sis { result, lhs, rhs } => Op::f32_not_lt_sis(result, lhs, rhs), + Op::F32NotEq_Sss { result, lhs, rhs } => Op::f32_eq_sss(result, lhs, rhs), + Op::F32NotEq_Ssi { result, lhs, rhs } => Op::f32_eq_ssi(result, lhs, rhs), + Op::F32NotLe_Sss { result, lhs, rhs } => Op::f32_le_sss(result, lhs, rhs), + Op::F32NotLe_Ssi { result, lhs, rhs } => Op::f32_le_ssi(result, lhs, rhs), + Op::F32NotLe_Sis { result, lhs, rhs } => Op::f32_le_sis(result, lhs, rhs), + Op::F32NotLt_Sss { result, lhs, rhs } => Op::f32_lt_sss(result, lhs, rhs), + Op::F32NotLt_Ssi { result, lhs, rhs } => Op::f32_lt_ssi(result, lhs, rhs), + Op::F32NotLt_Sis { result, lhs, rhs } => Op::f32_lt_sis(result, lhs, rhs), // f64 - Op::F64Eq { result, lhs, rhs } => Op::f64_ne(result, lhs, rhs), - Op::F64Ne { result, lhs, rhs } => Op::f64_eq(result, lhs, rhs), - Op::F64Le { result, lhs, rhs } => Op::f64_not_le(result, lhs, rhs), - Op::F64Lt { result, lhs, rhs } => Op::f64_not_lt(result, lhs, rhs), - Op::F64NotLe { result, lhs, rhs } => Op::f64_le(result, lhs, rhs), - Op::F64NotLt { result, lhs, rhs } => Op::f64_lt(result, lhs, rhs), + Op::F64Eq_Sss { result, lhs, rhs } => Op::f64_not_eq_sss(result, lhs, rhs), + Op::F64Eq_Ssi { result, lhs, rhs } => Op::f64_not_eq_ssi(result, lhs, rhs), + Op::F64Le_Sss { result, lhs, rhs } => Op::f64_not_le_sss(result, lhs, rhs), + Op::F64Le_Ssi { result, lhs, rhs } => Op::f64_not_le_ssi(result, lhs, rhs), + Op::F64Le_Sis { result, lhs, rhs } => Op::f64_not_le_sis(result, lhs, rhs), + Op::F64Lt_Sss { result, lhs, rhs } => Op::f64_not_lt_sss(result, lhs, rhs), + Op::F64Lt_Ssi { result, lhs, rhs } => Op::f64_not_lt_ssi(result, lhs, rhs), + Op::F64Lt_Sis { result, lhs, rhs } => Op::f64_not_lt_sis(result, lhs, rhs), + Op::F64NotEq_Sss { result, lhs, rhs } => Op::f64_eq_sss(result, lhs, rhs), + Op::F64NotEq_Ssi { result, lhs, rhs } => Op::f64_eq_ssi(result, lhs, rhs), + Op::F64NotLe_Sss { result, lhs, rhs } => Op::f64_le_sss(result, lhs, rhs), + Op::F64NotLe_Ssi { result, lhs, rhs } => Op::f64_le_ssi(result, lhs, rhs), + Op::F64NotLe_Sis { result, lhs, rhs } => Op::f64_le_sis(result, lhs, rhs), + Op::F64NotLt_Sss { result, lhs, rhs } => Op::f64_lt_sss(result, lhs, rhs), + Op::F64NotLt_Ssi { result, lhs, rhs } => Op::f64_lt_ssi(result, lhs, rhs), + Op::F64NotLt_Sis { result, lhs, rhs } => Op::f64_lt_sis(result, lhs, rhs), _ => return None, }; Some(negated) @@ -292,80 +245,104 @@ impl LogicalizeCmpInstr for Op { #[rustfmt::skip] let logicalized = match *self { // Bitwise -> Logical: i32 - Op::I32BitAnd { result, lhs, rhs } => Op::i32_and(result, lhs, rhs), - Op::I32BitOr { result, lhs, rhs } => Op::i32_or(result, lhs, rhs), - Op::I32BitXor { result, lhs, rhs } => Op::i32_ne(result, lhs, rhs), - Op::I32BitAndImm16 { result, lhs, rhs } => Op::i32_and_imm16(result, lhs, rhs), - Op::I32BitOrImm16 { result, lhs, rhs } => Op::i32_or_imm16(result, lhs, rhs), - Op::I32BitXorImm16 { result, lhs, rhs } => Op::i32_ne_imm16(result, lhs, rhs), + | Op::I32BitAnd_Sss { result, lhs, rhs } => Op::i32_and_sss(result, lhs, rhs), + | Op::I32BitOr_Sss { result, lhs, rhs } => Op::i32_or_sss(result, lhs, rhs), + | Op::I32BitXor_Sss { result, lhs, rhs } => Op::i32_ne_sss(result, lhs, rhs), + | Op::I32BitAnd_Ssi { result, lhs, rhs } => Op::i32_and_ssi(result, lhs, rhs), + | Op::I32BitOr_Ssi { result, lhs, rhs } => Op::i32_or_ssi(result, lhs, rhs), + | Op::I32BitXor_Ssi { result, lhs, rhs } => Op::i32_ne_ssi(result, lhs, rhs), // Bitwise -> Logical: i64 - Op::I64BitAnd { result, lhs, rhs } => Op::i64_and(result, lhs, rhs), - Op::I64BitOr { result, lhs, rhs } => Op::i64_or(result, lhs, rhs), - Op::I64BitXor { result, lhs, rhs } => Op::i64_ne(result, lhs, rhs), - Op::I64BitAndImm16 { result, lhs, rhs } => Op::i64_and_imm16(result, lhs, rhs), - Op::I64BitOrImm16 { result, lhs, rhs } => Op::i64_or_imm16(result, lhs, rhs), - Op::I64BitXorImm16 { result, lhs, rhs } => Op::i64_ne_imm16(result, lhs, rhs), + | Op::I64BitAnd_Sss { result, lhs, rhs } => Op::i64_and_sss(result, lhs, rhs), + | Op::I64BitOr_Sss { result, lhs, rhs } => Op::i64_or_sss(result, lhs, rhs), + | Op::I64BitXor_Sss { result, lhs, rhs } => Op::i64_ne_sss(result, lhs, rhs), + | Op::I64BitAnd_Ssi { result, lhs, rhs } => Op::i64_and_ssi(result, lhs, rhs), + | Op::I64BitOr_Ssi { result, lhs, rhs } => Op::i64_or_ssi(result, lhs, rhs), + | Op::I64BitXor_Ssi { result, lhs, rhs } => Op::i64_ne_ssi(result, lhs, rhs), // Logical -> Logical - Op::I32Eq { .. } | - Op::I32Ne { .. } | - Op::I32LeS { .. } | - Op::I32LeU { .. } | - Op::I32LtS { .. } | - Op::I32LtU { .. } | - Op::I32EqImm16 { .. } | - Op::I32NeImm16 { .. } | - Op::I32LeSImm16Rhs { .. } | - Op::I32LeUImm16Rhs { .. } | - Op::I32LtSImm16Rhs { .. } | - Op::I32LtUImm16Rhs { .. } | - Op::I32LeSImm16Lhs { .. } | - Op::I32LeUImm16Lhs { .. } | - Op::I32LtSImm16Lhs { .. } | - Op::I32LtUImm16Lhs { .. } | - Op::I32And { .. } | - Op::I32Or { .. } | - Op::I32AndImm16 { .. } | - Op::I32OrImm16 { .. } | - Op::I32Nand { .. } | - Op::I32Nor { .. } | - Op::I32NandImm16 { .. } | - Op::I32NorImm16 { .. } | - Op::I64Eq { .. } | - Op::I64Ne { .. } | - Op::I64LeS { .. } | - Op::I64LeU { .. } | - Op::I64LtS { .. } | - Op::I64LtU { .. } | - Op::I64EqImm16 { .. } | - Op::I64NeImm16 { .. } | - Op::I64LeSImm16Rhs { .. } | - Op::I64LeUImm16Rhs { .. } | - Op::I64LtSImm16Rhs { .. } | - Op::I64LtUImm16Rhs { .. } | - Op::I64LeSImm16Lhs { .. } | - Op::I64LeUImm16Lhs { .. } | - Op::I64LtSImm16Lhs { .. } | - Op::I64LtUImm16Lhs { .. } | - Op::I64And { .. } | - Op::I64Or { .. } | - Op::I64AndImm16 { .. } | - Op::I64OrImm16 { .. } | - Op::I64Nand { .. } | - Op::I64Nor { .. } | - Op::I64NandImm16 { .. } | - Op::I64NorImm16 { .. } | - Op::F32Eq { .. } | - Op::F32Ne { .. } | - Op::F32Lt { .. } | - Op::F32Le { .. } | - Op::F32NotLt { .. } | - Op::F32NotLe { .. } | - Op::F64Eq { .. } | - Op::F64Ne { .. } | - Op::F64Lt { .. } | - Op::F64Le { .. } | - Op::F64NotLt { .. } | - Op::F64NotLe { .. } => *self, + // i32 + | Op::I32Eq_Sss { .. } + | Op::I32Eq_Ssi { .. } + | Op::I32And_Sss { .. } + | Op::I32And_Ssi { .. } + | Op::I32Or_Sss { .. } + | Op::I32Or_Ssi { .. } + | Op::I32NotEq_Sss { .. } + | Op::I32NotEq_Ssi { .. } + | Op::I32NotAnd_Sss { .. } + | Op::I32NotAnd_Ssi { .. } + | Op::I32NotOr_Sss { .. } + | Op::I32NotOr_Ssi { .. } + | Op::I32Lt_Sss { .. } + | Op::I32Lt_Ssi { .. } + | Op::I32Lt_Sis { .. } + | Op::U32Lt_Sss { .. } + | Op::U32Lt_Ssi { .. } + | Op::U32Lt_Sis { .. } + | Op::I32Le_Sss { .. } + | Op::I32Le_Ssi { .. } + | Op::I32Le_Sis { .. } + | Op::U32Le_Sss { .. } + | Op::U32Le_Ssi { .. } + | Op::U32Le_Sis { .. } + // i64 + | Op::I64Eq_Sss { .. } + | Op::I64Eq_Ssi { .. } + | Op::I64And_Sss { .. } + | Op::I64And_Ssi { .. } + | Op::I64Or_Sss { .. } + | Op::I64Or_Ssi { .. } + | Op::I64NotEq_Sss { .. } + | Op::I64NotEq_Ssi { .. } + | Op::I64NotAnd_Sss { .. } + | Op::I64NotAnd_Ssi { .. } + | Op::I64NotOr_Sss { .. } + | Op::I64NotOr_Ssi { .. } + | Op::I64Lt_Sss { .. } + | Op::I64Lt_Ssi { .. } + | Op::I64Lt_Sis { .. } + | Op::U64Lt_Sss { .. } + | Op::U64Lt_Ssi { .. } + | Op::U64Lt_Sis { .. } + | Op::I64Le_Sss { .. } + | Op::I64Le_Ssi { .. } + | Op::I64Le_Sis { .. } + | Op::U64Le_Sss { .. } + | Op::U64Le_Ssi { .. } + | Op::U64Le_Sis { .. } + // f32 + | Op::F32Eq_Sss { .. } + | Op::F32Eq_Ssi { .. } + | Op::F32Le_Sss { .. } + | Op::F32Le_Ssi { .. } + | Op::F32Le_Sis { .. } + | Op::F32Lt_Sss { .. } + | Op::F32Lt_Ssi { .. } + | Op::F32Lt_Sis { .. } + | Op::F32NotEq_Sss { .. } + | Op::F32NotEq_Ssi { .. } + | Op::F32NotLe_Sss { .. } + | Op::F32NotLe_Ssi { .. } + | Op::F32NotLe_Sis { .. } + | Op::F32NotLt_Sss { .. } + | Op::F32NotLt_Ssi { .. } + | Op::F32NotLt_Sis { .. } + // f64 + | Op::F64Eq_Sss { .. } + | Op::F64Eq_Ssi { .. } + | Op::F64Le_Sss { .. } + | Op::F64Le_Ssi { .. } + | Op::F64Le_Sis { .. } + | Op::F64Lt_Sss { .. } + | Op::F64Lt_Ssi { .. } + | Op::F64Lt_Sis { .. } + | Op::F64NotEq_Sss { .. } + | Op::F64NotEq_Ssi { .. } + | Op::F64NotLe_Sss { .. } + | Op::F64NotLe_Ssi { .. } + | Op::F64NotLe_Sis { .. } + | Op::F64NotLt_Sss { .. } + | Op::F64NotLt_Ssi { .. } + | Op::F64NotLt_Sis { .. } => *self, _ => return None, }; Some(logicalized) @@ -376,424 +353,244 @@ pub trait TryIntoCmpSelectInstr: Sized { fn try_into_cmp_select_instr( &self, get_result: impl FnOnce() -> Result, + val_true: Slot, + val_false: Slot, ) -> Result; } /// The outcome of `cmp`+`select` op-code fusion. pub enum CmpSelectFusion { /// The `cmp`+`select` fusion was applied and may require swapping operands. - Applied { fused: Op, swap_operands: bool }, + Applied(Op), /// The `cmp`+`select` fusion was _not_ applied. Unapplied, } -/// Returns `true` if a `cmp`+`select` fused instruction required to swap its operands. -#[rustfmt::skip] -fn cmp_select_swap_operands(instr: &Op) -> bool { - matches!(instr, - | Op::I32Ne { .. } - | Op::I32NeImm16 { .. } - | Op::I32LeSImm16Lhs { .. } - | Op::I32LeUImm16Lhs { .. } - | Op::I32LtSImm16Lhs { .. } - | Op::I32LtUImm16Lhs { .. } - | Op::I32BitXor { .. } - | Op::I32BitXorImm16 { .. } - | Op::I64BitXor { .. } - | Op::I64BitXorImm16 { .. } - | Op::I32Nand { .. } - | Op::I32Nor { .. } - | Op::I32NandImm16 { .. } - | Op::I32NorImm16 { .. } - | Op::I64Ne { .. } - | Op::I64NeImm16 { .. } - | Op::I64LeSImm16Lhs { .. } - | Op::I64LeUImm16Lhs { .. } - | Op::I64LtSImm16Lhs { .. } - | Op::I64LtUImm16Lhs { .. } - | Op::I64Nand { .. } - | Op::I64Nor { .. } - | Op::I64NandImm16 { .. } - | Op::I64NorImm16 { .. } - | Op::F32Ne { .. } - | Op::F64Ne { .. } - | Op::F32NotLt { .. } - | Op::F32NotLe { .. } - | Op::F64NotLt { .. } - | Op::F64NotLe { .. } - ) -} - impl TryIntoCmpSelectInstr for Op { fn try_into_cmp_select_instr( &self, get_result: impl FnOnce() -> Result, + val_true: Slot, + val_false: Slot, ) -> Result { if !self.is_compare_instr() { return Ok(CmpSelectFusion::Unapplied); } - let swap_operands = cmp_select_swap_operands(self); let result = get_result()?; #[rustfmt::skip] let fused = match *self { // i32 - Op::I32Eq { lhs, rhs, .. } => Op::select_i32_eq(result, lhs, rhs), - Op::I32Ne { lhs, rhs, .. } => Op::select_i32_eq(result, lhs, rhs), - Op::I32LeS { lhs, rhs, .. } => Op::select_i32_le_s(result, lhs, rhs), - Op::I32LeU { lhs, rhs, .. } => Op::select_i32_le_u(result, lhs, rhs), - Op::I32LtS { lhs, rhs, .. } => Op::select_i32_lt_s(result, lhs, rhs), - Op::I32LtU { lhs, rhs, .. } => Op::select_i32_lt_u(result, lhs, rhs), - Op::I32EqImm16 { lhs, rhs, .. } => Op::select_i32_eq_imm16(result, lhs, rhs), - Op::I32NeImm16 { lhs, rhs, .. } => Op::select_i32_eq_imm16(result, lhs, rhs), - Op::I32LeSImm16Lhs { lhs, rhs, .. } => Op::select_i32_lt_s_imm16_rhs(result, rhs, lhs), - Op::I32LeUImm16Lhs { lhs, rhs, .. } => Op::select_i32_lt_u_imm16_rhs(result, rhs, lhs), - Op::I32LtSImm16Lhs { lhs, rhs, .. } => Op::select_i32_le_s_imm16_rhs(result, rhs, lhs), - Op::I32LtUImm16Lhs { lhs, rhs, .. } => Op::select_i32_le_u_imm16_rhs(result, rhs, lhs), - Op::I32LeSImm16Rhs { lhs, rhs, .. } => Op::select_i32_le_s_imm16_rhs(result, lhs, rhs), - Op::I32LeUImm16Rhs { lhs, rhs, .. } => Op::select_i32_le_u_imm16_rhs(result, lhs, rhs), - Op::I32LtSImm16Rhs { lhs, rhs, .. } => Op::select_i32_lt_s_imm16_rhs(result, lhs, rhs), - Op::I32LtUImm16Rhs { lhs, rhs, .. } => Op::select_i32_lt_u_imm16_rhs(result, lhs, rhs), - // i32 (and, or, xor) - Op::I32BitAnd { lhs, rhs, .. } => Op::select_i32_and(result, lhs, rhs), - Op::I32BitOr { lhs, rhs, .. } => Op::select_i32_or(result, lhs, rhs), - Op::I32BitXor { lhs, rhs, .. } => Op::select_i32_eq(result, lhs, rhs), - Op::I32And { lhs, rhs, .. } => Op::select_i32_and(result, lhs, rhs), - Op::I32Or { lhs, rhs, .. } => Op::select_i32_or(result, lhs, rhs), - Op::I32Nand { lhs, rhs, .. } => Op::select_i32_and(result, lhs, rhs), - Op::I32Nor { lhs, rhs, .. } => Op::select_i32_or(result, lhs, rhs), - Op::I32BitAndImm16 { lhs, rhs, .. } => Op::select_i32_and_imm16(result, lhs, rhs), - Op::I32BitOrImm16 { lhs, rhs, .. } => Op::select_i32_or_imm16(result, lhs, rhs), - Op::I32BitXorImm16 { lhs, rhs, .. } => Op::select_i32_eq_imm16(result, lhs, rhs), - Op::I32AndImm16 { lhs, rhs, .. } => Op::select_i32_and_imm16(result, lhs, rhs), - Op::I32OrImm16 { lhs, rhs, .. } => Op::select_i32_or_imm16(result, lhs, rhs), - Op::I32NandImm16 { lhs, rhs, .. } => Op::select_i32_and_imm16(result, lhs, rhs), - Op::I32NorImm16 { lhs, rhs, .. } => Op::select_i32_or_imm16(result, lhs, rhs), + | Op::I32And_Sss { lhs, rhs, .. } + | Op::I32BitAnd_Sss { lhs, rhs, .. } => Op::select_i32_and_sss(result, lhs, rhs, val_true, val_false), + | Op::I32And_Ssi { lhs, rhs, .. } + | Op::I32BitAnd_Ssi { lhs, rhs, .. } => Op::select_i32_and_ssi(result, lhs, rhs, val_true, val_false), + | Op::I32Or_Sss { lhs, rhs, .. } + | Op::I32BitOr_Sss { lhs, rhs, .. } => Op::select_i32_or_sss(result, lhs, rhs, val_true, val_false), + | Op::I32Or_Ssi { lhs, rhs, .. } + | Op::I32BitOr_Ssi { lhs, rhs, .. } => Op::select_i32_or_ssi(result, lhs, rhs, val_true, val_false), + | Op::I32NotEq_Sss { lhs, rhs, .. } + | Op::I32BitXor_Sss { lhs, rhs, .. } => Op::select_i32_eq_sss(result, lhs, rhs, val_false, val_true), + | Op::I32NotEq_Ssi { lhs, rhs, .. } + | Op::I32BitXor_Ssi { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, lhs, rhs, val_false, val_true), + | Op::I32Eq_Sss { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::I32Eq_Ssi { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::select_i32_and_ssi(result, lhs, rhs, val_false, val_true), + | Op::I32NotAnd_Ssi { lhs, rhs, .. } => Op::select_i32_and_ssi(result, lhs, rhs, val_false, val_true), + | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::select_i32_or_ssi(result, lhs, rhs, val_false, val_true), + | Op::I32NotOr_Ssi { lhs, rhs, .. } => Op::select_i32_or_ssi(result, lhs, rhs, val_false, val_true), + | Op::I32Lt_Sss { lhs, rhs, .. } => Op::select_i32_lt_sss(result, lhs, rhs, val_true, val_false), + | Op::I32Lt_Ssi { lhs, rhs, .. } => Op::select_i32_lt_ssi(result, lhs, rhs, val_true, val_false), + | Op::I32Lt_Sis { lhs, rhs, .. } => Op::select_i32_le_ssi(result, rhs, lhs, val_false, val_true), + | Op::U32Lt_Sss { lhs, rhs, .. } => Op::select_u32_lt_sss(result, lhs, rhs, val_true, val_false), + | Op::U32Lt_Ssi { lhs, rhs, .. } => Op::select_u32_lt_ssi(result, lhs, rhs, val_true, val_false), + | Op::U32Lt_Sis { lhs, rhs, .. } => Op::select_u32_le_ssi(result, rhs, lhs, val_false, val_true), + | Op::I32Le_Sss { lhs, rhs, .. } => Op::select_i32_le_sss(result, lhs, rhs, val_true, val_false), + | Op::I32Le_Ssi { lhs, rhs, .. } => Op::select_i32_le_ssi(result, lhs, rhs, val_true, val_false), + | Op::I32Le_Sis { lhs, rhs, .. } => Op::select_i32_lt_ssi(result, rhs, lhs, val_false, val_true), + | Op::U32Le_Sss { lhs, rhs, .. } => Op::select_u32_le_sss(result, lhs, rhs, val_true, val_false), + | Op::U32Le_Ssi { lhs, rhs, .. } => Op::select_u32_le_ssi(result, lhs, rhs, val_true, val_false), + | Op::U32Le_Sis { lhs, rhs, .. } => Op::select_u32_lt_ssi(result, rhs, lhs, val_false, val_true), // i64 - Op::I64Eq { lhs, rhs, .. } => Op::select_i64_eq(result, lhs, rhs), - Op::I64Ne { lhs, rhs, .. } => Op::select_i64_eq(result, lhs, rhs), - Op::I64LeS { lhs, rhs, .. } => Op::select_i64_le_s(result, lhs, rhs), - Op::I64LeU { lhs, rhs, .. } => Op::select_i64_le_u(result, lhs, rhs), - Op::I64LtS { lhs, rhs, .. } => Op::select_i64_lt_s(result, lhs, rhs), - Op::I64LtU { lhs, rhs, .. } => Op::select_i64_lt_u(result, lhs, rhs), - Op::I64EqImm16 { lhs, rhs, .. } => Op::select_i64_eq_imm16(result, lhs, rhs), - Op::I64NeImm16 { lhs, rhs, .. } => Op::select_i64_eq_imm16(result, lhs, rhs), - Op::I64LeSImm16Lhs { lhs, rhs, .. } => Op::select_i64_lt_s_imm16_rhs(result, rhs, lhs), - Op::I64LeUImm16Lhs { lhs, rhs, .. } => Op::select_i64_lt_u_imm16_rhs(result, rhs, lhs), - Op::I64LtSImm16Lhs { lhs, rhs, .. } => Op::select_i64_le_s_imm16_rhs(result, rhs, lhs), - Op::I64LtUImm16Lhs { lhs, rhs, .. } => Op::select_i64_le_u_imm16_rhs(result, rhs, lhs), - Op::I64LeSImm16Rhs { lhs, rhs, .. } => Op::select_i64_le_s_imm16_rhs(result, lhs, rhs), - Op::I64LeUImm16Rhs { lhs, rhs, .. } => Op::select_i64_le_u_imm16_rhs(result, lhs, rhs), - Op::I64LtSImm16Rhs { lhs, rhs, .. } => Op::select_i64_lt_s_imm16_rhs(result, lhs, rhs), - Op::I64LtUImm16Rhs { lhs, rhs, .. } => Op::select_i64_lt_u_imm16_rhs(result, lhs, rhs), - // i64 (and, or, xor) - Op::I64BitAnd { lhs, rhs, .. } => Op::select_i64_and(result, lhs, rhs), - Op::I64BitOr { lhs, rhs, .. } => Op::select_i64_or(result, lhs, rhs), - Op::I64BitXor { lhs, rhs, .. } => Op::select_i64_eq(result, lhs, rhs), - Op::I64And { lhs, rhs, .. } => Op::select_i64_and(result, lhs, rhs), - Op::I64Or { lhs, rhs, .. } => Op::select_i64_or(result, lhs, rhs), - Op::I64Nand { lhs, rhs, .. } => Op::select_i64_and(result, lhs, rhs), - Op::I64Nor { lhs, rhs, .. } => Op::select_i64_or(result, lhs, rhs), - Op::I64BitAndImm16 { lhs, rhs, .. } => Op::select_i64_and_imm16(result, lhs, rhs), - Op::I64BitOrImm16 { lhs, rhs, .. } => Op::select_i64_or_imm16(result, lhs, rhs), - Op::I64BitXorImm16 { lhs, rhs, .. } => Op::select_i64_eq_imm16(result, lhs, rhs), - Op::I64AndImm16 { lhs, rhs, .. } => Op::select_i64_and_imm16(result, lhs, rhs), - Op::I64OrImm16 { lhs, rhs, .. } => Op::select_i64_or_imm16(result, lhs, rhs), - Op::I64NandImm16 { lhs, rhs, .. } => Op::select_i64_and_imm16(result, lhs, rhs), - Op::I64NorImm16 { lhs, rhs, .. } => Op::select_i64_or_imm16(result, lhs, rhs), + | Op::I64And_Sss { lhs, rhs, .. } + | Op::I64BitAnd_Sss { lhs, rhs, .. } => Op::select_i64_and_sss(result, lhs, rhs, val_true, val_false), + | Op::I64And_Ssi { lhs, rhs, .. } + | Op::I64BitAnd_Ssi { lhs, rhs, .. } => Op::select_i64_and_ssi(result, lhs, rhs, val_true, val_false), + | Op::I64Or_Sss { lhs, rhs, .. } + | Op::I64BitOr_Sss { lhs, rhs, .. } => Op::select_i64_or_sss(result, lhs, rhs, val_true, val_false), + | Op::I64Or_Ssi { lhs, rhs, .. } + | Op::I64BitOr_Ssi { lhs, rhs, .. } => Op::select_i64_or_ssi(result, lhs, rhs, val_true, val_false), + | Op::I64NotEq_Sss { lhs, rhs, .. } + | Op::I64BitXor_Sss { lhs, rhs, .. } => Op::select_i64_eq_sss(result, lhs, rhs, val_false, val_true), + | Op::I64NotEq_Ssi { lhs, rhs, .. } + | Op::I64BitXor_Ssi { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, lhs, rhs, val_false, val_true), + | Op::I64Eq_Sss { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::I64Eq_Ssi { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::select_i64_and_ssi(result, lhs, rhs, val_false, val_true), + | Op::I64NotAnd_Ssi { lhs, rhs, .. } => Op::select_i64_and_ssi(result, lhs, rhs, val_false, val_true), + | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::select_i64_or_ssi(result, lhs, rhs, val_false, val_true), + | Op::I64NotOr_Ssi { lhs, rhs, .. } => Op::select_i64_or_ssi(result, lhs, rhs, val_false, val_true), + | Op::I64Lt_Sss { lhs, rhs, .. } => Op::select_i64_lt_sss(result, lhs, rhs, val_true, val_false), + | Op::I64Lt_Ssi { lhs, rhs, .. } => Op::select_i64_lt_ssi(result, lhs, rhs, val_true, val_false), + | Op::I64Lt_Sis { lhs, rhs, .. } => Op::select_i64_le_ssi(result, rhs, lhs, val_false, val_true), + | Op::U64Lt_Sss { lhs, rhs, .. } => Op::select_u64_lt_sss(result, lhs, rhs, val_true, val_false), + | Op::U64Lt_Ssi { lhs, rhs, .. } => Op::select_u64_lt_ssi(result, lhs, rhs, val_true, val_false), + | Op::U64Lt_Sis { lhs, rhs, .. } => Op::select_u64_le_ssi(result, rhs, lhs, val_false, val_true), + | Op::I64Le_Sss { lhs, rhs, .. } => Op::select_i64_le_sss(result, lhs, rhs, val_true, val_false), + | Op::I64Le_Ssi { lhs, rhs, .. } => Op::select_i64_le_ssi(result, lhs, rhs, val_true, val_false), + | Op::I64Le_Sis { lhs, rhs, .. } => Op::select_i64_lt_ssi(result, rhs, lhs, val_false, val_true), + | Op::U64Le_Sss { lhs, rhs, .. } => Op::select_u64_le_sss(result, lhs, rhs, val_true, val_false), + | Op::U64Le_Ssi { lhs, rhs, .. } => Op::select_u64_le_ssi(result, lhs, rhs, val_true, val_false), + | Op::U64Le_Sis { lhs, rhs, .. } => Op::select_u64_lt_ssi(result, rhs, lhs, val_false, val_true), // f32 - Op::F32Eq { lhs, rhs, .. } => Op::select_f32_eq(result, lhs, rhs), - Op::F32Ne { lhs, rhs, .. } => Op::select_f32_eq(result, lhs, rhs), - Op::F32Lt { lhs, rhs, .. } => Op::select_f32_lt(result, lhs, rhs), - Op::F32Le { lhs, rhs, .. } => Op::select_f32_le(result, lhs, rhs), - Op::F32NotLt { lhs, rhs, .. } => Op::select_f32_lt(result, lhs, rhs), - Op::F32NotLe { lhs, rhs, .. } => Op::select_f32_le(result, lhs, rhs), + | Op::F32Eq_Sss { lhs, rhs, .. } => Op::select_f32_eq_sss(result, lhs, rhs, val_true, val_false), + | Op::F32Eq_Ssi { lhs, rhs, .. } => Op::select_f32_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::F32Lt_Sss { lhs, rhs, .. } => Op::select_f32_lt_sss(result, lhs, rhs, val_true, val_false), + | Op::F32Lt_Ssi { lhs, rhs, .. } => Op::select_f32_lt_ssi(result, lhs, rhs, val_true, val_false), + | Op::F32Lt_Sis { lhs, rhs, .. } => Op::select_f32_lt_sis(result, lhs, rhs, val_true, val_false), + | Op::F32Le_Sss { lhs, rhs, .. } => Op::select_f32_le_sss(result, lhs, rhs, val_true, val_false), + | Op::F32Le_Ssi { lhs, rhs, .. } => Op::select_f32_le_ssi(result, lhs, rhs, val_true, val_false), + | Op::F32Le_Sis { lhs, rhs, .. } => Op::select_f32_le_sis(result, lhs, rhs, val_true, val_false), + | Op::F32NotEq_Sss { lhs, rhs, .. } => Op::select_f32_eq_sss(result, lhs, rhs, val_false, val_true), + | Op::F32NotEq_Ssi { lhs, rhs, .. } => Op::select_f32_eq_ssi(result, lhs, rhs, val_false, val_true), + | Op::F32NotLt_Sss { lhs, rhs, .. } => Op::select_f32_lt_sss(result, lhs, rhs, val_false, val_true), + | Op::F32NotLt_Ssi { lhs, rhs, .. } => Op::select_f32_lt_ssi(result, lhs, rhs, val_false, val_true), + | Op::F32NotLt_Sis { lhs, rhs, .. } => Op::select_f32_lt_sis(result, lhs, rhs, val_false, val_true), + | Op::F32NotLe_Sss { lhs, rhs, .. } => Op::select_f32_le_sss(result, lhs, rhs, val_false, val_true), + | Op::F32NotLe_Ssi { lhs, rhs, .. } => Op::select_f32_le_ssi(result, lhs, rhs, val_false, val_true), + | Op::F32NotLe_Sis { lhs, rhs, .. } => Op::select_f32_le_sis(result, lhs, rhs, val_false, val_true), // f64 - Op::F64Eq { lhs, rhs, .. } => Op::select_f64_eq(result, lhs, rhs), - Op::F64Ne { lhs, rhs, .. } => Op::select_f64_eq(result, lhs, rhs), - Op::F64Lt { lhs, rhs, .. } => Op::select_f64_lt(result, lhs, rhs), - Op::F64Le { lhs, rhs, .. } => Op::select_f64_le(result, lhs, rhs), - Op::F64NotLt { lhs, rhs, .. } => Op::select_f64_lt(result, lhs, rhs), - Op::F64NotLe { lhs, rhs, .. } => Op::select_f64_le(result, lhs, rhs), + | Op::F64Eq_Sss { lhs, rhs, .. } => Op::select_f64_eq_sss(result, lhs, rhs, val_true, val_false), + | Op::F64Eq_Ssi { lhs, rhs, .. } => Op::select_f64_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::F64Lt_Sss { lhs, rhs, .. } => Op::select_f64_lt_sss(result, lhs, rhs, val_true, val_false), + | Op::F64Lt_Ssi { lhs, rhs, .. } => Op::select_f64_lt_ssi(result, lhs, rhs, val_true, val_false), + | Op::F64Lt_Sis { lhs, rhs, .. } => Op::select_f64_lt_sis(result, lhs, rhs, val_true, val_false), + | Op::F64Le_Sss { lhs, rhs, .. } => Op::select_f64_le_sss(result, lhs, rhs, val_true, val_false), + | Op::F64Le_Ssi { lhs, rhs, .. } => Op::select_f64_le_ssi(result, lhs, rhs, val_true, val_false), + | Op::F64Le_Sis { lhs, rhs, .. } => Op::select_f64_le_sis(result, lhs, rhs, val_true, val_false), + | Op::F64NotEq_Sss { lhs, rhs, .. } => Op::select_f64_eq_sss(result, lhs, rhs, val_false, val_true), + | Op::F64NotEq_Ssi { lhs, rhs, .. } => Op::select_f64_eq_ssi(result, lhs, rhs, val_false, val_true), + | Op::F64NotLt_Sss { lhs, rhs, .. } => Op::select_f64_lt_sss(result, lhs, rhs, val_false, val_true), + | Op::F64NotLt_Ssi { lhs, rhs, .. } => Op::select_f64_lt_ssi(result, lhs, rhs, val_false, val_true), + | Op::F64NotLt_Sis { lhs, rhs, .. } => Op::select_f64_lt_sis(result, lhs, rhs, val_false, val_true), + | Op::F64NotLe_Sss { lhs, rhs, .. } => Op::select_f64_le_sss(result, lhs, rhs, val_false, val_true), + | Op::F64NotLe_Ssi { lhs, rhs, .. } => Op::select_f64_le_ssi(result, lhs, rhs, val_false, val_true), + | Op::F64NotLe_Sis { lhs, rhs, .. } => Op::select_f64_le_sis(result, lhs, rhs, val_false, val_true), _ => unreachable!("expected to successfully fuse cmp+select"), }; - Ok(CmpSelectFusion::Applied { - fused, - swap_operands, - }) + Ok(CmpSelectFusion::Applied(fused)) } } pub trait TryIntoCmpBranchInstr: Sized { - fn try_into_cmp_branch_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error>; + fn try_into_cmp_branch_instr(&self, offset: BranchOffset) -> Option; } impl TryIntoCmpBranchInstr for Op { - fn try_into_cmp_branch_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error> { - let Ok(offset) = BranchOffset16::try_from(offset) else { - return self.try_into_cmp_branch_fallback_instr(offset, stack); - }; + fn try_into_cmp_branch_instr(&self, offset: BranchOffset) -> Option { #[rustfmt::skip] let cmp_branch_instr = match *self { // i32 - Op::I32Eq { lhs, rhs, .. } => Op::branch_i32_eq(lhs, rhs, offset), - Op::I32Ne { lhs, rhs, .. } => Op::branch_i32_ne(lhs, rhs, offset), - Op::I32LeS { lhs, rhs, .. } => Op::branch_i32_le_s(lhs, rhs, offset), - Op::I32LeU { lhs, rhs, .. } => Op::branch_i32_le_u(lhs, rhs, offset), - Op::I32LtS { lhs, rhs, .. } => Op::branch_i32_lt_s(lhs, rhs, offset), - Op::I32LtU { lhs, rhs, .. } => Op::branch_i32_lt_u(lhs, rhs, offset), - Op::I32EqImm16 { lhs, rhs, .. } => Op::branch_i32_eq_imm16(lhs, rhs, offset), - Op::I32NeImm16 { lhs, rhs, .. } => Op::branch_i32_ne_imm16(lhs, rhs, offset), - Op::I32LeSImm16Lhs { lhs, rhs, .. } => Op::branch_i32_le_s_imm16_lhs(lhs, rhs, offset), - Op::I32LeUImm16Lhs { lhs, rhs, .. } => Op::branch_i32_le_u_imm16_lhs(lhs, rhs, offset), - Op::I32LtSImm16Lhs { lhs, rhs, .. } => Op::branch_i32_lt_s_imm16_lhs(lhs, rhs, offset), - Op::I32LtUImm16Lhs { lhs, rhs, .. } => Op::branch_i32_lt_u_imm16_lhs(lhs, rhs, offset), - Op::I32LeSImm16Rhs { lhs, rhs, .. } => Op::branch_i32_le_s_imm16_rhs(lhs, rhs, offset), - Op::I32LeUImm16Rhs { lhs, rhs, .. } => Op::branch_i32_le_u_imm16_rhs(lhs, rhs, offset), - Op::I32LtSImm16Rhs { lhs, rhs, .. } => Op::branch_i32_lt_s_imm16_rhs(lhs, rhs, offset), - Op::I32LtUImm16Rhs { lhs, rhs, .. } => Op::branch_i32_lt_u_imm16_rhs(lhs, rhs, offset), - // i32 (and, or, xor) - Op::I32BitAnd { lhs, rhs, .. } => Op::branch_i32_and(lhs, rhs, offset), - Op::I32BitOr { lhs, rhs, .. } => Op::branch_i32_or(lhs, rhs, offset), - Op::I32BitXor { lhs, rhs, .. } => Op::branch_i32_ne(lhs, rhs, offset), - Op::I32And { lhs, rhs, .. } => Op::branch_i32_and(lhs, rhs, offset), - Op::I32Or { lhs, rhs, .. } => Op::branch_i32_or(lhs, rhs, offset), - Op::I32Nand { lhs, rhs, .. } => Op::branch_i32_nand(lhs, rhs, offset), - Op::I32Nor { lhs, rhs, .. } => Op::branch_i32_nor(lhs, rhs, offset), - Op::I32BitAndImm16 { lhs, rhs, .. } => Op::branch_i32_and_imm16(lhs, rhs, offset), - Op::I32BitOrImm16 { lhs, rhs, .. } => Op::branch_i32_or_imm16(lhs, rhs, offset), - Op::I32BitXorImm16 { lhs, rhs, .. } => Op::branch_i32_ne_imm16(lhs, rhs, offset), - Op::I32AndImm16 { lhs, rhs, .. } => Op::branch_i32_and_imm16(lhs, rhs, offset), - Op::I32OrImm16 { lhs, rhs, .. } => Op::branch_i32_or_imm16(lhs, rhs, offset), - Op::I32NandImm16 { lhs, rhs, .. } => Op::branch_i32_nand_imm16(lhs, rhs, offset), - Op::I32NorImm16 { lhs, rhs, .. } => Op::branch_i32_nor_imm16(lhs, rhs, offset), + | Op::I32Eq_Sss { lhs, rhs, .. } => Op::branch_i32_eq_si(lhs, rhs, offset), + | Op::I32Eq_Ssi { lhs, rhs, .. } => Op::branch_i32_eq_si(lhs, rhs, offset), + | Op::I32And_Sss { lhs, rhs, .. } + | Op::I32BitAnd_Sss { lhs, rhs, .. } => Op::branch_i32_and_ss(lhs, rhs, offset), + | Op::I32And_Ssi { lhs, rhs, .. } + | Op::I32BitAnd_Ssi { lhs, rhs, .. } => Op::branch_i32_and_si(lhs, rhs, offset), + | Op::I32Or_Sss { lhs, rhs, .. } + | Op::I32BitOr_Sss { lhs, rhs, .. } => Op::branch_i32_or_ss(lhs, rhs, offset), + | Op::I32Or_Ssi { lhs, rhs, .. } + | Op::I32BitOr_Ssi { lhs, rhs, .. } => Op::branch_i32_or_si(lhs, rhs, offset), + | Op::I32NotEq_Sss { lhs, rhs, .. } + | Op::I32BitXor_Sss { lhs, rhs, .. } => Op::branch_i32_not_eq_ss(lhs, rhs, offset), + | Op::I32NotEq_Ssi { lhs, rhs, .. } + | Op::I32BitXor_Ssi { lhs, rhs, .. } => Op::branch_i32_not_eq_si(lhs, rhs, offset), + | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::branch_i32_not_and_si(lhs, rhs, offset), + | Op::I32NotAnd_Ssi { lhs, rhs, .. } => Op::branch_i32_not_and_si(lhs, rhs, offset), + | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::branch_i32_not_or_si(lhs, rhs, offset), + | Op::I32NotOr_Ssi { lhs, rhs, .. } => Op::branch_i32_not_or_si(lhs, rhs, offset), + | Op::I32Lt_Sss { lhs, rhs, .. } => Op::branch_i32_lt_ss(lhs, rhs, offset), + | Op::I32Lt_Ssi { lhs, rhs, .. } => Op::branch_i32_lt_si(lhs, rhs, offset), + | Op::I32Lt_Sis { lhs, rhs, .. } => Op::branch_i32_lt_is(rhs, lhs, offset), + | Op::U32Lt_Sss { lhs, rhs, .. } => Op::branch_u32_lt_ss(lhs, rhs, offset), + | Op::U32Lt_Ssi { lhs, rhs, .. } => Op::branch_u32_lt_si(lhs, rhs, offset), + | Op::U32Lt_Sis { lhs, rhs, .. } => Op::branch_u32_lt_is(rhs, lhs, offset), + | Op::I32Le_Sss { lhs, rhs, .. } => Op::branch_i32_le_ss(lhs, rhs, offset), + | Op::I32Le_Ssi { lhs, rhs, .. } => Op::branch_i32_le_si(lhs, rhs, offset), + | Op::I32Le_Sis { lhs, rhs, .. } => Op::branch_i32_le_is(rhs, lhs, offset), + | Op::U32Le_Sss { lhs, rhs, .. } => Op::branch_u32_le_ss(lhs, rhs, offset), + | Op::U32Le_Ssi { lhs, rhs, .. } => Op::branch_u32_le_si(lhs, rhs, offset), + | Op::U32Le_Sis { lhs, rhs, .. } => Op::branch_u32_le_is(rhs, lhs, offset), // i64 - Op::I64Eq { lhs, rhs, .. } => Op::branch_i64_eq(lhs, rhs, offset), - Op::I64Ne { lhs, rhs, .. } => Op::branch_i64_ne(lhs, rhs, offset), - Op::I64LeS { lhs, rhs, .. } => Op::branch_i64_le_s(lhs, rhs, offset), - Op::I64LeU { lhs, rhs, .. } => Op::branch_i64_le_u(lhs, rhs, offset), - Op::I64LtS { lhs, rhs, .. } => Op::branch_i64_lt_s(lhs, rhs, offset), - Op::I64LtU { lhs, rhs, .. } => Op::branch_i64_lt_u(lhs, rhs, offset), - Op::I64EqImm16 { lhs, rhs, .. } => Op::branch_i64_eq_imm16(lhs, rhs, offset), - Op::I64NeImm16 { lhs, rhs, .. } => Op::branch_i64_ne_imm16(lhs, rhs, offset), - Op::I64LeSImm16Lhs { lhs, rhs, .. } => Op::branch_i64_le_s_imm16_lhs(lhs, rhs, offset), - Op::I64LeUImm16Lhs { lhs, rhs, .. } => Op::branch_i64_le_u_imm16_lhs(lhs, rhs, offset), - Op::I64LtSImm16Lhs { lhs, rhs, .. } => Op::branch_i64_lt_s_imm16_lhs(lhs, rhs, offset), - Op::I64LtUImm16Lhs { lhs, rhs, .. } => Op::branch_i64_lt_u_imm16_lhs(lhs, rhs, offset), - Op::I64LeSImm16Rhs { lhs, rhs, .. } => Op::branch_i64_le_s_imm16_rhs(lhs, rhs, offset), - Op::I64LeUImm16Rhs { lhs, rhs, .. } => Op::branch_i64_le_u_imm16_rhs(lhs, rhs, offset), - Op::I64LtSImm16Rhs { lhs, rhs, .. } => Op::branch_i64_lt_s_imm16_rhs(lhs, rhs, offset), - Op::I64LtUImm16Rhs { lhs, rhs, .. } => Op::branch_i64_lt_u_imm16_rhs(lhs, rhs, offset), - // i64 (and, or, xor) - Op::I64BitAnd { lhs, rhs, .. } => Op::branch_i64_and(lhs, rhs, offset), - Op::I64BitOr { lhs, rhs, .. } => Op::branch_i64_or(lhs, rhs, offset), - Op::I64BitXor { lhs, rhs, .. } => Op::branch_i64_ne(lhs, rhs, offset), - Op::I64And { lhs, rhs, .. } => Op::branch_i64_and(lhs, rhs, offset), - Op::I64Or { lhs, rhs, .. } => Op::branch_i64_or(lhs, rhs, offset), - Op::I64Nand { lhs, rhs, .. } => Op::branch_i64_nand(lhs, rhs, offset), - Op::I64Nor { lhs, rhs, .. } => Op::branch_i64_nor(lhs, rhs, offset), - Op::I64BitAndImm16 { lhs, rhs, .. } => Op::branch_i64_and_imm16(lhs, rhs, offset), - Op::I64BitOrImm16 { lhs, rhs, .. } => Op::branch_i64_or_imm16(lhs, rhs, offset), - Op::I64BitXorImm16 { lhs, rhs, .. } => Op::branch_i64_ne_imm16(lhs, rhs, offset), - Op::I64AndImm16 { lhs, rhs, .. } => Op::branch_i64_and_imm16(lhs, rhs, offset), - Op::I64OrImm16 { lhs, rhs, .. } => Op::branch_i64_or_imm16(lhs, rhs, offset), - Op::I64NandImm16 { lhs, rhs, .. } => Op::branch_i64_nand_imm16(lhs, rhs, offset), - Op::I64NorImm16 { lhs, rhs, .. } => Op::branch_i64_nor_imm16(lhs, rhs, offset), + | Op::I64Eq_Sss { lhs, rhs, .. } => Op::branch_i64_eq_si(lhs, rhs, offset), + | Op::I64Eq_Ssi { lhs, rhs, .. } => Op::branch_i64_eq_si(lhs, rhs, offset), + | Op::I64And_Sss { lhs, rhs, .. } + | Op::I64BitAnd_Sss { lhs, rhs, .. } => Op::branch_i64_and_ss(lhs, rhs, offset), + | Op::I64And_Ssi { lhs, rhs, .. } + | Op::I64BitAnd_Ssi { lhs, rhs, .. } => Op::branch_i64_and_si(lhs, rhs, offset), + | Op::I64Or_Sss { lhs, rhs, .. } + | Op::I64BitOr_Sss { lhs, rhs, .. } => Op::branch_i64_or_ss(lhs, rhs, offset), + | Op::I64Or_Ssi { lhs, rhs, .. } + | Op::I64BitOr_Ssi { lhs, rhs, .. } => Op::branch_i64_or_si(lhs, rhs, offset), + | Op::I64NotEq_Sss { lhs, rhs, .. } + | Op::I64BitXor_Sss { lhs, rhs, .. } => Op::branch_i64_not_eq_ss(lhs, rhs, offset), + | Op::I64NotEq_Ssi { lhs, rhs, .. } + | Op::I64BitXor_Ssi { lhs, rhs, .. } => Op::branch_i64_not_eq_si(lhs, rhs, offset), + | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::branch_i64_not_and_si(lhs, rhs, offset), + | Op::I64NotAnd_Ssi { lhs, rhs, .. } => Op::branch_i64_not_and_si(lhs, rhs, offset), + | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::branch_i64_not_or_si(lhs, rhs, offset), + | Op::I64NotOr_Ssi { lhs, rhs, .. } => Op::branch_i64_not_or_si(lhs, rhs, offset), + | Op::I64Lt_Sss { lhs, rhs, .. } => Op::branch_i64_lt_ss(lhs, rhs, offset), + | Op::I64Lt_Ssi { lhs, rhs, .. } => Op::branch_i64_lt_si(lhs, rhs, offset), + | Op::I64Lt_Sis { lhs, rhs, .. } => Op::branch_i64_lt_is(rhs, lhs, offset), + | Op::U64Lt_Sss { lhs, rhs, .. } => Op::branch_u64_lt_ss(lhs, rhs, offset), + | Op::U64Lt_Ssi { lhs, rhs, .. } => Op::branch_u64_lt_si(lhs, rhs, offset), + | Op::U64Lt_Sis { lhs, rhs, .. } => Op::branch_u64_lt_is(rhs, lhs, offset), + | Op::I64Le_Sss { lhs, rhs, .. } => Op::branch_i64_le_ss(lhs, rhs, offset), + | Op::I64Le_Ssi { lhs, rhs, .. } => Op::branch_i64_le_si(lhs, rhs, offset), + | Op::I64Le_Sis { lhs, rhs, .. } => Op::branch_i64_le_is(rhs, lhs, offset), + | Op::U64Le_Sss { lhs, rhs, .. } => Op::branch_u64_le_ss(lhs, rhs, offset), + | Op::U64Le_Ssi { lhs, rhs, .. } => Op::branch_u64_le_si(lhs, rhs, offset), + | Op::U64Le_Sis { lhs, rhs, .. } => Op::branch_u64_le_is(rhs, lhs, offset), // f32 - Op::F32Eq { lhs, rhs, .. } => Op::branch_f32_eq(lhs, rhs, offset), - Op::F32Ne { lhs, rhs, .. } => Op::branch_f32_ne(lhs, rhs, offset), - Op::F32Lt { lhs, rhs, .. } => Op::branch_f32_lt(lhs, rhs, offset), - Op::F32Le { lhs, rhs, .. } => Op::branch_f32_le(lhs, rhs, offset), - Op::F32NotLt { lhs, rhs, .. } => Op::branch_f32_not_lt(lhs, rhs, offset), - Op::F32NotLe { lhs, rhs, .. } => Op::branch_f32_not_le(lhs, rhs, offset), + | Op::F32Eq_Sss { lhs, rhs, .. } => Op::branch_f32_eq_ss(lhs, rhs, offset), + | Op::F32Eq_Ssi { lhs, rhs, .. } => Op::branch_f32_eq_si(lhs, rhs, offset), + | Op::F32Lt_Sss { lhs, rhs, .. } => Op::branch_f32_lt_ss(lhs, rhs, offset), + | Op::F32Lt_Ssi { lhs, rhs, .. } => Op::branch_f32_lt_si(lhs, rhs, offset), + | Op::F32Lt_Sis { lhs, rhs, .. } => Op::branch_f32_lt_is(lhs, rhs, offset), + | Op::F32Le_Sss { lhs, rhs, .. } => Op::branch_f32_le_ss(lhs, rhs, offset), + | Op::F32Le_Ssi { lhs, rhs, .. } => Op::branch_f32_le_si(lhs, rhs, offset), + | Op::F32Le_Sis { lhs, rhs, .. } => Op::branch_f32_le_is(lhs, rhs, offset), + | Op::F32NotEq_Sss { lhs, rhs, .. } => Op::branch_f32_eq_ss(lhs, rhs, offset), + | Op::F32NotEq_Ssi { lhs, rhs, .. } => Op::branch_f32_eq_si(lhs, rhs, offset), + | Op::F32NotLt_Sss { lhs, rhs, .. } => Op::branch_f32_lt_ss(lhs, rhs, offset), + | Op::F32NotLt_Ssi { lhs, rhs, .. } => Op::branch_f32_lt_si(lhs, rhs, offset), + | Op::F32NotLt_Sis { lhs, rhs, .. } => Op::branch_f32_lt_is(lhs, rhs, offset), + | Op::F32NotLe_Sss { lhs, rhs, .. } => Op::branch_f32_le_ss(lhs, rhs, offset), + | Op::F32NotLe_Ssi { lhs, rhs, .. } => Op::branch_f32_le_si(lhs, rhs, offset), + | Op::F32NotLe_Sis { lhs, rhs, .. } => Op::branch_f32_le_is(lhs, rhs, offset), // f64 - Op::F64Eq { lhs, rhs, .. } => Op::branch_f64_eq(lhs, rhs, offset), - Op::F64Ne { lhs, rhs, .. } => Op::branch_f64_ne(lhs, rhs, offset), - Op::F64Lt { lhs, rhs, .. } => Op::branch_f64_lt(lhs, rhs, offset), - Op::F64Le { lhs, rhs, .. } => Op::branch_f64_le(lhs, rhs, offset), - Op::F64NotLt { lhs, rhs, .. } => Op::branch_f64_not_lt(lhs, rhs, offset), - Op::F64NotLe { lhs, rhs, .. } => Op::branch_f64_not_le(lhs, rhs, offset), - _ => return Ok(None), - }; - Ok(Some(cmp_branch_instr)) - } -} - -pub trait TryIntoCmpBranchFallbackInstr { - fn try_into_cmp_branch_fallback_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error>; -} - -impl TryIntoCmpBranchFallbackInstr for Op { - fn try_into_cmp_branch_fallback_instr( - &self, - offset: BranchOffset, - stack: &mut impl AllocConst, - ) -> Result, Error> { - debug_assert!(BranchOffset16::try_from(offset).is_err()); - let Some(comparator) = try_into_cmp_br_comparator(self) else { - return Ok(None); - }; - #[rustfmt::skip] - let (lhs, rhs) = match *self { - | Op::BranchI32And { lhs, rhs, .. } - | Op::BranchI32Or { lhs, rhs, .. } - | Op::BranchI32Nand { lhs, rhs, .. } - | Op::BranchI32Nor { lhs, rhs, .. } - | Op::BranchI32Eq { lhs, rhs, .. } - | Op::BranchI32Ne { lhs, rhs, .. } - | Op::BranchI32LtS { lhs, rhs, .. } - | Op::BranchI32LtU { lhs, rhs, .. } - | Op::BranchI32LeS { lhs, rhs, .. } - | Op::BranchI32LeU { lhs, rhs, .. } - | Op::BranchI64And { lhs, rhs, .. } - | Op::BranchI64Or { lhs, rhs, .. } - | Op::BranchI64Nand { lhs, rhs, .. } - | Op::BranchI64Nor { lhs, rhs, .. } - | Op::BranchI64Eq { lhs, rhs, .. } - | Op::BranchI64Ne { lhs, rhs, .. } - | Op::BranchI64LtS { lhs, rhs, .. } - | Op::BranchI64LtU { lhs, rhs, .. } - | Op::BranchI64LeS { lhs, rhs, .. } - | Op::BranchI64LeU { lhs, rhs, .. } - | Op::BranchF32Eq { lhs, rhs, .. } - | Op::BranchF32Ne { lhs, rhs, .. } - | Op::BranchF32Lt { lhs, rhs, .. } - | Op::BranchF32Le { lhs, rhs, .. } - | Op::BranchF32NotLt { lhs, rhs, .. } - | Op::BranchF32NotLe { lhs, rhs, .. } - | Op::BranchF64Eq { lhs, rhs, .. } - | Op::BranchF64Ne { lhs, rhs, .. } - | Op::BranchF64Lt { lhs, rhs, .. } - | Op::BranchF64Le { lhs, rhs, .. } - | Op::BranchF64NotLt { lhs, rhs, .. } - | Op::BranchF64NotLe { lhs, rhs, .. } => (lhs, rhs), - | Op::BranchI32AndImm16 { lhs, rhs, .. } - | Op::BranchI32OrImm16 { lhs, rhs, .. } - | Op::BranchI32NandImm16 { lhs, rhs, .. } - | Op::BranchI32NorImm16 { lhs, rhs, .. } - | Op::BranchI32EqImm16 { lhs, rhs, .. } - | Op::BranchI32NeImm16 { lhs, rhs, .. } - | Op::BranchI32LtSImm16Rhs { lhs, rhs, .. } - | Op::BranchI32LeSImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(i32::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI32LtSImm16Lhs { lhs, rhs, .. } - | Op::BranchI32LeSImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(i32::from(lhs))?; - (lhs, rhs) - } - | Op::BranchI32LtUImm16Rhs { lhs, rhs, .. } - | Op::BranchI32LeUImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(u32::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI32LtUImm16Lhs { lhs, rhs, .. } - | Op::BranchI32LeUImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(u32::from(lhs))?; - (lhs, rhs) - } - | Op::BranchI64AndImm16 { lhs, rhs, .. } - | Op::BranchI64OrImm16 { lhs, rhs, .. } - | Op::BranchI64NandImm16 { lhs, rhs, .. } - | Op::BranchI64NorImm16 { lhs, rhs, .. } - | Op::BranchI64EqImm16 { lhs, rhs, .. } - | Op::BranchI64NeImm16 { lhs, rhs, .. } - | Op::BranchI64LtSImm16Rhs { lhs, rhs, .. } - | Op::BranchI64LeSImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(i64::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI64LtSImm16Lhs { lhs, rhs, .. } - | Op::BranchI64LeSImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(i64::from(lhs))?; - (lhs, rhs) - } - | Op::BranchI64LtUImm16Rhs { lhs, rhs, .. } - | Op::BranchI64LeUImm16Rhs { lhs, rhs, .. } => { - let rhs = stack.alloc_const(u64::from(rhs))?; - (lhs, rhs) - } - | Op::BranchI64LtUImm16Lhs { lhs, rhs, .. } - | Op::BranchI64LeUImm16Lhs { lhs, rhs, .. } => { - let lhs = stack.alloc_const(u64::from(lhs))?; - (lhs, rhs) - } - _ => return Ok(None), + | Op::F64Eq_Sss { lhs, rhs, .. } => Op::branch_f64_eq_ss(lhs, rhs, offset), + | Op::F64Eq_Ssi { lhs, rhs, .. } => Op::branch_f64_eq_si(lhs, rhs, offset), + | Op::F64Lt_Sss { lhs, rhs, .. } => Op::branch_f64_lt_ss(lhs, rhs, offset), + | Op::F64Lt_Ssi { lhs, rhs, .. } => Op::branch_f64_lt_si(lhs, rhs, offset), + | Op::F64Lt_Sis { lhs, rhs, .. } => Op::branch_f64_lt_is(lhs, rhs, offset), + | Op::F64Le_Sss { lhs, rhs, .. } => Op::branch_f64_le_ss(lhs, rhs, offset), + | Op::F64Le_Ssi { lhs, rhs, .. } => Op::branch_f64_le_si(lhs, rhs, offset), + | Op::F64Le_Sis { lhs, rhs, .. } => Op::branch_f64_le_is(lhs, rhs, offset), + | Op::F64NotEq_Sss { lhs, rhs, .. } => Op::branch_f64_eq_ss(lhs, rhs, offset), + | Op::F64NotEq_Ssi { lhs, rhs, .. } => Op::branch_f64_eq_si(lhs, rhs, offset), + | Op::F64NotLt_Sss { lhs, rhs, .. } => Op::branch_f64_lt_ss(lhs, rhs, offset), + | Op::F64NotLt_Ssi { lhs, rhs, .. } => Op::branch_f64_lt_si(lhs, rhs, offset), + | Op::F64NotLt_Sis { lhs, rhs, .. } => Op::branch_f64_lt_is(lhs, rhs, offset), + | Op::F64NotLe_Sss { lhs, rhs, .. } => Op::branch_f64_le_ss(lhs, rhs, offset), + | Op::F64NotLe_Ssi { lhs, rhs, .. } => Op::branch_f64_le_si(lhs, rhs, offset), + | Op::F64NotLe_Sis { lhs, rhs, .. } => Op::branch_f64_le_is(lhs, rhs, offset), + _ => return None, }; - let params = stack.alloc_const(ComparatorAndOffset::new(comparator, offset))?; - Ok(Some(Op::branch_cmp_fallback(lhs, rhs, params))) + Some(cmp_branch_instr) } } -fn try_into_cmp_br_comparator(instr: &Op) -> Option { - #[rustfmt::skip] - let comparator = match *instr { - // i32 - | Op::BranchI32Eq { .. } | Op::BranchI32EqImm16 { .. } => Comparator::I32Eq, - | Op::BranchI32Ne { .. } | Op::BranchI32NeImm16 { .. } => Comparator::I32Ne, - | Op::BranchI32LtS { .. } - | Op::BranchI32LtSImm16Lhs { .. } - | Op::BranchI32LtSImm16Rhs { .. } => Comparator::I32LtS, - | Op::BranchI32LtU { .. } - | Op::BranchI32LtUImm16Lhs { .. } - | Op::BranchI32LtUImm16Rhs { .. } => Comparator::I32LtU, - | Op::BranchI32LeS { .. } - | Op::BranchI32LeSImm16Lhs { .. } - | Op::BranchI32LeSImm16Rhs { .. } => Comparator::I32LeS, - | Op::BranchI32LeU { .. } - | Op::BranchI32LeUImm16Lhs { .. } - | Op::BranchI32LeUImm16Rhs { .. } => Comparator::I32LeU, - // i32 (and,or,xor) - | Op::BranchI32And { .. } => Comparator::I32And, - | Op::BranchI32Or { .. } => Comparator::I32Or, - | Op::BranchI32Nand { .. } => Comparator::I32Nand, - | Op::BranchI32Nor { .. } => Comparator::I32Nor, - // i64 - | Op::BranchI64Eq { .. } | Op::BranchI64EqImm16 { .. } => Comparator::I64Eq, - | Op::BranchI64Ne { .. } | Op::BranchI64NeImm16 { .. } => Comparator::I64Ne, - | Op::BranchI64LtS { .. } - | Op::BranchI64LtSImm16Lhs { .. } - | Op::BranchI64LtSImm16Rhs { .. } => Comparator::I64LtS, - | Op::BranchI64LtU { .. } - | Op::BranchI64LtUImm16Lhs { .. } - | Op::BranchI64LtUImm16Rhs { .. } => Comparator::I64LtU, - | Op::BranchI64LeS { .. } - | Op::BranchI64LeSImm16Lhs { .. } - | Op::BranchI64LeSImm16Rhs { .. } => Comparator::I64LeS, - | Op::BranchI64LeU { .. } - | Op::BranchI64LeUImm16Lhs { .. } - | Op::BranchI64LeUImm16Rhs { .. } => Comparator::I64LeU, - // f32 - | Op::BranchF32Eq { .. } => Comparator::F32Eq, - | Op::BranchF32Ne { .. } => Comparator::F32Ne, - | Op::BranchF32Lt { .. } => Comparator::F32Lt, - | Op::BranchF32Le { .. } => Comparator::F32Le, - | Op::BranchF32NotLt { .. } => Comparator::F32NotLt, - | Op::BranchF32NotLe { .. } => Comparator::F32NotLe, - // f64 - | Op::BranchF64Eq { .. } => Comparator::F64Eq, - | Op::BranchF64Ne { .. } => Comparator::F64Ne, - | Op::BranchF64Lt { .. } => Comparator::F64Lt, - | Op::BranchF64Le { .. } => Comparator::F64Le, - | Op::BranchF64NotLt { .. } => Comparator::F64NotLt, - | Op::BranchF64NotLe { .. } => Comparator::F64NotLe, - _ => return None, - }; - Some(comparator) -} - /// Extension trait to update the branch offset of an [`Op`]. pub trait UpdateBranchOffset { /// Updates the [`BranchOffset`] for the branch [`Op`]. @@ -801,101 +598,106 @@ pub trait UpdateBranchOffset { /// # Panics /// /// If `self` is not a branch [`Op`]. - fn update_branch_offset( - &mut self, - stack: &mut impl AllocConst, - new_offset: BranchOffset, - ) -> Result<(), Error>; + fn update_branch_offset(&mut self, new_offset: BranchOffset) -> Result<(), Error>; } impl UpdateBranchOffset for Op { #[rustfmt::skip] fn update_branch_offset( &mut self, - stack: &mut impl AllocConst, new_offset: BranchOffset, ) -> Result<(), Error> { - match self { - | Op::Branch { offset } - | Op::BranchTableTarget { offset, .. } => { - offset.init(new_offset); - return Ok(()); - } - _ => {} - }; let offset = match self { - | Op::BranchI32And { offset, .. } - | Op::BranchI32Or { offset, .. } - | Op::BranchI32Nand { offset, .. } - | Op::BranchI32Nor { offset, .. } - | Op::BranchI32Eq { offset, .. } - | Op::BranchI32Ne { offset, .. } - | Op::BranchI32LtS { offset, .. } - | Op::BranchI32LtU { offset, .. } - | Op::BranchI32LeS { offset, .. } - | Op::BranchI32LeU { offset, .. } - | Op::BranchI64And { offset, .. } - | Op::BranchI64Or { offset, .. } - | Op::BranchI64Nand { offset, .. } - | Op::BranchI64Nor { offset, .. } - | Op::BranchI64Eq { offset, .. } - | Op::BranchI64Ne { offset, .. } - | Op::BranchI64LtS { offset, .. } - | Op::BranchI64LtU { offset, .. } - | Op::BranchI64LeS { offset, .. } - | Op::BranchI64LeU { offset, .. } - | Op::BranchF32Eq { offset, .. } - | Op::BranchF32Ne { offset, .. } - | Op::BranchF32Lt { offset, .. } - | Op::BranchF32Le { offset, .. } - | Op::BranchF32NotLt { offset, .. } - | Op::BranchF32NotLe { offset, .. } - | Op::BranchF64Eq { offset, .. } - | Op::BranchF64Ne { offset, .. } - | Op::BranchF64Lt { offset, .. } - | Op::BranchF64Le { offset, .. } - | Op::BranchF64NotLt { offset, .. } - | Op::BranchF64NotLe { offset, .. } - | Op::BranchI32AndImm16 { offset, .. } - | Op::BranchI32OrImm16 { offset, .. } - | Op::BranchI32NandImm16 { offset, .. } - | Op::BranchI32NorImm16 { offset, .. } - | Op::BranchI32EqImm16 { offset, .. } - | Op::BranchI32NeImm16 { offset, .. } - | Op::BranchI32LtSImm16Lhs { offset, .. } - | Op::BranchI32LtSImm16Rhs { offset, .. } - | Op::BranchI32LeSImm16Lhs { offset, .. } - | Op::BranchI32LeSImm16Rhs { offset, .. } - | Op::BranchI32LtUImm16Lhs { offset, .. } - | Op::BranchI32LtUImm16Rhs { offset, .. } - | Op::BranchI32LeUImm16Lhs { offset, .. } - | Op::BranchI32LeUImm16Rhs { offset, .. } - | Op::BranchI64AndImm16 { offset, .. } - | Op::BranchI64OrImm16 { offset, .. } - | Op::BranchI64NandImm16 { offset, .. } - | Op::BranchI64NorImm16 { offset, .. } - | Op::BranchI64EqImm16 { offset, .. } - | Op::BranchI64NeImm16 { offset, .. } - | Op::BranchI64LtSImm16Lhs { offset, .. } - | Op::BranchI64LtSImm16Rhs { offset, .. } - | Op::BranchI64LeSImm16Lhs { offset, .. } - | Op::BranchI64LeSImm16Rhs { offset, .. } - | Op::BranchI64LtUImm16Lhs { offset, .. } - | Op::BranchI64LtUImm16Rhs { offset, .. } - | Op::BranchI64LeUImm16Lhs { offset, .. } - | Op::BranchI64LeUImm16Rhs { offset, .. } => offset, + | Op::Branch { offset, .. } + // i32 + | Op::BranchI32Eq_Ss { offset, .. } + | Op::BranchI32Eq_Si { offset, .. } + | Op::BranchI32And_Ss { offset, .. } + | Op::BranchI32And_Si { offset, .. } + | Op::BranchI32Or_Ss { offset, .. } + | Op::BranchI32Or_Si { offset, .. } + | Op::BranchI32NotEq_Ss { offset, .. } + | Op::BranchI32NotEq_Si { offset, .. } + | Op::BranchI32NotAnd_Ss { offset, .. } + | Op::BranchI32NotAnd_Si { offset, .. } + | Op::BranchI32NotOr_Ss { offset, .. } + | Op::BranchI32NotOr_Si { offset, .. } + | Op::BranchI32Lt_Ss { offset, .. } + | Op::BranchI32Lt_Si { offset, .. } + | Op::BranchI32Lt_Is { offset, .. } + | Op::BranchU32Lt_Ss { offset, .. } + | Op::BranchU32Lt_Si { offset, .. } + | Op::BranchU32Lt_Is { offset, .. } + | Op::BranchI32Le_Ss { offset, .. } + | Op::BranchI32Le_Si { offset, .. } + | Op::BranchI32Le_Is { offset, .. } + | Op::BranchU32Le_Ss { offset, .. } + | Op::BranchU32Le_Si { offset, .. } + | Op::BranchU32Le_Is { offset, .. } + // i64 + | Op::BranchI64Eq_Ss { offset, .. } + | Op::BranchI64Eq_Si { offset, .. } + | Op::BranchI64And_Ss { offset, .. } + | Op::BranchI64And_Si { offset, .. } + | Op::BranchI64Or_Ss { offset, .. } + | Op::BranchI64Or_Si { offset, .. } + | Op::BranchI64NotEq_Ss { offset, .. } + | Op::BranchI64NotEq_Si { offset, .. } + | Op::BranchI64NotAnd_Ss { offset, .. } + | Op::BranchI64NotAnd_Si { offset, .. } + | Op::BranchI64NotOr_Ss { offset, .. } + | Op::BranchI64NotOr_Si { offset, .. } + | Op::BranchI64Lt_Ss { offset, .. } + | Op::BranchI64Lt_Si { offset, .. } + | Op::BranchI64Lt_Is { offset, .. } + | Op::BranchU64Lt_Ss { offset, .. } + | Op::BranchU64Lt_Si { offset, .. } + | Op::BranchU64Lt_Is { offset, .. } + | Op::BranchI64Le_Ss { offset, .. } + | Op::BranchI64Le_Si { offset, .. } + | Op::BranchI64Le_Is { offset, .. } + | Op::BranchU64Le_Ss { offset, .. } + | Op::BranchU64Le_Si { offset, .. } + | Op::BranchU64Le_Is { offset, .. } + // f32 + | Op::BranchF32Eq_Ss { offset, .. } + | Op::BranchF32Eq_Si { offset, .. } + | Op::BranchF32Lt_Ss { offset, .. } + | Op::BranchF32Lt_Si { offset, .. } + | Op::BranchF32Lt_Is { offset, .. } + | Op::BranchF32Le_Ss { offset, .. } + | Op::BranchF32Le_Si { offset, .. } + | Op::BranchF32Le_Is { offset, .. } + | Op::BranchF32NotEq_Ss { offset, .. } + | Op::BranchF32NotEq_Si { offset, .. } + | Op::BranchF32NotLt_Ss { offset, .. } + | Op::BranchF32NotLt_Si { offset, .. } + | Op::BranchF32NotLt_Is { offset, .. } + | Op::BranchF32NotLe_Ss { offset, .. } + | Op::BranchF32NotLe_Si { offset, .. } + | Op::BranchF32NotLe_Is { offset, .. } + // f64 + | Op::BranchF64Eq_Ss { offset, .. } + | Op::BranchF64Eq_Si { offset, .. } + | Op::BranchF64Lt_Ss { offset, .. } + | Op::BranchF64Lt_Si { offset, .. } + | Op::BranchF64Lt_Is { offset, .. } + | Op::BranchF64Le_Ss { offset, .. } + | Op::BranchF64Le_Si { offset, .. } + | Op::BranchF64Le_Is { offset, .. } + | Op::BranchF64NotEq_Ss { offset, .. } + | Op::BranchF64NotEq_Si { offset, .. } + | Op::BranchF64NotLt_Ss { offset, .. } + | Op::BranchF64NotLt_Si { offset, .. } + | Op::BranchF64NotLt_Is { offset, .. } + | Op::BranchF64NotLe_Ss { offset, .. } + | Op::BranchF64NotLe_Si { offset, .. } + | Op::BranchF64NotLe_Is { offset, .. } => offset, unexpected => { panic!("expected a Wasmi `cmp`+`branch` instruction but found: {unexpected:?}") } }; - if offset.init(new_offset).is_err() { - // Case: we need to convert `self` into its cmp+branch fallback instruction variant - // since adjusting the 16-bit offset failed. - let Some(fallback) = self.try_into_cmp_branch_fallback_instr(new_offset, stack)? else { - unreachable!("failed to create cmp+branch fallback instruction for: {self:?}"); - }; - *self = fallback; - } + offset.init(new_offset); Ok(()) } } From 86e1a34b13aad08f4f3607948ced423dae499d08 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:20:22 +0200 Subject: [PATCH 014/423] update StoreOperator impls and generalise --- crates/wasmi/src/engine/translator/func/op.rs | 250 ++++++++++-------- 1 file changed, 143 insertions(+), 107 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/op.rs b/crates/wasmi/src/engine/translator/func/op.rs index 0f7e0906e3..6f531b006d 100644 --- a/crates/wasmi/src/engine/translator/func/op.rs +++ b/crates/wasmi/src/engine/translator/func/op.rs @@ -4,160 +4,196 @@ use crate::{ }; /// Trait implemented by all Wasm operators that can be translated as wrapping store instructions. -pub trait StoreWrapOperator { +pub trait StoreOperator { /// The type of the value to the stored. type Value; - /// The type of the wrapped value. - type Wrapped; - /// The type of the value as (at most) 16-bit encoded instruction parameter. - type Param; - - fn store(ptr: Slot, offset_lo: Offset64Lo) -> Op; - fn store_imm(ptr: Slot, offset_lo: Offset64Lo) -> Op; - fn store_offset16(ptr: Slot, offset: Offset16, value: Slot) -> Op; - fn store_offset16_imm(ptr: Slot, offset: Offset16, value: Self::Param) -> Op; - fn store_at(value: Slot, address: Address32) -> Op; - fn store_at_imm(value: Self::Param, address: Address32) -> Op; + /// The type of immediate values. + type Immediate; + + /// Converts the value into the immediate value type. + /// + /// # Examples + /// + /// - Wrapping for wrapping stores. + /// - Conversion to bits type or identity for normal stores. + fn into_immediate(value: Self::Value) -> Self::Immediate; + + fn store_ss(ptr: Slot, offset: u64, value: Slot, memory: Memory) -> Op; + fn store_si(ptr: Slot, offset: u64, value: Self::Immediate, memory: Memory) -> Op; + fn store_is(address: Address, value: Slot, memory: Memory) -> Op; + fn store_ii(address: Address, value: Self::Value, memory: Memory) -> Op; + fn store_mem0_offset16_ss(ptr: Slot, offset: Offset16, value: Slot) -> Op; + fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Value) -> Op; } macro_rules! impl_store_wrap { ( $( - impl StoreWrapOperator for $name:ident { + impl StoreOperator for $name:ident { type Value = $value_ty:ty; - type Wrapped = $wrapped_ty:ty; - type Param = $param_ty:ty; - - fn store = $store:expr; - fn store_imm = $store_imm:expr; - fn store_offset16 = $store_offset16:expr; - fn store_offset16_imm = $store_offset16_imm:expr; - fn store_at = $store_at:expr; - fn store_at_imm = $store_at_imm:expr; + type Immediate = $wrapped_ty:ty; + + fn into_immediate = $apply:expr; + + fn store_ss = $store_ss:expr; + fn store_si = $store_si:expr; + fn store_is = $store_is:expr; + fn store_ii = $store_ii:expr; + fn store_mem0_offset16_ss = $store_mem0_offset16_ss:expr; + fn store_mem0_offset16_si = $store_mem0_offset16_si:expr; } )* ) => { $( pub enum $name {} - impl StoreWrapOperator for $name { + impl StoreOperator for $name { type Value = $value_ty; - type Wrapped = $wrapped_ty; - type Param = $param_ty; + type Immediate = $wrapped_ty; + + fn into_immediate(value: Self::Value) -> Self::Immediate { + $apply(value) + } - fn store(ptr: Slot, offset_lo: Offset64Lo) -> Op { - $store(ptr, offset_lo) + fn store_ss(ptr: Slot, offset: u64, value: Slot, memory: Memory) -> Op { + $store_ss(ptr, offset, value, memory) } - fn store_imm(ptr: Slot, offset_lo: Offset64Lo) -> Op { - $store_imm(ptr, offset_lo) + fn store_si(ptr: Slot, offset: u64, value: Self::Immediate, memory: Memory) -> Op { + $store_si(ptr, offset, value, memory) } - fn store_offset16(ptr: Slot, offset: Offset16, value: Slot) -> Op { - $store_offset16(ptr, offset, value) + fn store_is(address: Address, value: Slot, memory: Memory) -> Op { + $store_is(address, value, memory) } - fn store_offset16_imm(ptr: Slot, offset: Offset16, value: Self::Param) -> Op { - $store_offset16_imm(ptr, offset, value) + fn store_ii(address: Address, value: Self::Value, memory: Memory) -> Op { + $store_ii(address, value, memory) } - fn store_at(value: Slot, address: Address32) -> Op { - $store_at(value, address) + fn store_mem0_offset16_ss(ptr: Slot, offset: Offset16, value: Slot) -> Op { + $store_mem0_offset16_ss(ptr, offset, value) } - fn store_at_imm(value: Self::Param, address: Address32) -> Op { - $store_at_imm(value, address) + fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Value) -> Op { + $store_mem0_offset16_si(ptr, offset, value) } } )* }; } impl_store_wrap! { - impl StoreWrapOperator for I32Store { + impl StoreOperator for I32Store { type Value = i32; - type Wrapped = i32; - type Param = i16; - - fn store = Op::store32; - fn store_imm = Op::i32_store_imm16; - fn store_offset16 = Op::store32_offset16; - fn store_offset16_imm = Op::i32_store_offset16_imm16; - fn store_at = Op::store32_at; - fn store_at_imm = Op::i32_store_at_imm16; + type Immediate = u32; + + fn into_immediate = ::to_bits; + fn store_ss = Op::store32_ss; + fn store_si = Op::store32_si; + fn store_is = Op::store32_is; + fn store_ii = Op::store32_ii; + fn store_mem0_offset16_ss = Op::store32_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store32_mem0_offset16_si; } - impl StoreWrapOperator for I64Store { + impl StoreOperator for I64Store { type Value = i64; - type Wrapped = i64; - type Param = i16; - - fn store = Op::store64; - fn store_imm = Op::i64_store_imm16; - fn store_offset16 = Op::store64_offset16; - fn store_offset16_imm = Op::i64_store_offset16_imm16; - fn store_at = Op::store64_at; - fn store_at_imm = Op::i64_store_at_imm16; + type Immediate = i64; + + fn into_immediate = ::to_bits; + fn store_ss = Op::store64_ss; + fn store_si = Op::store64_si; + fn store_is = Op::store64_is; + fn store_ii = Op::store64_ii; + fn store_mem0_offset16_ss = Op::store64_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store64_mem0_offset16_si; + } + + impl StoreOperator for F32Store { + type Value = f32; + type Immediate = u32; + + fn into_immediate = ::to_bits; + fn store_ss = Op::store32_ss; + fn store_si = Op::store32_si; + fn store_is = Op::store32_is; + fn store_ii = Op::store32_ii; + fn store_mem0_offset16_ss = Op::store32_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store32_mem0_offset16_si; + } + + impl StoreOperator for F64Store { + type Value = f64; + type Immediate = u64; + + fn into_immediate = ::to_bits; + fn store_ss = Op::store64_ss; + fn store_si = Op::store64_si; + fn store_is = Op::store64_is; + fn store_ii = Op::store64_ii; + fn store_mem0_offset16_ss = Op::store64_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::store64_mem0_offset16_si; } - impl StoreWrapOperator for I32Store8 { + impl StoreOperator for I32Store8 { type Value = i32; - type Wrapped = i8; - type Param = i8; - - fn store = Op::i32_store8; - fn store_imm = Op::i32_store8_imm; - fn store_offset16 = Op::i32_store8_offset16; - fn store_offset16_imm = Op::i32_store8_offset16_imm; - fn store_at = Op::i32_store8_at; - fn store_at_imm = Op::i32_store8_at_imm; + type Immediate = i8; + + fn into_immediate = >::wrap; + fn store_ss = Op::i32_store8_ss; + fn store_si = Op::i32_store8_si; + fn store_is = Op::i32_store8_is; + fn store_ii = Op::i32_store8_ii; + fn store_mem0_offset16_ss = Op::i32_store8_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i32_store8_mem0_offset16_si; } - impl StoreWrapOperator for I32Store16 { + impl StoreOperator for I32Store16 { type Value = i32; - type Wrapped = i16; - type Param = i16; - - fn store = Op::i32_store16; - fn store_imm = Op::i32_store16_imm; - fn store_offset16 = Op::i32_store16_offset16; - fn store_offset16_imm = Op::i32_store16_offset16_imm; - fn store_at = Op::i32_store16_at; - fn store_at_imm = Op::i32_store16_at_imm; + type Immediate = i16; + + fn into_immediate = >::wrap; + fn store_ss = Op::i32_store16_ss; + fn store_si = Op::i32_store16_si; + fn store_is = Op::i32_store16_is; + fn store_ii = Op::i32_store16_ii; + fn store_mem0_offset16_ss = Op::i32_store16_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i32_store16_mem0_offset16_si; } - impl StoreWrapOperator for I64Store8 { + impl StoreOperator for I64Store8 { type Value = i64; - type Wrapped = i8; - type Param = i8; - - fn store = Op::i64_store8; - fn store_imm = Op::i64_store8_imm; - fn store_offset16 = Op::i64_store8_offset16; - fn store_offset16_imm = Op::i64_store8_offset16_imm; - fn store_at = Op::i64_store8_at; - fn store_at_imm = Op::i64_store8_at_imm; + type Immediate = i8; + + fn into_immediate = >::wrap; + fn store_ss = Op::i64_store8_ss; + fn store_si = Op::i64_store8_si; + fn store_is = Op::i64_store8_is; + fn store_ii = Op::i64_store8_ii; + fn store_mem0_offset16_ss = Op::i64_store8_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i64_store8_mem0_offset16_si; } - impl StoreWrapOperator for I64Store16 { + impl StoreOperator for I64Store16 { type Value = i64; - type Wrapped = i16; - type Param = i16; - - fn store = Op::i64_store16; - fn store_imm = Op::i64_store16_imm; - fn store_offset16 = Op::i64_store16_offset16; - fn store_offset16_imm = Op::i64_store16_offset16_imm; - fn store_at = Op::i64_store16_at; - fn store_at_imm = Op::i64_store16_at_imm; + type Immediate = i16; + + fn into_immediate = >::wrap; + fn store_ss = Op::i64_store16_ss; + fn store_si = Op::i64_store16_si; + fn store_is = Op::i64_store16_is; + fn store_ii = Op::i64_store16_ii; + fn store_mem0_offset16_ss = Op::i64_store16_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i64_store16_mem0_offset16_si; } - impl StoreWrapOperator for I64Store32 { + impl StoreOperator for I64Store32 { type Value = i64; - type Wrapped = i32; - type Param = i16; - - fn store = Op::i64_store32; - fn store_imm = Op::i64_store32_imm16; - fn store_offset16 = Op::i64_store32_offset16; - fn store_offset16_imm = Op::i64_store32_offset16_imm16; - fn store_at = Op::i64_store32_at; - fn store_at_imm = Op::i64_store32_at_imm16; + type Immediate = i32; + + fn into_immediate = >::wrap; + fn store_ss = Op::i64_store32_ss; + fn store_si = Op::i64_store32_si; + fn store_is = Op::i64_store32_is; + fn store_ii = Op::i64_store32_ii; + fn store_mem0_offset16_ss = Op::i64_store32_mem0_offset16_ss; + fn store_mem0_offset16_si = Op::i64_store32_mem0_offset16_si; } } From da0b3d214028f1161b85b2cec3452640ba8381d2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:20:44 +0200 Subject: [PATCH 015/423] update most FuncTranslator visit methods --- .../wasmi/src/engine/translator/func/visit.rs | 802 ++++++++++-------- 1 file changed, 456 insertions(+), 346 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 60204910d4..b0a06e9edd 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -11,7 +11,7 @@ use crate::{ }, BlockType, }, - ir::{self, Const16, Op}, + ir::{self, index, Op}, module::{self, FuncIdx, MemoryIdx, TableIdx, WasmiValueType}, Error, ExternRef, @@ -471,7 +471,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_global_set(&mut self, global_index: u32) -> Self::Output { bail_unreachable!(self); let global = ir::index::Global::from(global_index); - let input = match self.stack.pop() { + let input = self.stack.pop(); + let value = match input { Operand::Immediate(input) => input.val(), input => { // Case: `global.set` with simple register input. @@ -484,33 +485,21 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (global_type, _init_value) = self .module .get_global(module::GlobalIdx::from(global_index)); - debug_assert_eq!(global_type.content(), input.ty()); - match global_type.content() { - ValType::I32 => { - if let Ok(value) = Const16::try_from(i32::from(input)) { - // Case: `global.set` with 16-bit encoded `i32` value. - self.push_instr( - Op::global_set_i32imm16(value, global), - FuelCostsProvider::instance, - )?; - return Ok(()); - } - } - ValType::I64 => { - if let Ok(value) = Const16::try_from(i64::from(input)) { - // Case: `global.set` with 16-bit encoded `i64` value. - self.push_instr( - Op::global_set_i64imm16(value, global), - FuelCostsProvider::instance, - )?; - return Ok(()); - } + debug_assert_eq!(global_type.content(), value.ty()); + let global_set_instr = match global_type.content() { + ValType::I32 => Op::global_set32(global, u32::from(value)), + ValType::I64 => Op::global_set64(global, u64::from(value)), + ValType::F32 => Op::global_set32(global, f32::from(value).to_bits()), + ValType::F64 => Op::global_set64(global, f64::from(value).to_bits()), + ValType::FuncRef | ValType::ExternRef => Op::global_set64(global, u64::from(value)), + ValType::V128 => { + let consume_fuel = self.stack.consume_fuel_instr(); + let temp = self.copy_operand_to_temp(input, consume_fuel)?; + Op::global_set(global, temp) } - _ => {} }; // Note: at this point we have to allocate a function local constant. - let cref = self.layout.const_to_reg(input)?; - self.push_instr(Op::global_set(cref, global), FuelCostsProvider::instance)?; + self.push_instr(global_set_instr, FuelCostsProvider::instance)?; Ok(()) } @@ -519,9 +508,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I32, - Op::load32, - Op::load32_offset16, - Op::load32_at, + Op::load32_ss, + Op::load32_si, + Op::load32_mem0_offset16_ss, ) } @@ -530,9 +519,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I64, - Op::load64, - Op::load64_offset16, - Op::load64_at, + Op::load64_ss, + Op::load64_si, + Op::load64_mem0_offset16_ss, ) } @@ -541,9 +530,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::F32, - Op::load32, - Op::load32_offset16, - Op::load32_at, + Op::load32_ss, + Op::load32_si, + Op::load32_mem0_offset16_ss, ) } @@ -552,9 +541,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::F64, - Op::load64, - Op::load64_offset16, - Op::load64_at, + Op::load64_ss, + Op::load64_si, + Op::load64_mem0_offset16_ss, ) } @@ -563,9 +552,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I32, - Op::i32_load8_s, - Op::i32_load8_s_offset16, - Op::i32_load8_s_at, + Op::i32_load8_ss, + Op::i32_load8_si, + Op::i32_load8_mem0_offset16_ss, ) } @@ -574,9 +563,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I32, - Op::i32_load8_u, - Op::i32_load8_u_offset16, - Op::i32_load8_u_at, + Op::u32_load8_ss, + Op::u32_load8_si, + Op::u32_load8_mem0_offset16_ss, ) } @@ -585,9 +574,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I32, - Op::i32_load16_s, - Op::i32_load16_s_offset16, - Op::i32_load16_s_at, + Op::i32_load16_ss, + Op::i32_load16_si, + Op::i32_load16_mem0_offset16_ss, ) } @@ -596,9 +585,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I32, - Op::i32_load16_u, - Op::i32_load16_u_offset16, - Op::i32_load16_u_at, + Op::u32_load16_ss, + Op::u32_load16_si, + Op::u32_load16_mem0_offset16_ss, ) } @@ -607,9 +596,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I64, - Op::i64_load8_s, - Op::i64_load8_s_offset16, - Op::i64_load8_s_at, + Op::i64_load8_ss, + Op::i64_load8_si, + Op::i64_load8_mem0_offset16_ss, ) } @@ -618,9 +607,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I64, - Op::i64_load8_u, - Op::i64_load8_u_offset16, - Op::i64_load8_u_at, + Op::u64_load8_ss, + Op::u64_load8_si, + Op::u64_load8_mem0_offset16_ss, ) } @@ -629,9 +618,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I64, - Op::i64_load16_s, - Op::i64_load16_s_offset16, - Op::i64_load16_s_at, + Op::i64_load16_ss, + Op::i64_load16_si, + Op::i64_load16_mem0_offset16_ss, ) } @@ -640,9 +629,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I64, - Op::i64_load16_u, - Op::i64_load16_u_offset16, - Op::i64_load16_u_at, + Op::u64_load16_ss, + Op::u64_load16_si, + Op::u64_load16_mem0_offset16_ss, ) } @@ -651,9 +640,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I64, - Op::i64_load32_s, - Op::i64_load32_s_offset16, - Op::i64_load32_s_at, + Op::i64_load32_ss, + Op::i64_load32_si, + Op::i64_load32_mem0_offset16_ss, ) } @@ -662,55 +651,55 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.translate_load( memarg, ValType::I64, - Op::i64_load32_u, - Op::i64_load32_u_offset16, - Op::i64_load32_u_at, + Op::u64_load32_ss, + Op::u64_load32_si, + Op::u64_load32_mem0_offset16_ss, ) } #[inline(never)] fn visit_i32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_f32_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_store(memarg, Op::store32, Op::store32_offset16, Op::store32_at) + self.translate_store::(memarg) } #[inline(never)] fn visit_f64_store(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_store(memarg, Op::store64, Op::store64_offset16, Op::store64_at) + self.translate_store::(memarg) } #[inline(never)] fn visit_i32_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i32_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store8(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store16(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] fn visit_i64_store32(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_istore_wrap::(memarg) + self.translate_store::(memarg) } #[inline(never)] @@ -721,9 +710,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { .get_type_of_memory(MemoryIdx::from(mem)) .index_ty() .ty(); + let memory = index::Memory::from(mem); self.push_instr_with_result( index_ty, - |result| Op::memory_size(result, mem), + |result| Op::memory_size(result, memory), FuelCostsProvider::instance, )?; Ok(()) @@ -736,6 +726,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { .module .get_type_of_memory(MemoryIdx::from(mem)) .index_ty(); + let memory = index::Memory::from(mem); let delta = self.stack.pop(); if let Operand::Immediate(delta) = delta { let delta = delta.val(); @@ -751,7 +742,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { // as `memory.size` instruction instead. self.push_instr_with_result( index_ty.ty(), - |result| Op::memory_size(result, mem), + |result| Op::memory_size(result, memory), FuelCostsProvider::instance, )?; return Ok(()); @@ -761,10 +752,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let delta = self.immediate_to_reg(delta)?; self.push_instr_with_result( index_ty.ty(), - |result| Op::memory_grow(result, delta), + |result| Op::memory_grow(result, delta, memory), FuelCostsProvider::instance, )?; - self.push_param(Op::memory_index(mem))?; Ok(()) } @@ -808,8 +798,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_eq(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_eq, - Op::i32_eq_imm16, + Op::i32_eq_sss, + Op::i32_eq_ssi, wasm::i32_eq, FuncTranslator::fuse_eqz, ) @@ -818,8 +808,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_ne(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_ne, - Op::i32_ne_imm16, + Op::i32_not_eq_sss, + Op::i32_not_eq_ssi, wasm::i32_ne, FuncTranslator::fuse_nez, ) @@ -828,9 +818,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_lt_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_lt_s, - Op::i32_lt_s_imm16_rhs, - Op::i32_lt_s_imm16_lhs, + Op::i32_lt_sss, + Op::i32_lt_ssi, + Op::i32_lt_sis, wasm::i32_lt_s, ) } @@ -838,9 +828,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_lt_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_lt_u, - Op::i32_lt_u_imm16_rhs, - Op::i32_lt_u_imm16_lhs, + Op::u32_lt_sss, + Op::u32_lt_ssi, + Op::u32_lt_sis, wasm::i32_lt_u, ) } @@ -848,9 +838,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_gt_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_lt_s), - swap_ops!(Op::i32_lt_s_imm16_lhs), - swap_ops!(Op::i32_lt_s_imm16_rhs), + swap_ops!(Op::i32_lt_sss), + swap_ops!(Op::i32_lt_sis), + swap_ops!(Op::i32_lt_ssi), wasm::i32_gt_s, ) } @@ -858,9 +848,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_gt_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_lt_u), - swap_ops!(Op::i32_lt_u_imm16_lhs), - swap_ops!(Op::i32_lt_u_imm16_rhs), + swap_ops!(Op::u32_lt_sss), + swap_ops!(Op::u32_lt_sis), + swap_ops!(Op::u32_lt_ssi), wasm::i32_gt_u, ) } @@ -868,9 +858,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_le_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_le_s, - Op::i32_le_s_imm16_rhs, - Op::i32_le_s_imm16_lhs, + Op::i32_le_sss, + Op::i32_le_ssi, + Op::i32_le_sis, wasm::i32_le_s, ) } @@ -878,9 +868,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_le_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i32_le_u, - Op::i32_le_u_imm16_rhs, - Op::i32_le_u_imm16_lhs, + Op::u32_le_sss, + Op::u32_le_ssi, + Op::u32_le_sis, wasm::i32_le_u, ) } @@ -888,9 +878,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_ge_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_le_s), - swap_ops!(Op::i32_le_s_imm16_lhs), - swap_ops!(Op::i32_le_s_imm16_rhs), + swap_ops!(Op::i32_le_sss), + swap_ops!(Op::i32_le_sis), + swap_ops!(Op::i32_le_ssi), wasm::i32_ge_s, ) } @@ -898,9 +888,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_ge_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i32_le_u), - swap_ops!(Op::i32_le_u_imm16_lhs), - swap_ops!(Op::i32_le_u_imm16_rhs), + swap_ops!(Op::u32_le_sss), + swap_ops!(Op::u32_le_sis), + swap_ops!(Op::u32_le_ssi), wasm::i32_ge_u, ) } @@ -915,8 +905,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_eq(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_eq, - Op::i64_eq_imm16, + Op::i64_eq_sss, + Op::i64_eq_ssi, wasm::i64_eq, FuncTranslator::fuse_eqz, ) @@ -925,8 +915,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_ne(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_ne, - Op::i64_ne_imm16, + Op::i64_not_eq_sss, + Op::i64_not_eq_ssi, wasm::i64_ne, FuncTranslator::fuse_nez, ) @@ -935,9 +925,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_lt_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_lt_s, - Op::i64_lt_s_imm16_rhs, - Op::i64_lt_s_imm16_lhs, + Op::i64_lt_sss, + Op::i64_lt_ssi, + Op::i64_lt_sis, wasm::i64_lt_s, ) } @@ -945,9 +935,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_lt_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_lt_u, - Op::i64_lt_u_imm16_rhs, - Op::i64_lt_u_imm16_lhs, + Op::u64_lt_sss, + Op::u64_lt_ssi, + Op::u64_lt_sis, wasm::i64_lt_u, ) } @@ -955,9 +945,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_gt_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_lt_s), - swap_ops!(Op::i64_lt_s_imm16_lhs), - swap_ops!(Op::i64_lt_s_imm16_rhs), + swap_ops!(Op::i64_lt_sss), + swap_ops!(Op::i64_lt_sis), + swap_ops!(Op::i64_lt_ssi), wasm::i64_gt_s, ) } @@ -965,9 +955,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_gt_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_lt_u), - swap_ops!(Op::i64_lt_u_imm16_lhs), - swap_ops!(Op::i64_lt_u_imm16_rhs), + swap_ops!(Op::u64_lt_sss), + swap_ops!(Op::u64_lt_sis), + swap_ops!(Op::u64_lt_ssi), wasm::i64_gt_u, ) } @@ -975,9 +965,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_le_s(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_le_s, - Op::i64_le_s_imm16_rhs, - Op::i64_le_s_imm16_lhs, + Op::i64_le_sss, + Op::i64_le_ssi, + Op::i64_le_sis, wasm::i64_le_s, ) } @@ -985,9 +975,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_le_u(&mut self) -> Self::Output { self.translate_binary::( - Op::i64_le_u, - Op::i64_le_u_imm16_rhs, - Op::i64_le_u_imm16_lhs, + Op::u64_le_sss, + Op::u64_le_ssi, + Op::u64_le_sis, wasm::i64_le_u, ) } @@ -995,9 +985,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_ge_s(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_le_s), - swap_ops!(Op::i64_le_s_imm16_lhs), - swap_ops!(Op::i64_le_s_imm16_rhs), + swap_ops!(Op::i64_le_sss), + swap_ops!(Op::i64_le_sis), + swap_ops!(Op::i64_le_ssi), wasm::i64_ge_s, ) } @@ -1005,93 +995,133 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_ge_u(&mut self) -> Self::Output { self.translate_binary::( - swap_ops!(Op::i64_le_u), - swap_ops!(Op::i64_le_u_imm16_lhs), - swap_ops!(Op::i64_le_u_imm16_rhs), + swap_ops!(Op::u64_le_sss), + swap_ops!(Op::u64_le_sis), + swap_ops!(Op::u64_le_ssi), wasm::i64_ge_u, ) } #[inline(never)] fn visit_f32_eq(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_eq, wasm::f32_eq) + self.translate_binary_commutative( + Op::f32_eq_sss, + Op::f32_eq_ssi, + wasm::f32_eq, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f32_ne(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_ne, wasm::f32_ne) + self.translate_binary_commutative( + Op::f32_not_eq_sss, + Op::f32_not_eq_ssi, + wasm::f32_ne, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f32_lt(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_lt, wasm::f32_lt) + self.translate_binary(Op::f32_lt_sss, Op::f32_lt_ssi, Op::f32_lt_sis, wasm::f32_lt) } #[inline(never)] fn visit_f32_gt(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f32_lt), wasm::f32_gt) + self.translate_binary( + swap_ops!(Op::f32_lt_sss), + swap_ops!(Op::f32_lt_sis), + swap_ops!(Op::f32_lt_ssi), + wasm::f32_gt, + ) } #[inline(never)] fn visit_f32_le(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_le, wasm::f32_le) + self.translate_binary(Op::f32_le_sss, Op::f32_le_ssi, Op::f32_le_sis, wasm::f32_le) } #[inline(never)] fn visit_f32_ge(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f32_le), wasm::f32_ge) + self.translate_binary( + swap_ops!(Op::f32_le_sss), + swap_ops!(Op::f32_le_sis), + swap_ops!(Op::f32_le_ssi), + wasm::f32_ge, + ) } #[inline(never)] fn visit_f64_eq(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_eq, wasm::f64_eq) + self.translate_binary_commutative( + Op::f64_eq_sss, + Op::f64_eq_ssi, + wasm::f64_eq, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f64_ne(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_ne, wasm::f64_ne) + self.translate_binary_commutative( + Op::f64_not_eq_sss, + Op::f64_not_eq_ssi, + wasm::f64_ne, + Self::no_opt_ri, + ) } #[inline(never)] fn visit_f64_lt(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_lt, wasm::f64_lt) + self.translate_binary(Op::f64_lt_sss, Op::f64_lt_ssi, Op::f64_lt_sis, wasm::f64_lt) } #[inline(never)] fn visit_f64_gt(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f64_lt), wasm::f64_gt) + self.translate_binary( + swap_ops!(Op::f64_lt_sss), + swap_ops!(Op::f64_lt_sis), + swap_ops!(Op::f64_lt_ssi), + wasm::f64_gt, + ) } #[inline(never)] fn visit_f64_le(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_le, wasm::f64_le) + self.translate_binary(Op::f64_le_sss, Op::f64_le_ssi, Op::f64_le_sis, wasm::f64_le) } #[inline(never)] fn visit_f64_ge(&mut self) -> Self::Output { - self.translate_fbinary(swap_ops!(Op::f64_le), wasm::f64_ge) + self.translate_binary( + swap_ops!(Op::f64_le_sss), + swap_ops!(Op::f64_le_sis), + swap_ops!(Op::f64_le_ssi), + wasm::f64_ge, + ) } #[inline(never)] fn visit_i32_clz(&mut self) -> Self::Output { - self.translate_unary::(Op::i32_clz, wasm::i32_clz) + self.translate_unary::(Op::i32_clz_ss, wasm::i32_clz) } #[inline(never)] fn visit_i32_ctz(&mut self) -> Self::Output { - self.translate_unary::(Op::i32_ctz, wasm::i32_ctz) + self.translate_unary::(Op::i32_ctz_ss, wasm::i32_ctz) } #[inline(never)] fn visit_i32_popcnt(&mut self) -> Self::Output { - self.translate_unary::(Op::i32_popcnt, wasm::i32_popcnt) + self.translate_unary::(Op::i32_popcnt_ss, wasm::i32_popcnt) } #[inline(never)] fn visit_i32_add(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_add, - Op::i32_add_imm16, + Op::i32_add_sss, + Op::i32_add_ssi, wasm::i32_add, FuncTranslator::no_opt_ri, ) @@ -1099,10 +1129,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_sub(&mut self) -> Self::Output { - self.translate_isub( - Op::i32_sub, - Op::i32_add_imm16, - Op::i32_sub_imm16_lhs, + self.translate_binary( + Op::i32_sub_sss, + Op::i32_sub_ssi, + Op::i32_sub_sis, wasm::i32_sub, ) } @@ -1110,8 +1140,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_mul(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_mul, - Op::i32_mul_imm16, + Op::i32_mul_sss, + Op::i32_mul_ssi, wasm::i32_mul, FuncTranslator::no_opt_ri, ) @@ -1120,9 +1150,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_div_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_div_s, - Op::i32_div_s_imm16_rhs, - Op::i32_div_s_imm16_lhs, + Op::i32_div_sss, + Op::i32_div_ssi, + Op::i32_div_sis, wasm::i32_div_s, ) } @@ -1130,9 +1160,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_div_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_div_u, - Op::i32_div_u_imm16_rhs, - Op::i32_div_u_imm16_lhs, + Op::u32_div_sss, + Op::u32_div_ssi, + Op::u32_div_sis, wasm::i32_div_u, ) } @@ -1140,9 +1170,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rem_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_rem_s, - Op::i32_rem_s_imm16_rhs, - Op::i32_rem_s_imm16_lhs, + Op::i32_rem_sss, + Op::i32_rem_ssi, + Op::i32_rem_sis, wasm::i32_rem_s, ) } @@ -1150,9 +1180,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rem_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i32_rem_u, - Op::i32_rem_u_imm16_rhs, - Op::i32_rem_u_imm16_lhs, + Op::u32_rem_sss, + Op::u32_rem_ssi, + Op::u32_rem_sis, wasm::i32_rem_u, ) } @@ -1160,8 +1190,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bitand, - Op::i32_bitand_imm16, + Op::i32_bit_and_sss, + Op::i32_bit_and_ssi, wasm::i32_bitand, FuncTranslator::no_opt_ri, ) @@ -1170,8 +1200,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bitor, - Op::i32_bitor_imm16, + Op::i32_bit_or_sss, + Op::i32_bit_or_ssi, wasm::i32_bitor, FuncTranslator::no_opt_ri, ) @@ -1180,8 +1210,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bitxor, - Op::i32_bitxor_imm16, + Op::i32_bit_xor_sss, + Op::i32_bit_xor_ssi, wasm::i32_bitxor, FuncTranslator::no_opt_ri, ) @@ -1190,9 +1220,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_shl(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_shl, - Op::i32_shl_by, - Op::i32_shl_imm16, + Op::i32_shl_sss, + Op::i32_shl_ssi, + Op::i32_shl_sis, wasm::i32_shl, ) } @@ -1200,9 +1230,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_shr_s(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_shr_s, - Op::i32_shr_s_by, - Op::i32_shr_s_imm16, + Op::i32_shr_sss, + Op::i32_shr_ssi, + Op::i32_shr_sis, wasm::i32_shr_s, ) } @@ -1210,9 +1240,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_shr_u(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_shr_u, - Op::i32_shr_u_by, - Op::i32_shr_u_imm16, + Op::u32_shr_sss, + Op::u32_shr_ssi, + Op::u32_shr_sis, wasm::i32_shr_u, ) } @@ -1220,9 +1250,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rotl(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_rotl, - Op::i32_rotl_by, - Op::i32_rotl_imm16, + Op::i32_rotl_sss, + Op::i32_rotl_ssi, + Op::i32_rotl_sis, wasm::i32_rotl, ) } @@ -1230,33 +1260,33 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_rotr(&mut self) -> Self::Output { self.translate_shift::( - Op::i32_rotr, - Op::i32_rotr_by, - Op::i32_rotr_imm16, + Op::i32_rotr_sss, + Op::i32_rotr_ssi, + Op::i32_rotr_sis, wasm::i32_rotr, ) } #[inline(never)] fn visit_i64_clz(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_clz, wasm::i64_clz) + self.translate_unary::(Op::i64_clz_ss, wasm::i64_clz) } #[inline(never)] fn visit_i64_ctz(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_ctz, wasm::i64_ctz) + self.translate_unary::(Op::i64_ctz_ss, wasm::i64_ctz) } #[inline(never)] fn visit_i64_popcnt(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_popcnt, wasm::i64_popcnt) + self.translate_unary::(Op::i64_popcnt_ss, wasm::i64_popcnt) } #[inline(never)] fn visit_i64_add(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_add, - Op::i64_add_imm16, + Op::i64_add_sss, + Op::i64_add_ssi, wasm::i64_add, FuncTranslator::no_opt_ri, ) @@ -1264,10 +1294,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_sub(&mut self) -> Self::Output { - self.translate_isub( - Op::i64_sub, - Op::i64_add_imm16, - Op::i64_sub_imm16_lhs, + self.translate_binary( + Op::i64_sub_sss, + Op::i64_sub_ssi, + Op::i64_sub_sis, wasm::i64_sub, ) } @@ -1275,8 +1305,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_mul(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_mul, - Op::i64_mul_imm16, + Op::i64_mul_sss, + Op::i64_mul_ssi, wasm::i64_mul, FuncTranslator::no_opt_ri, ) @@ -1285,9 +1315,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_div_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_div_s, - Op::i64_div_s_imm16_rhs, - Op::i64_div_s_imm16_lhs, + Op::i64_div_sss, + Op::i64_div_ssi, + Op::i64_div_sis, wasm::i64_div_s, ) } @@ -1295,9 +1325,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_div_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_div_u, - Op::i64_div_u_imm16_rhs, - Op::i64_div_u_imm16_lhs, + Op::u64_div_sss, + Op::u64_div_ssi, + Op::u64_div_sis, wasm::i64_div_u, ) } @@ -1305,9 +1335,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rem_s(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_rem_s, - Op::i64_rem_s_imm16_rhs, - Op::i64_rem_s_imm16_lhs, + Op::i64_rem_sss, + Op::i64_rem_ssi, + Op::i64_rem_sis, wasm::i64_rem_s, ) } @@ -1315,9 +1345,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rem_u(&mut self) -> Self::Output { self.translate_divrem::( - Op::i64_rem_u, - Op::i64_rem_u_imm16_rhs, - Op::i64_rem_u_imm16_lhs, + Op::u64_rem_sss, + Op::u64_rem_ssi, + Op::u64_rem_sis, wasm::i64_rem_u, ) } @@ -1325,8 +1355,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bitand, - Op::i64_bitand_imm16, + Op::i64_bit_and_sss, + Op::i64_bit_and_ssi, wasm::i64_bitand, FuncTranslator::no_opt_ri, ) @@ -1335,8 +1365,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bitor, - Op::i64_bitor_imm16, + Op::i64_bit_or_sss, + Op::i64_bit_or_ssi, wasm::i64_bitor, FuncTranslator::no_opt_ri, ) @@ -1345,8 +1375,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bitxor, - Op::i64_bitxor_imm16, + Op::i64_bit_xor_sss, + Op::i64_bit_xor_ssi, wasm::i64_bitxor, FuncTranslator::no_opt_ri, ) @@ -1355,9 +1385,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_shl(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_shl, - Op::i64_shl_by, - Op::i64_shl_imm16, + Op::i64_shl_sss, + Op::i64_shl_ssi, + Op::i64_shl_sis, wasm::i64_shl, ) } @@ -1365,9 +1395,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_shr_s(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_shr_s, - Op::i64_shr_s_by, - Op::i64_shr_s_imm16, + Op::i64_shr_sss, + Op::i64_shr_ssi, + Op::i64_shr_sis, wasm::i64_shr_s, ) } @@ -1375,9 +1405,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_shr_u(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_shr_u, - Op::i64_shr_u_by, - Op::i64_shr_u_imm16, + Op::u64_shr_sss, + Op::u64_shr_ssi, + Op::u64_shr_sis, wasm::i64_shr_u, ) } @@ -1385,9 +1415,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rotl(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_rotl, - Op::i64_rotl_by, - Op::i64_rotl_imm16, + Op::i64_rotl_sss, + Op::i64_rotl_ssi, + Op::i64_rotl_sis, wasm::i64_rotl, ) } @@ -1395,181 +1425,251 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_rotr(&mut self) -> Self::Output { self.translate_shift::( - Op::i64_rotr, - Op::i64_rotr_by, - Op::i64_rotr_imm16, + Op::i64_rotr_sss, + Op::i64_rotr_ssi, + Op::i64_rotr_sis, wasm::i64_rotr, ) } #[inline(never)] fn visit_f32_abs(&mut self) -> Self::Output { - self.translate_unary(Op::f32_abs, wasm::f32_abs) + self.translate_unary(Op::f32_abs_ss, wasm::f32_abs) } #[inline(never)] fn visit_f32_neg(&mut self) -> Self::Output { - self.translate_unary(Op::f32_neg, wasm::f32_neg) + self.translate_unary(Op::f32_neg_ss, wasm::f32_neg) } #[inline(never)] fn visit_f32_ceil(&mut self) -> Self::Output { - self.translate_unary(Op::f32_ceil, wasm::f32_ceil) + self.translate_unary(Op::f32_ceil_ss, wasm::f32_ceil) } #[inline(never)] fn visit_f32_floor(&mut self) -> Self::Output { - self.translate_unary(Op::f32_floor, wasm::f32_floor) + self.translate_unary(Op::f32x4_floor_ss, wasm::f32_floor) } #[inline(never)] fn visit_f32_trunc(&mut self) -> Self::Output { - self.translate_unary(Op::f32_trunc, wasm::f32_trunc) + self.translate_unary(Op::f32_trunc_ss, wasm::f32_trunc) } #[inline(never)] fn visit_f32_nearest(&mut self) -> Self::Output { - self.translate_unary(Op::f32_nearest, wasm::f32_nearest) + self.translate_unary(Op::f32_nearest_ss, wasm::f32_nearest) } #[inline(never)] fn visit_f32_sqrt(&mut self) -> Self::Output { - self.translate_unary(Op::f32_sqrt, wasm::f32_sqrt) + self.translate_unary(Op::f32_sqrt_ss, wasm::f32_sqrt) } #[inline(never)] fn visit_f32_add(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_add, wasm::f32_add) + self.translate_binary( + Op::f32_add_sss, + Op::f32_add_ssi, + Op::f32_add_sis, + wasm::f32_add, + ) } #[inline(never)] fn visit_f32_sub(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_sub, wasm::f32_sub) + self.translate_binary( + Op::f32_sub_sss, + Op::f32_sub_ssi, + Op::f32_sub_sis, + wasm::f32_sub, + ) } #[inline(never)] fn visit_f32_mul(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_mul, wasm::f32_mul) + self.translate_binary( + Op::f32_mul_sss, + Op::f32_mul_ssi, + Op::f32_mul_sis, + wasm::f32_mul, + ) } #[inline(never)] fn visit_f32_div(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_div, wasm::f32_div) + self.translate_binary( + Op::f32_div_sss, + Op::f32_div_ssi, + Op::f32_div_sis, + wasm::f32_div, + ) } #[inline(never)] fn visit_f32_min(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_min, wasm::f32_min) + self.translate_binary( + Op::f32_min_sss, + Op::f32_min_ssi, + Op::f32_min_sis, + wasm::f32_min, + ) } #[inline(never)] fn visit_f32_max(&mut self) -> Self::Output { - self.translate_fbinary(Op::f32_max, wasm::f32_max) + self.translate_binary( + Op::f32_max_sss, + Op::f32_max_ssi, + Op::f32_max_sis, + wasm::f32_max, + ) } #[inline(never)] fn visit_f32_copysign(&mut self) -> Self::Output { - self.translate_fcopysign::(Op::f32_copysign, Op::f32_copysign_imm, wasm::f32_copysign) + self.translate_fcopysign::( + Op::f32_copysign_sss, + Op::f32_copysign_ssi, + Op::f32_copysign_sis, + wasm::f32_copysign, + ) } #[inline(never)] fn visit_f64_abs(&mut self) -> Self::Output { - self.translate_unary(Op::f64_abs, wasm::f64_abs) + self.translate_unary(Op::f64_abs_ss, wasm::f64_abs) } #[inline(never)] fn visit_f64_neg(&mut self) -> Self::Output { - self.translate_unary(Op::f64_neg, wasm::f64_neg) + self.translate_unary(Op::f64_neg_ss, wasm::f64_neg) } #[inline(never)] fn visit_f64_ceil(&mut self) -> Self::Output { - self.translate_unary(Op::f64_ceil, wasm::f64_ceil) + self.translate_unary(Op::f64_ceil_ss, wasm::f64_ceil) } #[inline(never)] fn visit_f64_floor(&mut self) -> Self::Output { - self.translate_unary(Op::f64_floor, wasm::f64_floor) + self.translate_unary(Op::f64_floor_ss, wasm::f64_floor) } #[inline(never)] fn visit_f64_trunc(&mut self) -> Self::Output { - self.translate_unary(Op::f64_trunc, wasm::f64_trunc) + self.translate_unary(Op::f64_trunc_ss, wasm::f64_trunc) } #[inline(never)] fn visit_f64_nearest(&mut self) -> Self::Output { - self.translate_unary(Op::f64_nearest, wasm::f64_nearest) + self.translate_unary(Op::f64_nearest_ss, wasm::f64_nearest) } #[inline(never)] fn visit_f64_sqrt(&mut self) -> Self::Output { - self.translate_unary(Op::f64_sqrt, wasm::f64_sqrt) + self.translate_unary(Op::f64_sqrt_ss, wasm::f64_sqrt) } #[inline(never)] fn visit_f64_add(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_add, wasm::f64_add) + self.translate_binary( + Op::f64_add_sss, + Op::f64_add_ssi, + Op::f64_add_sis, + wasm::f64_add, + ) } #[inline(never)] fn visit_f64_sub(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_sub, wasm::f64_sub) + self.translate_binary( + Op::f64_sub_sss, + Op::f64_sub_ssi, + Op::f64_sub_sis, + wasm::f64_sub, + ) } #[inline(never)] fn visit_f64_mul(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_mul, wasm::f64_mul) + self.translate_binary( + Op::f64_mul_sss, + Op::f64_mul_ssi, + Op::f64_mul_sis, + wasm::f64_mul, + ) } #[inline(never)] fn visit_f64_div(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_div, wasm::f64_div) + self.translate_binary( + Op::f64_div_sss, + Op::f64_div_ssi, + Op::f64_div_sis, + wasm::f64_div, + ) } #[inline(never)] fn visit_f64_min(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_min, wasm::f64_min) + self.translate_binary( + Op::f64_min_sss, + Op::f64_min_ssi, + Op::f64_min_sis, + wasm::f64_min, + ) } #[inline(never)] fn visit_f64_max(&mut self) -> Self::Output { - self.translate_fbinary(Op::f64_max, wasm::f64_max) + self.translate_binary( + Op::f64_max_sss, + Op::f64_max_ssi, + Op::f64_max_sis, + wasm::f64_max, + ) } #[inline(never)] fn visit_f64_copysign(&mut self) -> Self::Output { - self.translate_fcopysign::(Op::f64_copysign, Op::f64_copysign_imm, wasm::f64_copysign) + self.translate_fcopysign::( + Op::f64_copysign_sss, + Op::f64_copysign_ssi, + Op::f64_copysign_sis, + wasm::f64_copysign, + ) } #[inline(never)] fn visit_i32_wrap_i64(&mut self) -> Self::Output { - self.translate_unary(Op::i32_wrap_i64, wasm::i32_wrap_i64) + self.translate_unary(Op::i32_wrap_i64_ss, wasm::i32_wrap_i64) } #[inline(never)] fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f32_s, wasm::i32_trunc_f32_s) + self.translate_unary_fallible(Op::i32_trunc_f32_ss, wasm::i32_trunc_f32_s) } #[inline(never)] fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f32_u, wasm::i32_trunc_f32_u) + self.translate_unary_fallible(Op::u32_trunc_f32_ss, wasm::i32_trunc_f32_u) } #[inline(never)] fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f64_s, wasm::i32_trunc_f64_s) + self.translate_unary_fallible(Op::i32_trunc_f64_ss, wasm::i32_trunc_f64_s) } #[inline(never)] fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i32_trunc_f64_u, wasm::i32_trunc_f64_u) + self.translate_unary_fallible(Op::u32_trunc_f64_ss, wasm::i32_trunc_f64_u) } #[inline(never)] fn visit_i64_extend_i32_s(&mut self) -> Self::Output { - self.translate_unary::(Op::i64_extend32_s, wasm::i64_extend_i32_s) + self.translate_unary::(Op::i64_sext32_ss, wasm::i64_extend_i32_s) } #[inline(never)] @@ -1579,72 +1679,72 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f32_s, wasm::i64_trunc_f32_s) + self.translate_unary_fallible(Op::i64_trunc_f32_ss, wasm::i64_trunc_f32_s) } #[inline(never)] fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f32_u, wasm::i64_trunc_f32_u) + self.translate_unary_fallible(Op::u64_trunc_f32_ss, wasm::i64_trunc_f32_u) } #[inline(never)] fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f64_s, wasm::i64_trunc_f64_s) + self.translate_unary_fallible(Op::i64_trunc_f64_ss, wasm::i64_trunc_f64_s) } #[inline(never)] fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { - self.translate_unary_fallible(Op::i64_trunc_f64_u, wasm::i64_trunc_f64_u) + self.translate_unary_fallible(Op::u64_trunc_f64_ss, wasm::i64_trunc_f64_u) } #[inline(never)] fn visit_f32_convert_i32_s(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i32_s, wasm::f32_convert_i32_s) + self.translate_unary(Op::f32_convert_i32_ss, wasm::f32_convert_i32_s) } #[inline(never)] fn visit_f32_convert_i32_u(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i32_u, wasm::f32_convert_i32_u) + self.translate_unary(Op::f32_convert_u32_ss, wasm::f32_convert_i32_u) } #[inline(never)] fn visit_f32_convert_i64_s(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i64_s, wasm::f32_convert_i64_s) + self.translate_unary(Op::f32_convert_i64_ss, wasm::f32_convert_i64_s) } #[inline(never)] fn visit_f32_convert_i64_u(&mut self) -> Self::Output { - self.translate_unary(Op::f32_convert_i64_u, wasm::f32_convert_i64_u) + self.translate_unary(Op::f32_convert_u64_ss, wasm::f32_convert_i64_u) } #[inline(never)] fn visit_f32_demote_f64(&mut self) -> Self::Output { - self.translate_unary(Op::f32_demote_f64, wasm::f32_demote_f64) + self.translate_unary(Op::f32_demote_f64_ss, wasm::f32_demote_f64) } #[inline(never)] fn visit_f64_convert_i32_s(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i32_s, wasm::f64_convert_i32_s) + self.translate_unary(Op::f64_convert_i32_ss, wasm::f64_convert_i32_s) } #[inline(never)] fn visit_f64_convert_i32_u(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i32_u, wasm::f64_convert_i32_u) + self.translate_unary(Op::f64_convert_u32_ss, wasm::f64_convert_i32_u) } #[inline(never)] fn visit_f64_convert_i64_s(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i64_s, wasm::f64_convert_i64_s) + self.translate_unary(Op::f64_convert_i64_ss, wasm::f64_convert_i64_s) } #[inline(never)] fn visit_f64_convert_i64_u(&mut self) -> Self::Output { - self.translate_unary(Op::f64_convert_i64_u, wasm::f64_convert_i64_u) + self.translate_unary(Op::f64_convert_u64_ss, wasm::f64_convert_i64_u) } #[inline(never)] fn visit_f64_promote_f32(&mut self) -> Self::Output { - self.translate_unary(Op::f64_promote_f32, wasm::f64_promote_f32) + self.translate_unary(Op::f64_promote_f32_ss, wasm::f64_promote_f32) } #[inline(never)] @@ -1669,79 +1769,82 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_extend8_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_extend8_s, wasm::i32_extend8_s) + self.translate_unary(Op::i32_sext8_ss, wasm::i32_extend8_s) } #[inline(never)] fn visit_i32_extend16_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_extend16_s, wasm::i32_extend16_s) + self.translate_unary(Op::i32_sext16_ss, wasm::i32_extend16_s) } #[inline(never)] fn visit_i64_extend8_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_extend8_s, wasm::i64_extend8_s) + self.translate_unary(Op::i64_sext8_ss, wasm::i64_extend8_s) } #[inline(never)] fn visit_i64_extend16_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_extend16_s, wasm::i64_extend16_s) + self.translate_unary(Op::i64_sext16_ss, wasm::i64_extend16_s) } #[inline(never)] fn visit_i64_extend32_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_extend32_s, wasm::i64_extend32_s) + self.translate_unary(Op::i64_sext32_ss, wasm::i64_extend32_s) } #[inline(never)] fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f32_s, wasm::i32_trunc_sat_f32_s) + self.translate_unary(Op::i32_trunc_sat_f32_ss, wasm::i32_trunc_sat_f32_s) } #[inline(never)] fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f32_u, wasm::i32_trunc_sat_f32_u) + self.translate_unary(Op::u32_trunc_sat_f32_ss, wasm::i32_trunc_sat_f32_u) } #[inline(never)] fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f64_s, wasm::i32_trunc_sat_f64_s) + self.translate_unary(Op::i32_trunc_sat_f64_ss, wasm::i32_trunc_sat_f64_s) } #[inline(never)] fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { - self.translate_unary(Op::i32_trunc_sat_f64_u, wasm::i32_trunc_sat_f64_u) + self.translate_unary(Op::u32_trunc_sat_f64_ss, wasm::i32_trunc_sat_f64_u) } #[inline(never)] fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f32_s, wasm::i64_trunc_sat_f32_s) + self.translate_unary(Op::i64_trunc_sat_f32_ss, wasm::i64_trunc_sat_f32_s) } #[inline(never)] fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f32_u, wasm::i64_trunc_sat_f32_u) + self.translate_unary(Op::u64_trunc_sat_f32_ss, wasm::i64_trunc_sat_f32_u) } #[inline(never)] fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f64_s, wasm::i64_trunc_sat_f64_s) + self.translate_unary(Op::i64_trunc_sat_f64_ss, wasm::i64_trunc_sat_f64_s) } #[inline(never)] fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { - self.translate_unary(Op::i64_trunc_sat_f64_u, wasm::i64_trunc_sat_f64_u) + self.translate_unary(Op::u64_trunc_sat_f64_ss, wasm::i64_trunc_sat_f64_u) } #[inline(never)] fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); + let memory = index::Memory::from(mem); + let data = index::Data::from(data_index); let dst = self.immediate_to_reg(dst)?; let src = self.immediate_to_reg(src)?; let len = self.immediate_to_reg(len)?; - self.push_instr(Op::memory_init(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::memory_index(mem))?; - self.push_param(Op::data_index(data_index))?; + self.push_instr( + Op::memory_init(memory, data, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1756,12 +1859,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); + let dst_memory = index::Memory::from(dst_mem); + let src_memory = index::Memory::from(src_mem); let dst = self.immediate_to_reg(dst)?; let src = self.immediate_to_reg(src)?; let len = self.immediate_to_reg(len)?; - self.push_instr(Op::memory_copy(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::memory_index(dst_mem))?; - self.push_param(Op::memory_index(src_mem))?; + self.push_instr( + Op::memory_copy(dst_memory, src_memory, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1769,6 +1875,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, value, len) = self.stack.pop3(); + let memory = index::Memory::from(mem); let dst = self.immediate_to_reg(dst)?; let value = self.make_input(value, |_, value| { let byte = u32::from(value) as u8; @@ -1776,11 +1883,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { })?; let len = self.immediate_to_reg(len)?; let instr: Op = match value { - Input::Slot(value) => Op::memory_fill(dst, value, len), - Input::Immediate(value) => Op::memory_fill_imm(dst, value, len), + Input::Slot(value) => Op::memory_fill(memory, dst, value, len), + Input::Immediate(value) => Op::memory_fill_imm(memory, dst, value, len), }; self.push_instr(instr, FuelCostsProvider::instance)?; - self.push_param(Op::memory_index(mem))?; Ok(()) } @@ -1788,12 +1894,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_init(&mut self, elem_index: u32, table: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); + let table = index::Table::from(table); + let elem = index::Elem::from(elem_index); let dst = self.immediate_to_reg(dst)?; let src = self.immediate_to_reg(src)?; let len = self.immediate_to_reg(len)?; - self.push_instr(Op::table_init(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::table_index(table))?; - self.push_param(Op::elem_index(elem_index))?; + self.push_instr( + Op::table_init(table, elem, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1808,12 +1917,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); + let dst_table = index::Table::from(dst_table); + let src_table = index::Table::from(src_table); let dst = self.immediate_to_reg(dst)?; let src = self.immediate_to_reg(src)?; let len = self.immediate_to_reg(len)?; - self.push_instr(Op::table_copy(dst, src, len), FuelCostsProvider::instance)?; - self.push_param(Op::table_index(dst_table))?; - self.push_param(Op::table_index(src_table))?; + self.push_instr( + Op::table_copy(dst_table, src_table, dst, src, len), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1882,11 +1994,14 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_fill(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let (dst, value, len) = self.stack.pop3(); + let table = index::Table::from(table); let dst = self.immediate_to_reg(dst)?; let value = self.immediate_to_reg(value)?; let len = self.immediate_to_reg(len)?; - self.push_instr(Op::table_fill(dst, len, value), FuelCostsProvider::instance)?; - self.push_param(Op::table_index(table))?; + self.push_instr( + Op::table_fill(table, dst, len, value), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1894,6 +2009,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_get(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index = self.stack.pop(); let item_ty = table_type.element(); let index_ty = table_type.index_ty(); @@ -1901,12 +2017,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.push_instr_with_result( item_ty, |result| match index { - Input::Slot(index) => Op::table_get(result, index), - Input::Immediate(index) => Op::table_get_imm(result, index), + Input::Slot(index) => Op::table_get_ss(result, index, table), + Input::Immediate(index) => Op::table_get_si(result, index, table), }, FuelCostsProvider::instance, )?; - self.push_param(Op::table_index(table))?; Ok(()) } @@ -1914,16 +2029,16 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_set(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index_ty = table_type.index_ty(); let (index, value) = self.stack.pop2(); let index = self.make_index32(index, index_ty)?; let value = self.layout.operand_to_reg(value)?; let instr = match index { - Input::Slot(index) => Op::table_set(index, value), - Input::Immediate(index) => Op::table_set_at(value, index), + Input::Slot(index) => Op::table_set_ss(index, value, table), + Input::Immediate(index) => Op::table_set_si(value, index, table), }; self.push_instr(instr, FuelCostsProvider::instance)?; - self.push_param(Op::table_index(table))?; Ok(()) } @@ -1931,6 +2046,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_grow(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index_ty = table_type.index_ty(); let (value, delta) = self.stack.pop2(); if let Operand::Immediate(delta) = delta { @@ -1957,10 +2073,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let delta = self.immediate_to_reg(delta)?; self.push_instr_with_result( index_ty.ty(), - |result| Op::table_grow(result, delta, value), + |result| Op::table_grow(result, delta, value, table), FuelCostsProvider::instance, )?; - self.push_param(Op::table_index(table))?; Ok(()) } @@ -1968,6 +2083,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_size(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let table = index::Table::from(table); let index_ty = table_type.index_ty(); self.push_instr_with_result( index_ty.ty(), @@ -1983,28 +2099,21 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let func_idx = FuncIdx::from(function_index); let func_type = self.resolve_func_type(func_idx); let len_params = usize::from(func_type.len_params()); + let consume_fuel = self.stack.consume_fuel_instr(); + self.move_operands_to_temp(len_params, consume_fuel)?; let instr = match self.module.get_engine_func(func_idx) { Some(engine_func) => { // Case: We are calling an internal function and can optimize // this case by using the special instruction for it. - match len_params { - 0 => Op::return_call_internal_0(engine_func), - _ => Op::return_call_internal(engine_func), - } + Op::return_call_internal(engine_func) } None => { // Case: We are calling an imported function and must use the // general calling operator for it. - match len_params { - 0 => Op::return_call_imported_0(function_index), - _ => Op::return_call_imported(function_index), - } + Op::return_call_imported(function_index) } }; self.push_instr(instr, FuelCostsProvider::call)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; self.reachable = false; Ok(()) } @@ -2013,21 +2122,22 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { bail_unreachable!(self); let func_type = self.resolve_type(type_index); + let table = index::Table::from(table_index); + let index_ty = self + .module + .get_type_of_table(TableIdx::from(table_index)) + .index_ty(); let index = self.stack.pop(); - let indirect_params = self.call_indirect_params(index, table_index)?; let len_params = usize::from(func_type.len_params()); - let instr = match (len_params, indirect_params) { - (0, Op::CallIndirectParams { .. }) => Op::return_call_indirect_0(type_index), - (0, Op::CallIndirectParamsImm16 { .. }) => Op::return_call_indirect_0_imm16(type_index), - (_, Op::CallIndirectParams { .. }) => Op::return_call_indirect(type_index), - (_, Op::CallIndirectParamsImm16 { .. }) => Op::return_call_indirect_imm16(type_index), + let consume_fuel = self.stack.consume_fuel_instr(); + self.move_operands_to_temp(len_params, consume_fuel)?; + let index = self.make_index32(index, index_ty)?; + let instr = match index { + Input::Slot(index) => Op::return_call_indirect(index, type_index, table), + Input::Immediate(index) => Op::return_call_indirect_imm16(index, type_index, table), _ => unreachable!(), }; self.push_instr(instr, FuelCostsProvider::call)?; - self.push_param(indirect_params)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; self.reachable = false; Ok(()) } @@ -2044,11 +2154,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_mul_wide_s(&mut self) -> Self::Output { - self.translate_i64_mul_wide_sx(Op::i64_mul_wide_s, wasm::i64_mul_wide_s, true) + self.translate_i64_mul_wide_sx(Op::s64_mul_wide, wasm::i64_mul_wide_s, true) } #[inline(never)] fn visit_i64_mul_wide_u(&mut self) -> Self::Output { - self.translate_i64_mul_wide_sx(Op::i64_mul_wide_u, wasm::i64_mul_wide_u, false) + self.translate_i64_mul_wide_sx(Op::u64_mul_wide, wasm::i64_mul_wide_u, false) } } From 4b8b5a1b48050d6116b637f0378fe877957223ba Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:21:04 +0200 Subject: [PATCH 016/423] update some FuncTranslator visit simd methods --- .../src/engine/translator/func/simd/visit.rs | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index a1afea04ac..eb65e29861 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -34,9 +34,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load, - Op::v128_load_offset16, - Op::v128_load_at, + Op::v128_load_ss, + None, + Op::v128_load_mem0_offset16_ss, ) } @@ -44,9 +44,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load8x8_s, - Op::v128_load8x8_s_offset16, - Op::v128_load8x8_s_at, + Op::s16x8_load8x8_ss, + None, + Op::s16x8_load8x8_mem0_offset16_ss, ) } @@ -54,9 +54,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load8x8_u, - Op::v128_load8x8_u_offset16, - Op::v128_load8x8_u_at, + Op::u16x8_load8x8_ss, + None, + Op::u16x8_load8x8_mem0_offset16_ss, ) } @@ -64,9 +64,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load16x4_s, - Op::v128_load16x4_s_offset16, - Op::v128_load16x4_s_at, + Op::s32x4_load16x4_ss, + None, + Op::s32x4_load16x4_mem0_offset16_ss, ) } @@ -74,9 +74,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load16x4_u, - Op::v128_load16x4_u_offset16, - Op::v128_load16x4_u_at, + Op::u32x4_load16x4_ss, + None, + Op::u32x4_load16x4_mem0_offset16_ss, ) } @@ -84,9 +84,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load32x2_s, - Op::v128_load32x2_s_offset16, - Op::v128_load32x2_s_at, + Op::s64x2_load32x2_ss, + None, + Op::s64x2_load32x2_mem0_offset16_ss, ) } @@ -94,9 +94,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load32x2_u, - Op::v128_load32x2_u_offset16, - Op::v128_load32x2_u_at, + Op::u64x2_load32x2_ss, + None, + Op::u64x2_load32x2_mem0_offset16_ss, ) } @@ -104,9 +104,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load8_splat, - Op::v128_load8_splat_offset16, - Op::v128_load8_splat_at, + Op::v128_load8_splat_ss, + None, + Op::v128_load8_splat_mem0_offset16_ss, ) } @@ -114,9 +114,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load16_splat, - Op::v128_load16_splat_offset16, - Op::v128_load16_splat_at, + Op::v128_load16_splat_ss, + None, + Op::v128_load16_splat_mem0_offset16_ss, ) } @@ -124,9 +124,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load32_splat, - Op::v128_load32_splat_offset16, - Op::v128_load32_splat_at, + Op::v128_load32_splat_ss, + None, + Op::v128_load32_splat_mem0_offset16_ss, ) } @@ -134,9 +134,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load64_splat, - Op::v128_load64_splat_offset16, - Op::v128_load64_splat_at, + Op::v128_load64_splat_ss, + None, + Op::v128_load64_splat_mem0_offset16_ss, ) } @@ -144,9 +144,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load32_zero, - Op::v128_load32_zero_offset16, - Op::v128_load32_zero_at, + Op::v128_load32_zero_ss, + None, + Op::v128_load32_zero_mem0_offset16_ss, ) } @@ -154,9 +154,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_load( memarg, ValType::V128, - Op::v128_load64_zero, - Op::v128_load64_zero_offset16, - Op::v128_load64_zero_at, + Op::v128_load64_zero_ss, + None, + Op::v128_load64_zero_mem0_offset16_ss, ) } From 39a268d98046e9b22ba972e658856caa17211e28 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 10:21:26 +0200 Subject: [PATCH 017/423] initial update on FuncTranslator core/base methods --- .../wasmi/src/engine/translator/func/mod.rs | 677 ++++-------------- 1 file changed, 120 insertions(+), 557 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 095026e700..af9f895887 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -658,32 +658,22 @@ impl FuncTranslator { layout: &mut StackLayout, ) -> Result { let instr = match value.ty() { - ValType::I32 => Op::copy_imm32(result, i32::from(value)), - ValType::I64 => { - let value = i64::from(value); - match >::try_from(value) { - Ok(value) => Op::copy_i64imm32(result, value), - Err(_) => { - let value = layout.const_to_reg(value)?; - Op::copy(result, value) - } - } - } - ValType::F32 => Op::copy_imm32(result, f32::from(value)), - ValType::F64 => { - let value = f64::from(value); - match >::try_from(value) { - Ok(value) => Op::copy_f64imm32(result, value), - Err(_) => { - let value = layout.const_to_reg(value)?; - Op::copy(result, value) - } - } - } - ValType::V128 | ValType::FuncRef | ValType::ExternRef => { - let value = layout.const_to_reg(value)?; - Op::copy(result, value) - } + ValType::I32 => Op::copy32(result, i32::from(value)), + ValType::I64 => Op::copy64(result, i64::from(value)), + ValType::F32 => Op::copy32(result, f32::from(value).to_bits()), + ValType::F64 => Op::copy64(result, f64::from(value).to_bits()), + ValType::ExternRef | ValType::FuncRef => { + Op::copy64(result, u64::from(UntypedVal::from(value))) + } + #[cfg(feature = "simd")] + ValType::V128 => { + let value = V128::from(value).as_u128(); + let value_lo = (value & 0xFFFF_FFFF_FFFF_FFFF) as u64; + let value_hi = (value >> 64) as u64; + Op::copy128(result, value_lo, value_hi) + } + #[cfg(not(feature = "simd"))] + ValType::V128 => panic!("unexpected `v128` operand: {value}"), }; Ok(instr) } @@ -949,10 +939,10 @@ impl FuncTranslator { &mut self, operand: Operand, consume_fuel: Option, - ) -> Result<(), Error> { + ) -> Result { let result = self.layout.temp_to_reg(operand.index())?; self.encode_copy(result, operand, consume_fuel)?; - Ok(()) + Ok(result) } /// Efficiently converts the `operand` to a [`Slot`] if it is an immediate. @@ -1125,10 +1115,9 @@ impl FuncTranslator { let consume_fuel_instr = self.stack.consume_fuel_instr(); let values = self.try_form_regspan_or_move(usize::from(len_values), consume_fuel_instr)?; self.push_instr( - Op::branch_table_span(index, table.len() + 1), + Op::branch_table_span(index, table.len() + 1, values, len_values), FuelCostsProvider::base, )?; - self.push_param(Op::slot_span(BoundedSlotSpan::new(values, len_values)))?; // Encode the `br_table` targets: let targets = &self.immediates[..]; for target in targets { @@ -1158,52 +1147,36 @@ impl FuncTranslator { 1 => match self.stack.peek(0) { Operand::Local(operand) => { let value = self.layout.local_to_reg(operand.local_index())?; - Op::return_reg(value) + Op::return_slot(value) } Operand::Temp(operand) => { let value = self.layout.temp_to_reg(operand.operand_index())?; - Op::return_reg(value) + Op::return_slot(value) } Operand::Immediate(operand) => { let val = operand.val(); match operand.ty() { - ValType::I32 => Op::return_imm32(i32::from(val)), - ValType::I64 => match >::try_from(i64::from(val)) { - Ok(value) => Op::return_i64imm32(value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Op::return_reg(value) - } - }, - ValType::F32 => Op::return_imm32(f32::from(val)), - ValType::F64 => match >::try_from(f64::from(val)) { - Ok(value) => Op::return_f64imm32(value), - Err(_) => { - let value = self.layout.const_to_reg(val)?; - Op::return_reg(value) - } - }, - ValType::V128 | ValType::FuncRef | ValType::ExternRef => { - let value = self.layout.const_to_reg(val)?; - Op::return_reg(value) + ValType::I32 => Op::return32(i32::from(val)), + ValType::I64 => Op::return64(i64::from(val)), + ValType::F32 => Op::return32(f32::from(val).to_bits()), + ValType::F64 => Op::return64(f64::from(val).to_bits()), + ValType::FuncRef | ValType::ExternRef => { + Op::return64(u64::from(UntypedVal::from(val))) + } + ValType::V128 => { + let value = self.stack.peek(0); + let temp_slot = self.copy_operand_to_temp(operand, consume_fuel)?; + Op::return_slot(temp_slot) } } } }, - 2 => { - let (v0, v1) = self.stack.peek2(); - let v0 = self.layout.operand_to_reg(v0)?; - let v1 = self.layout.operand_to_reg(v1)?; - Op::return_reg2_ext(v0, v1) + _ => { + self.move_operands_to_temp(usize::from(len_results), consume_fuel)?; + let result0 = self.stack.peek(usize::from(len_results)); + let slot0 = self.layout.temp_to_reg(result0.index())?; + Op::return_span(SlotSpan::new(slot0)) } - 3 => { - let (v0, v1, v2) = self.stack.peek3(); - let v0 = self.layout.operand_to_reg(v0)?; - let v1 = self.layout.operand_to_reg(v1)?; - let v2 = self.layout.operand_to_reg(v2)?; - Op::return_reg3_ext(v0, v1, v2) - } - _ => return self.encode_return_many(len_results, consume_fuel), }; let instr = self .instrs @@ -1593,38 +1566,14 @@ impl FuncTranslator { }; let instr = self.instrs.next_instr(); let offset = self.labels.try_resolve_label(label, instr)?; - let instr = match BranchOffset16::try_from(offset) { - Ok(offset) => match branch_eqz { - true => Op::branch_i32_eq_imm16(condition, 0, offset), - false => Op::branch_i32_ne_imm16(condition, 0, offset), - }, - Err(_) => { - let zero = self.layout.const_to_reg(0_i32)?; - let comparator = match branch_eqz { - true => Comparator::I32Eq, - false => Comparator::I32Ne, - }; - self.make_branch_cmp_fallback(comparator, condition, zero, offset)? - } + let instr = match branch_eqz { + true => Op::branch_i32_eq_imm16(condition, 0, offset), + false => Op::branch_i32_ne_imm16(condition, 0, offset), }; self.push_instr(instr, FuelCostsProvider::base)?; Ok(()) } - /// Create an [`Op::BranchCmpFallback`]. - fn make_branch_cmp_fallback( - &mut self, - cmp: Comparator, - lhs: Slot, - rhs: Slot, - offset: BranchOffset, - ) -> Result { - let params = self - .layout - .const_to_reg(ComparatorAndOffset::new(cmp, offset))?; - Ok(Op::branch_cmp_fallback(lhs, rhs, params)) - } - /// Try to fuse a cmp+branch [`Op`] with optional negation. fn try_fuse_branch_cmp( &mut self, @@ -1704,7 +1653,7 @@ impl FuncTranslator { }; let offset = self.labels.try_resolve_label(label, instr)?; let fused = cmp_instr - .try_into_cmp_branch_instr(offset, &mut self.layout)? + .try_into_cmp_branch_instr(offset)? .expect("cmp+branch fusion must succeed"); Ok(Some(fused)) } @@ -1794,20 +1743,6 @@ impl FuncTranslator { Ok(()) } - /// Creates a new 16-bit encoded [`Input16`] from the given `value`. - pub fn make_imm16(&mut self, value: T) -> Result, Error> - where - T: Into + Copy + TryInto>, - { - match value.try_into() { - Ok(rhs) => Ok(Input::Immediate(rhs)), - Err(_) => { - let rhs = self.layout.const_to_reg(value)?; - Ok(Input::Slot(rhs)) - } - } - } - /// Create a new generic [`Input`] from the given `operand`. fn make_input( &mut self, @@ -1822,73 +1757,34 @@ impl FuncTranslator { Ok(Input::Slot(reg)) } - /// Converts the `provider` to a 16-bit index-type constant value. + /// Converts the `provider` to a 64-bit index-type constant value. /// /// # Note /// - /// - Turns immediates that cannot be 16-bit encoded into function local constants. - /// - The behavior is different whether `memory64` is enabled or disabled. - pub(super) fn make_index16( + /// This expects `operand` to be either `u32` or `u64` if `memory64` is enabled or disabled respectively. + pub(super) fn make_index64( &mut self, operand: Operand, index_type: IndexType, - ) -> Result, Error> { + ) -> Result, Error> { let value = match operand { Operand::Immediate(value) => value.val(), - operand => { + Operand::Local(value) => { debug_assert_eq!(operand.ty(), index_type.ty()); - let reg = self.layout.operand_to_reg(operand)?; + let reg = self.layout.local_to_reg(value.local_index())?; return Ok(Input::Slot(reg)); } - }; - match index_type { - IndexType::I64 => { - if let Ok(value) = Const16::try_from(u64::from(value)) { - return Ok(Input::Immediate(value)); - } - } - IndexType::I32 => { - if let Ok(value) = Const16::try_from(u32::from(value)) { - return Ok(Input::Immediate(>::cast(value))); - } - } - } - let reg = self.layout.const_to_reg(value)?; - Ok(Input::Slot(reg)) - } - - /// Converts the `provider` to a 32-bit index-type constant value. - /// - /// # Note - /// - /// - Turns immediates that cannot be 32-bit encoded into function local constants. - /// - The behavior is different whether `memory64` is enabled or disabled. - pub(super) fn make_index32( - &mut self, - operand: Operand, - index_type: IndexType, - ) -> Result, Error> { - let value = match operand { - Operand::Immediate(value) => value.val(), - operand => { + Operand::Temp(value) => { debug_assert_eq!(operand.ty(), index_type.ty()); - let reg = self.layout.operand_to_reg(operand)?; + let reg = self.layout.temp_to_reg(value.operand_index())?; return Ok(Input::Slot(reg)); } }; - match index_type { - IndexType::I64 => { - if let Ok(value) = Const32::try_from(u64::from(value)) { - return Ok(Input::Immediate(value)); - } - } - IndexType::I32 => { - let value32 = Const32::from(u32::from(value)); - return Ok(Input::Immediate(>::cast(value32))); - } - } - let reg = self.layout.const_to_reg(value)?; - Ok(Input::Slot(reg)) + let value = match index_type { + IndexType::I32 => u64::from(u32::from(value)), + IndexType::I64 => u64::from(value), + }; + Ok(Input::Immediate(value)) } /// Evaluates `consteval(lhs, rhs)` and pushed either its result or tranlates a `trap`. @@ -1939,13 +1835,13 @@ impl FuncTranslator { /// Translates a commutative binary Wasm operator to Wasmi bytecode. fn translate_binary_commutative( &mut self, - make_rr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_ri: fn(result: Slot, lhs: Slot, rhs: Const16) -> Op, + make_sss: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, + make_ssi: fn(result: Slot, lhs: Slot, rhs: T) -> Op, consteval: fn(T, T) -> R, - opt_ri: fn(this: &mut Self, lhs: Operand, rhs: T) -> Result, + opt_si: fn(this: &mut Self, lhs: Operand, rhs: T) -> Result, ) -> Result<(), Error> where - T: WasmInteger + TryInto>, + T: WasmInteger, R: Into + Typed, { bail_unreachable!(self); @@ -1955,17 +1851,13 @@ impl FuncTranslator { } (val, Operand::Immediate(imm)) | (Operand::Immediate(imm), val) => { let rhs = imm.val().into(); - if opt_ri(self, val, rhs)? { + if opt_si(self, val, rhs)? { return Ok(()); } let lhs = self.layout.operand_to_reg(val)?; - let rhs16 = self.make_imm16(rhs)?; self.push_instr_with_result( ::TY, - |result| match rhs16 { - Input::Immediate(rhs) => make_ri(result, lhs, rhs), - Input::Slot(rhs) => make_rr(result, lhs, rhs), - }, + |result| make_ssi(result, lhs, rhs), FuelCostsProvider::base, ) } @@ -1973,7 +1865,7 @@ impl FuncTranslator { ::TY, lhs, rhs, - make_rr, + make_sss, FuelCostsProvider::base, ), } @@ -1982,13 +1874,9 @@ impl FuncTranslator { /// Translates integer division and remainder Wasm operators to Wasmi bytecode. fn translate_divrem( &mut self, - make_instr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_instr_imm16_rhs: fn( - result: Slot, - lhs: Slot, - rhs: Const16<::NonZero>, - ) -> Op, - make_instr_imm16_lhs: fn(result: Slot, lhs: Const16, rhs: Slot) -> Op, + make_instr_sss: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, + make_instr_ssi: fn(result: Slot, lhs: Slot, rhs: ::NonZero) -> Op, + make_instr_sis: fn(result: Slot, lhs: T, rhs: Slot) -> Op, consteval: fn(T, T) -> Result, ) -> Result<(), Error> where @@ -2006,26 +1894,18 @@ impl FuncTranslator { // Optimization: division by zero always traps return self.translate_trap(TrapCode::IntegerDivisionByZero); }; - let rhs16 = self.make_imm16(non_zero_rhs)?; self.push_instr_with_result( ::TY, - |result| match rhs16 { - Input::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), - Input::Slot(rhs) => make_instr(result, lhs, rhs), - }, + |result| make_instr_ssi(result, lhs, rhs), FuelCostsProvider::base, ) } (Operand::Immediate(lhs), rhs) => { let lhs = T::from(lhs.val()); - let lhs16 = self.make_imm16(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ::TY, - |result| match lhs16 { - Input::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), - Input::Slot(lhs) => make_instr(result, lhs, rhs), - }, + |result| make_instr_sis(result, lhs, rhs), FuelCostsProvider::base, ) } @@ -2033,7 +1913,7 @@ impl FuncTranslator { ::TY, lhs, rhs, - make_instr, + make_instr_sss, FuelCostsProvider::base, ), } @@ -2042,9 +1922,9 @@ impl FuncTranslator { /// Translates binary non-commutative Wasm operators to Wasmi bytecode. fn translate_binary( &mut self, - make_instr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_instr_imm16_rhs: fn(result: Slot, lhs: Slot, rhs: Const16) -> Op, - make_instr_imm16_lhs: fn(result: Slot, lhs: Const16, rhs: Slot) -> Op, + make_instr_sss: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, + make_instr_ssi: fn(result: Slot, lhs: Slot, rhs: T) -> Op, + make_instr_sis: fn(result: Slot, lhs: T, rhs: Slot) -> Op, consteval: fn(T, T) -> R, ) -> Result<(), Error> where @@ -2059,13 +1939,9 @@ impl FuncTranslator { (lhs, Operand::Immediate(rhs)) => { let lhs = self.layout.operand_to_reg(lhs)?; let rhs = T::from(rhs.val()); - let rhs16 = self.make_imm16(rhs)?; self.push_instr_with_result( ::TY, - |result| match rhs16 { - Input::Immediate(rhs) => make_instr_imm16_rhs(result, lhs, rhs), - Input::Slot(rhs) => make_instr(result, lhs, rhs), - }, + |result| make_instr_ssi(result, lhs, rhs), FuelCostsProvider::base, ) } @@ -2075,69 +1951,7 @@ impl FuncTranslator { let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ::TY, - |result| match lhs16 { - Input::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), - Input::Slot(lhs) => make_instr(result, lhs, rhs), - }, - FuelCostsProvider::base, - ) - } - (lhs, rhs) => self.push_binary_instr_with_result( - ::TY, - lhs, - rhs, - make_instr, - FuelCostsProvider::base, - ), - } - } - - /// Translates Wasm `i{32,64}.sub` operators to Wasmi bytecode. - fn translate_isub( - &mut self, - make_sub_rr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_add_ri: fn(result: Slot, lhs: Slot, rhs: Const16) -> Op, - make_sub_ir: fn(result: Slot, lhs: Const16, rhs: Slot) -> Op, - consteval: fn(T, T) -> R, - ) -> Result<(), Error> - where - T: WasmInteger, - R: Into + Typed, - { - bail_unreachable!(self); - match self.stack.pop2() { - (Operand::Immediate(lhs), Operand::Immediate(rhs)) => { - self.translate_binary_consteval::(lhs, rhs, consteval) - } - (lhs, Operand::Immediate(rhs)) => { - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = T::from(rhs.val()); - let rhs16 = match rhs.wrapping_neg().try_into() { - Ok(rhs) => Input::Immediate(rhs), - Err(_) => { - let rhs = self.layout.const_to_reg(rhs)?; - Input::Slot(rhs) - } - }; - self.push_instr_with_result( - ::TY, - |result| match rhs16 { - Input::Immediate(rhs) => make_add_ri(result, lhs, rhs), - Input::Slot(rhs) => make_sub_rr(result, lhs, rhs), - }, - FuelCostsProvider::base, - ) - } - (Operand::Immediate(lhs), rhs) => { - let lhs = T::from(lhs.val()); - let lhs16 = self.make_imm16(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; - self.push_instr_with_result( - ::TY, - |result| match lhs16 { - Input::Immediate(lhs) => make_sub_ir(result, lhs, rhs), - Input::Slot(lhs) => make_sub_rr(result, lhs, rhs), - }, + |result| make_instr_sis(result, lhs, rhs), FuelCostsProvider::base, ) } @@ -2145,7 +1959,7 @@ impl FuncTranslator { ::TY, lhs, rhs, - make_sub_rr, + make_instr_sss, FuelCostsProvider::base, ), } @@ -2154,18 +1968,13 @@ impl FuncTranslator { /// Translates Wasm shift and rotate operators to Wasmi bytecode. fn translate_shift( &mut self, - make_instr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_instr_imm16_rhs: fn( - result: Slot, - lhs: Slot, - rhs: ::Output, - ) -> Op, - make_instr_imm16_lhs: fn(result: Slot, lhs: Const16, rhs: Slot) -> Op, + make_instr_sss: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, + make_instr_ssi: fn(result: Slot, lhs: Slot, rhs: ::Value) -> Op, + make_instr_sis: fn(result: Slot, lhs: T, rhs: Slot) -> Op, consteval: fn(T, T) -> T, ) -> Result<(), Error> where - T: WasmInteger + IntoShiftAmount>, - Const16: From, + T: WasmInteger + IntoShiftAmount, { bail_unreachable!(self); match self.stack.pop2() { @@ -2173,7 +1982,7 @@ impl FuncTranslator { self.translate_binary_consteval::(lhs, rhs, consteval) } (lhs, Operand::Immediate(rhs)) => { - let Some(rhs) = T::into_shift_amount(rhs.val().into()) else { + let Some(rhs) = T::into_shift_amount(T::from(rhs.val())) else { // Optimization: Shifting or rotating by zero bits is a no-op. self.stack.push_operand(lhs)?; return Ok(()); @@ -2181,7 +1990,7 @@ impl FuncTranslator { let lhs = self.layout.operand_to_reg(lhs)?; self.push_instr_with_result( ::TY, - |result| make_instr_imm16_rhs(result, lhs, rhs), + |result| make_instr_ssi(result, lhs, rhs), FuelCostsProvider::base, ) } @@ -2192,14 +2001,10 @@ impl FuncTranslator { self.stack.push_immediate(lhs)?; return Ok(()); } - let lhs16 = self.make_imm16(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ::TY, - |result| match lhs16 { - Input::Immediate(lhs) => make_instr_imm16_lhs(result, lhs, rhs), - Input::Slot(lhs) => make_instr(result, lhs, rhs), - }, + |result| make_instr_sis(result, lhs, rhs), FuelCostsProvider::base, ) } @@ -2207,7 +2012,7 @@ impl FuncTranslator { ::TY, lhs, rhs, - make_instr, + make_instr_sss, FuelCostsProvider::base, ), } @@ -2245,8 +2050,9 @@ impl FuncTranslator { /// - Applies constant evaluation if both operands are constant values. fn translate_fcopysign( &mut self, - make_instr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_instr_imm: fn(result: Slot, lhs: Slot, rhs: Sign) -> Op, + make_instr_sss: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, + make_instr_ssi: fn(result: Slot, lhs: Slot, rhs: Sign) -> Op, + make_instr_sis: fn(result: Slot, lhs: T, rhs: Slot) -> Op, consteval: fn(T, T) -> T, ) -> Result<(), Error> where @@ -2262,7 +2068,16 @@ impl FuncTranslator { let sign = T::from(rhs.val()).sign(); self.push_instr_with_result( ::TY, - |result| make_instr_imm(result, lhs, sign), + |result| make_instr_ssi(result, lhs, sign), + FuelCostsProvider::base, + ) + } + (Operand::Immediate(lhs), rhs) => { + let lhs = T::from(rhs.val()); + let rhs = self.layout.operand_to_reg(rhs)?; + self.push_instr_with_result( + ::TY, + |result| make_instr_sis(result, lhs, rhs), FuelCostsProvider::base, ) } @@ -2276,7 +2091,7 @@ impl FuncTranslator { ::TY, lhs, rhs, - make_instr, + make_instr_sss, FuelCostsProvider::base, ) } @@ -2362,17 +2177,6 @@ impl FuncTranslator { Ok(()) } - /// Create either [`Op::CallIndirectParams`] or [`Op::CallIndirectParamsImm16`] depending on the inputs. - fn call_indirect_params(&mut self, index: Operand, table_index: u32) -> Result { - let table_type = *self.module.get_type_of_table(TableIdx::from(table_index)); - let index = self.make_index16(index, table_type.index_ty())?; - let instr = match index { - Input::Slot(index) => Op::call_indirect_params(index, table_index), - Input::Immediate(index) => Op::call_indirect_params_imm16(index, table_index), - }; - Ok(instr) - } - /// Tries to fuse a Wasm `i32.eqz` (or `i32.eq` with 0 `rhs` value) instruction. /// /// Returns @@ -2479,61 +2283,53 @@ impl FuncTranslator { &mut self, memarg: MemArg, loaded_ty: ValType, - make_instr: fn(result: Slot, offset_lo: Offset64Lo) -> Op, - make_instr_offset16: fn(result: Slot, ptr: Slot, offset: Offset16) -> Op, - make_instr_at: fn(result: Slot, address: Address32) -> Op, + make_instr_ss: fn(result: Slot, ptr: Slot, offset: u64, memory: index::Memory) -> Op, + make_instr_si: impl Into< + Option Op>, + >, + make_instr_mem0_offset16_ss: fn(result: Slot, ptr: Slot, offset: Offset16) -> Op, ) -> Result<(), Error> { bail_unreachable!(self); let (memory, offset) = Self::decode_memarg(memarg); let ptr = self.stack.pop(); - let (ptr, offset) = match ptr { + let ptr = match ptr { Operand::Immediate(ptr) => { let ptr = ptr.val(); let Some(address) = self.effective_address(memory, ptr, offset) else { return self.translate_trap(TrapCode::MemoryOutOfBounds); }; - if let Ok(address) = Address32::try_from(address) { - self.push_instr_with_result( - loaded_ty, - |result| make_instr_at(result, address), - FuelCostsProvider::load, - )?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; + match make_instr_si.into() { + Some(make_instr_si) => { + self.push_instr_with_result( + loaded_ty, + |result| make_instr_si(result, address, memory), + FuelCostsProvider::load, + )?; + return Ok(()); + } + None => { + let consume_fuel = self.stack.consume_fuel_instr(); + self.copy_operand_to_temp(ptr, consume_fuel)? } - return Ok(()); } - // Case: we cannot use specialized encoding and thus have to fall back - // to the general case where `ptr` is zero and `offset` stores the - // `ptr+offset` address value. - let zero_ptr = self.layout.const_to_reg(0_u64)?; - (zero_ptr, u64::from(address)) - } - ptr => { - let ptr = self.layout.operand_to_reg(ptr)?; - (ptr, offset) } + ptr => self.layout.operand_to_reg(ptr)?, }; if memory.is_default() { if let Ok(offset) = Offset16::try_from(offset) { self.push_instr_with_result( loaded_ty, - |result| make_instr_offset16(result, ptr, offset), + |result| make_instr_mem0_offset16_ss(result, ptr, offset), FuelCostsProvider::load, )?; return Ok(()); } } - let (offset_hi, offset_lo) = Offset64::split(offset); self.push_instr_with_result( loaded_ty, - |result| make_instr(result, offset_lo), + |result| make_instr_ss(result, ptr, offset, memory), FuelCostsProvider::load, )?; - self.push_param(Op::slot_and_offset_hi(ptr, offset_hi))?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } Ok(()) } @@ -2549,247 +2345,14 @@ impl FuncTranslator { /// Used for translating the following Wasm operators to Wasmi bytecode: /// /// - `{i32, i64}.{store, store8, store16, store32}` - fn translate_istore_wrap( - &mut self, - memarg: MemArg, - ) -> Result<(), Error> + fn translate_store(&mut self, memarg: MemArg) -> Result<(), Error> where - T::Value: Copy + Wrap + From, - T::Param: TryFrom + Into, + T::Value: Copy + From, + T::Immediate: Copy, { bail_unreachable!(self); let (ptr, value) = self.stack.pop2(); - self.encode_istore_wrap::(memarg, ptr, value) - } - - /// Encodes Wasm integer `store` and `storeN` instructions as Wasmi bytecode. - fn encode_istore_wrap( - &mut self, - memarg: MemArg, - ptr: Operand, - value: Operand, - ) -> Result<(), Error> - where - T::Value: Copy + Wrap + From, - T::Param: TryFrom + Into, - { - let (memory, offset) = Self::decode_memarg(memarg); - let (ptr, offset) = match ptr { - Operand::Immediate(ptr) => { - let ptr = ptr.val(); - let Some(address) = self.effective_address(memory, ptr, offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; - if let Ok(address) = Address32::try_from(address) { - return self.encode_istore_wrap_at::(memory, address, value); - } - // Case: we cannot use specialized encoding and thus have to fall back - // to the general case where `ptr` is zero and `offset` stores the - // `ptr+offset` address value. - let zero_ptr = self.layout.const_to_reg(0_u64)?; - (zero_ptr, u64::from(address)) - } - ptr => { - let ptr = self.layout.operand_to_reg(ptr)?; - (ptr, offset) - } - }; - if memory.is_default() { - if let Some(_instr) = self.encode_istore_wrap_mem0::(ptr, offset, value)? { - return Ok(()); - } - } - let (offset_hi, offset_lo) = Offset64::split(offset); - let (instr, param) = { - match value { - Operand::Immediate(value) => { - let value = value.val(); - match T::Param::try_from(T::Value::from(value).wrap()).ok() { - Some(value) => ( - T::store_imm(ptr, offset_lo), - Op::imm16_and_offset_hi(value, offset_hi), - ), - None => ( - T::store(ptr, offset_lo), - Op::slot_and_offset_hi(self.layout.const_to_reg(value)?, offset_hi), - ), - } - } - value => { - let value = self.layout.operand_to_reg(value)?; - ( - T::store(ptr, offset_lo), - Op::slot_and_offset_hi(value, offset_hi), - ) - } - } - }; - self.push_instr(instr, FuelCostsProvider::store)?; - self.push_param(param)?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } - Ok(()) - } - - /// Encodes a Wasm integer `store` and `storeN` instructions as Wasmi bytecode. - /// - /// # Note - /// - /// This is used in cases where the `ptr` is a known constant value. - fn encode_istore_wrap_at( - &mut self, - memory: index::Memory, - address: Address32, - value: Operand, - ) -> Result<(), Error> - where - T::Value: Copy + From + Wrap, - T::Param: TryFrom, - { - match value { - Operand::Immediate(value) => { - let value = value.val(); - let wrapped = T::Value::from(value).wrap(); - if let Ok(value) = T::Param::try_from(wrapped) { - self.push_instr(T::store_at_imm(value, address), FuelCostsProvider::store)?; - } else { - let value = self.layout.const_to_reg(value)?; - self.push_instr(T::store_at(value, address), FuelCostsProvider::store)?; - } - } - value => { - let value = self.layout.operand_to_reg(value)?; - self.push_instr(T::store_at(value, address), FuelCostsProvider::store)?; - } - } - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } - Ok(()) - } - - /// Encodes a Wasm integer `store` and `storeN` instructions as Wasmi bytecode. - /// - /// # Note - /// - /// This optimizes for cases where the Wasm linear memory that is operated on is known - /// to be the default memory. - /// Returns `Some` in case the optimized instructions have been encoded. - fn encode_istore_wrap_mem0( - &mut self, - ptr: Slot, - offset: u64, - value: Operand, - ) -> Result, Error> - where - T::Value: Copy + From + Wrap, - T::Param: TryFrom, - { - let Ok(offset16) = Offset16::try_from(offset) else { - return Ok(None); - }; - let instr = match value { - Operand::Immediate(value) => { - let value = value.val(); - let wrapped = T::Value::from(value).wrap(); - match T::Param::try_from(wrapped) { - Ok(value) => self.push_instr( - T::store_offset16_imm(ptr, offset16, value), - FuelCostsProvider::store, - )?, - Err(_) => { - let value = self.layout.const_to_reg(value)?; - self.push_instr( - T::store_offset16(ptr, offset16, value), - FuelCostsProvider::store, - )? - } - } - } - value => { - let value = self.layout.operand_to_reg(value)?; - self.push_instr( - T::store_offset16(ptr, offset16, value), - FuelCostsProvider::store, - )? - } - }; - Ok(Some(instr)) - } - - /// Translates a general Wasm `store` instruction to Wasmi bytecode. - /// - /// # Note - /// - /// This chooses the most efficient encoding for the given `store` instruction. - /// If `ptr+offset` is a constant value the pointer address is pre-calculated. - /// - /// # Usage - /// - /// Used for translating the following Wasm operators to Wasmi bytecode: - /// - /// - `{f32, f64, v128}.store` - fn translate_store( - &mut self, - memarg: MemArg, - store: fn(ptr: Slot, offset_lo: Offset64Lo) -> Op, - store_offset16: fn(ptr: Slot, offset: Offset16, value: Slot) -> Op, - store_at: fn(value: Slot, address: Address32) -> Op, - ) -> Result<(), Error> { - bail_unreachable!(self); - let (memory, offset) = Self::decode_memarg(memarg); - let (ptr, value) = self.stack.pop2(); - let (ptr, offset) = match ptr { - Operand::Immediate(ptr) => { - let Some(address) = self.effective_address(memory, ptr.val(), offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; - if let Ok(address) = Address32::try_from(address) { - return self.encode_fstore_at(memory, address, value, store_at); - } - let zero_ptr = self.layout.const_to_reg(0_u64)?; - (zero_ptr, u64::from(address)) - } - ptr => { - let ptr = self.layout.operand_to_reg(ptr)?; - (ptr, offset) - } - }; - let (offset_hi, offset_lo) = Offset64::split(offset); - let value = self.layout.operand_to_reg(value)?; - if memory.is_default() { - if let Ok(offset) = Offset16::try_from(offset) { - self.push_instr(store_offset16(ptr, offset, value), FuelCostsProvider::store)?; - return Ok(()); - } - } - self.push_instr(store(ptr, offset_lo), FuelCostsProvider::store)?; - self.push_param(Op::slot_and_offset_hi(value, offset_hi))?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } - Ok(()) - } - - /// Encodes a Wasm `store` instruction with immediate address as Wasmi bytecode. - /// - /// # Note - /// - /// This is used in cases where the `ptr` is a known constant value. - fn encode_fstore_at( - &mut self, - memory: index::Memory, - address: Address32, - value: Operand, - make_instr_at: fn(value: Slot, address: Address32) -> Op, - ) -> Result<(), Error> { - let value = self.layout.operand_to_reg(value)?; - self.push_instr(make_instr_at(value, address), FuelCostsProvider::store)?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } - Ok(()) + todo!() } /// Returns the [`MemArg`] linear `memory` index and load/store `offset`. From 80223e364cad27fc60a22019c606170132365ba4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 13:37:32 +0200 Subject: [PATCH 018/423] redesign immediate_to_reg This now always encodes a copy for immediate operands and no longer uses function local constants. --- .../wasmi/src/engine/translator/func/mod.rs | 37 ++++--------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index af9f895887..9f40de083a 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -646,17 +646,13 @@ impl FuncTranslator { } Op::copy(result, value) } - Operand::Immediate(value) => Self::make_copy_imm_instr(result, value.val(), layout)?, + Operand::Immediate(value) => Self::make_copy_imm_instr(result, value.val())?, }; Ok(Some(instr)) } /// Returns the copy instruction to copy the given immediate `value` to `result`. - fn make_copy_imm_instr( - result: Slot, - value: TypedVal, - layout: &mut StackLayout, - ) -> Result { + fn make_copy_imm_instr(result: Slot, value: TypedVal) -> Result { let instr = match value.ty() { ValType::I32 => Op::copy32(result, i32::from(value)), ValType::I64 => Op::copy64(result, i64::from(value)), @@ -945,14 +941,10 @@ impl FuncTranslator { Ok(result) } - /// Efficiently converts the `operand` to a [`Slot`] if it is an immediate. + /// Copies the `operand` if it is an immediate and returns the [`Slot`]. /// /// # Note /// - /// - Preferrably, this encodes the immediate `operand` into a `copy` instruction - /// with the immediate encoded inline. - /// - If the immediate `operand` cannot be encoded as `copy` with inline immediate - /// a function local constant [`Slot`] will be allocated and returned. /// - Returns the associated [`Slot`] if `operand` is an [`Operand::Temp`] or [`Operand::Local`]. fn immediate_to_reg(&mut self, operand: Operand) -> Result { match operand { @@ -961,24 +953,11 @@ impl FuncTranslator { Operand::Immediate(operand) => { let value = operand.val(); let result = self.layout.temp_to_reg(operand.operand_index())?; - match Self::make_copy_imm_instr(result, value, &mut self.layout)? { - Op::Copy { value, .. } => { - // Case: not possible to craft a `copy` instruction - // with an inline immediate, so we can return - // the allocated function local constant [`Slot`] - // instead. - Ok(value) - } - copy_instr => { - let consume_fuel = self.stack.consume_fuel_instr(); - self.instrs.push_instr( - copy_instr, - consume_fuel, - FuelCostsProvider::base, - )?; - Ok(result) - } - } + let copy_instr = Self::make_copy_imm_instr(result, value)?; + let consume_fuel = self.stack.consume_fuel_instr(); + self.instrs + .push_instr(copy_instr, consume_fuel, FuelCostsProvider::base)?; + Ok(result) } } } From 38b49c032095675451daff94a7bdaa1abe87d5b6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 13:37:48 +0200 Subject: [PATCH 019/423] return SlotSpan from move_operand_to_temp --- crates/wasmi/src/engine/translator/func/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 9f40de083a..84b1a0549e 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -378,12 +378,14 @@ impl FuncTranslator { &mut self, len: usize, consume_fuel: Option, - ) -> Result<(), Error> { + ) -> Result { for n in 0..len { let operand = self.stack.operand_to_temp(n); self.copy_operand_to_temp(operand, consume_fuel)?; } - Ok(()) + let first_idx = self.stack.peek(len).index(); + let first = self.layout.temp_to_reg(first_idx)?; + Ok(SlotSpan::new(first)) } /// Convert all branch params up to `depth` to [`Operand::Temp`]. From c32ef6e082ff977b37e058549b052f68859cc7c6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 13:38:31 +0200 Subject: [PATCH 020/423] rename immediate_to_reg -> copy_if_immediate --- .../wasmi/src/engine/translator/func/mod.rs | 6 +-- .../wasmi/src/engine/translator/func/visit.rs | 40 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 84b1a0549e..3f0cfbebc6 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -948,7 +948,7 @@ impl FuncTranslator { /// # Note /// /// - Returns the associated [`Slot`] if `operand` is an [`Operand::Temp`] or [`Operand::Local`]. - fn immediate_to_reg(&mut self, operand: Operand) -> Result { + fn copy_if_immediate(&mut self, operand: Operand) -> Result { match operand { Operand::Local(operand) => self.layout.local_to_reg(operand.local_index()), Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index()), @@ -2134,8 +2134,8 @@ impl FuncTranslator { return Ok(()); } let condition = self.layout.operand_to_reg(condition)?; - let mut true_val = self.immediate_to_reg(true_val)?; - let mut false_val = self.immediate_to_reg(false_val)?; + let mut true_val = self.copy_if_immediate(true_val)?; + let mut false_val = self.copy_if_immediate(false_val)?; match self .instrs .try_fuse_select(ty, condition, &self.layout, &mut self.stack)? diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index b0a06e9edd..c969dc7443 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -749,7 +749,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } } // Case: fallback to generic `memory.grow` instruction - let delta = self.immediate_to_reg(delta)?; + let delta = self.copy_if_immediate(delta)?; self.push_instr_with_result( index_ty.ty(), |result| Op::memory_grow(result, delta, memory), @@ -1838,9 +1838,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (dst, src, len) = self.stack.pop3(); let memory = index::Memory::from(mem); let data = index::Data::from(data_index); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; self.push_instr( Op::memory_init(memory, data, dst, src, len), FuelCostsProvider::instance, @@ -1861,9 +1861,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (dst, src, len) = self.stack.pop3(); let dst_memory = index::Memory::from(dst_mem); let src_memory = index::Memory::from(src_mem); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; self.push_instr( Op::memory_copy(dst_memory, src_memory, dst, src, len), FuelCostsProvider::instance, @@ -1876,12 +1876,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { bail_unreachable!(self); let (dst, value, len) = self.stack.pop3(); let memory = index::Memory::from(mem); - let dst = self.immediate_to_reg(dst)?; + let dst = self.copy_if_immediate(dst)?; let value = self.make_input(value, |_, value| { let byte = u32::from(value) as u8; Ok(Input::Immediate(byte)) })?; - let len = self.immediate_to_reg(len)?; + let len = self.copy_if_immediate(len)?; let instr: Op = match value { Input::Slot(value) => Op::memory_fill(memory, dst, value, len), Input::Immediate(value) => Op::memory_fill_imm(memory, dst, value, len), @@ -1896,9 +1896,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (dst, src, len) = self.stack.pop3(); let table = index::Table::from(table); let elem = index::Elem::from(elem_index); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; self.push_instr( Op::table_init(table, elem, dst, src, len), FuelCostsProvider::instance, @@ -1919,9 +1919,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (dst, src, len) = self.stack.pop3(); let dst_table = index::Table::from(dst_table); let src_table = index::Table::from(src_table); - let dst = self.immediate_to_reg(dst)?; - let src = self.immediate_to_reg(src)?; - let len = self.immediate_to_reg(len)?; + let dst = self.copy_if_immediate(dst)?; + let src = self.copy_if_immediate(src)?; + let len = self.copy_if_immediate(len)?; self.push_instr( Op::table_copy(dst_table, src_table, dst, src, len), FuelCostsProvider::instance, @@ -1995,9 +1995,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { bail_unreachable!(self); let (dst, value, len) = self.stack.pop3(); let table = index::Table::from(table); - let dst = self.immediate_to_reg(dst)?; - let value = self.immediate_to_reg(value)?; - let len = self.immediate_to_reg(len)?; + let dst = self.copy_if_immediate(dst)?; + let value = self.copy_if_immediate(value)?; + let len = self.copy_if_immediate(len)?; self.push_instr( Op::table_fill(table, dst, len, value), FuelCostsProvider::instance, @@ -2069,8 +2069,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { return Ok(()); } } - let value = self.immediate_to_reg(value)?; - let delta = self.immediate_to_reg(delta)?; + let value = self.copy_if_immediate(value)?; + let delta = self.copy_if_immediate(delta)?; self.push_instr_with_result( index_ty.ty(), |result| Op::table_grow(result, delta, value, table), From 609c885fda3be0ac6bfbd5f0b3656565ccbd8aae Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 18:52:01 +0200 Subject: [PATCH 021/423] update docs --- crates/wasmi/src/engine/translator/func/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 3f0cfbebc6..fff41fc131 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -943,7 +943,9 @@ impl FuncTranslator { Ok(result) } - /// Copies the `operand` if it is an immediate and returns the [`Slot`]. + /// Copies the `operand` to its temporary [`Slot`] if it is an immediate. + /// + /// Returns the temporary [`Slot`] of the `operand`. /// /// # Note /// From 5a3f969d080b3b8d4e85d1ace333a6e086a43f3d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 18:52:10 +0200 Subject: [PATCH 022/423] update imports --- crates/wasmi/src/engine/translator/func/mod.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index fff41fc131..14707fe593 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -50,17 +50,7 @@ use crate::{ CompiledFuncEntity, TranslationError, }, - ir::{ - index, - Address, - BoundedSlotSpan, - FixedSlotSpan, - Offset16, - Op, - Sign, - Slot, - SlotSpan, - }, + ir::{index, Address, BoundedSlotSpan, FixedSlotSpan, Offset16, Op, Sign, Slot, SlotSpan}, module::{FuncIdx, FuncTypeIdx, MemoryIdx, ModuleHeader, WasmiValueType}, Engine, Error, From 6f97a095ab414ca4812d37e1c22f16da034ceb52 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 18:52:22 +0200 Subject: [PATCH 023/423] update visit_call_indirect --- .../wasmi/src/engine/translator/func/visit.rs | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index c969dc7443..794a3f81e2 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -381,28 +381,17 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { bail_unreachable!(self); - let func_type = self.resolve_type(type_index); let index = self.stack.pop(); - let indirect_params = self.call_indirect_params(index, table_index)?; + let consume_fuel = self.stack.consume_fuel_instr(); + let table = index::Table::from(table_index); + let func_type = self.resolve_type(type_index); + let index = self.copy_if_immediate(index)?; let len_params = usize::from(func_type.len_params()); - let results = self.call_regspan(len_params)?; - let instr = match (len_params, indirect_params) { - (0, Op::CallIndirectParams { .. }) => Op::call_indirect_0(results, type_index), - (0, Op::CallIndirectParamsImm16 { .. }) => { - Op::call_indirect_0_imm16(results, type_index) - } - (_, Op::CallIndirectParams { .. }) => Op::call_indirect(results, type_index), - (_, Op::CallIndirectParamsImm16 { .. }) => Op::call_indirect_imm16(results, type_index), - _ => unreachable!(), - }; - let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; - self.push_param(indirect_params)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; - if let Some(span) = self.push_results(call_instr, func_type.results())? { - debug_assert_eq!(span, results); - } + let results = self.move_operands_to_temp(len_params, consume_fuel)?; + let call_instr = self.push_instr( + Op::call_indirect(results, type_index, index, table), + FuelCostsProvider::call, + )?; Ok(()) } From 7edb5f95396ccbd8f05d787bf0faf1be184e5c7e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 19:07:18 +0200 Subject: [PATCH 024/423] clean-up WasmInteger utility trait --- crates/wasmi/src/engine/translator/utils.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 47fdb62fcd..3b4c479bff 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -46,17 +46,10 @@ impl_typed_for! { /// /// This trait provides some utility methods useful for translation. pub trait WasmInteger: - Copy - + Eq - + Typed - + From - + Into - + From - + Into - + TryInto> + Copy + Eq + Typed + From + Into + From + Into { /// The non-zero type of the [`WasmInteger`]. - type NonZero: Copy + Into + TryInto> + Into; + type NonZero: Copy + Into + Into; /// Returns `self` as [`Self::NonZero`] if possible. /// From bfebbe391beb1b81ed28f2c2ea1fd0e6a81ff22a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 19:07:25 +0200 Subject: [PATCH 025/423] extend ToBits impls --- crates/wasmi/src/engine/translator/utils.rs | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 3b4c479bff..30d22b09a2 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -261,6 +261,34 @@ pub trait ToBits { fn to_bits(self) -> Self::Out; } +impl ToBits for u8 { + type Out = u8; + fn to_bits(self) -> Self::Out { + self + } +} + +impl ToBits for i8 { + type Out = u8; + fn to_bits(self) -> Self::Out { + u8::from_ne_bytes(self.to_ne_bytes()) + } +} + +impl ToBits for u16 { + type Out = u16; + fn to_bits(self) -> Self::Out { + self + } +} + +impl ToBits for i16 { + type Out = u16; + fn to_bits(self) -> Self::Out { + u16::from_ne_bytes(self.to_ne_bytes()) + } +} + impl ToBits for u32 { type Out = u32; fn to_bits(self) -> Self::Out { From 89b16629a7f64d2219f1642736ff5971516c4510 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 19:07:47 +0200 Subject: [PATCH 026/423] update simd replace.lane translation --- .../src/engine/translator/func/simd/op.rs | 104 ++++++++---------- 1 file changed, 46 insertions(+), 58 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/op.rs b/crates/wasmi/src/engine/translator/func/simd/op.rs index 10702532a1..a29d2dc965 100644 --- a/crates/wasmi/src/engine/translator/func/simd/op.rs +++ b/crates/wasmi/src/engine/translator/func/simd/op.rs @@ -1,6 +1,7 @@ use super::IntoLaneIdx; use crate::{ core::{simd, Typed}, + engine::translator::utils::ToBits, ir::{Op, Slot}, V128, }; @@ -9,24 +10,27 @@ pub trait SimdReplaceLane { type Item: Typed + IntoLaneIdx + Copy; type Immediate: Copy; + fn into_immediate(value: Self::Item) -> Self::Immediate; + fn const_eval( input: V128, lane: ::LaneIdx, value: Self::Item, ) -> V128; - fn replace_lane(result: Slot, input: Slot, lane: ::LaneIdx) -> Op; + fn replace_lane_sss( + result: Slot, + input: Slot, + lane: ::LaneIdx, + value: Slot, + ) -> Op; - fn replace_lane_imm( + fn replace_lane_ssi( result: Slot, input: Slot, lane: ::LaneIdx, value: Self::Immediate, ) -> Op; - - fn replace_lane_imm_param(value: Self::Immediate) -> Option; - - fn value_to_imm(value: Self::Item) -> Option; } macro_rules! impl_replace_lane { @@ -37,10 +41,9 @@ macro_rules! impl_replace_lane { type Immediate = $imm_ty:ty; fn const_eval = $const_eval:expr; - fn replace_lane = $replace_lane:expr; - fn replace_lane_imm = $replace_lane_imm:expr; - fn replace_lane_imm_param = $replace_lane_imm_param:expr; - fn value_to_imm = $value_to_imm:expr; + fn into_immediate = $into_immediate:expr; + fn replace_lane_sss = $replace_lane_sss:expr; + fn replace_lane_ssi = $replace_lane_ssi:expr; } )* ) => { @@ -59,105 +62,90 @@ macro_rules! impl_replace_lane { $const_eval(input, lane, value) } - fn replace_lane( + fn into_immediate(value: Self::Item) -> Self::Immediate { + $into_immediate(value) + } + + fn replace_lane_sss( result: Slot, input: Slot, lane: ::LaneIdx, + value: Slot, ) -> Op { - $replace_lane(result, input, lane) + $replace_lane_sss(result, input, lane, value) } - fn replace_lane_imm( + fn replace_lane_ssi( result: Slot, input: Slot, lane: ::LaneIdx, value: Self::Immediate, ) -> Op { - $replace_lane_imm(result, input, lane, value) - } - - fn replace_lane_imm_param(value: Self::Immediate) -> Option { - $replace_lane_imm_param(value) - } - - fn value_to_imm(value: Self::Item) -> Option { - $value_to_imm(value) + $replace_lane_ssi(result, input, lane, value) } } )* }; } -macro_rules! wrap { - ($f:expr) => { - |result, input, lane, _value| $f(result, input, lane) - }; -} - impl_replace_lane! { impl SimdReplaceLane for I8x16ReplaceLane { type Item = i8; - type Immediate = i8; + type Immediate = u8; fn const_eval = simd::i8x16_replace_lane; - fn replace_lane = Op::i8x16_replace_lane; - fn replace_lane_imm = Op::i8x16_replace_lane_imm; - fn replace_lane_imm_param = |_| None; - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::v128_replace_lane8x16_sss; + fn replace_lane_ssi = Op::v128_replace_lane8x16_ssi; } impl SimdReplaceLane for I16x8ReplaceLane { type Item = i16; - type Immediate = i16; + type Immediate = u16; fn const_eval = simd::i16x8_replace_lane; - fn replace_lane = Op::i16x8_replace_lane; - fn replace_lane_imm = wrap!(Op::i16x8_replace_lane_imm); - fn replace_lane_imm_param = |value| Some(Op::const32(i32::from(value))); - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::i16x8_replace_lane; + fn replace_lane_ssi = Op::i16x8_replace_lane_imm; } impl SimdReplaceLane for I32x4ReplaceLane { type Item = i32; - type Immediate = i32; + type Immediate = u32; fn const_eval = simd::i32x4_replace_lane; - fn replace_lane = Op::i32x4_replace_lane; - fn replace_lane_imm = wrap!(Op::i32x4_replace_lane_imm); - fn replace_lane_imm_param = |value| Some(Op::const32(value)); - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::i32x4_replace_lane; + fn replace_lane_ssi = Op::i32x4_replace_lane_imm; } impl SimdReplaceLane for I64x2ReplaceLane { type Item = i64; - type Immediate = Const32; + type Immediate = u64; fn const_eval = simd::i64x2_replace_lane; - fn replace_lane = Op::i64x2_replace_lane; - fn replace_lane_imm = wrap!(Op::i64x2_replace_lane_imm32); - fn replace_lane_imm_param = |value| Some(Op::i64const32(value)); - fn value_to_imm = |value| >::try_from(value).ok(); + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::i64x2_replace_lane; + fn replace_lane_ssi = Op::i64x2_replace_lane_imm32; } impl SimdReplaceLane for F32x4ReplaceLane { type Item = f32; - type Immediate = f32; + type Immediate = u32; fn const_eval = simd::f32x4_replace_lane; - fn replace_lane = Op::f32x4_replace_lane; - fn replace_lane_imm = wrap!(Op::f32x4_replace_lane_imm); - fn replace_lane_imm_param = |value| Some(Op::const32(value)); - fn value_to_imm = Some; + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::f32x4_replace_lane; + fn replace_lane_ssi = Op::f32x4_replace_lane_imm; } impl SimdReplaceLane for F64x2ReplaceLane { type Item = f64; - type Immediate = Const32; + type Immediate = u64; fn const_eval = simd::f64x2_replace_lane; - fn replace_lane = Op::f64x2_replace_lane; - fn replace_lane_imm = wrap!(Op::f64x2_replace_lane_imm32); - fn replace_lane_imm_param = |value| Some(Op::f64const32(value)); - fn value_to_imm = |value| >::try_from(value).ok(); + fn into_immediate = ::to_bits; + fn replace_lane_sss = Op::f64x2_replace_lane; + fn replace_lane_ssi = Op::f64x2_replace_lane_imm32; } } From 49006e2d199122d47b06de420a3373fe9e7a9dc7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 19:08:03 +0200 Subject: [PATCH 027/423] update simd i8x16.shuffle translation --- crates/wasmi/src/engine/translator/func/simd/visit.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index eb65e29861..009800bd80 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -293,10 +293,9 @@ impl VisitSimdOperator<'_> for FuncTranslator { .const_to_reg(V128::from(u128::from_ne_bytes(lanes)))?; self.push_instr_with_result( ValType::V128, - |result| Op::i8x16_shuffle(result, lhs, rhs), + |result| Op::i8x16_shuffle(result, lhs, rhs, selector), FuelCostsProvider::simd, )?; - self.push_param(Op::slot(selector))?; Ok(()) } From f0243cbc17e1ffda6038a7fd6f728b0ca7f57f94 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 10 Sep 2025 23:21:40 +0200 Subject: [PATCH 028/423] fix compile errors in executor/instrs/binary.rs --- crates/wasmi/src/engine/executor/instrs/binary.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/executor/instrs/binary.rs b/crates/wasmi/src/engine/executor/instrs/binary.rs index d6b92ed4a8..958f5b0a93 100644 --- a/crates/wasmi/src/engine/executor/instrs/binary.rs +++ b/crates/wasmi/src/engine/executor/instrs/binary.rs @@ -68,7 +68,7 @@ macro_rules! impl_binary_imm16 { ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { + pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: $ty) { self.execute_binary_imm16_rhs(result, lhs, rhs, $op) } )* @@ -102,7 +102,7 @@ macro_rules! impl_shift_by { ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: ShiftAmount<$ty>) { + pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: u8) { self.execute_shift_by(result, lhs, rhs, $op) } )* @@ -128,7 +128,7 @@ macro_rules! impl_binary_imm16_lhs { ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Const16<$ty>, rhs: Slot) { + pub fn $fn_name(&mut self, result: Slot, lhs: $ty, rhs: Slot) { self.execute_binary_imm16_lhs(result, lhs, rhs, $op) } )* @@ -244,7 +244,7 @@ macro_rules! impl_divrem_s_imm16_rhs { ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) -> Result<(), Error> { + pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: $ty) -> Result<(), Error> { self.try_execute_divrem_imm16_rhs(result, lhs, rhs, $op) } )* @@ -264,7 +264,7 @@ macro_rules! impl_divrem_u_imm16_rhs { ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { + pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: $ty) { self.execute_divrem_imm16_rhs(result, lhs, rhs, $op) } )* @@ -284,7 +284,7 @@ macro_rules! impl_fallible_binary_imm16_lhs { ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Const16<$ty>, rhs: Slot) -> Result<(), Error> { + pub fn $fn_name(&mut self, result: Slot, lhs: $ty, rhs: Slot) -> Result<(), Error> { self.try_execute_binary_imm16_lhs(result, lhs, rhs, $op).map_err(Error::from) } )* From 88ca0d087bc6c4bbd3b2054b4d51109401cd46c7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 11:51:52 +0200 Subject: [PATCH 029/423] update v128.splat operators translation --- .../src/engine/translator/func/simd/mod.rs | 23 ++++++++++--------- .../src/engine/translator/func/simd/visit.rs | 12 +++++----- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index fa482d8a50..86ae9e2162 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -7,7 +7,7 @@ use crate::{ core::{simd::IntoLaneIdx, FuelCostsProvider, Typed, TypedVal}, engine::translator::{ func::{utils::Input, Operand}, - utils::{Instr, Wrap}, + utils::{Instr, ToBits, Wrap}, }, ir::{ index::{self, Memory}, @@ -25,24 +25,25 @@ impl FuncTranslator { /// Generically translate any of the Wasm `simd` splat instructions. fn translate_simd_splat( &mut self, - make_instr: fn(result: Slot, value: Slot) -> Op, - const_eval: fn(Wrapped) -> V128, + make_instr_ss: fn(result: Slot, value: Slot) -> Op, + make_instr_si: fn(result: Slot, value: ::Out) -> Op, ) -> Result<(), Error> where T: From + Wrap, + Wrapped: ToBits, { bail_unreachable!(self); let value = self.stack.pop(); - if let Operand::Immediate(value) = value { - let value = T::from(value.val()).wrap(); - let result = const_eval(value); - self.stack.push_immediate(result)?; - return Ok(()); - }; - let value = self.layout.operand_to_reg(value)?; + let value: Input = self.make_input(value, |_this, value| Ok(value))?; self.push_instr_with_result( ValType::V128, - |result| make_instr(result, value), + |result| match value { + Input::Slot(value) => make_instr_si(result, value), + Input::Immediate(value) => { + let value = T::from(value).wrap().to_bits(); + make_instr_si(result, value) + } + }, FuelCostsProvider::simd, )?; Ok(()) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 009800bd80..4371c99658 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -392,27 +392,27 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i8x16_splat, simd::i8x16_splat) + self.translate_simd_splat::(Op::v128_splat8_ss, Op::v128_splat8_si) } fn visit_i16x8_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i16x8_splat, simd::i16x8_splat) + self.translate_simd_splat::(Op::v128_splat16_ss, Op::v128_splat16_si) } fn visit_i32x4_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i32x4_splat, simd::i32x4_splat) + self.translate_simd_splat::(Op::v128_splat32_ss, Op::v128_splat32_si) } fn visit_i64x2_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::i64x2_splat, simd::i64x2_splat) + self.translate_simd_splat::(Op::v128_splat64_ss, Op::v128_splat64_si) } fn visit_f32x4_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::f32x4_splat, simd::f32x4_splat) + self.translate_simd_splat::(Op::v128_splat32_ss, Op::v128_splat32_si) } fn visit_f64x2_splat(&mut self) -> Self::Output { - self.translate_simd_splat::(Op::f64x2_splat, simd::f64x2_splat) + self.translate_simd_splat::(Op::v128_splat64_ss, Op::v128_splat64_si) } fn visit_i8x16_eq(&mut self) -> Self::Output { From 72c14370e16f6511971a210051ed118e2d23360d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 11:54:14 +0200 Subject: [PATCH 030/423] update remaining replace_lane operator translation --- .../src/engine/translator/func/simd/op.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/op.rs b/crates/wasmi/src/engine/translator/func/simd/op.rs index a29d2dc965..8ab9583bd6 100644 --- a/crates/wasmi/src/engine/translator/func/simd/op.rs +++ b/crates/wasmi/src/engine/translator/func/simd/op.rs @@ -105,8 +105,8 @@ impl_replace_lane! { fn const_eval = simd::i16x8_replace_lane; fn into_immediate = ::to_bits; - fn replace_lane_sss = Op::i16x8_replace_lane; - fn replace_lane_ssi = Op::i16x8_replace_lane_imm; + fn replace_lane_sss = Op::v128_replace_lane16x8_sss; + fn replace_lane_ssi = Op::v128_replace_lane16x8_ssi; } impl SimdReplaceLane for I32x4ReplaceLane { @@ -115,8 +115,8 @@ impl_replace_lane! { fn const_eval = simd::i32x4_replace_lane; fn into_immediate = ::to_bits; - fn replace_lane_sss = Op::i32x4_replace_lane; - fn replace_lane_ssi = Op::i32x4_replace_lane_imm; + fn replace_lane_sss = Op::v128_replace_lane32x4_sss; + fn replace_lane_ssi = Op::v128_replace_lane32x4_ssi; } impl SimdReplaceLane for I64x2ReplaceLane { @@ -125,8 +125,8 @@ impl_replace_lane! { fn const_eval = simd::i64x2_replace_lane; fn into_immediate = ::to_bits; - fn replace_lane_sss = Op::i64x2_replace_lane; - fn replace_lane_ssi = Op::i64x2_replace_lane_imm32; + fn replace_lane_sss = Op::v128_replace_lane64x2_sss; + fn replace_lane_ssi = Op::v128_replace_lane64x2_ssi; } impl SimdReplaceLane for F32x4ReplaceLane { @@ -135,8 +135,8 @@ impl_replace_lane! { fn const_eval = simd::f32x4_replace_lane; fn into_immediate = ::to_bits; - fn replace_lane_sss = Op::f32x4_replace_lane; - fn replace_lane_ssi = Op::f32x4_replace_lane_imm; + fn replace_lane_sss = Op::v128_replace_lane32x4_sss; + fn replace_lane_ssi = Op::v128_replace_lane32x4_ssi; } impl SimdReplaceLane for F64x2ReplaceLane { @@ -145,7 +145,7 @@ impl_replace_lane! { fn const_eval = simd::f64x2_replace_lane; fn into_immediate = ::to_bits; - fn replace_lane_sss = Op::f64x2_replace_lane; - fn replace_lane_ssi = Op::f64x2_replace_lane_imm32; + fn replace_lane_sss = Op::v128_replace_lane64x2_sss; + fn replace_lane_ssi = Op::v128_replace_lane64x2_ssi; } } From 7c251f73cfbc67df9086a309c1b94fb8186dc2c4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 12:55:34 +0200 Subject: [PATCH 031/423] fix i64_mul_wide translation --- crates/wasmi/src/engine/translator/func/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 794a3f81e2..cfcfa6e011 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -2143,7 +2143,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_mul_wide_s(&mut self) -> Self::Output { - self.translate_i64_mul_wide_sx(Op::s64_mul_wide, wasm::i64_mul_wide_s, true) + self.translate_i64_mul_wide_sx(Op::i64_mul_wide, wasm::i64_mul_wide_s, true) } #[inline(never)] From 0f84f6daa2069e1c01cdff5f1082ddb71c4842c1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 12:56:14 +0200 Subject: [PATCH 032/423] update [return_]call_indirect translation --- .../wasmi/src/engine/translator/func/mod.rs | 28 ++++++++++++++++ .../wasmi/src/engine/translator/func/visit.rs | 33 ++----------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 14707fe593..dd24438ed2 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -1631,6 +1631,34 @@ impl FuncTranslator { Ok(Some(fused)) } + /// Generically translates a `call_indirect` or `return_call_indirect` Wasm operator. + fn translate_call_indirect( + &mut self, + type_index: u32, + table_index: u32, + make_instr: fn( + params: BoundedSlotSpan, + index: Slot, + func_type: FuncType, + table: index::Table, + ) -> Op, + ) -> Result<(), Error> { + bail_unreachable!(self); + let index = self.stack.pop(); + let consume_fuel = self.stack.consume_fuel_instr(); + let table = index::Table::from(table_index); + let func_type = self.resolve_type(type_index); + let index = self.copy_if_immediate(index)?; + let len_params = func_type.len_params(); + let params = self.move_operands_to_temp(usize::from(len_params), consume_fuel)?; + let params = BoundedSlotSpan::new(params, len_params); + self.push_instr( + make_instr(params, index, index::FuncType::from(type_index), table), + FuelCostsProvider::call, + )?; + Ok(()) + } + /// Translates a unary Wasm instruction to Wasmi bytecode. fn translate_unary( &mut self, diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index cfcfa6e011..fbeab8c4fe 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -380,19 +380,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { - bail_unreachable!(self); - let index = self.stack.pop(); - let consume_fuel = self.stack.consume_fuel_instr(); - let table = index::Table::from(table_index); - let func_type = self.resolve_type(type_index); - let index = self.copy_if_immediate(index)?; - let len_params = usize::from(func_type.len_params()); - let results = self.move_operands_to_temp(len_params, consume_fuel)?; - let call_instr = self.push_instr( - Op::call_indirect(results, type_index, index, table), - FuelCostsProvider::call, - )?; - Ok(()) + self.translate_call_indirect(type_index, table_index, Op::call_indirect) } #[inline(never)] @@ -2109,24 +2097,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_return_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { - bail_unreachable!(self); - let func_type = self.resolve_type(type_index); - let table = index::Table::from(table_index); - let index_ty = self - .module - .get_type_of_table(TableIdx::from(table_index)) - .index_ty(); - let index = self.stack.pop(); - let len_params = usize::from(func_type.len_params()); - let consume_fuel = self.stack.consume_fuel_instr(); - self.move_operands_to_temp(len_params, consume_fuel)?; - let index = self.make_index32(index, index_ty)?; - let instr = match index { - Input::Slot(index) => Op::return_call_indirect(index, type_index, table), - Input::Immediate(index) => Op::return_call_indirect_imm16(index, type_index, table), - _ => unreachable!(), - }; - self.push_instr(instr, FuelCostsProvider::call)?; + self.translate_call_indirect(type_index, table_index, Op::return_call_indirect)?; self.reachable = false; Ok(()) } From 4ac4476a10f518499bb2f2826cc15331c6f13b40 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 13:00:19 +0200 Subject: [PATCH 033/423] update call and return_call translation --- .../wasmi/src/engine/translator/func/mod.rs | 29 ++++++++++ .../wasmi/src/engine/translator/func/visit.rs | 58 +++---------------- 2 files changed, 36 insertions(+), 51 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index dd24438ed2..937f1dcc09 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -1631,6 +1631,35 @@ impl FuncTranslator { Ok(Some(fused)) } + /// Generically translates a `call` or `return_call` Wasm operator. + fn translate_call( + &mut self, + function_index: u32, + call_internal: fn(params: BoundedSlotSpan, func: index::InternalFunc) -> Op, + call_imported: fn(params: BoundedSlotSpan, func: index::Func) -> Op, + ) -> Result<(), Error> { + bail_unreachable!(self); + let consume_fuel = self.stack.consume_fuel_instr(); + let func_idx = FuncIdx::from(function_index); + let len_params = self.resolve_func_type(func_idx).len_params(); + let params = self.move_operands_to_temp(usize::from(len_params), consume_fuel)?; + let params = BoundedSlotSpan::new(params, len_params); + let instr = match self.module.get_engine_func(func_idx) { + Some(engine_func) => { + // Case: We are calling an internal function and can optimize + // this case by using the special instruction for it. + call_internal(params, index::InternalFunc::from(engine_func)) + } + None => { + // Case: We are calling an imported function and must use the + // general calling operator for it. + call_imported(params, index::Func::from(function_index)) + } + }; + let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; + Ok(()) + } + /// Generically translates a `call_indirect` or `return_call_indirect` Wasm operator. fn translate_call_indirect( &mut self, diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index fbeab8c4fe..39d1aaf45a 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -11,7 +11,7 @@ use crate::{ }, BlockType, }, - ir::{self, index, Op}, + ir::{self, index, BoundedSlotSpan, Op}, module::{self, FuncIdx, MemoryIdx, TableIdx, WasmiValueType}, Error, ExternRef, @@ -345,37 +345,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_call(&mut self, function_index: u32) -> Self::Output { - bail_unreachable!(self); - let func_idx = FuncIdx::from(function_index); - let func_type = self.resolve_func_type(func_idx); - let len_params = usize::from(func_type.len_params()); - let results = self.call_regspan(len_params)?; - let instr = match self.module.get_engine_func(func_idx) { - Some(engine_func) => { - // Case: We are calling an internal function and can optimize - // this case by using the special instruction for it. - match len_params { - 0 => Op::call_internal_0(results, engine_func), - _ => Op::call_internal(results, engine_func), - } - } - None => { - // Case: We are calling an imported function and must use the - // general calling operator for it. - match len_params { - 0 => Op::call_imported_0(results, function_index), - _ => Op::call_imported(results, function_index), - } - } - }; - let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; - self.stack.pop_n(len_params, &mut self.operands); - self.instrs - .encode_register_list(&self.operands, &mut self.layout)?; - if let Some(span) = self.push_results(call_instr, func_type.results())? { - debug_assert_eq!(span, results); - } - Ok(()) + self.translate_call(function_index, Op::call_internal, Op::call_imported) } #[inline(never)] @@ -2072,25 +2042,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_return_call(&mut self, function_index: u32) -> Self::Output { - bail_unreachable!(self); - let func_idx = FuncIdx::from(function_index); - let func_type = self.resolve_func_type(func_idx); - let len_params = usize::from(func_type.len_params()); - let consume_fuel = self.stack.consume_fuel_instr(); - self.move_operands_to_temp(len_params, consume_fuel)?; - let instr = match self.module.get_engine_func(func_idx) { - Some(engine_func) => { - // Case: We are calling an internal function and can optimize - // this case by using the special instruction for it. - Op::return_call_internal(engine_func) - } - None => { - // Case: We are calling an imported function and must use the - // general calling operator for it. - Op::return_call_imported(function_index) - } - }; - self.push_instr(instr, FuelCostsProvider::call)?; + self.translate_call( + function_index, + Op::return_call_internal, + Op::return_call_imported, + )?; self.reachable = false; Ok(()) } From 5db9a4e9af6b8871a737a92e7b236f58a64db228 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 13:01:31 +0200 Subject: [PATCH 034/423] remove unused imports --- crates/wasmi/src/engine/translator/func/visit.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 39d1aaf45a..92caece79c 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -11,8 +11,8 @@ use crate::{ }, BlockType, }, - ir::{self, index, BoundedSlotSpan, Op}, - module::{self, FuncIdx, MemoryIdx, TableIdx, WasmiValueType}, + ir::{self, index, Op}, + module::{self, MemoryIdx, TableIdx, WasmiValueType}, Error, ExternRef, Func, From 284e9ab88c2bfd1b9ce010bd900d0e0e7f9c35b7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 13:05:53 +0200 Subject: [PATCH 035/423] fix bit{and,or,xor} translation --- .../wasmi/src/engine/translator/func/visit.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 92caece79c..4f64267a10 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1137,8 +1137,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bit_and_sss, - Op::i32_bit_and_ssi, + Op::i32_bitand_sss, + Op::i32_bitand_ssi, wasm::i32_bitand, FuncTranslator::no_opt_ri, ) @@ -1147,8 +1147,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bit_or_sss, - Op::i32_bit_or_ssi, + Op::i32_bitor_sss, + Op::i32_bitor_ssi, wasm::i32_bitor, FuncTranslator::no_opt_ri, ) @@ -1157,8 +1157,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i32_bit_xor_sss, - Op::i32_bit_xor_ssi, + Op::i32_bitxor_sss, + Op::i32_bitxor_ssi, wasm::i32_bitxor, FuncTranslator::no_opt_ri, ) @@ -1302,8 +1302,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_and(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bit_and_sss, - Op::i64_bit_and_ssi, + Op::i64_bitand_sss, + Op::i64_bitand_ssi, wasm::i64_bitand, FuncTranslator::no_opt_ri, ) @@ -1312,8 +1312,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_or(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bit_or_sss, - Op::i64_bit_or_ssi, + Op::i64_bitor_sss, + Op::i64_bitor_ssi, wasm::i64_bitor, FuncTranslator::no_opt_ri, ) @@ -1322,8 +1322,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_xor(&mut self) -> Self::Output { self.translate_binary_commutative::( - Op::i64_bit_xor_sss, - Op::i64_bit_xor_ssi, + Op::i64_bitxor_sss, + Op::i64_bitxor_ssi, wasm::i64_bitxor, FuncTranslator::no_opt_ri, ) From c62a188f8082d1e60da02407c730f2b37fb606a5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 13:06:03 +0200 Subject: [PATCH 036/423] flag visit_v128_store impl as todo --- .../wasmi/src/engine/translator/func/simd/visit.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 4371c99658..2a5388c4f9 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -161,12 +161,13 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { - self.translate_store( - memarg, - Op::v128_store, - Op::v128_store_offset16, - Op::v128_store_at, - ) + // self.translate_store( + // memarg, + // Op::v128_store, + // Op::v128_store_offset16, + // Op::v128_store_at, + // ) + todo!() } fn visit_v128_load8_lane(&mut self, memarg: MemArg, lane: u8) -> Self::Output { From ea452cca9d288de3df6e2b6059ac0daa5b7a85ce Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 15:02:37 +0200 Subject: [PATCH 037/423] update simd load_lane translation --- .../src/engine/translator/func/simd/mod.rs | 62 +++++++++---------- .../src/engine/translator/func/simd/visit.rs | 16 ++--- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 86ae9e2162..b68a54f889 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -11,6 +11,8 @@ use crate::{ }, ir::{ index::{self, Memory}, + Address, + Offset16, Op, Slot, }, @@ -247,49 +249,45 @@ impl FuncTranslator { &mut self, memarg: MemArg, lane: u8, - make_instr: fn(result: Slot, offset_lo: Offset64Lo) -> Op, - make_instr_at: fn(result: Slot, address: Address32) -> Op, + load_lane: fn( + result: Slot, + ptr: Slot, + offset: u64, + memory: index::Memory, + v128: Slot, + lane: T::LaneIdx, + ) -> Op, + load_lane_mem0_offset16: fn( + result: Slot, + address: Address, + memory: index::Memory, + v128: Slot, + lane: T::LaneIdx, + ) -> Op, ) -> Result<(), Error> { bail_unreachable!(self); let (memory, offset) = Self::decode_memarg(memarg); let Ok(lane) = ::try_from(lane) else { panic!("encountered out of bounds lane: {lane}"); }; - let (ptr, x) = self.stack.pop2(); - let x = self.layout.operand_to_reg(x)?; - let (ptr, offset) = match ptr { - Operand::Immediate(ptr) => { - let Some(address) = self.effective_address(memory, ptr.val(), offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; - if let Ok(address) = Address32::try_from(address) { - return self.translate_v128_load_lane_at::( - memory, - x, - lane, - address, - make_instr_at, - ); - } - let zero_ptr = self.layout.const_to_reg(0_u64)?; - (zero_ptr, u64::from(address)) - } - ptr => { - let ptr = self.layout.operand_to_reg(ptr)?; - (ptr, offset) + let (ptr, v128) = self.stack.pop2(); + let ptr = self.copy_if_immediate(ptr)?; + let v128 = self.copy_if_immediate(v128)?; + if memory.is_default() { + if let Ok(offset) = Offset16::try_from(offset) { + self.push_instr_with_result( + ::TY, + |result| load_lane_mem0_offset16(result, ptr, offset, v128, lane), + FuelCostsProvider::load, + )?; + return Ok(()); } - }; - let (offset_hi, offset_lo) = Offset64::split(offset); + } self.push_instr_with_result( ::TY, - |result| make_instr(result, offset_lo), + |result| load_lane(result, ptr, offset, memory, v128, lane), FuelCostsProvider::load, )?; - self.push_param(Op::slot_and_offset_hi(ptr, offset_hi))?; - self.push_param(Op::slot_and_lane(x, lane))?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 2a5388c4f9..d318980412 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -174,8 +174,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load8_lane, - Op::v128_load8_lane_at, + Op::v128_load_lane8_sss, + Op::v128_load_lane8_mem0_offset16_sss, ) } @@ -183,8 +183,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load16_lane, - Op::v128_load16_lane_at, + Op::v128_load_lane16_sss, + Op::v128_load_lane16_mem0_offset16_sss, ) } @@ -192,8 +192,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load32_lane, - Op::v128_load32_lane_at, + Op::v128_load_lane32_sss, + Op::v128_load_lane32_mem0_offset16_sss, ) } @@ -201,8 +201,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_load_lane::( memarg, lane, - Op::v128_load64_lane, - Op::v128_load64_lane_at, + Op::v128_load_lane64_sss, + Op::v128_load_lane64_mem0_offset16_sss, ) } From 45a00f917d5013ef5e85ab4afbdc2c8443e4d9d4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 15:03:51 +0200 Subject: [PATCH 038/423] remove no longer used translate_v128_load_lane_at method --- .../src/engine/translator/func/simd/mod.rs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index b68a54f889..eca4447069 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -291,29 +291,6 @@ impl FuncTranslator { Ok(()) } - fn translate_v128_load_lane_at( - &mut self, - memory: Memory, - x: Slot, - lane: LaneType, - address: Address32, - make_instr_at: fn(result: Slot, address: Address32) -> Op, - ) -> Result<(), Error> - where - LaneType: Into, - { - self.push_instr_with_result( - ::TY, - |result| make_instr_at(result, address), - FuelCostsProvider::load, - )?; - self.push_param(Op::slot_and_lane(x, lane))?; - if !memory.is_default() { - self.push_param(Op::memory_index(memory))?; - } - Ok(()) - } - #[allow(clippy::type_complexity)] fn translate_v128_store_lane( &mut self, From a080618478c3d2e34f27d8e83bc729e72c05686a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 23:31:05 +0200 Subject: [PATCH 039/423] fix bugs in comparator.rs --- .../wasmi/src/engine/translator/comparator.rs | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index ee9063bc5b..d3841bb5e7 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -247,17 +247,17 @@ impl LogicalizeCmpInstr for Op { // Bitwise -> Logical: i32 | Op::I32BitAnd_Sss { result, lhs, rhs } => Op::i32_and_sss(result, lhs, rhs), | Op::I32BitOr_Sss { result, lhs, rhs } => Op::i32_or_sss(result, lhs, rhs), - | Op::I32BitXor_Sss { result, lhs, rhs } => Op::i32_ne_sss(result, lhs, rhs), + | Op::I32BitXor_Sss { result, lhs, rhs } => Op::i32_not_eq_sss(result, lhs, rhs), | Op::I32BitAnd_Ssi { result, lhs, rhs } => Op::i32_and_ssi(result, lhs, rhs), | Op::I32BitOr_Ssi { result, lhs, rhs } => Op::i32_or_ssi(result, lhs, rhs), - | Op::I32BitXor_Ssi { result, lhs, rhs } => Op::i32_ne_ssi(result, lhs, rhs), + | Op::I32BitXor_Ssi { result, lhs, rhs } => Op::i32_not_eq_ssi(result, lhs, rhs), // Bitwise -> Logical: i64 | Op::I64BitAnd_Sss { result, lhs, rhs } => Op::i64_and_sss(result, lhs, rhs), | Op::I64BitOr_Sss { result, lhs, rhs } => Op::i64_or_sss(result, lhs, rhs), - | Op::I64BitXor_Sss { result, lhs, rhs } => Op::i64_ne_sss(result, lhs, rhs), + | Op::I64BitXor_Sss { result, lhs, rhs } => Op::i64_not_eq_sss(result, lhs, rhs), | Op::I64BitAnd_Ssi { result, lhs, rhs } => Op::i64_and_ssi(result, lhs, rhs), | Op::I64BitOr_Ssi { result, lhs, rhs } => Op::i64_or_ssi(result, lhs, rhs), - | Op::I64BitXor_Ssi { result, lhs, rhs } => Op::i64_ne_ssi(result, lhs, rhs), + | Op::I64BitXor_Ssi { result, lhs, rhs } => Op::i64_not_eq_ssi(result, lhs, rhs), // Logical -> Logical // i32 | Op::I32Eq_Sss { .. } @@ -392,11 +392,11 @@ impl TryIntoCmpSelectInstr for Op { | Op::I32BitXor_Sss { lhs, rhs, .. } => Op::select_i32_eq_sss(result, lhs, rhs, val_false, val_true), | Op::I32NotEq_Ssi { lhs, rhs, .. } | Op::I32BitXor_Ssi { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, lhs, rhs, val_false, val_true), - | Op::I32Eq_Sss { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::I32Eq_Sss { lhs, rhs, .. } => Op::select_i32_eq_sss(result, lhs, rhs, val_true, val_false), | Op::I32Eq_Ssi { lhs, rhs, .. } => Op::select_i32_eq_ssi(result, lhs, rhs, val_true, val_false), - | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::select_i32_and_ssi(result, lhs, rhs, val_false, val_true), + | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::select_i32_and_sss(result, lhs, rhs, val_false, val_true), | Op::I32NotAnd_Ssi { lhs, rhs, .. } => Op::select_i32_and_ssi(result, lhs, rhs, val_false, val_true), - | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::select_i32_or_ssi(result, lhs, rhs, val_false, val_true), + | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::select_i32_or_sss(result, lhs, rhs, val_false, val_true), | Op::I32NotOr_Ssi { lhs, rhs, .. } => Op::select_i32_or_ssi(result, lhs, rhs, val_false, val_true), | Op::I32Lt_Sss { lhs, rhs, .. } => Op::select_i32_lt_sss(result, lhs, rhs, val_true, val_false), | Op::I32Lt_Ssi { lhs, rhs, .. } => Op::select_i32_lt_ssi(result, lhs, rhs, val_true, val_false), @@ -423,11 +423,11 @@ impl TryIntoCmpSelectInstr for Op { | Op::I64BitXor_Sss { lhs, rhs, .. } => Op::select_i64_eq_sss(result, lhs, rhs, val_false, val_true), | Op::I64NotEq_Ssi { lhs, rhs, .. } | Op::I64BitXor_Ssi { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, lhs, rhs, val_false, val_true), - | Op::I64Eq_Sss { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, lhs, rhs, val_true, val_false), + | Op::I64Eq_Sss { lhs, rhs, .. } => Op::select_i64_eq_sss(result, lhs, rhs, val_true, val_false), | Op::I64Eq_Ssi { lhs, rhs, .. } => Op::select_i64_eq_ssi(result, lhs, rhs, val_true, val_false), - | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::select_i64_and_ssi(result, lhs, rhs, val_false, val_true), + | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::select_i64_and_sss(result, lhs, rhs, val_false, val_true), | Op::I64NotAnd_Ssi { lhs, rhs, .. } => Op::select_i64_and_ssi(result, lhs, rhs, val_false, val_true), - | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::select_i64_or_ssi(result, lhs, rhs, val_false, val_true), + | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::select_i64_or_sss(result, lhs, rhs, val_false, val_true), | Op::I64NotOr_Ssi { lhs, rhs, .. } => Op::select_i64_or_ssi(result, lhs, rhs, val_false, val_true), | Op::I64Lt_Sss { lhs, rhs, .. } => Op::select_i64_lt_sss(result, lhs, rhs, val_true, val_false), | Op::I64Lt_Ssi { lhs, rhs, .. } => Op::select_i64_lt_ssi(result, lhs, rhs, val_true, val_false), @@ -490,7 +490,7 @@ impl TryIntoCmpBranchInstr for Op { #[rustfmt::skip] let cmp_branch_instr = match *self { // i32 - | Op::I32Eq_Sss { lhs, rhs, .. } => Op::branch_i32_eq_si(lhs, rhs, offset), + | Op::I32Eq_Sss { lhs, rhs, .. } => Op::branch_i32_eq_ss(lhs, rhs, offset), | Op::I32Eq_Ssi { lhs, rhs, .. } => Op::branch_i32_eq_si(lhs, rhs, offset), | Op::I32And_Sss { lhs, rhs, .. } | Op::I32BitAnd_Sss { lhs, rhs, .. } => Op::branch_i32_and_ss(lhs, rhs, offset), @@ -504,24 +504,24 @@ impl TryIntoCmpBranchInstr for Op { | Op::I32BitXor_Sss { lhs, rhs, .. } => Op::branch_i32_not_eq_ss(lhs, rhs, offset), | Op::I32NotEq_Ssi { lhs, rhs, .. } | Op::I32BitXor_Ssi { lhs, rhs, .. } => Op::branch_i32_not_eq_si(lhs, rhs, offset), - | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::branch_i32_not_and_si(lhs, rhs, offset), + | Op::I32NotAnd_Sss { lhs, rhs, .. } => Op::branch_i32_not_and_ss(lhs, rhs, offset), | Op::I32NotAnd_Ssi { lhs, rhs, .. } => Op::branch_i32_not_and_si(lhs, rhs, offset), - | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::branch_i32_not_or_si(lhs, rhs, offset), + | Op::I32NotOr_Sss { lhs, rhs, .. } => Op::branch_i32_not_or_ss(lhs, rhs, offset), | Op::I32NotOr_Ssi { lhs, rhs, .. } => Op::branch_i32_not_or_si(lhs, rhs, offset), | Op::I32Lt_Sss { lhs, rhs, .. } => Op::branch_i32_lt_ss(lhs, rhs, offset), | Op::I32Lt_Ssi { lhs, rhs, .. } => Op::branch_i32_lt_si(lhs, rhs, offset), - | Op::I32Lt_Sis { lhs, rhs, .. } => Op::branch_i32_lt_is(rhs, lhs, offset), + | Op::I32Lt_Sis { lhs, rhs, .. } => Op::branch_i32_lt_is(lhs, rhs, offset), | Op::U32Lt_Sss { lhs, rhs, .. } => Op::branch_u32_lt_ss(lhs, rhs, offset), | Op::U32Lt_Ssi { lhs, rhs, .. } => Op::branch_u32_lt_si(lhs, rhs, offset), - | Op::U32Lt_Sis { lhs, rhs, .. } => Op::branch_u32_lt_is(rhs, lhs, offset), + | Op::U32Lt_Sis { lhs, rhs, .. } => Op::branch_u32_lt_is(lhs, rhs, offset), | Op::I32Le_Sss { lhs, rhs, .. } => Op::branch_i32_le_ss(lhs, rhs, offset), | Op::I32Le_Ssi { lhs, rhs, .. } => Op::branch_i32_le_si(lhs, rhs, offset), - | Op::I32Le_Sis { lhs, rhs, .. } => Op::branch_i32_le_is(rhs, lhs, offset), + | Op::I32Le_Sis { lhs, rhs, .. } => Op::branch_i32_le_is(lhs, rhs, offset), | Op::U32Le_Sss { lhs, rhs, .. } => Op::branch_u32_le_ss(lhs, rhs, offset), | Op::U32Le_Ssi { lhs, rhs, .. } => Op::branch_u32_le_si(lhs, rhs, offset), - | Op::U32Le_Sis { lhs, rhs, .. } => Op::branch_u32_le_is(rhs, lhs, offset), + | Op::U32Le_Sis { lhs, rhs, .. } => Op::branch_u32_le_is(lhs, rhs, offset), // i64 - | Op::I64Eq_Sss { lhs, rhs, .. } => Op::branch_i64_eq_si(lhs, rhs, offset), + | Op::I64Eq_Sss { lhs, rhs, .. } => Op::branch_i64_eq_ss(lhs, rhs, offset), | Op::I64Eq_Ssi { lhs, rhs, .. } => Op::branch_i64_eq_si(lhs, rhs, offset), | Op::I64And_Sss { lhs, rhs, .. } | Op::I64BitAnd_Sss { lhs, rhs, .. } => Op::branch_i64_and_ss(lhs, rhs, offset), @@ -535,22 +535,22 @@ impl TryIntoCmpBranchInstr for Op { | Op::I64BitXor_Sss { lhs, rhs, .. } => Op::branch_i64_not_eq_ss(lhs, rhs, offset), | Op::I64NotEq_Ssi { lhs, rhs, .. } | Op::I64BitXor_Ssi { lhs, rhs, .. } => Op::branch_i64_not_eq_si(lhs, rhs, offset), - | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::branch_i64_not_and_si(lhs, rhs, offset), + | Op::I64NotAnd_Sss { lhs, rhs, .. } => Op::branch_i64_not_and_ss(lhs, rhs, offset), | Op::I64NotAnd_Ssi { lhs, rhs, .. } => Op::branch_i64_not_and_si(lhs, rhs, offset), - | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::branch_i64_not_or_si(lhs, rhs, offset), + | Op::I64NotOr_Sss { lhs, rhs, .. } => Op::branch_i64_not_or_ss(lhs, rhs, offset), | Op::I64NotOr_Ssi { lhs, rhs, .. } => Op::branch_i64_not_or_si(lhs, rhs, offset), | Op::I64Lt_Sss { lhs, rhs, .. } => Op::branch_i64_lt_ss(lhs, rhs, offset), | Op::I64Lt_Ssi { lhs, rhs, .. } => Op::branch_i64_lt_si(lhs, rhs, offset), - | Op::I64Lt_Sis { lhs, rhs, .. } => Op::branch_i64_lt_is(rhs, lhs, offset), + | Op::I64Lt_Sis { lhs, rhs, .. } => Op::branch_i64_lt_is(lhs, rhs, offset), | Op::U64Lt_Sss { lhs, rhs, .. } => Op::branch_u64_lt_ss(lhs, rhs, offset), | Op::U64Lt_Ssi { lhs, rhs, .. } => Op::branch_u64_lt_si(lhs, rhs, offset), - | Op::U64Lt_Sis { lhs, rhs, .. } => Op::branch_u64_lt_is(rhs, lhs, offset), + | Op::U64Lt_Sis { lhs, rhs, .. } => Op::branch_u64_lt_is(lhs, rhs, offset), | Op::I64Le_Sss { lhs, rhs, .. } => Op::branch_i64_le_ss(lhs, rhs, offset), | Op::I64Le_Ssi { lhs, rhs, .. } => Op::branch_i64_le_si(lhs, rhs, offset), - | Op::I64Le_Sis { lhs, rhs, .. } => Op::branch_i64_le_is(rhs, lhs, offset), + | Op::I64Le_Sis { lhs, rhs, .. } => Op::branch_i64_le_is(lhs, rhs, offset), | Op::U64Le_Sss { lhs, rhs, .. } => Op::branch_u64_le_ss(lhs, rhs, offset), | Op::U64Le_Ssi { lhs, rhs, .. } => Op::branch_u64_le_si(lhs, rhs, offset), - | Op::U64Le_Sis { lhs, rhs, .. } => Op::branch_u64_le_is(rhs, lhs, offset), + | Op::U64Le_Sis { lhs, rhs, .. } => Op::branch_u64_le_is(lhs, rhs, offset), // f32 | Op::F32Eq_Sss { lhs, rhs, .. } => Op::branch_f32_eq_ss(lhs, rhs, offset), | Op::F32Eq_Ssi { lhs, rhs, .. } => Op::branch_f32_eq_si(lhs, rhs, offset), @@ -693,8 +693,8 @@ impl UpdateBranchOffset for Op { | Op::BranchF64NotLe_Ss { offset, .. } | Op::BranchF64NotLe_Si { offset, .. } | Op::BranchF64NotLe_Is { offset, .. } => offset, - unexpected => { - panic!("expected a Wasmi `cmp`+`branch` instruction but found: {unexpected:?}") + _unexpected => { + panic!("expected a Wasmi `cmp`+`branch` instruction but found: TODO") // TODO: add Debug for Op } }; offset.init(new_offset); From 3fc20581b59f7e37ecca729284256874c21d4d8a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 23:31:50 +0200 Subject: [PATCH 040/423] move IntoShiftAmount utility trait --- .../wasmi/src/engine/translator/func/mod.rs | 4 +-- .../src/engine/translator/func/simd/mod.rs | 2 +- .../wasmi/src/engine/translator/func/utils.rs | 28 ------------------- crates/wasmi/src/engine/translator/utils.rs | 28 +++++++++++++++++++ 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 937f1dcc09..f93229e654 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -30,7 +30,7 @@ use self::{ StackAllocations, TempOperand, }, - utils::{Input, IntoShiftAmount, Reset, ReusableAllocations}, + utils::{Input, Reset, ReusableAllocations}, }; use crate::{ core::{FuelCostsProvider, IndexType, Typed, TypedVal, UntypedVal}, @@ -43,7 +43,7 @@ use crate::{ TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, - utils::{Instr, WasmFloat, WasmInteger}, + utils::{Instr, IntoShiftAmount, WasmFloat, WasmInteger}, WasmTranslator, }, BlockType, diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index eca4447069..9d00795ac3 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -7,7 +7,7 @@ use crate::{ core::{simd::IntoLaneIdx, FuelCostsProvider, Typed, TypedVal}, engine::translator::{ func::{utils::Input, Operand}, - utils::{Instr, ToBits, Wrap}, + utils::{Instr, IntoShiftAmount, ToBits, Wrap}, }, ir::{ index::{self, Memory}, diff --git a/crates/wasmi/src/engine/translator/func/utils.rs b/crates/wasmi/src/engine/translator/func/utils.rs index bb198fe3be..8b9e9a13d2 100644 --- a/crates/wasmi/src/engine/translator/func/utils.rs +++ b/crates/wasmi/src/engine/translator/func/utils.rs @@ -56,31 +56,3 @@ pub enum Input { /// A 16-bit encoded immediate value operand. Immediate(T), } - -pub trait IntoShiftAmount { - /// The type denoting the shift amount. - /// - /// This is an unsigned integer ranging from `1..N` where `N` is the number of bits in `Self`. - type Value: Copy; - - /// Returns `self` wrapped into a proper shift amount for `Self`. - /// - /// Returns `None` if the resulting shift amount is 0, a.k.a. a no-op. - fn into_shift_amount(self) -> Option; -} - -macro_rules! impl_into_shift_amount { - ( $($ty:ty),* $(,)? ) => { - $( - impl IntoShiftAmount for $ty { - type Value = u8; - - fn into_shift_amount(self) -> Option { - let len_bits = (::core::mem::size_of::() * 8) as Self; - self.checked_rem_euclid(len_bits) - } - } - )* - }; -} -impl_into_shift_amount!(i32, u32, i64, u64, i128, u128); diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 30d22b09a2..9be62e4d23 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -310,6 +310,34 @@ impl ToBits for f32 { } } +pub trait IntoShiftAmount { + /// The type denoting the shift amount. + /// + /// This is an unsigned integer ranging from `1..N` where `N` is the number of bits in `Self`. + type Value: Copy; + + /// Returns `self` wrapped into a proper shift amount for `Self`. + /// + /// Returns `None` if the resulting shift amount is 0, a.k.a. a no-op. + fn into_shift_amount(self) -> Option; +} + +macro_rules! impl_into_shift_amount { + ( $($ty:ty),* $(,)? ) => { + $( + impl IntoShiftAmount for $ty { + type Value = u8; + + fn into_shift_amount(self) -> Option { + let len_bits = (::core::mem::size_of::() * 8) as Self; + self.checked_rem_euclid(len_bits) + } + } + )* + }; +} +impl_into_shift_amount!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128); + impl ToBits for u64 { type Out = u64; fn to_bits(self) -> Self::Out { From 4971b0b6d48d3885d14369002a4a66d4145ef3b8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 23:32:05 +0200 Subject: [PATCH 041/423] fix bugs in StoreOperator def and impls --- crates/wasmi/src/engine/translator/func/op.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/op.rs b/crates/wasmi/src/engine/translator/func/op.rs index 6f531b006d..20b67b0577 100644 --- a/crates/wasmi/src/engine/translator/func/op.rs +++ b/crates/wasmi/src/engine/translator/func/op.rs @@ -21,16 +21,16 @@ pub trait StoreOperator { fn store_ss(ptr: Slot, offset: u64, value: Slot, memory: Memory) -> Op; fn store_si(ptr: Slot, offset: u64, value: Self::Immediate, memory: Memory) -> Op; fn store_is(address: Address, value: Slot, memory: Memory) -> Op; - fn store_ii(address: Address, value: Self::Value, memory: Memory) -> Op; + fn store_ii(address: Address, value: Self::Immediate, memory: Memory) -> Op; fn store_mem0_offset16_ss(ptr: Slot, offset: Offset16, value: Slot) -> Op; - fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Value) -> Op; + fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Immediate) -> Op; } macro_rules! impl_store_wrap { ( $( impl StoreOperator for $name:ident { type Value = $value_ty:ty; - type Immediate = $wrapped_ty:ty; + type Immediate = $immediate_ty:ty; fn into_immediate = $apply:expr; @@ -46,7 +46,7 @@ macro_rules! impl_store_wrap { pub enum $name {} impl StoreOperator for $name { type Value = $value_ty; - type Immediate = $wrapped_ty; + type Immediate = $immediate_ty; fn into_immediate(value: Self::Value) -> Self::Immediate { $apply(value) @@ -64,7 +64,7 @@ macro_rules! impl_store_wrap { $store_is(address, value, memory) } - fn store_ii(address: Address, value: Self::Value, memory: Memory) -> Op { + fn store_ii(address: Address, value: Self::Immediate, memory: Memory) -> Op { $store_ii(address, value, memory) } @@ -72,7 +72,7 @@ macro_rules! impl_store_wrap { $store_mem0_offset16_ss(ptr, offset, value) } - fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Value) -> Op { + fn store_mem0_offset16_si(ptr: Slot, offset: Offset16, value: Self::Immediate) -> Op { $store_mem0_offset16_si(ptr, offset, value) } } @@ -95,7 +95,7 @@ impl_store_wrap! { impl StoreOperator for I64Store { type Value = i64; - type Immediate = i64; + type Immediate = u64; fn into_immediate = ::to_bits; fn store_ss = Op::store64_ss; From 8eec2d5b11ab8607f73c54a5975ea235d5f9eca5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 23:34:05 +0200 Subject: [PATCH 042/423] update simd shift operator translation --- .../src/engine/translator/func/simd/mod.rs | 12 +++++----- .../src/engine/translator/func/simd/visit.rs | 24 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 9d00795ac3..2aba95a9d2 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -206,12 +206,12 @@ impl FuncTranslator { /// Generically translate a Wasm SIMD shift instruction. fn translate_simd_shift( &mut self, - make_instr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - make_instr_imm: fn(result: Slot, lhs: Slot, rhs: ::Output) -> Op, + make_instr_sss: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, + make_instr_ssi: fn(result: Slot, lhs: Slot, rhs: ::Value) -> Op, const_eval: fn(lhs: V128, rhs: u32) -> V128, ) -> Result<(), Error> where - T: IntoShiftAmount>, + T: IntoShiftAmount + From, { bail_unreachable!(self); let (lhs, rhs) = self.stack.pop2(); @@ -227,10 +227,10 @@ impl FuncTranslator { self.stack.push_operand(lhs)?; return Ok(()); }; - let lhs = self.layout.operand_to_reg(lhs)?; + let lhs = self.copy_if_immediate(lhs)?; self.push_instr_with_result( ValType::V128, - |result| make_instr_imm(result, lhs, rhs), + |result| make_instr_ssi(result, lhs, rhs), FuelCostsProvider::simd, )?; return Ok(()); @@ -239,7 +239,7 @@ impl FuncTranslator { let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ValType::V128, - |result| make_instr(result, lhs, rhs), + |result| make_instr_sss(result, lhs, rhs), FuelCostsProvider::simd, )?; Ok(()) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index d318980412..98a19b5bd6 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -665,15 +665,15 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i8x16_shl, Op::i8x16_shl_by, simd::i8x16_shl) + self.translate_simd_shift::(Op::i8x16_shl_sss, Op::i8x16_shl_ssi, simd::i8x16_shl) } fn visit_i8x16_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i8x16_shr_s, Op::i8x16_shr_s_by, simd::i8x16_shr_s) + self.translate_simd_shift::(Op::s8x16_shr_sss, Op::s8x16_shr_ssi, simd::i8x16_shr_s) } fn visit_i8x16_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i8x16_shr_u, Op::i8x16_shr_u_by, simd::i8x16_shr_u) + self.translate_simd_shift::(Op::u8x16_shr_sss, Op::u8x16_shr_ssi, simd::i8x16_shr_u) } fn visit_i8x16_add(&mut self) -> Self::Output { @@ -785,15 +785,15 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i16x8_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i16x8_shl, Op::i16x8_shl_by, simd::i16x8_shl) + self.translate_simd_shift::(Op::i16x8_shl_sss, Op::i16x8_shl_ssi, simd::i16x8_shl) } fn visit_i16x8_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i16x8_shr_s, Op::i16x8_shr_s_by, simd::i16x8_shr_s) + self.translate_simd_shift::(Op::s16x8_shr_sss, Op::s16x8_shr_ssi, simd::i16x8_shr_s) } fn visit_i16x8_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i16x8_shr_u, Op::i16x8_shr_u_by, simd::i16x8_shr_u) + self.translate_simd_shift::(Op::u16x8_shr_sss, Op::u16x8_shr_ssi, simd::i16x8_shr_u) } fn visit_i16x8_add(&mut self) -> Self::Output { @@ -919,15 +919,15 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i32x4_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i32x4_shl, Op::i32x4_shl_by, simd::i32x4_shl) + self.translate_simd_shift::(Op::i32x4_shl_sss, Op::i32x4_shl_ssi, simd::i32x4_shl) } fn visit_i32x4_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i32x4_shr_s, Op::i32x4_shr_s_by, simd::i32x4_shr_s) + self.translate_simd_shift::(Op::s32x4_shr_sss, Op::s32x4_shr_ssi, simd::i32x4_shr_s) } fn visit_i32x4_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i32x4_shr_u, Op::i32x4_shr_u_by, simd::i32x4_shr_u) + self.translate_simd_shift::(Op::u32x4_shr_sss, Op::u32x4_shr_ssi, simd::i32x4_shr_u) } fn visit_i32x4_add(&mut self) -> Self::Output { @@ -1023,15 +1023,15 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i64x2_shl(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i64x2_shl, Op::i64x2_shl_by, simd::i64x2_shl) + self.translate_simd_shift::(Op::i64x2_shl_sss, Op::i64x2_shl_ssi, simd::i64x2_shl) } fn visit_i64x2_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i64x2_shr_s, Op::i64x2_shr_s_by, simd::i64x2_shr_s) + self.translate_simd_shift::(Op::s64x2_shr_sss, Op::s64x2_shr_ssi, simd::i64x2_shr_s) } fn visit_i64x2_shr_u(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::i64x2_shr_u, Op::i64x2_shr_u_by, simd::i64x2_shr_u) + self.translate_simd_shift::(Op::u64x2_shr_sss, Op::u64x2_shr_ssi, simd::i64x2_shr_u) } fn visit_i64x2_add(&mut self) -> Self::Output { From 565ea4386f7e833c9018c92112fdbe4fc972747b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 23:34:28 +0200 Subject: [PATCH 043/423] move ToBits impls a bit more to top --- crates/wasmi/src/engine/translator/utils.rs | 42 ++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 9be62e4d23..c3b0c98a86 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -310,6 +310,27 @@ impl ToBits for f32 { } } +impl ToBits for u64 { + type Out = u64; + fn to_bits(self) -> Self::Out { + self + } +} + +impl ToBits for i64 { + type Out = u64; + fn to_bits(self) -> Self::Out { + u64::from_ne_bytes(self.to_ne_bytes()) + } +} + +impl ToBits for f64 { + type Out = u64; + fn to_bits(self) -> Self::Out { + self.to_bits() + } +} + pub trait IntoShiftAmount { /// The type denoting the shift amount. /// @@ -337,24 +358,3 @@ macro_rules! impl_into_shift_amount { }; } impl_into_shift_amount!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128); - -impl ToBits for u64 { - type Out = u64; - fn to_bits(self) -> Self::Out { - self - } -} - -impl ToBits for i64 { - type Out = u64; - fn to_bits(self) -> Self::Out { - u64::from_ne_bytes(self.to_ne_bytes()) - } -} - -impl ToBits for f64 { - type Out = u64; - fn to_bits(self) -> Self::Out { - self.to_bits() - } -} From 865f70fc9c8448fa5c054dbfad398e0da9e2fc14 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 11 Sep 2025 23:34:41 +0200 Subject: [PATCH 044/423] remove no longer needed IsInstructionParameter extension trait --- crates/wasmi/src/engine/translator/utils.rs | 31 --------------------- 1 file changed, 31 deletions(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index c3b0c98a86..8ceefdbdac 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -168,37 +168,6 @@ impl BumpFuelConsumption for Op { } } -/// Extension trait to query if an [`Op`] is a parameter. -pub trait IsInstructionParameter { - /// Returns `true` if `self` is a parameter to an [`Op`]. - fn is_instruction_parameter(&self) -> bool; -} - -impl IsInstructionParameter for Op { - #[rustfmt::skip] - fn is_instruction_parameter(&self) -> bool { - matches!(self, - | Self::TableIndex { .. } - | Self::MemoryIndex { .. } - | Self::DataIndex { .. } - | Self::ElemIndex { .. } - | Self::Const32 { .. } - | Self::I64Const32 { .. } - | Self::F64Const32 { .. } - | Self::BranchTableTarget { .. } - | Self::Imm16AndImm32 { .. } - | Self::SlotAndImm32 { .. } - | Self::SlotSpan { .. } - | Self::Slot { .. } - | Self::Slot2 { .. } - | Self::Slot3 { .. } - | Self::SlotList { .. } - | Self::CallIndirectParams { .. } - | Self::CallIndirectParamsImm16 { .. } - ) - } -} - /// A reference to an encoded [`Op`]. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct Instr(u32); From 6389da8f76409807783de554d77f7fe3e507cd78 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 14:36:31 +0200 Subject: [PATCH 045/423] fix IntoShiftAmount trait impls --- crates/wasmi/src/engine/translator/utils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 8ceefdbdac..c500503dd6 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -320,7 +320,8 @@ macro_rules! impl_into_shift_amount { fn into_shift_amount(self) -> Option { let len_bits = (::core::mem::size_of::() * 8) as Self; - self.checked_rem_euclid(len_bits) + let shamt = self.checked_rem_euclid(len_bits)?; + Some(shamt as _) } } )* From 0cd11511e598558591aaf818d97d05e11b1f47ad Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 14:45:25 +0200 Subject: [PATCH 046/423] generate ToBits impls via macros --- crates/wasmi/src/engine/translator/utils.rs | 93 ++++++--------------- 1 file changed, 25 insertions(+), 68 deletions(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index c500503dd6..eaeafb27d0 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -7,7 +7,7 @@ use crate::{ Ref, ValType, }; -use core::num::NonZero; +use core::{convert::identity, num::NonZero}; impl Typed for ExternRef { const TY: ValType = ValType::ExternRef; @@ -230,74 +230,31 @@ pub trait ToBits { fn to_bits(self) -> Self::Out; } -impl ToBits for u8 { - type Out = u8; - fn to_bits(self) -> Self::Out { - self - } -} - -impl ToBits for i8 { - type Out = u8; - fn to_bits(self) -> Self::Out { - u8::from_ne_bytes(self.to_ne_bytes()) - } -} - -impl ToBits for u16 { - type Out = u16; - fn to_bits(self) -> Self::Out { - self - } -} - -impl ToBits for i16 { - type Out = u16; - fn to_bits(self) -> Self::Out { - u16::from_ne_bytes(self.to_ne_bytes()) - } -} - -impl ToBits for u32 { - type Out = u32; - fn to_bits(self) -> Self::Out { - self - } -} - -impl ToBits for i32 { - type Out = u32; - fn to_bits(self) -> Self::Out { - u32::from_ne_bytes(self.to_ne_bytes()) - } -} - -impl ToBits for f32 { - type Out = u32; - fn to_bits(self) -> Self::Out { - self.to_bits() - } -} - -impl ToBits for u64 { - type Out = u64; - fn to_bits(self) -> Self::Out { - self - } -} - -impl ToBits for i64 { - type Out = u64; - fn to_bits(self) -> Self::Out { - u64::from_ne_bytes(self.to_ne_bytes()) - } +macro_rules! impl_to_bits { + ( $($ty:ty as $bits_ty:ty = $expr:expr),* $(,)? ) => { + $( + impl ToBits for $ty { + type Out = $bits_ty; + fn to_bits(self) -> Self::Out { + $expr(self) + } + } + )* + }; } - -impl ToBits for f64 { - type Out = u64; - fn to_bits(self) -> Self::Out { - self.to_bits() - } +impl_to_bits! { + u8 as u8 = identity, + u16 as u16 = identity, + u32 as u32 = identity, + u64 as u64 = identity, + + f32 as u32 = f32::to_bits, + f64 as u64 = f64::to_bits, + + i8 as u8 = |v: i8| u8::from_ne_bytes(v.to_ne_bytes()), + i16 as u16 = |v: i16| u16::from_ne_bytes(v.to_ne_bytes()), + i32 as u32 = |v: i32| u32::from_ne_bytes(v.to_ne_bytes()), + i64 as u64 = |v: i64| u64::from_ne_bytes(v.to_ne_bytes()), } pub trait IntoShiftAmount { From fba0ffb43bd154c88678d8f52bd6d7e533f27f04 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 14:51:46 +0200 Subject: [PATCH 047/423] update and simplify RelinkResult --- .../src/engine/translator/func/instrs.rs | 3 +- .../wasmi/src/engine/translator/func/mod.rs | 2 +- .../src/engine/translator/relink_result.rs | 126 ++---------------- 3 files changed, 13 insertions(+), 118 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 3b28c02847..8431a20633 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -158,7 +158,6 @@ impl InstrEncoder { new_result: Slot, old_result: Slot, layout: &StackLayout, - module: &ModuleHeader, ) -> Result { if !matches!(layout.stack_space(new_result), StackSpace::Local) { // Case: cannot replace result if `new_result` isn't a local. @@ -170,7 +169,7 @@ impl InstrEncoder { }; if !self .get_mut(last_instr) - .relink_result(module, new_result, old_result)? + .relink_result(new_result, old_result)? { // Case: it was impossible to relink the result of `last_instr. return Ok(false); diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index f93229e654..0fcbc4525f 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -1487,7 +1487,7 @@ impl FuncTranslator { Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, }; self.instrs - .try_replace_result(result, old_result, &self.layout, &self.module) + .try_replace_result(result, old_result, &self.layout) } /// Encodes an unconditional Wasm `branch` instruction. diff --git a/crates/wasmi/src/engine/translator/relink_result.rs b/crates/wasmi/src/engine/translator/relink_result.rs index 7ec948b759..87240cc31d 100644 --- a/crates/wasmi/src/engine/translator/relink_result.rs +++ b/crates/wasmi/src/engine/translator/relink_result.rs @@ -1,10 +1,6 @@ use crate::{ - engine::EngineFunc, - ir::{index, Op, Slot, SlotSpan}, - module::ModuleHeader, - Engine, + ir::{Op, Slot}, Error, - FuncType, }; /// Extension trait for [`Op`] to conditionally relink result [`Slot`]s. @@ -16,118 +12,18 @@ pub trait RelinkResult { /// - `Ok(true)`: the result has been relinked /// - `Ok(false)`: the result has _not_ been relinked /// - `Err(_)`: translation error - fn relink_result( - &mut self, - module: &ModuleHeader, - new_result: Slot, - old_result: Slot, - ) -> Result; -} - -/// Visitor to implement [`RelinkResult`] for [`Op`]. -struct Visitor { - /// The new [`Slot`] that replaces the `old_result` [`Slot`]. - new_result: Slot, - /// The old result [`Slot`]. - old_result: Slot, - /// The return value of the visitation. - /// - /// For more information see docs of [`RelinkResult`]. - replaced: Result, -} - -impl Visitor { - /// Creates a new [`Visitor`]. - fn new(new_result: Slot, old_result: Slot) -> Self { - Self { - new_result, - old_result, - replaced: Ok(false), - } - } + fn relink_result(&mut self, new_result: Slot, old_result: Slot) -> Result; } impl RelinkResult for Op { - fn relink_result( - &mut self, - module: &ModuleHeader, - new_result: Slot, - old_result: Slot, - ) -> Result { - let mut visitor = Visitor::new(new_result, old_result); - self.visit_results(&mut visitor); - visitor.replaced - } -} - -fn relink_simple(result: &mut Slot, new_result: Slot, old_result: Slot) -> Result { - if *result != old_result { - // Note: This is a safeguard to prevent miscompilations. - return Ok(false); - } - debug_assert_ne!(*result, new_result); - *result = new_result; - Ok(true) -} - -fn get_engine(module: &ModuleHeader) -> Engine { - module.engine().upgrade().unwrap_or_else(|| { - panic!( - "engine for result relinking does not exist: {:?}", - module.engine() - ) - }) -} - -fn relink_call_internal( - results: &mut SlotSpan, - func: EngineFunc, - module: &ModuleHeader, - new_result: Slot, - old_result: Slot, -) -> Result { - let Some(module_func) = module.get_func_index(func) else { - panic!("missing module func for compiled func: {func:?}") - }; - let engine = get_engine(module); - let func_type = module.get_type_of_func(module_func); - let len_results = engine.resolve_func_type(func_type, FuncType::len_results); - if len_results != 1 { - return Ok(false); - } - relink_simple(results.head_mut(), new_result, old_result) -} - -fn relink_call_imported( - results: &mut SlotSpan, - func: index::Func, - module: &ModuleHeader, - new_result: Slot, - old_result: Slot, -) -> Result { - let engine = get_engine(module); - let func_idx = u32::from(func).into(); - let func_type = module.get_type_of_func(func_idx); - let len_results = engine.resolve_func_type(func_type, |func_type| func_type.results().len()); - if len_results != 1 { - return Ok(false); - } - relink_simple(results.head_mut(), new_result, old_result) -} - -fn relink_call_indirect( - results: &mut SlotSpan, - func_type: index::FuncType, - module: &ModuleHeader, - new_result: Slot, - old_result: Slot, -) -> Result { - let engine = get_engine(module); - let func_type_idx = u32::from(func_type).into(); - let func_type = module.get_func_type(func_type_idx); - let len_results = engine.resolve_func_type(func_type, |func_type| func_type.results().len()); - if len_results != 1 { - return Ok(false); + fn relink_result(&mut self, new_result: Slot, old_result: Slot) -> Result { + let Some(result_mut) = self.result_mut() else { + return Ok(false); + }; + if *result_mut != old_result { + return Ok(false); + } + *result_mut = new_result; + Ok(true) } - relink_simple(results.head_mut(), new_result, old_result) } From 5bcada3e58732ac7bf6587e1ea5acb19aed13a15 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 14:52:34 +0200 Subject: [PATCH 048/423] remove no longer needed debug_assert The new Wasmi ISA does no longer have instruction parameters. --- crates/wasmi/src/engine/translator/func/instrs.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 8431a20633..bf85fcc020 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -10,7 +10,7 @@ use crate::{ }, func::{Operand, Stack, StackLayout, StackSpace}, relink_result::RelinkResult, - utils::{BumpFuelConsumption as _, Instr, IsInstructionParameter as _}, + utils::{BumpFuelConsumption as _, Instr}, }, ir::{BranchOffset, Op, Slot}, module::ModuleHeader, @@ -109,10 +109,6 @@ impl InstrEncoder { /// Pushes a non-parameter [`Op`] to the [`InstrEncoder`]. fn push_instr_impl(&mut self, instruction: Op) -> Result { - debug_assert!( - !instruction.is_instruction_parameter(), - "parameter: {instruction:?}" - ); let instr = self.next_instr(); self.instrs.push(instruction); self.last_instr = Some(instr); From 73547b52701b08909ba3e66d7802f4842c8d2c5d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 20:42:40 +0200 Subject: [PATCH 049/423] use to_bits where needed --- crates/wasmi/src/engine/translator/func/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 0fcbc4525f..fa5a61ecc3 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -43,7 +43,7 @@ use crate::{ TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, - utils::{Instr, IntoShiftAmount, WasmFloat, WasmInteger}, + utils::{Instr, IntoShiftAmount, WasmFloat, WasmInteger, ToBits}, WasmTranslator, }, BlockType, @@ -646,8 +646,8 @@ impl FuncTranslator { /// Returns the copy instruction to copy the given immediate `value` to `result`. fn make_copy_imm_instr(result: Slot, value: TypedVal) -> Result { let instr = match value.ty() { - ValType::I32 => Op::copy32(result, i32::from(value)), - ValType::I64 => Op::copy64(result, i64::from(value)), + ValType::I32 => Op::copy32(result, i32::from(value).to_bits()), + ValType::I64 => Op::copy64(result, i64::from(value).to_bits()), ValType::F32 => Op::copy32(result, f32::from(value).to_bits()), ValType::F64 => Op::copy64(result, f64::from(value).to_bits()), ValType::ExternRef | ValType::FuncRef => { @@ -1129,8 +1129,8 @@ impl FuncTranslator { Operand::Immediate(operand) => { let val = operand.val(); match operand.ty() { - ValType::I32 => Op::return32(i32::from(val)), - ValType::I64 => Op::return64(i64::from(val)), + ValType::I32 => Op::return32(i32::from(val).to_bits()), + ValType::I64 => Op::return64(i64::from(val).to_bits()), ValType::F32 => Op::return32(f32::from(val).to_bits()), ValType::F64 => Op::return64(f64::from(val).to_bits()), ValType::FuncRef | ValType::ExternRef => { From 8061dc3c6b090d7663cfdb0dc79e63f347037294 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 20:43:20 +0200 Subject: [PATCH 050/423] update encode_br_if --- crates/wasmi/src/engine/translator/func/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index fa5a61ecc3..a62a2e3814 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -1540,8 +1540,8 @@ impl FuncTranslator { let instr = self.instrs.next_instr(); let offset = self.labels.try_resolve_label(label, instr)?; let instr = match branch_eqz { - true => Op::branch_i32_eq_imm16(condition, 0, offset), - false => Op::branch_i32_ne_imm16(condition, 0, offset), + true => Op::branch_i32_eq_si(condition, 0, offset), + false => Op::branch_i32_not_eq_si(condition, 0, offset), }; self.push_instr(instr, FuelCostsProvider::base)?; Ok(()) From 4fdcf0c98b46204a6042d0adca51e643db2bd7fe Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 20:44:24 +0200 Subject: [PATCH 051/423] update translate_select --- .../src/engine/translator/func/instrs.rs | 30 +++---- .../wasmi/src/engine/translator/func/mod.rs | 86 +++++++++---------- 2 files changed, 56 insertions(+), 60 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index bf85fcc020..d1a720b7bf 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -185,40 +185,40 @@ impl InstrEncoder { select_condition: Slot, layout: &StackLayout, stack: &mut Stack, - ) -> Result, Error> { + true_val: Slot, + false_val: Slot, + ) -> Result { let Some(last_instr) = self.last_instr else { // If there is no last instruction there is no comparison instruction to negate. - return Ok(None); + return Ok(false); }; let last_instruction = self.get(last_instr); let Some(last_result) = last_instruction.compare_result() else { // All negatable instructions have a single result register. - return Ok(None); + return Ok(false); }; if matches!(layout.stack_space(last_result), StackSpace::Local) { // The instruction stores its result into a local variable which // is an observable side effect which we are not allowed to mutate. - return Ok(None); + return Ok(false); } if last_result != select_condition { // The result of the last instruction and the select's `condition` // are not equal thus indicating that we cannot fuse the instructions. - return Ok(None); + return Ok(false); } - let CmpSelectFusion::Applied { - fused, - swap_operands, - } = last_instruction.try_into_cmp_select_instr(|| { - let select_result = stack.push_temp(ty, Some(last_instr))?; - let select_result = layout.temp_to_reg(select_result)?; - Ok(select_result) - })? + let CmpSelectFusion::Applied(fused) = + last_instruction.try_into_cmp_select_instr(true_val, false_val, || { + let select_result = stack.push_temp(ty, Some(last_instr))?; + let select_result = layout.temp_to_reg(select_result)?; + Ok(select_result) + })? else { - return Ok(None); + return Ok(false); }; let last_instr = self.get_mut(last_instr); *last_instr = fused; - Ok(Some(swap_operands)) + Ok(true) } /// Pushes an [`Op`] parameter to the [`InstrEncoder`]. diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index a62a2e3814..cd42bb9844 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -2156,54 +2156,50 @@ impl FuncTranslator { self.stack.push_operand(true_val)?; return Ok(()); } - if let Operand::Immediate(condition) = condition { - // Optimization: since condition is a constant value we can const-fold the `select` - // instruction and simply push the selected value back to the provider stack. - let condition = i32::from(condition.val()) != 0; - let selected = match condition { - true => true_val, - false => false_val, - }; - if let Operand::Temp(selected) = selected { - // Case: the selected operand is a temporary which needs to be copied - // if it was the `false_val` since it changed its index. This is - // not the case for the `true_val` since `true_val` is the first - // value popped from the stack. - if !condition { - let selected = self.layout.temp_to_reg(selected.operand_index())?; - self.push_instr_with_result( - ty, - |result| Op::copy(result, selected), - FuelCostsProvider::base, - )?; - return Ok(()); - } - } - self.stack.push_operand(selected)?; - return Ok(()); - } - let condition = self.layout.operand_to_reg(condition)?; - let mut true_val = self.copy_if_immediate(true_val)?; - let mut false_val = self.copy_if_immediate(false_val)?; - match self - .instrs - .try_fuse_select(ty, condition, &self.layout, &mut self.stack)? - { - Some(swap_operands) => { - if swap_operands { - mem::swap(&mut true_val, &mut false_val); + let condition = match condition { + Operand::Immediate(condition) => { + let condition = i32::from(condition.val()) != 0; + let selected = match condition { + true => true_val, + false => false_val, + }; + if let Operand::Temp(selected) = selected { + // Case: the selected operand is a temporary which needs to be copied + // if it was the `false_val` since it changed its index. This is + // not the case for the `true_val` since `true_val` is the first + // value popped from the stack. + if !condition { + let selected = self.layout.temp_to_reg(selected.operand_index())?; + self.push_instr_with_result( + ty, + |result| Op::copy(result, selected), + FuelCostsProvider::base, + )?; + return Ok(()); + } } + self.stack.push_operand(selected)?; + return Ok(()); } - None => { - self.push_instr_with_result( - ty, - |result| Op::select_i32_eq_imm16(result, condition, 0_i16), - FuelCostsProvider::base, - )?; - mem::swap(&mut true_val, &mut false_val); - } + Operand::Local(condition) => self.layout.local_to_reg(condition.local_index())?, + Operand::Temp(condition) => self.layout.temp_to_reg(condition.operand_index())?, + }; + let true_val = self.copy_if_immediate(true_val)?; + let false_val = self.copy_if_immediate(false_val)?; + if !self.instrs.try_fuse_select( + ty, + condition, + &self.layout, + &mut self.stack, + true_val, + false_val, + )? { + self.push_instr_with_result( + ty, + |result| Op::select_i32_eq_ssi(result, condition, 0_i32, false_val, true_val), + FuelCostsProvider::base, + )?; }; - self.push_param(Op::slot2_ext(true_val, false_val))?; Ok(()) } From f88599d89cc31ffc5999d8cc055e9019ce2b6ba6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 20:44:49 +0200 Subject: [PATCH 052/423] update visit_memory_fill translation --- crates/wasmi/src/engine/translator/func/visit.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 4f64267a10..38c2e8a212 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1824,16 +1824,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let (dst, value, len) = self.stack.pop3(); let memory = index::Memory::from(mem); let dst = self.copy_if_immediate(dst)?; - let value = self.make_input(value, |_, value| { - let byte = u32::from(value) as u8; - Ok(Input::Immediate(byte)) - })?; let len = self.copy_if_immediate(len)?; - let instr: Op = match value { - Input::Slot(value) => Op::memory_fill(memory, dst, value, len), - Input::Immediate(value) => Op::memory_fill_imm(memory, dst, value, len), - }; - self.push_instr(instr, FuelCostsProvider::instance)?; + let value = self.copy_if_immediate(value)?; + self.push_instr( + Op::memory_fill(memory, dst, value, len), + FuelCostsProvider::instance, + )?; Ok(()) } From 9056ffa8896b6e8ad9c949b480dc3b050926b3e1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 20:45:10 +0200 Subject: [PATCH 053/423] update data.drop and elem.drop translation --- crates/wasmi/src/engine/translator/func/visit.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 38c2e8a212..d9e3808fef 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1798,7 +1798,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { bail_unreachable!(self); - self.push_instr(Op::data_drop(data_index), FuelCostsProvider::instance)?; + self.push_instr( + Op::data_drop(index::Data::from(data_index)), + FuelCostsProvider::instance, + )?; Ok(()) } @@ -1852,7 +1855,10 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_elem_drop(&mut self, elem_index: u32) -> Self::Output { bail_unreachable!(self); - self.push_instr(Op::elem_drop(elem_index), FuelCostsProvider::instance)?; + self.push_instr( + Op::elem_drop(index::Elem::from(elem_index)), + FuelCostsProvider::instance, + )?; Ok(()) } From 2d2eaf0aced9e7593f924d0cfd29f0ff2adf71d7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 20:45:21 +0200 Subject: [PATCH 054/423] update global.set translation --- crates/wasmi/src/engine/translator/func/visit.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index d9e3808fef..b612de10a8 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -417,14 +417,14 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_global_set(&mut self, global_index: u32) -> Self::Output { bail_unreachable!(self); - let global = ir::index::Global::from(global_index); + let global = index::Global::from(global_index); let input = self.stack.pop(); let value = match input { Operand::Immediate(input) => input.val(), input => { // Case: `global.set` with simple register input. let input = self.layout.operand_to_reg(input)?; - self.push_instr(Op::global_set(input, global), FuelCostsProvider::instance)?; + self.push_instr(Op::global_set(global, input), FuelCostsProvider::instance)?; return Ok(()); } }; From 68bef979f582edc679d7302432d692ca423fa76b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 12 Sep 2025 20:45:58 +0200 Subject: [PATCH 055/423] fix various issues --- .../wasmi/src/engine/translator/comparator.rs | 4 ++-- .../wasmi/src/engine/translator/func/mod.rs | 23 ++++++++----------- .../wasmi/src/engine/translator/func/visit.rs | 4 ++-- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index d3841bb5e7..139a500ac7 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -352,9 +352,9 @@ impl LogicalizeCmpInstr for Op { pub trait TryIntoCmpSelectInstr: Sized { fn try_into_cmp_select_instr( &self, - get_result: impl FnOnce() -> Result, val_true: Slot, val_false: Slot, + get_result: impl FnOnce() -> Result, ) -> Result; } @@ -369,9 +369,9 @@ pub enum CmpSelectFusion { impl TryIntoCmpSelectInstr for Op { fn try_into_cmp_select_instr( &self, - get_result: impl FnOnce() -> Result, val_true: Slot, val_false: Slot, + get_result: impl FnOnce() -> Result, ) -> Result { if !self.is_compare_instr() { return Ok(CmpSelectFusion::Unapplied); diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index cd42bb9844..74a1451513 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -60,7 +60,6 @@ use crate::{ V128, }; use alloc::vec::Vec; -use core::mem; use wasmparser::{MemArg, WasmFeatures}; /// Type concerned with translating from Wasm bytecode to Wasmi bytecode. @@ -1626,7 +1625,7 @@ impl FuncTranslator { }; let offset = self.labels.try_resolve_label(label, instr)?; let fused = cmp_instr - .try_into_cmp_branch_instr(offset)? + .try_into_cmp_branch_instr(offset) .expect("cmp+branch fusion must succeed"); Ok(Some(fused)) } @@ -1656,7 +1655,7 @@ impl FuncTranslator { call_imported(params, index::Func::from(function_index)) } }; - let call_instr = self.push_instr(instr, FuelCostsProvider::call)?; + self.push_instr(instr, FuelCostsProvider::call)?; Ok(()) } @@ -1668,7 +1667,7 @@ impl FuncTranslator { make_instr: fn( params: BoundedSlotSpan, index: Slot, - func_type: FuncType, + func_type: index::FuncType, table: index::Table, ) -> Op, ) -> Result<(), Error> { @@ -1871,7 +1870,7 @@ impl FuncTranslator { opt_si: fn(this: &mut Self, lhs: Operand, rhs: T) -> Result, ) -> Result<(), Error> where - T: WasmInteger, + T: From + Copy, R: Into + Typed, { bail_unreachable!(self); @@ -1926,7 +1925,7 @@ impl FuncTranslator { }; self.push_instr_with_result( ::TY, - |result| make_instr_ssi(result, lhs, rhs), + |result| make_instr_ssi(result, lhs, non_zero_rhs), FuelCostsProvider::base, ) } @@ -1958,7 +1957,7 @@ impl FuncTranslator { consteval: fn(T, T) -> R, ) -> Result<(), Error> where - T: WasmInteger, + T: From + Copy, R: Into + Typed, { bail_unreachable!(self); @@ -1977,7 +1976,6 @@ impl FuncTranslator { } (Operand::Immediate(lhs), rhs) => { let lhs = T::from(lhs.val()); - let lhs16 = self.make_imm16(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ::TY, @@ -2103,7 +2101,7 @@ impl FuncTranslator { ) } (Operand::Immediate(lhs), rhs) => { - let lhs = T::from(rhs.val()); + let lhs = T::from(lhs.val()); let rhs = self.layout.operand_to_reg(rhs)?; self.push_instr_with_result( ::TY, @@ -2320,8 +2318,7 @@ impl FuncTranslator { let ptr = self.stack.pop(); let ptr = match ptr { Operand::Immediate(ptr) => { - let ptr = ptr.val(); - let Some(address) = self.effective_address(memory, ptr, offset) else { + let Some(address) = self.effective_address(memory, ptr.val(), offset) else { return self.translate_trap(TrapCode::MemoryOutOfBounds); }; match make_instr_si.into() { @@ -2335,7 +2332,7 @@ impl FuncTranslator { } None => { let consume_fuel = self.stack.consume_fuel_instr(); - self.copy_operand_to_temp(ptr, consume_fuel)? + self.copy_operand_to_temp(ptr.into(), consume_fuel)? } } } @@ -2395,7 +2392,7 @@ impl FuncTranslator { fn effective_address(&self, mem: index::Memory, ptr: TypedVal, offset: u64) -> Option
{ let memory_type = *self .module - .get_type_of_memory(MemoryIdx::from(u32::from(mem))); + .get_type_of_memory(MemoryIdx::from(u32::from(u16::from(mem)))); let ptr = match memory_type.is_64() { true => u64::from(ptr), false => u64::from(u32::from(ptr)), diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index b612de10a8..9f361add34 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1933,7 +1933,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { bail_unreachable!(self); self.push_instr_with_result( ValType::FuncRef, - |result| Op::ref_func(result, function_index), + |result| Op::ref_func(result, index::Func::from(function_index)), FuelCostsProvider::instance, )?; Ok(()) @@ -1962,7 +1962,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let index = self.stack.pop(); let item_ty = table_type.element(); let index_ty = table_type.index_ty(); - let index = self.make_index32(index, index_ty)?; + let index = self.make_index(index, index_ty)?; self.push_instr_with_result( item_ty, |result| match index { From 9d5d319e8e90ce6cbefbf56f211ef84c8777701c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 13 Sep 2025 12:51:18 +0200 Subject: [PATCH 056/423] add make_index32_or_copy utility method --- .../wasmi/src/engine/translator/func/mod.rs | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 74a1451513..3a3a7e415a 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -43,7 +43,7 @@ use crate::{ TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, - utils::{Instr, IntoShiftAmount, WasmFloat, WasmInteger, ToBits}, + utils::{Instr, IntoShiftAmount, ToBits, WasmFloat, WasmInteger}, WasmTranslator, }, BlockType, @@ -1816,6 +1816,27 @@ impl FuncTranslator { Ok(Input::Immediate(value)) } + /// Copies `operand` to a temporary stack slot if it is an immediate that cannot be encoded using 32-bits. + /// + /// - Returns [`Input::Slot`] if `operand` is a local or a temporary operand. + /// - Returns [`Input::Immediate`] if `operand` is an immediate that can be encoded as 32-bit value. + /// - Returns [`Input::Slot`] otherwise and encodes a copy storing the immediate into its temporary stack slot. + fn make_index32_or_copy( + &mut self, + operand: Operand, + index_ty: IndexType, + ) -> Result, Error> { + let index64 = match self.make_index64(operand, index_ty)? { + Input::Slot(index) => return Ok(Input::Slot(index)), + Input::Immediate(index) => index, + }; + let index32 = match u32::try_from(index64) { + Ok(index) => return Ok(Input::Immediate(index)), + Err(_) => self.copy_if_immediate(operand)?, + }; + Ok(Input::Slot(index32)) + } + /// Evaluates `consteval(lhs, rhs)` and pushed either its result or tranlates a `trap`. fn translate_binary_consteval_fallible( &mut self, From 3c69254ab56dae28fec287b6cf683f299ca27b36 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 13 Sep 2025 12:56:12 +0200 Subject: [PATCH 057/423] update table.{get,set} translation --- .../wasmi/src/engine/translator/func/visit.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 9f361add34..54b4e8d124 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1962,7 +1962,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let index = self.stack.pop(); let item_ty = table_type.element(); let index_ty = table_type.index_ty(); - let index = self.make_index(index, index_ty)?; + let index = self.make_index32_or_copy(index, index_ty)?; self.push_instr_with_result( item_ty, |result| match index { @@ -1981,11 +1981,16 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let table = index::Table::from(table); let index_ty = table_type.index_ty(); let (index, value) = self.stack.pop2(); - let index = self.make_index32(index, index_ty)?; - let value = self.layout.operand_to_reg(value)?; - let instr = match index { - Input::Slot(index) => Op::table_set_ss(index, value, table), - Input::Immediate(index) => Op::table_set_si(value, index, table), + let index = self.make_index32_or_copy(index, index_ty)?; + let value = + self.make_input(value, |_this, value| Ok(Input::Immediate(u64::from(value))))?; + let instr = match (index, value) { + (Input::Slot(index), Input::Slot(value)) => Op::table_set_ss(index, value, table), + (Input::Slot(index), Input::Immediate(value)) => Op::table_set_si(index, value, table), + (Input::Immediate(index), Input::Slot(value)) => Op::table_set_is(index, value, table), + (Input::Immediate(index), Input::Immediate(value)) => { + Op::table_set_ii(index, value, table) + } }; self.push_instr(instr, FuelCostsProvider::instance)?; Ok(()) From 17190f4b465c9410c01f951efebee301625696c5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 13 Sep 2025 12:56:31 +0200 Subject: [PATCH 058/423] fix generic param of i32.shr_u translation --- crates/wasmi/src/engine/translator/func/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 54b4e8d124..d47b67ad58 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1186,7 +1186,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_shr_u(&mut self) -> Self::Output { - self.translate_shift::( + self.translate_shift::( Op::u32_shr_sss, Op::u32_shr_ssi, Op::u32_shr_sis, From ccdc233d0cbfbfe235c6fc97ab0aa3b19f569a7f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 13 Sep 2025 14:33:22 +0200 Subject: [PATCH 059/423] more clean-ups towards getting the crate to compile again --- crates/wasmi/src/engine/translator/func/instrs.rs | 14 ++------------ crates/wasmi/src/engine/translator/func/mod.rs | 2 +- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index d1a720b7bf..aab34b27f3 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -13,7 +13,6 @@ use crate::{ utils::{BumpFuelConsumption as _, Instr}, }, ir::{BranchOffset, Op, Slot}, - module::ModuleHeader, Engine, Error, ValType, @@ -87,10 +86,7 @@ impl InstrEncoder { return Ok(None); }; let base_costs = fuel_costs.base(); - let Ok(base_costs) = u32::try_from(base_costs) else { - panic!("out of bounds base fuel costs: {base_costs}"); - }; - let instr = self.push_instr_impl(Op::consume_fuel(base_costs))?; + let instr = self.push_instr_impl(Op::consume_fuel(base_costs.into()))?; Ok(Some(instr)) } @@ -124,15 +120,10 @@ impl InstrEncoder { /// /// If `instr` or `new_instr` are [`Op`] parameters. pub fn try_replace_instr(&mut self, instr: Instr, new_instr: Op) -> Result { - debug_assert!( - !new_instr.is_instruction_parameter(), - "parameter: {new_instr:?}" - ); let Some(last_instr) = self.last_instr else { return Ok(false); }; let replace = self.get_mut(instr); - debug_assert!(!replace.is_instruction_parameter(), "parameter: {instr:?}"); if instr != last_instr { return Ok(false); } @@ -269,9 +260,8 @@ impl InstrEncoder { &mut self, instr: Instr, offset: BranchOffset, - layout: &mut StackLayout, ) -> Result<(), Error> { - self.get_mut(instr).update_branch_offset(layout, offset)?; + self.get_mut(instr).update_branch_offset(offset)?; Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 3a3a7e415a..eb242cf027 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -288,7 +288,7 @@ impl FuncTranslator { fn update_branch_offsets(&mut self) -> Result<(), Error> { for (user, offset) in self.labels.resolved_users() { self.instrs - .update_branch_offset(user, offset?, &mut self.layout)?; + .update_branch_offset(user, offset?)?; } Ok(()) } From 92f62d9362bacad782e5097ef9c549fc8e500821 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 13 Sep 2025 15:36:25 +0200 Subject: [PATCH 060/423] update return and copy instruction translation --- .../src/engine/translator/func/instrs.rs | 56 +--- .../wasmi/src/engine/translator/func/mod.rs | 259 +++--------------- 2 files changed, 33 insertions(+), 282 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index aab34b27f3..539d7f7c36 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -8,7 +8,7 @@ use crate::{ TryIntoCmpSelectInstr as _, UpdateBranchOffset as _, }, - func::{Operand, Stack, StackLayout, StackSpace}, + func::{Stack, StackLayout, StackSpace}, relink_result::RelinkResult, utils::{BumpFuelConsumption as _, Instr}, }, @@ -291,60 +291,6 @@ impl InstrEncoder { Ok(()) } - /// Encode the top-most `len` operands on the stack as register list. - /// - /// # Note - /// - /// This is used for the following n-ary instructions: - /// - /// - [`Op::ReturnMany`] - /// - [`Op::CopyMany`] - /// - [`Op::CallInternal`] - /// - [`Op::CallImported`] - /// - [`Op::CallIndirect`] - /// - [`Op::ReturnCallInternal`] - /// - [`Op::ReturnCallImported`] - /// - [`Op::ReturnCallIndirect`] - pub fn encode_register_list( - &mut self, - operands: &[Operand], - layout: &mut StackLayout, - ) -> Result<(), Error> { - let mut remaining = operands; - let mut operand_to_reg = - |operand: &Operand| -> Result { layout.operand_to_reg(*operand) }; - let instr = loop { - match remaining { - [] => return Ok(()), - [v0] => { - let v0 = operand_to_reg(v0)?; - break Op::slot(v0); - } - [v0, v1] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - break Op::slot2_ext(v0, v1); - } - [v0, v1, v2] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - let v2 = operand_to_reg(v2)?; - break Op::slot3_ext(v0, v1, v2); - } - [v0, v1, v2, rest @ ..] => { - let v0 = operand_to_reg(v0)?; - let v1 = operand_to_reg(v1)?; - let v2 = operand_to_reg(v2)?; - let instr = Op::slot_list_ext(v0, v1, v2); - self.push_param(instr); - remaining = rest; - } - }; - }; - self.push_param(instr); - Ok(()) - } - /// Returns an iterator yielding all [`Op`]s of the [`InstrEncoder`]. /// /// # Note diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index eb242cf027..9937b58b2f 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -287,8 +287,7 @@ impl FuncTranslator { /// If this is used before all branching labels have been pinned. fn update_branch_offsets(&mut self) -> Result<(), Error> { for (user, offset) in self.labels.resolved_users() { - self.instrs - .update_branch_offset(user, offset?)?; + self.instrs.update_branch_offset(user, offset?)?; } Ok(()) } @@ -439,11 +438,6 @@ impl FuncTranslator { self.encode_copy(result, value, consume_fuel_instr)?; Ok(()) } - 2 => { - let (val0, val1) = self.stack.peek2(); - self.encode_copy2(results, val0, val1, consume_fuel_instr)?; - Ok(()) - } _ => self.encode_copy_many(results, len_values, consume_fuel_instr), } } @@ -463,155 +457,12 @@ impl FuncTranslator { // Case: no-op copy instruction return Ok(None); }; - if let Some(fused_copy) = self.try_merge_copies(copy_instr)? { - // Case: successfully merged the copy instruction with its succeeding one. - return Ok(Some(fused_copy)); - } let instr = self.instrs .push_instr(copy_instr, consume_fuel_instr, FuelCostsProvider::base)?; Ok(Some(instr)) } - /// Tries to merge `copy_instr` with the succeeding copy instruction if one exists. - /// - /// - Returns `None` if merging was not applicable. - /// - Returns `Some(last_instr)` if merging was successful. - fn try_merge_copies(&mut self, copy_instr: Op) -> Result, Error> { - let Op::Copy { result, value } = copy_instr else { - // Case: `copy_instr` is not fusable. - return Ok(None); - }; - let Some(last_instr) = self.instrs.last_instr() else { - // Case: no `last_instr` to fuse with. - return Ok(None); - }; - let last_copy = *self.instrs.get(last_instr); - let fused_copy = match last_copy { - Op::Copy { .. } => Self::try_merge_copy_instr(last_copy, result, value), - Op::Copy2 { .. } => Self::try_merge_copy2_instr(last_copy, result, value), - Op::CopySpan { .. } => Self::try_merge_copy_span_instr(last_copy, result, value), - _ => return Ok(None), - }; - if let Some(fused_copy) = fused_copy { - let success = self.instrs.try_replace_instr(last_instr, fused_copy)?; - debug_assert!(success); - return Ok(Some(last_instr)); - } - Ok(None) - } - - /// Tries to merge two [`Op::Copy`] instructions and returns the result. - /// - /// Returns `None` if merging was not possible. - fn try_merge_copy_instr(last_copy: Op, result: Slot, value: Slot) -> Option { - let Op::Copy { - result: last_result, - value: last_value, - } = last_copy - else { - // Case: `last_copy` does not refer to a mergable copy instruction. - return None; - }; - // Try to fuse to a `copy2` instruction. - if value == last_result { - // Case: cannot merge since the succeeding copy overwrites the result of `copy_instr` - return None; - } - if result == last_result.next() { - // Case: we can append `copy_instrs`. - return Some(Op::copy2_ext(SlotSpan::new(last_result), last_value, value)); - } - if result == last_result.prev() { - // Case: we can prepend `copy_instr`. - return Some(Op::copy2_ext(SlotSpan::new(result), value, last_value)); - } - None - } - - /// Tries to merge an [`Op::Copy2`] and an [`Op::Copy`] and returns the result. - /// - /// Returns `None` if merging was not possible. - fn try_merge_copy2_instr(last_copy: Op, result: Slot, value: Slot) -> Option { - let Op::Copy2 { results, values } = last_copy else { - // Case: `last_copy` does not refer to a mergable copy instruction. - return None; - }; - // Try to fuse to a `copy_span` instruction. - let [last_result0, last_result1] = results.to_array(); - let [last_value0, last_value1] = values; - if last_value0.next() != last_value1 { - // Case: last `copy2` instruction itself is not convertible to a `copy_span`. - return None; - } - if result == last_result1.next() && value == last_value1.next() { - // Case: we can append `copy_instr`. - if value == last_result0 || value == last_result1 { - // Case: cannot merge since `value` is overwritten by `last_copy`. - return None; - } - let results = SlotSpan::new(last_result0); - let values = SlotSpan::new(last_value0); - let len = 3_u16; - debug_assert!(!SlotSpan::has_overlapping_copies(results, values, len)); - return Some(Op::copy_span(results, values, len)); - } - if result == last_result0.prev() && value == last_value0.prev() { - // Case: we can prepend `copy_instr`. - if result == last_value0 || result == last_value1 { - // Case: cannot merge since `result` overwrites results of `last_copy`. - return None; - } - let results = SlotSpan::new(result); - let values = SlotSpan::new(value); - let len = 3_u16; - debug_assert!(!SlotSpan::has_overlapping_copies(results, values, len)); - return Some(Op::copy_span(results, values, len)); - } - None - } - - /// Tries to merge an [`Op::CopySpan`] and an [`Op::Copy`] and returns the result. - /// - /// Returns `None` if merging was not possible. - fn try_merge_copy_span_instr(last_copy: Op, result: Slot, value: Slot) -> Option { - let Op::CopySpan { - results, - values, - len, - } = last_copy - else { - // Case: `last_copy` does not refer to a mergable copy instruction. - return None; - }; - let last_result0 = results.head(); - let last_value0 = values.head(); - // Try to fuse to a larger `copy_span` instruction. - if result == last_result0.next_n(len) && value == last_value0.next_n(len) { - // Case: we can append `copy_instr`. - let new_len = len + 1; - if SlotSpan::has_overlapping_copies(results, values, new_len) { - // Case: cannot merge since resulting `copy_span` has overlapping copies. - return None; - } - return Some(Op::copy_span(results, values, new_len)); - } - if result == last_result0.prev() && value == last_value0.prev() { - // Case: we can prepend `copy_instr`. - let new_len = len + 1; - if SlotSpan::has_overlapping_copies(results, values, new_len) { - // Case: cannot merge since resulting `copy_span` has overlapping copies. - return None; - } - return Some(Op::copy_span( - SlotSpan::new(result), - SlotSpan::new(value), - new_len, - )); - } - None - } - /// Returns the copy instruction to copy the given `operand` to `result`. /// /// Returns `None` if the resulting copy instruction is a no-op. @@ -665,34 +516,6 @@ impl FuncTranslator { Ok(instr) } - /// Encode a copy instruction that copies 2 values. - /// - /// # Note - /// - /// This won't encode a copy if the resulting copy instruction is a no-op. - fn encode_copy2( - &mut self, - results: SlotSpan, - val0: Operand, - val1: Operand, - consume_fuel_instr: Option, - ) -> Result<(), Error> { - let val0 = self.layout.operand_to_reg(val0)?; - let val1 = self.layout.operand_to_reg(val1)?; - let result0 = results.head(); - let result1 = result0.next(); - if result0 == val0 && result1 == val1 { - // Case: no-op copy instruction - return Ok(()); - } - self.instrs.push_instr( - Op::copy2_ext(results, val0, val1), - consume_fuel_instr, - FuelCostsProvider::base, - )?; - Ok(()) - } - /// Encode a copy instruction that copies a contiguous span of values. /// /// # Note @@ -741,30 +564,22 @@ impl FuncTranslator { &self.layout )?); match values { - [] => Ok(()), + [] => return Ok(()), [val0] => { let result = results.head(); let value = *val0; self.encode_copy(result, value, consume_fuel_instr)?; - Ok(()) + return Ok(()); } - [val0, val1] => self.encode_copy2(results, *val0, *val1, consume_fuel_instr), - [val0, val1, rest @ ..] => { - debug_assert!(!rest.is_empty()); + values => { + debug_assert!(!values.is_empty()); if let Some(values) = Self::try_form_regspan_of(values, &self.layout)? { return self.encode_copy_span(results, values, len, consume_fuel_instr); } - let val0 = self.layout.operand_to_reg(*val0)?; - let val1 = self.layout.operand_to_reg(*val1)?; - self.instrs.push_instr( - Op::copy_many_ext(results, val0, val1), - consume_fuel_instr, - |costs| costs.fuel_for_copying_values(u64::from(len)), - )?; - self.instrs.encode_register_list(rest, &mut self.layout)?; - Ok(()) } } + let values = self.move_operands_to_temp(usize::from(len), consume_fuel_instr)?; + self.encode_copy_span(results, values, len, consume_fuel_instr) } /// Tries to strip noop copies from the start of the `copy_many`. @@ -1053,7 +868,7 @@ impl FuncTranslator { fn encode_br_table_0(&mut self, table: wasmparser::BrTable, index: Slot) -> Result<(), Error> { debug_assert_eq!(self.immediates.len(), (table.len() + 1) as usize); self.push_instr( - Op::branch_table_0(index, table.len() + 1), + Op::branch_table(index, table.len() + 1), FuelCostsProvider::base, )?; // Encode the `br_table` targets: @@ -1104,8 +919,8 @@ impl FuncTranslator { let offset = self .labels .try_resolve_label(frame.label(), self.instrs.next_instr())?; - self.instrs - .push_param(Op::branch_table_target(results, offset)); + // self.instrs + // .push_param(Op::branch_table_target(results, offset)); // TODO: encode branch table targets properly frame.branch_to(); } Ok(()) @@ -1115,7 +930,7 @@ impl FuncTranslator { fn encode_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); let instr = match len_results { - 0 => Op::Return, + 0 => Op::Return {}, 1 => match self.stack.peek(0) { Operand::Local(operand) => { let value = self.layout.local_to_reg(operand.local_index())?; @@ -1137,7 +952,7 @@ impl FuncTranslator { } ValType::V128 => { let value = self.stack.peek(0); - let temp_slot = self.copy_operand_to_temp(operand, consume_fuel)?; + let temp_slot = self.copy_operand_to_temp(value, consume_fuel)?; Op::return_slot(temp_slot) } } @@ -1147,7 +962,7 @@ impl FuncTranslator { self.move_operands_to_temp(usize::from(len_results), consume_fuel)?; let result0 = self.stack.peek(usize::from(len_results)); let slot0 = self.layout.temp_to_reg(result0.index())?; - Op::return_span(SlotSpan::new(slot0)) + Op::return_span(BoundedSlotSpan::new(SlotSpan::new(slot0), len_results)) } }; let instr = self @@ -1162,37 +977,19 @@ impl FuncTranslator { self.operands.extend(self.stack.peek_n(len)); } - /// Encodes an [`Op::ReturnMany`] for `len` values. - /// - /// # Panics - /// - /// If `len` is not greater than or equal to 4. + /// Encodes an [`Op::ReturnSpan`] for `len` values. fn encode_return_many( &mut self, len: u16, consume_fuel_instr: Option, ) -> Result { - self.peek_operands_into_buffer(usize::from(len)); - if let Some(values) = Self::try_form_regspan_of(&self.operands, &self.layout)? { - let values = BoundedSlotSpan::new(values, len); - return self.instrs.push_instr( - Op::return_span(values), - consume_fuel_instr, - FuelCostsProvider::base, - ); - } - let [v0, v1, v2, rest @ ..] = &self.operands[..] else { - unreachable!("encode_return_many (pre-condition): len >= 4") - }; - let v0 = self.layout.operand_to_reg(*v0)?; - let v1 = self.layout.operand_to_reg(*v1)?; - let v2 = self.layout.operand_to_reg(*v2)?; + let values = self.move_operands_to_temp(usize::from(len), consume_fuel_instr)?; + let values = BoundedSlotSpan::new(values, len); let return_instr = self.instrs.push_instr( - Op::return_many_ext(v0, v1, v2), + Op::return_span(values), consume_fuel_instr, FuelCostsProvider::base, )?; - self.instrs.encode_register_list(rest, &mut self.layout)?; Ok(return_instr) } @@ -2444,7 +2241,13 @@ impl FuncTranslator { /// Translates a Wasm `i64.binop128` instruction from the `wide-arithmetic` proposal. fn translate_i64_binop128( &mut self, - make_instr: fn(results: [Slot; 2], lhs_lo: Slot) -> Op, + make_instr: fn( + results: FixedSlotSpan<2>, + lhs_lo: Slot, + lhs_hi: Slot, + rhs_lo: Slot, + rhs_hi: Slot, + ) -> Op, const_eval: fn(lhs_lo: i64, lhs_hi: i64, rhs_lo: i64, rhs_hi: i64) -> (i64, i64), ) -> Result<(), Error> { bail_unreachable!(self); @@ -2467,19 +2270,21 @@ impl FuncTranslator { self.stack.push_immediate(result_hi)?; return Ok(()); } - let rhs_lo = self.layout.operand_to_reg(rhs_lo)?; - let rhs_hi = self.layout.operand_to_reg(rhs_hi)?; - let lhs_lo = self.layout.operand_to_reg(lhs_lo)?; - let lhs_hi = self.layout.operand_to_reg(lhs_hi)?; + let rhs_lo = self.copy_if_immediate(rhs_lo)?; + let rhs_hi = self.copy_if_immediate(rhs_hi)?; + let lhs_lo = self.copy_if_immediate(lhs_lo)?; + let lhs_hi = self.copy_if_immediate(lhs_hi)?; let result_lo = self.stack.push_temp(ValType::I64, None)?; let result_hi = self.stack.push_temp(ValType::I64, None)?; let result_lo = self.layout.temp_to_reg(result_lo)?; let result_hi = self.layout.temp_to_reg(result_hi)?; + let Ok(results) = >::new(SlotSpan::new(result_lo)) else { + return Err(Error::from(TranslationError::AllocatedTooManySlots)); + }; self.push_instr( - make_instr([result_lo, result_hi], lhs_lo), + make_instr(results, lhs_lo, lhs_hi, rhs_lo, rhs_hi), FuelCostsProvider::base, )?; - self.push_param(Op::slot3_ext(lhs_hi, rhs_lo, rhs_hi))?; Ok(()) } From 3ce459532c92e76921bf8bf50278606874fc6ab7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:15:10 +0200 Subject: [PATCH 061/423] update load ops translation --- .../wasmi/src/engine/translator/func/mod.rs | 30 ++-- crates/wasmi/src/engine/translator/func/op.rs | 159 ++++++++++++++++++ .../wasmi/src/engine/translator/func/visit.rs | 112 ++---------- 3 files changed, 184 insertions(+), 117 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 9937b58b2f..9753b02f03 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -2121,29 +2121,22 @@ impl FuncTranslator { /// - `{i32, i64, f32, f64}.load` /// - `i32.{load8_s, load8_u, load16_s, load16_u}` /// - `i64.{load8_s, load8_u, load16_s, load16_u load32_s, load32_u}` - fn translate_load( - &mut self, - memarg: MemArg, - loaded_ty: ValType, - make_instr_ss: fn(result: Slot, ptr: Slot, offset: u64, memory: index::Memory) -> Op, - make_instr_si: impl Into< - Option Op>, - >, - make_instr_mem0_offset16_ss: fn(result: Slot, ptr: Slot, offset: Offset16) -> Op, - ) -> Result<(), Error> { + fn translate_load(&mut self, memarg: MemArg) -> Result<(), Error> { bail_unreachable!(self); let (memory, offset) = Self::decode_memarg(memarg); let ptr = self.stack.pop(); let ptr = match ptr { + Operand::Local(ptr) => self.layout.local_to_reg(ptr.local_index())?, + Operand::Temp(ptr) => self.layout.temp_to_reg(ptr.operand_index())?, Operand::Immediate(ptr) => { let Some(address) = self.effective_address(memory, ptr.val(), offset) else { return self.translate_trap(TrapCode::MemoryOutOfBounds); }; - match make_instr_si.into() { - Some(make_instr_si) => { + match T::load_si(address, memory) { + Some(load_si) => { self.push_instr_with_result( - loaded_ty, - |result| make_instr_si(result, address, memory), + T::LOADED_TY, + load_si, FuelCostsProvider::load, )?; return Ok(()); @@ -2154,21 +2147,20 @@ impl FuncTranslator { } } } - ptr => self.layout.operand_to_reg(ptr)?, }; if memory.is_default() { if let Ok(offset) = Offset16::try_from(offset) { self.push_instr_with_result( - loaded_ty, - |result| make_instr_mem0_offset16_ss(result, ptr, offset), + T::LOADED_TY, + |result| T::load_mem0_offset16_ss(result, ptr, offset), FuelCostsProvider::load, )?; return Ok(()); } } self.push_instr_with_result( - loaded_ty, - |result| make_instr_ss(result, ptr, offset, memory), + T::LOADED_TY, + |result| T::load_ss(result, ptr, offset, memory), FuelCostsProvider::load, )?; Ok(()) diff --git a/crates/wasmi/src/engine/translator/func/op.rs b/crates/wasmi/src/engine/translator/func/op.rs index 20b67b0577..15a0354106 100644 --- a/crates/wasmi/src/engine/translator/func/op.rs +++ b/crates/wasmi/src/engine/translator/func/op.rs @@ -1,6 +1,7 @@ use crate::{ engine::translator::utils::{ToBits, Wrap}, ir::{index::Memory, Address, Offset16, Op, Slot}, + ValType, }; /// Trait implemented by all Wasm operators that can be translated as wrapping store instructions. @@ -197,3 +198,161 @@ impl_store_wrap! { fn store_mem0_offset16_si = Op::i64_store32_mem0_offset16_si; } } + +/// Trait implemented by all Wasm operators that can be translated as load extend instructions. +pub trait LoadOperator { + /// The type of the loaded value. + const LOADED_TY: ValType; + + fn load_ss(result: Slot, ptr: Slot, offset: u64, memory: Memory) -> Op; + fn load_si(_address: Address, _memory: Memory) -> Option Op> { + Op>>::None + } + fn load_mem0_offset16_ss(result: Slot, ptr: Slot, offset: Offset16) -> Op; +} + +macro_rules! impl_load_extend { + ( $( + impl LoadOperator for $name:ident { + const LOADED_TY: ValType = $loaded_ty:expr; + + fn load_ss = $store_ss:expr; + $( fn load_si = $store_si:expr; )? + fn load_mem0_offset16_ss = $store_mem0_offset16_ss:expr; + } + )* ) => { + $( + pub enum $name {} + impl LoadOperator for $name { + const LOADED_TY: ValType = $loaded_ty; + + fn load_ss(result: Slot, ptr: Slot, offset: u64, memory: Memory) -> Op { + $store_ss(result, ptr, offset, memory) + } + + $( + fn load_si(address: Address, memory: Memory) -> Option Op> { + Some(move |result| $store_si(result, address, memory)) + } + )? + + fn load_mem0_offset16_ss(result: Slot, ptr: Slot, offset: Offset16) -> Op { + $store_mem0_offset16_ss(result, ptr, offset) + } + } + )* + }; +} +impl_load_extend! { + impl LoadOperator for I32Load { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::load32_ss; + fn load_si = Op::load32_si; + fn load_mem0_offset16_ss = Op::load32_mem0_offset16_ss; + } + + impl LoadOperator for I32Load8 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::i32_load8_ss; + fn load_si = Op::i32_load8_si; + fn load_mem0_offset16_ss = Op::i32_load8_mem0_offset16_ss; + } + + impl LoadOperator for U32Load8 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::u32_load8_ss; + fn load_si = Op::u32_load8_si; + fn load_mem0_offset16_ss = Op::u32_load8_mem0_offset16_ss; + } + + impl LoadOperator for I32Load16 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::i32_load16_ss; + fn load_si = Op::i32_load16_si; + fn load_mem0_offset16_ss = Op::i32_load16_mem0_offset16_ss; + } + + impl LoadOperator for U32Load16 { + const LOADED_TY: ValType = ValType::I32; + + fn load_ss = Op::u32_load16_ss; + fn load_si = Op::u32_load16_si; + fn load_mem0_offset16_ss = Op::u32_load16_mem0_offset16_ss; + } + + impl LoadOperator for I64Load { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::load64_ss; + fn load_si = Op::load64_si; + fn load_mem0_offset16_ss = Op::load64_mem0_offset16_ss; + } + + impl LoadOperator for I64Load8 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::i64_load8_ss; + fn load_si = Op::i64_load8_si; + fn load_mem0_offset16_ss = Op::i64_load8_mem0_offset16_ss; + } + + impl LoadOperator for U64Load8 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::u64_load8_ss; + fn load_si = Op::u64_load8_si; + fn load_mem0_offset16_ss = Op::u64_load8_mem0_offset16_ss; + } + + impl LoadOperator for I64Load16 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::i64_load16_ss; + fn load_si = Op::i64_load16_si; + fn load_mem0_offset16_ss = Op::i64_load16_mem0_offset16_ss; + } + + impl LoadOperator for U64Load16 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::u64_load16_ss; + fn load_si = Op::u64_load16_si; + fn load_mem0_offset16_ss = Op::u64_load16_mem0_offset16_ss; + } + + impl LoadOperator for I64Load32 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::i64_load32_ss; + fn load_si = Op::i64_load32_si; + fn load_mem0_offset16_ss = Op::i64_load32_mem0_offset16_ss; + } + + impl LoadOperator for U64Load32 { + const LOADED_TY: ValType = ValType::I64; + + fn load_ss = Op::u64_load32_ss; + fn load_si = Op::u64_load32_si; + fn load_mem0_offset16_ss = Op::u64_load32_mem0_offset16_ss; + } + + impl LoadOperator for F32Load { + const LOADED_TY: ValType = ValType::F32; + + fn load_ss = Op::load32_ss; + fn load_si = Op::load32_si; + fn load_mem0_offset16_ss = Op::load32_mem0_offset16_ss; + } + + impl LoadOperator for F64Load { + const LOADED_TY: ValType = ValType::F64; + + fn load_ss = Op::load64_ss; + fn load_si = Op::load64_si; + fn load_mem0_offset16_ss = Op::load64_mem0_offset16_ss; + } +} diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index d47b67ad58..cd9976c85f 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -452,156 +452,72 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::load32_ss, - Op::load32_si, - Op::load32_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::load64_ss, - Op::load64_si, - Op::load64_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_f32_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::F32, - Op::load32_ss, - Op::load32_si, - Op::load32_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_f64_load(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::F64, - Op::load64_ss, - Op::load64_si, - Op::load64_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::i32_load8_ss, - Op::i32_load8_si, - Op::i32_load8_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::u32_load8_ss, - Op::u32_load8_si, - Op::u32_load8_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::i32_load16_ss, - Op::i32_load16_si, - Op::i32_load16_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i32_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I32, - Op::u32_load16_ss, - Op::u32_load16_si, - Op::u32_load16_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load8_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load8_ss, - Op::i64_load8_si, - Op::i64_load8_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load8_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::u64_load8_ss, - Op::u64_load8_si, - Op::u64_load8_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load16_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load16_ss, - Op::i64_load16_si, - Op::i64_load16_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load16_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::u64_load16_ss, - Op::u64_load16_si, - Op::u64_load16_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load32_s(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::i64_load32_ss, - Op::i64_load32_si, - Op::i64_load32_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] fn visit_i64_load32_u(&mut self, memarg: wasmparser::MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::I64, - Op::u64_load32_ss, - Op::u64_load32_si, - Op::u64_load32_mem0_offset16_ss, - ) + self.translate_load::(memarg) } #[inline(never)] From 1e079f0536c6830a4ee4465fd5f4caafccf3a244 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:33:08 +0200 Subject: [PATCH 062/423] use new TryFrom for Memory impl --- crates/wasmi/src/engine/translator/func/visit.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index cd9976c85f..f4f76fdc10 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -573,7 +573,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { .get_type_of_memory(MemoryIdx::from(mem)) .index_ty() .ty(); - let memory = index::Memory::from(mem); + let memory = index::Memory::try_from(mem)?; self.push_instr_with_result( index_ty, |result| Op::memory_size(result, memory), @@ -589,7 +589,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { .module .get_type_of_memory(MemoryIdx::from(mem)) .index_ty(); - let memory = index::Memory::from(mem); + let memory = index::Memory::try_from(mem)?; let delta = self.stack.pop(); if let Operand::Immediate(delta) = delta { let delta = delta.val(); @@ -1699,7 +1699,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); - let memory = index::Memory::from(mem); + let memory = index::Memory::try_from(mem)?; let data = index::Data::from(data_index); let dst = self.copy_if_immediate(dst)?; let src = self.copy_if_immediate(src)?; @@ -1725,8 +1725,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.stack.pop3(); - let dst_memory = index::Memory::from(dst_mem); - let src_memory = index::Memory::from(src_mem); + let dst_memory = index::Memory::try_from(dst_mem)?; + let src_memory = index::Memory::try_from(src_mem)?; let dst = self.copy_if_immediate(dst)?; let src = self.copy_if_immediate(src)?; let len = self.copy_if_immediate(len)?; @@ -1741,7 +1741,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { bail_unreachable!(self); let (dst, value, len) = self.stack.pop3(); - let memory = index::Memory::from(mem); + let memory = index::Memory::try_from(mem)?; let dst = self.copy_if_immediate(dst)?; let len = self.copy_if_immediate(len)?; let value = self.copy_if_immediate(value)?; From 4937c854311d924bec01ae20699443c96b9d6453 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:33:21 +0200 Subject: [PATCH 063/423] refactor decode_memarg returning a Result --- crates/wasmi/src/engine/translator/func/mod.rs | 10 ++++++---- crates/wasmi/src/engine/translator/func/simd/mod.rs | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 9753b02f03..bde41d1ec9 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -2123,7 +2123,7 @@ impl FuncTranslator { /// - `i64.{load8_s, load8_u, load16_s, load16_u load32_s, load32_u}` fn translate_load(&mut self, memarg: MemArg) -> Result<(), Error> { bail_unreachable!(self); - let (memory, offset) = Self::decode_memarg(memarg); + let (memory, offset) = Self::decode_memarg(memarg)?; let ptr = self.stack.pop(); let ptr = match ptr { Operand::Local(ptr) => self.layout.local_to_reg(ptr.local_index())?, @@ -2185,6 +2185,7 @@ impl FuncTranslator { { bail_unreachable!(self); let (ptr, value) = self.stack.pop2(); + let (memory, offset) = Self::decode_memarg(memarg)?; todo!() } @@ -2193,9 +2194,9 @@ impl FuncTranslator { /// # Panics /// /// If the [`MemArg`] offset is not 32-bit. - fn decode_memarg(memarg: MemArg) -> (index::Memory, u64) { - let memory = index::Memory::from(memarg.memory); - (memory, memarg.offset) + fn decode_memarg(memarg: MemArg) -> Result<(index::Memory, u64), Error> { + let memory = index::Memory::try_from(memarg.memory)?; + Ok((memory, memarg.offset)) } /// Returns the effective address `ptr+offset` if it is valid. @@ -2273,6 +2274,7 @@ impl FuncTranslator { let Ok(results) = >::new(SlotSpan::new(result_lo)) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; + debug_assert_eq!(results.to_array(), [result_lo, result_hi]); self.push_instr( make_instr(results, lhs_lo, lhs_hi, rhs_lo, rhs_hi), FuelCostsProvider::base, diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 2aba95a9d2..8ff2bc7c50 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -266,7 +266,7 @@ impl FuncTranslator { ) -> Op, ) -> Result<(), Error> { bail_unreachable!(self); - let (memory, offset) = Self::decode_memarg(memarg); + let (memory, offset) = Self::decode_memarg(memarg)?; let Ok(lane) = ::try_from(lane) else { panic!("encountered out of bounds lane: {lane}"); }; @@ -320,7 +320,7 @@ impl FuncTranslator { } v128 => self.layout.operand_to_reg(v128)?, }; - let (memory, offset) = Self::decode_memarg(memarg); + let (memory, offset) = Self::decode_memarg(memarg)?; let (ptr, offset) = match ptr { Operand::Immediate(ptr) => { let Some(address) = self.effective_address(memory, ptr.val(), offset) else { From 3ec4d44c465052ceb8bf6ab228facf31787f857a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:39:16 +0200 Subject: [PATCH 064/423] update i64.shr_u translation --- crates/wasmi/src/engine/translator/func/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index f4f76fdc10..a5c191c4d8 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1267,7 +1267,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_i64_shr_u(&mut self) -> Self::Output { - self.translate_shift::( + self.translate_shift::( Op::u64_shr_sss, Op::u64_shr_ssi, Op::u64_shr_sis, From 26f61affed5b82489e198a8d381abe42eb70dd10 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:41:37 +0200 Subject: [PATCH 065/423] use unsigned in wasmi_core's wasm::shr_u APIs --- crates/core/src/value.rs | 17 +++-------------- crates/core/src/wasm.rs | 4 ++-- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index 6df7912fd3..a265888da0 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -95,7 +95,7 @@ pub trait Integer: Sized + Unsigned { /// Signed shift-right `self` by `other`. fn shr_s(lhs: Self, rhs: Self) -> Self; /// Unsigned shift-right `self` by `other`. - fn shr_u(lhs: Self, rhs: Self) -> Self; + fn shr_u(lhs: Self::Uint, rhs: Self::Uint) -> Self::Uint; /// Get left bit rotation result. fn rotl(lhs: Self, rhs: Self) -> Self; /// Get right bit rotation result. @@ -130,25 +130,14 @@ pub trait Integer: Sized + Unsigned { pub trait Unsigned { /// The unsigned type. type Uint; - - /// Converts `self` losslessly to the unsigned type. - fn to_unsigned(self) -> Self::Uint; } impl Unsigned for i32 { type Uint = u32; - #[inline] - fn to_unsigned(self) -> Self::Uint { - self as _ - } } impl Unsigned for i64 { type Uint = u64; - #[inline] - fn to_unsigned(self) -> Self::Uint { - self as _ - } } /// Float-point value. @@ -270,8 +259,8 @@ macro_rules! impl_integer { lhs.wrapping_shr(rhs as u32) } #[inline] - fn shr_u(lhs: Self, rhs: Self) -> Self { - lhs.to_unsigned().wrapping_shr(rhs as u32) as _ + fn shr_u(lhs: Self::Uint, rhs: Self::Uint) -> Self::Uint { + lhs.wrapping_shr(rhs as u32) as _ } #[inline] fn rotl(lhs: Self, rhs: Self) -> Self { diff --git a/crates/core/src/wasm.rs b/crates/core/src/wasm.rs index 61f51f49ae..3b195388e8 100644 --- a/crates/core/src/wasm.rs +++ b/crates/core/src/wasm.rs @@ -96,8 +96,8 @@ impl_untyped_val! { fn i64_shl(lhs: i64, rhs: i64) -> i64 = Integer::shl; fn i32_shr_s(lhs: i32, rhs: i32) -> i32 = Integer::shr_s; fn i64_shr_s(lhs: i64, rhs: i64) -> i64 = Integer::shr_s; - fn i32_shr_u(lhs: i32, rhs: i32) -> i32 = Integer::shr_u; - fn i64_shr_u(lhs: i64, rhs: i64) -> i64 = Integer::shr_u; + fn i32_shr_u(lhs: u32, rhs: u32) -> u32 = ::shr_u; + fn i64_shr_u(lhs: u64, rhs: u64) -> u64 = ::shr_u; fn i32_rotl(lhs: i32, rhs: i32) -> i32 = Integer::rotl; fn i64_rotl(lhs: i64, rhs: i64) -> i64 = Integer::rotl; fn i32_rotr(lhs: i32, rhs: i32) -> i32 = Integer::rotr; From ee1351b4eb2774e1a7b5d1808f1b77790b705e43 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:50:31 +0200 Subject: [PATCH 066/423] fix param order --- crates/wasmi/src/engine/translator/func/simd/op.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/op.rs b/crates/wasmi/src/engine/translator/func/simd/op.rs index 8ab9583bd6..c2d5f84670 100644 --- a/crates/wasmi/src/engine/translator/func/simd/op.rs +++ b/crates/wasmi/src/engine/translator/func/simd/op.rs @@ -72,7 +72,7 @@ macro_rules! impl_replace_lane { lane: ::LaneIdx, value: Slot, ) -> Op { - $replace_lane_sss(result, input, lane, value) + $replace_lane_sss(result, input, value, lane) } fn replace_lane_ssi( @@ -81,7 +81,7 @@ macro_rules! impl_replace_lane { lane: ::LaneIdx, value: Self::Immediate, ) -> Op { - $replace_lane_ssi(result, input, lane, value) + $replace_lane_ssi(result, input, value, lane) } } )* From 3a3579e6ac5e1d597365c2dce4736fe4e725c05d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:53:54 +0200 Subject: [PATCH 067/423] remove function local constant registry --- .../engine/translator/func/layout/consts.rs | 128 ------------------ .../src/engine/translator/func/layout/mod.rs | 42 +----- 2 files changed, 5 insertions(+), 165 deletions(-) delete mode 100644 crates/wasmi/src/engine/translator/func/layout/consts.rs diff --git a/crates/wasmi/src/engine/translator/func/layout/consts.rs b/crates/wasmi/src/engine/translator/func/layout/consts.rs deleted file mode 100644 index f224d3e700..0000000000 --- a/crates/wasmi/src/engine/translator/func/layout/consts.rs +++ /dev/null @@ -1,128 +0,0 @@ -use super::Reset; -use crate::{core::UntypedVal, engine::TranslationError, ir::Slot, Error}; -use alloc::{ - collections::{btree_map, BTreeMap}, - vec::Vec, -}; -use core::{iter::Rev, slice::Iter as SliceIter}; - -/// A pool of deduplicated function local constant values. -/// -/// - Those constant values are identified by their associated [`Slot`]. -/// - All constant values are also deduplicated so that no duplicates -/// are stored in a [`ConstRegistry`]. This also means that deciding if two -/// [`Slot`] values refer to the equal constant values can be efficiently -/// done by comparing the [`Slot`] indices without resolving to their -/// underlying constant values. -#[derive(Debug, Default)] -pub struct ConstRegistry { - /// Mapping from constant [`UntypedVal`] values to [`Slot`] indices. - const2idx: BTreeMap, - /// Mapping from [`Slot`] indices to constant [`UntypedVal`] values. - idx2const: Vec, - /// The [`Slot`] index for the next allocated function local constant value. - next_idx: i16, -} - -impl Reset for ConstRegistry { - fn reset(&mut self) { - self.const2idx.clear(); - self.idx2const.clear(); - self.next_idx = Self::first_index(); - } -} - -impl ConstRegistry { - /// The maximum index for [`Slot`] referring to function local constant values. - /// - /// # Note - /// - /// The maximum index is also the one to be assigned to the first allocated - /// function local constant value as indices are counting downwards. - fn first_index() -> i16 { - -1 - } - - /// The mininmum index for [`Slot`] referring to function local constant values. - /// - /// # Note - /// - /// This index is not assignable to a function local constant value and acts - /// as a bound to guard against overflowing the range of indices. - fn last_index() -> i16 { - i16::MIN - } - - /// Allocates a new constant `value` on the [`ConstRegistry`] and returns its identifier. - /// - /// # Note - /// - /// If the constant `value` already exists in this [`ConstRegistry`] no new value is - /// allocated and the identifier of the existing constant `value` returned instead. - /// - /// # Errors - /// - /// If too many constant values have been allocated for this [`ConstRegistry`]. - pub fn alloc(&mut self, value: UntypedVal) -> Result { - if self.next_idx == Self::last_index() { - return Err(Error::from(TranslationError::TooManyFuncLocalConstValues)); - } - match self.const2idx.entry(value) { - btree_map::Entry::Occupied(entry) => Ok(*entry.get()), - btree_map::Entry::Vacant(entry) => { - let register = Slot::from(self.next_idx); - self.next_idx -= 1; - entry.insert(register); - self.idx2const.push(value); - Ok(register) - } - } - } - - /// Returns an iterator yielding all function local constant values of the [`ConstRegistry`]. - /// - /// # Note - /// - /// The function local constant values are yielded in their allocation order. - pub fn iter(&self) -> ConstRegistryIter<'_> { - ConstRegistryIter::new(self) - } -} - -/// Iterator yielding all allocated function local constant values. -pub struct ConstRegistryIter<'a> { - /// The underlying iterator. - iter: Rev>, -} - -impl<'a> ConstRegistryIter<'a> { - /// Creates a new [`ConstRegistryIter`] from the given slice of [`UntypedVal`]. - pub fn new(consts: &'a ConstRegistry) -> Self { - // Note: we need to revert the iteration since we allocate new - // function local constants in reverse order of their absolute - // vector indices in the function call frame during execution. - Self { - iter: consts.idx2const.as_slice().iter().rev(), - } - } -} - -impl Iterator for ConstRegistryIter<'_> { - type Item = UntypedVal; - - fn next(&mut self) -> Option { - self.iter.next().copied() - } -} - -impl DoubleEndedIterator for ConstRegistryIter<'_> { - fn next_back(&mut self) -> Option { - self.iter.next_back().copied() - } -} - -impl ExactSizeIterator for ConstRegistryIter<'_> { - fn len(&self) -> usize { - self.iter.len() - } -} diff --git a/crates/wasmi/src/engine/translator/func/layout/mod.rs b/crates/wasmi/src/engine/translator/func/layout/mod.rs index bcf7330b00..e3c9d1d200 100644 --- a/crates/wasmi/src/engine/translator/func/layout/mod.rs +++ b/crates/wasmi/src/engine/translator/func/layout/mod.rs @@ -1,8 +1,5 @@ -mod consts; - -use self::consts::{ConstRegistry, ConstRegistryIter}; use super::{LocalIdx, Operand, OperandIdx, Reset}; -use crate::{core::UntypedVal, engine::TranslationError, ir::Slot, Error}; +use crate::{engine::TranslationError, ir::Slot, Error}; #[cfg(doc)] use super::Stack; @@ -12,14 +9,11 @@ use super::Stack; pub struct StackLayout { /// The number of locals registered to the function. len_locals: usize, - /// All function local constants. - consts: ConstRegistry, } impl Reset for StackLayout { fn reset(&mut self) { self.len_locals = 0; - self.consts.reset(); } } @@ -39,11 +33,7 @@ impl StackLayout { /// Returns `None` if the [`Slot`] is unknown to the [`Stack`]. #[must_use] pub fn stack_space(&self, slot: Slot) -> StackSpace { - let index = i16::from(slot); - if index.is_negative() { - return StackSpace::Const; - } - let index = index as u16; + let index = u16::from(slot); if usize::from(index) < self.len_locals { return StackSpace::Local; } @@ -58,7 +48,6 @@ impl StackLayout { /// /// - [`StackLayout::local_to_reg`] /// - [`StackLayout::temp_to_reg`] - /// - [`StackLayout::const_to_reg`] /// /// # Errors /// @@ -67,7 +56,7 @@ impl StackLayout { match operand { Operand::Local(operand) => self.local_to_reg(operand.local_index()), Operand::Temp(operand) => self.temp_to_reg(operand.operand_index()), - Operand::Immediate(operand) => self.const_to_reg(operand.val()), + Operand::Immediate(_) => panic!("function local constants have been removed"), // TODO: remove } } @@ -82,7 +71,7 @@ impl StackLayout { (u32::from(index) as usize) < self.len_locals, "out of bounds local operand index: {index:?}" ); - let Ok(index) = i16::try_from(u32::from(index)) else { + let Ok(index) = u16::try_from(u32::from(index)) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; Ok(Slot::from(index)) @@ -99,30 +88,11 @@ impl StackLayout { let Some(index) = index.checked_add(self.len_locals) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; - let Ok(index) = i16::try_from(index) else { + let Ok(index) = u16::try_from(index) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; Ok(Slot::from(index)) } - - /// Allocates a function local constant `value`. - /// - /// # Errors - /// - /// If too many function local constants have been allocated already. - #[inline] - pub fn const_to_reg(&mut self, value: impl Into) -> Result { - self.consts.alloc(value.into()) - } - - /// Returns an iterator yielding all function local constants. - /// - /// # Note - /// - /// The function local constant are yielded in reverse order of allocation. - pub fn consts(&self) -> ConstRegistryIter<'_> { - self.consts.iter() - } } /// The [`StackSpace`] of a [`Slot`]. @@ -130,8 +100,6 @@ impl StackLayout { pub enum StackSpace { /// Stack slot referring to a local variable. Local, - /// Stack slot referring to a function local constant value. - Const, /// Stack slot referring to a temporary stack operand. Temp, } From 1f34e41c4f2c6d5d03660af13e1b7ede1a633bad Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:54:23 +0200 Subject: [PATCH 068/423] rename layout/mod.rs -> layout.rs --- .../wasmi/src/engine/translator/func/{layout/mod.rs => layout.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crates/wasmi/src/engine/translator/func/{layout/mod.rs => layout.rs} (100%) diff --git a/crates/wasmi/src/engine/translator/func/layout/mod.rs b/crates/wasmi/src/engine/translator/func/layout.rs similarity index 100% rename from crates/wasmi/src/engine/translator/func/layout/mod.rs rename to crates/wasmi/src/engine/translator/func/layout.rs From 288d16b9b432cb3590aac379433649b1bb5469a5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 11:59:08 +0200 Subject: [PATCH 069/423] fix translate_simd_splat --- crates/wasmi/src/engine/translator/func/simd/mod.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 8ff2bc7c50..51920ee49d 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -36,15 +36,14 @@ impl FuncTranslator { { bail_unreachable!(self); let value = self.stack.pop(); - let value: Input = self.make_input(value, |_this, value| Ok(value))?; + let value = self.make_input(value, |_this, value| { + Ok(Input::Immediate(T::from(value).wrap().to_bits())) + })?; self.push_instr_with_result( ValType::V128, |result| match value { - Input::Slot(value) => make_instr_si(result, value), - Input::Immediate(value) => { - let value = T::from(value).wrap().to_bits(); - make_instr_si(result, value) - } + Input::Slot(value) => make_instr_ss(result, value), + Input::Immediate(value) => make_instr_si(result, value), }, FuelCostsProvider::simd, )?; From ffb8621260d94e9875a6f0f28fee6ff1eab72b00 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 12:08:41 +0200 Subject: [PATCH 070/423] fix mul_wide ops translation --- crates/wasmi/src/engine/translator/func/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index bde41d1ec9..844a47c92b 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -2298,21 +2298,21 @@ impl FuncTranslator { self.stack.push_immediate(result_hi)?; return Ok(()); } - (lhs, Operand::Immediate(rhs)) => { - let rhs = rhs.val(); - if self.try_opt_i64_mul_wide_sx(lhs, rhs, signed)? { + (lhs, Operand::Immediate(rhs_imm)) => { + let rhs_val = rhs_imm.val(); + if self.try_opt_i64_mul_wide_sx(lhs, rhs_val, signed)? { return Ok(()); } let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.const_to_reg(rhs)?; + let rhs = self.copy_if_immediate(rhs)?; (lhs, rhs) } - (Operand::Immediate(lhs), rhs) => { - let lhs = lhs.val(); - if self.try_opt_i64_mul_wide_sx(rhs, lhs, signed)? { + (Operand::Immediate(lhs_imm), rhs) => { + let lhs_val = lhs_imm.val(); + if self.try_opt_i64_mul_wide_sx(rhs, lhs_val, signed)? { return Ok(()); } - let lhs = self.layout.const_to_reg(lhs)?; + let lhs = self.copy_if_immediate(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; (lhs, rhs) } From 8361db0ef63cb3d4e0f7556280efda78028efa1c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 12:08:48 +0200 Subject: [PATCH 071/423] fix simd replace_lane ops translation --- .../src/engine/translator/func/simd/mod.rs | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 51920ee49d..3c969157b0 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -97,31 +97,17 @@ impl FuncTranslator { return Ok(()); } let input = self.layout.operand_to_reg(input)?; - let value = - self.make_input::(value, |this, value| { - match T::value_to_imm(T::Item::from(value)) { - Some(imm) => Ok(Input::Immediate(imm)), - None => { - let imm = this.layout.const_to_reg(value)?; - Ok(Input::Slot(imm)) - } - } - })?; - let param = match value { - Input::Slot(value) => Some(Op::slot(value)), - Input::Immediate(value) => T::replace_lane_imm_param(value), - }; + let value = self.make_input::(value, |_this, value| { + Ok(Input::Immediate(T::into_immediate(T::Item::from(value)))) + })?; self.push_instr_with_result( ::TY, |result| match value { - Input::Slot(_) => T::replace_lane(result, input, lane), - Input::Immediate(value) => T::replace_lane_imm(result, input, lane, value), + Input::Slot(value) => T::replace_lane_sss(result, input, lane, value), + Input::Immediate(value) => T::replace_lane_ssi(result, input, lane, value), }, FuelCostsProvider::simd, )?; - if let Some(param) = param { - self.push_param(param)?; - } Ok(()) } From 55f65d4b0ba2f639a36496ddd200d8c51ef9b771 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 12:11:17 +0200 Subject: [PATCH 072/423] fix simd swizzle translation --- crates/wasmi/src/engine/translator/func/simd/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 98a19b5bd6..ce11f5b0e5 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -389,7 +389,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_swizzle(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_swizzle, simd::i8x16_swizzle) + self.translate_simd_binary(Op::i8x16_swizzle_sss, simd::i8x16_swizzle) } fn visit_i8x16_splat(&mut self) -> Self::Output { From 5882a6ff65b6675c4ebcf731bf6a26399fa08352 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 12:11:28 +0200 Subject: [PATCH 073/423] fix simd extract_lane ops translation --- .../src/engine/translator/func/simd/visit.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index ce11f5b0e5..a3510dcc1c 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -303,7 +303,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i8x16_extract_lane_s, + Op::s8x16_extract_lane, simd::i8x16_extract_lane_s, ) } @@ -311,7 +311,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i8x16_extract_lane_u, + Op::u8x16_extract_lane, simd::i8x16_extract_lane_u, ) } @@ -319,7 +319,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i16x8_extract_lane_s, + Op::s16x8_extract_lane, simd::i16x8_extract_lane_s, ) } @@ -327,7 +327,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i16x8_extract_lane_u, + Op::u16x8_extract_lane, simd::i16x8_extract_lane_u, ) } @@ -335,7 +335,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i32x4_extract_lane, + Op::u32x4_extract_lane, simd::i32x4_extract_lane, ) } @@ -343,7 +343,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::i64x2_extract_lane, + Op::u64x2_extract_lane, simd::i64x2_extract_lane, ) } @@ -351,7 +351,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::f32x4_extract_lane, + Op::u32x4_extract_lane, simd::f32x4_extract_lane, ) } @@ -359,7 +359,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { self.translate_extract_lane::( lane, - Op::f64x2_extract_lane, + Op::u64x2_extract_lane, simd::f64x2_extract_lane, ) } From 62436e592347c37e6df3f41d22ec6c18af8d6a8e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 12:14:04 +0200 Subject: [PATCH 074/423] fix i8x16_shuffle translation --- crates/wasmi/src/engine/translator/func/simd/visit.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index a3510dcc1c..af628d2ab5 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -1,7 +1,7 @@ use super::FuncTranslator; use crate::{ core::{ - simd::{self, ImmLaneIdx32}, + simd::{self, ImmLaneIdx}, FuelCostsProvider, TypedVal, }, @@ -275,8 +275,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { bail_unreachable!(self); - let selector: [ImmLaneIdx32; 16] = array::from_fn(|i| { - let Ok(lane) = ImmLaneIdx32::try_from(lanes[i]) else { + let selector: [ImmLaneIdx<32>; 16] = array::from_fn(|i| { + let Ok(lane) = >::try_from(lanes[i]) else { panic!("encountered out of bounds lane at index {i}: {}", lanes[i]) }; lane @@ -289,9 +289,6 @@ impl VisitSimdOperator<'_> for FuncTranslator { } let lhs = self.layout.operand_to_reg(lhs)?; let rhs = self.layout.operand_to_reg(rhs)?; - let selector = self - .layout - .const_to_reg(V128::from(u128::from_ne_bytes(lanes)))?; self.push_instr_with_result( ValType::V128, |result| Op::i8x16_shuffle(result, lhs, rhs, selector), From 7ff7bfa35bdbcbccc59b11952fd3c2478a123bfa Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 12:18:28 +0200 Subject: [PATCH 075/423] fix most ternary simd ops translation --- crates/wasmi/src/engine/translator/func/simd/mod.rs | 11 +++++------ .../wasmi/src/engine/translator/func/simd/visit.rs | 12 ++++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 3c969157b0..e2d6359d79 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -164,7 +164,7 @@ impl FuncTranslator { /// Generically translate a Wasm ternary instruction. fn translate_simd_ternary( &mut self, - make_instr: fn(result: Slot, a: Slot, b: Slot) -> Op, + make_instr: fn(result: Slot, a: Slot, b: Slot, c: Slot) -> Op, const_eval: fn(lhas: V128, b: V128, c: V128) -> V128, ) -> Result<(), Error> { bail_unreachable!(self); @@ -176,15 +176,14 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let lhs = self.layout.operand_to_reg(a)?; - let rhs = self.layout.operand_to_reg(b)?; - let selector = self.layout.operand_to_reg(c)?; + let lhs = self.copy_if_immediate(a)?; + let rhs = self.copy_if_immediate(b)?; + let selector = self.copy_if_immediate(c)?; self.push_instr_with_result( ValType::V128, - |result| make_instr(result, lhs, rhs), + |result| make_instr(result, lhs, rhs, selector), FuelCostsProvider::simd, )?; - self.push_param(Op::slot(selector))?; Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index af628d2ab5..5f78d163c4 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -1258,19 +1258,19 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_f32x4_relaxed_madd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f32x4_relaxed_madd, simd::f32x4_relaxed_madd) + self.translate_simd_ternary(Op::f32x4_relaxed_madd_sss, simd::f32x4_relaxed_madd) } fn visit_f32x4_relaxed_nmadd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f32x4_relaxed_nmadd, simd::f32x4_relaxed_nmadd) + self.translate_simd_ternary(Op::f32x4_relaxed_nmadd_sss, simd::f32x4_relaxed_nmadd) } fn visit_f64x2_relaxed_madd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f64x2_relaxed_madd, simd::f64x2_relaxed_madd) + self.translate_simd_ternary(Op::f64x2_relaxed_madd_sss, simd::f64x2_relaxed_madd) } fn visit_f64x2_relaxed_nmadd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f64x2_relaxed_nmadd, simd::f64x2_relaxed_nmadd) + self.translate_simd_ternary(Op::f64x2_relaxed_nmadd_sss, simd::f64x2_relaxed_nmadd) } fn visit_i8x16_relaxed_laneselect(&mut self) -> Self::Output { @@ -1311,14 +1311,14 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i16x8_relaxed_dot_i8x16_i7x16_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i16x8_relaxed_dot_i8x16_i7x16_s, + Op::i16x8_relaxed_dot_i8x16_i7x16_sss, simd::i16x8_relaxed_dot_i8x16_i7x16_s, ) } fn visit_i32x4_relaxed_dot_i8x16_i7x16_add_s(&mut self) -> Self::Output { self.translate_simd_ternary( - Op::i32x4_relaxed_dot_i8x16_i7x16_add_s, + Op::i32x4_relaxed_dot_i8x16_i7x16_add_sss, simd::i32x4_relaxed_dot_i8x16_i7x16_add_s, ) } From 96f6c78af3d24487d7fd3e185040c0e2ed5582b8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 14:52:26 +0200 Subject: [PATCH 076/423] fix translate_v128_load_lane method --- crates/wasmi/src/engine/translator/func/simd/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index e2d6359d79..4b1418a9a4 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -243,8 +243,8 @@ impl FuncTranslator { ) -> Op, load_lane_mem0_offset16: fn( result: Slot, - address: Address, - memory: index::Memory, + ptr: Slot, + offset: Offset16, v128: Slot, lane: T::LaneIdx, ) -> Op, From d939cb4dd79bce1fc95920bf62c4f542d8f65aa6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 15:07:25 +0200 Subject: [PATCH 077/423] update simd store_lane ops translation --- .../wasmi/src/engine/translator/func/mod.rs | 13 ++++ .../src/engine/translator/func/simd/mod.rs | 60 +++++++------------ .../src/engine/translator/func/simd/visit.rs | 28 ++++----- 3 files changed, 46 insertions(+), 55 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 844a47c92b..afa7ae9493 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -2185,6 +2185,19 @@ impl FuncTranslator { { bail_unreachable!(self); let (ptr, value) = self.stack.pop2(); + self.encode_store::(memarg, ptr, value) + } + + fn encode_store( + &mut self, + memarg: MemArg, + ptr: Operand, + value: Operand, + ) -> Result<(), Error> + where + T::Value: Copy + From, + T::Immediate: Copy, + { let (memory, offset) = Self::decode_memarg(memarg)?; todo!() } diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 4b1418a9a4..00a95555ba 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -280,9 +280,13 @@ impl FuncTranslator { &mut self, memarg: MemArg, lane: u8, - make_instr: fn(ptr: Slot, offset_lo: Offset64Lo) -> Op, - make_instr_offset8: fn(ptr: Slot, value: Slot, offset: Offset8, lane: T::LaneIdx) -> Op, - make_instr_at: fn(value: Slot, address: Address32) -> Op, + make_instr: fn(ptr: Slot, offset: u64, value: Slot, memory: Memory, lane: T::LaneIdx) -> Op, + make_instr_mem0_offset16: fn( + ptr: Slot, + offset: Offset16, + value: Slot, + lane: T::LaneIdx, + ) -> Op, translate_imm: fn( &mut Self, memarg: MemArg, @@ -302,46 +306,24 @@ impl FuncTranslator { // lane value and translate as a more efficient non-SIMD operation. return translate_imm(self, memarg, ptr, lane, V128::from(v128.val())); } - v128 => self.layout.operand_to_reg(v128)?, + Operand::Local(v128) => self.layout.local_to_reg(v128.local_index())?, + Operand::Temp(v128) => self.layout.temp_to_reg(v128.operand_index())?, }; let (memory, offset) = Self::decode_memarg(memarg)?; - let (ptr, offset) = match ptr { - Operand::Immediate(ptr) => { - let Some(address) = self.effective_address(memory, ptr.val(), offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; - if let Ok(address) = Address32::try_from(address) { - return self.translate_v128_store_lane_at::( - memory, - address, - v128, - lane, - make_instr_at, - ); - } - // Case: we cannot use specialized encoding and thus have to fall back - // to the general case where `ptr` is zero and `offset` stores the - // `ptr+offset` address value. - let zero_ptr = self.layout.const_to_reg(0_u64)?; - (zero_ptr, u64::from(address)) - } - ptr => { - let ptr = self.layout.operand_to_reg(ptr)?; - (ptr, offset) + let ptr = self.copy_if_immediate(ptr)?; + if memory.is_default() { + if let Ok(offset16) = Offset16::try_from(offset) { + self.push_instr( + make_instr_mem0_offset16(ptr, offset16, v128, lane), + FuelCostsProvider::store, + )?; + return Ok(()); } - }; - if let Ok(Some(_)) = - self.translate_v128_store_lane_mem0(memory, ptr, offset, v128, lane, make_instr_offset8) - { - return Ok(()); } - let (offset_hi, offset_lo) = Offset64::split(offset); - let instr = make_instr(ptr, offset_lo); - let param = Op::slot_and_offset_hi(v128, offset_hi); - let param2 = Op::lane_and_memory_index(lane, memory); - self.push_instr(instr, FuelCostsProvider::store)?; - self.push_param(param)?; - self.push_param(param2)?; + self.push_instr( + make_instr(ptr, offset, v128, memory, lane), + FuelCostsProvider::store, + )?; Ok(()) } diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 5f78d163c4..c54d6a8548 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -210,13 +210,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store8_lane, - Op::v128_store8_lane_offset8, - Op::v128_store8_lane_at, + Op::v128_store8_lane_ss, + Op::v128_store8_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i8x16_extract_lane_s(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } @@ -225,13 +224,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store16_lane, - Op::v128_store16_lane_offset8, - Op::v128_store16_lane_at, + Op::v128_store16_lane_ss, + Op::v128_store16_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i16x8_extract_lane_s(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } @@ -240,13 +238,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store32_lane, - Op::v128_store32_lane_offset8, - Op::v128_store32_lane_at, + Op::v128_store32_lane_ss, + Op::v128_store32_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i32x4_extract_lane(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } @@ -255,13 +252,12 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.translate_v128_store_lane::( memarg, lane, - Op::v128_store64_lane, - Op::v128_store64_lane_offset8, - Op::v128_store64_lane_at, + Op::v128_store64_lane_ss, + Op::v128_store64_lane_mem0_offset16_ss, |this, memarg, ptr, lane, v128| { let value = simd::i64x2_extract_lane(v128, lane); let value = this.immediate_to_operand(value)?; - this.encode_istore_wrap::(memarg, ptr, value) + this.encode_store::(memarg, ptr, value) }, ) } From e2f9759833899ba7207204bfe4f8c610a7a30029 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 15:40:34 +0200 Subject: [PATCH 078/423] remove unused store_lane translation code --- .../src/engine/translator/func/simd/mod.rs | 39 +------------------ 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 00a95555ba..bdbaf468ad 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -7,17 +7,15 @@ use crate::{ core::{simd::IntoLaneIdx, FuelCostsProvider, Typed, TypedVal}, engine::translator::{ func::{utils::Input, Operand}, - utils::{Instr, IntoShiftAmount, ToBits, Wrap}, + utils::{IntoShiftAmount, ToBits, Wrap}, }, ir::{ index::{self, Memory}, - Address, Offset16, Op, Slot, }, Error, - TrapCode, ValType, V128, }; @@ -326,39 +324,4 @@ impl FuncTranslator { )?; Ok(()) } - - fn translate_v128_store_lane_at( - &mut self, - memory: index::Memory, - address: Address32, - value: Slot, - lane: T::LaneIdx, - make_instr_at: fn(value: Slot, address: Address32) -> Op, - ) -> Result<(), Error> { - self.push_instr(make_instr_at(value, address), FuelCostsProvider::store)?; - self.push_param(Op::lane_and_memory_index(lane, memory))?; - Ok(()) - } - - fn translate_v128_store_lane_mem0( - &mut self, - memory: Memory, - ptr: Slot, - offset: u64, - value: Slot, - lane: LaneType, - make_instr_offset8: fn(Slot, Slot, Offset8, LaneType) -> Op, - ) -> Result, Error> { - if !memory.is_default() { - return Ok(None); - } - let Ok(offset8) = Offset8::try_from(offset) else { - return Ok(None); - }; - let instr = self.push_instr( - make_instr_offset8(ptr, value, offset8, lane), - FuelCostsProvider::store, - )?; - Ok(Some(instr)) - } } From 66e593a6858c834428fac3ee0db649e9befd719c Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 15:40:47 +0200 Subject: [PATCH 079/423] update simd load operator translation --- .../src/engine/translator/func/simd/op.rs | 95 +++++++++++++++- .../src/engine/translator/func/simd/visit.rs | 106 +++--------------- 2 files changed, 107 insertions(+), 94 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/op.rs b/crates/wasmi/src/engine/translator/func/simd/op.rs index c2d5f84670..21e548d372 100644 --- a/crates/wasmi/src/engine/translator/func/simd/op.rs +++ b/crates/wasmi/src/engine/translator/func/simd/op.rs @@ -1,8 +1,9 @@ use super::IntoLaneIdx; use crate::{ core::{simd, Typed}, - engine::translator::utils::ToBits, - ir::{Op, Slot}, + engine::translator::{func::op::LoadOperator, utils::ToBits}, + ir::{index::Memory, Offset16, Op, Slot}, + ValType, V128, }; @@ -149,3 +150,93 @@ impl_replace_lane! { fn replace_lane_ssi = Op::v128_replace_lane64x2_ssi; } } + +macro_rules! impl_simd_load { + ( $( + impl LoadOperator for $name:ident { + fn load_ss = $store_ss:expr; + fn load_mem0_offset16_ss = $store_mem0_offset16_ss:expr; + } + )* ) => { + $( + pub enum $name {} + impl LoadOperator for $name { + const LOADED_TY: ValType = ValType::V128; + + fn load_ss(result: Slot, ptr: Slot, offset: u64, memory: Memory) -> Op { + $store_ss(result, ptr, offset, memory) + } + + fn load_mem0_offset16_ss(result: Slot, ptr: Slot, offset: Offset16) -> Op { + $store_mem0_offset16_ss(result, ptr, offset) + } + } + )* + }; +} +impl_simd_load! { + impl LoadOperator for V128Load { + fn load_ss = Op::v128_load_ss; + fn load_mem0_offset16_ss = Op::v128_load_mem0_offset16_ss; + } + + impl LoadOperator for I16x8Load8x8 { + fn load_ss = Op::i16x8_load8x8_ss; + fn load_mem0_offset16_ss = Op::i16x8_load8x8_mem0_offset16_ss; + } + + impl LoadOperator for U16x8Load8x8 { + fn load_ss = Op::u16x8_load8x8_ss; + fn load_mem0_offset16_ss = Op::u16x8_load8x8_mem0_offset16_ss; + } + + impl LoadOperator for I32x4Load16x4 { + fn load_ss = Op::i32x4_load16x4_ss; + fn load_mem0_offset16_ss = Op::i32x4_load16x4_mem0_offset16_ss; + } + + impl LoadOperator for U32x4Load16x4 { + fn load_ss = Op::u32x4_load16x4_ss; + fn load_mem0_offset16_ss = Op::u32x4_load16x4_mem0_offset16_ss; + } + + impl LoadOperator for I64x2Load32x2 { + fn load_ss = Op::i64x2_load32x2_ss; + fn load_mem0_offset16_ss = Op::i64x2_load32x2_mem0_offset16_ss; + } + + impl LoadOperator for U64x2Load32x2 { + fn load_ss = Op::u64x2_load32x2_ss; + fn load_mem0_offset16_ss = Op::u64x2_load32x2_mem0_offset16_ss; + } + + impl LoadOperator for V128Load8Splat { + fn load_ss = Op::v128_load8_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load8_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load16Splat { + fn load_ss = Op::v128_load16_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load16_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load32Splat { + fn load_ss = Op::v128_load32_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load32_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load64Splat { + fn load_ss = Op::v128_load64_splat_ss; + fn load_mem0_offset16_ss = Op::v128_load64_splat_mem0_offset16_ss; + } + + impl LoadOperator for V128Load32Zero { + fn load_ss = Op::v128_load32_zero_ss; + fn load_mem0_offset16_ss = Op::v128_load32_zero_mem0_offset16_ss; + } + + impl LoadOperator for V128Load64Zero { + fn load_ss = Op::v128_load64_zero_ss; + fn load_mem0_offset16_ss = Op::v128_load64_zero_mem0_offset16_ss; + } +} diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index c54d6a8548..140d9b10ec 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -31,136 +31,58 @@ macro_rules! swap_ops { impl VisitSimdOperator<'_> for FuncTranslator { fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load_ss, - None, - Op::v128_load_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::s16x8_load8x8_ss, - None, - Op::s16x8_load8x8_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::u16x8_load8x8_ss, - None, - Op::u16x8_load8x8_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::s32x4_load16x4_ss, - None, - Op::s32x4_load16x4_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::u32x4_load16x4_ss, - None, - Op::u32x4_load16x4_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::s64x2_load32x2_ss, - None, - Op::s64x2_load32x2_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::u64x2_load32x2_ss, - None, - Op::u64x2_load32x2_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load8_splat_ss, - None, - Op::v128_load8_splat_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load16_splat_ss, - None, - Op::v128_load16_splat_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load32_splat_ss, - None, - Op::v128_load32_splat_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load64_splat_ss, - None, - Op::v128_load64_splat_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load32_zero_ss, - None, - Op::v128_load32_zero_mem0_offset16_ss, - ) + self.translate_load::(memarg) } fn visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output { - self.translate_load( - memarg, - ValType::V128, - Op::v128_load64_zero_ss, - None, - Op::v128_load64_zero_mem0_offset16_ss, - ) + self.translate_load::(memarg) } - fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { + fn visit_v128_store(&mut self, _memarg: MemArg) -> Self::Output { // self.translate_store( // memarg, // Op::v128_store, From 331cefbb7153e04b5ea9f5143e975d9d64cacbb6 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 16:40:21 +0200 Subject: [PATCH 080/423] update many simd ops translation --- .../src/engine/translator/func/simd/visit.rs | 406 ++++++++++-------- 1 file changed, 224 insertions(+), 182 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 140d9b10ec..82e3ff954d 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -332,215 +332,215 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_eq, simd::i8x16_eq) + self.translate_simd_binary(Op::i8x16_eq_sss, simd::i8x16_eq) } fn visit_i8x16_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_ne, simd::i8x16_ne) + self.translate_simd_binary(Op::i8x16_not_eq_sss, simd::i8x16_ne) } fn visit_i8x16_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_lt_s, simd::i8x16_lt_s) + self.translate_simd_binary(Op::i8x16_lt_sss, simd::i8x16_lt_s) } fn visit_i8x16_lt_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_lt_u, simd::i8x16_lt_u) + self.translate_simd_binary(Op::u8x16_lt_sss, simd::i8x16_lt_u) } fn visit_i8x16_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_lt_s), simd::i8x16_gt_s) + self.translate_simd_binary(swap_ops!(Op::i8x16_lt_sss), simd::i8x16_gt_s) } fn visit_i8x16_gt_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_lt_u), simd::i8x16_gt_u) + self.translate_simd_binary(swap_ops!(Op::u8x16_lt_sss), simd::i8x16_gt_u) } fn visit_i8x16_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_le_s, simd::i8x16_le_s) + self.translate_simd_binary(Op::i8x16_le_sss, simd::i8x16_le_s) } fn visit_i8x16_le_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_le_u, simd::i8x16_le_u) + self.translate_simd_binary(Op::u8x16_le_sss, simd::i8x16_le_u) } fn visit_i8x16_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_le_s), simd::i8x16_ge_s) + self.translate_simd_binary(swap_ops!(Op::i8x16_le_sss), simd::i8x16_ge_s) } fn visit_i8x16_ge_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i8x16_le_u), simd::i8x16_ge_u) + self.translate_simd_binary(swap_ops!(Op::u8x16_le_sss), simd::i8x16_ge_u) } fn visit_i16x8_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_eq, simd::i16x8_eq) + self.translate_simd_binary(Op::i16x8_eq_sss, simd::i16x8_eq) } fn visit_i16x8_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_ne, simd::i16x8_ne) + self.translate_simd_binary(Op::i16x8_not_eq_sss, simd::i16x8_ne) } fn visit_i16x8_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_lt_s, simd::i16x8_lt_s) + self.translate_simd_binary(Op::i16x8_lt_sss, simd::i16x8_lt_s) } fn visit_i16x8_lt_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_lt_u, simd::i16x8_lt_u) + self.translate_simd_binary(Op::u16x8_lt_sss, simd::i16x8_lt_u) } fn visit_i16x8_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_lt_s), simd::i16x8_gt_s) + self.translate_simd_binary(swap_ops!(Op::i16x8_lt_sss), simd::i16x8_gt_s) } fn visit_i16x8_gt_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_lt_u), simd::i16x8_gt_u) + self.translate_simd_binary(swap_ops!(Op::u16x8_lt_sss), simd::i16x8_gt_u) } fn visit_i16x8_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_le_s, simd::i16x8_le_s) + self.translate_simd_binary(Op::i16x8_le_sss, simd::i16x8_le_s) } fn visit_i16x8_le_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_le_u, simd::i16x8_le_u) + self.translate_simd_binary(Op::u16x8_le_sss, simd::i16x8_le_u) } fn visit_i16x8_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_le_s), simd::i16x8_ge_s) + self.translate_simd_binary(swap_ops!(Op::i16x8_le_sss), simd::i16x8_ge_s) } fn visit_i16x8_ge_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i16x8_le_u), simd::i16x8_ge_u) + self.translate_simd_binary(swap_ops!(Op::u16x8_le_sss), simd::i16x8_ge_u) } fn visit_i32x4_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_eq, simd::i32x4_eq) + self.translate_simd_binary(Op::i32x4_eq_sss, simd::i32x4_eq) } fn visit_i32x4_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_ne, simd::i32x4_ne) + self.translate_simd_binary(Op::i32x4_not_eq_sss, simd::i32x4_ne) } fn visit_i32x4_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_lt_s, simd::i32x4_lt_s) + self.translate_simd_binary(Op::i32x4_lt_sss, simd::i32x4_lt_s) } fn visit_i32x4_lt_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_lt_u, simd::i32x4_lt_u) + self.translate_simd_binary(Op::u32x4_lt_sss, simd::i32x4_lt_u) } fn visit_i32x4_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_lt_s), simd::i32x4_gt_s) + self.translate_simd_binary(swap_ops!(Op::i32x4_lt_sss), simd::i32x4_gt_s) } fn visit_i32x4_gt_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_lt_u), simd::i32x4_gt_u) + self.translate_simd_binary(swap_ops!(Op::u32x4_lt_sss), simd::i32x4_gt_u) } fn visit_i32x4_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_le_s, simd::i32x4_le_s) + self.translate_simd_binary(Op::i32x4_le_sss, simd::i32x4_le_s) } fn visit_i32x4_le_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_le_u, simd::i32x4_le_u) + self.translate_simd_binary(Op::u32x4_le_sss, simd::i32x4_le_u) } fn visit_i32x4_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_le_s), simd::i32x4_ge_s) + self.translate_simd_binary(swap_ops!(Op::i32x4_le_sss), simd::i32x4_ge_s) } fn visit_i32x4_ge_u(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i32x4_le_u), simd::i32x4_ge_u) + self.translate_simd_binary(swap_ops!(Op::u32x4_le_sss), simd::i32x4_ge_u) } fn visit_i64x2_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_eq, simd::i64x2_eq) + self.translate_simd_binary(Op::i64x2_eq_sss, simd::i64x2_eq) } fn visit_i64x2_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_ne, simd::i64x2_ne) + self.translate_simd_binary(Op::i64x2_not_eq_sss, simd::i64x2_ne) } fn visit_i64x2_lt_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_lt_s, simd::i64x2_lt_s) + self.translate_simd_binary(Op::i64x2_lt_sss, simd::i64x2_lt_s) } fn visit_i64x2_gt_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i64x2_lt_s), simd::i64x2_gt_s) + self.translate_simd_binary(swap_ops!(Op::i64x2_lt_sss), simd::i64x2_gt_s) } fn visit_i64x2_le_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_le_s, simd::i64x2_le_s) + self.translate_simd_binary(Op::i64x2_le_sss, simd::i64x2_le_s) } fn visit_i64x2_ge_s(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::i64x2_le_s), simd::i64x2_ge_s) + self.translate_simd_binary(swap_ops!(Op::i64x2_le_sss), simd::i64x2_ge_s) } fn visit_f32x4_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_eq, simd::f32x4_eq) + self.translate_simd_binary(Op::f32x4_eq_sss, simd::f32x4_eq) } fn visit_f32x4_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_ne, simd::f32x4_ne) + self.translate_simd_binary(Op::f32x4_not_eq_sss, simd::f32x4_ne) } fn visit_f32x4_lt(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_lt, simd::f32x4_lt) + self.translate_simd_binary(Op::f32x4_lt_sss, simd::f32x4_lt) } fn visit_f32x4_gt(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f32x4_lt), simd::f32x4_gt) + self.translate_simd_binary(swap_ops!(Op::f32x4_lt_sss), simd::f32x4_gt) } fn visit_f32x4_le(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_le, simd::f32x4_le) + self.translate_simd_binary(Op::f32x4_le_sss, simd::f32x4_le) } fn visit_f32x4_ge(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f32x4_le), simd::f32x4_ge) + self.translate_simd_binary(swap_ops!(Op::f32x4_le_sss), simd::f32x4_ge) } fn visit_f64x2_eq(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_eq, simd::f64x2_eq) + self.translate_simd_binary(Op::f64x2_eq_sss, simd::f64x2_eq) } fn visit_f64x2_ne(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_ne, simd::f64x2_ne) + self.translate_simd_binary(Op::f64x2_not_eq_sss, simd::f64x2_ne) } fn visit_f64x2_lt(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_lt, simd::f64x2_lt) + self.translate_simd_binary(Op::f64x2_lt_sss, simd::f64x2_lt) } fn visit_f64x2_gt(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f64x2_lt), simd::f64x2_gt) + self.translate_simd_binary(swap_ops!(Op::f64x2_lt_sss), simd::f64x2_gt) } fn visit_f64x2_le(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_le, simd::f64x2_le) + self.translate_simd_binary(Op::f64x2_le_sss, simd::f64x2_le) } fn visit_f64x2_ge(&mut self) -> Self::Output { - self.translate_simd_binary(swap_ops!(Op::f64x2_le), simd::f64x2_ge) + self.translate_simd_binary(swap_ops!(Op::f64x2_le_sss), simd::f64x2_ge) } fn visit_v128_not(&mut self) -> Self::Output { - self.translate_simd_unary(Op::v128_not, simd::v128_not) + self.translate_simd_unary(Op::v128_not_ss, simd::v128_not) } fn visit_v128_and(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_and, simd::v128_and) + self.translate_simd_binary(Op::v128_and_sss, simd::v128_and) } fn visit_v128_andnot(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_andnot, simd::v128_andnot) + self.translate_simd_binary(Op::v128_and_not_sss, simd::v128_andnot) } fn visit_v128_or(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_or, simd::v128_or) + self.translate_simd_binary(Op::v128_or_sss, simd::v128_or) } fn visit_v128_xor(&mut self) -> Self::Output { - self.translate_simd_binary(Op::v128_xor, simd::v128_xor) + self.translate_simd_binary(Op::v128_xor_sss, simd::v128_xor) } fn visit_v128_bitselect(&mut self) -> Self::Output { @@ -548,35 +548,35 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_v128_any_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::v128_any_true, simd::v128_any_true) + self.translate_simd_unary(Op::v128_any_true_ss, simd::v128_any_true) } fn visit_i8x16_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_abs, simd::i8x16_abs) + self.translate_simd_unary(Op::i8x16_abs_ss, simd::i8x16_abs) } fn visit_i8x16_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_neg, simd::i8x16_neg) + self.translate_simd_unary(Op::i8x16_neg_ss, simd::i8x16_neg) } fn visit_i8x16_popcnt(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_popcnt, simd::i8x16_popcnt) + self.translate_simd_unary(Op::i8x16_popcnt_ss, simd::i8x16_popcnt) } fn visit_i8x16_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_all_true, simd::i8x16_all_true) + self.translate_simd_unary(Op::i8x16_all_true_ss, simd::i8x16_all_true) } fn visit_i8x16_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i8x16_bitmask, simd::i8x16_bitmask) + self.translate_simd_unary(Op::i8x16_bitmask_ss, simd::i8x16_bitmask) } fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_narrow_i16x8_s, simd::i8x16_narrow_i16x8_s) + self.translate_simd_binary(Op::i8x16_narrow_i16x8_sss, simd::i8x16_narrow_i16x8_s) } fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_narrow_i16x8_u, simd::i8x16_narrow_i16x8_u) + self.translate_simd_binary(Op::u8x16_narrow_i16x8_sss, simd::i8x16_narrow_i16x8_u) } fn visit_i8x16_shl(&mut self) -> Self::Output { @@ -584,7 +584,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::s8x16_shr_sss, Op::s8x16_shr_ssi, simd::i8x16_shr_s) + self.translate_simd_shift::(Op::i8x16_shr_sss, Op::i8x16_shr_ssi, simd::i8x16_shr_s) } fn visit_i8x16_shr_u(&mut self) -> Self::Output { @@ -592,109 +592,115 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i8x16_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_add, simd::i8x16_add) + self.translate_simd_binary(Op::i8x16_add_sss, simd::i8x16_add) } fn visit_i8x16_add_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_add_sat_s, simd::i8x16_add_sat_s) + self.translate_simd_binary(Op::i8x16_add_sat_sss, simd::i8x16_add_sat_s) } fn visit_i8x16_add_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_add_sat_u, simd::i8x16_add_sat_u) + self.translate_simd_binary(Op::u8x16_add_sat_sss, simd::i8x16_add_sat_u) } fn visit_i8x16_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_sub, simd::i8x16_sub) + self.translate_simd_binary(Op::i8x16_sub_sss, simd::i8x16_sub) } fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_sub_sat_s, simd::i8x16_sub_sat_s) + self.translate_simd_binary(Op::i8x16_sub_sat_sss, simd::i8x16_sub_sat_s) } fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_sub_sat_u, simd::i8x16_sub_sat_u) + self.translate_simd_binary(Op::u8x16_sub_sat_sss, simd::i8x16_sub_sat_u) } fn visit_i8x16_min_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_min_s, simd::i8x16_min_s) + self.translate_simd_binary(Op::i8x16_min_sss, simd::i8x16_min_s) } fn visit_i8x16_min_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_min_u, simd::i8x16_min_u) + self.translate_simd_binary(Op::u8x16_min_sss, simd::i8x16_min_u) } fn visit_i8x16_max_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_max_s, simd::i8x16_max_s) + self.translate_simd_binary(Op::i8x16_max_sss, simd::i8x16_max_s) } fn visit_i8x16_max_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_max_u, simd::i8x16_max_u) + self.translate_simd_binary(Op::u8x16_max_sss, simd::i8x16_max_u) } fn visit_i8x16_avgr_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i8x16_avgr_u, simd::i8x16_avgr_u) + self.translate_simd_binary(Op::u8x16_avgr_sss, simd::i8x16_avgr_u) } fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extadd_pairwise_i8x16_s, + Op::i16x8_extadd_pairwise_i8x16_ss, simd::i16x8_extadd_pairwise_i8x16_s, ) } fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extadd_pairwise_i8x16_u, + Op::u16x8_extadd_pairwise_i8x16_ss, simd::i16x8_extadd_pairwise_i8x16_u, ) } fn visit_i16x8_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_abs, simd::i16x8_abs) + self.translate_simd_unary(Op::i16x8_abs_ss, simd::i16x8_abs) } fn visit_i16x8_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_neg, simd::i16x8_neg) + self.translate_simd_unary(Op::i16x8_neg_ss, simd::i16x8_neg) } fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_q15mulr_sat_s, simd::i16x8_q15mulr_sat_s) + self.translate_simd_binary(Op::i16x8_q15_mulr_sat_sss, simd::i16x8_q15mulr_sat_s) } fn visit_i16x8_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_all_true, simd::i16x8_all_true) + self.translate_simd_unary(Op::i16x8_all_true_ss, simd::i16x8_all_true) } fn visit_i16x8_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_bitmask, simd::i16x8_bitmask) + self.translate_simd_unary(Op::i16x8_bitmask_ss, simd::i16x8_bitmask) } fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_narrow_i32x4_s, simd::i16x8_narrow_i32x4_s) + self.translate_simd_binary(Op::i16x8_narrow_i32x4_sss, simd::i16x8_narrow_i32x4_s) } fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_narrow_i32x4_u, simd::i16x8_narrow_i32x4_u) + self.translate_simd_binary(Op::u16x8_narrow_i32x4_sss, simd::i16x8_narrow_i32x4_u) } fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_extend_low_i8x16_s, simd::i16x8_extend_low_i8x16_s) + self.translate_simd_unary( + Op::i16x8_extend_low_i8x16_ss, + simd::i16x8_extend_low_i8x16_s, + ) } fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extend_high_i8x16_s, + Op::i16x8_extend_high_i8x16_ss, simd::i16x8_extend_high_i8x16_s, ) } fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i16x8_extend_low_i8x16_u, simd::i16x8_extend_low_i8x16_u) + self.translate_simd_unary( + Op::u16x8_extend_low_i8x16_ss, + simd::i16x8_extend_low_i8x16_u, + ) } fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i16x8_extend_high_i8x16_u, + Op::u16x8_extend_high_i8x16_ss, simd::i16x8_extend_high_i8x16_u, ) } @@ -704,7 +710,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i16x8_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::s16x8_shr_sss, Op::s16x8_shr_ssi, simd::i16x8_shr_s) + self.translate_simd_shift::(Op::i16x8_shr_sss, Op::i16x8_shr_ssi, simd::i16x8_shr_s) } fn visit_i16x8_shr_u(&mut self) -> Self::Output { @@ -712,123 +718,135 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i16x8_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_add, simd::i16x8_add) + self.translate_simd_binary(Op::i16x8_add_sss, simd::i16x8_add) } fn visit_i16x8_add_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_add_sat_s, simd::i16x8_add_sat_s) + self.translate_simd_binary(Op::i16x8_add_sat_sss, simd::i16x8_add_sat_s) } fn visit_i16x8_add_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_add_sat_u, simd::i16x8_add_sat_u) + self.translate_simd_binary(Op::u16x8_add_sat_sss, simd::i16x8_add_sat_u) } fn visit_i16x8_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_sub, simd::i16x8_sub) + self.translate_simd_binary(Op::i16x8_sub_sss, simd::i16x8_sub) } fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_sub_sat_s, simd::i16x8_sub_sat_s) + self.translate_simd_binary(Op::i16x8_sub_sat_sss, simd::i16x8_sub_sat_s) } fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_sub_sat_u, simd::i16x8_sub_sat_u) + self.translate_simd_binary(Op::u16x8_sub_sat_sss, simd::i16x8_sub_sat_u) } fn visit_i16x8_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_mul, simd::i16x8_mul) + self.translate_simd_binary(Op::i16x8_mul_sss, simd::i16x8_mul) } fn visit_i16x8_min_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_min_s, simd::i16x8_min_s) + self.translate_simd_binary(Op::i16x8_min_sss, simd::i16x8_min_s) } fn visit_i16x8_min_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_min_u, simd::i16x8_min_u) + self.translate_simd_binary(Op::u16x8_min_sss, simd::i16x8_min_u) } fn visit_i16x8_max_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_max_s, simd::i16x8_max_s) + self.translate_simd_binary(Op::i16x8_max_sss, simd::i16x8_max_s) } fn visit_i16x8_max_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_max_u, simd::i16x8_max_u) + self.translate_simd_binary(Op::u16x8_max_sss, simd::i16x8_max_u) } fn visit_i16x8_avgr_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_avgr_u, simd::i16x8_avgr_u) + self.translate_simd_binary(Op::u16x8_avgr_sss, simd::i16x8_avgr_u) } fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_extmul_low_i8x16_s, simd::i16x8_extmul_low_i8x16_s) + self.translate_simd_binary( + Op::i16x8_extmul_low_i8x16_sss, + simd::i16x8_extmul_low_i8x16_s, + ) } fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i16x8_extmul_high_i8x16_s, + Op::i16x8_extmul_high_i8x16_sss, simd::i16x8_extmul_high_i8x16_s, ) } fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i16x8_extmul_low_i8x16_u, simd::i16x8_extmul_low_i8x16_u) + self.translate_simd_binary( + Op::u16x8_extmul_low_i8x16_sss, + simd::i16x8_extmul_low_i8x16_u, + ) } fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i16x8_extmul_high_i8x16_u, + Op::u16x8_extmul_high_i8x16_sss, simd::i16x8_extmul_high_i8x16_u, ) } fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extadd_pairwise_i16x8_s, + Op::i32x4_extadd_pairwise_i16x8_ss, simd::i32x4_extadd_pairwise_i16x8_s, ) } fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extadd_pairwise_i16x8_u, + Op::u32x4_extadd_pairwise_i16x8_ss, simd::i32x4_extadd_pairwise_i16x8_u, ) } fn visit_i32x4_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_abs, simd::i32x4_abs) + self.translate_simd_unary(Op::i32x4_abs_ss, simd::i32x4_abs) } fn visit_i32x4_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_neg, simd::i32x4_neg) + self.translate_simd_unary(Op::i32x4_neg_ss, simd::i32x4_neg) } fn visit_i32x4_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_all_true, simd::i32x4_all_true) + self.translate_simd_unary(Op::i32x4_all_true_ss, simd::i32x4_all_true) } fn visit_i32x4_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_bitmask, simd::i32x4_bitmask) + self.translate_simd_unary(Op::i32x4_bitmask_ss, simd::i32x4_bitmask) } fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_extend_low_i16x8_s, simd::i32x4_extend_low_i16x8_s) + self.translate_simd_unary( + Op::i32x4_extend_low_i16x8_ss, + simd::i32x4_extend_low_i16x8_s, + ) } fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extend_high_i16x8_s, + Op::i32x4_extend_high_i16x8_ss, simd::i32x4_extend_high_i16x8_s, ) } fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_extend_low_i16x8_u, simd::i32x4_extend_low_i16x8_u) + self.translate_simd_unary( + Op::u32x4_extend_low_i16x8_ss, + simd::i32x4_extend_low_i16x8_u, + ) } fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_extend_high_i16x8_u, + Op::u32x4_extend_high_i16x8_ss, simd::i32x4_extend_high_i16x8_u, ) } @@ -838,7 +856,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i32x4_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::s32x4_shr_sss, Op::s32x4_shr_ssi, simd::i32x4_shr_s) + self.translate_simd_shift::(Op::i32x4_shr_sss, Op::i32x4_shr_ssi, simd::i32x4_shr_s) } fn visit_i32x4_shr_u(&mut self) -> Self::Output { @@ -846,93 +864,105 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i32x4_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_add, simd::i32x4_add) + self.translate_simd_binary(Op::i32x4_add_sss, simd::i32x4_add) } fn visit_i32x4_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_sub, simd::i32x4_sub) + self.translate_simd_binary(Op::i32x4_sub_sss, simd::i32x4_sub) } fn visit_i32x4_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_mul, simd::i32x4_mul) + self.translate_simd_binary(Op::i32x4_mul_sss, simd::i32x4_mul) } fn visit_i32x4_min_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_min_s, simd::i32x4_min_s) + self.translate_simd_binary(Op::i32x4_min_sss, simd::i32x4_min_s) } fn visit_i32x4_min_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_min_u, simd::i32x4_min_u) + self.translate_simd_binary(Op::u32x4_min_sss, simd::i32x4_min_u) } fn visit_i32x4_max_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_max_s, simd::i32x4_max_s) + self.translate_simd_binary(Op::i32x4_max_sss, simd::i32x4_max_s) } fn visit_i32x4_max_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_max_u, simd::i32x4_max_u) + self.translate_simd_binary(Op::u32x4_max_sss, simd::i32x4_max_u) } fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_dot_i16x8_s, simd::i32x4_dot_i16x8_s) + self.translate_simd_binary(Op::i32x4_dot_i16x8_sss, simd::i32x4_dot_i16x8_s) } fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_extmul_low_i16x8_s, simd::i32x4_extmul_low_i16x8_s) + self.translate_simd_binary( + Op::i32x4_extmul_low_i16x8_sss, + simd::i32x4_extmul_low_i16x8_s, + ) } fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i32x4_extmul_high_i16x8_s, + Op::i32x4_extmul_high_i16x8_sss, simd::i32x4_extmul_high_i16x8_s, ) } fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i32x4_extmul_low_i16x8_u, simd::i32x4_extmul_low_i16x8_u) + self.translate_simd_binary( + Op::u32x4_extmul_low_i16x8_sss, + simd::i32x4_extmul_low_i16x8_u, + ) } fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i32x4_extmul_high_i16x8_u, + Op::u32x4_extmul_high_i16x8_sss, simd::i32x4_extmul_high_i16x8_u, ) } fn visit_i64x2_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_abs, simd::i64x2_abs) + self.translate_simd_unary(Op::i64x2_abs_ss, simd::i64x2_abs) } fn visit_i64x2_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_neg, simd::i64x2_neg) + self.translate_simd_unary(Op::i64x2_neg_ss, simd::i64x2_neg) } fn visit_i64x2_all_true(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_all_true, simd::i64x2_all_true) + self.translate_simd_unary(Op::i64x2_all_true_ss, simd::i64x2_all_true) } fn visit_i64x2_bitmask(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_bitmask, simd::i64x2_bitmask) + self.translate_simd_unary(Op::i64x2_bitmask_ss, simd::i64x2_bitmask) } fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_extend_low_i32x4_s, simd::i64x2_extend_low_i32x4_s) + self.translate_simd_unary( + Op::i64x2_extend_low_i32x4_ss, + simd::i64x2_extend_low_i32x4_s, + ) } fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i64x2_extend_high_i32x4_s, + Op::i64x2_extend_high_i32x4_ss, simd::i64x2_extend_high_i32x4_s, ) } fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i64x2_extend_low_i32x4_u, simd::i64x2_extend_low_i32x4_u) + self.translate_simd_unary( + Op::u64x2_extend_low_i32x4_ss, + simd::i64x2_extend_low_i32x4_u, + ) } fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i64x2_extend_high_i32x4_u, + Op::u64x2_extend_high_i32x4_ss, simd::i64x2_extend_high_i32x4_u, ) } @@ -942,7 +972,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i64x2_shr_s(&mut self) -> Self::Output { - self.translate_simd_shift::(Op::s64x2_shr_sss, Op::s64x2_shr_ssi, simd::i64x2_shr_s) + self.translate_simd_shift::(Op::i64x2_shr_sss, Op::i64x2_shr_ssi, simd::i64x2_shr_s) } fn visit_i64x2_shr_u(&mut self) -> Self::Output { @@ -950,209 +980,221 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_i64x2_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_add, simd::i64x2_add) + self.translate_simd_binary(Op::i64x2_add_sss, simd::i64x2_add) } fn visit_i64x2_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_sub, simd::i64x2_sub) + self.translate_simd_binary(Op::i64x2_sub_sss, simd::i64x2_sub) } fn visit_i64x2_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_mul, simd::i64x2_mul) + self.translate_simd_binary(Op::i64x2_mul_sss, simd::i64x2_mul) } fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_extmul_low_i32x4_s, simd::i64x2_extmul_low_i32x4_s) + self.translate_simd_binary( + Op::i64x2_extmul_low_i32x4_sss, + simd::i64x2_extmul_low_i32x4_s, + ) } fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i64x2_extmul_high_i32x4_s, + Op::i64x2_extmul_high_i32x4_sss, simd::i64x2_extmul_high_i32x4_s, ) } fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_binary(Op::i64x2_extmul_low_i32x4_u, simd::i64x2_extmul_low_i32x4_u) + self.translate_simd_binary( + Op::i64x2_extmul_low_i32x4_sss, + simd::i64x2_extmul_low_i32x4_u, + ) } fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output { self.translate_simd_binary( - Op::i64x2_extmul_high_i32x4_u, + Op::u64x2_extmul_high_i32x4_sss, simd::i64x2_extmul_high_i32x4_u, ) } fn visit_f32x4_ceil(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_ceil, simd::f32x4_ceil) + self.translate_simd_unary(Op::f32x4_ceil_ss, simd::f32x4_ceil) } fn visit_f32x4_floor(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_floor, simd::f32x4_floor) + self.translate_simd_unary(Op::f32x4_floor_ss, simd::f32x4_floor) } fn visit_f32x4_trunc(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_trunc, simd::f32x4_trunc) + self.translate_simd_unary(Op::f32x4_trunc_ss, simd::f32x4_trunc) } fn visit_f32x4_nearest(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_nearest, simd::f32x4_nearest) + self.translate_simd_unary(Op::f32x4_nearest_ss, simd::f32x4_nearest) } fn visit_f32x4_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_abs, simd::f32x4_abs) + self.translate_simd_unary(Op::f32x4_abs_ss, simd::f32x4_abs) } fn visit_f32x4_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_neg, simd::f32x4_neg) + self.translate_simd_unary(Op::f32x4_neg_ss, simd::f32x4_neg) } fn visit_f32x4_sqrt(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_sqrt, simd::f32x4_sqrt) + self.translate_simd_unary(Op::f32x4_sqrt_ss, simd::f32x4_sqrt) } fn visit_f32x4_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_add, simd::f32x4_add) + self.translate_simd_binary(Op::f32x4_add_sss, simd::f32x4_add) } fn visit_f32x4_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_sub, simd::f32x4_sub) + self.translate_simd_binary(Op::f32x4_sub_sss, simd::f32x4_sub) } fn visit_f32x4_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_mul, simd::f32x4_mul) + self.translate_simd_binary(Op::f32x4_mul_sss, simd::f32x4_mul) } fn visit_f32x4_div(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_div, simd::f32x4_div) + self.translate_simd_binary(Op::f32x4_div_sss, simd::f32x4_div) } fn visit_f32x4_min(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_min, simd::f32x4_min) + self.translate_simd_binary(Op::f32x4_min_sss, simd::f32x4_min) } fn visit_f32x4_max(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_max, simd::f32x4_max) + self.translate_simd_binary(Op::f32x4_max_sss, simd::f32x4_max) } fn visit_f32x4_pmin(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_pmin, simd::f32x4_pmin) + self.translate_simd_binary(Op::f32x4_pmin_sss, simd::f32x4_pmin) } fn visit_f32x4_pmax(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f32x4_pmax, simd::f32x4_pmax) + self.translate_simd_binary(Op::f32x4_pmax_sss, simd::f32x4_pmax) } fn visit_f64x2_ceil(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_ceil, simd::f64x2_ceil) + self.translate_simd_unary(Op::f64x2_ceil_ss, simd::f64x2_ceil) } fn visit_f64x2_floor(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_floor, simd::f64x2_floor) + self.translate_simd_unary(Op::f64x2_floor_ss, simd::f64x2_floor) } fn visit_f64x2_trunc(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_trunc, simd::f64x2_trunc) + self.translate_simd_unary(Op::f64x2_trunc_ss, simd::f64x2_trunc) } fn visit_f64x2_nearest(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_nearest, simd::f64x2_nearest) + self.translate_simd_unary(Op::f64x2_nearest_ss, simd::f64x2_nearest) } fn visit_f64x2_abs(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_abs, simd::f64x2_abs) + self.translate_simd_unary(Op::f64x2_abs_ss, simd::f64x2_abs) } fn visit_f64x2_neg(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_neg, simd::f64x2_neg) + self.translate_simd_unary(Op::f64x2_neg_ss, simd::f64x2_neg) } fn visit_f64x2_sqrt(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_sqrt, simd::f64x2_sqrt) + self.translate_simd_unary(Op::f64x2_sqrt_ss, simd::f64x2_sqrt) } fn visit_f64x2_add(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_add, simd::f64x2_add) + self.translate_simd_binary(Op::f64x2_add_sss, simd::f64x2_add) } fn visit_f64x2_sub(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_sub, simd::f64x2_sub) + self.translate_simd_binary(Op::f64x2_sub_sss, simd::f64x2_sub) } fn visit_f64x2_mul(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_mul, simd::f64x2_mul) + self.translate_simd_binary(Op::f64x2_mul_sss, simd::f64x2_mul) } fn visit_f64x2_div(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_div, simd::f64x2_div) + self.translate_simd_binary(Op::f64x2_div_sss, simd::f64x2_div) } fn visit_f64x2_min(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_min, simd::f64x2_min) + self.translate_simd_binary(Op::f64x2_min_sss, simd::f64x2_min) } fn visit_f64x2_max(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_max, simd::f64x2_max) + self.translate_simd_binary(Op::f64x2_max_sss, simd::f64x2_max) } fn visit_f64x2_pmin(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_pmin, simd::f64x2_pmin) + self.translate_simd_binary(Op::f64x2_pmin_sss, simd::f64x2_pmin) } fn visit_f64x2_pmax(&mut self) -> Self::Output { - self.translate_simd_binary(Op::f64x2_pmax, simd::f64x2_pmax) + self.translate_simd_binary(Op::f64x2_pmax_sss, simd::f64x2_pmax) } fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_trunc_sat_f32x4_s, simd::i32x4_trunc_sat_f32x4_s) + self.translate_simd_unary(Op::i32x4_trunc_sat_f32x4_ss, simd::i32x4_trunc_sat_f32x4_s) } fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::i32x4_trunc_sat_f32x4_u, simd::i32x4_trunc_sat_f32x4_u) + self.translate_simd_unary(Op::u32x4_trunc_sat_f32x4_ss, simd::i32x4_trunc_sat_f32x4_u) } fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_convert_i32x4_s, simd::f32x4_convert_i32x4_s) + self.translate_simd_unary(Op::f32x4_convert_i32x4_ss, simd::f32x4_convert_i32x4_s) } fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_convert_i32x4_u, simd::f32x4_convert_i32x4_u) + self.translate_simd_unary(Op::f32x4_convert_u32x4_ss, simd::f32x4_convert_i32x4_u) } fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_trunc_sat_f64x2_s_zero, + Op::i32x4_trunc_sat_f64x2_zero_ss, simd::i32x4_trunc_sat_f64x2_s_zero, ) } fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_trunc_sat_f64x2_u_zero, + Op::u32x4_trunc_sat_f64x2_zero_ss, simd::i32x4_trunc_sat_f64x2_u_zero, ) } fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output { self.translate_simd_unary( - Op::f64x2_convert_low_i32x4_s, + Op::f64x2_convert_low_i32x4_ss, simd::f64x2_convert_low_i32x4_s, ) } fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output { self.translate_simd_unary( - Op::f64x2_convert_low_i32x4_u, + Op::f64x2_convert_low_u32x4_ss, simd::f64x2_convert_low_i32x4_u, ) } fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f32x4_demote_f64x2_zero, simd::f32x4_demote_f64x2_zero) + self.translate_simd_unary( + Op::f32x4_demote_f64x2_zero_ss, + simd::f32x4_demote_f64x2_zero, + ) } fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output { - self.translate_simd_unary(Op::f64x2_promote_low_f32x4, simd::f64x2_promote_low_f32x4) + self.translate_simd_unary( + Op::f64x2_promote_low_f32x4_ss, + simd::f64x2_promote_low_f32x4, + ) } fn visit_i8x16_relaxed_swizzle(&mut self) -> Self::Output { From 745f39ba8098afde241f7ae276e3ec0d2e9d4950 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Mon, 15 Sep 2025 16:40:47 +0200 Subject: [PATCH 081/423] remove consts from CompiledFuncEntity Function local constants will no longer exist after this refactoring. --- crates/wasmi/src/engine/code_map.rs | 18 ++---------------- crates/wasmi/src/engine/translator/func/mod.rs | 12 ++---------- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/crates/wasmi/src/engine/code_map.rs b/crates/wasmi/src/engine/code_map.rs index 75959cc91a..0ffe559e0e 100644 --- a/crates/wasmi/src/engine/code_map.rs +++ b/crates/wasmi/src/engine/code_map.rs @@ -8,7 +8,7 @@ use super::{FuncTranslationDriver, FuncTranslator, TranslationError, ValidatingFuncTranslator}; use crate::{ collections::arena::{Arena, ArenaIndex}, - core::{Fuel, FuelCostsProvider, UntypedVal}, + core::{Fuel, FuelCostsProvider}, engine::{utils::unreachable_unchecked, ResumableOutOfFuelError}, errors::FuelError, ir::{index::InternalFunc, Op}, @@ -786,8 +786,6 @@ impl<'a> From<&'a [u8]> for SmallByteSlice { pub struct CompiledFuncEntity { /// The sequence of [`Op`] of the [`CompiledFuncEntity`]. instrs: Pin>, - /// The constant values local to the [`EngineFunc`]. - consts: Pin>, /// The number of stack slots used by the [`EngineFunc`] in total. /// /// # Note @@ -804,13 +802,11 @@ impl CompiledFuncEntity { /// /// - If `instrs` is empty. /// - If `instrs` contains more than `i32::MAX` instructions. - pub fn new(len_stack_slots: u16, instrs: I, consts: C) -> Self + pub fn new(len_stack_slots: u16, instrs: I) -> Self where I: IntoIterator, - C: IntoIterator, { let instrs: Pin> = Pin::new(instrs.into_iter().collect()); - let consts: Pin> = Pin::new(consts.into_iter().collect()); assert!( !instrs.is_empty(), "compiled functions must have at least one instruction" @@ -826,7 +822,6 @@ impl CompiledFuncEntity { ); Self { instrs, - consts, len_stack_slots, } } @@ -837,8 +832,6 @@ impl CompiledFuncEntity { pub struct CompiledFuncRef<'a> { /// The sequence of [`Op`] of the [`CompiledFuncEntity`]. instrs: Pin<&'a [Op]>, - /// The constant values local to the [`EngineFunc`]. - consts: Pin<&'a [UntypedVal]>, /// The number of stack slots used by the [`EngineFunc`] in total. len_stack_slots: u16, } @@ -848,7 +841,6 @@ impl<'a> From<&'a CompiledFuncEntity> for CompiledFuncRef<'a> { fn from(func: &'a CompiledFuncEntity) -> Self { Self { instrs: func.instrs.as_ref(), - consts: func.consts.as_ref(), len_stack_slots: func.len_stack_slots, } } @@ -866,10 +858,4 @@ impl<'a> CompiledFuncRef<'a> { pub fn len_stack_slots(&self) -> u16 { self.len_stack_slots } - - /// Returns the function local constant values of the [`EngineFunc`]. - #[inline] - pub fn consts(&self) -> &'a [UntypedVal] { - self.consts.get_ref() - } } diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index afa7ae9493..5acdb79bb8 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -169,11 +169,7 @@ impl WasmTranslator<'_> for FuncTranslator { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; self.update_branch_offsets()?; - finalize(CompiledFuncEntity::new( - frame_size, - self.instrs.drain(), - self.layout.consts(), - )); + finalize(CompiledFuncEntity::new(frame_size, self.instrs.drain())); Ok(self.into_allocations()) } } @@ -272,11 +268,7 @@ impl FuncTranslator { /// /// Returns `None` if the frame size is out of bounds. fn frame_size(&self) -> Option { - let frame_size = self - .stack - .max_height() - .checked_add(self.locals.len())? - .checked_add(self.layout.consts().len())?; + let frame_size = self.stack.max_height().checked_add(self.locals.len())?; u16::try_from(frame_size).ok() } From f93ae765e7036fff65f2b9b3a134a102383255d2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 16 Sep 2025 12:40:07 +0200 Subject: [PATCH 082/423] update translation of remaining simd ops --- .../src/engine/translator/func/simd/visit.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 82e3ff954d..03cd55b021 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -544,7 +544,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_v128_bitselect(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::v128_bitselect, simd::v128_bitselect) + self.translate_simd_ternary(Op::v128_bitselect_ssss, simd::v128_bitselect) } fn visit_v128_any_true(&mut self) -> Self::Output { @@ -1157,14 +1157,14 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { self.translate_simd_unary( - Op::i32x4_trunc_sat_f64x2_zero_ss, + Op::i32x4_trunc_sat_zero_f64x2_ss, simd::i32x4_trunc_sat_f64x2_s_zero, ) } fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { self.translate_simd_unary( - Op::u32x4_trunc_sat_f64x2_zero_ss, + Op::u32x4_trunc_sat_zero_f64x2_ss, simd::i32x4_trunc_sat_f64x2_u_zero, ) } @@ -1185,7 +1185,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { self.translate_simd_unary( - Op::f32x4_demote_f64x2_zero_ss, + Op::f32x4_demote_zero_f64x2_ss, simd::f32x4_demote_f64x2_zero, ) } @@ -1218,19 +1218,19 @@ impl VisitSimdOperator<'_> for FuncTranslator { } fn visit_f32x4_relaxed_madd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f32x4_relaxed_madd_sss, simd::f32x4_relaxed_madd) + self.translate_simd_ternary(Op::f32x4_relaxed_madd_ssss, simd::f32x4_relaxed_madd) } fn visit_f32x4_relaxed_nmadd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f32x4_relaxed_nmadd_sss, simd::f32x4_relaxed_nmadd) + self.translate_simd_ternary(Op::f32x4_relaxed_nmadd_ssss, simd::f32x4_relaxed_nmadd) } fn visit_f64x2_relaxed_madd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f64x2_relaxed_madd_sss, simd::f64x2_relaxed_madd) + self.translate_simd_ternary(Op::f64x2_relaxed_madd_ssss, simd::f64x2_relaxed_madd) } fn visit_f64x2_relaxed_nmadd(&mut self) -> Self::Output { - self.translate_simd_ternary(Op::f64x2_relaxed_nmadd_sss, simd::f64x2_relaxed_nmadd) + self.translate_simd_ternary(Op::f64x2_relaxed_nmadd_ssss, simd::f64x2_relaxed_nmadd) } fn visit_i8x16_relaxed_laneselect(&mut self) -> Self::Output { @@ -1278,7 +1278,7 @@ impl VisitSimdOperator<'_> for FuncTranslator { fn visit_i32x4_relaxed_dot_i8x16_i7x16_add_s(&mut self) -> Self::Output { self.translate_simd_ternary( - Op::i32x4_relaxed_dot_i8x16_i7x16_add_sss, + Op::i32x4_relaxed_dot_i8x16_i7x16_add_ssss, simd::i32x4_relaxed_dot_i8x16_i7x16_add_s, ) } From 28f602b02f3ee1f6bce3d2c19d6190ea7cdbe723 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 16 Sep 2025 12:42:37 +0200 Subject: [PATCH 083/423] fix CachedInstance::get_memory --- crates/wasmi/src/engine/executor/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/executor/cache.rs b/crates/wasmi/src/engine/executor/cache.rs index 2476f5d3e4..ab91d1be8f 100644 --- a/crates/wasmi/src/engine/executor/cache.rs +++ b/crates/wasmi/src/engine/executor/cache.rs @@ -123,7 +123,7 @@ impl CachedInstance { #[inline] pub unsafe fn get_memory(&self, index: index::Memory) -> Option { let instance = unsafe { self.as_ref() }; - instance.get_memory(u32::from(index)) + instance.get_memory(u32::from(u16::from(index))) } /// Returns the [`Table`] at the `index` if any. From 201833e0438bff1404ae114471baa5e03f567483 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 16 Sep 2025 12:46:09 +0200 Subject: [PATCH 084/423] update ValueStack after function local constants removal --- crates/wasmi/src/engine/executor/stack/values.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/crates/wasmi/src/engine/executor/stack/values.rs b/crates/wasmi/src/engine/executor/stack/values.rs index 499218e7bf..c6693529d9 100644 --- a/crates/wasmi/src/engine/executor/stack/values.rs +++ b/crates/wasmi/src/engine/executor/stack/values.rs @@ -224,19 +224,11 @@ impl ValueStack { on_resize: impl FnMut(&mut Self), ) -> Result<(FrameParams, StackOffsets), TrapCode> { let len_stack_slots = func.len_stack_slots(); - let len_consts = func.consts().len(); let len = self.len(); - let mut spare = self - .extend_by(len_stack_slots as usize, on_resize)? - .iter_mut(); - (&mut spare) - .zip(func.consts()) - .for_each(|(uninit, const_value)| { - uninit.write(*const_value); - }); - let params = FrameParams::new(spare.into_slice()); + let spare = self.extend_by(len_stack_slots as usize, on_resize)?; + let params = FrameParams::new(spare); let frame = ValueStackOffset(len); - let base = ValueStackOffset(len + len_consts); + let base = ValueStackOffset(len); Ok(( params, StackOffsets { @@ -451,6 +443,6 @@ impl FrameSlots { /// Returns the underlying pointer offset by the [`Slot`] index. unsafe fn register_offset(&self, slot: Slot) -> *mut UntypedVal { - unsafe { self.ptr.offset(isize::from(i16::from(slot))) } + unsafe { self.ptr.add(usize::from(u16::from(slot))) } } } From 4c2fedc0802a3b20c4655d7f197caca006a177e2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Tue, 16 Sep 2025 12:56:15 +0200 Subject: [PATCH 085/423] add and use UpdateResultSlot extension trait This will be helpful once we add actual registers as results. --- .../wasmi/src/engine/translator/func/mod.rs | 14 +++++------ .../wasmi/src/engine/translator/func/utils.rs | 24 ++++++++++++++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 5acdb79bb8..4461721bfc 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -30,7 +30,7 @@ use self::{ StackAllocations, TempOperand, }, - utils::{Input, Reset, ReusableAllocations}, + utils::{Input, Reset, ReusableAllocations, UpdateResultSlot}, }; use crate::{ core::{FuelCostsProvider, IndexType, Typed, TypedVal, UntypedVal}, @@ -904,11 +904,11 @@ impl FuncTranslator { panic!("out of bounds `br_table` target does not fit `usize`: {target:?}"); }; let mut frame = self.stack.peek_control_mut(depth).control_frame(); - let Some(results) = Self::frame_results_impl(&frame, &self.engine, &self.layout)? + let Some(_results) = Self::frame_results_impl(&frame, &self.engine, &self.layout)? else { panic!("must have frame results since `br_table` requires to copy values"); }; - let offset = self + let _offset = self .labels .try_resolve_label(frame.label(), self.instrs.next_instr())?; // self.instrs @@ -2090,7 +2090,7 @@ impl FuncTranslator { // Need to replace `cmp` instruction result register since it might // have been misaligned if `lhs` originally referred to the zero operand. let new_result = self.layout.temp_to_reg(result_idx)?; - let Some(negated) = negated.replace_cmp_result(new_result) else { + let Some(negated) = negated.update_result_slot(new_result) else { unreachable!("`negated` has been asserted as `cmp` instruction"); }; if !self.instrs.try_replace_instr(last_instr, negated)? { @@ -2183,14 +2183,14 @@ impl FuncTranslator { fn encode_store( &mut self, memarg: MemArg, - ptr: Operand, - value: Operand, + _ptr: Operand, + _value: Operand, ) -> Result<(), Error> where T::Value: Copy + From, T::Immediate: Copy, { - let (memory, offset) = Self::decode_memarg(memarg)?; + let (_memory, _offset) = Self::decode_memarg(memarg)?; todo!() } diff --git a/crates/wasmi/src/engine/translator/func/utils.rs b/crates/wasmi/src/engine/translator/func/utils.rs index 8b9e9a13d2..a596bf0c68 100644 --- a/crates/wasmi/src/engine/translator/func/utils.rs +++ b/crates/wasmi/src/engine/translator/func/utils.rs @@ -1,4 +1,4 @@ -use crate::ir::Slot; +use crate::ir::{Op, Slot}; /// Bail out early in case the current code is unreachable. /// @@ -56,3 +56,25 @@ pub enum Input { /// A 16-bit encoded immediate value operand. Immediate(T), } + +/// Extension trait to update the result [`Slot`] of an [`Op`]. +pub trait UpdateResultSlot: Sized { + /// Updates the result [`Slot`] of `self` if possible. + /// + /// # Note + /// + /// - Returns `Some` resulting `Self` if the update was successful. + /// - Returns `None` if the result update could not be applied. + fn update_result_slot(&self, new_result: Slot) -> Option; +} + +impl UpdateResultSlot for Op { + fn update_result_slot(&self, new_result: Slot) -> Option { + let mut op = *self; + let Some(result_mut) = op.result_mut() else { + return None; + }; + *result_mut = new_result; + Some(op) + } +} From 02da0349831756eea442d98882df76eded5e315e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 11:47:09 +0200 Subject: [PATCH 086/423] remove most of wasmi's executor since it will be rewritten --- crates/wasmi/src/engine/code_map.rs | 1 + crates/wasmi/src/engine/executor/instrs.rs | 2472 +---------------- .../src/engine/executor/instrs/binary.rs | 323 --- .../src/engine/executor/instrs/branch.rs | 328 --- .../wasmi/src/engine/executor/instrs/call.rs | 713 ----- .../src/engine/executor/instrs/comparison.rs | 91 - .../src/engine/executor/instrs/conversion.rs | 60 - .../wasmi/src/engine/executor/instrs/copy.rs | 117 - .../src/engine/executor/instrs/global.rs | 82 - .../wasmi/src/engine/executor/instrs/load.rs | 411 --- .../src/engine/executor/instrs/memory.rs | 281 -- .../src/engine/executor/instrs/return_.rs | 237 -- .../src/engine/executor/instrs/select.rs | 140 - .../wasmi/src/engine/executor/instrs/simd.rs | 814 ------ .../wasmi/src/engine/executor/instrs/store.rs | 470 ---- .../wasmi/src/engine/executor/instrs/table.rs | 255 -- .../wasmi/src/engine/executor/instrs/unary.rs | 33 - .../wasmi/src/engine/executor/instrs/utils.rs | 117 - .../engine/executor/instrs/wide_arithmetic.rs | 95 - crates/wasmi/src/engine/executor/mod.rs | 30 +- crates/wasmi/src/engine/executor/stack/mod.rs | 2 +- 21 files changed, 22 insertions(+), 7050 deletions(-) delete mode 100644 crates/wasmi/src/engine/executor/instrs/binary.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/branch.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/call.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/comparison.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/conversion.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/copy.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/global.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/load.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/memory.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/return_.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/select.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/simd.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/store.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/table.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/unary.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/utils.rs delete mode 100644 crates/wasmi/src/engine/executor/instrs/wide_arithmetic.rs diff --git a/crates/wasmi/src/engine/code_map.rs b/crates/wasmi/src/engine/code_map.rs index 0ffe559e0e..c970eecfcd 100644 --- a/crates/wasmi/src/engine/code_map.rs +++ b/crates/wasmi/src/engine/code_map.rs @@ -139,6 +139,7 @@ impl EngineFuncSpan { /// Returns the `u32` index of the [`EngineFunc`] in `self` if any. /// /// Returns `None` if `func` is not contained in `self`. + #[allow(unused)] pub fn position(&self, func: EngineFunc) -> Option { debug_assert!(self.start <= self.end); if func < self.start || func >= self.end { diff --git a/crates/wasmi/src/engine/executor/instrs.rs b/crates/wasmi/src/engine/executor/instrs.rs index b588b51e54..9d312d6dfb 100644 --- a/crates/wasmi/src/engine/executor/instrs.rs +++ b/crates/wasmi/src/engine/executor/instrs.rs @@ -1,63 +1,26 @@ -pub use self::call::dispatch_host_func; use super::{cache::CachedInstance, InstructionPtr, Stack}; use crate::{ - core::{hint, wasm, ReadAs, UntypedVal, WriteAs}, + core::{wasm, ReadAs, UntypedVal, WriteAs}, engine::{ code_map::CodeMap, - executor::stack::{CallFrame, FrameSlots, ValueStack}, + executor::stack::FrameSlots, utils::unreachable_unchecked, DedupFuncType, - EngineFunc, }, - ir::{index, BlockFuel, Op, Slot}, + ir::{index, Slot}, memory::DataSegment, - store::{PrunedStore, StoreInner}, + store::PrunedStore, table::ElementSegment, Error, Func, Global, Memory, - Ref, Table, - TrapCode, }; #[cfg(doc)] use crate::Instance; -#[macro_use] -mod utils; - -#[cfg(feature = "simd")] -mod simd; - -mod binary; -mod branch; -mod call; -mod comparison; -mod conversion; -mod copy; -mod global; -mod load; -mod memory; -mod return_; -mod select; -mod store; -mod table; -mod unary; -mod wide_arithmetic; - -macro_rules! forward_return { - ($expr:expr) => {{ - if hint::unlikely($expr.is_break()) { - return Ok(()); - } - }}; -} - -/// Tells if execution loop shall continue or break (return) to the execution's caller. -type ControlFlow = ::core::ops::ControlFlow<(), ()>; - /// Executes compiled function instructions until execution returns from the root function. /// /// # Errors @@ -130,2105 +93,8 @@ impl<'engine> Executor<'engine> { /// Executes the function frame until it returns or traps. #[inline(always)] - fn execute(&mut self, store: &mut PrunedStore) -> Result<(), Error> { - use Op as Instr; - loop { - match *self.ip.get() { - Instr::Trap { trap_code } => self.execute_trap(trap_code)?, - Instr::ConsumeFuel { fuel } => { - self.execute_consume_fuel(store.inner_mut(), fuel)? - } - Instr::Return => { - forward_return!(self.execute_return(store.inner_mut())) - } - Instr::ReturnSlot { value } => { - forward_return!(self.execute_return_reg(store.inner_mut(), value)) - } - Instr::ReturnSlot2 { values } => { - forward_return!(self.execute_return_reg2(store.inner_mut(), values)) - } - Instr::ReturnSlot3 { values } => { - forward_return!(self.execute_return_reg3(store.inner_mut(), values)) - } - Instr::ReturnImm32 { value } => { - forward_return!(self.execute_return_imm32(store.inner_mut(), value)) - } - Instr::ReturnI64Imm32 { value } => { - forward_return!(self.execute_return_i64imm32(store.inner_mut(), value)) - } - Instr::ReturnF64Imm32 { value } => { - forward_return!(self.execute_return_f64imm32(store.inner_mut(), value)) - } - Instr::ReturnSpan { values } => { - forward_return!(self.execute_return_span(store.inner_mut(), values)) - } - Instr::ReturnMany { values } => { - forward_return!(self.execute_return_many(store.inner_mut(), values)) - } - Instr::Branch { offset } => self.execute_branch(offset), - Instr::BranchTable0 { index, len_targets } => { - self.execute_branch_table_0(index, len_targets) - } - Instr::BranchTableSpan { index, len_targets } => { - self.execute_branch_table_span(index, len_targets) - } - Instr::BranchCmpFallback { lhs, rhs, params } => { - self.execute_branch_cmp_fallback(lhs, rhs, params) - } - Instr::BranchI32And { lhs, rhs, offset } => { - self.execute_branch_i32_and(lhs, rhs, offset) - } - Instr::BranchI32AndImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_and_imm16(lhs, rhs, offset) - } - Instr::BranchI32Or { lhs, rhs, offset } => { - self.execute_branch_i32_or(lhs, rhs, offset) - } - Instr::BranchI32OrImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_or_imm16(lhs, rhs, offset) - } - Instr::BranchI32Nand { lhs, rhs, offset } => { - self.execute_branch_i32_nand(lhs, rhs, offset) - } - Instr::BranchI32NandImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_nand_imm16(lhs, rhs, offset) - } - Instr::BranchI32Nor { lhs, rhs, offset } => { - self.execute_branch_i32_nor(lhs, rhs, offset) - } - Instr::BranchI32NorImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_nor_imm16(lhs, rhs, offset) - } - Instr::BranchI32Eq { lhs, rhs, offset } => { - self.execute_branch_i32_eq(lhs, rhs, offset) - } - Instr::BranchI32EqImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_eq_imm16(lhs, rhs, offset) - } - Instr::BranchI32Ne { lhs, rhs, offset } => { - self.execute_branch_i32_ne(lhs, rhs, offset) - } - Instr::BranchI32NeImm16 { lhs, rhs, offset } => { - self.execute_branch_i32_ne_imm16(lhs, rhs, offset) - } - Instr::BranchI32LtS { lhs, rhs, offset } => { - self.execute_branch_i32_lt_s(lhs, rhs, offset) - } - Instr::BranchI32LtSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LtSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI32LtU { lhs, rhs, offset } => { - self.execute_branch_i32_lt_u(lhs, rhs, offset) - } - Instr::BranchI32LtUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LtUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_lt_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI32LeS { lhs, rhs, offset } => { - self.execute_branch_i32_le_s(lhs, rhs, offset) - } - Instr::BranchI32LeSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LeSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI32LeU { lhs, rhs, offset } => { - self.execute_branch_i32_le_u(lhs, rhs, offset) - } - Instr::BranchI32LeUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI32LeUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i32_le_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64And { lhs, rhs, offset } => { - self.execute_branch_i64_and(lhs, rhs, offset) - } - Instr::BranchI64AndImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_and_imm16(lhs, rhs, offset) - } - Instr::BranchI64Or { lhs, rhs, offset } => { - self.execute_branch_i64_or(lhs, rhs, offset) - } - Instr::BranchI64OrImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_or_imm16(lhs, rhs, offset) - } - Instr::BranchI64Nand { lhs, rhs, offset } => { - self.execute_branch_i64_nand(lhs, rhs, offset) - } - Instr::BranchI64NandImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_nand_imm16(lhs, rhs, offset) - } - Instr::BranchI64Nor { lhs, rhs, offset } => { - self.execute_branch_i64_nor(lhs, rhs, offset) - } - Instr::BranchI64NorImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_nor_imm16(lhs, rhs, offset) - } - Instr::BranchI64Eq { lhs, rhs, offset } => { - self.execute_branch_i64_eq(lhs, rhs, offset) - } - Instr::BranchI64EqImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_eq_imm16(lhs, rhs, offset) - } - Instr::BranchI64Ne { lhs, rhs, offset } => { - self.execute_branch_i64_ne(lhs, rhs, offset) - } - Instr::BranchI64NeImm16 { lhs, rhs, offset } => { - self.execute_branch_i64_ne_imm16(lhs, rhs, offset) - } - Instr::BranchI64LtS { lhs, rhs, offset } => { - self.execute_branch_i64_lt_s(lhs, rhs, offset) - } - Instr::BranchI64LtSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LtSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64LtU { lhs, rhs, offset } => { - self.execute_branch_i64_lt_u(lhs, rhs, offset) - } - Instr::BranchI64LtUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LtUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_lt_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64LeS { lhs, rhs, offset } => { - self.execute_branch_i64_le_s(lhs, rhs, offset) - } - Instr::BranchI64LeSImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_s_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LeSImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_s_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchI64LeU { lhs, rhs, offset } => { - self.execute_branch_i64_le_u(lhs, rhs, offset) - } - Instr::BranchI64LeUImm16Lhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_u_imm16_lhs(lhs, rhs, offset) - } - Instr::BranchI64LeUImm16Rhs { lhs, rhs, offset } => { - self.execute_branch_i64_le_u_imm16_rhs(lhs, rhs, offset) - } - Instr::BranchF32Eq { lhs, rhs, offset } => { - self.execute_branch_f32_eq(lhs, rhs, offset) - } - Instr::BranchF32Ne { lhs, rhs, offset } => { - self.execute_branch_f32_ne(lhs, rhs, offset) - } - Instr::BranchF32Lt { lhs, rhs, offset } => { - self.execute_branch_f32_lt(lhs, rhs, offset) - } - Instr::BranchF32Le { lhs, rhs, offset } => { - self.execute_branch_f32_le(lhs, rhs, offset) - } - Instr::BranchF32NotLt { lhs, rhs, offset } => { - self.execute_branch_f32_not_lt(lhs, rhs, offset) - } - Instr::BranchF32NotLe { lhs, rhs, offset } => { - self.execute_branch_f32_not_le(lhs, rhs, offset) - } - Instr::BranchF64Eq { lhs, rhs, offset } => { - self.execute_branch_f64_eq(lhs, rhs, offset) - } - Instr::BranchF64Ne { lhs, rhs, offset } => { - self.execute_branch_f64_ne(lhs, rhs, offset) - } - Instr::BranchF64Lt { lhs, rhs, offset } => { - self.execute_branch_f64_lt(lhs, rhs, offset) - } - Instr::BranchF64Le { lhs, rhs, offset } => { - self.execute_branch_f64_le(lhs, rhs, offset) - } - Instr::BranchF64NotLt { lhs, rhs, offset } => { - self.execute_branch_f64_not_lt(lhs, rhs, offset) - } - Instr::BranchF64NotLe { lhs, rhs, offset } => { - self.execute_branch_f64_not_le(lhs, rhs, offset) - } - Instr::Copy { result, value } => self.execute_copy(result, value), - Instr::Copy2 { results, values } => self.execute_copy_2(results, values), - Instr::CopyImm32 { result, value } => self.execute_copy_imm32(result, value), - Instr::CopyI64Imm32 { result, value } => self.execute_copy_i64imm32(result, value), - Instr::CopyF64Imm32 { result, value } => self.execute_copy_f64imm32(result, value), - Instr::CopySpan { - results, - values, - len, - } => self.execute_copy_span(results, values, len), - Instr::CopyMany { results, values } => self.execute_copy_many(results, values), - Instr::ReturnCallInternal0 { func } => { - self.execute_return_call_internal_0(store.inner_mut(), EngineFunc::from(func))? - } - Instr::ReturnCallInternal { func } => { - self.execute_return_call_internal(store.inner_mut(), EngineFunc::from(func))? - } - Instr::ReturnCallImported0 { func } => { - forward_return!(self.execute_return_call_imported_0(store, func)?) - } - Instr::ReturnCallImported { func } => { - forward_return!(self.execute_return_call_imported(store, func)?) - } - Instr::ReturnCallIndirect0 { func_type } => { - forward_return!(self.execute_return_call_indirect_0(store, func_type)?) - } - Instr::ReturnCallIndirect0Imm16 { func_type } => { - forward_return!(self.execute_return_call_indirect_0_imm16(store, func_type)?) - } - Instr::ReturnCallIndirect { func_type } => { - forward_return!(self.execute_return_call_indirect(store, func_type)?) - } - Instr::ReturnCallIndirectImm16 { func_type } => { - forward_return!(self.execute_return_call_indirect_imm16(store, func_type)?) - } - Instr::CallInternal0 { results, func } => self.execute_call_internal_0( - store.inner_mut(), - results, - EngineFunc::from(func), - )?, - Instr::CallInternal { results, func } => { - self.execute_call_internal(store.inner_mut(), results, EngineFunc::from(func))? - } - Instr::CallImported0 { results, func } => { - self.execute_call_imported_0(store, results, func)? - } - Instr::CallImported { results, func } => { - self.execute_call_imported(store, results, func)? - } - Instr::CallIndirect0 { results, func_type } => { - self.execute_call_indirect_0(store, results, func_type)? - } - Instr::CallIndirect0Imm16 { results, func_type } => { - self.execute_call_indirect_0_imm16(store, results, func_type)? - } - Instr::CallIndirect { results, func_type } => { - self.execute_call_indirect(store, results, func_type)? - } - Instr::CallIndirectImm16 { results, func_type } => { - self.execute_call_indirect_imm16(store, results, func_type)? - } - Instr::SelectI32Eq { result, lhs, rhs } => { - self.execute_select_i32_eq(result, lhs, rhs) - } - Instr::SelectI32EqImm16 { result, lhs, rhs } => { - self.execute_select_i32_eq_imm16(result, lhs, rhs) - } - Instr::SelectI32LtS { result, lhs, rhs } => { - self.execute_select_i32_lt_s(result, lhs, rhs) - } - Instr::SelectI32LtSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32LtU { result, lhs, rhs } => { - self.execute_select_i32_lt_u(result, lhs, rhs) - } - Instr::SelectI32LtUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32LeS { result, lhs, rhs } => { - self.execute_select_i32_le_s(result, lhs, rhs) - } - Instr::SelectI32LeSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32LeU { result, lhs, rhs } => { - self.execute_select_i32_le_u(result, lhs, rhs) - } - Instr::SelectI32LeUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i32_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI32And { result, lhs, rhs } => { - self.execute_select_i32_and(result, lhs, rhs) - } - Instr::SelectI32AndImm16 { result, lhs, rhs } => { - self.execute_select_i32_and_imm16(result, lhs, rhs) - } - Instr::SelectI32Or { result, lhs, rhs } => { - self.execute_select_i32_or(result, lhs, rhs) - } - Instr::SelectI32OrImm16 { result, lhs, rhs } => { - self.execute_select_i32_or_imm16(result, lhs, rhs) - } - Instr::SelectI64Eq { result, lhs, rhs } => { - self.execute_select_i64_eq(result, lhs, rhs) - } - Instr::SelectI64EqImm16 { result, lhs, rhs } => { - self.execute_select_i64_eq_imm16(result, lhs, rhs) - } - Instr::SelectI64LtS { result, lhs, rhs } => { - self.execute_select_i64_lt_s(result, lhs, rhs) - } - Instr::SelectI64LtSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64LtU { result, lhs, rhs } => { - self.execute_select_i64_lt_u(result, lhs, rhs) - } - Instr::SelectI64LtUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64LeS { result, lhs, rhs } => { - self.execute_select_i64_le_s(result, lhs, rhs) - } - Instr::SelectI64LeSImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64LeU { result, lhs, rhs } => { - self.execute_select_i64_le_u(result, lhs, rhs) - } - Instr::SelectI64LeUImm16Rhs { result, lhs, rhs } => { - self.execute_select_i64_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::SelectI64And { result, lhs, rhs } => { - self.execute_select_i64_and(result, lhs, rhs) - } - Instr::SelectI64AndImm16 { result, lhs, rhs } => { - self.execute_select_i64_and_imm16(result, lhs, rhs) - } - Instr::SelectI64Or { result, lhs, rhs } => { - self.execute_select_i64_or(result, lhs, rhs) - } - Instr::SelectI64OrImm16 { result, lhs, rhs } => { - self.execute_select_i64_or_imm16(result, lhs, rhs) - } - Instr::SelectF32Eq { result, lhs, rhs } => { - self.execute_select_f32_eq(result, lhs, rhs) - } - Instr::SelectF32Lt { result, lhs, rhs } => { - self.execute_select_f32_lt(result, lhs, rhs) - } - Instr::SelectF32Le { result, lhs, rhs } => { - self.execute_select_f32_le(result, lhs, rhs) - } - Instr::SelectF64Eq { result, lhs, rhs } => { - self.execute_select_f64_eq(result, lhs, rhs) - } - Instr::SelectF64Lt { result, lhs, rhs } => { - self.execute_select_f64_lt(result, lhs, rhs) - } - Instr::SelectF64Le { result, lhs, rhs } => { - self.execute_select_f64_le(result, lhs, rhs) - } - Instr::RefFunc { result, func } => self.execute_ref_func(result, func), - Instr::GlobalGet { result, global } => { - self.execute_global_get(store.inner(), result, global) - } - Instr::GlobalSet { global, input } => { - self.execute_global_set(store.inner_mut(), global, input) - } - Instr::GlobalSetI32Imm16 { global, input } => { - self.execute_global_set_i32imm16(store.inner_mut(), global, input) - } - Instr::GlobalSetI64Imm16 { global, input } => { - self.execute_global_set_i64imm16(store.inner_mut(), global, input) - } - Instr::Load32 { result, offset_lo } => { - self.execute_load32(store.inner(), result, offset_lo)? - } - Instr::Load32At { result, address } => { - self.execute_load32_at(store.inner(), result, address)? - } - Instr::Load32Offset16 { - result, - ptr, - offset, - } => self.execute_load32_offset16(result, ptr, offset)?, - Instr::Load64 { result, offset_lo } => { - self.execute_load64(store.inner(), result, offset_lo)? - } - Instr::Load64At { result, address } => { - self.execute_load64_at(store.inner(), result, address)? - } - Instr::Load64Offset16 { - result, - ptr, - offset, - } => self.execute_load64_offset16(result, ptr, offset)?, - Instr::I32Load8s { result, offset_lo } => { - self.execute_i32_load8_s(store.inner(), result, offset_lo)? - } - Instr::I32Load8sAt { result, address } => { - self.execute_i32_load8_s_at(store.inner(), result, address)? - } - Instr::I32Load8sOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load8_s_offset16(result, ptr, offset)?, - Instr::I32Load8u { result, offset_lo } => { - self.execute_i32_load8_u(store.inner(), result, offset_lo)? - } - Instr::I32Load8uAt { result, address } => { - self.execute_i32_load8_u_at(store.inner(), result, address)? - } - Instr::I32Load8uOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load8_u_offset16(result, ptr, offset)?, - Instr::I32Load16s { result, offset_lo } => { - self.execute_i32_load16_s(store.inner(), result, offset_lo)? - } - Instr::I32Load16sAt { result, address } => { - self.execute_i32_load16_s_at(store.inner(), result, address)? - } - Instr::I32Load16sOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load16_s_offset16(result, ptr, offset)?, - Instr::I32Load16u { result, offset_lo } => { - self.execute_i32_load16_u(store.inner(), result, offset_lo)? - } - Instr::I32Load16uAt { result, address } => { - self.execute_i32_load16_u_at(store.inner(), result, address)? - } - Instr::I32Load16uOffset16 { - result, - ptr, - offset, - } => self.execute_i32_load16_u_offset16(result, ptr, offset)?, - Instr::I64Load8s { result, offset_lo } => { - self.execute_i64_load8_s(store.inner(), result, offset_lo)? - } - Instr::I64Load8sAt { result, address } => { - self.execute_i64_load8_s_at(store.inner(), result, address)? - } - Instr::I64Load8sOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load8_s_offset16(result, ptr, offset)?, - Instr::I64Load8u { result, offset_lo } => { - self.execute_i64_load8_u(store.inner(), result, offset_lo)? - } - Instr::I64Load8uAt { result, address } => { - self.execute_i64_load8_u_at(store.inner(), result, address)? - } - Instr::I64Load8uOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load8_u_offset16(result, ptr, offset)?, - Instr::I64Load16s { result, offset_lo } => { - self.execute_i64_load16_s(store.inner(), result, offset_lo)? - } - Instr::I64Load16sAt { result, address } => { - self.execute_i64_load16_s_at(store.inner(), result, address)? - } - Instr::I64Load16sOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load16_s_offset16(result, ptr, offset)?, - Instr::I64Load16u { result, offset_lo } => { - self.execute_i64_load16_u(store.inner(), result, offset_lo)? - } - Instr::I64Load16uAt { result, address } => { - self.execute_i64_load16_u_at(store.inner(), result, address)? - } - Instr::I64Load16uOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load16_u_offset16(result, ptr, offset)?, - Instr::I64Load32s { result, offset_lo } => { - self.execute_i64_load32_s(store.inner(), result, offset_lo)? - } - Instr::I64Load32sAt { result, address } => { - self.execute_i64_load32_s_at(store.inner(), result, address)? - } - Instr::I64Load32sOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load32_s_offset16(result, ptr, offset)?, - Instr::I64Load32u { result, offset_lo } => { - self.execute_i64_load32_u(store.inner(), result, offset_lo)? - } - Instr::I64Load32uAt { result, address } => { - self.execute_i64_load32_u_at(store.inner(), result, address)? - } - Instr::I64Load32uOffset16 { - result, - ptr, - offset, - } => self.execute_i64_load32_u_offset16(result, ptr, offset)?, - Instr::Store32 { ptr, offset_lo } => { - self.execute_store32(store.inner_mut(), ptr, offset_lo)? - } - Instr::Store32Offset16 { ptr, offset, value } => { - self.execute_store32_offset16(ptr, offset, value)? - } - Instr::Store32At { address, value } => { - self.execute_store32_at(store.inner_mut(), address, value)? - } - Instr::Store64 { ptr, offset_lo } => { - self.execute_store64(store.inner_mut(), ptr, offset_lo)? - } - Instr::Store64Offset16 { ptr, offset, value } => { - self.execute_store64_offset16(ptr, offset, value)? - } - Instr::Store64At { address, value } => { - self.execute_store64_at(store.inner_mut(), address, value)? - } - Instr::I32StoreImm16 { ptr, offset_lo } => { - self.execute_i32_store_imm16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32StoreOffset16Imm16 { ptr, offset, value } => { - self.execute_i32_store_offset16_imm16(ptr, offset, value)? - } - Instr::I32StoreAtImm16 { address, value } => { - self.execute_i32_store_at_imm16(store.inner_mut(), address, value)? - } - Instr::I32Store8 { ptr, offset_lo } => { - self.execute_i32_store8(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store8Imm { ptr, offset_lo } => { - self.execute_i32_store8_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store8Offset16 { ptr, offset, value } => { - self.execute_i32_store8_offset16(ptr, offset, value)? - } - Instr::I32Store8Offset16Imm { ptr, offset, value } => { - self.execute_i32_store8_offset16_imm(ptr, offset, value)? - } - Instr::I32Store8At { address, value } => { - self.execute_i32_store8_at(store.inner_mut(), address, value)? - } - Instr::I32Store8AtImm { address, value } => { - self.execute_i32_store8_at_imm(store.inner_mut(), address, value)? - } - Instr::I32Store16 { ptr, offset_lo } => { - self.execute_i32_store16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store16Imm { ptr, offset_lo } => { - self.execute_i32_store16_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I32Store16Offset16 { ptr, offset, value } => { - self.execute_i32_store16_offset16(ptr, offset, value)? - } - Instr::I32Store16Offset16Imm { ptr, offset, value } => { - self.execute_i32_store16_offset16_imm(ptr, offset, value)? - } - Instr::I32Store16At { address, value } => { - self.execute_i32_store16_at(store.inner_mut(), address, value)? - } - Instr::I32Store16AtImm { address, value } => { - self.execute_i32_store16_at_imm(store.inner_mut(), address, value)? - } - Instr::I64StoreImm16 { ptr, offset_lo } => { - self.execute_i64_store_imm16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64StoreOffset16Imm16 { ptr, offset, value } => { - self.execute_i64_store_offset16_imm16(ptr, offset, value)? - } - Instr::I64StoreAtImm16 { address, value } => { - self.execute_i64_store_at_imm16(store.inner_mut(), address, value)? - } - Instr::I64Store8 { ptr, offset_lo } => { - self.execute_i64_store8(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store8Imm { ptr, offset_lo } => { - self.execute_i64_store8_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store8Offset16 { ptr, offset, value } => { - self.execute_i64_store8_offset16(ptr, offset, value)? - } - Instr::I64Store8Offset16Imm { ptr, offset, value } => { - self.execute_i64_store8_offset16_imm(ptr, offset, value)? - } - Instr::I64Store8At { address, value } => { - self.execute_i64_store8_at(store.inner_mut(), address, value)? - } - Instr::I64Store8AtImm { address, value } => { - self.execute_i64_store8_at_imm(store.inner_mut(), address, value)? - } - Instr::I64Store16 { ptr, offset_lo } => { - self.execute_i64_store16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store16Imm { ptr, offset_lo } => { - self.execute_i64_store16_imm(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store16Offset16 { ptr, offset, value } => { - self.execute_i64_store16_offset16(ptr, offset, value)? - } - Instr::I64Store16Offset16Imm { ptr, offset, value } => { - self.execute_i64_store16_offset16_imm(ptr, offset, value)? - } - Instr::I64Store16At { address, value } => { - self.execute_i64_store16_at(store.inner_mut(), address, value)? - } - Instr::I64Store16AtImm { address, value } => { - self.execute_i64_store16_at_imm(store.inner_mut(), address, value)? - } - Instr::I64Store32 { ptr, offset_lo } => { - self.execute_i64_store32(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store32Imm16 { ptr, offset_lo } => { - self.execute_i64_store32_imm16(store.inner_mut(), ptr, offset_lo)? - } - Instr::I64Store32Offset16 { ptr, offset, value } => { - self.execute_i64_store32_offset16(ptr, offset, value)? - } - Instr::I64Store32Offset16Imm16 { ptr, offset, value } => { - self.execute_i64_store32_offset16_imm16(ptr, offset, value)? - } - Instr::I64Store32At { address, value } => { - self.execute_i64_store32_at(store.inner_mut(), address, value)? - } - Instr::I64Store32AtImm16 { address, value } => { - self.execute_i64_store32_at_imm16(store.inner_mut(), address, value)? - } - Instr::I32Eq { result, lhs, rhs } => self.execute_i32_eq(result, lhs, rhs), - Instr::I32EqImm16 { result, lhs, rhs } => { - self.execute_i32_eq_imm16(result, lhs, rhs) - } - Instr::I32Ne { result, lhs, rhs } => self.execute_i32_ne(result, lhs, rhs), - Instr::I32NeImm16 { result, lhs, rhs } => { - self.execute_i32_ne_imm16(result, lhs, rhs) - } - Instr::I32LtS { result, lhs, rhs } => self.execute_i32_lt_s(result, lhs, rhs), - Instr::I32LtSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_lt_s_imm16_lhs(result, lhs, rhs) - } - Instr::I32LtSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::I32LtU { result, lhs, rhs } => self.execute_i32_lt_u(result, lhs, rhs), - Instr::I32LtUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_lt_u_imm16_lhs(result, lhs, rhs) - } - Instr::I32LtUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::I32LeS { result, lhs, rhs } => self.execute_i32_le_s(result, lhs, rhs), - Instr::I32LeSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_le_s_imm16_lhs(result, lhs, rhs) - } - Instr::I32LeSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::I32LeU { result, lhs, rhs } => self.execute_i32_le_u(result, lhs, rhs), - Instr::I32LeUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_le_u_imm16_lhs(result, lhs, rhs) - } - Instr::I32LeUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64Eq { result, lhs, rhs } => self.execute_i64_eq(result, lhs, rhs), - Instr::I64EqImm16 { result, lhs, rhs } => { - self.execute_i64_eq_imm16(result, lhs, rhs) - } - Instr::I64Ne { result, lhs, rhs } => self.execute_i64_ne(result, lhs, rhs), - Instr::I64NeImm16 { result, lhs, rhs } => { - self.execute_i64_ne_imm16(result, lhs, rhs) - } - Instr::I64LtS { result, lhs, rhs } => self.execute_i64_lt_s(result, lhs, rhs), - Instr::I64LtSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_lt_s_imm16_lhs(result, lhs, rhs) - } - Instr::I64LtSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_lt_s_imm16_rhs(result, lhs, rhs) - } - Instr::I64LtU { result, lhs, rhs } => self.execute_i64_lt_u(result, lhs, rhs), - Instr::I64LtUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_lt_u_imm16_lhs(result, lhs, rhs) - } - Instr::I64LtUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_lt_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64LeS { result, lhs, rhs } => self.execute_i64_le_s(result, lhs, rhs), - Instr::I64LeSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_le_s_imm16_lhs(result, lhs, rhs) - } - Instr::I64LeSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_le_s_imm16_rhs(result, lhs, rhs) - } - Instr::I64LeU { result, lhs, rhs } => self.execute_i64_le_u(result, lhs, rhs), - Instr::I64LeUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_le_u_imm16_lhs(result, lhs, rhs) - } - Instr::I64LeUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_le_u_imm16_rhs(result, lhs, rhs) - } - Instr::F32Eq { result, lhs, rhs } => self.execute_f32_eq(result, lhs, rhs), - Instr::F32Ne { result, lhs, rhs } => self.execute_f32_ne(result, lhs, rhs), - Instr::F32Lt { result, lhs, rhs } => self.execute_f32_lt(result, lhs, rhs), - Instr::F32Le { result, lhs, rhs } => self.execute_f32_le(result, lhs, rhs), - Instr::F32NotLt { result, lhs, rhs } => self.execute_f32_not_lt(result, lhs, rhs), - Instr::F32NotLe { result, lhs, rhs } => self.execute_f32_not_le(result, lhs, rhs), - Instr::F64Eq { result, lhs, rhs } => self.execute_f64_eq(result, lhs, rhs), - Instr::F64Ne { result, lhs, rhs } => self.execute_f64_ne(result, lhs, rhs), - Instr::F64Lt { result, lhs, rhs } => self.execute_f64_lt(result, lhs, rhs), - Instr::F64Le { result, lhs, rhs } => self.execute_f64_le(result, lhs, rhs), - Instr::F64NotLt { result, lhs, rhs } => self.execute_f64_not_lt(result, lhs, rhs), - Instr::F64NotLe { result, lhs, rhs } => self.execute_f64_not_le(result, lhs, rhs), - Instr::I32Clz { result, input } => self.execute_i32_clz(result, input), - Instr::I32Ctz { result, input } => self.execute_i32_ctz(result, input), - Instr::I32Popcnt { result, input } => self.execute_i32_popcnt(result, input), - Instr::I32Add { result, lhs, rhs } => self.execute_i32_add(result, lhs, rhs), - Instr::I32AddImm16 { result, lhs, rhs } => { - self.execute_i32_add_imm16(result, lhs, rhs) - } - Instr::I32Sub { result, lhs, rhs } => self.execute_i32_sub(result, lhs, rhs), - Instr::I32SubImm16Lhs { result, lhs, rhs } => { - self.execute_i32_sub_imm16_lhs(result, lhs, rhs) - } - Instr::I32Mul { result, lhs, rhs } => self.execute_i32_mul(result, lhs, rhs), - Instr::I32MulImm16 { result, lhs, rhs } => { - self.execute_i32_mul_imm16(result, lhs, rhs) - } - Instr::I32DivS { result, lhs, rhs } => self.execute_i32_div_s(result, lhs, rhs)?, - Instr::I32DivSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_div_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I32DivSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_div_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I32DivU { result, lhs, rhs } => self.execute_i32_div_u(result, lhs, rhs)?, - Instr::I32DivUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_div_u_imm16_rhs(result, lhs, rhs) - } - Instr::I32DivUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_div_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I32RemS { result, lhs, rhs } => self.execute_i32_rem_s(result, lhs, rhs)?, - Instr::I32RemSImm16Rhs { result, lhs, rhs } => { - self.execute_i32_rem_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I32RemSImm16Lhs { result, lhs, rhs } => { - self.execute_i32_rem_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I32RemU { result, lhs, rhs } => self.execute_i32_rem_u(result, lhs, rhs)?, - Instr::I32RemUImm16Rhs { result, lhs, rhs } => { - self.execute_i32_rem_u_imm16_rhs(result, lhs, rhs) - } - Instr::I32RemUImm16Lhs { result, lhs, rhs } => { - self.execute_i32_rem_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I32BitAnd { result, lhs, rhs } => self.execute_i32_bitand(result, lhs, rhs), - Instr::I32BitAndImm16 { result, lhs, rhs } => { - self.execute_i32_bitand_imm16(result, lhs, rhs) - } - Instr::I32BitOr { result, lhs, rhs } => self.execute_i32_bitor(result, lhs, rhs), - Instr::I32BitOrImm16 { result, lhs, rhs } => { - self.execute_i32_bitor_imm16(result, lhs, rhs) - } - Instr::I32BitXor { result, lhs, rhs } => self.execute_i32_bitxor(result, lhs, rhs), - Instr::I32BitXorImm16 { result, lhs, rhs } => { - self.execute_i32_bitxor_imm16(result, lhs, rhs) - } - Instr::I32And { result, lhs, rhs } => self.execute_i32_and(result, lhs, rhs), - Instr::I32AndImm16 { result, lhs, rhs } => { - self.execute_i32_and_imm16(result, lhs, rhs) - } - Instr::I32Or { result, lhs, rhs } => self.execute_i32_or(result, lhs, rhs), - Instr::I32OrImm16 { result, lhs, rhs } => { - self.execute_i32_or_imm16(result, lhs, rhs) - } - Instr::I32Nand { result, lhs, rhs } => self.execute_i32_nand(result, lhs, rhs), - Instr::I32NandImm16 { result, lhs, rhs } => { - self.execute_i32_nand_imm16(result, lhs, rhs) - } - Instr::I32Nor { result, lhs, rhs } => self.execute_i32_nor(result, lhs, rhs), - Instr::I32NorImm16 { result, lhs, rhs } => { - self.execute_i32_nor_imm16(result, lhs, rhs) - } - Instr::I32Shl { result, lhs, rhs } => self.execute_i32_shl(result, lhs, rhs), - Instr::I32ShlBy { result, lhs, rhs } => self.execute_i32_shl_by(result, lhs, rhs), - Instr::I32ShlImm16 { result, lhs, rhs } => { - self.execute_i32_shl_imm16(result, lhs, rhs) - } - Instr::I32ShrU { result, lhs, rhs } => self.execute_i32_shr_u(result, lhs, rhs), - Instr::I32ShrUBy { result, lhs, rhs } => { - self.execute_i32_shr_u_by(result, lhs, rhs) - } - Instr::I32ShrUImm16 { result, lhs, rhs } => { - self.execute_i32_shr_u_imm16(result, lhs, rhs) - } - Instr::I32ShrS { result, lhs, rhs } => self.execute_i32_shr_s(result, lhs, rhs), - Instr::I32ShrSBy { result, lhs, rhs } => { - self.execute_i32_shr_s_by(result, lhs, rhs) - } - Instr::I32ShrSImm16 { result, lhs, rhs } => { - self.execute_i32_shr_s_imm16(result, lhs, rhs) - } - Instr::I32Rotl { result, lhs, rhs } => self.execute_i32_rotl(result, lhs, rhs), - Instr::I32RotlBy { result, lhs, rhs } => self.execute_i32_rotl_by(result, lhs, rhs), - Instr::I32RotlImm16 { result, lhs, rhs } => { - self.execute_i32_rotl_imm16(result, lhs, rhs) - } - Instr::I32Rotr { result, lhs, rhs } => self.execute_i32_rotr(result, lhs, rhs), - Instr::I32RotrBy { result, lhs, rhs } => self.execute_i32_rotr_by(result, lhs, rhs), - Instr::I32RotrImm16 { result, lhs, rhs } => { - self.execute_i32_rotr_imm16(result, lhs, rhs) - } - Instr::I64Clz { result, input } => self.execute_i64_clz(result, input), - Instr::I64Ctz { result, input } => self.execute_i64_ctz(result, input), - Instr::I64Popcnt { result, input } => self.execute_i64_popcnt(result, input), - Instr::I64Add { result, lhs, rhs } => self.execute_i64_add(result, lhs, rhs), - Instr::I64AddImm16 { result, lhs, rhs } => { - self.execute_i64_add_imm16(result, lhs, rhs) - } - Instr::I64Sub { result, lhs, rhs } => self.execute_i64_sub(result, lhs, rhs), - Instr::I64SubImm16Lhs { result, lhs, rhs } => { - self.execute_i64_sub_imm16_lhs(result, lhs, rhs) - } - Instr::I64Mul { result, lhs, rhs } => self.execute_i64_mul(result, lhs, rhs), - Instr::I64MulImm16 { result, lhs, rhs } => { - self.execute_i64_mul_imm16(result, lhs, rhs) - } - Instr::I64DivS { result, lhs, rhs } => self.execute_i64_div_s(result, lhs, rhs)?, - Instr::I64DivSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_div_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I64DivSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_div_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I64DivU { result, lhs, rhs } => self.execute_i64_div_u(result, lhs, rhs)?, - Instr::I64DivUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_div_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64DivUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_div_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I64RemS { result, lhs, rhs } => self.execute_i64_rem_s(result, lhs, rhs)?, - Instr::I64RemSImm16Rhs { result, lhs, rhs } => { - self.execute_i64_rem_s_imm16_rhs(result, lhs, rhs)? - } - Instr::I64RemSImm16Lhs { result, lhs, rhs } => { - self.execute_i64_rem_s_imm16_lhs(result, lhs, rhs)? - } - Instr::I64RemU { result, lhs, rhs } => self.execute_i64_rem_u(result, lhs, rhs)?, - Instr::I64RemUImm16Rhs { result, lhs, rhs } => { - self.execute_i64_rem_u_imm16_rhs(result, lhs, rhs) - } - Instr::I64RemUImm16Lhs { result, lhs, rhs } => { - self.execute_i64_rem_u_imm16_lhs(result, lhs, rhs)? - } - Instr::I64BitAnd { result, lhs, rhs } => self.execute_i64_bitand(result, lhs, rhs), - Instr::I64BitAndImm16 { result, lhs, rhs } => { - self.execute_i64_bitand_imm16(result, lhs, rhs) - } - Instr::I64BitOr { result, lhs, rhs } => self.execute_i64_bitor(result, lhs, rhs), - Instr::I64BitOrImm16 { result, lhs, rhs } => { - self.execute_i64_bitor_imm16(result, lhs, rhs) - } - Instr::I64BitXor { result, lhs, rhs } => self.execute_i64_bitxor(result, lhs, rhs), - Instr::I64BitXorImm16 { result, lhs, rhs } => { - self.execute_i64_bitxor_imm16(result, lhs, rhs) - } - Instr::I64And { result, lhs, rhs } => self.execute_i64_and(result, lhs, rhs), - Instr::I64AndImm16 { result, lhs, rhs } => { - self.execute_i64_and_imm16(result, lhs, rhs) - } - Instr::I64Or { result, lhs, rhs } => self.execute_i64_or(result, lhs, rhs), - Instr::I64OrImm16 { result, lhs, rhs } => { - self.execute_i64_or_imm16(result, lhs, rhs) - } - Instr::I64Nand { result, lhs, rhs } => self.execute_i64_nand(result, lhs, rhs), - Instr::I64NandImm16 { result, lhs, rhs } => { - self.execute_i64_nand_imm16(result, lhs, rhs) - } - Instr::I64Nor { result, lhs, rhs } => self.execute_i64_nor(result, lhs, rhs), - Instr::I64NorImm16 { result, lhs, rhs } => { - self.execute_i64_nor_imm16(result, lhs, rhs) - } - Instr::I64Shl { result, lhs, rhs } => self.execute_i64_shl(result, lhs, rhs), - Instr::I64ShlBy { result, lhs, rhs } => self.execute_i64_shl_by(result, lhs, rhs), - Instr::I64ShlImm16 { result, lhs, rhs } => { - self.execute_i64_shl_imm16(result, lhs, rhs) - } - Instr::I64ShrU { result, lhs, rhs } => self.execute_i64_shr_u(result, lhs, rhs), - Instr::I64ShrUBy { result, lhs, rhs } => { - self.execute_i64_shr_u_by(result, lhs, rhs) - } - Instr::I64ShrUImm16 { result, lhs, rhs } => { - self.execute_i64_shr_u_imm16(result, lhs, rhs) - } - Instr::I64ShrS { result, lhs, rhs } => self.execute_i64_shr_s(result, lhs, rhs), - Instr::I64ShrSBy { result, lhs, rhs } => { - self.execute_i64_shr_s_by(result, lhs, rhs) - } - Instr::I64ShrSImm16 { result, lhs, rhs } => { - self.execute_i64_shr_s_imm16(result, lhs, rhs) - } - Instr::I64Rotl { result, lhs, rhs } => self.execute_i64_rotl(result, lhs, rhs), - Instr::I64RotlBy { result, lhs, rhs } => self.execute_i64_rotl_by(result, lhs, rhs), - Instr::I64RotlImm16 { result, lhs, rhs } => { - self.execute_i64_rotl_imm16(result, lhs, rhs) - } - Instr::I64Rotr { result, lhs, rhs } => self.execute_i64_rotr(result, lhs, rhs), - Instr::I64RotrBy { result, lhs, rhs } => self.execute_i64_rotr_by(result, lhs, rhs), - Instr::I64RotrImm16 { result, lhs, rhs } => { - self.execute_i64_rotr_imm16(result, lhs, rhs) - } - Instr::I64Add128 { results, lhs_lo } => self.execute_i64_add128(results, lhs_lo), - Instr::I64Sub128 { results, lhs_lo } => self.execute_i64_sub128(results, lhs_lo), - Instr::I64MulWideS { results, lhs, rhs } => { - self.execute_i64_mul_wide_s(results, lhs, rhs) - } - Instr::I64MulWideU { results, lhs, rhs } => { - self.execute_i64_mul_wide_u(results, lhs, rhs) - } - Instr::I32WrapI64 { result, input } => self.execute_i32_wrap_i64(result, input), - Instr::I32Extend8S { result, input } => self.execute_i32_extend8_s(result, input), - Instr::I32Extend16S { result, input } => self.execute_i32_extend16_s(result, input), - Instr::I64Extend8S { result, input } => self.execute_i64_extend8_s(result, input), - Instr::I64Extend16S { result, input } => self.execute_i64_extend16_s(result, input), - Instr::I64Extend32S { result, input } => self.execute_i64_extend32_s(result, input), - Instr::F32Abs { result, input } => self.execute_f32_abs(result, input), - Instr::F32Neg { result, input } => self.execute_f32_neg(result, input), - Instr::F32Ceil { result, input } => self.execute_f32_ceil(result, input), - Instr::F32Floor { result, input } => self.execute_f32_floor(result, input), - Instr::F32Trunc { result, input } => self.execute_f32_trunc(result, input), - Instr::F32Nearest { result, input } => self.execute_f32_nearest(result, input), - Instr::F32Sqrt { result, input } => self.execute_f32_sqrt(result, input), - Instr::F32Add { result, lhs, rhs } => self.execute_f32_add(result, lhs, rhs), - Instr::F32Sub { result, lhs, rhs } => self.execute_f32_sub(result, lhs, rhs), - Instr::F32Mul { result, lhs, rhs } => self.execute_f32_mul(result, lhs, rhs), - Instr::F32Div { result, lhs, rhs } => self.execute_f32_div(result, lhs, rhs), - Instr::F32Min { result, lhs, rhs } => self.execute_f32_min(result, lhs, rhs), - Instr::F32Max { result, lhs, rhs } => self.execute_f32_max(result, lhs, rhs), - Instr::F32Copysign { result, lhs, rhs } => { - self.execute_f32_copysign(result, lhs, rhs) - } - Instr::F32CopysignImm { result, lhs, rhs } => { - self.execute_f32_copysign_imm(result, lhs, rhs) - } - Instr::F64Abs { result, input } => self.execute_f64_abs(result, input), - Instr::F64Neg { result, input } => self.execute_f64_neg(result, input), - Instr::F64Ceil { result, input } => self.execute_f64_ceil(result, input), - Instr::F64Floor { result, input } => self.execute_f64_floor(result, input), - Instr::F64Trunc { result, input } => self.execute_f64_trunc(result, input), - Instr::F64Nearest { result, input } => self.execute_f64_nearest(result, input), - Instr::F64Sqrt { result, input } => self.execute_f64_sqrt(result, input), - Instr::F64Add { result, lhs, rhs } => self.execute_f64_add(result, lhs, rhs), - Instr::F64Sub { result, lhs, rhs } => self.execute_f64_sub(result, lhs, rhs), - Instr::F64Mul { result, lhs, rhs } => self.execute_f64_mul(result, lhs, rhs), - Instr::F64Div { result, lhs, rhs } => self.execute_f64_div(result, lhs, rhs), - Instr::F64Min { result, lhs, rhs } => self.execute_f64_min(result, lhs, rhs), - Instr::F64Max { result, lhs, rhs } => self.execute_f64_max(result, lhs, rhs), - Instr::F64Copysign { result, lhs, rhs } => { - self.execute_f64_copysign(result, lhs, rhs) - } - Instr::F64CopysignImm { result, lhs, rhs } => { - self.execute_f64_copysign_imm(result, lhs, rhs) - } - Instr::I32TruncF32S { result, input } => { - self.execute_i32_trunc_f32_s(result, input)? - } - Instr::I32TruncF32U { result, input } => { - self.execute_i32_trunc_f32_u(result, input)? - } - Instr::I32TruncF64S { result, input } => { - self.execute_i32_trunc_f64_s(result, input)? - } - Instr::I32TruncF64U { result, input } => { - self.execute_i32_trunc_f64_u(result, input)? - } - Instr::I64TruncF32S { result, input } => { - self.execute_i64_trunc_f32_s(result, input)? - } - Instr::I64TruncF32U { result, input } => { - self.execute_i64_trunc_f32_u(result, input)? - } - Instr::I64TruncF64S { result, input } => { - self.execute_i64_trunc_f64_s(result, input)? - } - Instr::I64TruncF64U { result, input } => { - self.execute_i64_trunc_f64_u(result, input)? - } - Instr::I32TruncSatF32S { result, input } => { - self.execute_i32_trunc_sat_f32_s(result, input) - } - Instr::I32TruncSatF32U { result, input } => { - self.execute_i32_trunc_sat_f32_u(result, input) - } - Instr::I32TruncSatF64S { result, input } => { - self.execute_i32_trunc_sat_f64_s(result, input) - } - Instr::I32TruncSatF64U { result, input } => { - self.execute_i32_trunc_sat_f64_u(result, input) - } - Instr::I64TruncSatF32S { result, input } => { - self.execute_i64_trunc_sat_f32_s(result, input) - } - Instr::I64TruncSatF32U { result, input } => { - self.execute_i64_trunc_sat_f32_u(result, input) - } - Instr::I64TruncSatF64S { result, input } => { - self.execute_i64_trunc_sat_f64_s(result, input) - } - Instr::I64TruncSatF64U { result, input } => { - self.execute_i64_trunc_sat_f64_u(result, input) - } - Instr::F32DemoteF64 { result, input } => self.execute_f32_demote_f64(result, input), - Instr::F64PromoteF32 { result, input } => { - self.execute_f64_promote_f32(result, input) - } - Instr::F32ConvertI32S { result, input } => { - self.execute_f32_convert_i32_s(result, input) - } - Instr::F32ConvertI32U { result, input } => { - self.execute_f32_convert_i32_u(result, input) - } - Instr::F32ConvertI64S { result, input } => { - self.execute_f32_convert_i64_s(result, input) - } - Instr::F32ConvertI64U { result, input } => { - self.execute_f32_convert_i64_u(result, input) - } - Instr::F64ConvertI32S { result, input } => { - self.execute_f64_convert_i32_s(result, input) - } - Instr::F64ConvertI32U { result, input } => { - self.execute_f64_convert_i32_u(result, input) - } - Instr::F64ConvertI64S { result, input } => { - self.execute_f64_convert_i64_s(result, input) - } - Instr::F64ConvertI64U { result, input } => { - self.execute_f64_convert_i64_u(result, input) - } - Instr::TableGet { result, index } => { - self.execute_table_get(store.inner(), result, index)? - } - Instr::TableGetImm { result, index } => { - self.execute_table_get_imm(store.inner(), result, index)? - } - Instr::TableSize { result, table } => { - self.execute_table_size(store.inner(), result, table) - } - Instr::TableSet { index, value } => { - self.execute_table_set(store.inner_mut(), index, value)? - } - Instr::TableSetAt { index, value } => { - self.execute_table_set_at(store.inner_mut(), index, value)? - } - Instr::TableCopy { dst, src, len } => { - self.execute_table_copy(store.inner_mut(), dst, src, len)? - } - Instr::TableInit { dst, src, len } => { - self.execute_table_init(store.inner_mut(), dst, src, len)? - } - Instr::TableFill { dst, len, value } => { - self.execute_table_fill(store.inner_mut(), dst, len, value)? - } - Instr::TableGrow { - result, - delta, - value, - } => self.execute_table_grow(store, result, delta, value)?, - Instr::ElemDrop { index } => self.execute_element_drop(store.inner_mut(), index), - Instr::DataDrop { index } => self.execute_data_drop(store.inner_mut(), index), - Instr::MemorySize { result, memory } => { - self.execute_memory_size(store.inner(), result, memory) - } - Instr::MemoryGrow { result, delta } => { - self.execute_memory_grow(store, result, delta)? - } - Instr::MemoryCopy { dst, src, len } => { - self.execute_memory_copy(store.inner_mut(), dst, src, len)? - } - Instr::MemoryFill { dst, value, len } => { - self.execute_memory_fill(store.inner_mut(), dst, value, len)? - } - Instr::MemoryFillImm { dst, value, len } => { - self.execute_memory_fill_imm(store.inner_mut(), dst, value, len)? - } - Instr::MemoryInit { dst, src, len } => { - self.execute_memory_init(store.inner_mut(), dst, src, len)? - } - Instr::TableIndex { .. } - | Instr::MemoryIndex { .. } - | Instr::DataIndex { .. } - | Instr::ElemIndex { .. } - | Instr::Const32 { .. } - | Instr::I64Const32 { .. } - | Instr::F64Const32 { .. } - | Instr::BranchTableTarget { .. } - | Instr::Slot { .. } - | Instr::Slot2 { .. } - | Instr::Slot3 { .. } - | Instr::SlotAndImm32 { .. } - | Instr::Imm16AndImm32 { .. } - | Instr::SlotSpan { .. } - | Instr::SlotList { .. } - | Instr::CallIndirectParams { .. } - | Instr::CallIndirectParamsImm16 { .. } => self.invalid_instruction_word()?, - #[cfg(feature = "simd")] - Instr::I8x16Splat { result, value } => self.execute_i8x16_splat(result, value), - #[cfg(feature = "simd")] - Instr::I16x8Splat { result, value } => self.execute_i16x8_splat(result, value), - #[cfg(feature = "simd")] - Instr::I32x4Splat { result, value } => self.execute_i32x4_splat(result, value), - #[cfg(feature = "simd")] - Instr::I64x2Splat { result, value } => self.execute_i64x2_splat(result, value), - #[cfg(feature = "simd")] - Instr::F32x4Splat { result, value } => self.execute_f32x4_splat(result, value), - #[cfg(feature = "simd")] - Instr::F64x2Splat { result, value } => self.execute_f64x2_splat(result, value), - #[cfg(feature = "simd")] - Instr::I8x16ExtractLaneS { - result, - value, - lane, - } => self.i8x16_extract_lane_s(result, value, lane), - #[cfg(feature = "simd")] - Instr::I8x16ExtractLaneU { - result, - value, - lane, - } => self.i8x16_extract_lane_u(result, value, lane), - #[cfg(feature = "simd")] - Instr::I16x8ExtractLaneS { - result, - value, - lane, - } => self.i16x8_extract_lane_s(result, value, lane), - #[cfg(feature = "simd")] - Instr::I16x8ExtractLaneU { - result, - value, - lane, - } => self.i16x8_extract_lane_u(result, value, lane), - #[cfg(feature = "simd")] - Instr::I32x4ExtractLane { - result, - value, - lane, - } => self.i32x4_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::I64x2ExtractLane { - result, - value, - lane, - } => self.i64x2_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::F32x4ExtractLane { - result, - value, - lane, - } => self.f32x4_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::F64x2ExtractLane { - result, - value, - lane, - } => self.f64x2_extract_lane(result, value, lane), - #[cfg(feature = "simd")] - Instr::I8x16ReplaceLane { - result, - input, - lane, - } => self.execute_i8x16_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I8x16ReplaceLaneImm { - result, - input, - lane, - value, - } => self.execute_i8x16_replace_lane_imm(result, input, lane, value), - #[cfg(feature = "simd")] - Instr::I16x8ReplaceLane { - result, - input, - lane, - } => self.execute_i16x8_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I16x8ReplaceLaneImm { - result, - input, - lane, - } => self.execute_i16x8_replace_lane_imm(result, input, lane), - #[cfg(feature = "simd")] - Instr::I32x4ReplaceLane { - result, - input, - lane, - } => self.execute_i32x4_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I32x4ReplaceLaneImm { - result, - input, - lane, - } => self.execute_i32x4_replace_lane_imm(result, input, lane), - #[cfg(feature = "simd")] - Instr::I64x2ReplaceLane { - result, - input, - lane, - } => self.execute_i64x2_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::I64x2ReplaceLaneImm32 { - result, - input, - lane, - } => self.execute_i64x2_replace_lane_imm32(result, input, lane), - #[cfg(feature = "simd")] - Instr::F32x4ReplaceLane { - result, - input, - lane, - } => self.execute_f32x4_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::F32x4ReplaceLaneImm { - result, - input, - lane, - } => self.execute_f32x4_replace_lane_imm(result, input, lane), - #[cfg(feature = "simd")] - Instr::F64x2ReplaceLane { - result, - input, - lane, - } => self.execute_f64x2_replace_lane(result, input, lane), - #[cfg(feature = "simd")] - Instr::F64x2ReplaceLaneImm32 { - result, - input, - lane, - } => self.execute_f64x2_replace_lane_imm32(result, input, lane), - #[cfg(feature = "simd")] - Instr::I8x16Shuffle { result, lhs, rhs } => { - self.execute_i8x16_shuffle(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Swizzle { - result, - input, - selector, - } => self.execute_i8x16_swizzle(result, input, selector), - #[cfg(feature = "simd")] - Instr::I8x16Add { result, lhs, rhs } => self.execute_i8x16_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Add { result, lhs, rhs } => self.execute_i16x8_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Add { result, lhs, rhs } => self.execute_i32x4_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Add { result, lhs, rhs } => self.execute_i64x2_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16Sub { result, lhs, rhs } => self.execute_i8x16_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Sub { result, lhs, rhs } => self.execute_i16x8_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Sub { result, lhs, rhs } => self.execute_i32x4_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Sub { result, lhs, rhs } => self.execute_i64x2_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Mul { result, lhs, rhs } => self.execute_i16x8_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Mul { result, lhs, rhs } => self.execute_i32x4_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Mul { result, lhs, rhs } => self.execute_i64x2_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4DotI16x8S { result, lhs, rhs } => { - self.execute_i32x4_dot_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Neg { result, input } => self.execute_i8x16_neg(result, input), - #[cfg(feature = "simd")] - Instr::I16x8Neg { result, input } => self.execute_i16x8_neg(result, input), - #[cfg(feature = "simd")] - Instr::I32x4Neg { result, input } => self.execute_i32x4_neg(result, input), - #[cfg(feature = "simd")] - Instr::I64x2Neg { result, input } => self.execute_i64x2_neg(result, input), - #[cfg(feature = "simd")] - Instr::I16x8ExtmulLowI8x16S { result, lhs, rhs } => { - self.execute_i16x8_extmul_low_i8x16_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtmulHighI8x16S { result, lhs, rhs } => { - self.execute_i16x8_extmul_high_i8x16_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtmulLowI8x16U { result, lhs, rhs } => { - self.execute_i16x8_extmul_low_i8x16_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtmulHighI8x16U { result, lhs, rhs } => { - self.execute_i16x8_extmul_high_i8x16_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulLowI16x8S { result, lhs, rhs } => { - self.execute_i32x4_extmul_low_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulHighI16x8S { result, lhs, rhs } => { - self.execute_i32x4_extmul_high_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulLowI16x8U { result, lhs, rhs } => { - self.execute_i32x4_extmul_low_i16x8_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtmulHighI16x8U { result, lhs, rhs } => { - self.execute_i32x4_extmul_high_i16x8_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulLowI32x4S { result, lhs, rhs } => { - self.execute_i64x2_extmul_low_i32x4_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulHighI32x4S { result, lhs, rhs } => { - self.execute_i64x2_extmul_high_i32x4_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulLowI32x4U { result, lhs, rhs } => { - self.execute_i64x2_extmul_low_i32x4_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtmulHighI32x4U { result, lhs, rhs } => { - self.execute_i64x2_extmul_high_i32x4_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtaddPairwiseI8x16S { result, input } => { - self.execute_i16x8_extadd_pairwise_i8x16_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtaddPairwiseI8x16U { result, input } => { - self.execute_i16x8_extadd_pairwise_i8x16_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtaddPairwiseI16x8S { result, input } => { - self.execute_i32x4_extadd_pairwise_i16x8_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtaddPairwiseI16x8U { result, input } => { - self.execute_i32x4_extadd_pairwise_i16x8_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I8x16AddSatS { result, lhs, rhs } => { - self.execute_i8x16_add_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16AddSatU { result, lhs, rhs } => { - self.execute_i8x16_add_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8AddSatS { result, lhs, rhs } => { - self.execute_i16x8_add_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8AddSatU { result, lhs, rhs } => { - self.execute_i16x8_add_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16SubSatS { result, lhs, rhs } => { - self.execute_i8x16_sub_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16SubSatU { result, lhs, rhs } => { - self.execute_i8x16_sub_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8SubSatS { result, lhs, rhs } => { - self.execute_i16x8_sub_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8SubSatU { result, lhs, rhs } => { - self.execute_i16x8_sub_sat_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8Q15MulrSatS { result, lhs, rhs } => { - self.execute_i16x8_q15mulr_sat_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16MinS { result, lhs, rhs } => self.execute_i8x16_min_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16MinU { result, lhs, rhs } => self.execute_i8x16_min_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MinS { result, lhs, rhs } => self.execute_i16x8_min_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MinU { result, lhs, rhs } => self.execute_i16x8_min_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MinS { result, lhs, rhs } => self.execute_i32x4_min_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MinU { result, lhs, rhs } => self.execute_i32x4_min_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16MaxS { result, lhs, rhs } => self.execute_i8x16_max_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16MaxU { result, lhs, rhs } => self.execute_i8x16_max_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MaxS { result, lhs, rhs } => self.execute_i16x8_max_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8MaxU { result, lhs, rhs } => self.execute_i16x8_max_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MaxS { result, lhs, rhs } => self.execute_i32x4_max_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4MaxU { result, lhs, rhs } => self.execute_i32x4_max_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16AvgrU { result, lhs, rhs } => { - self.execute_i8x16_avgr_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8AvgrU { result, lhs, rhs } => { - self.execute_i16x8_avgr_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Abs { result, input } => self.execute_i8x16_abs(result, input), - #[cfg(feature = "simd")] - Instr::I16x8Abs { result, input } => self.execute_i16x8_abs(result, input), - #[cfg(feature = "simd")] - Instr::I32x4Abs { result, input } => self.execute_i32x4_abs(result, input), - #[cfg(feature = "simd")] - Instr::I64x2Abs { result, input } => self.execute_i64x2_abs(result, input), - #[cfg(feature = "simd")] - Instr::I8x16Shl { result, lhs, rhs } => self.execute_i8x16_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16ShlBy { result, lhs, rhs } => { - self.execute_i8x16_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8Shl { result, lhs, rhs } => self.execute_i16x8_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8ShlBy { result, lhs, rhs } => { - self.execute_i16x8_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4Shl { result, lhs, rhs } => self.execute_i32x4_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4ShlBy { result, lhs, rhs } => { - self.execute_i32x4_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2Shl { result, lhs, rhs } => self.execute_i64x2_shl(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2ShlBy { result, lhs, rhs } => { - self.execute_i64x2_shl_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16ShrS { result, lhs, rhs } => self.execute_i8x16_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16ShrSBy { result, lhs, rhs } => { - self.execute_i8x16_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16ShrU { result, lhs, rhs } => self.execute_i8x16_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16ShrUBy { result, lhs, rhs } => { - self.execute_i8x16_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ShrS { result, lhs, rhs } => self.execute_i16x8_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8ShrSBy { result, lhs, rhs } => { - self.execute_i16x8_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ShrU { result, lhs, rhs } => self.execute_i16x8_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8ShrUBy { result, lhs, rhs } => { - self.execute_i16x8_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ShrS { result, lhs, rhs } => self.execute_i32x4_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4ShrSBy { result, lhs, rhs } => { - self.execute_i32x4_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4ShrU { result, lhs, rhs } => self.execute_i32x4_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4ShrUBy { result, lhs, rhs } => { - self.execute_i32x4_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ShrS { result, lhs, rhs } => self.execute_i64x2_shr_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2ShrSBy { result, lhs, rhs } => { - self.execute_i64x2_shr_s_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I64x2ShrU { result, lhs, rhs } => self.execute_i64x2_shr_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2ShrUBy { result, lhs, rhs } => { - self.execute_i64x2_shr_u_by(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::V128And { result, lhs, rhs } => self.execute_v128_and(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::V128Or { result, lhs, rhs } => self.execute_v128_or(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::V128Xor { result, lhs, rhs } => self.execute_v128_xor(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::V128Andnot { result, lhs, rhs } => { - self.execute_v128_andnot(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::V128Not { result, input } => self.execute_v128_not(result, input), - #[cfg(feature = "simd")] - Instr::V128Bitselect { result, lhs, rhs } => { - self.execute_v128_bitselect(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16Popcnt { result, input } => self.execute_i8x16_popcnt(result, input), - #[cfg(feature = "simd")] - Instr::V128AnyTrue { result, input } => self.execute_v128_any_true(result, input), - #[cfg(feature = "simd")] - Instr::I8x16AllTrue { result, input } => self.execute_i8x16_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I16x8AllTrue { result, input } => self.execute_i16x8_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I32x4AllTrue { result, input } => self.execute_i32x4_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I64x2AllTrue { result, input } => self.execute_i64x2_all_true(result, input), - #[cfg(feature = "simd")] - Instr::I8x16Bitmask { result, input } => self.execute_i8x16_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I16x8Bitmask { result, input } => self.execute_i16x8_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I32x4Bitmask { result, input } => self.execute_i32x4_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I64x2Bitmask { result, input } => self.execute_i64x2_bitmask(result, input), - #[cfg(feature = "simd")] - Instr::I8x16Eq { result, lhs, rhs } => self.execute_i8x16_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Eq { result, lhs, rhs } => self.execute_i16x8_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Eq { result, lhs, rhs } => self.execute_i32x4_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Eq { result, lhs, rhs } => self.execute_i64x2_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Eq { result, lhs, rhs } => self.execute_f32x4_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Eq { result, lhs, rhs } => self.execute_f64x2_eq(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16Ne { result, lhs, rhs } => self.execute_i8x16_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8Ne { result, lhs, rhs } => self.execute_i16x8_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4Ne { result, lhs, rhs } => self.execute_i32x4_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2Ne { result, lhs, rhs } => self.execute_i64x2_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Ne { result, lhs, rhs } => self.execute_f32x4_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Ne { result, lhs, rhs } => self.execute_f64x2_ne(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LtS { result, lhs, rhs } => self.execute_i8x16_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LtU { result, lhs, rhs } => self.execute_i8x16_lt_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LtS { result, lhs, rhs } => self.execute_i16x8_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LtU { result, lhs, rhs } => self.execute_i16x8_lt_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LtS { result, lhs, rhs } => self.execute_i32x4_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LtU { result, lhs, rhs } => self.execute_i32x4_lt_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2LtS { result, lhs, rhs } => self.execute_i64x2_lt_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Lt { result, lhs, rhs } => self.execute_f32x4_lt(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Lt { result, lhs, rhs } => self.execute_f64x2_lt(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LeS { result, lhs, rhs } => self.execute_i8x16_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I8x16LeU { result, lhs, rhs } => self.execute_i8x16_le_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LeS { result, lhs, rhs } => self.execute_i16x8_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I16x8LeU { result, lhs, rhs } => self.execute_i16x8_le_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LeS { result, lhs, rhs } => self.execute_i32x4_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I32x4LeU { result, lhs, rhs } => self.execute_i32x4_le_u(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::I64x2LeS { result, lhs, rhs } => self.execute_i64x2_le_s(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Le { result, lhs, rhs } => self.execute_f32x4_le(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Le { result, lhs, rhs } => self.execute_f64x2_le(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Neg { result, input } => self.execute_f32x4_neg(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Neg { result, input } => self.execute_f64x2_neg(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Abs { result, input } => self.execute_f32x4_abs(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Abs { result, input } => self.execute_f64x2_abs(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Min { result, lhs, rhs } => self.execute_f32x4_min(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Min { result, lhs, rhs } => self.execute_f64x2_min(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Max { result, lhs, rhs } => self.execute_f32x4_max(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Max { result, lhs, rhs } => self.execute_f64x2_max(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Pmin { result, lhs, rhs } => self.execute_f32x4_pmin(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Pmin { result, lhs, rhs } => self.execute_f64x2_pmin(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Pmax { result, lhs, rhs } => self.execute_f32x4_pmax(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Pmax { result, lhs, rhs } => self.execute_f64x2_pmax(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Add { result, lhs, rhs } => self.execute_f32x4_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Add { result, lhs, rhs } => self.execute_f64x2_add(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Sub { result, lhs, rhs } => self.execute_f32x4_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Sub { result, lhs, rhs } => self.execute_f64x2_sub(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Div { result, lhs, rhs } => self.execute_f32x4_div(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Div { result, lhs, rhs } => self.execute_f64x2_div(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Mul { result, lhs, rhs } => self.execute_f32x4_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F64x2Mul { result, lhs, rhs } => self.execute_f64x2_mul(result, lhs, rhs), - #[cfg(feature = "simd")] - Instr::F32x4Sqrt { result, input } => self.execute_f32x4_sqrt(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Sqrt { result, input } => self.execute_f64x2_sqrt(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Ceil { result, input } => self.execute_f32x4_ceil(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Ceil { result, input } => self.execute_f64x2_ceil(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Floor { result, input } => self.execute_f32x4_floor(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Floor { result, input } => self.execute_f64x2_floor(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Trunc { result, input } => self.execute_f32x4_trunc(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Trunc { result, input } => self.execute_f64x2_trunc(result, input), - #[cfg(feature = "simd")] - Instr::F32x4Nearest { result, input } => self.execute_f32x4_nearest(result, input), - #[cfg(feature = "simd")] - Instr::F64x2Nearest { result, input } => self.execute_f64x2_nearest(result, input), - #[cfg(feature = "simd")] - Instr::F32x4ConvertI32x4S { result, input } => { - self.execute_f32x4_convert_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::F32x4ConvertI32x4U { result, input } => { - self.execute_f32x4_convert_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::F64x2ConvertLowI32x4S { result, input } => { - self.execute_f64x2_convert_low_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::F64x2ConvertLowI32x4U { result, input } => { - self.execute_f64x2_convert_low_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF32x4S { result, input } => { - self.execute_i32x4_trunc_sat_f32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF32x4U { result, input } => { - self.execute_i32x4_trunc_sat_f32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF64x2SZero { result, input } => { - self.execute_i32x4_trunc_sat_f64x2_s_zero(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4TruncSatF64x2UZero { result, input } => { - self.execute_i32x4_trunc_sat_f64x2_u_zero(result, input) - } - #[cfg(feature = "simd")] - Instr::F32x4DemoteF64x2Zero { result, input } => { - self.execute_f32x4_demote_f64x2_zero(result, input) - } - #[cfg(feature = "simd")] - Instr::F64x2PromoteLowF32x4 { result, input } => { - self.execute_f64x2_promote_low_f32x4(result, input) - } - #[cfg(feature = "simd")] - Instr::I8x16NarrowI16x8S { result, lhs, rhs } => { - self.execute_i8x16_narrow_i16x8_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I8x16NarrowI16x8U { result, lhs, rhs } => { - self.execute_i8x16_narrow_i16x8_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8NarrowI32x4S { result, lhs, rhs } => { - self.execute_i16x8_narrow_i32x4_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8NarrowI32x4U { result, lhs, rhs } => { - self.execute_i16x8_narrow_i32x4_u(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendLowI8x16S { result, input } => { - self.execute_i16x8_extend_low_i8x16_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendHighI8x16S { result, input } => { - self.execute_i16x8_extend_high_i8x16_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendLowI8x16U { result, input } => { - self.execute_i16x8_extend_low_i8x16_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I16x8ExtendHighI8x16U { result, input } => { - self.execute_i16x8_extend_high_i8x16_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendLowI16x8S { result, input } => { - self.execute_i32x4_extend_low_i16x8_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendHighI16x8S { result, input } => { - self.execute_i32x4_extend_high_i16x8_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendLowI16x8U { result, input } => { - self.execute_i32x4_extend_low_i16x8_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I32x4ExtendHighI16x8U { result, input } => { - self.execute_i32x4_extend_high_i16x8_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendLowI32x4S { result, input } => { - self.execute_i64x2_extend_low_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendHighI32x4S { result, input } => { - self.execute_i64x2_extend_high_i32x4_s(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendLowI32x4U { result, input } => { - self.execute_i64x2_extend_low_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::I64x2ExtendHighI32x4U { result, input } => { - self.execute_i64x2_extend_high_i32x4_u(result, input) - } - #[cfg(feature = "simd")] - Instr::V128Store { ptr, offset_lo } => { - self.execute_v128_store(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128StoreOffset16 { ptr, value, offset } => { - self.execute_v128_store_offset16(ptr, offset, value)? - } - #[cfg(feature = "simd")] - Instr::V128StoreAt { value, address } => { - self.execute_v128_store_at(store.inner_mut(), address, value)? - } - #[cfg(feature = "simd")] - Instr::V128Store8Lane { ptr, offset_lo } => { - self.execute_v128_store8_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store8LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store8_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store8LaneAt { value, address } => { - self.execute_v128_store8_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Store16Lane { ptr, offset_lo } => { - self.execute_v128_store16_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store16LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store16_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store16LaneAt { value, address } => { - self.execute_v128_store16_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Store32Lane { ptr, offset_lo } => { - self.execute_v128_store32_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store32LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store32_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store32LaneAt { value, address } => { - self.execute_v128_store32_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Store64Lane { ptr, offset_lo } => { - self.execute_v128_store64_lane(store.inner_mut(), ptr, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Store64LaneOffset8 { - ptr, - value, - offset, - lane, - } => self.execute_v128_store64_lane_offset8(ptr, value, offset, lane)?, - #[cfg(feature = "simd")] - Instr::V128Store64LaneAt { value, address } => { - self.execute_v128_store64_lane_at(store.inner_mut(), value, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load { result, offset_lo } => { - self.execute_v128_load(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128LoadAt { result, address } => { - self.execute_v128_load_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128LoadOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32Zero { result, offset_lo } => { - self.execute_v128_load32_zero(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32ZeroAt { result, address } => { - self.execute_v128_load32_zero_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32ZeroOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32_zero_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load64Zero { result, offset_lo } => { - self.execute_v128_load64_zero(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load64ZeroAt { result, address } => { - self.execute_v128_load64_zero_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load64ZeroOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load64_zero_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8Splat { result, offset_lo } => { - self.execute_v128_load8_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8SplatAt { result, address } => { - self.execute_v128_load8_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load8SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load8_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load16Splat { result, offset_lo } => { - self.execute_v128_load16_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16SplatAt { result, address } => { - self.execute_v128_load16_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load16_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32Splat { result, offset_lo } => { - self.execute_v128_load32_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32SplatAt { result, address } => { - self.execute_v128_load32_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load64Splat { result, offset_lo } => { - self.execute_v128_load64_splat(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load64SplatAt { result, address } => { - self.execute_v128_load64_splat_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load64SplatOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load64_splat_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8x8S { result, offset_lo } => { - self.execute_v128_load8x8_s(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8SAt { result, address } => { - self.execute_v128_load8x8_s_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8SOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load8x8_s_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8x8U { result, offset_lo } => { - self.execute_v128_load8x8_u(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8UAt { result, address } => { - self.execute_v128_load8x8_u_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load8x8UOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load8x8_u_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load16x4S { result, offset_lo } => { - self.execute_v128_load16x4_s(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4SAt { result, address } => { - self.execute_v128_load16x4_s_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4SOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load16x4_s_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load16x4U { result, offset_lo } => { - self.execute_v128_load16x4_u(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4UAt { result, address } => { - self.execute_v128_load16x4_u_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16x4UOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load16x4_u_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32x2S { result, offset_lo } => { - self.execute_v128_load32x2_s(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2SAt { result, address } => { - self.execute_v128_load32x2_s_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2SOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32x2_s_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load32x2U { result, offset_lo } => { - self.execute_v128_load32x2_u(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2UAt { result, address } => { - self.execute_v128_load32x2_u_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32x2UOffset16 { - result, - ptr, - offset, - } => self.execute_v128_load32x2_u_offset16(result, ptr, offset)?, - #[cfg(feature = "simd")] - Instr::V128Load8Lane { result, offset_lo } => { - self.execute_v128_load8_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load8LaneAt { result, address } => { - self.execute_v128_load8_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load16Lane { result, offset_lo } => { - self.execute_v128_load16_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load16LaneAt { result, address } => { - self.execute_v128_load16_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load32Lane { result, offset_lo } => { - self.execute_v128_load32_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load32LaneAt { result, address } => { - self.execute_v128_load32_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::V128Load64Lane { result, offset_lo } => { - self.execute_v128_load64_lane(store.inner(), result, offset_lo)? - } - #[cfg(feature = "simd")] - Instr::V128Load64LaneAt { result, address } => { - self.execute_v128_load64_lane_at(store.inner(), result, address)? - } - #[cfg(feature = "simd")] - Instr::I16x8RelaxedDotI8x16I7x16S { result, lhs, rhs } => { - self.execute_i16x8_relaxed_dot_i8x16_i7x16_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::I32x4RelaxedDotI8x16I7x16AddS { result, lhs, rhs } => { - self.execute_i32x4_relaxed_dot_i8x16_i7x16_add_s(result, lhs, rhs) - } - #[cfg(feature = "simd")] - Instr::F32x4RelaxedMadd { result, a, b } => { - self.execute_f32x4_relaxed_madd(result, a, b) - } - #[cfg(feature = "simd")] - Instr::F32x4RelaxedNmadd { result, a, b } => { - self.execute_f32x4_relaxed_nmadd(result, a, b) - } - #[cfg(feature = "simd")] - Instr::F64x2RelaxedMadd { result, a, b } => { - self.execute_f64x2_relaxed_madd(result, a, b) - } - #[cfg(feature = "simd")] - Instr::F64x2RelaxedNmadd { result, a, b } => { - self.execute_f64x2_relaxed_nmadd(result, a, b) - } - unsupported => panic!("encountered unsupported Wasmi instruction: {unsupported:?}"), - } - } + fn execute(&mut self, _store: &mut PrunedStore) -> Result<(), Error> { + todo!() } } @@ -2323,332 +189,6 @@ impl Executor<'_> { // the heap underlying the value stack is changed. unsafe { self.sp.write_as::(slot, value) }; } - - /// Shifts the instruction pointer to the next instruction. - #[inline(always)] - fn next_instr(&mut self) { - self.next_instr_at(1) - } - - /// Shifts the instruction pointer to the next instruction. - /// - /// Has a parameter `skip` to denote how many instruction words - /// to skip to reach the next actual instruction. - /// - /// # Note - /// - /// This is used by Wasmi instructions that have a fixed - /// encoding size of two instruction words such as [`Op::Branch`]. - #[inline(always)] - fn next_instr_at(&mut self, skip: usize) { - self.ip.add(skip) - } - - /// Shifts the instruction pointer to the next instruction and returns `Ok(())`. - /// - /// # Note - /// - /// This is a convenience function for fallible instructions. - #[inline(always)] - fn try_next_instr(&mut self) -> Result<(), Error> { - self.try_next_instr_at(1) - } - - /// Shifts the instruction pointer to the next instruction and returns `Ok(())`. - /// - /// Has a parameter `skip` to denote how many instruction words - /// to skip to reach the next actual instruction. - /// - /// # Note - /// - /// This is a convenience function for fallible instructions. - #[inline(always)] - fn try_next_instr_at(&mut self, skip: usize) -> Result<(), Error> { - self.next_instr_at(skip); - Ok(()) - } - - /// Returns the [`FrameSlots`] of the [`CallFrame`]. - fn frame_stack_ptr_impl(value_stack: &mut ValueStack, frame: &CallFrame) -> FrameSlots { - // Safety: We are using the frame's own base offset as input because it is - // guaranteed by the Wasm validation and translation phase to be - // valid for all register indices used by the associated function body. - unsafe { value_stack.stack_ptr_at(frame.base_offset()) } - } - - /// Initializes the [`Executor`] state for the [`CallFrame`]. - /// - /// # Note - /// - /// The initialization of the [`Executor`] allows for efficient execution. - fn init_call_frame(&mut self, frame: &CallFrame) { - Self::init_call_frame_impl(&mut self.stack.values, &mut self.sp, &mut self.ip, frame) - } - - /// Initializes the [`Executor`] state for the [`CallFrame`]. - /// - /// # Note - /// - /// The initialization of the [`Executor`] allows for efficient execution. - fn init_call_frame_impl( - value_stack: &mut ValueStack, - sp: &mut FrameSlots, - ip: &mut InstructionPtr, - frame: &CallFrame, - ) { - *sp = Self::frame_stack_ptr_impl(value_stack, frame); - *ip = frame.instr_ptr(); - } - - /// Executes a generic unary [`Op`]. - #[inline(always)] - fn execute_unary(&mut self, result: Slot, input: Slot, op: fn(P) -> R) - where - UntypedVal: ReadAs

+ WriteAs, - { - let value = self.get_stack_slot_as::

(input); - self.set_stack_slot_as::(result, op(value)); - self.next_instr(); - } - - /// Executes a fallible generic unary [`Op`]. - #[inline(always)] - fn try_execute_unary( - &mut self, - result: Slot, - input: Slot, - op: fn(P) -> Result, - ) -> Result<(), Error> - where - UntypedVal: ReadAs

+ WriteAs, - { - let value = self.get_stack_slot_as::

(input); - self.set_stack_slot_as::(result, op(value)?); - self.try_next_instr() - } - - /// Executes a generic binary [`Op`]. - #[inline(always)] - fn execute_binary( - &mut self, - result: Slot, - lhs: Slot, - rhs: Slot, - op: fn(Lhs, Rhs) -> Result, - ) where - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a generic binary [`Op`]. - #[inline(always)] - fn execute_binary_imm16_rhs( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - op: fn(Lhs, Rhs) -> T, - ) where - Rhs: From>, - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = Rhs::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a generic binary [`Op`] with reversed operands. - #[inline(always)] - fn execute_binary_imm16_lhs( - &mut self, - result: Slot, - lhs: Const16, - rhs: Slot, - op: fn(Lhs, Rhs) -> T, - ) where - Lhs: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = Lhs::from(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a generic shift or rotate [`Op`]. - #[inline(always)] - fn execute_shift_by( - &mut self, - result: Slot, - lhs: Slot, - rhs: ShiftAmount, - op: fn(Lhs, Rhs) -> T, - ) where - Rhs: From>, - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = Rhs::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } - - /// Executes a fallible generic binary [`Op`]. - #[inline(always)] - fn try_execute_binary( - &mut self, - result: Slot, - lhs: Slot, - rhs: Slot, - op: fn(Lhs, Rhs) -> Result, - ) -> Result<(), Error> - where - UntypedVal: ReadAs + ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)?); - self.try_next_instr() - } - - /// Executes a fallible generic binary [`Op`]. - #[inline(always)] - fn try_execute_divrem_imm16_rhs( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - op: fn(Lhs, Rhs) -> Result, - ) -> Result<(), Error> - where - Rhs: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = Rhs::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)?); - self.try_next_instr() - } - - /// Executes a fallible generic binary [`Op`]. - #[inline(always)] - fn execute_divrem_imm16_rhs( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - op: fn(Lhs, NonZeroT) -> T, - ) where - NonZeroT: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = ::from(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr() - } - - /// Executes a fallible generic binary [`Op`] with reversed operands. - #[inline(always)] - fn try_execute_binary_imm16_lhs( - &mut self, - result: Slot, - lhs: Const16, - rhs: Slot, - op: fn(Lhs, Rhs) -> Result, - ) -> Result<(), Error> - where - Lhs: From>, - UntypedVal: ReadAs + WriteAs, - { - let lhs = Lhs::from(lhs); - let rhs = self.get_stack_slot_as::(rhs); - self.set_stack_slot_as::(result, op(lhs, rhs)?); - self.try_next_instr() - } - - /// Returns the optional `memory` parameter for a `load_at` [`Op`]. - /// - /// # Note - /// - /// - Returns the default [`index::Memory`] if the parameter is missing. - /// - Bumps `self.ip` if a [`Op::MemoryIndex`] parameter was found. - #[inline(always)] - fn fetch_optional_memory(&mut self, delta: usize) -> index::Memory { - let mut addr: InstructionPtr = self.ip; - addr.add(delta); - match *addr.get() { - Op::MemoryIndex { index } => { - hint::cold(); - self.ip.add(1); - index - } - _ => index::Memory::from(0), - } - } - - /// Fetches the [`Slot`] and [`Offset64Hi`] parameters for a load or store [`Op`]. - unsafe fn fetch_reg_and_offset_hi(&self) -> (Slot, Offset64Hi) { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match addr.get().filter_register_and_offset_hi() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::SlotAndImm32` but found: {instr:?}") - }, - } - } -} - -impl Executor<'_> { - /// Used for all [`Op`] words that are not meant for execution. - /// - /// # Note - /// - /// This includes [`Op`] variants such as [`Op::TableIndex`] - /// that primarily carry parameters for actually executable [`Op`]. - fn invalid_instruction_word(&mut self) -> Result<(), Error> { - // Safety: Wasmi translation guarantees that branches are never taken to instruction parameters directly. - unsafe { - unreachable_unchecked!( - "expected instruction but found instruction parameter: {:?}", - *self.ip.get() - ) - } - } - - /// Executes a Wasm `unreachable` instruction. - fn execute_trap(&mut self, trap_code: TrapCode) -> Result<(), Error> { - Err(Error::from(trap_code)) - } - - /// Executes an [`Op::ConsumeFuel`]. - fn execute_consume_fuel( - &mut self, - store: &mut StoreInner, - block_fuel: BlockFuel, - ) -> Result<(), Error> { - // We do not have to check if fuel metering is enabled since - // [`Op::ConsumeFuel`] are only generated if fuel metering - // is enabled to begin with. - store - .fuel_mut() - .consume_fuel_unchecked(block_fuel.to_u64())?; - self.try_next_instr() - } - - /// Executes an [`Op::RefFunc`]. - fn execute_ref_func(&mut self, result: Slot, func_index: index::Func) { - let func = self.get_func(func_index); - let funcref = >::from(func); - self.set_stack_slot(result, funcref); - self.next_instr(); - } } /// Extension method for [`UntypedVal`] required by the [`Executor`]. diff --git a/crates/wasmi/src/engine/executor/instrs/binary.rs b/crates/wasmi/src/engine/executor/instrs/binary.rs deleted file mode 100644 index 958f5b0a93..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/binary.rs +++ /dev/null @@ -1,323 +0,0 @@ -use super::{Executor, UntypedValueExt}; -use crate::{ - core::wasm, - ir::{Sign, Slot}, - Error, - TrapCode, -}; -use core::num::{NonZeroI32, NonZeroI64, NonZeroU32, NonZeroU64}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - impl_binary_executors! { - (Op::I32Add, execute_i32_add, wasm::i32_add), - (Op::I32Sub, execute_i32_sub, wasm::i32_sub), - (Op::I32Mul, execute_i32_mul, wasm::i32_mul), - (Op::I32BitAnd, execute_i32_bitand, wasm::i32_bitand), - (Op::I32BitOr, execute_i32_bitor, wasm::i32_bitor), - (Op::I32BitXor, execute_i32_bitxor, wasm::i32_bitxor), - (Op::I32And, execute_i32_and, ::and), - (Op::I32Or, execute_i32_or, ::or), - (Op::I32Nand, execute_i32_nand, ::nand), - (Op::I32Nor, execute_i32_nor, ::nor), - - (Op::I64Add, execute_i64_add, wasm::i64_add), - (Op::I64Sub, execute_i64_sub, wasm::i64_sub), - (Op::I64Mul, execute_i64_mul, wasm::i64_mul), - (Op::I64BitAnd, execute_i64_bitand, wasm::i64_bitand), - (Op::I64BitOr, execute_i64_bitor, wasm::i64_bitor), - (Op::I64BitXor, execute_i64_bitxor, wasm::i64_bitxor), - (Op::I64And, execute_i64_and, ::and), - (Op::I64Or, execute_i64_or, ::or), - (Op::I64Nand, execute_i64_nand, ::nand), - (Op::I64Nor, execute_i64_nor, ::nor), - - (Op::I32Shl, execute_i32_shl, wasm::i32_shl), - (Op::I32ShrU, execute_i32_shr_u, wasm::i32_shr_u), - (Op::I32ShrS, execute_i32_shr_s, wasm::i32_shr_s), - (Op::I32Rotl, execute_i32_rotl, wasm::i32_rotl), - (Op::I32Rotr, execute_i32_rotr, wasm::i32_rotr), - - (Op::I64Shl, execute_i64_shl, wasm::i64_shl), - (Op::I64ShrU, execute_i64_shr_u, wasm::i64_shr_u), - (Op::I64ShrS, execute_i64_shr_s, wasm::i64_shr_s), - (Op::I64Rotl, execute_i64_rotl, wasm::i64_rotl), - (Op::I64Rotr, execute_i64_rotr, wasm::i64_rotr), - - (Op::F32Add, execute_f32_add, wasm::f32_add), - (Op::F32Sub, execute_f32_sub, wasm::f32_sub), - (Op::F32Mul, execute_f32_mul, wasm::f32_mul), - (Op::F32Div, execute_f32_div, wasm::f32_div), - (Op::F32Min, execute_f32_min, wasm::f32_min), - (Op::F32Max, execute_f32_max, wasm::f32_max), - (Op::F32Copysign, execute_f32_copysign, wasm::f32_copysign), - - (Op::F64Add, execute_f64_add, wasm::f64_add), - (Op::F64Sub, execute_f64_sub, wasm::f64_sub), - (Op::F64Mul, execute_f64_mul, wasm::f64_mul), - (Op::F64Div, execute_f64_div, wasm::f64_div), - (Op::F64Min, execute_f64_min, wasm::f64_min), - (Op::F64Max, execute_f64_max, wasm::f64_max), - (Op::F64Copysign, execute_f64_copysign, wasm::f64_copysign), - } -} - -macro_rules! impl_binary_imm16 { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: $ty) { - self.execute_binary_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_binary_imm16! { - (i32, Op::I32AddImm16, execute_i32_add_imm16, wasm::i32_add), - (i32, Op::I32MulImm16, execute_i32_mul_imm16, wasm::i32_mul), - (i32, Op::I32BitAndImm16, execute_i32_bitand_imm16, wasm::i32_bitand), - (i32, Op::I32BitOrImm16, execute_i32_bitor_imm16, wasm::i32_bitor), - (i32, Op::I32BitXorImm16, execute_i32_bitxor_imm16, wasm::i32_bitxor), - (i32, Op::I32AndImm16, execute_i32_and_imm16, ::and), - (i32, Op::I32OrImm16, execute_i32_or_imm16, ::or), - (i32, Op::I32NandImm16, execute_i32_nand_imm16, ::nand), - (i32, Op::I32NorImm16, execute_i32_nor_imm16, ::nor), - - (i64, Op::I64AddImm16, execute_i64_add_imm16, wasm::i64_add), - (i64, Op::I64MulImm16, execute_i64_mul_imm16, wasm::i64_mul), - (i64, Op::I64BitAndImm16, execute_i64_bitand_imm16, wasm::i64_bitand), - (i64, Op::I64BitOrImm16, execute_i64_bitor_imm16, wasm::i64_bitor), - (i64, Op::I64BitXorImm16, execute_i64_bitxor_imm16, wasm::i64_bitxor), - (i64, Op::I64AndImm16, execute_i64_and_imm16, ::and), - (i64, Op::I64OrImm16, execute_i64_or_imm16, ::or), - (i64, Op::I64NandImm16, execute_i64_nand_imm16, ::nand), - (i64, Op::I64NorImm16, execute_i64_nor_imm16, ::nor), - } -} - -macro_rules! impl_shift_by { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: u8) { - self.execute_shift_by(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_shift_by! { - (i32, Op::I32ShlBy, execute_i32_shl_by, wasm::i32_shl), - (i32, Op::I32ShrUBy, execute_i32_shr_u_by, wasm::i32_shr_u), - (i32, Op::I32ShrSBy, execute_i32_shr_s_by, wasm::i32_shr_s), - (i32, Op::I32RotlBy, execute_i32_rotl_by, wasm::i32_rotl), - (i32, Op::I32RotrBy, execute_i32_rotr_by, wasm::i32_rotr), - - (i64, Op::I64ShlBy, execute_i64_shl_by, wasm::i64_shl), - (i64, Op::I64ShrUBy, execute_i64_shr_u_by, wasm::i64_shr_u), - (i64, Op::I64ShrSBy, execute_i64_shr_s_by, wasm::i64_shr_s), - (i64, Op::I64RotlBy, execute_i64_rotl_by, wasm::i64_rotl), - (i64, Op::I64RotrBy, execute_i64_rotr_by, wasm::i64_rotr), - } -} - -macro_rules! impl_binary_imm16_lhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: $ty, rhs: Slot) { - self.execute_binary_imm16_lhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_binary_imm16_lhs! { - (i32, Op::I32SubImm16Lhs, execute_i32_sub_imm16_lhs, wasm::i32_sub), - (i64, Op::I64SubImm16Lhs, execute_i64_sub_imm16_lhs, wasm::i64_sub), - - (i32, Op::I32ShlImm16, execute_i32_shl_imm16, wasm::i32_shl), - (i32, Op::I32ShrUImm16, execute_i32_shr_u_imm16, wasm::i32_shr_u), - (i32, Op::I32ShrSImm16, execute_i32_shr_s_imm16, wasm::i32_shr_s), - (i32, Op::I32RotlImm16, execute_i32_rotl_imm16, wasm::i32_rotl), - (i32, Op::I32RotrImm16, execute_i32_rotr_imm16, wasm::i32_rotr), - - (i64, Op::I64ShlImm16, execute_i64_shl_imm16, wasm::i64_shl), - (i64, Op::I64ShrUImm16, execute_i64_shr_u_imm16, wasm::i64_shr_u), - (i64, Op::I64ShrSImm16, execute_i64_shr_s_imm16, wasm::i64_shr_s), - (i64, Op::I64RotlImm16, execute_i64_rotl_imm16, wasm::i64_rotl), - (i64, Op::I64RotrImm16, execute_i64_rotr_imm16, wasm::i64_rotr), - } -} - -macro_rules! impl_fallible_binary { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Slot) -> Result<(), Error> { - self.try_execute_binary(result, lhs, rhs, $op).map_err(Error::from) - } - )* - }; -} -impl Executor<'_> { - impl_fallible_binary! { - (Op::I32DivS, execute_i32_div_s, wasm::i32_div_s), - (Op::I32DivU, execute_i32_div_u, wasm::i32_div_u), - (Op::I32RemS, execute_i32_rem_s, wasm::i32_rem_s), - (Op::I32RemU, execute_i32_rem_u, wasm::i32_rem_u), - - (Op::I64DivS, execute_i64_div_s, wasm::i64_div_s), - (Op::I64DivU, execute_i64_div_u, wasm::i64_div_u), - (Op::I64RemS, execute_i64_rem_s, wasm::i64_rem_s), - (Op::I64RemU, execute_i64_rem_u, wasm::i64_rem_u), - } -} - -/// Extension trait to provide more optimized divide and remainder implementations. -pub trait DivRemExt: Sized { - /// Signed non-zero value type. - type NonZeroS; - /// Unsigned non-zero value type. - type NonZeroU; - - /// Optimized variant of Wasm `i{32,64}.div_s` for immutable non-zero `rhs` values. - fn div_s(self, rhs: Self::NonZeroS) -> Result; - /// Optimized variant of Wasm `i{32,64}.div_u` for immutable non-zero `rhs` values. - fn div_u(self, rhs: Self::NonZeroU) -> Self; - /// Optimized variant of Wasm `i{32,64}.rem_s` for immutable non-zero `rhs` values. - fn rem_s(self, rhs: Self::NonZeroS) -> Result; - /// Optimized variant of Wasm `i{32,64}.rem_u` for immutable non-zero `rhs` values. - fn rem_u(self, rhs: Self::NonZeroU) -> Self; -} - -impl DivRemExt for i32 { - type NonZeroS = NonZeroI32; - type NonZeroU = NonZeroU32; - - fn div_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_div(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn div_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u32) / rhs) as Self - } - - fn rem_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_rem(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn rem_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u32) % rhs) as Self - } -} - -impl DivRemExt for i64 { - type NonZeroS = NonZeroI64; - type NonZeroU = NonZeroU64; - - fn div_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_div(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn div_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u64) / rhs) as Self - } - - fn rem_s(self, rhs: Self::NonZeroS) -> Result { - self.checked_rem(rhs.get()) - .ok_or_else(|| Error::from(TrapCode::IntegerOverflow)) - } - - fn rem_u(self, rhs: Self::NonZeroU) -> Self { - ((self as u64) % rhs) as Self - } -} - -macro_rules! impl_divrem_s_imm16_rhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: $ty) -> Result<(), Error> { - self.try_execute_divrem_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_divrem_s_imm16_rhs! { - (NonZeroI32, Op::I32DivSImm16Rhs, execute_i32_div_s_imm16_rhs, ::div_s), - (NonZeroI32, Op::I32RemSImm16Rhs, execute_i32_rem_s_imm16_rhs, ::rem_s), - - (NonZeroI64, Op::I64DivSImm16Rhs, execute_i64_div_s_imm16_rhs, ::div_s), - (NonZeroI64, Op::I64RemSImm16Rhs, execute_i64_rem_s_imm16_rhs, ::rem_s), - } -} - -macro_rules! impl_divrem_u_imm16_rhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: $ty) { - self.execute_divrem_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_divrem_u_imm16_rhs! { - (NonZeroU32, Op::I32DivUImm16Rhs, execute_i32_div_u_imm16_rhs, ::div_u), - (NonZeroU32, Op::I32RemUImm16Rhs, execute_i32_rem_u_imm16_rhs, ::rem_u), - - (NonZeroU64, Op::I64DivUImm16Rhs, execute_i64_div_u_imm16_rhs, ::div_u), - (NonZeroU64, Op::I64RemUImm16Rhs, execute_i64_rem_u_imm16_rhs, ::rem_u), - } -} - -macro_rules! impl_fallible_binary_imm16_lhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: $ty, rhs: Slot) -> Result<(), Error> { - self.try_execute_binary_imm16_lhs(result, lhs, rhs, $op).map_err(Error::from) - } - )* - }; -} -impl Executor<'_> { - impl_fallible_binary_imm16_lhs! { - (i32, Op::I32DivSImm16Lhs, execute_i32_div_s_imm16_lhs, wasm::i32_div_s), - (u32, Op::I32DivUImm16Lhs, execute_i32_div_u_imm16_lhs, wasm::i32_div_u), - (i32, Op::I32RemSImm16Lhs, execute_i32_rem_s_imm16_lhs, wasm::i32_rem_s), - (u32, Op::I32RemUImm16Lhs, execute_i32_rem_u_imm16_lhs, wasm::i32_rem_u), - - (i64, Op::I64DivSImm16Lhs, execute_i64_div_s_imm16_lhs, wasm::i64_div_s), - (u64, Op::I64DivUImm16Lhs, execute_i64_div_u_imm16_lhs, wasm::i64_div_u), - (i64, Op::I64RemSImm16Lhs, execute_i64_rem_s_imm16_lhs, wasm::i64_rem_s), - (u64, Op::I64RemUImm16Lhs, execute_i64_rem_u_imm16_lhs, wasm::i64_rem_u), - } -} - -impl Executor<'_> { - /// Executes an [`Op::F32CopysignImm`]. - pub fn execute_f32_copysign_imm(&mut self, result: Slot, lhs: Slot, rhs: Sign) { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = f32::from(rhs); - self.set_stack_slot_as::(result, wasm::f32_copysign(lhs, rhs)); - self.next_instr() - } - - /// Executes an [`Op::F64CopysignImm`]. - pub fn execute_f64_copysign_imm(&mut self, result: Slot, lhs: Slot, rhs: Sign) { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = f64::from(rhs); - self.set_stack_slot_as::(result, wasm::f64_copysign(lhs, rhs)); - self.next_instr() - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/branch.rs b/crates/wasmi/src/engine/executor/instrs/branch.rs deleted file mode 100644 index 84e0adcfeb..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/branch.rs +++ /dev/null @@ -1,328 +0,0 @@ -use super::{Executor, UntypedValueCmpExt, UntypedValueExt}; -use crate::{ - core::{ReadAs, UntypedVal}, - engine::utils::unreachable_unchecked, - ir::{BranchOffset, Op, Slot}, -}; -use core::cmp; - -impl Executor<'_> { - /// Branches and adjusts the value stack. - /// - /// # Note - /// - /// Offsets the instruction pointer using the given [`BranchOffset`]. - fn branch_to(&mut self, offset: BranchOffset) { - self.ip.offset(offset.to_i32() as isize) - } - - /// Branches and adjusts the value stack. - /// - /// # Note - /// - /// Offsets the instruction pointer using the given [`BranchOffset`]. - fn branch_to16(&mut self, offset: BranchOffset16) { - self.ip.offset(offset.to_i16() as isize) - } - - pub fn execute_branch(&mut self, offset: BranchOffset) { - self.branch_to(offset) - } - - /// Fetches the branch table index value and normalizes it to clamp between `0..len_targets`. - fn fetch_branch_table_offset(&self, index: Slot, len_targets: u32) -> usize { - let index: u32 = self.get_stack_slot_as::(index); - // The index of the default target which is the last target of the slice. - let max_index = len_targets - 1; - // A normalized index will always yield a target without panicking. - cmp::min(index, max_index) as usize + 1 - } - - pub fn execute_branch_table_0(&mut self, index: Slot, len_targets: u32) { - let offset = self.fetch_branch_table_offset(index, len_targets); - self.ip.add(offset); - } - - pub fn execute_branch_table_span(&mut self, index: Slot, len_targets: u32) { - let offset = self.fetch_branch_table_offset(index, len_targets); - self.ip.add(1); - let values = match *self.ip.get() { - Op::SlotSpan { span } => span, - unexpected => { - // Safety: Wasmi translation guarantees that `Op::SlotSpan` follows. - unsafe { - unreachable_unchecked!("expected `Op::SlotSpan` but found: {unexpected:?}") - } - } - }; - let len = values.len(); - let values = values.span(); - self.ip.add(offset); - match *self.ip.get() { - // Note: we explicitly do _not_ handle branch table returns here for technical reasons. - // They are executed as the next conventional instruction in the pipeline, no special treatment required. - Op::BranchTableTarget { results, offset } => { - self.execute_copy_span_impl(results, values, len); - self.execute_branch(offset) - } - unexpected => { - // Safety: Wasmi translator guarantees that one of the above `Op` variants exists. - unsafe { - unreachable_unchecked!( - "expected target for `Op::BranchTableSpan` but found: {unexpected:?}" - ) - } - } - } - } - - /// Executes a generic fused compare and branch instruction with raw inputs. - #[inline(always)] - fn execute_branch_binop( - &mut self, - lhs: Slot, - rhs: Slot, - offset: impl Into, - f: fn(T, T) -> bool, - ) where - UntypedVal: ReadAs, - { - let lhs: T = self.get_stack_slot_as(lhs); - let rhs: T = self.get_stack_slot_as(rhs); - if f(lhs, rhs) { - return self.branch_to(offset.into()); - } - self.next_instr() - } - - /// Executes a generic fused compare and branch instruction with immediate `rhs` operand. - fn execute_branch_binop_imm16_rhs( - &mut self, - lhs: Slot, - rhs: Const16, - offset: BranchOffset16, - f: fn(T, T) -> bool, - ) where - T: From>, - UntypedVal: ReadAs, - { - let lhs: T = self.get_stack_slot_as(lhs); - let rhs = T::from(rhs); - if f(lhs, rhs) { - return self.branch_to16(offset); - } - self.next_instr() - } - - /// Executes a generic fused compare and branch instruction with immediate `rhs` operand. - fn execute_branch_binop_imm16_lhs( - &mut self, - lhs: Const16, - rhs: Slot, - offset: BranchOffset16, - f: fn(T, T) -> bool, - ) where - T: From>, - UntypedVal: ReadAs, - { - let lhs = T::from(lhs); - let rhs: T = self.get_stack_slot_as(rhs); - if f(lhs, rhs) { - return self.branch_to16(offset); - } - self.next_instr() - } -} - -fn cmp_eq(a: T, b: T) -> bool -where - T: PartialEq, -{ - a == b -} - -fn cmp_ne(a: T, b: T) -> bool -where - T: PartialEq, -{ - a != b -} - -fn cmp_lt(a: T, b: T) -> bool -where - T: PartialOrd, -{ - a < b -} - -fn cmp_le(a: T, b: T) -> bool -where - T: PartialOrd, -{ - a <= b -} - -macro_rules! impl_execute_branch_binop { - ( $( ($ty:ty, Op::$op_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - impl<'engine> Executor<'engine> { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op_name), "`].")] - #[inline(always)] - pub fn $fn_name(&mut self, lhs: Slot, rhs: Slot, offset: BranchOffset16) { - self.execute_branch_binop::<$ty>(lhs, rhs, offset, $op) - } - )* - } - } -} -impl_execute_branch_binop! { - (i32, Op::BranchI32And, execute_branch_i32_and, UntypedValueExt::and), - (i32, Op::BranchI32Or, execute_branch_i32_or, UntypedValueExt::or), - (i32, Op::BranchI32Nand, execute_branch_i32_nand, UntypedValueExt::nand), - (i32, Op::BranchI32Nor, execute_branch_i32_nor, UntypedValueExt::nor), - (i32, Op::BranchI32Eq, execute_branch_i32_eq, cmp_eq), - (i32, Op::BranchI32Ne, execute_branch_i32_ne, cmp_ne), - (i32, Op::BranchI32LtS, execute_branch_i32_lt_s, cmp_lt), - (u32, Op::BranchI32LtU, execute_branch_i32_lt_u, cmp_lt), - (i32, Op::BranchI32LeS, execute_branch_i32_le_s, cmp_le), - (u32, Op::BranchI32LeU, execute_branch_i32_le_u, cmp_le), - - (i64, Op::BranchI64And, execute_branch_i64_and, UntypedValueExt::and), - (i64, Op::BranchI64Or, execute_branch_i64_or, UntypedValueExt::or), - (i64, Op::BranchI64Nand, execute_branch_i64_nand, UntypedValueExt::nand), - (i64, Op::BranchI64Nor, execute_branch_i64_nor, UntypedValueExt::nor), - (i64, Op::BranchI64Eq, execute_branch_i64_eq, cmp_eq), - (i64, Op::BranchI64Ne, execute_branch_i64_ne, cmp_ne), - (i64, Op::BranchI64LtS, execute_branch_i64_lt_s, cmp_lt), - (u64, Op::BranchI64LtU, execute_branch_i64_lt_u, cmp_lt), - (i64, Op::BranchI64LeS, execute_branch_i64_le_s, cmp_le), - (u64, Op::BranchI64LeU, execute_branch_i64_le_u, cmp_le), - - (f32, Op::BranchF32Eq, execute_branch_f32_eq, cmp_eq), - (f32, Op::BranchF32Ne, execute_branch_f32_ne, cmp_ne), - (f32, Op::BranchF32Lt, execute_branch_f32_lt, cmp_lt), - (f32, Op::BranchF32Le, execute_branch_f32_le, cmp_le), - (f32, Op::BranchF32NotLt, execute_branch_f32_not_lt, UntypedValueCmpExt::not_lt), - (f32, Op::BranchF32NotLe, execute_branch_f32_not_le, UntypedValueCmpExt::not_le), - - (f64, Op::BranchF64Eq, execute_branch_f64_eq, cmp_eq), - (f64, Op::BranchF64Ne, execute_branch_f64_ne, cmp_ne), - (f64, Op::BranchF64Lt, execute_branch_f64_lt, cmp_lt), - (f64, Op::BranchF64Le, execute_branch_f64_le, cmp_le), - (f64, Op::BranchF64NotLt, execute_branch_f64_not_lt, UntypedValueCmpExt::not_lt), - (f64, Op::BranchF64NotLe, execute_branch_f64_not_le, UntypedValueCmpExt::not_le), -} - -macro_rules! impl_execute_branch_binop_imm16_rhs { - ( $( ($ty:ty, Op::$op_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - impl<'engine> Executor<'engine> { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op_name), "`].")] - pub fn $fn_name(&mut self, lhs: Slot, rhs: Const16<$ty>, offset: BranchOffset16) { - self.execute_branch_binop_imm16_rhs::<$ty>(lhs, rhs, offset, $op) - } - )* - } - } -} -impl_execute_branch_binop_imm16_rhs! { - (i32, Op::BranchI32AndImm16, execute_branch_i32_and_imm16, UntypedValueExt::and), - (i32, Op::BranchI32OrImm16, execute_branch_i32_or_imm16, UntypedValueExt::or), - (i32, Op::BranchI32NandImm16, execute_branch_i32_nand_imm16, UntypedValueExt::nand), - (i32, Op::BranchI32NorImm16, execute_branch_i32_nor_imm16, UntypedValueExt::nor), - (i32, Op::BranchI32EqImm16, execute_branch_i32_eq_imm16, cmp_eq), - (i32, Op::BranchI32NeImm16, execute_branch_i32_ne_imm16, cmp_ne), - (i32, Op::BranchI32LtSImm16Rhs, execute_branch_i32_lt_s_imm16_rhs, cmp_lt), - (u32, Op::BranchI32LtUImm16Rhs, execute_branch_i32_lt_u_imm16_rhs, cmp_lt), - (i32, Op::BranchI32LeSImm16Rhs, execute_branch_i32_le_s_imm16_rhs, cmp_le), - (u32, Op::BranchI32LeUImm16Rhs, execute_branch_i32_le_u_imm16_rhs, cmp_le), - - (i64, Op::BranchI64AndImm16, execute_branch_i64_and_imm16, UntypedValueExt::and), - (i64, Op::BranchI64OrImm16, execute_branch_i64_or_imm16, UntypedValueExt::or), - (i64, Op::BranchI64NandImm16, execute_branch_i64_nand_imm16, UntypedValueExt::nand), - (i64, Op::BranchI64NorImm16, execute_branch_i64_nor_imm16, UntypedValueExt::nor), - (i64, Op::BranchI64EqImm16, execute_branch_i64_eq_imm16, cmp_eq), - (i64, Op::BranchI64NeImm16, execute_branch_i64_ne_imm16, cmp_ne), - (i64, Op::BranchI64LtSImm16Rhs, execute_branch_i64_lt_s_imm16_rhs, cmp_lt), - (u64, Op::BranchI64LtUImm16Rhs, execute_branch_i64_lt_u_imm16_rhs, cmp_lt), - (i64, Op::BranchI64LeSImm16Rhs, execute_branch_i64_le_s_imm16_rhs, cmp_le), - (u64, Op::BranchI64LeUImm16Rhs, execute_branch_i64_le_u_imm16_rhs, cmp_le), -} - -macro_rules! impl_execute_branch_binop_imm16_lhs { - ( $( ($ty:ty, Op::$op_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - impl<'engine> Executor<'engine> { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op_name), "`].")] - pub fn $fn_name(&mut self, lhs: Const16<$ty>, rhs: Slot, offset: BranchOffset16) { - self.execute_branch_binop_imm16_lhs::<$ty>(lhs, rhs, offset, $op) - } - )* - } - } -} -impl_execute_branch_binop_imm16_lhs! { - (i32, Op::BranchI32LtSImm16Lhs, execute_branch_i32_lt_s_imm16_lhs, cmp_lt), - (u32, Op::BranchI32LtUImm16Lhs, execute_branch_i32_lt_u_imm16_lhs, cmp_lt), - (i32, Op::BranchI32LeSImm16Lhs, execute_branch_i32_le_s_imm16_lhs, cmp_le), - (u32, Op::BranchI32LeUImm16Lhs, execute_branch_i32_le_u_imm16_lhs, cmp_le), - - (i64, Op::BranchI64LtSImm16Lhs, execute_branch_i64_lt_s_imm16_lhs, cmp_lt), - (u64, Op::BranchI64LtUImm16Lhs, execute_branch_i64_lt_u_imm16_lhs, cmp_lt), - (i64, Op::BranchI64LeSImm16Lhs, execute_branch_i64_le_s_imm16_lhs, cmp_le), - (u64, Op::BranchI64LeUImm16Lhs, execute_branch_i64_le_u_imm16_lhs, cmp_le), -} - -impl Executor<'_> { - /// Executes an [`Op::BranchCmpFallback`]. - pub fn execute_branch_cmp_fallback(&mut self, lhs: Slot, rhs: Slot, params: Slot) { - use Comparator as C; - let params: u64 = self.get_stack_slot_as(params); - let Some(params) = ComparatorAndOffset::from_u64(params) else { - panic!("encountered invalidaly encoded ComparatorOffsetParam: {params:?}") - }; - let offset = params.offset; - match params.cmp { - C::I32Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::I32Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::I32LtS => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I32LtU => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I32LeS => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I32LeU => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I32And => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::and), - C::I32Or => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::or), - C::I32Nand => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nand), - C::I32Nor => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nor), - C::I64Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::I64Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::I64LtS => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I64LtU => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::I64LeS => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I64LeU => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::I64And => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::and), - C::I64Or => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::or), - C::I64Nand => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nand), - C::I64Nor => self.execute_branch_binop::(lhs, rhs, offset, UntypedValueExt::nor), - C::F32Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::F32Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::F32Lt => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::F32Le => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::F32NotLt => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_lt) - } - C::F32NotLe => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_le) - } - C::F64Eq => self.execute_branch_binop::(lhs, rhs, offset, cmp_eq), - C::F64Ne => self.execute_branch_binop::(lhs, rhs, offset, cmp_ne), - C::F64Lt => self.execute_branch_binop::(lhs, rhs, offset, cmp_lt), - C::F64Le => self.execute_branch_binop::(lhs, rhs, offset, cmp_le), - C::F64NotLt => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_lt) - } - C::F64NotLe => { - self.execute_branch_binop::(lhs, rhs, offset, UntypedValueCmpExt::not_le) - } - }; - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/call.rs b/crates/wasmi/src/engine/executor/instrs/call.rs deleted file mode 100644 index 4fae2d8372..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/call.rs +++ /dev/null @@ -1,713 +0,0 @@ -use super::{ControlFlow, Executor, InstructionPtr}; -use crate::{ - engine::{ - code_map::CompiledFuncRef, - executor::stack::{CallFrame, FrameParams, ValueStack}, - utils::unreachable_unchecked, - EngineFunc, - FuncInOut, - ResumableHostTrapError, - }, - func::{FuncEntity, HostFuncEntity}, - ir::{index, Op, Slot, SlotSpan}, - store::{CallHooks, PrunedStore, StoreError, StoreInner}, - Error, - Func, - Instance, - Ref, - TrapCode, -}; -use core::array; - -/// Dispatches and executes the host function. -/// -/// Returns the number of parameters and results of the called host function. -/// -/// # Errors -/// -/// Returns the error of the host function if an error occurred. -pub fn dispatch_host_func( - store: &mut PrunedStore, - value_stack: &mut ValueStack, - host_func: HostFuncEntity, - instance: Option<&Instance>, - call_hooks: CallHooks, -) -> Result<(u16, u16), Error> { - let len_params = host_func.len_params(); - let len_results = host_func.len_results(); - let max_inout = len_params.max(len_results); - let values = value_stack.as_slice_mut(); - let params_results = FuncInOut::new( - values.split_at_mut(values.len() - usize::from(max_inout)).1, - usize::from(len_params), - usize::from(len_results), - ); - match store.call_host_func(&host_func, instance, params_results, call_hooks) { - Err(StoreError::Internal(error)) => { - panic!("`call.host`: internal interpreter error: {error}") - } - Err(StoreError::External(error)) => { - // Note: We drop the values that have been temporarily added to - // the stack to act as parameter and result buffer for the - // called host function. Since the host function failed we - // need to clean up the temporary buffer values here. - // This is required for resumable calls to work properly. - value_stack.drop(usize::from(max_inout)); - return Err(error); - } - _ => {} - } - Ok((len_params, len_results)) -} - -/// The kind of a function call. -#[derive(Debug, Copy, Clone)] -pub enum CallKind { - /// A nested function call. - Nested, - /// A tailing function call. - Tail, -} - -trait CallContext { - const KIND: CallKind; - const HAS_PARAMS: bool; -} -trait ReturnCallContext: CallContext {} - -mod marker { - use super::{CallContext, CallKind, ReturnCallContext}; - - pub enum ReturnCall0 {} - impl CallContext for ReturnCall0 { - const KIND: CallKind = CallKind::Tail; - const HAS_PARAMS: bool = false; - } - impl ReturnCallContext for ReturnCall0 {} - - pub enum ReturnCall {} - impl CallContext for ReturnCall { - const KIND: CallKind = CallKind::Tail; - const HAS_PARAMS: bool = true; - } - impl ReturnCallContext for ReturnCall {} - - pub enum NestedCall0 {} - impl CallContext for NestedCall0 { - const KIND: CallKind = CallKind::Nested; - const HAS_PARAMS: bool = false; - } - - pub enum NestedCall {} - impl CallContext for NestedCall { - const KIND: CallKind = CallKind::Nested; - const HAS_PARAMS: bool = true; - } -} - -impl Executor<'_> { - /// Updates the [`InstructionPtr`] of the caller [`CallFrame`] before dispatching a call. - /// - /// # Note - /// - /// The `offset` denotes how many [`Op`] words make up the call instruction. - #[inline(always)] - fn update_instr_ptr_at(&mut self, offset: usize) { - // Note: we explicitly do not mutate `self.ip` since that would make - // other parts of the code more fragile with respect to instruction ordering. - self.ip.add(offset); - let caller = self - .stack - .calls - .peek_mut() - .expect("caller call frame must be on the stack"); - caller.update_instr_ptr(self.ip); - } - - /// Fetches the [`Op::CallIndirectParams`] parameter for a call [`Op`]. - /// - /// # Note - /// - /// - This advances the [`InstructionPtr`] to the next [`Op`]. - /// - This is done by encoding an [`Op::TableGet`] instruction - /// word following the actual instruction where the [`index::Table`] - /// parameter belongs to. - /// - This is required for some instructions that do not fit into - /// a single instruction word and store a [`index::Table`] value in - /// another instruction word. - fn pull_call_indirect_params(&mut self) -> (u64, index::Table) { - self.ip.add(1); - match *self.ip.get() { - Op::CallIndirectParams { index, table } => { - let index: u64 = self.get_stack_slot_as(index); - (index, table) - } - unexpected => { - // Safety: Wasmi translation guarantees that correct instruction parameter follows. - unsafe { - unreachable_unchecked!( - "expected `Op::CallIndirectParams` but found {unexpected:?}" - ) - } - } - } - } - - /// Fetches the [`Op::CallIndirectParamsImm16`] parameter for a call [`Op`]. - /// - /// # Note - /// - /// - This advances the [`InstructionPtr`] to the next [`Op`]. - /// - This is done by encoding an [`Op::TableGet`] instruction - /// word following the actual instruction where the [`index::Table`] - /// parameter belongs to. - /// - This is required for some instructions that do not fit into - /// a single instruction word and store a [`index::Table`] value in - /// another instruction word. - fn pull_call_indirect_params_imm16(&mut self) -> (u64, index::Table) { - self.ip.add(1); - match *self.ip.get() { - Op::CallIndirectParamsImm16 { index, table } => { - let index: u64 = index.into(); - (index, table) - } - unexpected => { - // Safety: Wasmi translation guarantees that correct instruction parameter follows. - unsafe { - unreachable_unchecked!( - "expected `Op::CallIndirectParamsImm16` but found {unexpected:?}" - ) - } - } - } - } - - /// Creates a [`CallFrame`] for calling the [`EngineFunc`]. - #[inline(always)] - fn dispatch_compiled_func( - &mut self, - results: SlotSpan, - func: CompiledFuncRef, - ) -> Result { - // We have to reinstantiate the `self.sp` [`FrameSlots`] since we just called - // [`ValueStack::alloc_call_frame`] which might invalidate all live [`FrameSlots`]. - let caller = self - .stack - .calls - .peek() - .expect("need to have a caller on the call stack"); - let (mut uninit_params, offsets) = self.stack.values.alloc_call_frame(func, |this| { - // Safety: We use the base offset of a live call frame on the call stack. - self.sp = unsafe { this.stack_ptr_at(caller.base_offset()) }; - })?; - let instr_ptr = InstructionPtr::new(func.instrs().as_ptr()); - let frame = CallFrame::new(instr_ptr, offsets, results); - if ::HAS_PARAMS { - self.copy_call_params(&mut uninit_params); - } - uninit_params.init_zeroes(); - Ok(frame) - } - - /// Copies the parameters from caller for the callee [`CallFrame`]. - /// - /// This will also adjust the instruction pointer to point to the - /// last call parameter [`Op`] if any. - fn copy_call_params(&mut self, uninit_params: &mut FrameParams) { - self.ip.add(1); - if let Op::SlotList { .. } = self.ip.get() { - self.copy_call_params_list(uninit_params); - } - match self.ip.get() { - Op::Slot { slot } => { - self.copy_regs(uninit_params, array::from_ref(slot)); - } - Op::Slot2 { slots } => { - self.copy_regs(uninit_params, slots); - } - Op::Slot3 { slots } => { - self.copy_regs(uninit_params, slots); - } - unexpected => { - // Safety: Wasmi translation guarantees that register list finalizer exists. - unsafe { - unreachable_unchecked!( - "expected register-list finalizer but found: {unexpected:?}" - ) - } - } - } - } - - /// Copies an array of [`Slot`] to the `dst` [`Slot`] span. - fn copy_regs(&self, uninit_params: &mut FrameParams, regs: &[Slot; N]) { - for value in regs { - let value = self.get_stack_slot(*value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { uninit_params.init_next(value) } - } - } - - /// Copies a list of [`Op::SlotList`] to the `dst` [`Slot`] span. - /// Copies the parameters from `src` for the called [`CallFrame`]. - /// - /// This will make the [`InstructionPtr`] point to the [`Op`] following the - /// last [`Op::SlotList`] if any. - #[cold] - fn copy_call_params_list(&mut self, uninit_params: &mut FrameParams) { - while let Op::SlotList { regs } = self.ip.get() { - self.copy_regs(uninit_params, regs); - self.ip.add(1); - } - } - - /// Prepares a [`EngineFunc`] call with optional call parameters. - #[inline(always)] - fn prepare_compiled_func_call( - &mut self, - store: &mut StoreInner, - results: SlotSpan, - func: EngineFunc, - mut instance: Option, - ) -> Result<(), Error> { - let func = self.code_map.get(Some(store.fuel_mut()), func)?; - let mut called = self.dispatch_compiled_func::(results, func)?; - match ::KIND { - CallKind::Nested => { - // We need to update the instruction pointer of the caller call frame. - self.update_instr_ptr_at(1); - } - CallKind::Tail => { - // In case of a tail call we have to remove the caller call frame after - // allocating the callee call frame. This moves all cells of the callee frame - // and may invalidate pointers to it. - // - // Safety: - // - // We provide `merge_call_frames` properly with `frame` that has just been allocated - // on the value stack which is what the function expects. After this operation we ensure - // that `self.sp` is adjusted via a call to `init_call_frame` since it may have been - // invalidated by this method. - let caller_instance = unsafe { self.stack.merge_call_frames(&mut called) }; - if let Some(caller_instance) = caller_instance { - instance.get_or_insert(caller_instance); - } - } - } - self.init_call_frame(&called); - self.stack.calls.push(called, instance)?; - Ok(()) - } - - /// Executes an [`Op::ReturnCallInternal0`]. - #[inline(always)] - pub fn execute_return_call_internal_0( - &mut self, - store: &mut StoreInner, - func: EngineFunc, - ) -> Result<(), Error> { - self.execute_return_call_internal_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallInternal`]. - #[inline(always)] - pub fn execute_return_call_internal( - &mut self, - store: &mut StoreInner, - func: EngineFunc, - ) -> Result<(), Error> { - self.execute_return_call_internal_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallInternal`] or [`Op::ReturnCallInternal0`]. - #[inline(always)] - fn execute_return_call_internal_impl( - &mut self, - store: &mut StoreInner, - func: EngineFunc, - ) -> Result<(), Error> { - let results = self.caller_results(); - self.prepare_compiled_func_call::(store, results, func, None) - } - - /// Returns the `results` [`SlotSpan`] of the top-most [`CallFrame`] on the [`CallStack`]. - /// - /// # Note - /// - /// We refer to the top-most [`CallFrame`] as the `caller` since this method is used for - /// tail call instructions for which the top-most [`CallFrame`] is the caller. - /// - /// [`CallStack`]: crate::engine::executor::stack::CallStack - #[inline(always)] - fn caller_results(&self) -> SlotSpan { - self.stack - .calls - .peek() - .expect("must have caller on the stack") - .results() - } - - /// Executes an [`Op::CallInternal0`]. - #[inline(always)] - pub fn execute_call_internal_0( - &mut self, - store: &mut StoreInner, - results: SlotSpan, - func: EngineFunc, - ) -> Result<(), Error> { - self.prepare_compiled_func_call::(store, results, func, None) - } - - /// Executes an [`Op::CallInternal`]. - #[inline(always)] - pub fn execute_call_internal( - &mut self, - store: &mut StoreInner, - results: SlotSpan, - func: EngineFunc, - ) -> Result<(), Error> { - self.prepare_compiled_func_call::(store, results, func, None) - } - - /// Executes an [`Op::ReturnCallImported0`]. - pub fn execute_return_call_imported_0( - &mut self, - store: &mut PrunedStore, - func: index::Func, - ) -> Result { - self.execute_return_call_imported_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallImported`]. - pub fn execute_return_call_imported( - &mut self, - store: &mut PrunedStore, - func: index::Func, - ) -> Result { - self.execute_return_call_imported_impl::(store, func) - } - - /// Executes an [`Op::ReturnCallImported`] or [`Op::ReturnCallImported0`]. - fn execute_return_call_imported_impl( - &mut self, - store: &mut PrunedStore, - func: index::Func, - ) -> Result { - let func = self.get_func(func); - self.execute_call_imported_impl::(store, None, &func) - } - - /// Executes an [`Op::CallImported0`]. - pub fn execute_call_imported_0( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func: index::Func, - ) -> Result<(), Error> { - let func = self.get_func(func); - _ = self.execute_call_imported_impl::(store, Some(results), &func)?; - Ok(()) - } - - /// Executes an [`Op::CallImported`]. - pub fn execute_call_imported( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func: index::Func, - ) -> Result<(), Error> { - let func = self.get_func(func); - _ = self.execute_call_imported_impl::(store, Some(results), &func)?; - Ok(()) - } - - /// Executes an imported or indirect (tail) call instruction. - fn execute_call_imported_impl( - &mut self, - store: &mut PrunedStore, - results: Option, - func: &Func, - ) -> Result { - match store.inner().resolve_func(func) { - FuncEntity::Wasm(func) => { - let instance = *func.instance(); - let func_body = func.func_body(); - let results = results.unwrap_or_else(|| self.caller_results()); - self.prepare_compiled_func_call::( - store.inner_mut(), - results, - func_body, - Some(instance), - )?; - self.cache.update(store.inner_mut(), &instance); - Ok(ControlFlow::Continue(())) - } - FuncEntity::Host(host_func) => { - let host_func = *host_func; - self.execute_host_func::(store, results, func, host_func) - } - } - } - - /// Executes a host function. - /// - /// # Note - /// - /// This uses the value stack to store parameters and results of the host function call. - /// Returns an [`ErrorKind::ResumableHostTrap`] variant if the host function returned an error - /// and there are still call frames on the call stack making it possible to resume the - /// execution at a later point in time. - /// - /// [`ErrorKind::ResumableHostTrap`]: crate::error::ErrorKind::ResumableHostTrap - fn execute_host_func( - &mut self, - store: &mut PrunedStore, - results: Option, - func: &Func, - host_func: HostFuncEntity, - ) -> Result { - let len_params = host_func.len_params(); - let len_results = host_func.len_results(); - let max_inout = usize::from(len_params.max(len_results)); - let instance = *self.stack.calls.instance_expect(); - // We have to reinstantiate the `self.sp` [`FrameSlots`] since we just called - // [`ValueStack::reserve`] which might invalidate all live [`FrameSlots`]. - let (caller, popped_instance) = match ::KIND { - CallKind::Nested => self.stack.calls.peek().copied().map(|frame| (frame, None)), - CallKind::Tail => self.stack.calls.pop(), - } - .expect("need to have a caller on the call stack"); - let buffer = self.stack.values.extend_by(max_inout, |this| { - // Safety: we use the base offset of a live call frame on the call stack. - self.sp = unsafe { this.stack_ptr_at(caller.base_offset()) }; - })?; - if ::HAS_PARAMS { - let mut uninit_params = FrameParams::new(buffer); - self.copy_call_params(&mut uninit_params); - } - if matches!(::KIND, CallKind::Nested) { - self.update_instr_ptr_at(1); - } - let results = results.unwrap_or_else(|| caller.results()); - self.dispatch_host_func(store, host_func, &instance) - .map_err(|error| match self.stack.calls.is_empty() { - true => error, - false => ResumableHostTrapError::new(error, *func, results).into(), - })?; - self.cache.update(store.inner_mut(), &instance); - let results = results.iter(len_results); - match ::KIND { - CallKind::Nested => { - let returned = self.stack.values.drop_return(max_inout); - for (result, value) in results.zip(returned) { - // # Safety (1) - // - // We can safely acquire the stack pointer to the caller's and callee's (host) - // call frames because we just allocated the host call frame and can be sure that - // they are different. - // In the following we make sure to not access registers out of bounds of each - // call frame since we rely on Wasm validation and proper Wasm translation to - // provide us with valid result registers. - unsafe { self.sp.set(result, *value) }; - } - Ok(ControlFlow::Continue(())) - } - CallKind::Tail => { - let (mut regs, cf) = match self.stack.calls.peek() { - Some(frame) => { - // Case: return the caller's caller frame registers. - let sp = unsafe { self.stack.values.stack_ptr_at(frame.base_offset()) }; - (sp, ControlFlow::Continue(())) - } - None => { - // Case: call stack is empty -> return the root frame registers. - let sp = self.stack.values.root_stack_ptr(); - (sp, ControlFlow::Break(())) - } - }; - let returned = self.stack.values.drop_return(max_inout); - for (result, value) in results.zip(returned) { - // # Safety (1) - // - // We can safely acquire the stack pointer to the caller's and callee's (host) - // call frames because we just allocated the host call frame and can be sure that - // they are different. - // In the following we make sure to not access registers out of bounds of each - // call frame since we rely on Wasm validation and proper Wasm translation to - // provide us with valid result registers. - unsafe { regs.set(result, *value) }; - } - self.stack.values.truncate(caller.frame_offset()); - let new_instance = popped_instance.and_then(|_| self.stack.calls.instance()); - if let Some(new_instance) = new_instance { - self.cache.update(store.inner_mut(), new_instance); - } - if let Some(caller) = self.stack.calls.peek() { - Self::init_call_frame_impl( - &mut self.stack.values, - &mut self.sp, - &mut self.ip, - caller, - ); - } - Ok(cf) - } - } - } - - /// Convenience forwarder to [`dispatch_host_func`]. - fn dispatch_host_func( - &mut self, - store: &mut PrunedStore, - host_func: HostFuncEntity, - instance: &Instance, - ) -> Result<(u16, u16), Error> { - dispatch_host_func( - store, - &mut self.stack.values, - host_func, - Some(instance), - CallHooks::Call, - ) - } - - /// Executes an [`Op::CallIndirect0`]. - pub fn execute_return_call_indirect_0( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0Imm16`]. - pub fn execute_return_call_indirect_0_imm16( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params_imm16(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0`]. - pub fn execute_return_call_indirect( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0Imm16`]. - pub fn execute_return_call_indirect_imm16( - &mut self, - store: &mut PrunedStore, - func_type: index::FuncType, - ) -> Result { - let (index, table) = self.pull_call_indirect_params_imm16(); - self.execute_call_indirect_impl::(store, None, func_type, index, table) - } - - /// Executes an [`Op::CallIndirect0`]. - pub fn execute_call_indirect_0( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirect0Imm16`]. - pub fn execute_call_indirect_0_imm16( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params_imm16(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirect`]. - pub fn execute_call_indirect( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirectImm16`]. - pub fn execute_call_indirect_imm16( - &mut self, - store: &mut PrunedStore, - results: SlotSpan, - func_type: index::FuncType, - ) -> Result<(), Error> { - let (index, table) = self.pull_call_indirect_params_imm16(); - _ = self.execute_call_indirect_impl::( - store, - Some(results), - func_type, - index, - table, - )?; - Ok(()) - } - - /// Executes an [`Op::CallIndirect`] and [`Op::CallIndirect0`]. - fn execute_call_indirect_impl( - &mut self, - store: &mut PrunedStore, - results: Option, - func_type: index::FuncType, - index: u64, - table: index::Table, - ) -> Result { - let table = self.get_table(table); - let funcref = store - .inner() - .resolve_table(&table) - .get_untyped(index) - .map(>::from) - .ok_or(TrapCode::TableOutOfBounds)?; - let func = funcref.val().ok_or(TrapCode::IndirectCallToNull)?; - let actual_signature = store.inner().resolve_func(func).ty_dedup(); - let expected_signature = &self.get_func_type_dedup(func_type); - if actual_signature != expected_signature { - return Err(Error::from(TrapCode::BadSignature)); - } - self.execute_call_imported_impl::(store, results, func) - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/comparison.rs b/crates/wasmi/src/engine/executor/instrs/comparison.rs deleted file mode 100644 index 9d6f0b44d9..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/comparison.rs +++ /dev/null @@ -1,91 +0,0 @@ -use super::{Executor, UntypedValueCmpExt}; -use crate::{core::wasm, ir::Slot}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - impl_binary_executors! { - (Op::I32Eq, execute_i32_eq, wasm::i32_eq), - (Op::I32Ne, execute_i32_ne, wasm::i32_ne), - (Op::I32LtS, execute_i32_lt_s, wasm::i32_lt_s), - (Op::I32LtU, execute_i32_lt_u, wasm::i32_lt_u), - (Op::I32LeS, execute_i32_le_s, wasm::i32_le_s), - (Op::I32LeU, execute_i32_le_u, wasm::i32_le_u), - - (Op::I64Eq, execute_i64_eq, wasm::i64_eq), - (Op::I64Ne, execute_i64_ne, wasm::i64_ne), - (Op::I64LtS, execute_i64_lt_s, wasm::i64_lt_s), - (Op::I64LtU, execute_i64_lt_u, wasm::i64_lt_u), - (Op::I64LeS, execute_i64_le_s, wasm::i64_le_s), - (Op::I64LeU, execute_i64_le_u, wasm::i64_le_u), - - (Op::F32Eq, execute_f32_eq, wasm::f32_eq), - (Op::F32Ne, execute_f32_ne, wasm::f32_ne), - (Op::F32Lt, execute_f32_lt, wasm::f32_lt), - (Op::F32Le, execute_f32_le, wasm::f32_le), - (Op::F32NotLt, execute_f32_not_lt, ::not_lt), - (Op::F32NotLe, execute_f32_not_le, ::not_le), - - (Op::F64Eq, execute_f64_eq, wasm::f64_eq), - (Op::F64Ne, execute_f64_ne, wasm::f64_ne), - (Op::F64Lt, execute_f64_lt, wasm::f64_lt), - (Op::F64Le, execute_f64_le, wasm::f64_le), - (Op::F64NotLt, execute_f64_not_lt, ::not_lt), - (Op::F64NotLe, execute_f64_not_le, ::not_le), - } -} - -macro_rules! impl_comparison_imm16_rhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { - self.execute_binary_imm16_rhs(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_comparison_imm16_rhs! { - (i32, Op::I32EqImm16, execute_i32_eq_imm16, wasm::i32_eq), - (i32, Op::I32NeImm16, execute_i32_ne_imm16, wasm::i32_ne), - (i32, Op::I32LtSImm16Rhs, execute_i32_lt_s_imm16_rhs, wasm::i32_lt_s), - (u32, Op::I32LtUImm16Rhs, execute_i32_lt_u_imm16_rhs, wasm::i32_lt_u), - (i32, Op::I32LeSImm16Rhs, execute_i32_le_s_imm16_rhs, wasm::i32_le_s), - (u32, Op::I32LeUImm16Rhs, execute_i32_le_u_imm16_rhs, wasm::i32_le_u), - - (i64, Op::I64EqImm16, execute_i64_eq_imm16, wasm::i64_eq), - (i64, Op::I64NeImm16, execute_i64_ne_imm16, wasm::i64_ne), - (i64, Op::I64LtSImm16Rhs, execute_i64_lt_s_imm16_rhs, wasm::i64_lt_s), - (u64, Op::I64LtUImm16Rhs, execute_i64_lt_u_imm16_rhs, wasm::i64_lt_u), - (i64, Op::I64LeSImm16Rhs, execute_i64_le_s_imm16_rhs, wasm::i64_le_s), - (u64, Op::I64LeUImm16Rhs, execute_i64_le_u_imm16_rhs, wasm::i64_le_u), - } -} - -macro_rules! impl_comparison_imm16_lhs { - ( $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Const16<$ty>, rhs: Slot) { - self.execute_binary_imm16_lhs(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_comparison_imm16_lhs! { - (i32, Op::I32LtSImm16Lhs, execute_i32_lt_s_imm16_lhs, wasm::i32_lt_s), - (u32, Op::I32LtUImm16Lhs, execute_i32_lt_u_imm16_lhs, wasm::i32_lt_u), - (i32, Op::I32LeSImm16Lhs, execute_i32_le_s_imm16_lhs, wasm::i32_le_s), - (u32, Op::I32LeUImm16Lhs, execute_i32_le_u_imm16_lhs, wasm::i32_le_u), - - (i64, Op::I64LtSImm16Lhs, execute_i64_lt_s_imm16_lhs, wasm::i64_lt_s), - (u64, Op::I64LtUImm16Lhs, execute_i64_lt_u_imm16_lhs, wasm::i64_lt_u), - (i64, Op::I64LeSImm16Lhs, execute_i64_le_s_imm16_lhs, wasm::i64_le_s), - (u64, Op::I64LeUImm16Lhs, execute_i64_le_u_imm16_lhs, wasm::i64_le_u), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/conversion.rs b/crates/wasmi/src/engine/executor/instrs/conversion.rs deleted file mode 100644 index e291aeb208..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/conversion.rs +++ /dev/null @@ -1,60 +0,0 @@ -use super::Executor; -use crate::{core::wasm, ir::Slot, Error}; - -#[cfg(doc)] -use crate::ir::Op; - -macro_rules! impl_fallible_conversion_impls { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, input: Slot) -> Result<(), Error> { - self.try_execute_unary(result, input, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_unary_executors! { - (Op::I32WrapI64, execute_i32_wrap_i64, wasm::i32_wrap_i64), - - (Op::I32TruncSatF32S, execute_i32_trunc_sat_f32_s, wasm::i32_trunc_sat_f32_s), - (Op::I32TruncSatF32U, execute_i32_trunc_sat_f32_u, wasm::i32_trunc_sat_f32_u), - (Op::I32TruncSatF64S, execute_i32_trunc_sat_f64_s, wasm::i32_trunc_sat_f64_s), - (Op::I32TruncSatF64U, execute_i32_trunc_sat_f64_u, wasm::i32_trunc_sat_f64_u), - (Op::I64TruncSatF32S, execute_i64_trunc_sat_f32_s, wasm::i64_trunc_sat_f32_s), - (Op::I64TruncSatF32U, execute_i64_trunc_sat_f32_u, wasm::i64_trunc_sat_f32_u), - (Op::I64TruncSatF64S, execute_i64_trunc_sat_f64_s, wasm::i64_trunc_sat_f64_s), - (Op::I64TruncSatF64U, execute_i64_trunc_sat_f64_u, wasm::i64_trunc_sat_f64_u), - - (Op::I32Extend8S, execute_i32_extend8_s, wasm::i32_extend8_s), - (Op::I32Extend16S, execute_i32_extend16_s, wasm::i32_extend16_s), - (Op::I64Extend8S, execute_i64_extend8_s, wasm::i64_extend8_s), - (Op::I64Extend16S, execute_i64_extend16_s, wasm::i64_extend16_s), - (Op::I64Extend32S, execute_i64_extend32_s, wasm::i64_extend32_s), - - (Op::F32DemoteF64, execute_f32_demote_f64, wasm::f32_demote_f64), - (Op::F64PromoteF32, execute_f64_promote_f32, wasm::f64_promote_f32), - - (Op::F32ConvertI32S, execute_f32_convert_i32_s, wasm::f32_convert_i32_s), - (Op::F32ConvertI32U, execute_f32_convert_i32_u, wasm::f32_convert_i32_u), - (Op::F32ConvertI64S, execute_f32_convert_i64_s, wasm::f32_convert_i64_s), - (Op::F32ConvertI64U, execute_f32_convert_i64_u, wasm::f32_convert_i64_u), - (Op::F64ConvertI32S, execute_f64_convert_i32_s, wasm::f64_convert_i32_s), - (Op::F64ConvertI32U, execute_f64_convert_i32_u, wasm::f64_convert_i32_u), - (Op::F64ConvertI64S, execute_f64_convert_i64_s, wasm::f64_convert_i64_s), - (Op::F64ConvertI64U, execute_f64_convert_i64_u, wasm::f64_convert_i64_u), - } - - impl_fallible_conversion_impls! { - (Op::I32TruncF32S, execute_i32_trunc_f32_s, wasm::i32_trunc_f32_s), - (Op::I32TruncF32U, execute_i32_trunc_f32_u, wasm::i32_trunc_f32_u), - (Op::I32TruncF64S, execute_i32_trunc_f64_s, wasm::i32_trunc_f64_s), - (Op::I32TruncF64U, execute_i32_trunc_f64_u, wasm::i32_trunc_f64_u), - (Op::I64TruncF32S, execute_i64_trunc_f32_s, wasm::i64_trunc_f32_s), - (Op::I64TruncF32U, execute_i64_trunc_f32_u, wasm::i64_trunc_f32_u), - (Op::I64TruncF64S, execute_i64_trunc_f64_s, wasm::i64_trunc_f64_s), - (Op::I64TruncF64U, execute_i64_trunc_f64_u, wasm::i64_trunc_f64_u), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/copy.rs b/crates/wasmi/src/engine/executor/instrs/copy.rs deleted file mode 100644 index 7447ac6727..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/copy.rs +++ /dev/null @@ -1,117 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::UntypedVal, - engine::utils::unreachable_unchecked, - ir::{FixedSlotSpan, Op, Slot, SlotSpan}, -}; -use core::slice; - -impl Executor<'_> { - /// Executes a generic `copy` [`Op`]. - fn execute_copy_impl(&mut self, result: Slot, value: T, f: fn(&mut Self, T) -> UntypedVal) { - let value = f(self, value); - self.set_stack_slot(result, value); - self.next_instr() - } - - /// Executes an [`Op::Copy`]. - pub fn execute_copy(&mut self, result: Slot, value: Slot) { - self.execute_copy_impl(result, value, |this, value| this.get_stack_slot(value)) - } - - /// Executes an [`Op::Copy2`]. - pub fn execute_copy_2(&mut self, results: FixedSlotSpan<2>, values: [Slot; 2]) { - self.execute_copy_2_impl(results, values); - self.next_instr() - } - - /// Internal implementation of [`Op::Copy2`] execution. - fn execute_copy_2_impl(&mut self, results: FixedSlotSpan<2>, values: [Slot; 2]) { - let result0 = results.span().head(); - let result1 = result0.next(); - // We need `tmp` in case `results[0] == values[1]` to avoid overwriting `values[1]` before reading it. - let tmp = self.get_stack_slot(values[1]); - self.set_stack_slot(result0, self.get_stack_slot(values[0])); - self.set_stack_slot(result1, tmp); - } - - /// Executes an [`Op::CopyImm32`]. - pub fn execute_copy_imm32(&mut self, result: Slot, value: AnyConst32) { - self.execute_copy_impl(result, value, |_, value| UntypedVal::from(u32::from(value))) - } - - /// Executes an [`Op::CopyI64Imm32`]. - pub fn execute_copy_i64imm32(&mut self, result: Slot, value: Const32) { - self.execute_copy_impl(result, value, |_, value| UntypedVal::from(i64::from(value))) - } - - /// Executes an [`Op::CopyF64Imm32`]. - pub fn execute_copy_f64imm32(&mut self, result: Slot, value: Const32) { - self.execute_copy_impl(result, value, |_, value| UntypedVal::from(f64::from(value))) - } - - /// Executes an [`Op::CopySpan`]. - /// - /// # Note - /// - /// - This instruction assumes that `results` and `values` do _not_ overlap - /// and thus can copy all the elements without a costly temporary buffer. - /// - If `results` and `values` _do_ overlap [`Op::CopySpan`] is used. - pub fn execute_copy_span(&mut self, results: SlotSpan, values: SlotSpan, len: u16) { - self.execute_copy_span_impl(results, values, len); - self.next_instr(); - } - - /// Internal implementation of [`Op::CopySpan`] execution. - pub fn execute_copy_span_impl(&mut self, results: SlotSpan, values: SlotSpan, len: u16) { - let results = results.iter(len); - let values = values.iter(len); - for (result, value) in results.into_iter().zip(values.into_iter()) { - let value = self.get_stack_slot(value); - self.set_stack_slot(result, value); - } - } - - /// Executes an [`Op::CopyMany`]. - pub fn execute_copy_many(&mut self, results: SlotSpan, values: [Slot; 2]) { - self.ip.add(1); - self.ip = self.execute_copy_many_impl(self.ip, results, &values); - self.next_instr() - } - - /// Internal implementation of [`Op::CopyMany`] execution. - pub fn execute_copy_many_impl( - &mut self, - ip: InstructionPtr, - results: SlotSpan, - values: &[Slot], - ) -> InstructionPtr { - let mut ip = ip; - let mut result = results.head(); - let mut copy_values = |values: &[Slot]| { - for &value in values { - let value = self.get_stack_slot(value); - self.set_stack_slot(result, value); - result = result.next(); - } - }; - copy_values(values); - while let Op::SlotList { regs } = ip.get() { - copy_values(regs); - ip.add(1); - } - let values = match ip.get() { - Op::Slot { slot } => slice::from_ref(slot), - Op::Slot2 { slots } => slots, - Op::Slot3 { slots } => slots, - unexpected => { - // Safety: Wasmi translator guarantees that slot-list finalizer exists. - unsafe { - unreachable_unchecked!("expected slot-list finalizer but found: {unexpected:?}") - } - } - }; - copy_values(values); - ip - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/global.rs b/crates/wasmi/src/engine/executor/instrs/global.rs deleted file mode 100644 index 9ab3882cda..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/global.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::Executor; -use crate::{ - core::{hint, UntypedVal}, - ir::{index, Slot}, - store::StoreInner, -}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - /// Executes an [`Op::GlobalGet`]. - pub fn execute_global_get(&mut self, store: &StoreInner, result: Slot, global: index::Global) { - let value = match u32::from(global) { - 0 => unsafe { self.cache.global.get() }, - _ => { - hint::cold(); - let global = self.get_global(global); - *store.resolve_global(&global).get_untyped() - } - }; - self.set_stack_slot(result, value); - self.next_instr() - } - - /// Executes an [`Op::GlobalSet`]. - pub fn execute_global_set( - &mut self, - store: &mut StoreInner, - global: index::Global, - input: Slot, - ) { - let input = self.get_stack_slot(input); - self.execute_global_set_impl(store, global, input) - } - - /// Executes an [`Op::GlobalSetI32Imm16`]. - pub fn execute_global_set_i32imm16( - &mut self, - store: &mut StoreInner, - global: index::Global, - input: Const16, - ) { - let input = i32::from(input).into(); - self.execute_global_set_impl(store, global, input) - } - - /// Executes an [`Op::GlobalSetI64Imm16`]. - pub fn execute_global_set_i64imm16( - &mut self, - store: &mut StoreInner, - global: index::Global, - input: Const16, - ) { - let input = i64::from(input).into(); - self.execute_global_set_impl(store, global, input) - } - - /// Executes a generic `global.set` instruction. - fn execute_global_set_impl( - &mut self, - store: &mut StoreInner, - global: index::Global, - new_value: UntypedVal, - ) { - match u32::from(global) { - 0 => unsafe { self.cache.global.set(new_value) }, - _ => { - hint::cold(); - let global = self.get_global(global); - let mut ptr = store.resolve_global_mut(&global).get_untyped_ptr(); - // Safety: - // - Wasmi translation won't create `global.set` instructions for immutable globals. - // - Wasm validation ensures that values with matching types are written to globals. - unsafe { - *ptr.as_mut() = new_value; - } - } - }; - self.next_instr() - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/load.rs b/crates/wasmi/src/engine/executor/instrs/load.rs deleted file mode 100644 index 01e7139521..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/load.rs +++ /dev/null @@ -1,411 +0,0 @@ -use super::Executor; -use crate::{ - core::{wasm, UntypedVal, WriteAs}, - ir::{index::Memory, Offset16, Slot}, - store::StoreInner, - Error, - TrapCode, -}; - -#[cfg(feature = "simd")] -use crate::{core::simd, V128}; - -#[cfg(doc)] -use crate::ir::Op; - -/// The function signature of Wasm load operations. -type WasmLoadOp = fn(memory: &[u8], ptr: u64, offset: u64) -> Result; - -/// The function signature of Wasm load operations. -type WasmLoadAtOp = fn(memory: &[u8], address: usize) -> Result; - -impl Executor<'_> { - /// Returns the register `value` and `offset` parameters for a `load` [`Op`]. - fn fetch_ptr_and_offset_hi(&self) -> (Slot, Offset64Hi) { - // Safety: Wasmi translation guarantees that `Op::SlotAndImm32` exists. - unsafe { self.fetch_reg_and_offset_hi() } - } - - /// Executes a generic Wasm `load[N_{s|u}]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.load` - /// - `{i32, i64}.load8_s` - /// - `{i32, i64}.load8_u` - /// - `{i32, i64}.load16_s` - /// - `{i32, i64}.load16_u` - /// - `i64.load32_s` - /// - `i64.load32_u` - fn execute_load_extend( - &mut self, - store: &StoreInner, - memory: Memory, - result: Slot, - address: u64, - offset: Offset64, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_memory_bytes(memory, store); - let loaded_value = load_extend(memory, address, u64::from(offset))?; - self.set_stack_slot_as::(result, loaded_value); - Ok(()) - } - - /// Executes a generic Wasm `load[N_{s|u}]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.load` - /// - `{i32, i64}.load8_s` - /// - `{i32, i64}.load8_u` - /// - `{i32, i64}.load16_s` - /// - `{i32, i64}.load16_u` - /// - `i64.load32_s` - /// - `i64.load32_u` - fn execute_load_extend_at( - &mut self, - store: &StoreInner, - memory: Memory, - result: Slot, - address: Address32, - load_extend_at: WasmLoadAtOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_memory_bytes(memory, store); - let loaded_value = load_extend_at(memory, usize::from(address))?; - self.set_stack_slot_as::(result, loaded_value); - Ok(()) - } - - /// Executes a generic Wasm `store[N_{s|u}]` operation on the default memory. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.load` - /// - `{i32, i64}.load8_s` - /// - `{i32, i64}.load8_u` - /// - `{i32, i64}.load16_s` - /// - `{i32, i64}.load16_u` - /// - `i64.load32_s` - /// - `i64.load32_u` - fn execute_load_extend_mem0( - &mut self, - result: Slot, - address: u64, - offset: Offset64, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_default_memory_bytes(); - let loaded_value = load_extend(memory, address, u64::from(offset))?; - self.set_stack_slot_as::(result, loaded_value); - Ok(()) - } - - /// Executes a generic `load` [`Op`]. - fn execute_load_impl( - &mut self, - store: &StoreInner, - result: Slot, - offset_lo: Offset64Lo, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let (ptr, offset_hi) = self.fetch_ptr_and_offset_hi(); - let memory = self.fetch_optional_memory(2); - let address = self.get_stack_slot_as::(ptr); - let offset = Offset64::combine(offset_hi, offset_lo); - self.execute_load_extend::(store, memory, result, address, offset, load_extend)?; - self.try_next_instr_at(2) - } - - /// Executes a generic `load_at` [`Op`]. - fn execute_load_at_impl( - &mut self, - store: &StoreInner, - result: Slot, - address: Address32, - load_extend_at: WasmLoadAtOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let memory = self.fetch_optional_memory(1); - self.execute_load_extend_at::(store, memory, result, address, load_extend_at)?; - self.try_next_instr() - } - - /// Executes a generic `load_offset16` [`Op`]. - fn execute_load_offset16_impl( - &mut self, - result: Slot, - ptr: Slot, - offset: Offset16, - load_extend: WasmLoadOp, - ) -> Result<(), Error> - where - UntypedVal: WriteAs, - { - let address = self.get_stack_slot_as::(ptr); - let offset = Offset64::from(offset); - self.execute_load_extend_mem0::(result, address, offset, load_extend)?; - self.try_next_instr() - } -} - -macro_rules! impl_execute_load { - ( $( - ( - $ty:ty, - (Op::$var_load:expr, $fn_load:ident), - (Op::$var_load_at:expr, $fn_load_at:ident), - (Op::$var_load_off16:expr, $fn_load_off16:ident), - $load_fn:expr, - $load_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_load), "`].")] - pub fn $fn_load(&mut self, store: &StoreInner, result: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - self.execute_load_impl(store, result, offset_lo, $load_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_load_at), "`].")] - pub fn $fn_load_at(&mut self, store: &StoreInner, result: Slot, address: Address32) -> Result<(), Error> { - self.execute_load_at_impl(store, result, address, $load_at_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_load_off16), "`].")] - pub fn $fn_load_off16(&mut self, result: Slot, ptr: Slot, offset: Offset16) -> Result<(), Error> { - self.execute_load_offset16_impl::<$ty>(result, ptr, offset, $load_fn) - } - )* - } -} - -impl Executor<'_> { - #[cfg(feature = "simd")] - impl_execute_load! { - ( - V128, - (Op::V128Load, execute_v128_load), - (Op::V128LoadAt, execute_v128_load_at), - (Op::V128LoadOffset16, execute_v128_load_offset16), - simd::v128_load, - simd::v128_load_at, - ), - ( - V128, - (Op::V128Load8x8S, execute_v128_load8x8_s), - (Op::V128Load8x8SAt, execute_v128_load8x8_s_at), - (Op::V128Load8x8SOffset16, execute_v128_load8x8_s_offset16), - simd::v128_load8x8_s, - simd::v128_load8x8_s_at, - ), - ( - V128, - (Op::V128Load8x8U, execute_v128_load8x8_u), - (Op::V128Load8x8UAt, execute_v128_load8x8_u_at), - (Op::V128Load8x8UOffset16, execute_v128_load8x8_u_offset16), - simd::v128_load8x8_u, - simd::v128_load8x8_u_at, - ), - ( - V128, - (Op::V128Load16x4S, execute_v128_load16x4_s), - (Op::V128Load16x4SAt, execute_v128_load16x4_s_at), - (Op::V128Load16x4SOffset16, execute_v128_load16x4_s_offset16), - simd::v128_load16x4_s, - simd::v128_load16x4_s_at, - ), - ( - V128, - (Op::V128Load16x4U, execute_v128_load16x4_u), - (Op::V128Load16x4UAt, execute_v128_load16x4_u_at), - (Op::V128Load16x4UOffset16, execute_v128_load16x4_u_offset16), - simd::v128_load16x4_u, - simd::v128_load16x4_u_at, - ), - ( - V128, - (Op::V128Load32x2S, execute_v128_load32x2_s), - (Op::V128Load32x2SAt, execute_v128_load32x2_s_at), - (Op::V128Load32x2SOffset16, execute_v128_load32x2_s_offset16), - simd::v128_load32x2_s, - simd::v128_load32x2_s_at, - ), - ( - V128, - (Op::V128Load32x2U, execute_v128_load32x2_u), - (Op::V128Load32x2UAt, execute_v128_load32x2_u_at), - (Op::V128Load32x2UOffset16, execute_v128_load32x2_u_offset16), - simd::v128_load32x2_u, - simd::v128_load32x2_u_at, - ), - ( - V128, - (Op::V128Load8Splat, execute_v128_load8_splat), - (Op::V128Load8SplatAt, execute_v128_load8_splat_at), - (Op::V128Load8SplatOffset16, execute_v128_load8_splat_offset16), - simd::v128_load8_splat, - simd::v128_load8_splat_at, - ), - ( - V128, - (Op::V128Load16Splat, execute_v128_load16_splat), - (Op::V128Load16SplatAt, execute_v128_load16_splat_at), - (Op::V128Load16SplatOffset16, execute_v128_load16_splat_offset16), - simd::v128_load16_splat, - simd::v128_load16_splat_at, - ), - ( - V128, - (Op::V128Load32Splat, execute_v128_load32_splat), - (Op::V128Load32SplatAt, execute_v128_load32_splat_at), - (Op::V128Load32SplatOffset16, execute_v128_load32_splat_offset16), - simd::v128_load32_splat, - simd::v128_load32_splat_at, - ), - ( - V128, - (Op::V128Load64Splat, execute_v128_load64_splat), - (Op::V128Load64SplatAt, execute_v128_load64_splat_at), - (Op::V128Load64SplatOffset16, execute_v128_load64_splat_offset16), - simd::v128_load64_splat, - simd::v128_load64_splat_at, - ), - ( - V128, - (Op::V128Load32Zero, execute_v128_load32_zero), - (Op::V128Load32ZeroAt, execute_v128_load32_zero_at), - (Op::V128Load32ZeroOffset16, execute_v128_load32_zero_offset16), - simd::v128_load32_zero, - simd::v128_load32_zero_at, - ), - ( - V128, - (Op::V128Load64Zero, execute_v128_load64_zero), - (Op::V128Load64ZeroAt, execute_v128_load64_zero_at), - (Op::V128Load64ZeroOffset16, execute_v128_load64_zero_offset16), - simd::v128_load64_zero, - simd::v128_load64_zero_at, - ), - } - - impl_execute_load! { - ( - u32, - (Op::Load32, execute_load32), - (Op::Load32At, execute_load32_at), - (Op::Load32Offset16, execute_load32_offset16), - wasm::load32, - wasm::load32_at, - ), - ( - u64, - (Op::Load64, execute_load64), - (Op::Load64At, execute_load64_at), - (Op::Load64Offset16, execute_load64_offset16), - wasm::load64, - wasm::load64_at, - ), - - ( - i32, - (Op::I32Load8s, execute_i32_load8_s), - (Op::I32Load8sAt, execute_i32_load8_s_at), - (Op::I32Load8sOffset16, execute_i32_load8_s_offset16), - wasm::i32_load8_s, - wasm::i32_load8_s_at, - ), - ( - i32, - (Op::I32Load8u, execute_i32_load8_u), - (Op::I32Load8uAt, execute_i32_load8_u_at), - (Op::I32Load8uOffset16, execute_i32_load8_u_offset16), - wasm::i32_load8_u, - wasm::i32_load8_u_at, - ), - ( - i32, - (Op::I32Load16s, execute_i32_load16_s), - (Op::I32Load16sAt, execute_i32_load16_s_at), - (Op::I32Load16sOffset16, execute_i32_load16_s_offset16), - wasm::i32_load16_s, - wasm::i32_load16_s_at, - ), - ( - i32, - (Op::I32Load16u, execute_i32_load16_u), - (Op::I32Load16uAt, execute_i32_load16_u_at), - (Op::I32Load16uOffset16, execute_i32_load16_u_offset16), - wasm::i32_load16_u, - wasm::i32_load16_u_at, - ), - - ( - i64, - (Op::I64Load8s, execute_i64_load8_s), - (Op::I64Load8sAt, execute_i64_load8_s_at), - (Op::I64Load8sOffset16, execute_i64_load8_s_offset16), - wasm::i64_load8_s, - wasm::i64_load8_s_at, - ), - ( - i64, - (Op::I64Load8u, execute_i64_load8_u), - (Op::I64Load8uAt, execute_i64_load8_u_at), - (Op::I64Load8uOffset16, execute_i64_load8_u_offset16), - wasm::i64_load8_u, - wasm::i64_load8_u_at, - ), - ( - i64, - (Op::I64Load16s, execute_i64_load16_s), - (Op::I64Load16sAt, execute_i64_load16_s_at), - (Op::I64Load16sOffset16, execute_i64_load16_s_offset16), - wasm::i64_load16_s, - wasm::i64_load16_s_at, - ), - ( - i64, - (Op::I64Load16u, execute_i64_load16_u), - (Op::I64Load16uAt, execute_i64_load16_u_at), - (Op::I64Load16uOffset16, execute_i64_load16_u_offset16), - wasm::i64_load16_u, - wasm::i64_load16_u_at, - ), - ( - i64, - (Op::I64Load32s, execute_i64_load32_s), - (Op::I64Load32sAt, execute_i64_load32_s_at), - (Op::I64Load32sOffset16, execute_i64_load32_s_offset16), - wasm::i64_load32_s, - wasm::i64_load32_s_at, - ), - ( - i64, - (Op::I64Load32u, execute_i64_load32_u), - (Op::I64Load32uAt, execute_i64_load32_u_at), - (Op::I64Load32uOffset16, execute_i64_load32_u_offset16), - wasm::i64_load32_u, - wasm::i64_load32_u_at, - ), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/memory.rs b/crates/wasmi/src/engine/executor/instrs/memory.rs deleted file mode 100644 index 5cbf1b7b7d..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/memory.rs +++ /dev/null @@ -1,281 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - engine::{utils::unreachable_unchecked, ResumableOutOfFuelError}, - errors::MemoryError, - ir::{ - index::{Data, Memory}, - Op, - Slot, - }, - store::{PrunedStore, StoreError, StoreInner}, - Error, - TrapCode, -}; - -impl Executor<'_> { - /// Returns the [`Op::MemoryIndex`] parameter for an [`Op`]. - fn fetch_memory_index(&self, offset: usize) -> Memory { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::MemoryIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::MemoryIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::MemoryIndex` but found: {unexpected:?}") - } - } - } - } - - /// Returns the [`Op::DataIndex`] parameter for an [`Op`]. - fn fetch_data_segment_index(&self, offset: usize) -> Data { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::DataIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::DataIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::DataIndex` but found: {unexpected:?}") - } - } - } - } - - /// Executes an [`Op::DataDrop`]. - pub fn execute_data_drop(&mut self, store: &mut StoreInner, segment_index: Data) { - let segment = self.get_data_segment(segment_index); - store.resolve_data_mut(&segment).drop_bytes(); - self.next_instr(); - } - - /// Executes an [`Op::MemorySize`]. - pub fn execute_memory_size(&mut self, store: &StoreInner, result: Slot, memory: Memory) { - self.execute_memory_size_impl(store, result, memory); - self.next_instr() - } - - /// Underlying implementation of [`Op::MemorySize`]. - fn execute_memory_size_impl(&mut self, store: &StoreInner, result: Slot, memory: Memory) { - let memory = self.get_memory(memory); - let size = store.resolve_memory(&memory).size(); - self.set_stack_slot(result, size); - } - - /// Executes an [`Op::MemoryGrow`]. - pub fn execute_memory_grow( - &mut self, - store: &mut PrunedStore, - result: Slot, - delta: Slot, - ) -> Result<(), Error> { - let delta: u64 = self.get_stack_slot_as(delta); - let memory = self.fetch_memory_index(1); - if delta == 0 { - // Case: growing by 0 pages means there is nothing to do - self.execute_memory_size_impl(store.inner(), result, memory); - return self.try_next_instr_at(2); - } - let memory = self.get_memory(memory); - let return_value = match store.grow_memory(&memory, delta) { - Ok(return_value) => { - // The `memory.grow` operation might have invalidated the cached - // linear memory so we need to reset it in order for the cache to - // reload in case it is used again. - // - // Safety: the instance has not changed thus calling this is valid. - unsafe { self.cache.update_memory(store.inner_mut()) }; - return_value - } - Err(StoreError::External( - MemoryError::OutOfBoundsGrowth | MemoryError::OutOfSystemMemory, - )) => { - let memory_ty = store.inner().resolve_memory(&memory).ty(); - match memory_ty.is_64() { - true => u64::MAX, - false => u64::from(u32::MAX), - } - } - Err(StoreError::External(MemoryError::OutOfFuel { required_fuel })) => { - return Err(Error::from(ResumableOutOfFuelError::new(required_fuel))) - } - Err(StoreError::External(MemoryError::ResourceLimiterDeniedAllocation)) => { - return Err(Error::from(TrapCode::GrowthOperationLimited)) - } - Err(error) => { - panic!("`table.grow`: internal interpreter error: {error}") - } - }; - self.set_stack_slot(result, return_value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::MemoryCopy`]. - pub fn execute_memory_copy( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u64 = self.get_stack_slot_as(src); - let len: u64 = self.get_stack_slot_as(len); - let Ok(dst_index) = usize::try_from(dst) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(src_index) = usize::try_from(src) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(len) = usize::try_from(len) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let dst_memory = self.fetch_memory_index(1); - let src_memory = self.fetch_memory_index(2); - if src_memory == dst_memory { - return self - .execute_memory_copy_within_impl(store, src_memory, dst_index, src_index, len); - } - let (src_memory, dst_memory, fuel) = store.resolve_memory_pair_and_fuel( - &self.get_memory(src_memory), - &self.get_memory(dst_memory), - ); - // These accesses just perform the bounds checks required by the Wasm spec. - let src_bytes = src_memory - .data() - .get(src_index..) - .and_then(|memory| memory.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - let dst_bytes = dst_memory - .data_mut() - .get_mut(dst_index..) - .and_then(|memory| memory.get_mut(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - dst_bytes.copy_from_slice(src_bytes); - self.try_next_instr_at(3) - } - - /// Executes a generic `memory.copy` instruction. - fn execute_memory_copy_within_impl( - &mut self, - store: &mut StoreInner, - memory: Memory, - dst_index: usize, - src_index: usize, - len: usize, - ) -> Result<(), Error> { - let memory = self.get_memory(memory); - let (memory, fuel) = store.resolve_memory_and_fuel_mut(&memory); - let bytes = memory.data_mut(); - // These accesses just perform the bounds checks required by the Wasm spec. - bytes - .get(src_index..) - .and_then(|memory| memory.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - bytes - .get(dst_index..) - .and_then(|memory| memory.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - bytes.copy_within(src_index..src_index.wrapping_add(len), dst_index); - self.try_next_instr_at(3) - } - - /// Executes an [`Op::MemoryFill`]. - pub fn execute_memory_fill( - &mut self, - store: &mut StoreInner, - dst: Slot, - value: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let value: u8 = self.get_stack_slot_as(value); - let len: u64 = self.get_stack_slot_as(len); - self.execute_memory_fill_impl(store, dst, value, len) - } - - /// Executes an [`Op::MemoryFillImm`]. - pub fn execute_memory_fill_imm( - &mut self, - store: &mut StoreInner, - dst: Slot, - value: u8, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let len: u64 = self.get_stack_slot_as(len); - self.execute_memory_fill_impl(store, dst, value, len) - } - - /// Executes a generic `memory.fill` instruction. - #[inline(never)] - fn execute_memory_fill_impl( - &mut self, - store: &mut StoreInner, - dst: u64, - value: u8, - len: u64, - ) -> Result<(), Error> { - let Ok(dst) = usize::try_from(dst) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(len) = usize::try_from(len) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let memory = self.fetch_memory_index(1); - let memory = self.get_memory(memory); - let (memory, fuel) = store.resolve_memory_and_fuel_mut(&memory); - let slice = memory - .data_mut() - .get_mut(dst..) - .and_then(|memory| memory.get_mut(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - slice.fill(value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::MemoryInit`]. - pub fn execute_memory_init( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u32 = self.get_stack_slot_as(src); - let len: u32 = self.get_stack_slot_as(len); - let Ok(dst_index) = usize::try_from(dst) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(src_index) = usize::try_from(src) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let Ok(len) = usize::try_from(len) else { - return Err(Error::from(TrapCode::MemoryOutOfBounds)); - }; - let memory_index: Memory = self.fetch_memory_index(1); - let data_index: Data = self.fetch_data_segment_index(2); - let (memory, data, fuel) = store.resolve_memory_init_params( - &self.get_memory(memory_index), - &self.get_data_segment(data_index), - ); - let memory = memory - .data_mut() - .get_mut(dst_index..) - .and_then(|memory| memory.get_mut(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - let data = data - .bytes() - .get(src_index..) - .and_then(|data| data.get(..len)) - .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_copying_bytes(len as u64))?; - memory.copy_from_slice(data); - self.try_next_instr_at(3) - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/return_.rs b/crates/wasmi/src/engine/executor/instrs/return_.rs deleted file mode 100644 index f29139c589..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/return_.rs +++ /dev/null @@ -1,237 +0,0 @@ -use super::{ControlFlow, Executor, InstructionPtr}; -use crate::{ - core::UntypedVal, - engine::{executor::stack::FrameSlots, utils::unreachable_unchecked}, - ir::{BoundedSlotSpan, Op, Slot, SlotSpan}, - store::StoreInner, -}; -use core::slice; - -impl Executor<'_> { - /// Returns the execution to the caller. - /// - /// Any return values are expected to already have been transferred - /// from the returning callee to the caller. - fn return_impl(&mut self, store: &mut StoreInner) -> ControlFlow { - let (returned, popped_instance) = self - .stack - .calls - .pop() - .expect("the executing call frame is always on the stack"); - self.stack.values.truncate(returned.frame_offset()); - let new_instance = popped_instance.and_then(|_| self.stack.calls.instance()); - if let Some(new_instance) = new_instance { - self.cache.update(store, new_instance); - } - match self.stack.calls.peek() { - Some(caller) => { - Self::init_call_frame_impl( - &mut self.stack.values, - &mut self.sp, - &mut self.ip, - caller, - ); - ControlFlow::Continue(()) - } - None => ControlFlow::Break(()), - } - } - - /// Execute an [`Op::Return`]. - pub fn execute_return(&mut self, store: &mut StoreInner) -> ControlFlow { - self.return_impl(store) - } - - /// Returns the [`FrameSlots`] of the caller and the [`SlotSpan`] of the results. - /// - /// The returned [`FrameSlots`] is valid for all [`Slot`] in the returned [`SlotSpan`]. - fn return_caller_results(&mut self) -> (FrameSlots, SlotSpan) { - let (callee, caller) = self - .stack - .calls - .peek_2() - .expect("the callee must exist on the call stack"); - match caller { - Some(caller) => { - // Case: we need to return the `value` back to the caller frame. - // - // In this case we transfer the single return `value` to the `results` - // register span of the caller's call frame. - // - // Safety: The caller call frame is still live on the value stack - // and therefore it is safe to acquire its value stack pointer. - let caller_sp = unsafe { self.stack.values.stack_ptr_at(caller.base_offset()) }; - let results = callee.results(); - (caller_sp, results) - } - None => { - // Case: the root call frame is returning. - // - // In this case we transfer the single return `value` to the root - // register span of the entire value stack which is simply its zero index. - let dst_sp = self.stack.values.root_stack_ptr(); - let results = SlotSpan::new(Slot::from(0)); - (dst_sp, results) - } - } - } - - /// Execute a generic return [`Op`] returning a single value. - fn execute_return_value( - &mut self, - store: &mut StoreInner, - value: T, - f: fn(&Self, T) -> UntypedVal, - ) -> ControlFlow { - let (mut caller_sp, results) = self.return_caller_results(); - let value = f(self, value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { caller_sp.set(results.head(), value) } - self.return_impl(store) - } - - /// Execute an [`Op::ReturnSlot`] returning a single [`Slot`] value. - pub fn execute_return_reg(&mut self, store: &mut StoreInner, value: Slot) -> ControlFlow { - self.execute_return_value(store, value, Self::get_stack_slot) - } - - /// Execute an [`Op::ReturnSlot2`] returning two [`Slot`] values. - pub fn execute_return_reg2( - &mut self, - store: &mut StoreInner, - values: [Slot; 2], - ) -> ControlFlow { - self.execute_return_reg_n_impl::<2>(store, values) - } - - /// Execute an [`Op::ReturnSlot3`] returning three [`Slot`] values. - pub fn execute_return_reg3( - &mut self, - store: &mut StoreInner, - values: [Slot; 3], - ) -> ControlFlow { - self.execute_return_reg_n_impl::<3>(store, values) - } - - /// Executes an [`Op::ReturnSlot2`] or [`Op::ReturnSlot3`] generically. - fn execute_return_reg_n_impl( - &mut self, - store: &mut StoreInner, - values: [Slot; N], - ) -> ControlFlow { - let (mut caller_sp, results) = self.return_caller_results(); - debug_assert!(u16::try_from(N).is_ok()); - for (result, value) in results.iter(N as u16).zip(values) { - let value = self.get_stack_slot(value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { caller_sp.set(result, value) } - } - self.return_impl(store) - } - - /// Execute an [`Op::ReturnImm32`] returning a single 32-bit value. - pub fn execute_return_imm32( - &mut self, - store: &mut StoreInner, - value: AnyConst32, - ) -> ControlFlow { - self.execute_return_value(store, value, |_, value| u32::from(value).into()) - } - - /// Execute an [`Op::ReturnI64Imm32`] returning a single 32-bit encoded `i64` value. - pub fn execute_return_i64imm32( - &mut self, - store: &mut StoreInner, - value: Const32, - ) -> ControlFlow { - self.execute_return_value(store, value, |_, value| i64::from(value).into()) - } - - /// Execute an [`Op::ReturnF64Imm32`] returning a single 32-bit encoded `f64` value. - pub fn execute_return_f64imm32( - &mut self, - store: &mut StoreInner, - value: Const32, - ) -> ControlFlow { - self.execute_return_value(store, value, |_, value| f64::from(value).into()) - } - - /// Execute an [`Op::ReturnSpan`] returning many values. - pub fn execute_return_span( - &mut self, - store: &mut StoreInner, - values: BoundedSlotSpan, - ) -> ControlFlow { - let (mut caller_sp, results) = self.return_caller_results(); - let results = results.iter(values.len()); - for (result, value) in results.zip(values) { - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - let value = self.get_stack_slot(value); - unsafe { caller_sp.set(result, value) } - } - self.return_impl(store) - } - - /// Execute an [`Op::ReturnMany`] returning many values. - pub fn execute_return_many( - &mut self, - store: &mut StoreInner, - values: [Slot; 3], - ) -> ControlFlow { - self.ip.add(1); - self.copy_many_return_values(self.ip, &values); - self.return_impl(store) - } - - /// Copies many return values to the caller frame. - /// - /// # Note - /// - /// Used by the execution logic for - /// - /// - [`Op::ReturnMany`] - pub fn copy_many_return_values(&mut self, ip: InstructionPtr, values: &[Slot]) { - let (mut caller_sp, results) = self.return_caller_results(); - let mut result = results.head(); - let mut copy_results = |values: &[Slot]| { - for value in values { - let value = self.get_stack_slot(*value); - // Safety: The `callee.results()` always refer to a span of valid - // registers of the `caller` that does not overlap with the - // registers of the callee since they reside in different - // call frames. Therefore this access is safe. - unsafe { caller_sp.set(result, value) } - result = result.next(); - } - }; - copy_results(values); - let mut ip = ip; - while let Op::SlotList { regs } = ip.get() { - copy_results(regs); - ip.add(1); - } - let values = match ip.get() { - Op::Slot { slot } => slice::from_ref(slot), - Op::Slot2 { slots } => slots, - Op::Slot3 { slots } => slots, - unexpected => { - // Safety: Wasmi translation guarantees that a slot-list finalizer exists. - unsafe { - unreachable_unchecked!( - "unexpected slot-list finalizer but found: {unexpected:?}" - ) - } - } - }; - copy_results(values); - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/select.rs b/crates/wasmi/src/engine/executor/instrs/select.rs deleted file mode 100644 index a578f640d1..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/select.rs +++ /dev/null @@ -1,140 +0,0 @@ -use super::{Executor, InstructionPtr, UntypedValueExt}; -use crate::{ - core::{wasm, ReadAs, UntypedVal}, - engine::utils::unreachable_unchecked, - ir::{Op, Slot}, -}; - -impl Executor<'_> { - /// Fetches two [`Slot`]s. - fn fetch_register_2(&self) -> (Slot, Slot) { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Slot2 { - slots: [slot0, slot1], - } => (slot0, slot1), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::Slot2`] exists. - unsafe { unreachable_unchecked!("expected `Op::Slot2` but found {unexpected:?}") } - } - } - } - - /// Executes a fused `cmp`+`select` instruction. - #[inline(always)] - fn execute_cmp_select_impl( - &mut self, - result: Slot, - lhs: Slot, - rhs: Slot, - f: fn(T, T) -> bool, - ) where - UntypedVal: ReadAs, - { - let (true_val, false_val) = self.fetch_register_2(); - let lhs: T = self.get_stack_slot_as(lhs); - let rhs: T = self.get_stack_slot_as(rhs); - let selected = self.get_stack_slot(match f(lhs, rhs) { - true => true_val, - false => false_val, - }); - self.set_stack_slot(result, selected); - self.next_instr_at(2); - } - - /// Executes a fused `cmp`+`select` instruction with immediate `rhs` parameter. - #[inline(always)] - fn execute_cmp_select_imm_rhs_impl( - &mut self, - result: Slot, - lhs: Slot, - rhs: Const16, - f: fn(T, T) -> bool, - ) where - UntypedVal: ReadAs, - T: From>, - { - let (true_val, false_val) = self.fetch_register_2(); - let lhs: T = self.get_stack_slot_as(lhs); - let rhs: T = rhs.into(); - let selected = self.get_stack_slot(match f(lhs, rhs) { - true => true_val, - false => false_val, - }); - self.set_stack_slot(result, selected); - self.next_instr_at(2); - } -} - -macro_rules! impl_cmp_select_for { - ( - $( - (Op::$doc_name:ident, $fn_name:ident, $op:expr) - ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($doc_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Slot) { - self.execute_cmp_select_impl(result, lhs, rhs, $op) - } - )* - }; -} - -macro_rules! impl_cmp_select_imm_rhs_for { - ( - $( - ($ty:ty, Op::$doc_name:ident, $fn_name:ident, $op:expr) - ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($doc_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Const16<$ty>) { - self.execute_cmp_select_imm_rhs_impl::<$ty>(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - impl_cmp_select_for! { - (Op::SelectI32Eq, execute_select_i32_eq, wasm::i32_eq), - (Op::SelectI32LtS, execute_select_i32_lt_s, wasm::i32_lt_s), - (Op::SelectI32LtU, execute_select_i32_lt_u, wasm::i32_lt_u), - (Op::SelectI32LeS, execute_select_i32_le_s, wasm::i32_le_s), - (Op::SelectI32LeU, execute_select_i32_le_u, wasm::i32_le_u), - (Op::SelectI32And, execute_select_i32_and, ::and), - (Op::SelectI32Or, execute_select_i32_or, ::or), - (Op::SelectI64Eq, execute_select_i64_eq, wasm::i64_eq), - (Op::SelectI64LtS, execute_select_i64_lt_s, wasm::i64_lt_s), - (Op::SelectI64LtU, execute_select_i64_lt_u, wasm::i64_lt_u), - (Op::SelectI64LeS, execute_select_i64_le_s, wasm::i64_le_s), - (Op::SelectI64LeU, execute_select_i64_le_u, wasm::i64_le_u), - (Op::SelectI64And, execute_select_i64_and, ::and), - (Op::SelectI64Or, execute_select_i64_or, ::or), - (Op::SelectF32Eq, execute_select_f32_eq, wasm::f32_eq), - (Op::SelectF32Lt, execute_select_f32_lt, wasm::f32_lt), - (Op::SelectF32Le, execute_select_f32_le, wasm::f32_le), - (Op::SelectF64Eq, execute_select_f64_eq, wasm::f64_eq), - (Op::SelectF64Lt, execute_select_f64_lt, wasm::f64_lt), - (Op::SelectF64Le, execute_select_f64_le, wasm::f64_le), - } - - impl_cmp_select_imm_rhs_for! { - (i32, Op::SelectI32EqImm16, execute_select_i32_eq_imm16, wasm::i32_eq), - (i32, Op::SelectI32LtSImm16Rhs, execute_select_i32_lt_s_imm16_rhs, wasm::i32_lt_s), - (u32, Op::SelectI32LtUImm16Rhs, execute_select_i32_lt_u_imm16_rhs, wasm::i32_lt_u), - (i32, Op::SelectI32LeSImm16Rhs, execute_select_i32_le_s_imm16_rhs, wasm::i32_le_s), - (u32, Op::SelectI32LeUImm16Rhs, execute_select_i32_le_u_imm16_rhs, wasm::i32_le_u), - (i32, Op::SelectI32AndImm16, execute_select_i32_and_imm16, UntypedValueExt::and), - (i32, Op::SelectI32OrImm16, execute_select_i32_or_imm16, UntypedValueExt::or), - (i64, Op::SelectI64EqImm16, execute_select_i64_eq_imm16, wasm::i64_eq), - (i64, Op::SelectI64LtSImm16Rhs, execute_select_i64_lt_s_imm16_rhs, wasm::i64_lt_s), - (u64, Op::SelectI64LtUImm16Rhs, execute_select_i64_lt_u_imm16_rhs, wasm::i64_lt_u), - (i64, Op::SelectI64LeSImm16Rhs, execute_select_i64_le_s_imm16_rhs, wasm::i64_le_s), - (u64, Op::SelectI64LeUImm16Rhs, execute_select_i64_le_u_imm16_rhs, wasm::i64_le_u), - (i64, Op::SelectI64AndImm16, execute_select_i64_and_imm16, UntypedValueExt::and), - (i64, Op::SelectI64OrImm16, execute_select_i64_or_imm16, UntypedValueExt::or), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/simd.rs b/crates/wasmi/src/engine/executor/instrs/simd.rs deleted file mode 100644 index 1ab02d8c27..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/simd.rs +++ /dev/null @@ -1,814 +0,0 @@ -use super::Executor; -use crate::{ - core::{ - simd::{ - self, - ImmLaneIdx16, - ImmLaneIdx2, - ImmLaneIdx32, - ImmLaneIdx4, - ImmLaneIdx8, - IntoLaneIdx, - }, - UntypedVal, - WriteAs, - }, - engine::{executor::InstructionPtr, utils::unreachable_unchecked}, - ir::{index, Op, Slot}, - store::StoreInner, - Error, - TrapCode, - V128, -}; - -#[cfg(doc)] -use crate::ir::Offset64Hi; - -impl Executor<'_> { - /// Fetches a [`Slot`] from an [`Op::Slot`] instruction parameter. - fn fetch_register(&self) -> Slot { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Slot { slot } => slot, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::Slot`] exists. - unsafe { unreachable_unchecked!("expected `Op::Slot` but found {unexpected:?}") } - } - } - } - - /// Fetches the [`Slot`] and [`Offset64Hi`] parameters for a load or store [`Op`]. - unsafe fn fetch_reg_and_lane(&self, delta: usize) -> (Slot, LaneType) - where - LaneType: TryFrom, - { - let mut addr: InstructionPtr = self.ip; - addr.add(delta); - match addr.get().filter_register_and_lane::() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::SlotAndImm32` but found: {instr:?}") - }, - } - } - - /// Returns the register `value` and `lane` parameters for a `load` [`Op`]. - pub fn fetch_value_and_lane(&self, delta: usize) -> (Slot, LaneType) - where - LaneType: TryFrom, - { - // Safety: Wasmi translation guarantees that `Op::SlotAndImm32` exists. - unsafe { self.fetch_reg_and_lane::(delta) } - } - - /// Fetches a [`Slot`] from an [`Op::Const32`] instruction parameter. - fn fetch_const32_as(&self) -> T - where - T: From, - { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Const32 { value } => value.into(), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::Const32`] exists. - unsafe { unreachable_unchecked!("expected `Op::Const32` but found {unexpected:?}") } - } - } - } - - /// Fetches a [`Slot`] from an [`Op::I64Const32`] instruction parameter. - fn fetch_i64const32(&self) -> i64 { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::I64Const32 { value } => value.into(), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::I64Const32`] exists. - unsafe { - unreachable_unchecked!("expected `Op::I64Const32` but found {unexpected:?}") - } - } - } - } - - /// Fetches a [`Slot`] from an [`Op::F64Const32`] instruction parameter. - fn fetch_f64const32(&self) -> f64 { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::F64Const32 { value } => value.into(), - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::F64Const32`] exists. - unsafe { - unreachable_unchecked!("expected `Op::F64Const32` but found {unexpected:?}") - } - } - } - } - - /// Executes an [`Op::I8x16Shuffle`] instruction. - pub fn execute_i8x16_shuffle(&mut self, result: Slot, lhs: Slot, rhs: Slot) { - let selector = self.fetch_register(); - let lhs = self.get_stack_slot_as::(lhs); - let rhs = self.get_stack_slot_as::(rhs); - let selector = self - .get_stack_slot_as::(selector) - .as_u128() - .to_ne_bytes() - .map(|lane| { - match ImmLaneIdx32::try_from(lane) { - Ok(lane) => lane, - Err(_) => { - // Safety: Wasmi translation guarantees that the indices are within bounds. - unsafe { unreachable_unchecked!("unexpected out of bounds index: {lane}") } - } - } - }); - self.set_stack_slot_as::(result, simd::i8x16_shuffle(lhs, rhs, selector)); - self.next_instr_at(2); - } -} - -macro_rules! impl_ternary_simd_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr $(,)?) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, a: Slot, b: Slot) { - let c = self.fetch_register(); - let a = self.get_stack_slot_as::(a); - let b = self.get_stack_slot_as::(b); - let c = self.get_stack_slot_as::(c); - self.set_stack_slot_as::(result, $op(a, b, c)); - self.next_instr_at(2); - } - )* - }; -} - -impl Executor<'_> { - impl_ternary_simd_executors! { - (Op::V128Bitselect, execute_v128_bitselect, simd::v128_bitselect), - ( - Op::I32x4RelaxedDotI8x16I7x16AddS, - execute_i32x4_relaxed_dot_i8x16_i7x16_add_s, - simd::i32x4_relaxed_dot_i8x16_i7x16_add_s, - ), - (Op::F32x4RelaxedMadd, execute_f32x4_relaxed_madd, simd::f32x4_relaxed_madd), - (Op::F32x4RelaxedNmadd, execute_f32x4_relaxed_nmadd, simd::f32x4_relaxed_nmadd), - (Op::F64x2RelaxedMadd, execute_f64x2_relaxed_madd, simd::f64x2_relaxed_madd), - (Op::F64x2RelaxedNmadd, execute_f64x2_relaxed_nmadd, simd::f64x2_relaxed_nmadd), - } -} - -macro_rules! impl_replace_lane_ops { - ( - $( - ($ty:ty, Op::$instr_name:ident, $exec_name:ident, $execute:expr) - ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($instr_name), "`].")] - pub fn $exec_name(&mut self, result: Slot, input: Slot, lane: <$ty as IntoLaneIdx>::LaneIdx) { - let value = self.fetch_register(); - let input = self.get_stack_slot_as::(input); - let value = self.get_stack_slot_as::<$ty>(value); - self.set_stack_slot_as::(result, $execute(input, lane, value)); - self.next_instr_at(2); - } - )* - }; -} - -impl Executor<'_> { - impl_replace_lane_ops! { - (i8, Op::I8x16ReplaceLane, execute_i8x16_replace_lane, simd::i8x16_replace_lane), - (i16, Op::I16x8ReplaceLane, execute_i16x8_replace_lane, simd::i16x8_replace_lane), - (i32, Op::I32x4ReplaceLane, execute_i32x4_replace_lane, simd::i32x4_replace_lane), - (i64, Op::I64x2ReplaceLane, execute_i64x2_replace_lane, simd::i64x2_replace_lane), - (f32, Op::F32x4ReplaceLane, execute_f32x4_replace_lane, simd::f32x4_replace_lane), - (f64, Op::F64x2ReplaceLane, execute_f64x2_replace_lane, simd::f64x2_replace_lane), - } - - /// Executes an [`Op::I8x16ReplaceLaneImm`] instruction. - pub fn execute_i8x16_replace_lane_imm( - &mut self, - result: Slot, - input: Slot, - lane: ImmLaneIdx16, - value: i8, - ) { - self.execute_replace_lane_impl(result, input, lane, value, 1, simd::i8x16_replace_lane) - } - - /// Executes an [`Op::I16x8ReplaceLaneImm`] instruction. - pub fn execute_i16x8_replace_lane_imm(&mut self, result: Slot, input: Slot, lane: ImmLaneIdx8) { - let value = self.fetch_const32_as::() as i16; - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::i16x8_replace_lane) - } - - /// Executes an [`Op::I32x4ReplaceLaneImm`] instruction. - pub fn execute_i32x4_replace_lane_imm(&mut self, result: Slot, input: Slot, lane: ImmLaneIdx4) { - let value = self.fetch_const32_as::(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::i32x4_replace_lane) - } - - /// Executes an [`Op::I64x2ReplaceLaneImm32`] instruction. - pub fn execute_i64x2_replace_lane_imm32( - &mut self, - result: Slot, - input: Slot, - lane: ImmLaneIdx2, - ) { - let value = self.fetch_i64const32(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::i64x2_replace_lane) - } - - /// Executes an [`Op::F32x4ReplaceLaneImm`] instruction. - pub fn execute_f32x4_replace_lane_imm(&mut self, result: Slot, input: Slot, lane: ImmLaneIdx4) { - let value = self.fetch_const32_as::(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::f32x4_replace_lane) - } - - /// Executes an [`Op::F64x2ReplaceLaneImm32`] instruction. - pub fn execute_f64x2_replace_lane_imm32( - &mut self, - result: Slot, - input: Slot, - lane: ImmLaneIdx2, - ) { - let value = self.fetch_f64const32(); - self.execute_replace_lane_impl(result, input, lane, value, 2, simd::f64x2_replace_lane) - } - - /// Generically execute a SIMD replace lane instruction. - fn execute_replace_lane_impl( - &mut self, - result: Slot, - input: Slot, - lane: LaneType, - value: T, - delta: usize, - eval: fn(V128, LaneType, T) -> V128, - ) { - let input = self.get_stack_slot_as::(input); - self.set_stack_slot_as::(result, eval(input, lane, value)); - self.next_instr_at(delta); - } - - impl_unary_executors! { - (Op::V128AnyTrue, execute_v128_any_true, simd::v128_any_true), - (Op::I8x16AllTrue, execute_i8x16_all_true, simd::i8x16_all_true), - (Op::I8x16Bitmask, execute_i8x16_bitmask, simd::i8x16_bitmask), - (Op::I16x8AllTrue, execute_i16x8_all_true, simd::i16x8_all_true), - (Op::I16x8Bitmask, execute_i16x8_bitmask, simd::i16x8_bitmask), - (Op::I32x4AllTrue, execute_i32x4_all_true, simd::i32x4_all_true), - (Op::I32x4Bitmask, execute_i32x4_bitmask, simd::i32x4_bitmask), - (Op::I64x2AllTrue, execute_i64x2_all_true, simd::i64x2_all_true), - (Op::I64x2Bitmask, execute_i64x2_bitmask, simd::i64x2_bitmask), - - (Op::I8x16Neg, execute_i8x16_neg, simd::i8x16_neg), - (Op::I16x8Neg, execute_i16x8_neg, simd::i16x8_neg), - (Op::I16x8Neg, execute_i32x4_neg, simd::i32x4_neg), - (Op::I16x8Neg, execute_i64x2_neg, simd::i64x2_neg), - (Op::I16x8Neg, execute_f32x4_neg, simd::f32x4_neg), - (Op::I16x8Neg, execute_f64x2_neg, simd::f64x2_neg), - - (Op::I8x16Abs, execute_i8x16_abs, simd::i8x16_abs), - (Op::I16x8Abs, execute_i16x8_abs, simd::i16x8_abs), - (Op::I16x8Abs, execute_i32x4_abs, simd::i32x4_abs), - (Op::I16x8Abs, execute_i64x2_abs, simd::i64x2_abs), - (Op::I16x8Abs, execute_f32x4_abs, simd::f32x4_abs), - (Op::I16x8Abs, execute_f64x2_abs, simd::f64x2_abs), - - (Op::I8x16Splat, execute_i8x16_splat, simd::i8x16_splat), - (Op::I16x8Splat, execute_i16x8_splat, simd::i16x8_splat), - (Op::I32x4Splat, execute_i32x4_splat, simd::i32x4_splat), - (Op::I64x2Splat, execute_i64x2_splat, simd::i64x2_splat), - (Op::F32x4Splat, execute_f32x4_splat, simd::f32x4_splat), - (Op::F64x2Splat, execute_f64x2_splat, simd::f64x2_splat), - - (Op::I16x8ExtaddPairwiseI8x16S, execute_i16x8_extadd_pairwise_i8x16_s, simd::i16x8_extadd_pairwise_i8x16_s), - (Op::I16x8ExtaddPairwiseI8x16U, execute_i16x8_extadd_pairwise_i8x16_u, simd::i16x8_extadd_pairwise_i8x16_u), - (Op::I32x4ExtaddPairwiseI16x8S, execute_i32x4_extadd_pairwise_i16x8_s, simd::i32x4_extadd_pairwise_i16x8_s), - (Op::I32x4ExtaddPairwiseI16x8U, execute_i32x4_extadd_pairwise_i16x8_u, simd::i32x4_extadd_pairwise_i16x8_u), - - (Op::F32x4Ceil, execute_f32x4_ceil, simd::f32x4_ceil), - (Op::F32x4Floor, execute_f32x4_floor, simd::f32x4_floor), - (Op::F32x4Trunc, execute_f32x4_trunc, simd::f32x4_trunc), - (Op::F32x4Nearest, execute_f32x4_nearest, simd::f32x4_nearest), - (Op::F32x4Sqrt, execute_f32x4_sqrt, simd::f32x4_sqrt), - (Op::F64x2Ceil, execute_f64x2_ceil, simd::f64x2_ceil), - (Op::F64x2Floor, execute_f64x2_floor, simd::f64x2_floor), - (Op::F64x2Trunc, execute_f64x2_trunc, simd::f64x2_trunc), - (Op::F64x2Nearest, execute_f64x2_nearest, simd::f64x2_nearest), - (Op::F64x2Sqrt, execute_f64x2_sqrt, simd::f64x2_sqrt), - - (Op::V128Not, execute_v128_not, simd::v128_not), - (Op::I8x16Popcnt, execute_i8x16_popcnt, simd::i8x16_popcnt), - - (Op::i16x8_extend_low_i8x16_s, execute_i16x8_extend_low_i8x16_s, simd::i16x8_extend_low_i8x16_s), - (Op::i16x8_extend_high_i8x16_s, execute_i16x8_extend_high_i8x16_s, simd::i16x8_extend_high_i8x16_s), - (Op::i16x8_extend_low_i8x16_u, execute_i16x8_extend_low_i8x16_u, simd::i16x8_extend_low_i8x16_u), - (Op::i16x8_extend_high_i8x16_u, execute_i16x8_extend_high_i8x16_u, simd::i16x8_extend_high_i8x16_u), - (Op::i32x4_extend_low_i16x8_s, execute_i32x4_extend_low_i16x8_s, simd::i32x4_extend_low_i16x8_s), - (Op::i32x4_extend_high_i16x8_s, execute_i32x4_extend_high_i16x8_s, simd::i32x4_extend_high_i16x8_s), - (Op::i32x4_extend_low_i16x8_u, execute_i32x4_extend_low_i16x8_u, simd::i32x4_extend_low_i16x8_u), - (Op::i32x4_extend_high_i16x8_u, execute_i32x4_extend_high_i16x8_u, simd::i32x4_extend_high_i16x8_u), - (Op::i64x2_extend_low_i32x4_s, execute_i64x2_extend_low_i32x4_s, simd::i64x2_extend_low_i32x4_s), - (Op::i64x2_extend_high_i32x4_s, execute_i64x2_extend_high_i32x4_s, simd::i64x2_extend_high_i32x4_s), - (Op::i64x2_extend_low_i32x4_u, execute_i64x2_extend_low_i32x4_u, simd::i64x2_extend_low_i32x4_u), - (Op::i64x2_extend_high_i32x4_u, execute_i64x2_extend_high_i32x4_u, simd::i64x2_extend_high_i32x4_u), - - (Op::I32x4TruncSatF32x4S, execute_i32x4_trunc_sat_f32x4_s, simd::i32x4_trunc_sat_f32x4_s), - (Op::I32x4TruncSatF32x4U, execute_i32x4_trunc_sat_f32x4_u, simd::i32x4_trunc_sat_f32x4_u), - (Op::F32x4ConvertI32x4S, execute_f32x4_convert_i32x4_s, simd::f32x4_convert_i32x4_s), - (Op::F32x4ConvertI32x4U, execute_f32x4_convert_i32x4_u, simd::f32x4_convert_i32x4_u), - (Op::I32x4TruncSatF64x2SZero, execute_i32x4_trunc_sat_f64x2_s_zero, simd::i32x4_trunc_sat_f64x2_s_zero), - (Op::I32x4TruncSatF64x2UZero, execute_i32x4_trunc_sat_f64x2_u_zero, simd::i32x4_trunc_sat_f64x2_u_zero), - (Op::F64x2ConvertLowI32x4S, execute_f64x2_convert_low_i32x4_s, simd::f64x2_convert_low_i32x4_s), - (Op::F64x2ConvertLowI32x4U, execute_f64x2_convert_low_i32x4_u, simd::f64x2_convert_low_i32x4_u), - (Op::F32x4DemoteF64x2Zero, execute_f32x4_demote_f64x2_zero, simd::f32x4_demote_f64x2_zero), - (Op::F64x2PromoteLowF32x4, execute_f64x2_promote_low_f32x4, simd::f64x2_promote_low_f32x4), - } - - impl_binary_executors! { - (Op::I8x16Swizzle, execute_i8x16_swizzle, simd::i8x16_swizzle), - - (Op::I16x8Q15MulrSatS, execute_i16x8_q15mulr_sat_s, simd::i16x8_q15mulr_sat_s), - (Op::I32x4DotI16x8S, execute_i32x4_dot_i16x8_s, simd::i32x4_dot_i16x8_s), - (Op::I16x8RelaxedDotI8x16I7x16S, execute_i16x8_relaxed_dot_i8x16_i7x16_s, simd::i16x8_relaxed_dot_i8x16_i7x16_s), - - (Op::I16x8ExtmulLowI8x16S, execute_i16x8_extmul_low_i8x16_s, simd::i16x8_extmul_low_i8x16_s), - (Op::I16x8ExtmulHighI8x16S, execute_i16x8_extmul_high_i8x16_s, simd::i16x8_extmul_high_i8x16_s), - (Op::I16x8ExtmulLowI8x16U, execute_i16x8_extmul_low_i8x16_u, simd::i16x8_extmul_low_i8x16_u), - (Op::I16x8ExtmulHighI8x16U, execute_i16x8_extmul_high_i8x16_u, simd::i16x8_extmul_high_i8x16_u), - (Op::I32x4ExtmulLowI16x8S, execute_i32x4_extmul_low_i16x8_s, simd::i32x4_extmul_low_i16x8_s), - (Op::I32x4ExtmulHighI16x8S, execute_i32x4_extmul_high_i16x8_s, simd::i32x4_extmul_high_i16x8_s), - (Op::I32x4ExtmulLowI16x8U, execute_i32x4_extmul_low_i16x8_u, simd::i32x4_extmul_low_i16x8_u), - (Op::I32x4ExtmulHighI16x8U, execute_i32x4_extmul_high_i16x8_u, simd::i32x4_extmul_high_i16x8_u), - (Op::I64x2ExtmulLowI32x4S, execute_i64x2_extmul_low_i32x4_s, simd::i64x2_extmul_low_i32x4_s), - (Op::I64x2ExtmulHighI32x4S, execute_i64x2_extmul_high_i32x4_s, simd::i64x2_extmul_high_i32x4_s), - (Op::I64x2ExtmulLowI32x4U, execute_i64x2_extmul_low_i32x4_u, simd::i64x2_extmul_low_i32x4_u), - (Op::I64x2ExtmulHighI32x4U, execute_i64x2_extmul_high_i32x4_u, simd::i64x2_extmul_high_i32x4_u), - - (Op::I32x4Add, execute_i32x4_add, simd::i32x4_add), - (Op::I32x4Sub, execute_i32x4_sub, simd::i32x4_sub), - (Op::I32x4Mul, execute_i32x4_mul, simd::i32x4_mul), - - (Op::I64x2Add, execute_i64x2_add, simd::i64x2_add), - (Op::I64x2Sub, execute_i64x2_sub, simd::i64x2_sub), - (Op::I64x2Mul, execute_i64x2_mul, simd::i64x2_mul), - - (Op::I8x16Eq, execute_i8x16_eq, simd::i8x16_eq), - (Op::I8x16Ne, execute_i8x16_ne, simd::i8x16_ne), - (Op::I8x16LtS, execute_i8x16_lt_s, simd::i8x16_lt_s), - (Op::I8x16LtU, execute_i8x16_lt_u, simd::i8x16_lt_u), - (Op::I8x16LeS, execute_i8x16_le_s, simd::i8x16_le_s), - (Op::I8x16LeU, execute_i8x16_le_u, simd::i8x16_le_u), - (Op::I16x8Eq, execute_i16x8_eq, simd::i16x8_eq), - (Op::I16x8Ne, execute_i16x8_ne, simd::i16x8_ne), - (Op::I16x8LtS, execute_i16x8_lt_s, simd::i16x8_lt_s), - (Op::I16x8LtU, execute_i16x8_lt_u, simd::i16x8_lt_u), - (Op::I16x8LeS, execute_i16x8_le_s, simd::i16x8_le_s), - (Op::I16x8LeU, execute_i16x8_le_u, simd::i16x8_le_u), - (Op::I32x4Eq, execute_i32x4_eq, simd::i32x4_eq), - (Op::I32x4Ne, execute_i32x4_ne, simd::i32x4_ne), - (Op::I32x4LtS, execute_i32x4_lt_s, simd::i32x4_lt_s), - (Op::I32x4LtU, execute_i32x4_lt_u, simd::i32x4_lt_u), - (Op::I32x4LeS, execute_i32x4_le_s, simd::i32x4_le_s), - (Op::I32x4LeU, execute_i32x4_le_u, simd::i32x4_le_u), - (Op::I64x2Eq, execute_i64x2_eq, simd::i64x2_eq), - (Op::I64x2Ne, execute_i64x2_ne, simd::i64x2_ne), - (Op::I64x2LtS, execute_i64x2_lt_s, simd::i64x2_lt_s), - (Op::I64x2LeS, execute_i64x2_le_s, simd::i64x2_le_s), - (Op::F32x4Eq, execute_f32x4_eq, simd::f32x4_eq), - (Op::F32x4Ne, execute_f32x4_ne, simd::f32x4_ne), - (Op::F32x4Lt, execute_f32x4_lt, simd::f32x4_lt), - (Op::F32x4Le, execute_f32x4_le, simd::f32x4_le), - (Op::F64x2Eq, execute_f64x2_eq, simd::f64x2_eq), - (Op::F64x2Ne, execute_f64x2_ne, simd::f64x2_ne), - (Op::F64x2Lt, execute_f64x2_lt, simd::f64x2_lt), - (Op::F64x2Le, execute_f64x2_le, simd::f64x2_le), - - (Op::I8x16MinS, execute_i8x16_min_s, simd::i8x16_min_s), - (Op::I8x16MinU, execute_i8x16_min_u, simd::i8x16_min_u), - (Op::I8x16MaxS, execute_i8x16_max_s, simd::i8x16_max_s), - (Op::I8x16MaxU, execute_i8x16_max_u, simd::i8x16_max_u), - (Op::I8x16AvgrU, execute_i8x16_avgr_u, simd::i8x16_avgr_u), - (Op::I16x8MinS, execute_i16x8_min_s, simd::i16x8_min_s), - (Op::I16x8MinU, execute_i16x8_min_u, simd::i16x8_min_u), - (Op::I16x8MaxS, execute_i16x8_max_s, simd::i16x8_max_s), - (Op::I16x8MaxU, execute_i16x8_max_u, simd::i16x8_max_u), - (Op::I16x8AvgrU, execute_i16x8_avgr_u, simd::i16x8_avgr_u), - (Op::I32x4MinS, execute_i32x4_min_s, simd::i32x4_min_s), - (Op::I32x4MinU, execute_i32x4_min_u, simd::i32x4_min_u), - (Op::I32x4MaxS, execute_i32x4_max_s, simd::i32x4_max_s), - (Op::I32x4MaxU, execute_i32x4_max_u, simd::i32x4_max_u), - - (Op::I8x16Shl, execute_i8x16_shl, simd::i8x16_shl), - (Op::I8x16ShrS, execute_i8x16_shr_s, simd::i8x16_shr_s), - (Op::I8x16ShrU, execute_i8x16_shr_u, simd::i8x16_shr_u), - (Op::I16x8Shl, execute_i16x8_shl, simd::i16x8_shl), - (Op::I16x8ShrS, execute_i16x8_shr_s, simd::i16x8_shr_s), - (Op::I16x8ShrU, execute_i16x8_shr_u, simd::i16x8_shr_u), - (Op::I32x4Shl, execute_i32x4_shl, simd::i32x4_shl), - (Op::I32x4ShrS, execute_i32x4_shr_s, simd::i32x4_shr_s), - (Op::I32x4ShrU, execute_i32x4_shr_u, simd::i32x4_shr_u), - (Op::I64x2Shl, execute_i64x2_shl, simd::i64x2_shl), - (Op::I64x2ShrS, execute_i64x2_shr_s, simd::i64x2_shr_s), - (Op::I64x2ShrU, execute_i64x2_shr_u, simd::i64x2_shr_u), - - (Op::I8x16Add, execute_i8x16_add, simd::i8x16_add), - (Op::I8x16AddSatS, execute_i8x16_add_sat_s, simd::i8x16_add_sat_s), - (Op::I8x16AddSatU, execute_i8x16_add_sat_u, simd::i8x16_add_sat_u), - (Op::I8x16Sub, execute_i8x16_sub, simd::i8x16_sub), - (Op::I8x16SubSatS, execute_i8x16_sub_sat_s, simd::i8x16_sub_sat_s), - (Op::I8x16SubSatU, execute_i8x16_sub_sat_u, simd::i8x16_sub_sat_u), - - (Op::I16x8Add, execute_i16x8_add, simd::i16x8_add), - (Op::I16x8AddSatS, execute_i16x8_add_sat_s, simd::i16x8_add_sat_s), - (Op::I16x8AddSatU, execute_i16x8_add_sat_u, simd::i16x8_add_sat_u), - (Op::I16x8Sub, execute_i16x8_sub, simd::i16x8_sub), - (Op::I16x8SubSatS, execute_i16x8_sub_sat_s, simd::i16x8_sub_sat_s), - (Op::I16x8SubSatU, execute_i16x8_sub_sat_u, simd::i16x8_sub_sat_u), - (Op::I16x8Sub, execute_i16x8_mul, simd::i16x8_mul), - - (Op::V128And, execute_v128_and, simd::v128_and), - (Op::V128Andnot, execute_v128_andnot, simd::v128_andnot), - (Op::V128Or, execute_v128_or, simd::v128_or), - (Op::V128Xor, execute_v128_xor, simd::v128_xor), - - (Op::F32x4Add, execute_f32x4_add, simd::f32x4_add), - (Op::F32x4Sub, execute_f32x4_sub, simd::f32x4_sub), - (Op::F32x4Mul, execute_f32x4_mul, simd::f32x4_mul), - (Op::F32x4Div, execute_f32x4_div, simd::f32x4_div), - (Op::F32x4Min, execute_f32x4_min, simd::f32x4_min), - (Op::F32x4Max, execute_f32x4_max, simd::f32x4_max), - (Op::F32x4Pmin, execute_f32x4_pmin, simd::f32x4_pmin), - (Op::F32x4Pmax, execute_f32x4_pmax, simd::f32x4_pmax), - - (Op::F64x2Add, execute_f64x2_add, simd::f64x2_add), - (Op::F64x2Sub, execute_f64x2_sub, simd::f64x2_sub), - (Op::F64x2Mul, execute_f64x2_mul, simd::f64x2_mul), - (Op::F64x2Div, execute_f64x2_div, simd::f64x2_div), - (Op::F64x2Min, execute_f64x2_min, simd::f64x2_min), - (Op::F64x2Max, execute_f64x2_max, simd::f64x2_max), - (Op::F64x2Pmin, execute_f64x2_pmin, simd::f64x2_pmin), - (Op::F64x2Pmax, execute_f64x2_pmax, simd::f64x2_pmax), - - (Op::I8x16NarrowI16x8S, execute_i8x16_narrow_i16x8_s, simd::i8x16_narrow_i16x8_s), - (Op::I8x16NarrowI16x8U, execute_i8x16_narrow_i16x8_u, simd::i8x16_narrow_i16x8_u), - (Op::I16x8NarrowI32x4S, execute_i16x8_narrow_i32x4_s, simd::i16x8_narrow_i32x4_s), - (Op::I16x8NarrowI32x4U, execute_i16x8_narrow_i32x4_u, simd::i16x8_narrow_i32x4_u), - } -} - -impl Executor<'_> { - /// Executes a generic SIMD extract-lane [`Op`]. - #[inline(always)] - fn execute_extract_lane( - &mut self, - result: Slot, - input: Slot, - lane: Lane, - op: fn(V128, Lane) -> T, - ) where - UntypedVal: WriteAs, - { - let input = self.get_stack_slot_as::(input); - self.set_stack_slot_as::(result, op(input, lane)); - self.next_instr(); - } -} - -macro_rules! impl_extract_lane_executors { - ( - $( ($ty:ty, Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, input: Slot, lane: <$ty as IntoLaneIdx>::LaneIdx) { - self.execute_extract_lane(result, input, lane, $op) - } - )* - }; -} -impl Executor<'_> { - impl_extract_lane_executors! { - (i8, Op::I8x16ExtractLaneS, i8x16_extract_lane_s, simd::i8x16_extract_lane_s), - (u8, Op::I8x16ExtractLaneU, i8x16_extract_lane_u, simd::i8x16_extract_lane_u), - (i16, Op::I16x8ExtractLaneS, i16x8_extract_lane_s, simd::i16x8_extract_lane_s), - (u16, Op::I16x8ExtractLaneU, i16x8_extract_lane_u, simd::i16x8_extract_lane_u), - (i32, Op::I32x4ExtractLane, i32x4_extract_lane, simd::i32x4_extract_lane), - (u32, Op::F32x4ExtractLane, f32x4_extract_lane, simd::f32x4_extract_lane), - (i64, Op::I64x2ExtractLane, i64x2_extract_lane, simd::i64x2_extract_lane), - (u64, Op::F64x2ExtractLane, f64x2_extract_lane, simd::f64x2_extract_lane), - } -} - -impl Executor<'_> { - /// Generically execute a SIMD shift operation with immediate shift amount. - #[inline(always)] - fn execute_simd_shift_by( - &mut self, - result: Slot, - lhs: Slot, - rhs: ShiftAmount, - op: fn(V128, u32) -> V128, - ) { - let lhs = self.get_stack_slot_as::(lhs); - let rhs = rhs.into(); - self.set_stack_slot_as::(result, op(lhs, rhs)); - self.next_instr(); - } -} - -macro_rules! impl_simd_shift_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: ShiftAmount) { - self.execute_simd_shift_by(result, lhs, rhs, $op) - } - )* - }; -} -impl Executor<'_> { - impl_simd_shift_executors! { - (Op::I8x16ShlBy, execute_i8x16_shl_by, simd::i8x16_shl), - (Op::I8x16ShrSBy, execute_i8x16_shr_s_by, simd::i8x16_shr_s), - (Op::I8x16ShrUBy, execute_i8x16_shr_u_by, simd::i8x16_shr_u), - (Op::I16x8ShlBy, execute_i16x8_shl_by, simd::i16x8_shl), - (Op::I16x8ShrSBy, execute_i16x8_shr_s_by, simd::i16x8_shr_s), - (Op::I16x8ShrUBy, execute_i16x8_shr_u_by, simd::i16x8_shr_u), - (Op::I32x4ShlBy, execute_i32x4_shl_by, simd::i32x4_shl), - (Op::I32x4ShrSBy, execute_i32x4_shr_s_by, simd::i32x4_shr_s), - (Op::I32x4ShrUBy, execute_i32x4_shr_u_by, simd::i32x4_shr_u), - (Op::I64x2ShlBy, execute_i64x2_shl_by, simd::i64x2_shl), - (Op::I64x2ShrSBy, execute_i64x2_shr_s_by, simd::i64x2_shr_s), - (Op::I64x2ShrUBy, execute_i64x2_shr_u_by, simd::i64x2_shr_u), - } -} - -impl Executor<'_> { - /// Returns the optional `memory` parameter for a `load_at` [`Op`]. - /// - /// # Note - /// - /// - Returns the default [`index::Memory`] if the parameter is missing. - /// - Bumps `self.ip` if a [`Op::MemoryIndex`] parameter was found. - #[inline(always)] - fn fetch_lane_and_memory(&mut self, delta: usize) -> (LaneType, index::Memory) - where - LaneType: TryFrom, - { - let mut addr: InstructionPtr = self.ip; - addr.add(delta); - match addr.get().filter_lane_and_memory() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::Imm16AndImm32` but found: {instr:?}") - }, - } - } -} - -type V128LoadLane = - fn(memory: &[u8], ptr: u64, offset: u64, x: V128, lane: LaneType) -> Result; - -type V128LoadLaneAt = - fn(memory: &[u8], address: usize, x: V128, lane: LaneType) -> Result; - -macro_rules! impl_execute_v128_load_lane { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &StoreInner, - result: Slot, - offset_lo: Offset64Lo, - ) -> Result<(), Error> { - self.execute_v128_load_lane_impl::<<$ty as IntoLaneIdx>::LaneIdx>(store, result, offset_lo, $eval) - } - )* - }; -} - -macro_rules! impl_execute_v128_load_lane_at { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &StoreInner, - result: Slot, - address: Address32, - ) -> Result<(), Error> { - self.execute_v128_load_lane_at_impl::<<$ty as IntoLaneIdx>::LaneIdx>(store, result, address, $eval) - } - )* - }; -} - -impl Executor<'_> { - fn execute_v128_load_lane_impl( - &mut self, - store: &StoreInner, - result: Slot, - offset_lo: Offset64Lo, - load: V128LoadLane, - ) -> Result<(), Error> - where - LaneType: TryFrom + Into + Copy, - { - let (ptr, offset_hi) = self.fetch_value_and_offset_hi(); - let (v128, lane) = self.fetch_value_and_lane::(2); - let memory = self.fetch_optional_memory(3); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - let v128 = self.get_stack_slot_as::(v128); - let memory = self.fetch_memory_bytes(memory, store); - let loaded = load(memory, ptr, u64::from(offset), v128, lane)?; - self.set_stack_slot_as::(result, loaded); - self.try_next_instr_at(3) - } - - impl_execute_v128_load_lane! { - (u8, Op::V128Load8Lane, execute_v128_load8_lane, simd::v128_load8_lane), - (u16, Op::V128Load16Lane, execute_v128_load16_lane, simd::v128_load16_lane), - (u32, Op::V128Load32Lane, execute_v128_load32_lane, simd::v128_load32_lane), - (u64, Op::V128Load64Lane, execute_v128_load64_lane, simd::v128_load64_lane), - } - - fn execute_v128_load_lane_at_impl( - &mut self, - store: &StoreInner, - result: Slot, - address: Address32, - load_at: V128LoadLaneAt, - ) -> Result<(), Error> - where - LaneType: TryFrom + Into + Copy, - { - let (v128, lane) = self.fetch_value_and_lane::(1); - let memory = self.fetch_optional_memory(2); - let v128 = self.get_stack_slot_as::(v128); - let memory = self.fetch_memory_bytes(memory, store); - let loaded = load_at(memory, usize::from(address), v128, lane)?; - self.set_stack_slot_as::(result, loaded); - self.try_next_instr_at(2) - } - - impl_execute_v128_load_lane_at! { - (u8, Op::V128Load8LaneAt, execute_v128_load8_lane_at, simd::v128_load8_lane_at), - (u16, Op::V128Load16LaneAt, execute_v128_load16_lane_at, simd::v128_load16_lane_at), - (u32, Op::V128Load32LaneAt, execute_v128_load32_lane_at, simd::v128_load32_lane_at), - (u64, Op::V128Load64LaneAt, execute_v128_load64_lane_at, simd::v128_load64_lane_at), - } -} - -macro_rules! impl_execute_v128_store_lane { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - ) -> Result<(), Error> { - self.execute_v128_store_lane::<$ty>(store, ptr, offset_lo, $eval) - } - )* - }; -} - -macro_rules! impl_execute_v128_store_lane_offset16 { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - ptr: Slot, - value: Slot, - offset: Offset8, - lane: <$ty as IntoLaneIdx>::LaneIdx, - ) -> Result<(), Error> { - self.execute_v128_store_lane_offset8::<$ty>(ptr, value, offset, lane, $eval) - } - )* - }; -} - -macro_rules! impl_execute_v128_store_lane_at { - ( - $( ($ty:ty, Op::$op:ident, $exec:ident, $eval:expr) ),* $(,)? - ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($op), "`] instruction.")] - pub fn $exec( - &mut self, - store: &mut StoreInner, - value: Slot, - address: Address32, - ) -> Result<(), Error> { - self.execute_v128_store_lane_at::<$ty>(store, value, address, $eval) - } - )* - }; -} - -type V128StoreLane = fn( - memory: &mut [u8], - ptr: u64, - offset: u64, - value: V128, - lane: LaneType, -) -> Result<(), TrapCode>; - -type V128StoreLaneAt = - fn(memory: &mut [u8], address: usize, value: V128, lane: LaneType) -> Result<(), TrapCode>; - -impl Executor<'_> { - fn execute_v128_store_lane( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - eval: V128StoreLane, - ) -> Result<(), Error> { - let (value, offset_hi) = self.fetch_value_and_offset_hi(); - let (lane, memory) = self.fetch_lane_and_memory(2); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - let v128 = self.get_stack_slot_as::(value); - let memory = self.fetch_memory_bytes_mut(memory, store); - eval(memory, ptr, u64::from(offset), v128, lane)?; - self.try_next_instr_at(3) - } - - impl_execute_v128_store_lane! { - (u8, Op::V128Store8Lane, execute_v128_store8_lane, simd::v128_store8_lane), - (u16, Op::V128Store16Lane, execute_v128_store16_lane, simd::v128_store16_lane), - (u32, Op::V128Store32Lane, execute_v128_store32_lane, simd::v128_store32_lane), - (u64, Op::V128Store64Lane, execute_v128_store64_lane, simd::v128_store64_lane), - } - - fn execute_v128_store_lane_offset8( - &mut self, - ptr: Slot, - value: Slot, - offset: Offset8, - lane: T::LaneIdx, - eval: V128StoreLane, - ) -> Result<(), Error> { - let ptr = self.get_stack_slot_as::(ptr); - let offset = u64::from(Offset64::from(offset)); - let v128 = self.get_stack_slot_as::(value); - let memory = self.fetch_default_memory_bytes_mut(); - eval(memory, ptr, offset, v128, lane)?; - self.try_next_instr() - } - - impl_execute_v128_store_lane_offset16! { - (u8, Op::V128Store8LaneOffset8, execute_v128_store8_lane_offset8, simd::v128_store8_lane), - (u16, Op::V128Store16LaneOffset8, execute_v128_store16_lane_offset8, simd::v128_store16_lane), - (u32, Op::V128Store32LaneOffset8, execute_v128_store32_lane_offset8, simd::v128_store32_lane), - (u64, Op::V128Store64LaneOffset8, execute_v128_store64_lane_offset8, simd::v128_store64_lane), - } - - fn execute_v128_store_lane_at( - &mut self, - store: &mut StoreInner, - value: Slot, - address: Address32, - eval: V128StoreLaneAt, - ) -> Result<(), Error> - where - T::LaneIdx: TryFrom + Into, - { - let (lane, memory) = self.fetch_lane_and_memory::(1); - let v128 = self.get_stack_slot_as::(value); - let memory = self.fetch_memory_bytes_mut(memory, store); - eval(memory, usize::from(address), v128, lane)?; - self.try_next_instr_at(2) - } - - impl_execute_v128_store_lane_at! { - (u8, Op::V128Store8LaneAt, execute_v128_store8_lane_at, simd::v128_store8_lane_at), - (u16, Op::V128Store16LaneAt, execute_v128_store16_lane_at, simd::v128_store16_lane_at), - (u32, Op::V128Store32LaneAt, execute_v128_store32_lane_at, simd::v128_store32_lane_at), - (u64, Op::V128Store64LaneAt, execute_v128_store64_lane_at, simd::v128_store64_lane_at), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/store.rs b/crates/wasmi/src/engine/executor/instrs/store.rs deleted file mode 100644 index 3eb51684d7..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/store.rs +++ /dev/null @@ -1,470 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::{wasm, ReadAs, UntypedVal}, - engine::utils::unreachable_unchecked, - ir::{index::Memory, Offset16, Slot}, - store::StoreInner, - Error, - TrapCode, -}; - -#[cfg(feature = "simd")] -use crate::{core::simd, V128}; - -#[cfg(doc)] -use crate::ir::Op; - -/// The function signature of Wasm store operations. -type WasmStoreOp = - fn(memory: &mut [u8], address: u64, offset: u64, value: T) -> Result<(), TrapCode>; - -/// The function signature of Wasm store operations. -type WasmStoreAtOp = fn(memory: &mut [u8], address: usize, value: T) -> Result<(), TrapCode>; - -impl Executor<'_> { - /// Returns the immediate `value` and `offset_hi` parameters for a `load` [`Op`]. - fn fetch_value_and_offset_imm(&self) -> (T, Offset64Hi) - where - T: From, - { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match addr.get().filter_imm16_and_offset_hi::() { - Ok(value) => value, - Err(instr) => unsafe { - unreachable_unchecked!("expected an `Op::SlotAndImm32` but found: {instr:?}") - }, - } - } - - /// Executes a generic Wasm `store[N]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.store` - /// - `{i32, i64}.store8` - /// - `{i32, i64}.store16` - /// - `i64.store32` - pub(super) fn execute_store_wrap( - &mut self, - store: &mut StoreInner, - memory: Memory, - address: u64, - offset: Offset64, - value: T, - store_wrap: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_memory_bytes_mut(memory, store); - store_wrap(memory, address, u64::from(offset), value)?; - Ok(()) - } - - /// Executes a generic Wasm `store[N]` operation. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.store` - /// - `{i32, i64}.store8` - /// - `{i32, i64}.store16` - /// - `i64.store32` - fn execute_store_wrap_at( - &mut self, - store: &mut StoreInner, - memory: Memory, - address: Address32, - value: T, - store_wrap_at: WasmStoreAtOp, - ) -> Result<(), Error> { - let memory = self.fetch_memory_bytes_mut(memory, store); - store_wrap_at(memory, usize::from(address), value)?; - Ok(()) - } - - /// Executes a generic Wasm `store[N]` operation for the default memory. - /// - /// # Note - /// - /// This can be used to emulate the following Wasm operands: - /// - /// - `{i32, i64, f32, f64}.store` - /// - `{i32, i64}.store8` - /// - `{i32, i64}.store16` - /// - `i64.store32` - fn execute_store_wrap_mem0( - &mut self, - address: u64, - offset: Offset64, - value: T, - store_wrap: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_default_memory_bytes_mut(); - store_wrap(memory, address, u64::from(offset), value)?; - Ok(()) - } - - fn execute_store( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let (value, offset_hi) = self.fetch_value_and_offset_hi(); - let memory = self.fetch_optional_memory(2); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - let value = self.get_stack_slot_as::(value); - self.execute_store_wrap::(store, memory, ptr, offset, value, store_op)?; - self.try_next_instr_at(2) - } - - fn execute_store_imm( - &mut self, - store: &mut StoreInner, - ptr: Slot, - offset_lo: Offset64Lo, - offset_hi: Offset64Hi, - value: T, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_optional_memory(2); - let offset = Offset64::combine(offset_hi, offset_lo); - let ptr = self.get_stack_slot_as::(ptr); - self.execute_store_wrap::(store, memory, ptr, offset, value, store_op)?; - self.try_next_instr_at(2) - } - - fn execute_store_offset16( - &mut self, - ptr: Slot, - offset: Offset16, - value: Slot, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let ptr = self.get_stack_slot_as::(ptr); - let value = self.get_stack_slot_as::(value); - self.execute_store_wrap_mem0::(ptr, Offset64::from(offset), value, store_op)?; - self.try_next_instr() - } - - fn execute_store_offset16_imm16( - &mut self, - ptr: Slot, - offset: Offset16, - value: T, - store_op: WasmStoreOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let ptr = self.get_stack_slot_as::(ptr); - self.execute_store_wrap_mem0::(ptr, Offset64::from(offset), value, store_op)?; - self.try_next_instr() - } - - fn execute_store_at( - &mut self, - store: &mut StoreInner, - address: Address32, - value: Slot, - store_at_op: WasmStoreAtOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_optional_memory(1); - self.execute_store_wrap_at::( - store, - memory, - address, - self.get_stack_slot_as::(value), - store_at_op, - )?; - self.try_next_instr() - } - - fn execute_store_at_imm16( - &mut self, - store: &mut StoreInner, - address: Address32, - value: T, - store_at_op: WasmStoreAtOp, - ) -> Result<(), Error> - where - UntypedVal: ReadAs, - { - let memory = self.fetch_optional_memory(1); - self.execute_store_wrap_at::(store, memory, address, value, store_at_op)?; - self.try_next_instr() - } -} - -macro_rules! impl_execute_istore { - ( $( - ( - $ty:ty, - ($from_ty:ty => $to_ty:ty), - (Op::$var_store_imm:ident, $fn_store_imm:ident), - (Op::$var_store_off16_imm16:ident, $fn_store_off16_imm16:ident), - (Op::$var_store_at_imm16:ident, $fn_store_at_imm16:ident), - $store_fn:expr, - $store_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_store_imm), "`].")] - #[allow(clippy::cast_lossless)] - pub fn $fn_store_imm(&mut self, store: &mut StoreInner, ptr: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - let (value, offset_hi) = self.fetch_value_and_offset_imm::<$to_ty>(); - self.execute_store_imm::<$ty>(store, ptr, offset_lo, offset_hi, value as $ty, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_off16_imm16), "`].")] - #[allow(clippy::cast_lossless)] - pub fn $fn_store_off16_imm16( - &mut self, - ptr: Slot, - offset: Offset16, - value: $from_ty, - ) -> Result<(), Error> { - self.execute_store_offset16_imm16::<$ty>(ptr, offset, <$to_ty>::from(value) as _, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_at_imm16), "`].")] - #[allow(clippy::cast_lossless)] - pub fn $fn_store_at_imm16( - &mut self, - store: &mut StoreInner, - address: Address32, - value: $from_ty, - ) -> Result<(), Error> { - #[allow(clippy::cast_lossless)] - self.execute_store_at_imm16::<$ty>(store, address, <$to_ty>::from(value) as _, $store_at_fn) - } - )* - }; -} -impl Executor<'_> { - impl_execute_istore! { - ( - u32, - (Const16 => i32), - (Op::I32StoreImm16, execute_i32_store_imm16), - (Op::I32StoreOffset16Imm16, execute_i32_store_offset16_imm16), - (Op::I32StoreAtImm16, execute_i32_store_at_imm16), - wasm::store32, - wasm::store32_at, - ), - ( - u64, - (Const16 => i64), - (Op::I64StoreImm16, execute_i64_store_imm16), - (Op::I64StoreOffset16Imm16, execute_i64_store_offset16_imm16), - (Op::I64StoreAtImm16, execute_i64_store_at_imm16), - wasm::store64, - wasm::store64_at, - ), - } -} - -macro_rules! impl_execute_istore_trunc { - ( $( - ( - $ty:ty, - ($from_ty:ty => $to_ty:ty), - (Op::$var_store:ident, $fn_store:ident), - (Op::$var_store_imm:ident, $fn_store_imm:ident), - (Op::$var_store_off16:ident, $fn_store_off16:ident), - (Op::$var_store_off16_imm16:ident, $fn_store_off16_imm16:ident), - (Op::$var_store_at:ident, $fn_store_at:ident), - (Op::$var_store_at_imm16:ident, $fn_store_at_imm16:ident), - $store_fn:expr, - $store_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - impl_execute_istore! { - ( - $ty, - ($from_ty => $to_ty), - (Op::$var_store_imm, $fn_store_imm), - (Op::$var_store_off16_imm16, $fn_store_off16_imm16), - (Op::$var_store_at_imm16, $fn_store_at_imm16), - $store_fn, - $store_at_fn, - ) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store), "`].")] - pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - self.execute_store::<$ty>(store, ptr, offset_lo, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_off16), "`].")] - pub fn $fn_store_off16( - &mut self, - ptr: Slot, - offset: Offset16, - value: Slot, - ) -> Result<(), Error> { - self.execute_store_offset16::<$ty>(ptr, offset, value, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_at), "`].")] - pub fn $fn_store_at(&mut self, store: &mut StoreInner, address: Address32, value: Slot) -> Result<(), Error> { - self.execute_store_at::<$ty>(store, address, value, $store_at_fn) - } - )* - }; -} -impl Executor<'_> { - impl_execute_istore_trunc! { - ( - i32, - (i8 => i8), - (Op::I32Store8, execute_i32_store8), - (Op::I32Store8Imm, execute_i32_store8_imm), - (Op::I32Store8Offset16, execute_i32_store8_offset16), - (Op::I32Store8Offset16Imm, execute_i32_store8_offset16_imm), - (Op::I32Store8At, execute_i32_store8_at), - (Op::I32Store8AtImm, execute_i32_store8_at_imm), - wasm::i32_store8, - wasm::i32_store8_at, - ), - ( - i32, - (i16 => i16), - (Op::I32Store16, execute_i32_store16), - (Op::I32Store16Imm, execute_i32_store16_imm), - (Op::I32Store16Offset16, execute_i32_store16_offset16), - (Op::I32Store16Offset16Imm, execute_i32_store16_offset16_imm), - (Op::I32Store16At, execute_i32_store16_at), - (Op::I32Store16AtImm, execute_i32_store16_at_imm), - wasm::i32_store16, - wasm::i32_store16_at, - ), - ( - i64, - (i8 => i8), - (Op::I64Store8, execute_i64_store8), - (Op::I64Store8Imm, execute_i64_store8_imm), - (Op::I64Store8Offset16, execute_i64_store8_offset16), - (Op::I64Store8Offset16Imm, execute_i64_store8_offset16_imm), - (Op::I64Store8At, execute_i64_store8_at), - (Op::I64Store8AtImm, execute_i64_store8_at_imm), - wasm::i64_store8, - wasm::i64_store8_at, - ), - ( - i64, - (i16 => i16), - (Op::I64Store16, execute_i64_store16), - (Op::I64Store16Imm, execute_i64_store16_imm), - (Op::I64Store16Offset16, execute_i64_store16_offset16), - (Op::I64Store16Offset16Imm, execute_i64_store16_offset16_imm), - (Op::I64Store16At, execute_i64_store16_at), - (Op::I64Store16AtImm, execute_i64_store16_at_imm), - wasm::i64_store16, - wasm::i64_store16_at, - ), - ( - i64, - (Const16 => i32), - (Op::I64Store32, execute_i64_store32), - (Op::I64Store32Imm16, execute_i64_store32_imm16), - (Op::I64Store32Offset16, execute_i64_store32_offset16), - (Op::I64Store32Offset16Imm16, execute_i64_store32_offset16_imm16), - (Op::I64Store32At, execute_i64_store32_at), - (Op::I64Store32AtImm16, execute_i64_store32_at_imm16), - wasm::i64_store32, - wasm::i64_store32_at, - ), - } -} - -macro_rules! impl_execute_store { - ( $( - ( - $ty:ty, - (Op::$var_store:ident, $fn_store:ident), - (Op::$var_store_off16:ident, $fn_store_off16:ident), - (Op::$var_store_at:ident, $fn_store_at:ident), - $store_fn:expr, - $store_at_fn:expr $(,)? - ) - ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_store), "`].")] - pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Slot, offset_lo: Offset64Lo) -> Result<(), Error> { - self.execute_store::<$ty>(store, ptr, offset_lo, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_off16), "`].")] - pub fn $fn_store_off16( - &mut self, - ptr: Slot, - offset: Offset16, - value: Slot, - ) -> Result<(), Error> { - self.execute_store_offset16::<$ty>(ptr, offset, value, $store_fn) - } - - #[doc = concat!("Executes an [`Op::", stringify!($var_store_at), "`].")] - pub fn $fn_store_at(&mut self, store: &mut StoreInner, address: Address32, value: Slot) -> Result<(), Error> { - self.execute_store_at::<$ty>(store, address, value, $store_at_fn) - } - )* - } -} - -impl Executor<'_> { - #[cfg(feature = "simd")] - impl_execute_store! { - ( - V128, - (Op::V128Store, execute_v128_store), - (Op::V128StoreOffset16, execute_v128_store_offset16), - (Op::V128StoreAt, execute_v128_store_at), - simd::v128_store, - simd::v128_store_at, - ), - } - - impl_execute_store! { - ( - u32, - (Op::Store32, execute_store32), - (Op::Store32Offset16, execute_store32_offset16), - (Op::Store32At, execute_store32_at), - wasm::store32, - wasm::store32_at, - ), - ( - u64, - (Op::Store64, execute_store64), - (Op::Store64Offset16, execute_store64_offset16), - (Op::Store64At, execute_store64_at), - wasm::store64, - wasm::store64_at, - ), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/table.rs b/crates/wasmi/src/engine/executor/instrs/table.rs deleted file mode 100644 index 923d246043..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/table.rs +++ /dev/null @@ -1,255 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::CoreTable, - engine::{utils::unreachable_unchecked, ResumableOutOfFuelError}, - errors::TableError, - ir::{ - index::{Elem, Table}, - Op, - Slot, - }, - store::{PrunedStore, StoreError, StoreInner}, - Error, - TrapCode, -}; - -impl Executor<'_> { - /// Returns the [`Op::TableIndex`] parameter for an [`Op`]. - fn fetch_table_index(&self, offset: usize) -> Table { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::TableIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::TableIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::TableIndex` but found: {unexpected:?}") - } - } - } - } - - /// Returns the [`Op::ElemIndex`] parameter for an [`Op`]. - fn fetch_element_segment_index(&self, offset: usize) -> Elem { - let mut addr: InstructionPtr = self.ip; - addr.add(offset); - match *addr.get() { - Op::ElemIndex { index } => index, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::ElemIndex`] exists. - unsafe { - unreachable_unchecked!("expected `Op::ElemIndex` but found: {unexpected:?}") - } - } - } - } - - /// Executes an [`Op::TableGet`]. - pub fn execute_table_get( - &mut self, - store: &StoreInner, - result: Slot, - index: Slot, - ) -> Result<(), Error> { - let index: u64 = self.get_stack_slot_as(index); - self.execute_table_get_impl(store, result, index) - } - - /// Executes an [`Op::TableGetImm`]. - pub fn execute_table_get_imm( - &mut self, - store: &StoreInner, - result: Slot, - index: Const32, - ) -> Result<(), Error> { - let index: u64 = index.into(); - self.execute_table_get_impl(store, result, index) - } - - /// Executes a `table.get` instruction generically. - fn execute_table_get_impl( - &mut self, - store: &StoreInner, - result: Slot, - index: u64, - ) -> Result<(), Error> { - let table_index = self.fetch_table_index(1); - let table = self.get_table(table_index); - let value = store - .resolve_table(&table) - .get_untyped(index) - .ok_or(TrapCode::TableOutOfBounds)?; - self.set_stack_slot(result, value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::TableSize`]. - pub fn execute_table_size(&mut self, store: &StoreInner, result: Slot, table_index: Table) { - self.execute_table_size_impl(store, result, table_index); - self.next_instr(); - } - - /// Executes a generic `table.size` instruction. - fn execute_table_size_impl(&mut self, store: &StoreInner, result: Slot, table_index: Table) { - let table = self.get_table(table_index); - let size = store.resolve_table(&table).size(); - self.set_stack_slot(result, size); - } - - /// Executes an [`Op::TableSet`]. - pub fn execute_table_set( - &mut self, - store: &mut StoreInner, - index: Slot, - value: Slot, - ) -> Result<(), Error> { - let index: u64 = self.get_stack_slot_as(index); - self.execute_table_set_impl(store, index, value) - } - - /// Executes an [`Op::TableSetAt`]. - pub fn execute_table_set_at( - &mut self, - store: &mut StoreInner, - index: Const32, - value: Slot, - ) -> Result<(), Error> { - let index: u64 = index.into(); - self.execute_table_set_impl(store, index, value) - } - - /// Executes a generic `table.set` instruction. - fn execute_table_set_impl( - &mut self, - store: &mut StoreInner, - index: u64, - value: Slot, - ) -> Result<(), Error> { - let table_index = self.fetch_table_index(1); - let table = self.get_table(table_index); - let value = self.get_stack_slot(value); - store - .resolve_table_mut(&table) - .set_untyped(index, value) - .map_err(|_| TrapCode::TableOutOfBounds)?; - self.try_next_instr_at(2) - } - - /// Executes an [`Op::TableCopy`]. - pub fn execute_table_copy( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u64 = self.get_stack_slot_as(src); - let len: u64 = self.get_stack_slot_as(len); - let dst_table_index = self.fetch_table_index(1); - let src_table_index = self.fetch_table_index(2); - if dst_table_index == src_table_index { - // Case: copy within the same table - let table = self.get_table(dst_table_index); - let (table, fuel) = store.resolve_table_and_fuel_mut(&table); - table.copy_within(dst, src, len, Some(fuel))?; - } else { - // Case: copy between two different tables - let dst_table = self.get_table(dst_table_index); - let src_table = self.get_table(src_table_index); - // Copy from one table to another table: - let (dst_table, src_table, fuel) = - store.resolve_table_pair_and_fuel(&dst_table, &src_table); - CoreTable::copy(dst_table, dst, src_table, src, len, Some(fuel))?; - } - self.try_next_instr_at(3) - } - - /// Executes an [`Op::TableInit`]. - pub fn execute_table_init( - &mut self, - store: &mut StoreInner, - dst: Slot, - src: Slot, - len: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let src: u32 = self.get_stack_slot_as(src); - let len: u32 = self.get_stack_slot_as(len); - let table_index = self.fetch_table_index(1); - let element_index = self.fetch_element_segment_index(2); - let (table, element, fuel) = store.resolve_table_init_params( - &self.get_table(table_index), - &self.get_element_segment(element_index), - ); - table.init(element.as_ref(), dst, src, len, Some(fuel))?; - self.try_next_instr_at(3) - } - - /// Executes an [`Op::TableFill`]. - pub fn execute_table_fill( - &mut self, - store: &mut StoreInner, - dst: Slot, - len: Slot, - value: Slot, - ) -> Result<(), Error> { - let dst: u64 = self.get_stack_slot_as(dst); - let len: u64 = self.get_stack_slot_as(len); - let table_index = self.fetch_table_index(1); - let value = self.get_stack_slot(value); - let table = self.get_table(table_index); - let (table, fuel) = store.resolve_table_and_fuel_mut(&table); - table.fill_untyped(dst, value, len, Some(fuel))?; - self.try_next_instr_at(2) - } - - /// Executes an [`Op::TableGrow`]. - pub fn execute_table_grow( - &mut self, - store: &mut PrunedStore, - result: Slot, - delta: Slot, - value: Slot, - ) -> Result<(), Error> { - let delta: u64 = self.get_stack_slot_as(delta); - let table_index = self.fetch_table_index(1); - if delta == 0 { - // Case: growing by 0 elements means there is nothing to do - self.execute_table_size_impl(store.inner(), result, table_index); - return self.try_next_instr_at(2); - } - let table = self.get_table(table_index); - let value = self.get_stack_slot(value); - let return_value = match store.grow_table(&table, delta, value) { - Ok(return_value) => return_value, - Err(StoreError::External( - TableError::GrowOutOfBounds | TableError::OutOfSystemMemory, - )) => { - let table_ty = store.inner().resolve_table(&table).ty(); - match table_ty.is_64() { - true => u64::MAX, - false => u64::from(u32::MAX), - } - } - Err(StoreError::External(TableError::OutOfFuel { required_fuel })) => { - return Err(Error::from(ResumableOutOfFuelError::new(required_fuel))) - } - Err(StoreError::External(TableError::ResourceLimiterDeniedAllocation)) => { - return Err(Error::from(TrapCode::GrowthOperationLimited)) - } - Err(error) => { - panic!("`memory.grow`: internal interpreter error: {error}") - } - }; - self.set_stack_slot(result, return_value); - self.try_next_instr_at(2) - } - - /// Executes an [`Op::ElemDrop`]. - pub fn execute_element_drop(&mut self, store: &mut StoreInner, segment_index: Elem) { - let segment = self.get_element_segment(segment_index); - store.resolve_element_mut(&segment).drop_items(); - self.next_instr(); - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/unary.rs b/crates/wasmi/src/engine/executor/instrs/unary.rs deleted file mode 100644 index 063db5142e..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/unary.rs +++ /dev/null @@ -1,33 +0,0 @@ -use super::Executor; -use crate::{core::wasm, ir::Slot}; - -#[cfg(doc)] -use crate::ir::Op; - -impl Executor<'_> { - impl_unary_executors! { - (Op::I32Clz, execute_i32_clz, wasm::i32_clz), - (Op::I32Ctz, execute_i32_ctz, wasm::i32_ctz), - (Op::I32Popcnt, execute_i32_popcnt, wasm::i32_popcnt), - - (Op::I64Clz, execute_i64_clz, wasm::i64_clz), - (Op::I64Ctz, execute_i64_ctz, wasm::i64_ctz), - (Op::I64Popcnt, execute_i64_popcnt, wasm::i64_popcnt), - - (Op::F32Abs, execute_f32_abs, wasm::f32_abs), - (Op::F32Neg, execute_f32_neg, wasm::f32_neg), - (Op::F32Ceil, execute_f32_ceil, wasm::f32_ceil), - (Op::F32Floor, execute_f32_floor, wasm::f32_floor), - (Op::F32Trunc, execute_f32_trunc, wasm::f32_trunc), - (Op::F32Nearest, execute_f32_nearest, wasm::f32_nearest), - (Op::F32Sqrt, execute_f32_sqrt, wasm::f32_sqrt), - - (Op::F64Abs, execute_f64_abs, wasm::f64_abs), - (Op::F64Neg, execute_f64_neg, wasm::f64_neg), - (Op::F64Ceil, execute_f64_ceil, wasm::f64_ceil), - (Op::F64Floor, execute_f64_floor, wasm::f64_floor), - (Op::F64Trunc, execute_f64_trunc, wasm::f64_trunc), - (Op::F64Nearest, execute_f64_nearest, wasm::f64_nearest), - (Op::F64Sqrt, execute_f64_sqrt, wasm::f64_sqrt), - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/utils.rs b/crates/wasmi/src/engine/executor/instrs/utils.rs deleted file mode 100644 index 9c81432186..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/utils.rs +++ /dev/null @@ -1,117 +0,0 @@ -use super::Executor; -use crate::{ - ir::{index::Memory, Slot}, - store::StoreInner, -}; - -#[cfg(doc)] -use crate::ir::Op; - -macro_rules! impl_unary_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, input: Slot) { - self.execute_unary(result, input, $op) - } - )* - }; -} - -macro_rules! impl_binary_executors { - ( $( (Op::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { - $( - #[doc = concat!("Executes an [`Op::", stringify!($var_name), "`].")] - pub fn $fn_name(&mut self, result: Slot, lhs: Slot, rhs: Slot) { - self.execute_binary(result, lhs, rhs, $op) - } - )* - }; -} - -impl Executor<'_> { - /// Returns the register `value` and `offset` parameters for a `load` [`Op`]. - pub fn fetch_value_and_offset_hi(&self) -> (Slot, Offset64Hi) { - // Safety: Wasmi translation guarantees that `Op::SlotAndImm32` exists. - unsafe { self.fetch_reg_and_offset_hi() } - } - - /// Fetches the bytes of the default memory at index 0. - pub fn fetch_default_memory_bytes(&self) -> &[u8] { - // Safety: the `self.cache.memory` pointer is always synchronized - // conservatively whenever it could have been invalidated. - unsafe { self.cache.memory.data() } - } - - /// Fetches the bytes of the given `memory`. - pub fn fetch_memory_bytes<'exec, 'store, 'bytes>( - &'exec self, - memory: Memory, - store: &'store StoreInner, - ) -> &'bytes [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - match memory.is_default() { - true => self.fetch_default_memory_bytes(), - false => self.fetch_non_default_memory_bytes(memory, store), - } - } - - /// Fetches the bytes of the given non-default `memory`. - #[cold] - pub fn fetch_non_default_memory_bytes<'exec, 'store, 'bytes>( - &'exec self, - memory: Memory, - store: &'store StoreInner, - ) -> &'bytes [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - let memory = self.get_memory(memory); - store.resolve_memory(&memory).data() - } - - /// Fetches the bytes of the default memory at index 0. - #[inline] - pub fn fetch_default_memory_bytes_mut(&mut self) -> &mut [u8] { - // Safety: the `self.cache.memory` pointer is always synchronized - // conservatively whenever it could have been invalidated. - unsafe { self.cache.memory.data_mut() } - } - - /// Fetches the bytes of the given `memory`. - #[inline] - pub fn fetch_memory_bytes_mut<'exec, 'store, 'bytes>( - &'exec mut self, - memory: Memory, - store: &'store mut StoreInner, - ) -> &'bytes mut [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - match memory.is_default() { - true => self.fetch_default_memory_bytes_mut(), - false => self.fetch_non_default_memory_bytes_mut(memory, store), - } - } - - /// Fetches the bytes of the given non-default `memory`. - #[cold] - #[inline] - pub fn fetch_non_default_memory_bytes_mut<'exec, 'store, 'bytes>( - &'exec mut self, - memory: Memory, - store: &'store mut StoreInner, - ) -> &'bytes mut [u8] - where - 'exec: 'bytes, - 'store: 'bytes, - { - let memory = self.get_memory(memory); - store.resolve_memory_mut(&memory).data_mut() - } -} diff --git a/crates/wasmi/src/engine/executor/instrs/wide_arithmetic.rs b/crates/wasmi/src/engine/executor/instrs/wide_arithmetic.rs deleted file mode 100644 index 6d39b556ba..0000000000 --- a/crates/wasmi/src/engine/executor/instrs/wide_arithmetic.rs +++ /dev/null @@ -1,95 +0,0 @@ -use super::{Executor, InstructionPtr}; -use crate::{ - core::wasm, - engine::utils::unreachable_unchecked, - ir::{FixedSlotSpan, Op, Slot}, -}; - -/// Parameters for the `i64.add128` and `i64.sub128` instructions. -struct Params128 { - /// The register storing the high 64-bit part of the `lhs` parameter value. - lhs_hi: Slot, - /// The register storing the low 64-bit part of the `rhs` parameter value. - rhs_lo: Slot, - /// The register storing the low 64-bit part of the `rhs` parameter value. - rhs_hi: Slot, -} - -/// Function signature for `i64.binop128` handlers. -type BinOp128Fn = fn(lhs_lo: i64, lhs_hi: i64, rhs_lo: i64, rhs_hi: i64) -> (i64, i64); - -/// Function signature for `i64.mul_wide_sx` handlers. -type I64MulWideFn = fn(lhs: i64, rhs: i64) -> (i64, i64); - -impl Executor<'_> { - /// Fetches the parameters required by the `i64.add128` and `i64.sub128` instructions. - fn fetch_params128(&self) -> Params128 { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Op::Slot3 { slots } => Params128 { - lhs_hi: slots[0], - rhs_lo: slots[1], - rhs_hi: slots[2], - }, - unexpected => { - // Safety: Wasmi translation guarantees that [`Op::MemoryIndex`] exists. - unsafe { unreachable_unchecked!("expected `Op::Slot3` but found: {unexpected:?}") } - } - } - } - - /// Executes a generic Wasm `i64.binop128` instruction. - fn execute_i64_binop128(&mut self, results: [Slot; 2], lhs_lo: Slot, binop: BinOp128Fn) { - let Params128 { - lhs_hi, - rhs_lo, - rhs_hi, - } = self.fetch_params128(); - let lhs_lo: i64 = self.get_stack_slot_as(lhs_lo); - let lhs_hi: i64 = self.get_stack_slot_as(lhs_hi); - let rhs_lo: i64 = self.get_stack_slot_as(rhs_lo); - let rhs_hi: i64 = self.get_stack_slot_as(rhs_hi); - let (result_lo, result_hi) = binop(lhs_lo, lhs_hi, rhs_lo, rhs_hi); - self.set_stack_slot(results[0], result_lo); - self.set_stack_slot(results[1], result_hi); - self.next_instr_at(2) - } - - /// Executes an [`Op::I64Add128`]. - pub fn execute_i64_add128(&mut self, results: [Slot; 2], lhs_lo: Slot) { - self.execute_i64_binop128(results, lhs_lo, wasm::i64_add128) - } - - /// Executes an [`Op::I64Sub128`]. - pub fn execute_i64_sub128(&mut self, results: [Slot; 2], lhs_lo: Slot) { - self.execute_i64_binop128(results, lhs_lo, wasm::i64_sub128) - } - - /// Executes a generic Wasm `i64.mul_wide_sx` instruction. - fn execute_i64_mul_wide_sx( - &mut self, - results: FixedSlotSpan<2>, - lhs: Slot, - rhs: Slot, - mul_wide: I64MulWideFn, - ) { - let lhs: i64 = self.get_stack_slot_as(lhs); - let rhs: i64 = self.get_stack_slot_as(rhs); - let (result_lo, result_hi) = mul_wide(lhs, rhs); - let results = results.to_array(); - self.set_stack_slot(results[0], result_lo); - self.set_stack_slot(results[1], result_hi); - self.next_instr() - } - - /// Executes an [`Op::I64MulWideS`]. - pub fn execute_i64_mul_wide_s(&mut self, results: FixedSlotSpan<2>, lhs: Slot, rhs: Slot) { - self.execute_i64_mul_wide_sx(results, lhs, rhs, wasm::i64_mul_wide_s) - } - - /// Executes an [`Op::I64MulWideU`]. - pub fn execute_i64_mul_wide_u(&mut self, results: FixedSlotSpan<2>, lhs: Slot, rhs: Slot) { - self.execute_i64_mul_wide_sx(results, lhs, rhs, wasm::i64_mul_wide_u) - } -} diff --git a/crates/wasmi/src/engine/executor/mod.rs b/crates/wasmi/src/engine/executor/mod.rs index 257cf04f45..b14d25e9f3 100644 --- a/crates/wasmi/src/engine/executor/mod.rs +++ b/crates/wasmi/src/engine/executor/mod.rs @@ -1,9 +1,7 @@ +#![expect(dead_code)] + pub(crate) use self::stack::Stack; -use self::{ - instr_ptr::InstructionPtr, - instrs::{dispatch_host_func, execute_instrs}, - stack::CallFrame, -}; +use self::{instr_ptr::InstructionPtr, instrs::execute_instrs, stack::CallFrame}; use crate::{ engine::{ CallParams, @@ -15,7 +13,6 @@ use crate::{ }, func::HostFuncEntity, ir::{Slot, SlotSpan}, - store::CallHooks, CallHook, Error, Func, @@ -371,17 +368,18 @@ impl<'engine> EngineExecutor<'engine> { #[inline(always)] fn dispatch_host_func( &mut self, - store: &mut Store, - host_func: HostFuncEntity, + _store: &mut Store, + _host_func: HostFuncEntity, ) -> Result<(), Error> { - dispatch_host_func( - store.prune(), - &mut self.stack.values, - host_func, - None, - CallHooks::Ignore, - )?; - Ok(()) + // dispatch_host_func( + // store.prune(), + // &mut self.stack.values, + // host_func, + // None, + // CallHooks::Ignore, + // )?; + // Ok(()) + todo!() } /// Writes the results of the function execution back into the `results` buffer. diff --git a/crates/wasmi/src/engine/executor/stack/mod.rs b/crates/wasmi/src/engine/executor/stack/mod.rs index d74a268f2f..4bbb9bc96b 100644 --- a/crates/wasmi/src/engine/executor/stack/mod.rs +++ b/crates/wasmi/src/engine/executor/stack/mod.rs @@ -3,7 +3,7 @@ mod values; pub use self::{ calls::{CallFrame, CallStack, StackOffsets}, - values::{BaseValueStackOffset, FrameParams, FrameSlots, FrameValueStackOffset, ValueStack}, + values::{BaseValueStackOffset, FrameSlots, FrameValueStackOffset, ValueStack}, }; use crate::{engine::StackConfig, Instance, TrapCode}; From 602a6e66f6ea7e85ab1f6f60032ae712f2c77a5d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 11:47:20 +0200 Subject: [PATCH 087/423] remove some no longer needed code --- crates/wasmi/src/engine/mod.rs | 1 - crates/wasmi/src/engine/resumable.rs | 1 + .../wasmi/src/engine/translator/func/mod.rs | 76 ------------------- .../src/engine/translator/func/stack/mod.rs | 2 + crates/wasmi/src/engine/translator/utils.rs | 7 -- 5 files changed, 3 insertions(+), 84 deletions(-) diff --git a/crates/wasmi/src/engine/mod.rs b/crates/wasmi/src/engine/mod.rs index 4149a47c65..835da10acd 100644 --- a/crates/wasmi/src/engine/mod.rs +++ b/crates/wasmi/src/engine/mod.rs @@ -49,7 +49,6 @@ pub use self::{ }; use crate::{ collections::arena::{ArenaIndex, GuardedEntity}, - func::FuncInOut, module::{FuncIdx, ModuleHeader}, Error, Func, diff --git a/crates/wasmi/src/engine/resumable.rs b/crates/wasmi/src/engine/resumable.rs index 642e8d468e..989d81d76a 100644 --- a/crates/wasmi/src/engine/resumable.rs +++ b/crates/wasmi/src/engine/resumable.rs @@ -71,6 +71,7 @@ impl fmt::Display for ResumableHostTrapError { impl ResumableHostTrapError { /// Creates a new [`ResumableHostTrapError`]. #[cold] + #[expect(unused)] pub(crate) fn new(host_error: Error, host_func: Func, caller_results: SlotSpan) -> Self { Self { host_error, diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 4461721bfc..b2ef70baf6 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -313,36 +313,6 @@ impl FuncTranslator { .resolve_func_type(dedup_func_type, Clone::clone) } - /// Returns the [`SlotSpan`] of a call instruction before manipulating the operand stack. - fn call_regspan(&self, len_params: usize) -> Result { - let height = self.stack.height(); - let Some(start) = height.checked_sub(len_params) else { - panic!("operand stack underflow while evaluating call `SlotSpan`"); - }; - let start = self.layout.temp_to_reg(OperandIdx::from(start))?; - Ok(SlotSpan::new(start)) - } - - /// Push `results` as [`TempOperand`] onto the [`Stack`] tagged to `instr`. - /// - /// Returns the [`SlotSpan`] identifying the pushed operands if any. - fn push_results( - &mut self, - instr: Instr, - results: &[ValType], - ) -> Result, Error> { - let (first, rest) = match results.split_first() { - Some((first, rest)) => (first, rest), - None => return Ok(None), - }; - let first = self.stack.push_temp(*first, Some(instr))?; - for result in rest { - self.stack.push_temp(*result, Some(instr))?; - } - let start = self.layout.temp_to_reg(first)?; - Ok(Some(SlotSpan::new(start))) - } - /// Returns the [`Engine`] for which the function is compiled. fn engine(&self) -> &Engine { &self.engine @@ -826,12 +796,6 @@ impl FuncTranslator { self.push_instr_with_result(result_ty, |result| make_instr(result, lhs, rhs), fuel_costs) } - /// Pushes an instruction parameter `param` to the list of instructions. - fn push_param(&mut self, param: Op) -> Result<(), Error> { - self.instrs.push_param(param); - Ok(()) - } - /// Populate the `buffer` with the `table` targets including the `table` default target. /// /// Returns a shared slice to the `buffer` after it has been filled. @@ -969,22 +933,6 @@ impl FuncTranslator { self.operands.extend(self.stack.peek_n(len)); } - /// Encodes an [`Op::ReturnSpan`] for `len` values. - fn encode_return_many( - &mut self, - len: u16, - consume_fuel_instr: Option, - ) -> Result { - let values = self.move_operands_to_temp(usize::from(len), consume_fuel_instr)?; - let values = BoundedSlotSpan::new(values, len); - let return_instr = self.instrs.push_instr( - Op::return_span(values), - consume_fuel_instr, - FuelCostsProvider::base, - )?; - Ok(return_instr) - } - /// Tries to form a [`SlotSpan`] from the top-most `n` operands on the [`Stack`]. /// /// Returns `None` if forming a [`SlotSpan`] was not possible. @@ -1856,30 +1804,6 @@ impl FuncTranslator { } } - /// Translate a binary float Wasm operation. - fn translate_fbinary( - &mut self, - make_instr: fn(result: Slot, lhs: Slot, rhs: Slot) -> Op, - consteval: fn(T, T) -> R, - ) -> Result<(), Error> - where - T: WasmFloat, - R: Into + Typed, - { - bail_unreachable!(self); - let (lhs, rhs) = self.stack.pop2(); - if let (Operand::Immediate(lhs), Operand::Immediate(rhs)) = (lhs, rhs) { - return self.translate_binary_consteval::(lhs, rhs, consteval); - } - self.push_binary_instr_with_result( - ::TY, - lhs, - rhs, - make_instr, - FuelCostsProvider::base, - ) - } - /// Translate Wasmi `{f32,f64}.copysign` instructions. /// /// # Note diff --git a/crates/wasmi/src/engine/translator/func/stack/mod.rs b/crates/wasmi/src/engine/translator/func/stack/mod.rs index d42de743cf..5f1095bc04 100644 --- a/crates/wasmi/src/engine/translator/func/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func/stack/mod.rs @@ -1,3 +1,5 @@ +#![expect(unused)] + mod control; mod locals; mod operand; diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index eaeafb27d0..d36e0801c8 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -58,9 +58,6 @@ pub trait WasmInteger: /// Returns `true` if `self` is equal to zero (0). fn is_zero(self) -> bool; - - /// Returns the wrapped negated `self`. - fn wrapping_neg(self) -> Self; } macro_rules! impl_wasm_integer { @@ -76,10 +73,6 @@ macro_rules! impl_wasm_integer { fn is_zero(self) -> bool { self == 0 } - - fn wrapping_neg(self) -> Self { - Self::wrapping_neg(self) - } } )* }; From d008273f8dbd6b319dc96e8c7372f5f1100095bd Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:22:21 +0200 Subject: [PATCH 088/423] warning: allow -> expect --- crates/wasmi/src/engine/code_map.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/code_map.rs b/crates/wasmi/src/engine/code_map.rs index c970eecfcd..99490ffb4f 100644 --- a/crates/wasmi/src/engine/code_map.rs +++ b/crates/wasmi/src/engine/code_map.rs @@ -139,7 +139,7 @@ impl EngineFuncSpan { /// Returns the `u32` index of the [`EngineFunc`] in `self` if any. /// /// Returns `None` if `func` is not contained in `self`. - #[allow(unused)] + #[expect(unused)] pub fn position(&self, func: EngineFunc) -> Option { debug_assert!(self.start <= self.end); if func < self.start || func >= self.end { From 01ec29ce86f96b755269653483104d4da1f2fce1 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:22:40 +0200 Subject: [PATCH 089/423] generalize usage of StackLayout::{local,temp}_to_reg --- .../src/engine/translator/func/layout.rs | 59 ++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index e3c9d1d200..90d2065b47 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -1,9 +1,60 @@ use super::{LocalIdx, Operand, OperandIdx, Reset}; -use crate::{engine::TranslationError, ir::Slot, Error}; +use crate::{ + engine::{ + translator::func::{LocalOperand, TempOperand}, + TranslationError, + }, + ir::Slot, + Error, +}; #[cfg(doc)] use super::Stack; +/// Allows conversion from `Self` to [`LocalIdx`]. +/// +/// # Note +/// +/// This allows to use [`StackLayout::local_to_reg`] with [`LocalIdx`] and [`LocalOperand`]. +pub trait IntoLocalIdx { + /// Converts `self` into [`LocalIdx`]. + fn into_local_idx(self) -> LocalIdx; +} + +impl IntoLocalIdx for LocalIdx { + fn into_local_idx(self) -> LocalIdx { + self + } +} + +impl IntoLocalIdx for LocalOperand { + fn into_local_idx(self) -> LocalIdx { + self.local_index() + } +} + +/// Allows conversion from `Self` to [`OperandIdx`]. +/// +/// # Note +/// +/// This allows to use [`StackLayout::temp_to_reg`] with [`LocalIdx`] and [`TempOperand`]. +pub trait IntoOperandIdx { + /// Converts `self` into [`OperandIdx`]. + fn into_operand_idx(self) -> OperandIdx; +} + +impl IntoOperandIdx for OperandIdx { + fn into_operand_idx(self) -> OperandIdx { + self + } +} + +impl IntoOperandIdx for TempOperand { + fn into_operand_idx(self) -> OperandIdx { + self.operand_index() + } +} + /// The layout of the [`Stack`]. #[derive(Debug, Default)] pub struct StackLayout { @@ -66,7 +117,8 @@ impl StackLayout { /// /// If `index` cannot be converted into a [`Slot`]. #[inline] - pub fn local_to_reg(&self, index: LocalIdx) -> Result { + pub fn local_to_reg(&self, item: impl IntoLocalIdx) -> Result { + let index = item.into_local_idx(); debug_assert!( (u32::from(index) as usize) < self.len_locals, "out of bounds local operand index: {index:?}" @@ -83,7 +135,8 @@ impl StackLayout { /// /// If `index` cannot be converted into a [`Slot`]. #[inline] - pub fn temp_to_reg(&self, index: OperandIdx) -> Result { + pub fn temp_to_reg(&self, item: impl IntoOperandIdx) -> Result { + let index = item.into_operand_idx(); let index = usize::from(index); let Some(index) = index.checked_add(self.len_locals) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); From 15267c485921c34330235bb49e9609df36d281ea Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:23:01 +0200 Subject: [PATCH 090/423] add missing export of LocalOperand --- crates/wasmi/src/engine/translator/func/stack/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/stack/mod.rs b/crates/wasmi/src/engine/translator/func/stack/mod.rs index 5f1095bc04..c156feb25b 100644 --- a/crates/wasmi/src/engine/translator/func/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func/stack/mod.rs @@ -23,7 +23,7 @@ pub use self::{ IfReachability, LoopControlFrame, }, - operand::{ImmediateOperand, Operand, TempOperand}, + operand::{ImmediateOperand, LocalOperand, Operand, TempOperand}, operands::{OperandIdx, PreservedAllLocalsIter, PreservedLocalsIter}, }; use super::{Reset, ReusableAllocations}; From d7e96dd2db7247e9d0ae2054cf6976e585677d8d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:23:11 +0200 Subject: [PATCH 091/423] implement store ops translation --- .../wasmi/src/engine/translator/func/mod.rs | 113 +++++++++++++++++- crates/wasmi/src/engine/translator/func/op.rs | 3 +- 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index b2ef70baf6..4f13881e17 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -23,6 +23,7 @@ use self::{ IfControlFrame, IfReachability, ImmediateOperand, + LocalOperand, LoopControlFrame, Operand, OperandIdx, @@ -2104,18 +2105,122 @@ impl FuncTranslator { self.encode_store::(memarg, ptr, value) } + /// Encodes a Wasm store operator to Wasmi bytecode. fn encode_store( &mut self, memarg: MemArg, - _ptr: Operand, - _value: Operand, + ptr: Operand, + value: Operand, ) -> Result<(), Error> where T::Value: Copy + From, T::Immediate: Copy, { - let (_memory, _offset) = Self::decode_memarg(memarg)?; - todo!() + let (memory, offset) = Self::decode_memarg(memarg)?; + let ptr = match ptr { + Operand::Local(ptr) => self.layout.local_to_reg(ptr)?, + Operand::Temp(ptr) => self.layout.temp_to_reg(ptr)?, + Operand::Immediate(ptr) => { + return self.encode_store_ix::(ptr, offset, memory, value) + } + }; + if self.encode_store_mem0_offset16::(ptr, offset, memory, value)? { + return Ok(()); + } + let store_op = match value { + Operand::Local(value) => { + let value = self.layout.local_to_reg(value)?; + T::store_ss(ptr, offset, value, memory) + } + Operand::Temp(value) => { + let value = self.layout.temp_to_reg(value)?; + T::store_ss(ptr, offset, value, memory) + } + Operand::Immediate(value) => { + let value = ::from(value.val()); + let immediate = ::into_immediate(value); + T::store_si(ptr, offset, immediate, memory) + } + }; + self.push_instr(store_op, FuelCostsProvider::store)?; + Ok(()) + } + + /// Encodes a Wasm store operator with immediate `ptr` to Wasmi bytecode. + fn encode_store_ix( + &mut self, + ptr: ImmediateOperand, + offset: u64, + memory: index::Memory, + value: Operand, + ) -> Result<(), Error> + where + T::Value: Copy + From, + T::Immediate: Copy, + { + let Some(address) = self.effective_address(memory, ptr.val(), offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + let store_op = match value { + Operand::Local(value) => { + let value = self.layout.local_to_reg(value)?; + T::store_is(address, value, memory) + } + Operand::Temp(value) => { + let value = self.layout.temp_to_reg(value)?; + T::store_is(address, value, memory) + } + Operand::Immediate(value) => { + let value = ::from(value.val()); + let immediate = ::into_immediate(value); + T::store_ii(address, immediate, memory) + } + }; + self.push_instr(store_op, FuelCostsProvider::store)?; + return Ok(()); + } + + /// Encodes a Wasm store operator with `(mem 0)` and 16-bit encodable `offset` to Wasmi bytecode. + /// + /// # Note + /// + /// - Returns `Ok(true)` if encoding was successfull. + /// - Returns `Ok(false)` if encoding was unsuccessful. + /// - Returns `Err(_)` if an error occurred. + fn encode_store_mem0_offset16( + &mut self, + ptr: Slot, + offset: u64, + memory: index::Memory, + value: Operand, + ) -> Result + where + T::Value: Copy + From, + T::Immediate: Copy, + { + if !memory.is_default() { + return Ok(false); + } + let Ok(offset16) = Offset16::try_from(offset) else { + return Ok(false); + }; + let store_op = match value { + Operand::Local(value) => { + let value = self.layout.local_to_reg(value)?; + T::store_mem0_offset16_ss(ptr, offset16, value) + } + Operand::Temp(value) => { + let value = self.layout.temp_to_reg(value)?; + T::store_mem0_offset16_ss(ptr, offset16, value) + } + Operand::Immediate(value) => { + let value = ::from(value.val()); + let immediate = ::into_immediate(value); + T::store_mem0_offset16_si(ptr, offset16, immediate) + } + }; + self.push_instr(store_op, FuelCostsProvider::store)?; + Ok(true) } /// Returns the [`MemArg`] linear `memory` index and load/store `offset`. diff --git a/crates/wasmi/src/engine/translator/func/op.rs b/crates/wasmi/src/engine/translator/func/op.rs index 15a0354106..eb090f18ee 100644 --- a/crates/wasmi/src/engine/translator/func/op.rs +++ b/crates/wasmi/src/engine/translator/func/op.rs @@ -1,4 +1,5 @@ use crate::{ + core::Typed, engine::translator::utils::{ToBits, Wrap}, ir::{index::Memory, Address, Offset16, Op, Slot}, ValType, @@ -7,7 +8,7 @@ use crate::{ /// Trait implemented by all Wasm operators that can be translated as wrapping store instructions. pub trait StoreOperator { /// The type of the value to the stored. - type Value; + type Value: Typed; /// The type of immediate values. type Immediate; From 6a2d83b45befbb15776791faf467ea0a137452d0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:26:40 +0200 Subject: [PATCH 092/423] rename operand_to_reg -> operand_to_slot --- .../src/engine/translator/func/layout.rs | 2 +- .../wasmi/src/engine/translator/func/mod.rs | 34 +++++++++---------- .../src/engine/translator/func/simd/mod.rs | 14 ++++---- .../src/engine/translator/func/simd/visit.rs | 4 +-- .../wasmi/src/engine/translator/func/visit.rs | 4 +-- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index 90d2065b47..d55dba2e9d 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -103,7 +103,7 @@ impl StackLayout { /// # Errors /// /// If the forwarded method returned an error. - pub fn operand_to_reg(&mut self, operand: Operand) -> Result { + pub fn operand_to_slot(&mut self, operand: Operand) -> Result { match operand { Operand::Local(operand) => self.local_to_reg(operand.local_index()), Operand::Temp(operand) => self.temp_to_reg(operand.operand_index()), diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 4f13881e17..a67ade92b7 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -792,8 +792,8 @@ impl FuncTranslator { fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, ) -> Result<(), Error> { debug_assert_eq!(lhs.ty(), rhs.ty()); - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result(result_ty, |result| make_instr(result, lhs, rhs), fuel_costs) } @@ -1441,7 +1441,7 @@ impl FuncTranslator { self.stack.push_immediate(consteval(input.val().into()))?; return Ok(()); } - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; self.push_instr_with_result( ::TY, |result| make_instr(result, input), @@ -1473,7 +1473,7 @@ impl FuncTranslator { } return Ok(()); } - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; self.push_instr_with_result( ::TY, |result| make_instr(result, input), @@ -1642,7 +1642,7 @@ impl FuncTranslator { if opt_si(self, val, rhs)? { return Ok(()); } - let lhs = self.layout.operand_to_reg(val)?; + let lhs = self.layout.operand_to_slot(val)?; self.push_instr_with_result( ::TY, |result| make_ssi(result, lhs, rhs), @@ -1676,7 +1676,7 @@ impl FuncTranslator { self.translate_binary_consteval_fallible::(lhs, rhs, consteval) } (lhs, Operand::Immediate(rhs)) => { - let lhs = self.layout.operand_to_reg(lhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; let rhs = T::from(rhs.val()); let Some(non_zero_rhs) = ::non_zero(rhs) else { // Optimization: division by zero always traps @@ -1690,7 +1690,7 @@ impl FuncTranslator { } (Operand::Immediate(lhs), rhs) => { let lhs = T::from(lhs.val()); - let rhs = self.layout.operand_to_reg(rhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ::TY, |result| make_instr_sis(result, lhs, rhs), @@ -1725,7 +1725,7 @@ impl FuncTranslator { self.translate_binary_consteval::(lhs, rhs, consteval) } (lhs, Operand::Immediate(rhs)) => { - let lhs = self.layout.operand_to_reg(lhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; let rhs = T::from(rhs.val()); self.push_instr_with_result( ::TY, @@ -1735,7 +1735,7 @@ impl FuncTranslator { } (Operand::Immediate(lhs), rhs) => { let lhs = T::from(lhs.val()); - let rhs = self.layout.operand_to_reg(rhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ::TY, |result| make_instr_sis(result, lhs, rhs), @@ -1774,7 +1774,7 @@ impl FuncTranslator { self.stack.push_operand(lhs)?; return Ok(()); }; - let lhs = self.layout.operand_to_reg(lhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; self.push_instr_with_result( ::TY, |result| make_instr_ssi(result, lhs, rhs), @@ -1788,7 +1788,7 @@ impl FuncTranslator { self.stack.push_immediate(lhs)?; return Ok(()); } - let rhs = self.layout.operand_to_reg(rhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ::TY, |result| make_instr_sis(result, lhs, rhs), @@ -1827,7 +1827,7 @@ impl FuncTranslator { self.translate_binary_consteval::(lhs, rhs, consteval) } (lhs, Operand::Immediate(rhs)) => { - let lhs = self.layout.operand_to_reg(lhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; let sign = T::from(rhs.val()).sign(); self.push_instr_with_result( ::TY, @@ -1837,7 +1837,7 @@ impl FuncTranslator { } (Operand::Immediate(lhs), rhs) => { let lhs = T::from(lhs.val()); - let rhs = self.layout.operand_to_reg(rhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ::TY, |result| make_instr_sis(result, lhs, rhs), @@ -2337,7 +2337,7 @@ impl FuncTranslator { if self.try_opt_i64_mul_wide_sx(lhs, rhs_val, signed)? { return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; let rhs = self.copy_if_immediate(rhs)?; (lhs, rhs) } @@ -2347,12 +2347,12 @@ impl FuncTranslator { return Ok(()); } let lhs = self.copy_if_immediate(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; (lhs, rhs) } (lhs, rhs) => { - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; (lhs, rhs) } }; diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index bdbaf468ad..642509fe81 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -68,7 +68,7 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); }; - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; self.push_instr_with_result( ::TY, |result| make_instr(result, input, lane), @@ -94,7 +94,7 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; let value = self.make_input::(value, |_this, value| { Ok(Input::Immediate(T::into_immediate(T::Item::from(value)))) })?; @@ -126,7 +126,7 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); }; - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; self.push_instr_with_result( ::TY, |result| make_instr(result, input), @@ -149,8 +149,8 @@ impl FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ValType::V128, |result| make_instr(result, lhs, rhs), @@ -217,8 +217,8 @@ impl FuncTranslator { )?; return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ValType::V128, |result| make_instr_sss(result, lhs, rhs), diff --git a/crates/wasmi/src/engine/translator/func/simd/visit.rs b/crates/wasmi/src/engine/translator/func/simd/visit.rs index 03cd55b021..5fb56d6dae 100644 --- a/crates/wasmi/src/engine/translator/func/simd/visit.rs +++ b/crates/wasmi/src/engine/translator/func/simd/visit.rs @@ -205,8 +205,8 @@ impl VisitSimdOperator<'_> for FuncTranslator { self.stack.push_immediate(result)?; return Ok(()); } - let lhs = self.layout.operand_to_reg(lhs)?; - let rhs = self.layout.operand_to_reg(rhs)?; + let lhs = self.layout.operand_to_slot(lhs)?; + let rhs = self.layout.operand_to_slot(rhs)?; self.push_instr_with_result( ValType::V128, |result| Op::i8x16_shuffle(result, lhs, rhs, selector), diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index a5c191c4d8..a6e38e553f 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -317,7 +317,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let Ok(default_target) = usize::try_from(default_target) else { panic!("out of bounds `default_target` does not fit into `usize`: {default_target}"); }; - let index = self.layout.operand_to_reg(index)?; + let index = self.layout.operand_to_slot(index)?; let len_branch_params = self .stack .peek_control(default_target) @@ -423,7 +423,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { Operand::Immediate(input) => input.val(), input => { // Case: `global.set` with simple register input. - let input = self.layout.operand_to_reg(input)?; + let input = self.layout.operand_to_slot(input)?; self.push_instr(Op::global_set(global, input), FuelCostsProvider::instance)?; return Ok(()); } From 9512b1759505d0c72768b26b5a1238a7df63e3de Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:27:26 +0200 Subject: [PATCH 093/423] rename local_to_reg -> local_to_slot --- .../src/engine/translator/func/layout.rs | 8 ++-- .../wasmi/src/engine/translator/func/mod.rs | 40 +++++++++---------- .../src/engine/translator/func/simd/mod.rs | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index d55dba2e9d..832e5d89e9 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -15,7 +15,7 @@ use super::Stack; /// /// # Note /// -/// This allows to use [`StackLayout::local_to_reg`] with [`LocalIdx`] and [`LocalOperand`]. +/// This allows to use [`StackLayout::local_to_slot`] with [`LocalIdx`] and [`LocalOperand`]. pub trait IntoLocalIdx { /// Converts `self` into [`LocalIdx`]. fn into_local_idx(self) -> LocalIdx; @@ -97,7 +97,7 @@ impl StackLayout { /// /// Forwards to one of /// - /// - [`StackLayout::local_to_reg`] + /// - [`StackLayout::local_to_slot`] /// - [`StackLayout::temp_to_reg`] /// /// # Errors @@ -105,7 +105,7 @@ impl StackLayout { /// If the forwarded method returned an error. pub fn operand_to_slot(&mut self, operand: Operand) -> Result { match operand { - Operand::Local(operand) => self.local_to_reg(operand.local_index()), + Operand::Local(operand) => self.local_to_slot(operand.local_index()), Operand::Temp(operand) => self.temp_to_reg(operand.operand_index()), Operand::Immediate(_) => panic!("function local constants have been removed"), // TODO: remove } @@ -117,7 +117,7 @@ impl StackLayout { /// /// If `index` cannot be converted into a [`Slot`]. #[inline] - pub fn local_to_reg(&self, item: impl IntoLocalIdx) -> Result { + pub fn local_to_slot(&self, item: impl IntoLocalIdx) -> Result { let index = item.into_local_idx(); debug_assert!( (u32::from(index) as usize) < self.len_locals, diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index a67ade92b7..6595c5aabf 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -444,7 +444,7 @@ impl FuncTranslator { Op::copy(result, value) } Operand::Local(value) => { - let value = layout.local_to_reg(value.local_index())?; + let value = layout.local_to_slot(value.local_index())?; if result == value { // Case: no-op copy return Ok(None); @@ -557,7 +557,7 @@ impl FuncTranslator { let mut values = values; while let Some((value, rest)) = values.split_first() { let value = match value { - Operand::Local(value) => layout.local_to_reg(value.local_index())?, + Operand::Local(value) => layout.local_to_slot(value.local_index())?, Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, Operand::Immediate(_) => { // Immediate values will never yield no-op copies. @@ -589,7 +589,7 @@ impl FuncTranslator { let mut values = values; while let Some((value, rest)) = values.split_last() { let value = match value { - Operand::Local(value) => layout.local_to_reg(value.local_index())?, + Operand::Local(value) => layout.local_to_slot(value.local_index())?, Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, Operand::Immediate(_) => { // Immediate values will never yield no-op copies. @@ -632,7 +632,7 @@ impl FuncTranslator { // Note: We only have to check the register case since constant value // copies can never overlap. let value = match value { - Operand::Local(value) => layout.local_to_reg(value.local_index())?, + Operand::Local(value) => layout.local_to_slot(value.local_index())?, Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, Operand::Immediate(_) => { // Immediates are allocated as function local constants @@ -719,7 +719,7 @@ impl FuncTranslator { /// - Returns the associated [`Slot`] if `operand` is an [`Operand::Temp`] or [`Operand::Local`]. fn copy_if_immediate(&mut self, operand: Operand) -> Result { match operand { - Operand::Local(operand) => self.layout.local_to_reg(operand.local_index()), + Operand::Local(operand) => self.layout.local_to_slot(operand.local_index()), Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index()), Operand::Immediate(operand) => { let value = operand.val(); @@ -890,7 +890,7 @@ impl FuncTranslator { 0 => Op::Return {}, 1 => match self.stack.peek(0) { Operand::Local(operand) => { - let value = self.layout.local_to_reg(operand.local_index())?; + let value = self.layout.local_to_slot(operand.local_index())?; Op::return_slot(value) } Operand::Temp(operand) => { @@ -956,7 +956,7 @@ impl FuncTranslator { return Ok(None); }; let mut head = match head.as_ref() { - Operand::Local(start) => layout.local_to_reg(start.local_index())?, + Operand::Local(start) => layout.local_to_slot(start.local_index())?, Operand::Temp(start) => layout.temp_to_reg(start.operand_index())?, Operand::Immediate(_) => return Ok(None), }; @@ -964,7 +964,7 @@ impl FuncTranslator { for value in values { let cur = match value.as_ref() { Operand::Immediate(_) => return Ok(None), - Operand::Local(value) => layout.local_to_reg(value.local_index())?, + Operand::Local(value) => layout.local_to_slot(value.local_index())?, Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, }; if head != cur.prev() { @@ -1171,7 +1171,7 @@ impl FuncTranslator { let consume_fuel_instr = self.stack.consume_fuel_instr(); for preserved in self.stack.preserve_locals(local_idx) { let result = self.layout.temp_to_reg(preserved)?; - let value = self.layout.local_to_reg(local_idx)?; + let value = self.layout.local_to_slot(local_idx)?; self.instrs.push_instr( Op::copy(result, value), consume_fuel_instr, @@ -1194,7 +1194,7 @@ impl FuncTranslator { return Ok(()); } // At this point we need to encode a copy instruction. - let result = self.layout.local_to_reg(local_idx)?; + let result = self.layout.local_to_slot(local_idx)?; let outcome = self.encode_copy(result, input, consume_fuel_instr)?; debug_assert!( outcome.is_some(), @@ -1211,7 +1211,7 @@ impl FuncTranslator { new_result: LocalIdx, old_result: Operand, ) -> Result { - let result = self.layout.local_to_reg(new_result)?; + let result = self.layout.local_to_slot(new_result)?; let old_result = match old_result { Operand::Immediate(_) => { // Case: cannot replace immediate value result. @@ -1256,7 +1256,7 @@ impl FuncTranslator { return Ok(()); } let condition = match condition { - Operand::Local(condition) => self.layout.local_to_reg(condition.local_index())?, + Operand::Local(condition) => self.layout.local_to_slot(condition.local_index())?, Operand::Temp(condition) => self.layout.temp_to_reg(condition.operand_index())?, Operand::Immediate(condition) => { let condition = i32::from(condition.val()); @@ -1517,7 +1517,7 @@ impl FuncTranslator { f: impl FnOnce(&mut Self, TypedVal) -> Result, Error>, ) -> Result, Error> { let reg = match operand { - Operand::Local(operand) => self.layout.local_to_reg(operand.local_index())?, + Operand::Local(operand) => self.layout.local_to_slot(operand.local_index())?, Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, Operand::Immediate(operand) => return f(self, operand.val()), }; @@ -1538,7 +1538,7 @@ impl FuncTranslator { Operand::Immediate(value) => value.val(), Operand::Local(value) => { debug_assert_eq!(operand.ty(), index_type.ty()); - let reg = self.layout.local_to_reg(value.local_index())?; + let reg = self.layout.local_to_slot(value.local_index())?; return Ok(Input::Slot(reg)); } Operand::Temp(value) => { @@ -1914,7 +1914,7 @@ impl FuncTranslator { self.stack.push_operand(selected)?; return Ok(()); } - Operand::Local(condition) => self.layout.local_to_reg(condition.local_index())?, + Operand::Local(condition) => self.layout.local_to_slot(condition.local_index())?, Operand::Temp(condition) => self.layout.temp_to_reg(condition.operand_index())?, }; let true_val = self.copy_if_immediate(true_val)?; @@ -2043,7 +2043,7 @@ impl FuncTranslator { let (memory, offset) = Self::decode_memarg(memarg)?; let ptr = self.stack.pop(); let ptr = match ptr { - Operand::Local(ptr) => self.layout.local_to_reg(ptr.local_index())?, + Operand::Local(ptr) => self.layout.local_to_slot(ptr.local_index())?, Operand::Temp(ptr) => self.layout.temp_to_reg(ptr.operand_index())?, Operand::Immediate(ptr) => { let Some(address) = self.effective_address(memory, ptr.val(), offset) else { @@ -2118,7 +2118,7 @@ impl FuncTranslator { { let (memory, offset) = Self::decode_memarg(memarg)?; let ptr = match ptr { - Operand::Local(ptr) => self.layout.local_to_reg(ptr)?, + Operand::Local(ptr) => self.layout.local_to_slot(ptr)?, Operand::Temp(ptr) => self.layout.temp_to_reg(ptr)?, Operand::Immediate(ptr) => { return self.encode_store_ix::(ptr, offset, memory, value) @@ -2129,7 +2129,7 @@ impl FuncTranslator { } let store_op = match value { Operand::Local(value) => { - let value = self.layout.local_to_reg(value)?; + let value = self.layout.local_to_slot(value)?; T::store_ss(ptr, offset, value, memory) } Operand::Temp(value) => { @@ -2163,7 +2163,7 @@ impl FuncTranslator { }; let store_op = match value { Operand::Local(value) => { - let value = self.layout.local_to_reg(value)?; + let value = self.layout.local_to_slot(value)?; T::store_is(address, value, memory) } Operand::Temp(value) => { @@ -2206,7 +2206,7 @@ impl FuncTranslator { }; let store_op = match value { Operand::Local(value) => { - let value = self.layout.local_to_reg(value)?; + let value = self.layout.local_to_slot(value)?; T::store_mem0_offset16_ss(ptr, offset16, value) } Operand::Temp(value) => { diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 642509fe81..8beabbcab9 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -304,7 +304,7 @@ impl FuncTranslator { // lane value and translate as a more efficient non-SIMD operation. return translate_imm(self, memarg, ptr, lane, V128::from(v128.val())); } - Operand::Local(v128) => self.layout.local_to_reg(v128.local_index())?, + Operand::Local(v128) => self.layout.local_to_slot(v128.local_index())?, Operand::Temp(v128) => self.layout.temp_to_reg(v128.operand_index())?, }; let (memory, offset) = Self::decode_memarg(memarg)?; From 222a6e2fa685bd6459c41f941b9ddbdf970b0c1a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:28:13 +0200 Subject: [PATCH 094/423] rename temp_to_reg -> temp_to_slot --- .../src/engine/translator/func/instrs.rs | 2 +- .../src/engine/translator/func/layout.rs | 8 +-- .../wasmi/src/engine/translator/func/mod.rs | 68 +++++++++---------- .../src/engine/translator/func/simd/mod.rs | 2 +- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 539d7f7c36..3566d9b4b0 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -201,7 +201,7 @@ impl InstrEncoder { let CmpSelectFusion::Applied(fused) = last_instruction.try_into_cmp_select_instr(true_val, false_val, || { let select_result = stack.push_temp(ty, Some(last_instr))?; - let select_result = layout.temp_to_reg(select_result)?; + let select_result = layout.temp_to_slot(select_result)?; Ok(select_result) })? else { diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index 832e5d89e9..baa9a94106 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -37,7 +37,7 @@ impl IntoLocalIdx for LocalOperand { /// /// # Note /// -/// This allows to use [`StackLayout::temp_to_reg`] with [`LocalIdx`] and [`TempOperand`]. +/// This allows to use [`StackLayout::temp_to_slot`] with [`LocalIdx`] and [`TempOperand`]. pub trait IntoOperandIdx { /// Converts `self` into [`OperandIdx`]. fn into_operand_idx(self) -> OperandIdx; @@ -98,7 +98,7 @@ impl StackLayout { /// Forwards to one of /// /// - [`StackLayout::local_to_slot`] - /// - [`StackLayout::temp_to_reg`] + /// - [`StackLayout::temp_to_slot`] /// /// # Errors /// @@ -106,7 +106,7 @@ impl StackLayout { pub fn operand_to_slot(&mut self, operand: Operand) -> Result { match operand { Operand::Local(operand) => self.local_to_slot(operand.local_index()), - Operand::Temp(operand) => self.temp_to_reg(operand.operand_index()), + Operand::Temp(operand) => self.temp_to_slot(operand.operand_index()), Operand::Immediate(_) => panic!("function local constants have been removed"), // TODO: remove } } @@ -135,7 +135,7 @@ impl StackLayout { /// /// If `index` cannot be converted into a [`Slot`]. #[inline] - pub fn temp_to_reg(&self, item: impl IntoOperandIdx) -> Result { + pub fn temp_to_slot(&self, item: impl IntoOperandIdx) -> Result { let index = item.into_operand_idx(); let index = usize::from(index); let Some(index) = index.checked_add(self.len_locals) else { diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 6595c5aabf..4d0673ac5f 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -335,7 +335,7 @@ impl FuncTranslator { self.copy_operand_to_temp(operand, consume_fuel)?; } let first_idx = self.stack.peek(len).index(); - let first = self.layout.temp_to_reg(first_idx)?; + let first = self.layout.temp_to_slot(first_idx)?; Ok(SlotSpan::new(first)) } @@ -436,7 +436,7 @@ impl FuncTranslator { ) -> Result, Error> { let instr = match value { Operand::Temp(value) => { - let value = layout.temp_to_reg(value.operand_index())?; + let value = layout.temp_to_slot(value.operand_index())?; if result == value { // Case: no-op copy return Ok(None); @@ -558,7 +558,7 @@ impl FuncTranslator { while let Some((value, rest)) = values.split_first() { let value = match value { Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, + Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, Operand::Immediate(_) => { // Immediate values will never yield no-op copies. break; @@ -590,7 +590,7 @@ impl FuncTranslator { while let Some((value, rest)) = values.split_last() { let value = match value { Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, + Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, Operand::Immediate(_) => { // Immediate values will never yield no-op copies. break; @@ -633,7 +633,7 @@ impl FuncTranslator { // copies can never overlap. let value = match value { Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, + Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, Operand::Immediate(_) => { // Immediates are allocated as function local constants // which can not collide with the result registers. @@ -666,7 +666,7 @@ impl FuncTranslator { return Ok(None); } let height = frame.height(); - let start = layout.temp_to_reg(OperandIdx::from(height))?; + let start = layout.temp_to_slot(OperandIdx::from(height))?; let span = SlotSpan::new(start); Ok(Some(span)) } @@ -705,7 +705,7 @@ impl FuncTranslator { operand: Operand, consume_fuel: Option, ) -> Result { - let result = self.layout.temp_to_reg(operand.index())?; + let result = self.layout.temp_to_slot(operand.index())?; self.encode_copy(result, operand, consume_fuel)?; Ok(result) } @@ -720,10 +720,10 @@ impl FuncTranslator { fn copy_if_immediate(&mut self, operand: Operand) -> Result { match operand { Operand::Local(operand) => self.layout.local_to_slot(operand.local_index()), - Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index()), + Operand::Temp(operand) => self.layout.temp_to_slot(operand.operand_index()), Operand::Immediate(operand) => { let value = operand.val(); - let result = self.layout.temp_to_reg(operand.operand_index())?; + let result = self.layout.temp_to_slot(operand.operand_index())?; let copy_instr = Self::make_copy_imm_instr(result, value)?; let consume_fuel = self.stack.consume_fuel_instr(); self.instrs @@ -742,7 +742,7 @@ impl FuncTranslator { let consume_fuel_instr = self.stack.consume_fuel_instr(); for local in self.stack.preserve_all_locals() { debug_assert!(matches!(local, Operand::Local(_))); - let result = self.layout.temp_to_reg(local.index())?; + let result = self.layout.temp_to_slot(local.index())?; let Some(copy_instr) = Self::make_copy_instr(result, local, &mut self.layout)? else { unreachable!("`result` and `local` refer to different stack spaces"); }; @@ -774,7 +774,7 @@ impl FuncTranslator { let expected_iidx = self.instrs.next_instr(); let result = self .layout - .temp_to_reg(self.stack.push_temp(result_ty, Some(expected_iidx))?)?; + .temp_to_slot(self.stack.push_temp(result_ty, Some(expected_iidx))?)?; let actual_iidx = self.instrs .push_instr(make_instr(result), consume_fuel_instr, fuel_costs)?; @@ -894,7 +894,7 @@ impl FuncTranslator { Op::return_slot(value) } Operand::Temp(operand) => { - let value = self.layout.temp_to_reg(operand.operand_index())?; + let value = self.layout.temp_to_slot(operand.operand_index())?; Op::return_slot(value) } Operand::Immediate(operand) => { @@ -918,7 +918,7 @@ impl FuncTranslator { _ => { self.move_operands_to_temp(usize::from(len_results), consume_fuel)?; let result0 = self.stack.peek(usize::from(len_results)); - let slot0 = self.layout.temp_to_reg(result0.index())?; + let slot0 = self.layout.temp_to_slot(result0.index())?; Op::return_span(BoundedSlotSpan::new(SlotSpan::new(slot0), len_results)) } }; @@ -957,7 +957,7 @@ impl FuncTranslator { }; let mut head = match head.as_ref() { Operand::Local(start) => layout.local_to_slot(start.local_index())?, - Operand::Temp(start) => layout.temp_to_reg(start.operand_index())?, + Operand::Temp(start) => layout.temp_to_slot(start.operand_index())?, Operand::Immediate(_) => return Ok(None), }; let start = head; @@ -965,7 +965,7 @@ impl FuncTranslator { let cur = match value.as_ref() { Operand::Immediate(_) => return Ok(None), Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_reg(value.operand_index())?, + Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, }; if head != cur.prev() { return Ok(None); @@ -1170,7 +1170,7 @@ impl FuncTranslator { let local_idx = LocalIdx::from(local_index); let consume_fuel_instr = self.stack.consume_fuel_instr(); for preserved in self.stack.preserve_locals(local_idx) { - let result = self.layout.temp_to_reg(preserved)?; + let result = self.layout.temp_to_slot(preserved)?; let value = self.layout.local_to_slot(local_idx)?; self.instrs.push_instr( Op::copy(result, value), @@ -1221,7 +1221,7 @@ impl FuncTranslator { // Case: cannot replace local with another local due to observable behavior. return Ok(false); } - Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, + Operand::Temp(operand) => self.layout.temp_to_slot(operand.operand_index())?, }; self.instrs .try_replace_result(result, old_result, &self.layout) @@ -1257,7 +1257,7 @@ impl FuncTranslator { } let condition = match condition { Operand::Local(condition) => self.layout.local_to_slot(condition.local_index())?, - Operand::Temp(condition) => self.layout.temp_to_reg(condition.operand_index())?, + Operand::Temp(condition) => self.layout.temp_to_slot(condition.operand_index())?, Operand::Immediate(condition) => { let condition = i32::from(condition.val()); let take_branch = match branch_eqz { @@ -1346,7 +1346,7 @@ impl FuncTranslator { // Note: cannot fuse cmp instructions with observable semantics. return Ok(None); } - if result != self.layout.temp_to_reg(condition.operand_index())? { + if result != self.layout.temp_to_slot(condition.operand_index())? { // Note: cannot fuse cmp instruction with a result that differs // from the condition operand. return Ok(None); @@ -1518,7 +1518,7 @@ impl FuncTranslator { ) -> Result, Error> { let reg = match operand { Operand::Local(operand) => self.layout.local_to_slot(operand.local_index())?, - Operand::Temp(operand) => self.layout.temp_to_reg(operand.operand_index())?, + Operand::Temp(operand) => self.layout.temp_to_slot(operand.operand_index())?, Operand::Immediate(operand) => return f(self, operand.val()), }; Ok(Input::Slot(reg)) @@ -1543,7 +1543,7 @@ impl FuncTranslator { } Operand::Temp(value) => { debug_assert_eq!(operand.ty(), index_type.ty()); - let reg = self.layout.temp_to_reg(value.operand_index())?; + let reg = self.layout.temp_to_slot(value.operand_index())?; return Ok(Input::Slot(reg)); } }; @@ -1902,7 +1902,7 @@ impl FuncTranslator { // not the case for the `true_val` since `true_val` is the first // value popped from the stack. if !condition { - let selected = self.layout.temp_to_reg(selected.operand_index())?; + let selected = self.layout.temp_to_slot(selected.operand_index())?; self.push_instr_with_result( ty, |result| Op::copy(result, selected), @@ -1915,7 +1915,7 @@ impl FuncTranslator { return Ok(()); } Operand::Local(condition) => self.layout.local_to_slot(condition.local_index())?, - Operand::Temp(condition) => self.layout.temp_to_reg(condition.operand_index())?, + Operand::Temp(condition) => self.layout.temp_to_slot(condition.operand_index())?, }; let true_val = self.copy_if_immediate(true_val)?; let false_val = self.copy_if_immediate(false_val)?; @@ -1995,7 +1995,7 @@ impl FuncTranslator { // Case: `lhs`'s origin instruction does not match the last instruction return Ok(false); } - let lhs_reg = self.layout.temp_to_reg(lhs.operand_index())?; + let lhs_reg = self.layout.temp_to_slot(lhs.operand_index())?; let last_instruction = self.instrs.get(last_instr); let Some(result) = last_instruction.compare_result() else { // Case: cannot fuse non-cmp instructions @@ -2014,7 +2014,7 @@ impl FuncTranslator { let result_idx = self.stack.push_temp(ValType::I32, lhs.instr())?; // Need to replace `cmp` instruction result register since it might // have been misaligned if `lhs` originally referred to the zero operand. - let new_result = self.layout.temp_to_reg(result_idx)?; + let new_result = self.layout.temp_to_slot(result_idx)?; let Some(negated) = negated.update_result_slot(new_result) else { unreachable!("`negated` has been asserted as `cmp` instruction"); }; @@ -2044,7 +2044,7 @@ impl FuncTranslator { let ptr = self.stack.pop(); let ptr = match ptr { Operand::Local(ptr) => self.layout.local_to_slot(ptr.local_index())?, - Operand::Temp(ptr) => self.layout.temp_to_reg(ptr.operand_index())?, + Operand::Temp(ptr) => self.layout.temp_to_slot(ptr.operand_index())?, Operand::Immediate(ptr) => { let Some(address) = self.effective_address(memory, ptr.val(), offset) else { return self.translate_trap(TrapCode::MemoryOutOfBounds); @@ -2119,7 +2119,7 @@ impl FuncTranslator { let (memory, offset) = Self::decode_memarg(memarg)?; let ptr = match ptr { Operand::Local(ptr) => self.layout.local_to_slot(ptr)?, - Operand::Temp(ptr) => self.layout.temp_to_reg(ptr)?, + Operand::Temp(ptr) => self.layout.temp_to_slot(ptr)?, Operand::Immediate(ptr) => { return self.encode_store_ix::(ptr, offset, memory, value) } @@ -2133,7 +2133,7 @@ impl FuncTranslator { T::store_ss(ptr, offset, value, memory) } Operand::Temp(value) => { - let value = self.layout.temp_to_reg(value)?; + let value = self.layout.temp_to_slot(value)?; T::store_ss(ptr, offset, value, memory) } Operand::Immediate(value) => { @@ -2167,7 +2167,7 @@ impl FuncTranslator { T::store_is(address, value, memory) } Operand::Temp(value) => { - let value = self.layout.temp_to_reg(value)?; + let value = self.layout.temp_to_slot(value)?; T::store_is(address, value, memory) } Operand::Immediate(value) => { @@ -2210,7 +2210,7 @@ impl FuncTranslator { T::store_mem0_offset16_ss(ptr, offset16, value) } Operand::Temp(value) => { - let value = self.layout.temp_to_reg(value)?; + let value = self.layout.temp_to_slot(value)?; T::store_mem0_offset16_ss(ptr, offset16, value) } Operand::Immediate(value) => { @@ -2303,8 +2303,8 @@ impl FuncTranslator { let lhs_hi = self.copy_if_immediate(lhs_hi)?; let result_lo = self.stack.push_temp(ValType::I64, None)?; let result_hi = self.stack.push_temp(ValType::I64, None)?; - let result_lo = self.layout.temp_to_reg(result_lo)?; - let result_hi = self.layout.temp_to_reg(result_hi)?; + let result_lo = self.layout.temp_to_slot(result_lo)?; + let result_hi = self.layout.temp_to_slot(result_hi)?; let Ok(results) = >::new(SlotSpan::new(result_lo)) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; @@ -2358,7 +2358,7 @@ impl FuncTranslator { }; let result0 = self.stack.push_temp(ValType::I64, None)?; let _result1 = self.stack.push_temp(ValType::I64, None)?; - let result0 = self.layout.temp_to_reg(result0)?; + let result0 = self.layout.temp_to_slot(result0)?; let Ok(results) = >::new(SlotSpan::new(result0)) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; @@ -2390,7 +2390,7 @@ impl FuncTranslator { if matches!(lhs, Operand::Temp(_)) { // Case: `lhs` is temporary and thus might need a copy to its new result. let consume_fuel_instr = self.stack.consume_fuel_instr(); - let result = self.layout.temp_to_reg(result)?; + let result = self.layout.temp_to_slot(result)?; self.encode_copy(result, lhs, consume_fuel_instr)?; } self.stack.push_immediate(0_i64)?; // hi-bits diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 8beabbcab9..6137783139 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -305,7 +305,7 @@ impl FuncTranslator { return translate_imm(self, memarg, ptr, lane, V128::from(v128.val())); } Operand::Local(v128) => self.layout.local_to_slot(v128.local_index())?, - Operand::Temp(v128) => self.layout.temp_to_reg(v128.operand_index())?, + Operand::Temp(v128) => self.layout.temp_to_slot(v128.operand_index())?, }; let (memory, offset) = Self::decode_memarg(memarg)?; let ptr = self.copy_if_immediate(ptr)?; From a667783ccf8d29b6f9f885cba8ea05f9578fb3e3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:29:33 +0200 Subject: [PATCH 095/423] add #Panics section to operand_to_slot --- crates/wasmi/src/engine/translator/func/layout.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index baa9a94106..6fbfaeeb29 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -103,6 +103,12 @@ impl StackLayout { /// # Errors /// /// If the forwarded method returned an error. + /// + /// # Panics + /// + /// If `operand` is an [`ImmediateOperand`]. + /// + /// [`ImmediateOperand`]: crate::engine::translator::func::ImmediateOperand pub fn operand_to_slot(&mut self, operand: Operand) -> Result { match operand { Operand::Local(operand) => self.local_to_slot(operand.local_index()), From ae4658aab432eea5e2d020bb6594e18c6baaa6d3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:30:36 +0200 Subject: [PATCH 096/423] improve panic message in operand_to_slot --- crates/wasmi/src/engine/translator/func/layout.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index 6fbfaeeb29..34d80867b7 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -113,7 +113,9 @@ impl StackLayout { match operand { Operand::Local(operand) => self.local_to_slot(operand.local_index()), Operand::Temp(operand) => self.temp_to_slot(operand.operand_index()), - Operand::Immediate(_) => panic!("function local constants have been removed"), // TODO: remove + Operand::Immediate(operand) => { + panic!("cannot convert `ImmediateOperand` to stack `Slot` but got: {operand:?}") + } } } From b39b5feed413264ab75dcefc577327aa69bfaf77 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:31:10 +0200 Subject: [PATCH 097/423] simplify operand_to_slot impl --- crates/wasmi/src/engine/translator/func/layout.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index 34d80867b7..9a07ba2420 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -111,8 +111,8 @@ impl StackLayout { /// [`ImmediateOperand`]: crate::engine::translator::func::ImmediateOperand pub fn operand_to_slot(&mut self, operand: Operand) -> Result { match operand { - Operand::Local(operand) => self.local_to_slot(operand.local_index()), - Operand::Temp(operand) => self.temp_to_slot(operand.operand_index()), + Operand::Local(operand) => self.local_to_slot(operand), + Operand::Temp(operand) => self.temp_to_slot(operand), Operand::Immediate(operand) => { panic!("cannot convert `ImmediateOperand` to stack `Slot` but got: {operand:?}") } From a9cb1b8b1835cdd38adfa6b03fba8be71272e3ff Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:31:59 +0200 Subject: [PATCH 098/423] add #[inline] annotations --- crates/wasmi/src/engine/translator/func/layout.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index 9a07ba2420..6d37f2045c 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -22,12 +22,14 @@ pub trait IntoLocalIdx { } impl IntoLocalIdx for LocalIdx { + #[inline] fn into_local_idx(self) -> LocalIdx { self } } impl IntoLocalIdx for LocalOperand { + #[inline] fn into_local_idx(self) -> LocalIdx { self.local_index() } @@ -44,12 +46,14 @@ pub trait IntoOperandIdx { } impl IntoOperandIdx for OperandIdx { + #[inline] fn into_operand_idx(self) -> OperandIdx { self } } impl IntoOperandIdx for TempOperand { + #[inline] fn into_operand_idx(self) -> OperandIdx { self.operand_index() } From 5c96f6f6f8be67fd4b308db98bd36fcbeaf92a6f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:37:48 +0200 Subject: [PATCH 099/423] add Into{Local,Operand}Idx impls for &'_ {Local,Temp}Operand --- crates/wasmi/src/engine/translator/func/layout.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func/layout.rs b/crates/wasmi/src/engine/translator/func/layout.rs index 6d37f2045c..1af347efa8 100644 --- a/crates/wasmi/src/engine/translator/func/layout.rs +++ b/crates/wasmi/src/engine/translator/func/layout.rs @@ -35,6 +35,13 @@ impl IntoLocalIdx for LocalOperand { } } +impl IntoLocalIdx for &'_ LocalOperand { + #[inline] + fn into_local_idx(self) -> LocalIdx { + self.local_index() + } +} + /// Allows conversion from `Self` to [`OperandIdx`]. /// /// # Note @@ -59,6 +66,13 @@ impl IntoOperandIdx for TempOperand { } } +impl IntoOperandIdx for &'_ TempOperand { + #[inline] + fn into_operand_idx(self) -> OperandIdx { + self.operand_index() + } +} + /// The layout of the [`Stack`]. #[derive(Debug, Default)] pub struct StackLayout { From 240b9811b7f190dea3aab5775f8239af030f5a5d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 12:38:05 +0200 Subject: [PATCH 100/423] simplify {local,temp}_to_slot usage where possible --- .../wasmi/src/engine/translator/func/mod.rs | 60 +++++++++---------- .../src/engine/translator/func/simd/mod.rs | 4 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 4d0673ac5f..0fde5e853d 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -436,7 +436,7 @@ impl FuncTranslator { ) -> Result, Error> { let instr = match value { Operand::Temp(value) => { - let value = layout.temp_to_slot(value.operand_index())?; + let value = layout.temp_to_slot(value)?; if result == value { // Case: no-op copy return Ok(None); @@ -444,7 +444,7 @@ impl FuncTranslator { Op::copy(result, value) } Operand::Local(value) => { - let value = layout.local_to_slot(value.local_index())?; + let value = layout.local_to_slot(value)?; if result == value { // Case: no-op copy return Ok(None); @@ -557,8 +557,8 @@ impl FuncTranslator { let mut values = values; while let Some((value, rest)) = values.split_first() { let value = match value { - Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, + Operand::Local(value) => layout.local_to_slot(value)?, + Operand::Temp(value) => layout.temp_to_slot(value)?, Operand::Immediate(_) => { // Immediate values will never yield no-op copies. break; @@ -589,8 +589,8 @@ impl FuncTranslator { let mut values = values; while let Some((value, rest)) = values.split_last() { let value = match value { - Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, + Operand::Local(value) => layout.local_to_slot(value)?, + Operand::Temp(value) => layout.temp_to_slot(value)?, Operand::Immediate(_) => { // Immediate values will never yield no-op copies. break; @@ -632,8 +632,8 @@ impl FuncTranslator { // Note: We only have to check the register case since constant value // copies can never overlap. let value = match value { - Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, + Operand::Local(value) => layout.local_to_slot(value)?, + Operand::Temp(value) => layout.temp_to_slot(value)?, Operand::Immediate(_) => { // Immediates are allocated as function local constants // which can not collide with the result registers. @@ -719,8 +719,8 @@ impl FuncTranslator { /// - Returns the associated [`Slot`] if `operand` is an [`Operand::Temp`] or [`Operand::Local`]. fn copy_if_immediate(&mut self, operand: Operand) -> Result { match operand { - Operand::Local(operand) => self.layout.local_to_slot(operand.local_index()), - Operand::Temp(operand) => self.layout.temp_to_slot(operand.operand_index()), + Operand::Local(operand) => self.layout.local_to_slot(operand), + Operand::Temp(operand) => self.layout.temp_to_slot(operand), Operand::Immediate(operand) => { let value = operand.val(); let result = self.layout.temp_to_slot(operand.operand_index())?; @@ -890,11 +890,11 @@ impl FuncTranslator { 0 => Op::Return {}, 1 => match self.stack.peek(0) { Operand::Local(operand) => { - let value = self.layout.local_to_slot(operand.local_index())?; + let value = self.layout.local_to_slot(operand)?; Op::return_slot(value) } Operand::Temp(operand) => { - let value = self.layout.temp_to_slot(operand.operand_index())?; + let value = self.layout.temp_to_slot(operand)?; Op::return_slot(value) } Operand::Immediate(operand) => { @@ -956,16 +956,16 @@ impl FuncTranslator { return Ok(None); }; let mut head = match head.as_ref() { - Operand::Local(start) => layout.local_to_slot(start.local_index())?, - Operand::Temp(start) => layout.temp_to_slot(start.operand_index())?, + Operand::Local(start) => layout.local_to_slot(start)?, + Operand::Temp(start) => layout.temp_to_slot(start)?, Operand::Immediate(_) => return Ok(None), }; let start = head; for value in values { let cur = match value.as_ref() { Operand::Immediate(_) => return Ok(None), - Operand::Local(value) => layout.local_to_slot(value.local_index())?, - Operand::Temp(value) => layout.temp_to_slot(value.operand_index())?, + Operand::Local(value) => layout.local_to_slot(value)?, + Operand::Temp(value) => layout.temp_to_slot(value)?, }; if head != cur.prev() { return Ok(None); @@ -1221,7 +1221,7 @@ impl FuncTranslator { // Case: cannot replace local with another local due to observable behavior. return Ok(false); } - Operand::Temp(operand) => self.layout.temp_to_slot(operand.operand_index())?, + Operand::Temp(operand) => self.layout.temp_to_slot(operand)?, }; self.instrs .try_replace_result(result, old_result, &self.layout) @@ -1256,8 +1256,8 @@ impl FuncTranslator { return Ok(()); } let condition = match condition { - Operand::Local(condition) => self.layout.local_to_slot(condition.local_index())?, - Operand::Temp(condition) => self.layout.temp_to_slot(condition.operand_index())?, + Operand::Local(condition) => self.layout.local_to_slot(condition)?, + Operand::Temp(condition) => self.layout.temp_to_slot(condition)?, Operand::Immediate(condition) => { let condition = i32::from(condition.val()); let take_branch = match branch_eqz { @@ -1346,7 +1346,7 @@ impl FuncTranslator { // Note: cannot fuse cmp instructions with observable semantics. return Ok(None); } - if result != self.layout.temp_to_slot(condition.operand_index())? { + if result != self.layout.temp_to_slot(condition)? { // Note: cannot fuse cmp instruction with a result that differs // from the condition operand. return Ok(None); @@ -1517,8 +1517,8 @@ impl FuncTranslator { f: impl FnOnce(&mut Self, TypedVal) -> Result, Error>, ) -> Result, Error> { let reg = match operand { - Operand::Local(operand) => self.layout.local_to_slot(operand.local_index())?, - Operand::Temp(operand) => self.layout.temp_to_slot(operand.operand_index())?, + Operand::Local(operand) => self.layout.local_to_slot(operand)?, + Operand::Temp(operand) => self.layout.temp_to_slot(operand)?, Operand::Immediate(operand) => return f(self, operand.val()), }; Ok(Input::Slot(reg)) @@ -1538,12 +1538,12 @@ impl FuncTranslator { Operand::Immediate(value) => value.val(), Operand::Local(value) => { debug_assert_eq!(operand.ty(), index_type.ty()); - let reg = self.layout.local_to_slot(value.local_index())?; + let reg = self.layout.local_to_slot(value)?; return Ok(Input::Slot(reg)); } Operand::Temp(value) => { debug_assert_eq!(operand.ty(), index_type.ty()); - let reg = self.layout.temp_to_slot(value.operand_index())?; + let reg = self.layout.temp_to_slot(value)?; return Ok(Input::Slot(reg)); } }; @@ -1902,7 +1902,7 @@ impl FuncTranslator { // not the case for the `true_val` since `true_val` is the first // value popped from the stack. if !condition { - let selected = self.layout.temp_to_slot(selected.operand_index())?; + let selected = self.layout.temp_to_slot(selected)?; self.push_instr_with_result( ty, |result| Op::copy(result, selected), @@ -1914,8 +1914,8 @@ impl FuncTranslator { self.stack.push_operand(selected)?; return Ok(()); } - Operand::Local(condition) => self.layout.local_to_slot(condition.local_index())?, - Operand::Temp(condition) => self.layout.temp_to_slot(condition.operand_index())?, + Operand::Local(condition) => self.layout.local_to_slot(condition)?, + Operand::Temp(condition) => self.layout.temp_to_slot(condition)?, }; let true_val = self.copy_if_immediate(true_val)?; let false_val = self.copy_if_immediate(false_val)?; @@ -1995,7 +1995,7 @@ impl FuncTranslator { // Case: `lhs`'s origin instruction does not match the last instruction return Ok(false); } - let lhs_reg = self.layout.temp_to_slot(lhs.operand_index())?; + let lhs_reg = self.layout.temp_to_slot(lhs)?; let last_instruction = self.instrs.get(last_instr); let Some(result) = last_instruction.compare_result() else { // Case: cannot fuse non-cmp instructions @@ -2043,8 +2043,8 @@ impl FuncTranslator { let (memory, offset) = Self::decode_memarg(memarg)?; let ptr = self.stack.pop(); let ptr = match ptr { - Operand::Local(ptr) => self.layout.local_to_slot(ptr.local_index())?, - Operand::Temp(ptr) => self.layout.temp_to_slot(ptr.operand_index())?, + Operand::Local(ptr) => self.layout.local_to_slot(ptr)?, + Operand::Temp(ptr) => self.layout.temp_to_slot(ptr)?, Operand::Immediate(ptr) => { let Some(address) = self.effective_address(memory, ptr.val(), offset) else { return self.translate_trap(TrapCode::MemoryOutOfBounds); diff --git a/crates/wasmi/src/engine/translator/func/simd/mod.rs b/crates/wasmi/src/engine/translator/func/simd/mod.rs index 6137783139..c41adb0109 100644 --- a/crates/wasmi/src/engine/translator/func/simd/mod.rs +++ b/crates/wasmi/src/engine/translator/func/simd/mod.rs @@ -304,8 +304,8 @@ impl FuncTranslator { // lane value and translate as a more efficient non-SIMD operation. return translate_imm(self, memarg, ptr, lane, V128::from(v128.val())); } - Operand::Local(v128) => self.layout.local_to_slot(v128.local_index())?, - Operand::Temp(v128) => self.layout.temp_to_slot(v128.operand_index())?, + Operand::Local(v128) => self.layout.local_to_slot(v128)?, + Operand::Temp(v128) => self.layout.temp_to_slot(v128)?, }; let (memory, offset) = Self::decode_memarg(memarg)?; let ptr = self.copy_if_immediate(ptr)?; From 8af6aa8af6fc78d5d156349401126aa857d5a585 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:33:16 +0200 Subject: [PATCH 101/423] rename InstrEncoder -> OpEncoder --- .../src/engine/translator/func/instrs.rs | 28 +++++++++---------- .../wasmi/src/engine/translator/func/mod.rs | 6 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 3566d9b4b0..3dec7f096e 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -19,9 +19,9 @@ use crate::{ }; use alloc::vec::{self, Vec}; -/// Creates and encodes the list of [`Op`]s for a function. +/// Creates and encodes the buffer of encoded [`Op`]s for a function. #[derive(Debug, Default)] -pub struct InstrEncoder { +pub struct OpEncoder { /// The list of constructed instructions and their parameters. instrs: Vec, /// The fuel costs of instructions. @@ -32,7 +32,7 @@ pub struct InstrEncoder { last_instr: Option, } -impl ReusableAllocations for InstrEncoder { +impl ReusableAllocations for OpEncoder { type Allocations = InstrEncoderAllocations; fn into_allocations(self) -> Self::Allocations { @@ -42,7 +42,7 @@ impl ReusableAllocations for InstrEncoder { } } -/// The reusable heap allocations of the [`InstrEncoder`]. +/// The reusable heap allocations of the [`OpEncoder`]. #[derive(Debug, Default)] pub struct InstrEncoderAllocations { /// The list of constructed instructions and their parameters. @@ -55,8 +55,8 @@ impl Reset for InstrEncoderAllocations { } } -impl InstrEncoder { - /// Creates a new [`InstrEncoder`]. +impl OpEncoder { + /// Creates a new [`OpEncoder`]. pub fn new(engine: &Engine, alloc: InstrEncoderAllocations) -> Self { let config = engine.config(); let fuel_costs = config @@ -90,7 +90,7 @@ impl InstrEncoder { Ok(Some(instr)) } - /// Pushes a non-parameter [`Op`] to the [`InstrEncoder`]. + /// Pushes a non-parameter [`Op`] to the [`OpEncoder`]. /// /// Returns an [`Instr`] that refers to the pushed [`Op`]. pub fn push_instr( @@ -103,7 +103,7 @@ impl InstrEncoder { self.push_instr_impl(instruction) } - /// Pushes a non-parameter [`Op`] to the [`InstrEncoder`]. + /// Pushes a non-parameter [`Op`] to the [`OpEncoder`]. fn push_instr_impl(&mut self, instruction: Op) -> Result { let instr = self.next_instr(); self.instrs.push(instruction); @@ -212,7 +212,7 @@ impl InstrEncoder { Ok(true) } - /// Pushes an [`Op`] parameter to the [`InstrEncoder`]. + /// Pushes an [`Op`] parameter to the [`OpEncoder`]. /// /// The parameter is associated to the last pushed [`Op`]. pub fn push_param(&mut self, instruction: Op) { @@ -237,7 +237,7 @@ impl InstrEncoder { &mut self.instrs[instr.into_usize()] } - /// Resets the [`Instr`] last created via [`InstrEncoder::push_instr`]. + /// Resets the [`Instr`] last created via [`OpEncoder::push_instr`]. /// /// # Note /// @@ -291,24 +291,24 @@ impl InstrEncoder { Ok(()) } - /// Returns an iterator yielding all [`Op`]s of the [`InstrEncoder`]. + /// Returns an iterator yielding all [`Op`]s of the [`OpEncoder`]. /// /// # Note /// - /// The [`InstrEncoder`] will be empty after this operation. + /// The [`OpEncoder`] will be empty after this operation. pub fn drain(&mut self) -> InstrEncoderIter<'_> { InstrEncoderIter { iter: self.instrs.drain(..), } } - /// Returns the last instruction of the [`InstrEncoder`] if any. + /// Returns the last instruction of the [`OpEncoder`] if any. pub fn last_instr(&self) -> Option { self.last_instr } } -/// Iterator yielding all [`Op`]s of the [`InstrEncoder`]. +/// Iterator yielding all [`Op`]s of the [`OpEncoder`]. #[derive(Debug)] pub struct InstrEncoderIter<'a> { /// The underlying iterator. diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 0fde5e853d..7f6979577d 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -10,7 +10,7 @@ mod stack; mod visit; use self::{ - instrs::{InstrEncoder, InstrEncoderAllocations}, + instrs::{OpEncoder, InstrEncoderAllocations}, layout::{StackLayout, StackSpace}, locals::{LocalIdx, LocalsRegistry}, stack::{ @@ -97,7 +97,7 @@ pub struct FuncTranslator { /// Slots and pins labels and tracks their users. labels: LabelRegistry, /// Constructs and encodes function instructions. - instrs: InstrEncoder, + instrs: OpEncoder, /// Temporary buffer for operands. operands: Vec, /// Temporary buffer for immediate values. @@ -214,7 +214,7 @@ impl FuncTranslator { immediates, } = alloc.into_reset(); let stack = Stack::new(&engine, stack); - let instrs = InstrEncoder::new(&engine, instrs); + let instrs = OpEncoder::new(&engine, instrs); let mut translator = Self { func, engine, From 77d95ddee7aca50c812b1def295894cd9507e3e7 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:33:25 +0200 Subject: [PATCH 102/423] fix bug in visit_f32_floor translation --- crates/wasmi/src/engine/translator/func/visit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index a6e38e553f..94f8c147ef 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -1312,7 +1312,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { #[inline(never)] fn visit_f32_floor(&mut self) -> Self::Output { - self.translate_unary(Op::f32x4_floor_ss, wasm::f32_floor) + self.translate_unary(Op::f32_floor_ss, wasm::f32_floor) } #[inline(never)] From d8ec119d73ab7e1898c233066daf340d0621e65d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:33:44 +0200 Subject: [PATCH 103/423] fix compile error in panic for make_copy_imm_instr --- crates/wasmi/src/engine/translator/func/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 7f6979577d..e342c47877 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -58,8 +58,9 @@ use crate::{ FuncType, TrapCode, ValType, - V128, }; +#[cfg(feature = "simd")] +use crate::V128; use alloc::vec::Vec; use wasmparser::{MemArg, WasmFeatures}; @@ -474,7 +475,7 @@ impl FuncTranslator { Op::copy128(result, value_lo, value_hi) } #[cfg(not(feature = "simd"))] - ValType::V128 => panic!("unexpected `v128` operand: {value}"), + ValType::V128 => panic!("unexpected `v128` operand: {value:?}"), }; Ok(instr) } From d5d948e8e774d8540e74e57adba56f4cffe017b5 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:34:30 +0200 Subject: [PATCH 104/423] rename InstrEncoderAllocations -> OpEncoderAllocations --- crates/wasmi/src/engine/translator/func/instrs.rs | 8 ++++---- crates/wasmi/src/engine/translator/func/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 3dec7f096e..d77a472c70 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -33,7 +33,7 @@ pub struct OpEncoder { } impl ReusableAllocations for OpEncoder { - type Allocations = InstrEncoderAllocations; + type Allocations = OpEncoderAllocations; fn into_allocations(self) -> Self::Allocations { Self::Allocations { @@ -44,12 +44,12 @@ impl ReusableAllocations for OpEncoder { /// The reusable heap allocations of the [`OpEncoder`]. #[derive(Debug, Default)] -pub struct InstrEncoderAllocations { +pub struct OpEncoderAllocations { /// The list of constructed instructions and their parameters. instrs: Vec, } -impl Reset for InstrEncoderAllocations { +impl Reset for OpEncoderAllocations { fn reset(&mut self) { self.instrs.clear(); } @@ -57,7 +57,7 @@ impl Reset for InstrEncoderAllocations { impl OpEncoder { /// Creates a new [`OpEncoder`]. - pub fn new(engine: &Engine, alloc: InstrEncoderAllocations) -> Self { + pub fn new(engine: &Engine, alloc: OpEncoderAllocations) -> Self { let config = engine.config(); let fuel_costs = config .get_consume_fuel() diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index e342c47877..549e016f4f 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -10,7 +10,7 @@ mod stack; mod visit; use self::{ - instrs::{OpEncoder, InstrEncoderAllocations}, + instrs::{OpEncoder, OpEncoderAllocations}, layout::{StackLayout, StackSpace}, locals::{LocalIdx, LocalsRegistry}, stack::{ @@ -117,7 +117,7 @@ pub struct FuncTranslatorAllocations { /// Slots and pins labels and tracks their users. labels: LabelRegistry, /// Constructs and encodes function instructions. - instrs: InstrEncoderAllocations, + instrs: OpEncoderAllocations, /// Temporary buffer for operands. operands: Vec, /// Temporary buffer for immediate values. From ca0fab0d4890e8660e75548a728a7295fc88b2d0 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:37:12 +0200 Subject: [PATCH 105/423] rename InstrEncoderIter -> OpEncoderIter --- crates/wasmi/src/engine/translator/func/instrs.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index d77a472c70..2154479064 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -296,8 +296,8 @@ impl OpEncoder { /// # Note /// /// The [`OpEncoder`] will be empty after this operation. - pub fn drain(&mut self) -> InstrEncoderIter<'_> { - InstrEncoderIter { + pub fn drain(&mut self) -> OpEncoderIter<'_> { + OpEncoderIter { iter: self.instrs.drain(..), } } @@ -310,12 +310,12 @@ impl OpEncoder { /// Iterator yielding all [`Op`]s of the [`OpEncoder`]. #[derive(Debug)] -pub struct InstrEncoderIter<'a> { +pub struct OpEncoderIter<'a> { /// The underlying iterator. iter: vec::Drain<'a, Op>, } -impl<'a> Iterator for InstrEncoderIter<'a> { +impl<'a> Iterator for OpEncoderIter<'a> { type Item = Op; fn next(&mut self) -> Option { @@ -327,7 +327,7 @@ impl<'a> Iterator for InstrEncoderIter<'a> { } } -impl ExactSizeIterator for InstrEncoderIter<'_> { +impl ExactSizeIterator for OpEncoderIter<'_> { fn len(&self) -> usize { self.iter.len() } From 2d219c3a3a183cc4f0aad8a5e571913616d962cb Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:38:28 +0200 Subject: [PATCH 106/423] rename OpEncoder::last_instr -> last --- .../src/engine/translator/func/instrs.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 2154479064..c2358c8fa2 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -28,8 +28,11 @@ pub struct OpEncoder { /// /// This is `Some` if fuel metering is enabled, otherwise `None`. fuel_costs: Option, - /// The last pushed non-parameter [`Op`]. - last_instr: Option, + /// The last pushed [`Op`]. + /// + /// This is special in that it allows being peeked and manipulated. + /// This is useful to perform op-code fusion or adjusting the result slot. + last: Option, } impl ReusableAllocations for OpEncoder { @@ -66,7 +69,7 @@ impl OpEncoder { Self { instrs: alloc.instrs, fuel_costs, - last_instr: None, + last: None, } } @@ -107,7 +110,7 @@ impl OpEncoder { fn push_instr_impl(&mut self, instruction: Op) -> Result { let instr = self.next_instr(); self.instrs.push(instruction); - self.last_instr = Some(instr); + self.last = Some(instr); Ok(instr) } @@ -120,7 +123,7 @@ impl OpEncoder { /// /// If `instr` or `new_instr` are [`Op`] parameters. pub fn try_replace_instr(&mut self, instr: Instr, new_instr: Op) -> Result { - let Some(last_instr) = self.last_instr else { + let Some(last_instr) = self.last else { return Ok(false); }; let replace = self.get_mut(instr); @@ -150,7 +153,7 @@ impl OpEncoder { // Case: cannot replace result if `new_result` isn't a local. return Ok(false); } - let Some(last_instr) = self.last_instr else { + let Some(last_instr) = self.last else { // Case: cannot replace result without last instruction. return Ok(false); }; @@ -179,7 +182,7 @@ impl OpEncoder { true_val: Slot, false_val: Slot, ) -> Result { - let Some(last_instr) = self.last_instr else { + let Some(last_instr) = self.last else { // If there is no last instruction there is no comparison instruction to negate. return Ok(false); }; @@ -248,7 +251,7 @@ impl OpEncoder { /// needs to be reset to `None` to signal that no such optimization is /// valid across control flow boundaries. pub fn reset_last_instr(&mut self) { - self.last_instr = None; + self.last = None; } /// Updates the branch offset of `instr` to `offset`. @@ -304,7 +307,7 @@ impl OpEncoder { /// Returns the last instruction of the [`OpEncoder`] if any. pub fn last_instr(&self) -> Option { - self.last_instr + self.last } } From d377e174c62dcbf4d1b5b85af0b53d3a7e25f0a4 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:39:08 +0200 Subject: [PATCH 107/423] reorder OpEncoder fields --- crates/wasmi/src/engine/translator/func/instrs.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index c2358c8fa2..949fcc1e14 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -22,17 +22,17 @@ use alloc::vec::{self, Vec}; /// Creates and encodes the buffer of encoded [`Op`]s for a function. #[derive(Debug, Default)] pub struct OpEncoder { - /// The list of constructed instructions and their parameters. - instrs: Vec, - /// The fuel costs of instructions. - /// - /// This is `Some` if fuel metering is enabled, otherwise `None`. - fuel_costs: Option, /// The last pushed [`Op`]. /// /// This is special in that it allows being peeked and manipulated. /// This is useful to perform op-code fusion or adjusting the result slot. last: Option, + /// The fuel costs of instructions. + /// + /// This is `Some` if fuel metering is enabled, otherwise `None`. + fuel_costs: Option, + /// The list of constructed instructions and their parameters. + instrs: Vec, } impl ReusableAllocations for OpEncoder { From 15d0e461342f1daf8232635577cad7b150e9d148 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:39:50 +0200 Subject: [PATCH 108/423] improve docs for last field --- crates/wasmi/src/engine/translator/func/instrs.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 949fcc1e14..23bfe2cf2f 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -24,8 +24,10 @@ use alloc::vec::{self, Vec}; pub struct OpEncoder { /// The last pushed [`Op`]. /// - /// This is special in that it allows being peeked and manipulated. - /// This is useful to perform op-code fusion or adjusting the result slot. + /// # Note + /// + /// - This allows the last [`Op`] to be peeked and manipulated. + /// - For example, this is useful to perform op-code fusion or adjusting the result slot. last: Option, /// The fuel costs of instructions. /// From cd840ba87c1fc8039935b01b60c3c548aecb68f3 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 14:41:48 +0200 Subject: [PATCH 109/423] rename OpEncoder::instrs -> ops --- .../wasmi/src/engine/translator/func/instrs.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 23bfe2cf2f..06aa639367 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -34,7 +34,7 @@ pub struct OpEncoder { /// This is `Some` if fuel metering is enabled, otherwise `None`. fuel_costs: Option, /// The list of constructed instructions and their parameters. - instrs: Vec, + ops: Vec, } impl ReusableAllocations for OpEncoder { @@ -42,7 +42,7 @@ impl ReusableAllocations for OpEncoder { fn into_allocations(self) -> Self::Allocations { Self::Allocations { - instrs: self.instrs, + instrs: self.ops, } } } @@ -69,7 +69,7 @@ impl OpEncoder { .then(|| config.fuel_costs()) .cloned(); Self { - instrs: alloc.instrs, + ops: alloc.instrs, fuel_costs, last: None, } @@ -78,7 +78,7 @@ impl OpEncoder { /// Returns the next [`Instr`]. #[must_use] pub fn next_instr(&self) -> Instr { - Instr::from_usize(self.instrs.len()) + Instr::from_usize(self.ops.len()) } /// Pushes an [`Op::ConsumeFuel`] instruction to `self`. @@ -111,7 +111,7 @@ impl OpEncoder { /// Pushes a non-parameter [`Op`] to the [`OpEncoder`]. fn push_instr_impl(&mut self, instruction: Op) -> Result { let instr = self.next_instr(); - self.instrs.push(instruction); + self.ops.push(instruction); self.last = Some(instr); Ok(instr) } @@ -221,7 +221,7 @@ impl OpEncoder { /// /// The parameter is associated to the last pushed [`Op`]. pub fn push_param(&mut self, instruction: Op) { - self.instrs.push(instruction); + self.ops.push(instruction); } /// Returns a shared reference to the [`Op`] associated to [`Instr`]. @@ -230,7 +230,7 @@ impl OpEncoder { /// /// If `instr` is out of bounds for `self`. pub fn get(&self, instr: Instr) -> &Op { - &self.instrs[instr.into_usize()] + &self.ops[instr.into_usize()] } /// Returns an exclusive reference to the [`Op`] associated to [`Instr`]. @@ -239,7 +239,7 @@ impl OpEncoder { /// /// If `instr` is out of bounds for `self`. fn get_mut(&mut self, instr: Instr) -> &mut Op { - &mut self.instrs[instr.into_usize()] + &mut self.ops[instr.into_usize()] } /// Resets the [`Instr`] last created via [`OpEncoder::push_instr`]. @@ -303,7 +303,7 @@ impl OpEncoder { /// The [`OpEncoder`] will be empty after this operation. pub fn drain(&mut self) -> OpEncoderIter<'_> { OpEncoderIter { - iter: self.instrs.drain(..), + iter: self.ops.drain(..), } } From d20324ec5790b4249b3729a6f669ab9ae42ff99a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 15:04:50 +0200 Subject: [PATCH 110/423] rename Instr -> OpPos --- .../src/engine/translator/func/instrs.rs | 38 +++++++++---------- .../wasmi/src/engine/translator/func/mod.rs | 32 ++++++++-------- .../engine/translator/func/stack/control.rs | 36 +++++++++--------- .../src/engine/translator/func/stack/mod.rs | 16 ++++---- .../engine/translator/func/stack/operand.rs | 10 ++--- .../engine/translator/func/stack/operands.rs | 6 +-- crates/wasmi/src/engine/translator/labels.rs | 30 +++++++-------- crates/wasmi/src/engine/translator/utils.rs | 18 ++++----- 8 files changed, 93 insertions(+), 93 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 06aa639367..d98da3b0f6 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -10,7 +10,7 @@ use crate::{ }, func::{Stack, StackLayout, StackSpace}, relink_result::RelinkResult, - utils::{BumpFuelConsumption as _, Instr}, + utils::{BumpFuelConsumption as _, OpPos}, }, ir::{BranchOffset, Op, Slot}, Engine, @@ -28,7 +28,7 @@ pub struct OpEncoder { /// /// - This allows the last [`Op`] to be peeked and manipulated. /// - For example, this is useful to perform op-code fusion or adjusting the result slot. - last: Option, + last: Option, /// The fuel costs of instructions. /// /// This is `Some` if fuel metering is enabled, otherwise `None`. @@ -75,10 +75,10 @@ impl OpEncoder { } } - /// Returns the next [`Instr`]. + /// Returns the next [`OpPos`]. #[must_use] - pub fn next_instr(&self) -> Instr { - Instr::from_usize(self.ops.len()) + pub fn next_instr(&self) -> OpPos { + OpPos::from_usize(self.ops.len()) } /// Pushes an [`Op::ConsumeFuel`] instruction to `self`. @@ -86,7 +86,7 @@ impl OpEncoder { /// # Note /// /// The pushes [`Op::ConsumeFuel`] is initialized with base fuel costs. - pub fn push_consume_fuel_instr(&mut self) -> Result, Error> { + pub fn push_consume_fuel_instr(&mut self) -> Result, Error> { let Some(fuel_costs) = &self.fuel_costs else { return Ok(None); }; @@ -97,19 +97,19 @@ impl OpEncoder { /// Pushes a non-parameter [`Op`] to the [`OpEncoder`]. /// - /// Returns an [`Instr`] that refers to the pushed [`Op`]. + /// Returns an [`OpPos`] that refers to the pushed [`Op`]. pub fn push_instr( &mut self, instruction: Op, - consume_fuel: Option, + consume_fuel: Option, f: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result { + ) -> Result { self.bump_fuel_consumption(consume_fuel, f)?; self.push_instr_impl(instruction) } /// Pushes a non-parameter [`Op`] to the [`OpEncoder`]. - fn push_instr_impl(&mut self, instruction: Op) -> Result { + fn push_instr_impl(&mut self, instruction: Op) -> Result { let instr = self.next_instr(); self.ops.push(instruction); self.last = Some(instr); @@ -124,7 +124,7 @@ impl OpEncoder { /// # Panics (Debug) /// /// If `instr` or `new_instr` are [`Op`] parameters. - pub fn try_replace_instr(&mut self, instr: Instr, new_instr: Op) -> Result { + pub fn try_replace_instr(&mut self, instr: OpPos, new_instr: Op) -> Result { let Some(last_instr) = self.last else { return Ok(false); }; @@ -224,25 +224,25 @@ impl OpEncoder { self.ops.push(instruction); } - /// Returns a shared reference to the [`Op`] associated to [`Instr`]. + /// Returns a shared reference to the [`Op`] associated to [`OpPos`]. /// /// # Panics /// /// If `instr` is out of bounds for `self`. - pub fn get(&self, instr: Instr) -> &Op { + pub fn get(&self, instr: OpPos) -> &Op { &self.ops[instr.into_usize()] } - /// Returns an exclusive reference to the [`Op`] associated to [`Instr`]. + /// Returns an exclusive reference to the [`Op`] associated to [`OpPos`]. /// /// # Panics /// /// If `instr` is out of bounds for `self`. - fn get_mut(&mut self, instr: Instr) -> &mut Op { + fn get_mut(&mut self, instr: OpPos) -> &mut Op { &mut self.ops[instr.into_usize()] } - /// Resets the [`Instr`] last created via [`OpEncoder::push_instr`]. + /// Resets the [`OpPos`] last created via [`OpEncoder::push_instr`]. /// /// # Note /// @@ -263,7 +263,7 @@ impl OpEncoder { /// If the branch offset could not be updated for `instr`. pub fn update_branch_offset( &mut self, - instr: Instr, + instr: OpPos, offset: BranchOffset, ) -> Result<(), Error> { self.get_mut(instr).update_branch_offset(offset)?; @@ -277,7 +277,7 @@ impl OpEncoder { /// If consumed fuel is out of bounds after this operation. pub fn bump_fuel_consumption( &mut self, - consume_fuel: Option, + consume_fuel: Option, f: impl FnOnce(&FuelCostsProvider) -> u64, ) -> Result<(), Error> { let (fuel_costs, consume_fuel) = match (&self.fuel_costs, consume_fuel) { @@ -308,7 +308,7 @@ impl OpEncoder { } /// Returns the last instruction of the [`OpEncoder`] if any. - pub fn last_instr(&self) -> Option { + pub fn last_instr(&self) -> Option { self.last } } diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 549e016f4f..1eeb85304b 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -44,7 +44,7 @@ use crate::{ TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, - utils::{Instr, IntoShiftAmount, ToBits, WasmFloat, WasmInteger}, + utils::{OpPos, IntoShiftAmount, ToBits, WasmFloat, WasmInteger}, WasmTranslator, }, BlockType, @@ -329,7 +329,7 @@ impl FuncTranslator { fn move_operands_to_temp( &mut self, len: usize, - consume_fuel: Option, + consume_fuel: Option, ) -> Result { for n in 0..len { let operand = self.stack.operand_to_temp(n); @@ -349,7 +349,7 @@ impl FuncTranslator { fn copy_branch_params( &mut self, target: &impl ControlFrameBase, - consume_fuel_instr: Option, + consume_fuel_instr: Option, ) -> Result<(), Error> { let len_branch_params = target.len_branch_params(&self.engine); let Some(branch_results) = self.frame_results(target)? else { @@ -392,7 +392,7 @@ impl FuncTranslator { &mut self, results: SlotSpan, len_values: u16, - consume_fuel_instr: Option, + consume_fuel_instr: Option, ) -> Result<(), Error> { match len_values { 0 => Ok(()), @@ -415,8 +415,8 @@ impl FuncTranslator { &mut self, result: Slot, value: Operand, - consume_fuel_instr: Option, - ) -> Result, Error> { + consume_fuel_instr: Option, + ) -> Result, Error> { let Some(copy_instr) = Self::make_copy_instr(result, value, &mut self.layout)? else { // Case: no-op copy instruction return Ok(None); @@ -490,7 +490,7 @@ impl FuncTranslator { results: SlotSpan, values: SlotSpan, len: u16, - consume_fuel_instr: Option, + consume_fuel_instr: Option, ) -> Result<(), Error> { if results == values { // Case: results and values are equal and therefore the copy is a no-op @@ -516,7 +516,7 @@ impl FuncTranslator { &mut self, results: SlotSpan, len: u16, - consume_fuel_instr: Option, + consume_fuel_instr: Option, ) -> Result<(), Error> { self.peek_operands_into_buffer(usize::from(len)); let values = &self.operands[..]; @@ -689,7 +689,7 @@ impl FuncTranslator { !can_avoid_copies } - /// Pins the `label` to the next [`Instr`]. + /// Pins the `label` to the next [`OpPos`]. fn pin_label(&mut self, label: LabelRef) { self.labels .pin_label(label, self.instrs.next_instr()) @@ -704,7 +704,7 @@ impl FuncTranslator { fn copy_operand_to_temp( &mut self, operand: Operand, - consume_fuel: Option, + consume_fuel: Option, ) -> Result { let result = self.layout.temp_to_slot(operand.index())?; self.encode_copy(result, operand, consume_fuel)?; @@ -758,7 +758,7 @@ impl FuncTranslator { &mut self, instr: Op, fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result { + ) -> Result { let consume_fuel = self.stack.consume_fuel_instr(); let instr = self.instrs.push_instr(instr, consume_fuel, fuel_costs)?; Ok(instr) @@ -885,7 +885,7 @@ impl FuncTranslator { } /// Encodes a generic return instruction. - fn encode_return(&mut self, consume_fuel: Option) -> Result { + fn encode_return(&mut self, consume_fuel: Option) -> Result { let len_results = self.func_type_with(FuncType::len_results); let instr = match len_results { 0 => Op::Return {}, @@ -982,7 +982,7 @@ impl FuncTranslator { fn try_form_regspan_or_move( &mut self, len: usize, - consume_fuel_instr: Option, + consume_fuel_instr: Option, ) -> Result { if let Some(span) = self.try_form_regspan(len)? { return Ok(span); @@ -1099,7 +1099,7 @@ impl FuncTranslator { _ => true, }; if end_of_else_reachable { - let consume_fuel_instr: Option = frame.consume_fuel_instr(); + let consume_fuel_instr: Option = frame.consume_fuel_instr(); self.copy_branch_params(&frame, consume_fuel_instr)?; } self.push_frame_results(&frame)?; @@ -1229,7 +1229,7 @@ impl FuncTranslator { } /// Encodes an unconditional Wasm `branch` instruction. - fn encode_br(&mut self, label: LabelRef) -> Result { + fn encode_br(&mut self, label: LabelRef) -> Result { let instr = self.instrs.next_instr(); let offset = self.labels.try_resolve_label(label, instr)?; let br_instr = self.push_instr(Op::branch(offset), FuelCostsProvider::base)?; @@ -1333,7 +1333,7 @@ impl FuncTranslator { /// - Returns `Ok(None)`, otherwise. fn try_make_fused_branch_cmp_instr( &mut self, - instr: Instr, + instr: OpPos, condition: TempOperand, label: LabelRef, negate: bool, diff --git a/crates/wasmi/src/engine/translator/func/stack/control.rs b/crates/wasmi/src/engine/translator/func/stack/control.rs index 399662855f..139263a7e8 100644 --- a/crates/wasmi/src/engine/translator/func/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func/stack/control.rs @@ -1,7 +1,7 @@ use super::{Operand, Reset}; use crate::{ engine::{ - translator::{labels::LabelRef, utils::Instr}, + translator::{labels::LabelRef, utils::OpPos}, BlockType, }, Engine, @@ -43,7 +43,7 @@ pub struct ControlStack { /// fuel consumption instruction since this information is accessed commonly. /// /// [`Op`]: crate::ir::Op - consume_fuel_instr: Option, + consume_fuel_instr: Option, /// Special operand stack to memorize operands for `else` control frames. else_operands: ElseOperands, /// This is `true` if an `if` with else providers was just popped from the stack. @@ -101,7 +101,7 @@ impl ControlStack { /// /// Returns `None` otherwise. #[inline] - pub fn consume_fuel_instr(&self) -> Option { + pub fn consume_fuel_instr(&self) -> Option { debug_assert!(!self.is_empty()); self.consume_fuel_instr } @@ -128,7 +128,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option, ) { debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(BlockControlFrame { @@ -147,7 +147,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option, ) { debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(LoopControlFrame { @@ -166,7 +166,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option, reachability: IfReachability, else_operands: impl IntoIterator, ) { @@ -191,7 +191,7 @@ impl ControlStack { pub fn push_else( &mut self, if_frame: IfControlFrame, - consume_fuel: Option, + consume_fuel: Option, is_end_of_then_reachable: bool, ) { debug_assert!(!self.orphaned_else_operands); @@ -305,7 +305,7 @@ impl<'a> ControlFrameBase for ControlFrameMut<'a> { self.0.len_branch_params(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option { self.0.consume_fuel_instr() } } @@ -418,7 +418,7 @@ pub trait ControlFrameBase { /// Returns a reference to the [`Op::ConsumeFuel`] of `self`. /// /// Returns `None` if fuel metering is disabled. - fn consume_fuel_instr(&self) -> Option; + fn consume_fuel_instr(&self) -> Option; } impl ControlFrameBase for ControlFrame { @@ -496,7 +496,7 @@ impl ControlFrameBase for ControlFrame { } } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option { match self { ControlFrame::Block(frame) => frame.consume_fuel_instr(), ControlFrame::Loop(frame) => frame.consume_fuel_instr(), @@ -523,7 +523,7 @@ pub struct BlockControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option, /// The label used to branch to the [`BlockControlFrame`]. label: LabelRef, } @@ -553,7 +553,7 @@ impl ControlFrameBase for BlockControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option { self.consume_fuel } } @@ -572,7 +572,7 @@ pub struct LoopControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option, /// The label used to branch to the [`LoopControlFrame`]. label: LabelRef, } @@ -602,7 +602,7 @@ impl ControlFrameBase for LoopControlFrame { self.ty.len_params(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option { self.consume_fuel } } @@ -621,7 +621,7 @@ pub struct IfControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option, /// The label used to branch to the [`IfControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -672,7 +672,7 @@ impl ControlFrameBase for IfControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option { self.consume_fuel } } @@ -716,7 +716,7 @@ pub struct ElseControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option, /// The label used to branch to the [`ElseControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -807,7 +807,7 @@ impl ControlFrameBase for ElseControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option { self.consume_fuel } } diff --git a/crates/wasmi/src/engine/translator/func/stack/mod.rs b/crates/wasmi/src/engine/translator/func/stack/mod.rs index c156feb25b..f9d7350df4 100644 --- a/crates/wasmi/src/engine/translator/func/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func/stack/mod.rs @@ -33,7 +33,7 @@ use crate::{ translator::{ func::{stack::operands::PeekedOperands, LocalIdx}, labels::LabelRef, - utils::Instr, + utils::OpPos, }, BlockType, }, @@ -160,7 +160,7 @@ impl Stack { &mut self, ty: BlockType, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option, ) -> Result<(), Error> { debug_assert!(self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -172,7 +172,7 @@ impl Stack { /// /// # Note /// - /// This inherits the `consume_fuel` [`Instr`] from the parent [`ControlFrame`]. + /// This inherits the `consume_fuel` [`OpPos`] from the parent [`ControlFrame`]. /// /// # Errors /// @@ -201,7 +201,7 @@ impl Stack { &mut self, ty: BlockType, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option, ) -> Result<(), Error> { debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -230,7 +230,7 @@ impl Stack { ty: BlockType, label: LabelRef, reachability: IfReachability, - consume_fuel: Option, + consume_fuel: Option, ) -> Result<(), Error> { debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -262,7 +262,7 @@ impl Stack { &mut self, if_frame: IfControlFrame, is_end_of_then_reachable: bool, - consume_fuel: Option, + consume_fuel: Option, ) -> Result<(), Error> { debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); self.push_else_operands(&if_frame)?; @@ -361,7 +361,7 @@ impl Stack { /// /// If too many operands have been pushed onto the [`Stack`]. #[inline] - pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { + pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { self.operands.push_temp(ty, instr) } @@ -523,7 +523,7 @@ impl Stack { /// /// Returns `None` otherwise. #[inline] - pub fn consume_fuel_instr(&self) -> Option { + pub fn consume_fuel_instr(&self) -> Option { self.controls.consume_fuel_instr() } } diff --git a/crates/wasmi/src/engine/translator/func/stack/operand.rs b/crates/wasmi/src/engine/translator/func/stack/operand.rs index 1ff14745d8..819ea422d6 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operand.rs @@ -1,5 +1,5 @@ use super::{LocalIdx, OperandIdx, StackOperand}; -use crate::{core::TypedVal, engine::translator::utils::Instr, ValType}; +use crate::{core::TypedVal, engine::translator::utils::OpPos, ValType}; #[cfg(doc)] use super::Stack; @@ -47,7 +47,7 @@ impl Operand { } /// Creates a temporary [`Operand`]. - pub(super) fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { + pub(super) fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { Self::Temp(TempOperand { operand_index, ty, @@ -126,7 +126,7 @@ pub struct TempOperand { /// The type of the temporary. ty: ValType, /// The instruction which created this [`TempOperand`] as its result if any. - instr: Option, + instr: Option, } impl From for Operand { @@ -146,8 +146,8 @@ impl TempOperand { self.ty } - /// Returns the instruction which created this [`TempOperand`] as its result if any. - pub fn instr(&self) -> Option { + /// Returns the instruction whcih created this [`TempOperand`] as its result if any. + pub fn instr(&self) -> Option { self.instr } } diff --git a/crates/wasmi/src/engine/translator/func/stack/operands.rs b/crates/wasmi/src/engine/translator/func/stack/operands.rs index f2a7f26c8c..f873862378 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operands.rs @@ -1,5 +1,5 @@ use super::{LocalIdx, LocalsHead, Operand, Reset}; -use crate::{core::TypedVal, engine::translator::utils::Instr, Error, ValType}; +use crate::{core::TypedVal, engine::translator::utils::OpPos, Error, ValType}; use alloc::vec::Vec; use core::{num::NonZero, slice}; @@ -48,7 +48,7 @@ pub enum StackOperand { /// The type of the temporary value. ty: ValType, /// The instruction which has this [`StackOperand`] as result if any. - instr: Option, + instr: Option, }, /// An immediate value on the [`OperandStack`]. Immediate { @@ -183,7 +183,7 @@ impl OperandStack { /// /// If too many operands have been pushed onto the [`OperandStack`]. #[inline] - pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { + pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { let idx = self.next_index(); self.operands.push(StackOperand::Temp { ty, instr }); self.update_max_stack_height(); diff --git a/crates/wasmi/src/engine/translator/labels.rs b/crates/wasmi/src/engine/translator/labels.rs index eab8c906ac..ebc9f2c899 100644 --- a/crates/wasmi/src/engine/translator/labels.rs +++ b/crates/wasmi/src/engine/translator/labels.rs @@ -1,4 +1,4 @@ -use crate::{engine::translator::utils::Instr, ir::BranchOffset, Error}; +use crate::{engine::translator::utils::OpPos, ir::BranchOffset, Error}; use alloc::vec::Vec; use core::{ fmt::{self, Display}, @@ -8,8 +8,8 @@ use core::{ /// A label during the Wasmi compilation process. #[derive(Debug, Copy, Clone)] pub enum Label { - /// The label has already been pinned to a particular [`Instr`]. - Pinned(Instr), + /// The label has already been pinned to a particular [`OpPos`]. + Pinned(OpPos), /// The label is still unpinned. Unpinned, } @@ -41,12 +41,12 @@ pub struct LabelUser { /// The label in use by the user. label: LabelRef, /// The reference to the using instruction. - user: Instr, + user: OpPos, } impl LabelUser { /// Creates a new [`LabelUser`]. - pub fn new(label: LabelRef, user: Instr) -> Self { + pub fn new(label: LabelRef, user: OpPos) -> Self { Self { label, user } } } @@ -55,7 +55,7 @@ impl LabelUser { #[derive(Debug, Copy, Clone)] pub enum LabelError { /// When trying to pin an already pinned [`Label`]. - AlreadyPinned { label: LabelRef, pinned_to: Instr }, + AlreadyPinned { label: LabelRef, pinned_to: OpPos }, /// When trying to resolve an unpinned [`Label`]. Unpinned { label: LabelRef }, } @@ -112,8 +112,8 @@ impl LabelRegistry { /// /// # Errors /// - /// If the `label` has already been pinned to some other [`Instr`]. - pub fn pin_label(&mut self, label: LabelRef, instr: Instr) -> Result<(), LabelError> { + /// If the `label` has already been pinned to some other [`OpPos`]. + pub fn pin_label(&mut self, label: LabelRef, instr: OpPos) -> Result<(), LabelError> { match self.get_label_mut(label) { Label::Pinned(pinned) => Err(LabelError::AlreadyPinned { label, @@ -127,7 +127,7 @@ impl LabelRegistry { } /// Pins the `label` to the given `instr` if unpinned. - pub fn try_pin_label(&mut self, label: LabelRef, instr: Instr) { + pub fn try_pin_label(&mut self, label: LabelRef, instr: OpPos) { if let unpinned @ Label::Unpinned = self.get_label_mut(label) { *unpinned = Label::Pinned(instr) } @@ -143,7 +143,7 @@ impl LabelRegistry { pub fn try_resolve_label( &mut self, label: LabelRef, - user: Instr, + user: OpPos, ) -> Result { let offset = match *self.get_label(label) { Label::Pinned(target) => { @@ -157,19 +157,19 @@ impl LabelRegistry { Ok(offset) } - /// Resolves a `label` to its pinned [`Instr`]. + /// Resolves a `label` to its pinned [`OpPos`]. /// /// # Errors /// /// If the `label` is unpinned. - fn resolve_label(&self, label: LabelRef) -> Result { + fn resolve_label(&self, label: LabelRef) -> Result { match self.get_label(label) { Label::Pinned(instr) => Ok(*instr), Label::Unpinned => Err(LabelError::Unpinned { label }), } } - /// Returns an iterator over pairs of user [`Instr`] and their [`BranchOffset`]. + /// Returns an iterator over pairs of user [`OpPos`] and their [`BranchOffset`]. /// /// # Panics /// @@ -184,7 +184,7 @@ impl LabelRegistry { /// Iterator over resolved label users. /// -/// Iterates over pairs of user [`Instr`] and its respective [`BranchOffset`] +/// Iterates over pairs of user [`OpPos`] and its respective [`BranchOffset`] /// which allows the [`InstructionsBuilder`] to properly update the branching /// offsets. /// @@ -196,7 +196,7 @@ pub struct ResolvedUserIter<'a> { } impl Iterator for ResolvedUserIter<'_> { - type Item = (Instr, Result); + type Item = (OpPos, Result); fn next(&mut self) -> Option { let next = self.users.next()?; diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index d36e0801c8..ba8d5b1217 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -163,22 +163,22 @@ impl BumpFuelConsumption for Op { /// A reference to an encoded [`Op`]. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct Instr(u32); +pub struct OpPos(u32); -impl From for Instr { +impl From for OpPos { fn from(index: u32) -> Self { Self(index) } } -impl From for u32 { - fn from(instr: Instr) -> Self { +impl From for u32 { + fn from(instr: OpPos) -> Self { instr.0 } } -impl Instr { - /// Creates an [`Instr`] from the given `usize` value. +impl OpPos { + /// Creates an [`OpPos`] from the given `usize` value. /// /// # Note /// @@ -186,10 +186,10 @@ impl Instr { /// /// # Panics /// - /// If the `value` exceeds limitations for [`Instr`]. + /// If the `value` exceeds limitations for [`OpPos`]. pub fn from_usize(value: usize) -> Self { let Ok(index) = u32::try_from(value) else { - panic!("out of bounds index {value} for `Instr`") + panic!("out of bounds index {value} for `OpPos`") }; Self(index) } @@ -199,7 +199,7 @@ impl Instr { match usize::try_from(self.0) { Ok(index) => index, Err(error) => { - panic!("out of bound index {} for `Instr`: {error}", self.0) + panic!("out of bound index {} for `OpPos`: {error}", self.0) } } } From cfb838ac563e6efcd7f5fc4c80bee6081324227e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 15:18:22 +0200 Subject: [PATCH 111/423] add LabelRegistry::trace_branch_offset utility --- crates/wasmi/src/engine/translator/labels.rs | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/labels.rs b/crates/wasmi/src/engine/translator/labels.rs index ebc9f2c899..d974aa7101 100644 --- a/crates/wasmi/src/engine/translator/labels.rs +++ b/crates/wasmi/src/engine/translator/labels.rs @@ -1,4 +1,4 @@ -use crate::{engine::translator::utils::OpPos, ir::BranchOffset, Error}; +use crate::{engine::{translator::utils::OpPos, TranslationError}, ir::BranchOffset, Error}; use alloc::vec::Vec; use core::{ fmt::{self, Display}, @@ -133,6 +133,24 @@ impl LabelRegistry { } } + /// Creates an initialized [`BranchOffset`] from `src` to `dst`. + /// + /// # Errors + /// + /// If the resulting [`BranchOffset`] is out of bounds. + pub fn trace_branch_offset(src: OpPos, dst: OpPos) -> Result { + fn trace_offset32(src: OpPos, dst: OpPos) -> Option { + let src = isize::try_from(usize::from(src)).ok()?; + let dst = isize::try_from(usize::from(dst)).ok()?; + let offset = dst.checked_sub(src)?; + i32::try_from(offset).ok() + } + let Some(offset) = trace_offset32(src, dst) else { + return Err(Error::from(TranslationError::BranchOffsetOutOfBounds)) + }; + Ok(BranchOffset::from(offset)) + } + /// Tries to resolve the `label`. /// /// Returns the proper `BranchOffset` in case the `label` has already been @@ -147,7 +165,7 @@ impl LabelRegistry { ) -> Result { let offset = match *self.get_label(label) { Label::Pinned(target) => { - BranchOffset::from_src_to_dst(u32::from(user), u32::from(target))? + Self::trace_branch_offset(user, target)? } Label::Unpinned => { self.users.push(LabelUser::new(label, user)); @@ -205,8 +223,7 @@ impl Iterator for ResolvedUserIter<'_> { .registry .resolve_label(next.label) .unwrap_or_else(|err| panic!("failed to resolve user: {err}")); - let offset = - BranchOffset::from_src_to_dst(u32::from(src), u32::from(dst)).map_err(Into::into); + let offset = LabelRegistry::trace_branch_offset(src, dst); Some((src, offset)) } } From 8c7eda8e8c9fe2ae4a451e99d28be114d4b4dfad Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 15:18:50 +0200 Subject: [PATCH 112/423] remove unused OpPos::distance method --- crates/wasmi/src/engine/translator/utils.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index ba8d5b1217..b67eb1b7e8 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -203,15 +203,6 @@ impl OpPos { } } } - - /// Returns the absolute distance between `self` and `other`. - /// - /// - Returns `0` if `self == other`. - /// - Returns `1` if `self` is adjacent to `other` in the sequence of instructions. - /// - etc.. - pub fn distance(self, other: Self) -> u32 { - self.0.abs_diff(other.0) - } } /// Types that can be converted into bits. From 12beefa80e9b5cd713ebdd6f45e8f7d0ec9dfd81 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 15:21:40 +0200 Subject: [PATCH 113/423] apply rustfmt --- .../src/engine/translator/func/instrs.rs | 29 ++++++++++--------- .../wasmi/src/engine/translator/func/mod.rs | 6 ++-- crates/wasmi/src/engine/translator/labels.rs | 12 ++++---- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index d98da3b0f6..546f0cc31f 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -1,16 +1,19 @@ use super::{Reset, ReusableAllocations}; use crate::{ core::FuelCostsProvider, - engine::translator::{ - comparator::{ - CmpSelectFusion, - CompareResult as _, - TryIntoCmpSelectInstr as _, - UpdateBranchOffset as _, + engine::{ + translator::{ + comparator::{ + CmpSelectFusion, + CompareResult as _, + TryIntoCmpSelectInstr as _, + UpdateBranchOffset as _, + }, + func::{Stack, StackLayout, StackSpace}, + relink_result::RelinkResult, + utils::{BumpFuelConsumption as _, OpPos}, }, - func::{Stack, StackLayout, StackSpace}, - relink_result::RelinkResult, - utils::{BumpFuelConsumption as _, OpPos}, + TranslationError, }, ir::{BranchOffset, Op, Slot}, Engine, @@ -23,9 +26,9 @@ use alloc::vec::{self, Vec}; #[derive(Debug, Default)] pub struct OpEncoder { /// The last pushed [`Op`]. - /// + /// /// # Note - /// + /// /// - This allows the last [`Op`] to be peeked and manipulated. /// - For example, this is useful to perform op-code fusion or adjusting the result slot. last: Option, @@ -41,9 +44,7 @@ impl ReusableAllocations for OpEncoder { type Allocations = OpEncoderAllocations; fn into_allocations(self) -> Self::Allocations { - Self::Allocations { - instrs: self.ops, - } + Self::Allocations { instrs: self.ops } } } diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 1eeb85304b..33620efee3 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -33,6 +33,8 @@ use self::{ }, utils::{Input, Reset, ReusableAllocations, UpdateResultSlot}, }; +#[cfg(feature = "simd")] +use crate::V128; use crate::{ core::{FuelCostsProvider, IndexType, Typed, TypedVal, UntypedVal}, engine::{ @@ -44,7 +46,7 @@ use crate::{ TryIntoCmpBranchInstr as _, }, labels::{LabelRef, LabelRegistry}, - utils::{OpPos, IntoShiftAmount, ToBits, WasmFloat, WasmInteger}, + utils::{IntoShiftAmount, OpPos, ToBits, WasmFloat, WasmInteger}, WasmTranslator, }, BlockType, @@ -59,8 +61,6 @@ use crate::{ TrapCode, ValType, }; -#[cfg(feature = "simd")] -use crate::V128; use alloc::vec::Vec; use wasmparser::{MemArg, WasmFeatures}; diff --git a/crates/wasmi/src/engine/translator/labels.rs b/crates/wasmi/src/engine/translator/labels.rs index d974aa7101..9e76ed8e4b 100644 --- a/crates/wasmi/src/engine/translator/labels.rs +++ b/crates/wasmi/src/engine/translator/labels.rs @@ -1,4 +1,8 @@ -use crate::{engine::{translator::utils::OpPos, TranslationError}, ir::BranchOffset, Error}; +use crate::{ + engine::{translator::utils::OpPos, TranslationError}, + ir::BranchOffset, + Error, +}; use alloc::vec::Vec; use core::{ fmt::{self, Display}, @@ -146,7 +150,7 @@ impl LabelRegistry { i32::try_from(offset).ok() } let Some(offset) = trace_offset32(src, dst) else { - return Err(Error::from(TranslationError::BranchOffsetOutOfBounds)) + return Err(Error::from(TranslationError::BranchOffsetOutOfBounds)); }; Ok(BranchOffset::from(offset)) } @@ -164,9 +168,7 @@ impl LabelRegistry { user: OpPos, ) -> Result { let offset = match *self.get_label(label) { - Label::Pinned(target) => { - Self::trace_branch_offset(user, target)? - } + Label::Pinned(target) => Self::trace_branch_offset(user, target)?, Label::Unpinned => { self.users.push(LabelUser::new(label, user)); BranchOffset::uninit() From 6f1a6ee9946e84010e3dd61f05d396d56ede3219 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 15:22:01 +0200 Subject: [PATCH 114/423] make OpPos wrap usize instead of u32 --- .../src/engine/translator/func/instrs.rs | 10 ++--- crates/wasmi/src/engine/translator/utils.rs | 40 +++---------------- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index 546f0cc31f..ec3ce1206f 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -79,7 +79,7 @@ impl OpEncoder { /// Returns the next [`OpPos`]. #[must_use] pub fn next_instr(&self) -> OpPos { - OpPos::from_usize(self.ops.len()) + OpPos::from(self.ops.len()) } /// Pushes an [`Op::ConsumeFuel`] instruction to `self`. @@ -230,8 +230,8 @@ impl OpEncoder { /// # Panics /// /// If `instr` is out of bounds for `self`. - pub fn get(&self, instr: OpPos) -> &Op { - &self.ops[instr.into_usize()] + pub fn get(&self, pos: OpPos) -> &Op { + &self.ops[usize::from(pos)] } /// Returns an exclusive reference to the [`Op`] associated to [`OpPos`]. @@ -239,8 +239,8 @@ impl OpEncoder { /// # Panics /// /// If `instr` is out of bounds for `self`. - fn get_mut(&mut self, instr: OpPos) -> &mut Op { - &mut self.ops[instr.into_usize()] + fn get_mut(&mut self, pos: OpPos) -> &mut Op { + &mut self.ops[usize::from(pos)] } /// Resets the [`OpPos`] last created via [`OpEncoder::push_instr`]. diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index b67eb1b7e8..7b1714f2f9 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -163,45 +163,17 @@ impl BumpFuelConsumption for Op { /// A reference to an encoded [`Op`]. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct OpPos(u32); +pub struct OpPos(usize); -impl From for OpPos { - fn from(index: u32) -> Self { +impl From for OpPos { + fn from(index: usize) -> Self { Self(index) } } -impl From for u32 { - fn from(instr: OpPos) -> Self { - instr.0 - } -} - -impl OpPos { - /// Creates an [`OpPos`] from the given `usize` value. - /// - /// # Note - /// - /// This intentionally is an API intended for test purposes only. - /// - /// # Panics - /// - /// If the `value` exceeds limitations for [`OpPos`]. - pub fn from_usize(value: usize) -> Self { - let Ok(index) = u32::try_from(value) else { - panic!("out of bounds index {value} for `OpPos`") - }; - Self(index) - } - - /// Returns an `usize` representation of the instruction index. - pub fn into_usize(self) -> usize { - match usize::try_from(self.0) { - Ok(index) => index, - Err(error) => { - panic!("out of bound index {} for `OpPos`: {error}", self.0) - } - } +impl From for usize { + fn from(pos: OpPos) -> Self { + pos.0 } } From bc79e6ef36e21537f13860cb2cd56aeb113805d8 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 17 Sep 2025 17:40:24 +0200 Subject: [PATCH 115/423] add initial EncodeOps type --- .../src/engine/translator/func/instrs.rs | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs index ec3ce1206f..a8df945e0a 100644 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ b/crates/wasmi/src/engine/translator/func/instrs.rs @@ -15,13 +15,55 @@ use crate::{ }, TranslationError, }, - ir::{BranchOffset, Op, Slot}, + ir::{self, BranchOffset, Encode as _, Op, Slot}, Engine, Error, ValType, }; use alloc::vec::{self, Vec}; +#[derive(Debug, Default)] +#[expect(unused)] +pub struct EncodedOps { + buffer: Vec, +} + +impl ir::Encoder for EncodedOps { + type Pos = OpPos; + type Error = TranslationError; + + fn write_bytes(&mut self, bytes: &[u8]) -> Result { + let pos = self.buffer.len(); + self.buffer.extend(bytes); + Ok(OpPos::from(pos)) + } + + fn encode_op_code(&mut self, code: ir::OpCode) -> Result { + // Note: this implements encoding for indirect threading. + // + // For direct threading we need to know ahead of time about the + // function pointers of all operator execution handlers which + // are defined in the Wasmi executor and available to the translator. + u16::from(code).encode(self) + } + + fn branch_offset( + &mut self, + _pos: Self::Pos, + _branch_offset: BranchOffset, + ) -> Result<(), Self::Error> { + todo!() + } + + fn block_fuel( + &mut self, + _pos: Self::Pos, + _block_fuel: ir::BlockFuel, + ) -> Result<(), Self::Error> { + todo!() + } +} + /// Creates and encodes the buffer of encoded [`Op`]s for a function. #[derive(Debug, Default)] pub struct OpEncoder { From 04dbd9cb15b54de14033169f8c686b9ecff1da72 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 18 Sep 2025 12:20:00 +0200 Subject: [PATCH 116/423] expect unused warning for now --- crates/wasmi/src/module/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasmi/src/module/mod.rs b/crates/wasmi/src/module/mod.rs index ed941c9b22..18fef29c3b 100644 --- a/crates/wasmi/src/module/mod.rs +++ b/crates/wasmi/src/module/mod.rs @@ -129,6 +129,7 @@ impl ModuleHeader { } /// Returns the [`FuncIdx`] for the given [`EngineFunc`]. + #[expect(unused)] pub fn get_func_index(&self, func: EngineFunc) -> Option { let position = self.inner.engine_funcs.position(func)?; let len_imports = self.inner.imports.len_funcs as u32; From e892f27753a7328731955911e4a793e2ee99c1c2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 18 Sep 2025 12:21:11 +0200 Subject: [PATCH 117/423] remove unnecessary expect attr --- crates/wasmi/src/engine/code_map.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/wasmi/src/engine/code_map.rs b/crates/wasmi/src/engine/code_map.rs index 99490ffb4f..0ffe559e0e 100644 --- a/crates/wasmi/src/engine/code_map.rs +++ b/crates/wasmi/src/engine/code_map.rs @@ -139,7 +139,6 @@ impl EngineFuncSpan { /// Returns the `u32` index of the [`EngineFunc`] in `self` if any. /// /// Returns `None` if `func` is not contained in `self`. - #[expect(unused)] pub fn position(&self, func: EngineFunc) -> Option { debug_assert!(self.start <= self.end); if func < self.start || func >= self.end { From d02587d73ae9004f5dbc7e3f36479d0fbd7d722b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Fri, 19 Sep 2025 15:03:15 +0200 Subject: [PATCH 118/423] apply clippy suggestions --- crates/wasmi/src/engine/translator/func/mod.rs | 2 +- crates/wasmi/src/engine/translator/func/utils.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 33620efee3..89d98d1566 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -2178,7 +2178,7 @@ impl FuncTranslator { } }; self.push_instr(store_op, FuelCostsProvider::store)?; - return Ok(()); + Ok(()) } /// Encodes a Wasm store operator with `(mem 0)` and 16-bit encodable `offset` to Wasmi bytecode. diff --git a/crates/wasmi/src/engine/translator/func/utils.rs b/crates/wasmi/src/engine/translator/func/utils.rs index a596bf0c68..3f0a88c18a 100644 --- a/crates/wasmi/src/engine/translator/func/utils.rs +++ b/crates/wasmi/src/engine/translator/func/utils.rs @@ -71,9 +71,7 @@ pub trait UpdateResultSlot: Sized { impl UpdateResultSlot for Op { fn update_result_slot(&self, new_result: Slot) -> Option { let mut op = *self; - let Some(result_mut) = op.result_mut() else { - return None; - }; + let result_mut = op.result_mut()?; *result_mut = new_result; Some(op) } From 91a14780c5d2c753426efebda4965be88845e2ae Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 27 Sep 2025 11:08:11 +0200 Subject: [PATCH 119/423] initial re-design of OpEncoder's new API --- crates/wasmi/src/engine/code_map.rs | 35 +- crates/wasmi/src/engine/executor/mod.rs | 2 +- .../wasmi/src/engine/translator/comparator.rs | 111 --- crates/wasmi/src/engine/translator/error.rs | 61 +- .../src/engine/translator/func/encoder.rs | 657 ++++++++++++++++++ .../src/engine/translator/func/instrs.rs | 382 ---------- .../engine/translator/{ => func}/labels.rs | 24 +- .../wasmi/src/engine/translator/func/mod.rs | 247 ++++--- .../engine/translator/func/stack/control.rs | 37 +- .../src/engine/translator/func/stack/mod.rs | 21 +- .../engine/translator/func/stack/operand.rs | 8 +- .../engine/translator/func/stack/operands.rs | 6 +- .../wasmi/src/engine/translator/func/visit.rs | 18 +- crates/wasmi/src/engine/translator/mod.rs | 1 - crates/wasmi/src/engine/translator/utils.rs | 16 - 15 files changed, 903 insertions(+), 723 deletions(-) create mode 100644 crates/wasmi/src/engine/translator/func/encoder.rs delete mode 100644 crates/wasmi/src/engine/translator/func/instrs.rs rename crates/wasmi/src/engine/translator/{ => func}/labels.rs (89%) diff --git a/crates/wasmi/src/engine/code_map.rs b/crates/wasmi/src/engine/code_map.rs index 0ffe559e0e..4375a3e85a 100644 --- a/crates/wasmi/src/engine/code_map.rs +++ b/crates/wasmi/src/engine/code_map.rs @@ -11,7 +11,7 @@ use crate::{ core::{Fuel, FuelCostsProvider}, engine::{utils::unreachable_unchecked, ResumableOutOfFuelError}, errors::FuelError, - ir::{index::InternalFunc, Op}, + ir::index::InternalFunc, module::{FuncIdx, ModuleHeader}, Config, Error, @@ -785,7 +785,7 @@ impl<'a> From<&'a [u8]> for SmallByteSlice { #[derive(Debug)] pub struct CompiledFuncEntity { /// The sequence of [`Op`] of the [`CompiledFuncEntity`]. - instrs: Pin>, + ops: Pin>, /// The number of stack slots used by the [`EngineFunc`] in total. /// /// # Note @@ -800,15 +800,12 @@ impl CompiledFuncEntity { /// /// # Panics /// - /// - If `instrs` is empty. - /// - If `instrs` contains more than `i32::MAX` instructions. - pub fn new(len_stack_slots: u16, instrs: I) -> Self - where - I: IntoIterator, - { - let instrs: Pin> = Pin::new(instrs.into_iter().collect()); + /// - If `ops` is empty. + /// - If `ops` contains more than `i32::MAX` encoded bytes. + pub fn new(len_stack_slots: u16, ops: &[u8]) -> Self { + let ops: Pin> = Pin::new(ops.into()); assert!( - !instrs.is_empty(), + !ops.is_empty(), "compiled functions must have at least one instruction" ); assert!( @@ -816,12 +813,12 @@ impl CompiledFuncEntity { // However, Wasmi's branch instructions can jump across at most `i32::MAX` // forwards or `i32::MIN` instructions backwards and thus having more than // `i32::MAX` instructions might introduce problems. - instrs.len() <= i32::MAX as usize, + ops.len() <= i32::MAX as usize, "compiled function has too many instructions: {}", - instrs.len(), + ops.len(), ); Self { - instrs, + ops, len_stack_slots, } } @@ -830,8 +827,8 @@ impl CompiledFuncEntity { /// A shared reference to the data of a [`EngineFunc`]. #[derive(Debug, Copy, Clone)] pub struct CompiledFuncRef<'a> { - /// The sequence of [`Op`] of the [`CompiledFuncEntity`]. - instrs: Pin<&'a [Op]>, + /// The sequence of encoded [`Op`]s of the [`CompiledFuncEntity`]. + ops: Pin<&'a [u8]>, /// The number of stack slots used by the [`EngineFunc`] in total. len_stack_slots: u16, } @@ -840,17 +837,17 @@ impl<'a> From<&'a CompiledFuncEntity> for CompiledFuncRef<'a> { #[inline] fn from(func: &'a CompiledFuncEntity) -> Self { Self { - instrs: func.instrs.as_ref(), + ops: func.ops.as_ref(), len_stack_slots: func.len_stack_slots, } } } impl<'a> CompiledFuncRef<'a> { - /// Returns the sequence of [`Op`] of the [`EngineFunc`]. + /// Returns the sequence of encoded [`Op`]s of the [`EngineFunc`]. #[inline] - pub fn instrs(&self) -> &'a [Op] { - self.instrs.get_ref() + pub fn ops(&self) -> &'a [u8] { + self.ops.get_ref() } /// Returns the number of stack slots used by the [`EngineFunc`]. diff --git a/crates/wasmi/src/engine/executor/mod.rs b/crates/wasmi/src/engine/executor/mod.rs index b14d25e9f3..f112cf2187 100644 --- a/crates/wasmi/src/engine/executor/mod.rs +++ b/crates/wasmi/src/engine/executor/mod.rs @@ -265,7 +265,7 @@ impl<'engine> EngineExecutor<'engine> { uninit_params.init_zeroes(); self.stack.calls.push( CallFrame::new( - InstructionPtr::new(compiled_func.instrs().as_ptr()), + InstructionPtr::new(compiled_func.ops().as_ptr()), offsets, SlotSpan::new(Slot::from(0)), ), diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index 139a500ac7..85fe991fd9 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -590,114 +590,3 @@ impl TryIntoCmpBranchInstr for Op { Some(cmp_branch_instr) } } - -/// Extension trait to update the branch offset of an [`Op`]. -pub trait UpdateBranchOffset { - /// Updates the [`BranchOffset`] for the branch [`Op`]. - /// - /// # Panics - /// - /// If `self` is not a branch [`Op`]. - fn update_branch_offset(&mut self, new_offset: BranchOffset) -> Result<(), Error>; -} - -impl UpdateBranchOffset for Op { - #[rustfmt::skip] - fn update_branch_offset( - &mut self, - new_offset: BranchOffset, - ) -> Result<(), Error> { - let offset = match self { - | Op::Branch { offset, .. } - // i32 - | Op::BranchI32Eq_Ss { offset, .. } - | Op::BranchI32Eq_Si { offset, .. } - | Op::BranchI32And_Ss { offset, .. } - | Op::BranchI32And_Si { offset, .. } - | Op::BranchI32Or_Ss { offset, .. } - | Op::BranchI32Or_Si { offset, .. } - | Op::BranchI32NotEq_Ss { offset, .. } - | Op::BranchI32NotEq_Si { offset, .. } - | Op::BranchI32NotAnd_Ss { offset, .. } - | Op::BranchI32NotAnd_Si { offset, .. } - | Op::BranchI32NotOr_Ss { offset, .. } - | Op::BranchI32NotOr_Si { offset, .. } - | Op::BranchI32Lt_Ss { offset, .. } - | Op::BranchI32Lt_Si { offset, .. } - | Op::BranchI32Lt_Is { offset, .. } - | Op::BranchU32Lt_Ss { offset, .. } - | Op::BranchU32Lt_Si { offset, .. } - | Op::BranchU32Lt_Is { offset, .. } - | Op::BranchI32Le_Ss { offset, .. } - | Op::BranchI32Le_Si { offset, .. } - | Op::BranchI32Le_Is { offset, .. } - | Op::BranchU32Le_Ss { offset, .. } - | Op::BranchU32Le_Si { offset, .. } - | Op::BranchU32Le_Is { offset, .. } - // i64 - | Op::BranchI64Eq_Ss { offset, .. } - | Op::BranchI64Eq_Si { offset, .. } - | Op::BranchI64And_Ss { offset, .. } - | Op::BranchI64And_Si { offset, .. } - | Op::BranchI64Or_Ss { offset, .. } - | Op::BranchI64Or_Si { offset, .. } - | Op::BranchI64NotEq_Ss { offset, .. } - | Op::BranchI64NotEq_Si { offset, .. } - | Op::BranchI64NotAnd_Ss { offset, .. } - | Op::BranchI64NotAnd_Si { offset, .. } - | Op::BranchI64NotOr_Ss { offset, .. } - | Op::BranchI64NotOr_Si { offset, .. } - | Op::BranchI64Lt_Ss { offset, .. } - | Op::BranchI64Lt_Si { offset, .. } - | Op::BranchI64Lt_Is { offset, .. } - | Op::BranchU64Lt_Ss { offset, .. } - | Op::BranchU64Lt_Si { offset, .. } - | Op::BranchU64Lt_Is { offset, .. } - | Op::BranchI64Le_Ss { offset, .. } - | Op::BranchI64Le_Si { offset, .. } - | Op::BranchI64Le_Is { offset, .. } - | Op::BranchU64Le_Ss { offset, .. } - | Op::BranchU64Le_Si { offset, .. } - | Op::BranchU64Le_Is { offset, .. } - // f32 - | Op::BranchF32Eq_Ss { offset, .. } - | Op::BranchF32Eq_Si { offset, .. } - | Op::BranchF32Lt_Ss { offset, .. } - | Op::BranchF32Lt_Si { offset, .. } - | Op::BranchF32Lt_Is { offset, .. } - | Op::BranchF32Le_Ss { offset, .. } - | Op::BranchF32Le_Si { offset, .. } - | Op::BranchF32Le_Is { offset, .. } - | Op::BranchF32NotEq_Ss { offset, .. } - | Op::BranchF32NotEq_Si { offset, .. } - | Op::BranchF32NotLt_Ss { offset, .. } - | Op::BranchF32NotLt_Si { offset, .. } - | Op::BranchF32NotLt_Is { offset, .. } - | Op::BranchF32NotLe_Ss { offset, .. } - | Op::BranchF32NotLe_Si { offset, .. } - | Op::BranchF32NotLe_Is { offset, .. } - // f64 - | Op::BranchF64Eq_Ss { offset, .. } - | Op::BranchF64Eq_Si { offset, .. } - | Op::BranchF64Lt_Ss { offset, .. } - | Op::BranchF64Lt_Si { offset, .. } - | Op::BranchF64Lt_Is { offset, .. } - | Op::BranchF64Le_Ss { offset, .. } - | Op::BranchF64Le_Si { offset, .. } - | Op::BranchF64Le_Is { offset, .. } - | Op::BranchF64NotEq_Ss { offset, .. } - | Op::BranchF64NotEq_Si { offset, .. } - | Op::BranchF64NotLt_Ss { offset, .. } - | Op::BranchF64NotLt_Si { offset, .. } - | Op::BranchF64NotLt_Is { offset, .. } - | Op::BranchF64NotLe_Ss { offset, .. } - | Op::BranchF64NotLe_Si { offset, .. } - | Op::BranchF64NotLe_Is { offset, .. } => offset, - _unexpected => { - panic!("expected a Wasmi `cmp`+`branch` instruction but found: TODO") // TODO: add Debug for Op - } - }; - offset.init(new_offset); - Ok(()) - } -} diff --git a/crates/wasmi/src/engine/translator/error.rs b/crates/wasmi/src/engine/translator/error.rs index 93ca88f4f8..c326440338 100644 --- a/crates/wasmi/src/engine/translator/error.rs +++ b/crates/wasmi/src/engine/translator/error.rs @@ -34,6 +34,8 @@ pub enum TranslationError { TooManyLocalVariables, /// The function failed to compiled lazily. LazyCompilationFailed, + /// Ran out of system memory during translation. + OutOfSystemMemory, } impl TranslationError { @@ -52,64 +54,41 @@ impl Error for TranslationError {} impl Display for TranslationError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { + let message = match self { Self::UnsupportedBlockType(error) => { - write!(f, "encountered unsupported Wasm block type: {error:?}") + return write!(f, "encountered unsupported Wasm block type: {error:?}") } Self::UnsupportedValueType(error) => { - write!(f, "encountered unsupported Wasm value type: {error:?}") + return write!(f, "encountered unsupported Wasm value type: {error:?}") } Self::BranchTableTargetsOutOfBounds => { - write!( - f, - "branch table targets are out of bounds for wasmi bytecode" - ) - } - Self::BranchOffsetOutOfBounds => { - write!(f, "branching offset is out of bounds for wasmi bytecode") + "branch table targets are out of bounds for wasmi bytecode" } + Self::BranchOffsetOutOfBounds => "branching offset is out of bounds for wasmi bytecode", Self::BlockFuelOutOfBounds => { - write!( - f, - "fuel required to execute a block is out of bounds for wasmi bytecode" - ) + "fuel required to execute a block is out of bounds for wasmi bytecode" } Self::AllocatedTooManySlots => { - write!( - f, - "translation requires more registers for a function than available" - ) - } - Self::SlotOutOfBounds => { - write!(f, "tried to access out of bounds register index") + "translation requires more registers for a function than available" } + Self::SlotOutOfBounds => "tried to access out of bounds register index", Self::EmulatedValueStackOverflow => { - write!(f, "function requires value stack with out of bounds depth") + "function requires value stack with out of bounds depth" } Self::ProviderSliceOverflow => { - write!(f, "tried to allocate too many or too large provider slices") + "tried to allocate too many or too large provider slices" } Self::TooManyFuncLocalConstValues => { - write!( - f, - "tried to allocate too many function local constant values" - ) - } - Self::TooManyFunctionResults => { - write!(f, "encountered function with too many function results") - } - Self::TooManyFunctionParams => { - write!(f, "encountered function with too many function parameters") - } - Self::TooManyLocalVariables => { - write!(f, "encountered function with too many local variables") + "tried to allocate too many function local constant values" } + Self::TooManyFunctionResults => "encountered function with too many function results", + Self::TooManyFunctionParams => "encountered function with too many function parameters", + Self::TooManyLocalVariables => "encountered function with too many local variables", Self::LazyCompilationFailed => { - write!( - f, - "lazy function compilation encountered a Wasm validation or translation error" - ) + "lazy function compilation encountered a Wasm validation or translation error" } - } + Self::OutOfSystemMemory => "ran out of system memory during translation", + }; + f.write_str(message) } } diff --git a/crates/wasmi/src/engine/translator/func/encoder.rs b/crates/wasmi/src/engine/translator/func/encoder.rs new file mode 100644 index 0000000000..bd51e23504 --- /dev/null +++ b/crates/wasmi/src/engine/translator/func/encoder.rs @@ -0,0 +1,657 @@ +use super::{Reset, ReusableAllocations}; +use crate::{ + core::FuelCostsProvider, + engine::TranslationError, + ir::{self, BlockFuel, BranchOffset, Encode as _, Op}, + Engine, + Error, +}; +use alloc::vec::Vec; +use core::{cmp, fmt, marker::PhantomData, mem}; + +/// A byte position within the encoded byte buffer. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct BytePos(usize); + +impl From for BytePos { + fn from(index: usize) -> Self { + Self(index) + } +} + +impl From for usize { + fn from(pos: BytePos) -> Self { + pos.0 + } +} + +/// A position within the encoded byte buffer and its known encoded type. +pub struct Pos { + /// The underlying byte position. + value: BytePos, + /// The type marker denoting what value type has been encoded. + marker: PhantomData T>, +} + +impl From for Pos { + fn from(value: BytePos) -> Self { + Self { + value, + marker: PhantomData, + } + } +} +impl From> for BytePos { + fn from(pos: Pos) -> Self { + pos.value + } +} +impl Copy for Pos {} +impl Clone for Pos { + fn clone(&self) -> Self { + Self { + value: self.value.clone(), + marker: PhantomData, + } + } +} +impl PartialEq for Pos { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } +} +impl Eq for Pos {} +impl PartialOrd for Pos { + fn partial_cmp(&self, other: &Self) -> Option { + self.value.partial_cmp(&other.value) + } +} +impl Ord for Pos { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.value.cmp(&other.value) + } +} +impl fmt::Debug for Pos { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pos") + .field("value", &self.value) + .field("marker", &self.marker) + .finish() + } +} + +#[derive(Debug, Default)] +#[expect(unused)] +pub struct EncodedOps { + buffer: Vec, + temp: Option<(BytePos, TempBuffer)>, +} + +/// The kind of temporary/scratch stored object. +#[derive(Debug)] +pub enum TempBuffer { + /// The temporary object is a [`BranchOffset`]. + BranchOffset(ir::BranchOffset), + /// The temporary object is a [`BlockFuel`]. + BlockFuel(ir::BlockFuel), +} + +impl Reset for EncodedOps { + fn reset(&mut self) { + self.buffer.clear(); + } +} + +impl EncodedOps { + /// Returns the next [`BytePos`]. + #[must_use] + pub fn next_pos(&self) -> BytePos { + BytePos::from(self.buffer.len()) + } + + /// Takes the temporay buffer if any exists. + #[must_use] + pub fn take_temp(&mut self) -> Option<(BytePos, TempBuffer)> { + self.temp.take() + } +} + +impl ir::Encoder for EncodedOps { + type Pos = BytePos; + type Error = TranslationError; + + fn write_bytes(&mut self, bytes: &[u8]) -> Result { + let pos = self.buffer.len(); + if self.buffer.try_reserve(bytes.len()).is_err() { + return Err(TranslationError::OutOfSystemMemory); + } + self.buffer.extend(bytes); + Ok(BytePos::from(pos)) + } + + fn encode_op_code(&mut self, code: ir::OpCode) -> Result { + encode_op_code(self, code) + } + + fn branch_offset( + &mut self, + pos: Self::Pos, + branch_offset: BranchOffset, + ) -> Result<(), Self::Error> { + debug_assert!(self.temp.is_none()); + self.temp = Some((pos, TempBuffer::BranchOffset(branch_offset))); + Ok(()) + } + + fn block_fuel(&mut self, pos: Self::Pos, block_fuel: ir::BlockFuel) -> Result<(), Self::Error> { + debug_assert!(self.temp.is_none()); + self.temp = Some((pos, TempBuffer::BlockFuel(block_fuel))); + Ok(()) + } +} + +/// Creates and encodes the buffer of encoded [`Op`]s for a function. +#[derive(Debug, Default)] +pub struct OpEncoder { + /// The currently staged [`Op`]. + /// + /// # Note + /// + /// - This allows the last [`Op`] to be peeked, inspected and manipulated. + /// - For example, this is useful to perform op-code fusion or adjusting the result slot. + staged: Option, + /// The fuel costs of instructions. + /// + /// This is `Some` if fuel metering is enabled, otherwise `None`. + fuel_costs: Option, + /// The list of constructed instructions and their parameters. + ops: EncodedOps, +} + +/// The staged [`Op`] and information about its fuel consumption. +#[derive(Debug, Copy, Clone)] +pub struct StagedOp { + /// The staged [`Op`]. + op: Op, + /// Fuel information for the staged [`Op`]. + /// + /// - The [`Op::ConsumeFuel`] operator associated to the staged [`Op`] if any. + /// - The fuel required by the staged [`Op`]. + fuel: Option<(Pos, BlockFuel)>, +} + +impl StagedOp { + /// Creates a new [`StagedOp`] from `op` and `fuel`. + pub fn new(op: Op, fuel: Option<(Pos, BlockFuel)>) -> Self { + Self { op, fuel } + } + + /// Replaces the current [`Op`] with `op`. + /// + /// Returns the [`Op`] being replaced. + pub fn replace(&mut self, op: Op) -> Op { + mem::replace(&mut self.op, op) + } +} + +impl ReusableAllocations for OpEncoder { + type Allocations = OpEncoderAllocations; + + fn into_allocations(self) -> Self::Allocations { + Self::Allocations { ops: self.ops } + } +} + +/// The reusable heap allocations of the [`OpEncoder`]. +#[derive(Debug, Default)] +pub struct OpEncoderAllocations { + /// The list of constructed instructions and their parameters. + ops: EncodedOps, +} + +impl Reset for OpEncoderAllocations { + fn reset(&mut self) { + self.ops.reset(); + } +} + +impl OpEncoder { + /// Creates a new [`OpEncoder`]. + pub fn new(engine: &Engine, alloc: OpEncoderAllocations) -> Self { + let config = engine.config(); + let fuel_costs = config + .get_consume_fuel() + .then(|| config.fuel_costs()) + .cloned(); + Self { + staged: None, + fuel_costs, + ops: alloc.ops, + } + } + + /// Returns the next allocated [`Pos`]. + /// + /// # Panics (Debug) + /// + /// If there is a staged [`Op`]. + pub fn next_pos(&self) -> Pos { + // TODO: we should probably remove this API again from `OpEncoder` + debug_assert!(self.staged.is_none()); + Pos::from(self.ops.next_pos()) + } + + /// Returns the staged [`Op`] if any. + pub fn peek_staged(&self) -> Option { + self.staged.map(|staged| staged.op) + } + + /// Sets the staged [`Op`] to `new_staged` and encodes the previously staged [`Op`] if any. + /// + /// Returns the [`Pos`] of the staged [`Op`] if it was encoded. + pub fn stage( + &mut self, + new_staged: Op, + fuel_op: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result, Error> { + let fuel = match (fuel_op, &self.fuel_costs) { + (None, None) => None, + (Some(fuel_op), Some(fuel_costs)) => Some((fuel_op, fuel_selector.select(fuel_costs))), + _ => unreachable!(), + }; + let new_staged = StagedOp::new(new_staged, fuel); + if let Some(old_staged) = self.staged.replace(new_staged) { + self.encode_staged(old_staged)?; + } + Ok(Pos::from(self.ops.next_pos())) + } + + /// Encodes the staged [`Op`] if there is any. + /// + /// # Note + /// + /// - After this operation there will be no more staged [`Op`]. + /// - Does nothing if there is no staged [`Op`]. + pub fn try_encode_staged(&mut self) -> Result<(), Error> { + if let Some(staged) = self.staged.take() { + self.encode_staged(staged)?; + } + debug_assert!(self.staged.is_none()); + Ok(()) + } + + /// Encodes the `staged_op`. + /// + /// - Bumps fuel consumption of the associated [`Op::ConsumeFuel`] operator. + /// - Returns the [`Pos`] of the encoded [`StagedOp`]. + /// + /// # Panics (Debug) + /// + /// If the staged operator unexpectedly issued [`BranchOffset`] or [`BlockFuel`] fields. + /// Those operators may never be staged and must be taken care of directly. + fn encode_staged(&mut self, staged: StagedOp) -> Result, Error> { + if let Some((fuel_pos, fuel_used)) = staged.fuel { + self.bump_fuel_consumption_by(Some(fuel_pos), fuel_used)?; + } + let pos = self.encode_impl(staged.op)?; + debug_assert!(self.ops.temp.is_none()); + Ok(pos) + } + + /// Replaces the staged [`Op`] with `new_staged`. + /// + /// - This does __not__ encode the currently staged [`Op`] but merely replaces it. + /// - Returns the [`Pos`] of the newly staged [`Op`]. + /// + /// # Panics (Debug) + /// + /// If there currently is no staged [`Op`] that can be replaced. + pub fn replace_staged(&mut self, new_staged: Op) -> Result, Error> { + let Some(staged) = self.staged.as_mut() else { + panic!("expected a staged `Op` but found `None`") + }; + staged.replace(new_staged); + Ok(Pos::from(self.ops.next_pos())) + } + + /// Encodes the staged [`Op`] as `op`. + /// + /// # Note + /// + /// - This re-uses the fuel consumption information of the staged [`Op`]. + /// - After this operation there will no longer be a staged [`Op`]. + /// + /// # Panics (Debug) + /// + /// If there currently is no staged [`Op`] that can be removed. + pub fn encode_staged_as(&mut self, op: Op) -> Result, Error> { + let Some(mut staged) = self.staged.take() else { + panic!("expected a staged `Op` but found `None`") + }; + staged.replace(op); + let pos = self.encode_staged(staged)?; + debug_assert!(self.staged.is_none()); + debug_assert!(self.ops.temp.is_none()); + Ok(pos) + } + + /// Encodes an [`Op`] to the [`OpEncoder`] and returns its [`Pos`]. + /// + /// # Note + /// + /// Bumps the `fuel` of the [`Op::ConsumeFuel`] accordingly. + pub fn encode( + &mut self, + op: Op, + fuel_pos: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result, Error> { + self.try_encode_staged()?; + self.bump_fuel_consumption(fuel_pos, fuel_selector)?; + let pos = self.encode_impl(op)?; + debug_assert!(self.ops.take_temp().is_none()); + debug_assert!(self.staged.is_none()); + Ok(pos) + } + + /// Encodes an [`Op::ConsumeFuel`] operator to `self`. + /// + /// # Note + /// + /// The pushed [`Op::ConsumeFuel`] is initialized with base fuel costs. + pub fn encode_consume_fuel(&mut self) -> Result>, Error> { + let Some(fuel_costs) = &self.fuel_costs else { + return Ok(None); + }; + let consumed_fuel = BlockFuel::from(fuel_costs.base()); + self.try_encode_staged()?; + Op::consume_fuel(consumed_fuel).encode(&mut self.ops)?; + let Some((pos, TempBuffer::BlockFuel(_))) = self.ops.take_temp() else { + unreachable!("expected encoded `BlockFuel` entry but found none") + }; + debug_assert!(self.staged.is_none()); + Ok(Some(Pos::from(pos))) + } + + /// Encodes a branch [`Op`] to the [`OpEncoder`] and returns its [`Pos`] and [`Pos`]. + /// + /// # Note + /// + /// Bumps the `fuel` of the [`Op::ConsumeFuel`] accordingly. + pub fn encode_branch( + &mut self, + op: Op, + fuel_pos: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result<(Pos, Pos), Error> { + self.try_encode_staged()?; + self.bump_fuel_consumption(fuel_pos, fuel_selector)?; + let pos_op = self.encode_impl(op)?; + let pos_offset = match self.ops.take_temp() { + Some((pos, TempBuffer::BranchOffset(_))) => Pos::from(pos), + _ => panic!("missing encoded position for `BranchOffset`"), + }; + debug_assert!(self.staged.is_none()); + Ok((pos_op, pos_offset)) + } + + /// Encodes an [`Op`] to the [`OpEncoder`] and returns its [`Pos`]. + /// + /// # Note + /// + /// - Encodes `last` [`Op`] prior to `op` if `last` is `Some`. + /// - After this call `last` will yield `None`. + fn encode_impl(&mut self, op: Op) -> Result, Error> { + let pos = self.ops.next_pos(); + op.encode(&mut self.ops)?; + Ok(Pos::from(pos)) + } + + /// Updates the encoded [`BranchOffset`] at `pos` to `offset`. + /// + /// # Panics + /// + /// - If `pos` was out of bounds for `self`. + /// - If the [`BranchOffset`] at `pos` failed to be decoded, updated or re-encoded. + pub fn update_branch_offset( + &mut self, + pos: Pos, + offset: BranchOffset, + ) -> Result<(), Error> { + self.update_encoded(pos, |_| Some(offset)); + Ok(()) + } + + /// Bumps consumed fuel for [`Op::ConsumeFuel`] at `fuel_pos` by `fuel_selector(fuel_costs)`. + /// + /// Does nothing if fuel metering is disabled. + /// + /// # Errors + /// + /// If consumed fuel is out of bounds after this operation. + fn bump_fuel_consumption( + &mut self, + fuel_pos: Option>, + fuel_selector: impl FuelCostsSelector, + ) -> Result<(), Error> { + debug_assert_eq!(fuel_pos.is_some(), self.fuel_costs.is_some()); + let fuel_used = match &self.fuel_costs { + None => return Ok(()), + Some(fuel_costs) => fuel_selector.select(fuel_costs), + }; + self.bump_fuel_consumption_by(fuel_pos, fuel_used) + } + + /// Bumps consumed fuel for [`Op::ConsumeFuel`] at `fuel_pos` by `delta`. + /// + /// Does nothing if fuel metering is disabled. + /// + /// # Errors + /// + /// If consumed fuel is out of bounds after this operation. + fn bump_fuel_consumption_by( + &mut self, + fuel_pos: Option>, + delta: BlockFuel, + ) -> Result<(), Error> { + debug_assert_eq!(fuel_pos.is_some(), self.fuel_costs.is_some()); + let fuel_pos = match fuel_pos { + None => return Ok(()), + Some(fuel_pos) => fuel_pos, + }; + self.update_encoded(fuel_pos, |mut fuel| -> Option { + fuel.bump_by(u64::from(delta)).ok()?; + Some(fuel) + }); + Ok(()) + } + + /// Returns an iterator yielding all encoded [`Op`]s of the [`OpEncoder`] as bytes. + pub fn encoded_ops(&mut self) -> &[u8] { + debug_assert!(self.staged.is_none()); + &self.ops.buffer[..] + } +} + +/// Error indicating that in-place updating of encoded items failed. +struct UpdateEncodedError { + /// The underlying kind of error. + kind: UpdateEncodedErrorKind, + /// The type that is decoded, updated and re-encoded. + marker: PhantomData T>, +} + +impl From for UpdateEncodedError { + fn from(kind: UpdateEncodedErrorKind) -> Self { + Self { + kind, + marker: PhantomData, + } + } +} +impl Clone for UpdateEncodedError { + fn clone(&self) -> Self { + Self { + kind: self.kind, + marker: PhantomData, + } + } +} +impl Copy for UpdateEncodedError {} +impl fmt::Debug for UpdateEncodedError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("UpdateEncodedError") + .field("kind", &self.kind) + .field("marker", &self.marker) + .finish() + } +} +impl fmt::Display for UpdateEncodedError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self.kind { + UpdateEncodedErrorKind::BufferOutOfBounds => "buffer out of bounds", + UpdateEncodedErrorKind::FailedToDecode => "failed to decode", + UpdateEncodedErrorKind::FailedToEncode => "failed to encode", + UpdateEncodedErrorKind::FailedToUpdateEncoded => "failed to update encoded", + }; + let type_name = core::any::type_name::(); + write!(f, "{message}: {type_name}") + } +} + +/// Kinds of errors indicating that in-place updating of encoded items failed. +#[derive(Debug, Copy, Clone)] +enum UpdateEncodedErrorKind { + /// Buffer is out of bounds for the position of the update. + BufferOutOfBounds, + /// Failed to decode the encoded item. + FailedToDecode, + /// Failed to encode the updated item. + FailedToEncode, + /// Failed to update the encoded item. + FailedToUpdateEncoded, +} + +impl OpEncoder { + /// Updates an encoded value `v` of type `T` at `pos` in-place using the result of `f(v)`. + /// + /// # Panics + /// + /// - If the underlying bytes buffer is out of bounds for `pos`. + /// - If decodiing of `T` at `pos` fails. + /// - If encodiing of `T` at `pos` fails. + fn update_encoded(&mut self, pos: Pos, f: impl FnOnce(T) -> Option) + where + T: ir::Encode + ir::Decode, + { + if let Err(error) = self + .update_encoded_or_err(pos, f) + .map_err(>::from) + { + panic!("`OpEncoder::update_encoded` unexpectedly failed: {error}") + } + } + + /// Updates a value of type `T` at `pos` using `f` in the encoded buffer. + /// + /// # Errors + /// + /// - If the underlying bytes buffer is out of bounds for `pos`. + /// - If decodiing of `T` at `pos` fails. + /// - If encodiing of `T` at `pos` fails. + /// - If `f(value)` returns `None` and thus updating failed. + fn update_encoded_or_err( + &mut self, + pos: Pos, + f: impl FnOnce(T) -> Option, + ) -> Result<(), UpdateEncodedErrorKind> + where + T: ir::Decode + ir::Encode, + { + let at = usize::from(BytePos::from(pos)); + let Some(buffer) = self.ops.buffer.get_mut(at..) else { + return Err(UpdateEncodedErrorKind::BufferOutOfBounds); + }; + let Ok(decoded) = T::decode(&mut &buffer[..]) else { + return Err(UpdateEncodedErrorKind::FailedToDecode); + }; + let Some(updated) = f(decoded) else { + return Err(UpdateEncodedErrorKind::FailedToUpdateEncoded); + }; + if updated.encode(&mut SliceEncoder::from(buffer)).is_err() { + return Err(UpdateEncodedErrorKind::FailedToEncode); + } + Ok(()) + } +} + +/// Utility type to encode items to a slice of bytes. +pub struct SliceEncoder<'a> { + /// The underlying bytes that will store the encoded items. + bytes: &'a mut [u8], +} + +/// An error that may occur upon encoding items to a byte slice. +#[derive(Debug, Copy, Clone)] +pub struct SliceEncoderError; + +impl<'a> From<&'a mut [u8]> for SliceEncoder<'a> { + fn from(bytes: &'a mut [u8]) -> Self { + Self { bytes } + } +} + +impl<'a> ir::Encoder for SliceEncoder<'a> { + type Pos = (); + type Error = SliceEncoderError; + + fn write_bytes(&mut self, bytes: &[u8]) -> Result { + let Some(buffer) = self.bytes.get_mut(..bytes.len()) else { + return Err(SliceEncoderError); + }; + buffer.copy_from_slice(bytes); + Ok(()) + } + + fn encode_op_code(&mut self, code: ir::OpCode) -> Result { + encode_op_code(self, code) + } + + fn branch_offset( + &mut self, + _pos: Self::Pos, + _branch_offset: BranchOffset, + ) -> Result<(), Self::Error> { + Ok(()) + } + + fn block_fuel(&mut self, _pos: Self::Pos, _block_fuel: BlockFuel) -> Result<(), Self::Error> { + Ok(()) + } +} + +/// Convenience trait to wrap type usable as fuel costs selectors. +pub trait FuelCostsSelector { + /// Selects the fuel usage from the [`FuelCostsProvider`]. + fn select(self, costs: &FuelCostsProvider) -> BlockFuel; +} + +impl FuelCostsSelector for T +where + T: FnOnce(&FuelCostsProvider) -> u64, +{ + fn select(self, costs: &FuelCostsProvider) -> BlockFuel { + BlockFuel::from(self(costs)) + } +} + +/// Encodes an [`ir::OpCode`] to a generic [`ir::Encoder`]. +fn encode_op_code(encoder: &mut E, code: ir::OpCode) -> Result { + // Note: this implements encoding for indirect threading. + // + // For direct threading we need to know ahead of time about the + // function pointers of all operator execution handlers which + // are defined in the Wasmi executor and available to the translator. + u16::from(code).encode(encoder) +} diff --git a/crates/wasmi/src/engine/translator/func/instrs.rs b/crates/wasmi/src/engine/translator/func/instrs.rs deleted file mode 100644 index a8df945e0a..0000000000 --- a/crates/wasmi/src/engine/translator/func/instrs.rs +++ /dev/null @@ -1,382 +0,0 @@ -use super::{Reset, ReusableAllocations}; -use crate::{ - core::FuelCostsProvider, - engine::{ - translator::{ - comparator::{ - CmpSelectFusion, - CompareResult as _, - TryIntoCmpSelectInstr as _, - UpdateBranchOffset as _, - }, - func::{Stack, StackLayout, StackSpace}, - relink_result::RelinkResult, - utils::{BumpFuelConsumption as _, OpPos}, - }, - TranslationError, - }, - ir::{self, BranchOffset, Encode as _, Op, Slot}, - Engine, - Error, - ValType, -}; -use alloc::vec::{self, Vec}; - -#[derive(Debug, Default)] -#[expect(unused)] -pub struct EncodedOps { - buffer: Vec, -} - -impl ir::Encoder for EncodedOps { - type Pos = OpPos; - type Error = TranslationError; - - fn write_bytes(&mut self, bytes: &[u8]) -> Result { - let pos = self.buffer.len(); - self.buffer.extend(bytes); - Ok(OpPos::from(pos)) - } - - fn encode_op_code(&mut self, code: ir::OpCode) -> Result { - // Note: this implements encoding for indirect threading. - // - // For direct threading we need to know ahead of time about the - // function pointers of all operator execution handlers which - // are defined in the Wasmi executor and available to the translator. - u16::from(code).encode(self) - } - - fn branch_offset( - &mut self, - _pos: Self::Pos, - _branch_offset: BranchOffset, - ) -> Result<(), Self::Error> { - todo!() - } - - fn block_fuel( - &mut self, - _pos: Self::Pos, - _block_fuel: ir::BlockFuel, - ) -> Result<(), Self::Error> { - todo!() - } -} - -/// Creates and encodes the buffer of encoded [`Op`]s for a function. -#[derive(Debug, Default)] -pub struct OpEncoder { - /// The last pushed [`Op`]. - /// - /// # Note - /// - /// - This allows the last [`Op`] to be peeked and manipulated. - /// - For example, this is useful to perform op-code fusion or adjusting the result slot. - last: Option, - /// The fuel costs of instructions. - /// - /// This is `Some` if fuel metering is enabled, otherwise `None`. - fuel_costs: Option, - /// The list of constructed instructions and their parameters. - ops: Vec, -} - -impl ReusableAllocations for OpEncoder { - type Allocations = OpEncoderAllocations; - - fn into_allocations(self) -> Self::Allocations { - Self::Allocations { instrs: self.ops } - } -} - -/// The reusable heap allocations of the [`OpEncoder`]. -#[derive(Debug, Default)] -pub struct OpEncoderAllocations { - /// The list of constructed instructions and their parameters. - instrs: Vec, -} - -impl Reset for OpEncoderAllocations { - fn reset(&mut self) { - self.instrs.clear(); - } -} - -impl OpEncoder { - /// Creates a new [`OpEncoder`]. - pub fn new(engine: &Engine, alloc: OpEncoderAllocations) -> Self { - let config = engine.config(); - let fuel_costs = config - .get_consume_fuel() - .then(|| config.fuel_costs()) - .cloned(); - Self { - ops: alloc.instrs, - fuel_costs, - last: None, - } - } - - /// Returns the next [`OpPos`]. - #[must_use] - pub fn next_instr(&self) -> OpPos { - OpPos::from(self.ops.len()) - } - - /// Pushes an [`Op::ConsumeFuel`] instruction to `self`. - /// - /// # Note - /// - /// The pushes [`Op::ConsumeFuel`] is initialized with base fuel costs. - pub fn push_consume_fuel_instr(&mut self) -> Result, Error> { - let Some(fuel_costs) = &self.fuel_costs else { - return Ok(None); - }; - let base_costs = fuel_costs.base(); - let instr = self.push_instr_impl(Op::consume_fuel(base_costs.into()))?; - Ok(Some(instr)) - } - - /// Pushes a non-parameter [`Op`] to the [`OpEncoder`]. - /// - /// Returns an [`OpPos`] that refers to the pushed [`Op`]. - pub fn push_instr( - &mut self, - instruction: Op, - consume_fuel: Option, - f: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result { - self.bump_fuel_consumption(consume_fuel, f)?; - self.push_instr_impl(instruction) - } - - /// Pushes a non-parameter [`Op`] to the [`OpEncoder`]. - fn push_instr_impl(&mut self, instruction: Op) -> Result { - let instr = self.next_instr(); - self.ops.push(instruction); - self.last = Some(instr); - Ok(instr) - } - - /// Replaces `instr` with `new_instr` in `self`. - /// - /// - Returns `Ok(true)` if replacement was successful. - /// - Returns `Ok(false)` if replacement was unsuccessful. - /// - /// # Panics (Debug) - /// - /// If `instr` or `new_instr` are [`Op`] parameters. - pub fn try_replace_instr(&mut self, instr: OpPos, new_instr: Op) -> Result { - let Some(last_instr) = self.last else { - return Ok(false); - }; - let replace = self.get_mut(instr); - if instr != last_instr { - return Ok(false); - } - *replace = new_instr; - Ok(true) - } - - /// Tries to replace the result of the last instruction with `new_result` if possible. - /// - /// # Note - /// - /// - `old_result`: - /// just required for additional safety to check if the last instruction - /// really is the source of the `local.set` or `local.tee`. - /// - `new_result`: - /// the new result which shall replace the `old_result`. - pub fn try_replace_result( - &mut self, - new_result: Slot, - old_result: Slot, - layout: &StackLayout, - ) -> Result { - if !matches!(layout.stack_space(new_result), StackSpace::Local) { - // Case: cannot replace result if `new_result` isn't a local. - return Ok(false); - } - let Some(last_instr) = self.last else { - // Case: cannot replace result without last instruction. - return Ok(false); - }; - if !self - .get_mut(last_instr) - .relink_result(new_result, old_result)? - { - // Case: it was impossible to relink the result of `last_instr. - return Ok(false); - } - Ok(true) - } - - /// Tries to fuse a compare instruction with a Wasm `select` instruction. - /// - /// # Returns - /// - /// - Returns `Some` if fusion was successful. - /// - Returns `None` if fusion could not be applied. - pub fn try_fuse_select( - &mut self, - ty: ValType, - select_condition: Slot, - layout: &StackLayout, - stack: &mut Stack, - true_val: Slot, - false_val: Slot, - ) -> Result { - let Some(last_instr) = self.last else { - // If there is no last instruction there is no comparison instruction to negate. - return Ok(false); - }; - let last_instruction = self.get(last_instr); - let Some(last_result) = last_instruction.compare_result() else { - // All negatable instructions have a single result register. - return Ok(false); - }; - if matches!(layout.stack_space(last_result), StackSpace::Local) { - // The instruction stores its result into a local variable which - // is an observable side effect which we are not allowed to mutate. - return Ok(false); - } - if last_result != select_condition { - // The result of the last instruction and the select's `condition` - // are not equal thus indicating that we cannot fuse the instructions. - return Ok(false); - } - let CmpSelectFusion::Applied(fused) = - last_instruction.try_into_cmp_select_instr(true_val, false_val, || { - let select_result = stack.push_temp(ty, Some(last_instr))?; - let select_result = layout.temp_to_slot(select_result)?; - Ok(select_result) - })? - else { - return Ok(false); - }; - let last_instr = self.get_mut(last_instr); - *last_instr = fused; - Ok(true) - } - - /// Pushes an [`Op`] parameter to the [`OpEncoder`]. - /// - /// The parameter is associated to the last pushed [`Op`]. - pub fn push_param(&mut self, instruction: Op) { - self.ops.push(instruction); - } - - /// Returns a shared reference to the [`Op`] associated to [`OpPos`]. - /// - /// # Panics - /// - /// If `instr` is out of bounds for `self`. - pub fn get(&self, pos: OpPos) -> &Op { - &self.ops[usize::from(pos)] - } - - /// Returns an exclusive reference to the [`Op`] associated to [`OpPos`]. - /// - /// # Panics - /// - /// If `instr` is out of bounds for `self`. - fn get_mut(&mut self, pos: OpPos) -> &mut Op { - &mut self.ops[usize::from(pos)] - } - - /// Resets the [`OpPos`] last created via [`OpEncoder::push_instr`]. - /// - /// # Note - /// - /// The `last_instr` info is used for op-code fusion of `local.set` - /// `local.tee`, compare, conditional branch and `select` instructions. - /// - /// Whenever ending a control frame during Wasm translation `last_instr` - /// needs to be reset to `None` to signal that no such optimization is - /// valid across control flow boundaries. - pub fn reset_last_instr(&mut self) { - self.last = None; - } - - /// Updates the branch offset of `instr` to `offset`. - /// - /// # Errors - /// - /// If the branch offset could not be updated for `instr`. - pub fn update_branch_offset( - &mut self, - instr: OpPos, - offset: BranchOffset, - ) -> Result<(), Error> { - self.get_mut(instr).update_branch_offset(offset)?; - Ok(()) - } - - /// Bumps consumed fuel for [`Op::ConsumeFuel`] of `instr` by `delta`. - /// - /// # Errors - /// - /// If consumed fuel is out of bounds after this operation. - pub fn bump_fuel_consumption( - &mut self, - consume_fuel: Option, - f: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result<(), Error> { - let (fuel_costs, consume_fuel) = match (&self.fuel_costs, consume_fuel) { - (None, None) => return Ok(()), - (Some(fuel_costs), Some(consume_fuel)) => (fuel_costs, consume_fuel), - _ => { - panic!( - "fuel metering state mismatch: fuel_costs: {:?}, fuel_instr: {:?}", - self.fuel_costs, consume_fuel, - ); - } - }; - let fuel_consumed = f(fuel_costs); - self.get_mut(consume_fuel) - .bump_fuel_consumption(fuel_consumed)?; - Ok(()) - } - - /// Returns an iterator yielding all [`Op`]s of the [`OpEncoder`]. - /// - /// # Note - /// - /// The [`OpEncoder`] will be empty after this operation. - pub fn drain(&mut self) -> OpEncoderIter<'_> { - OpEncoderIter { - iter: self.ops.drain(..), - } - } - - /// Returns the last instruction of the [`OpEncoder`] if any. - pub fn last_instr(&self) -> Option { - self.last - } -} - -/// Iterator yielding all [`Op`]s of the [`OpEncoder`]. -#[derive(Debug)] -pub struct OpEncoderIter<'a> { - /// The underlying iterator. - iter: vec::Drain<'a, Op>, -} - -impl<'a> Iterator for OpEncoderIter<'a> { - type Item = Op; - - fn next(&mut self) -> Option { - self.iter.next() - } - - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl ExactSizeIterator for OpEncoderIter<'_> { - fn len(&self) -> usize { - self.iter.len() - } -} diff --git a/crates/wasmi/src/engine/translator/labels.rs b/crates/wasmi/src/engine/translator/func/labels.rs similarity index 89% rename from crates/wasmi/src/engine/translator/labels.rs rename to crates/wasmi/src/engine/translator/func/labels.rs index 9e76ed8e4b..9204c0cf2a 100644 --- a/crates/wasmi/src/engine/translator/labels.rs +++ b/crates/wasmi/src/engine/translator/func/labels.rs @@ -1,5 +1,5 @@ use crate::{ - engine::{translator::utils::OpPos, TranslationError}, + engine::{translator::func::encoder::BytePos, TranslationError}, ir::BranchOffset, Error, }; @@ -13,7 +13,7 @@ use core::{ #[derive(Debug, Copy, Clone)] pub enum Label { /// The label has already been pinned to a particular [`OpPos`]. - Pinned(OpPos), + Pinned(BytePos), /// The label is still unpinned. Unpinned, } @@ -45,12 +45,12 @@ pub struct LabelUser { /// The label in use by the user. label: LabelRef, /// The reference to the using instruction. - user: OpPos, + user: BytePos, } impl LabelUser { /// Creates a new [`LabelUser`]. - pub fn new(label: LabelRef, user: OpPos) -> Self { + pub fn new(label: LabelRef, user: BytePos) -> Self { Self { label, user } } } @@ -59,7 +59,7 @@ impl LabelUser { #[derive(Debug, Copy, Clone)] pub enum LabelError { /// When trying to pin an already pinned [`Label`]. - AlreadyPinned { label: LabelRef, pinned_to: OpPos }, + AlreadyPinned { label: LabelRef, pinned_to: BytePos }, /// When trying to resolve an unpinned [`Label`]. Unpinned { label: LabelRef }, } @@ -117,7 +117,7 @@ impl LabelRegistry { /// # Errors /// /// If the `label` has already been pinned to some other [`OpPos`]. - pub fn pin_label(&mut self, label: LabelRef, instr: OpPos) -> Result<(), LabelError> { + pub fn pin_label(&mut self, label: LabelRef, instr: BytePos) -> Result<(), LabelError> { match self.get_label_mut(label) { Label::Pinned(pinned) => Err(LabelError::AlreadyPinned { label, @@ -131,7 +131,7 @@ impl LabelRegistry { } /// Pins the `label` to the given `instr` if unpinned. - pub fn try_pin_label(&mut self, label: LabelRef, instr: OpPos) { + pub fn try_pin_label(&mut self, label: LabelRef, instr: BytePos) { if let unpinned @ Label::Unpinned = self.get_label_mut(label) { *unpinned = Label::Pinned(instr) } @@ -142,8 +142,8 @@ impl LabelRegistry { /// # Errors /// /// If the resulting [`BranchOffset`] is out of bounds. - pub fn trace_branch_offset(src: OpPos, dst: OpPos) -> Result { - fn trace_offset32(src: OpPos, dst: OpPos) -> Option { + pub fn trace_branch_offset(src: BytePos, dst: BytePos) -> Result { + fn trace_offset32(src: BytePos, dst: BytePos) -> Option { let src = isize::try_from(usize::from(src)).ok()?; let dst = isize::try_from(usize::from(dst)).ok()?; let offset = dst.checked_sub(src)?; @@ -165,7 +165,7 @@ impl LabelRegistry { pub fn try_resolve_label( &mut self, label: LabelRef, - user: OpPos, + user: BytePos, ) -> Result { let offset = match *self.get_label(label) { Label::Pinned(target) => Self::trace_branch_offset(user, target)?, @@ -182,7 +182,7 @@ impl LabelRegistry { /// # Errors /// /// If the `label` is unpinned. - fn resolve_label(&self, label: LabelRef) -> Result { + fn resolve_label(&self, label: LabelRef) -> Result { match self.get_label(label) { Label::Pinned(instr) => Ok(*instr), Label::Unpinned => Err(LabelError::Unpinned { label }), @@ -216,7 +216,7 @@ pub struct ResolvedUserIter<'a> { } impl Iterator for ResolvedUserIter<'_> { - type Item = (OpPos, Result); + type Item = (BytePos, Result); fn next(&mut self) -> Option { let next = self.users.next()?; diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 89d98d1566..ad22a2d5df 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -1,6 +1,7 @@ #[macro_use] mod utils; -mod instrs; +mod encoder; +mod labels; mod layout; mod locals; mod op; @@ -10,7 +11,8 @@ mod stack; mod visit; use self::{ - instrs::{OpEncoder, OpEncoderAllocations}, + encoder::{OpEncoder, OpEncoderAllocations, Pos}, + labels::{LabelRef, LabelRegistry}, layout::{StackLayout, StackSpace}, locals::{LocalIdx, LocalsRegistry}, stack::{ @@ -40,19 +42,21 @@ use crate::{ engine::{ translator::{ comparator::{ + CmpSelectFusion, CompareResult as _, LogicalizeCmpInstr, NegateCmpInstr, TryIntoCmpBranchInstr as _, + TryIntoCmpSelectInstr as _, }, - labels::{LabelRef, LabelRegistry}, - utils::{IntoShiftAmount, OpPos, ToBits, WasmFloat, WasmInteger}, + utils::{IntoShiftAmount, ToBits, WasmFloat, WasmInteger}, WasmTranslator, }, BlockType, CompiledFuncEntity, TranslationError, }, + ir, ir::{index, Address, BoundedSlotSpan, FixedSlotSpan, Offset16, Op, Sign, Slot, SlotSpan}, module::{FuncIdx, FuncTypeIdx, MemoryIdx, ModuleHeader, WasmiValueType}, Engine, @@ -171,7 +175,10 @@ impl WasmTranslator<'_> for FuncTranslator { return Err(Error::from(TranslationError::AllocatedTooManySlots)); }; self.update_branch_offsets()?; - finalize(CompiledFuncEntity::new(frame_size, self.instrs.drain())); + finalize(CompiledFuncEntity::new( + frame_size, + self.instrs.encoded_ops(), + )); Ok(self.into_allocations()) } } @@ -239,7 +246,7 @@ impl FuncTranslator { let func_ty = self.module.get_type_of_func(self.func); let block_ty = BlockType::func_type(func_ty); let end_label = self.labels.new_label(); - let consume_fuel = self.instrs.push_consume_fuel_instr()?; + let consume_fuel = self.instrs.encode_consume_fuel()?; self.stack .push_func_block(block_ty, end_label, consume_fuel)?; Ok(()) @@ -329,7 +336,7 @@ impl FuncTranslator { fn move_operands_to_temp( &mut self, len: usize, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result { for n in 0..len { let operand = self.stack.operand_to_temp(n); @@ -349,7 +356,7 @@ impl FuncTranslator { fn copy_branch_params( &mut self, target: &impl ControlFrameBase, - consume_fuel_instr: Option, + consume_fuel_instr: Option>, ) -> Result<(), Error> { let len_branch_params = target.len_branch_params(&self.engine); let Some(branch_results) = self.frame_results(target)? else { @@ -374,7 +381,7 @@ impl FuncTranslator { .ty() .func_type_with(&self.engine, |func_ty| -> Result<(), Error> { for result in func_ty.results() { - self.stack.push_temp(*result, None)?; + self.stack.push_temp(*result)?; } Ok(()) })?; @@ -392,7 +399,7 @@ impl FuncTranslator { &mut self, results: SlotSpan, len_values: u16, - consume_fuel_instr: Option, + consume_fuel_instr: Option>, ) -> Result<(), Error> { match len_values { 0 => Ok(()), @@ -415,16 +422,23 @@ impl FuncTranslator { &mut self, result: Slot, value: Operand, - consume_fuel_instr: Option, - ) -> Result, Error> { + consume_fuel_instr: Option>, + ) -> Result>, Error> { let Some(copy_instr) = Self::make_copy_instr(result, value, &mut self.layout)? else { // Case: no-op copy instruction return Ok(None); }; - let instr = - self.instrs - .push_instr(copy_instr, consume_fuel_instr, FuelCostsProvider::base)?; - Ok(Some(instr)) + let encode_strategy = match self.layout.stack_space(result) { + StackSpace::Local => OpEncoder::encode, + StackSpace::Temp => OpEncoder::stage, + }; + let pos = encode_strategy( + &mut self.instrs, + copy_instr, + consume_fuel_instr, + FuelCostsProvider::base, + )?; + Ok(Some(pos)) } /// Returns the copy instruction to copy the given `operand` to `result`. @@ -490,17 +504,17 @@ impl FuncTranslator { results: SlotSpan, values: SlotSpan, len: u16, - consume_fuel_instr: Option, + consume_fuel_instr: Option>, ) -> Result<(), Error> { if results == values { // Case: results and values are equal and therefore the copy is a no-op return Ok(()); } debug_assert!(!SlotSpan::has_overlapping_copies(results, values, len)); - self.instrs.push_instr( + self.instrs.encode( Op::copy_span(results, values, len), consume_fuel_instr, - |costs| costs.fuel_for_copying_values(u64::from(len)), + |costs: &FuelCostsProvider| costs.fuel_for_copying_values(u64::from(len)), )?; Ok(()) } @@ -516,7 +530,7 @@ impl FuncTranslator { &mut self, results: SlotSpan, len: u16, - consume_fuel_instr: Option, + consume_fuel_instr: Option>, ) -> Result<(), Error> { self.peek_operands_into_buffer(usize::from(len)); let values = &self.operands[..]; @@ -692,7 +706,7 @@ impl FuncTranslator { /// Pins the `label` to the next [`OpPos`]. fn pin_label(&mut self, label: LabelRef) { self.labels - .pin_label(label, self.instrs.next_instr()) + .pin_label(label, self.instrs.next_pos()) .unwrap_or_else(|err| panic!("failed to pin label to next instruction: {err}")); } @@ -704,7 +718,7 @@ impl FuncTranslator { fn copy_operand_to_temp( &mut self, operand: Operand, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result { let result = self.layout.temp_to_slot(operand.index())?; self.encode_copy(result, operand, consume_fuel)?; @@ -728,7 +742,7 @@ impl FuncTranslator { let copy_instr = Self::make_copy_imm_instr(result, value)?; let consume_fuel = self.stack.consume_fuel_instr(); self.instrs - .push_instr(copy_instr, consume_fuel, FuelCostsProvider::base)?; + .encode(copy_instr, consume_fuel, FuelCostsProvider::base)?; Ok(result) } } @@ -748,7 +762,7 @@ impl FuncTranslator { unreachable!("`result` and `local` refer to different stack spaces"); }; self.instrs - .push_instr(copy_instr, consume_fuel_instr, FuelCostsProvider::base)?; + .encode(copy_instr, consume_fuel_instr, FuelCostsProvider::base)?; } Ok(()) } @@ -758,9 +772,9 @@ impl FuncTranslator { &mut self, instr: Op, fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, - ) -> Result { + ) -> Result, Error> { let consume_fuel = self.stack.consume_fuel_instr(); - let instr = self.instrs.push_instr(instr, consume_fuel, fuel_costs)?; + let instr = self.instrs.stage(instr, consume_fuel, fuel_costs)?; // TODO: maybe use OpEncoder::encode here instead Ok(instr) } @@ -772,13 +786,11 @@ impl FuncTranslator { fuel_costs: impl FnOnce(&FuelCostsProvider) -> u64, ) -> Result<(), Error> { let consume_fuel_instr = self.stack.consume_fuel_instr(); - let expected_iidx = self.instrs.next_instr(); - let result = self - .layout - .temp_to_slot(self.stack.push_temp(result_ty, Some(expected_iidx))?)?; - let actual_iidx = - self.instrs - .push_instr(make_instr(result), consume_fuel_instr, fuel_costs)?; + let expected_iidx = self.instrs.next_pos(); + let result = self.layout.temp_to_slot(self.stack.push_temp(result_ty)?)?; + let actual_iidx = self + .instrs + .stage(make_instr(result), consume_fuel_instr, fuel_costs)?; assert_eq!(expected_iidx, actual_iidx); Ok(()) } @@ -838,8 +850,8 @@ impl FuncTranslator { let mut frame = self.stack.peek_control_mut(depth).control_frame(); let offset = self .labels - .try_resolve_label(frame.label(), self.instrs.next_instr())?; - self.instrs.push_param(Op::branch(offset)); + .try_resolve_label(frame.label(), self.instrs.next_pos())?; + // self.instrs.push_param(Op::branch(offset)); // TODO: finish encoding impl frame.branch_to(); } Ok(()) @@ -876,7 +888,7 @@ impl FuncTranslator { }; let _offset = self .labels - .try_resolve_label(frame.label(), self.instrs.next_instr())?; + .try_resolve_label(frame.label(), self.instrs.next_pos())?; // self.instrs // .push_param(Op::branch_table_target(results, offset)); // TODO: encode branch table targets properly frame.branch_to(); @@ -885,7 +897,10 @@ impl FuncTranslator { } /// Encodes a generic return instruction. - fn encode_return(&mut self, consume_fuel: Option) -> Result { + fn encode_return( + &mut self, + consume_fuel: Option>, + ) -> Result, Error> { let len_results = self.func_type_with(FuncType::len_results); let instr = match len_results { 0 => Op::Return {}, @@ -925,7 +940,7 @@ impl FuncTranslator { }; let instr = self .instrs - .push_instr(instr, consume_fuel, FuelCostsProvider::base)?; + .encode(instr, consume_fuel, FuelCostsProvider::base)?; Ok(instr) } @@ -982,7 +997,7 @@ impl FuncTranslator { fn try_form_regspan_or_move( &mut self, len: usize, - consume_fuel_instr: Option, + consume_fuel_instr: Option>, ) -> Result { if let Some(span) = self.try_form_regspan(len)? { return Ok(span); @@ -1003,10 +1018,7 @@ impl FuncTranslator { } self.push_frame_results(&frame)?; } - if let Err(err) = self - .labels - .pin_label(frame.label(), self.instrs.next_instr()) - { + if let Err(err) = self.labels.pin_label(frame.label(), self.instrs.next_pos()) { panic!("failed to pin label: {err}") } self.reachable |= frame.is_branched_to(); @@ -1015,7 +1027,7 @@ impl FuncTranslator { } if frame.is_branched_to() { // No need to reset `last_instr` if there was no branch to the end of a Wasm `block`. - self.instrs.reset_last_instr(); + self.instrs.try_encode_staged(); } Ok(()) } @@ -1048,16 +1060,16 @@ impl FuncTranslator { self.copy_branch_params(&frame, consume_fuel_instr)?; let end_offset = self .labels - .try_resolve_label(frame.label(), self.instrs.next_instr()) + .try_resolve_label(frame.label(), self.instrs.next_pos()) .unwrap(); - self.instrs.push_instr( + self.instrs.encode( Op::branch(end_offset), consume_fuel_instr, FuelCostsProvider::base, )?; } self.labels - .try_pin_label(else_label, self.instrs.next_instr()); + .try_pin_label(else_label, self.instrs.next_pos()); self.stack.push_else_operands(&frame)?; if has_results { // We haven't visited the `else` block and thus the `else` @@ -1065,16 +1077,16 @@ impl FuncTranslator { // be popped. We use them to restore the stack to the state // when entering the `if` block so that we can properly copy // the `else` results to were they are expected. - let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + let consume_fuel_instr = self.instrs.encode_consume_fuel()?; self.copy_branch_params(&frame, consume_fuel_instr)?; } self.push_frame_results(&frame)?; self.labels - .pin_label(frame.label(), self.instrs.next_instr()) + .pin_label(frame.label(), self.instrs.next_pos()) .unwrap(); self.reachable = true; // Need to reset `last_instr` since end of `if` is a control flow boundary. - self.instrs.reset_last_instr(); + self.instrs.try_encode_staged(); Ok(()) } @@ -1099,16 +1111,16 @@ impl FuncTranslator { _ => true, }; if end_of_else_reachable { - let consume_fuel_instr: Option = frame.consume_fuel_instr(); + let consume_fuel_instr: Option> = frame.consume_fuel_instr(); self.copy_branch_params(&frame, consume_fuel_instr)?; } self.push_frame_results(&frame)?; self.labels - .pin_label(frame.label(), self.instrs.next_instr()) + .pin_label(frame.label(), self.instrs.next_pos()) .unwrap(); self.reachable = reachable; // Need to reset `last_instr` since end of `else` is a control flow boundary. - self.instrs.reset_last_instr(); + self.instrs.try_encode_staged(); Ok(()) } @@ -1126,13 +1138,13 @@ impl FuncTranslator { self.push_frame_results(&frame)?; } self.labels - .pin_label(frame.label(), self.instrs.next_instr()) + .pin_label(frame.label(), self.instrs.next_pos()) .unwrap(); self.reachable = end_is_reachable || frame.is_branched_to(); if frame.is_branched_to() { // No need to reset `last_instr` if there was no branch to the // end of a Wasm `if` where only `then` or `else` is reachable. - self.instrs.reset_last_instr(); + self.instrs.try_encode_staged(); } Ok(()) } @@ -1141,7 +1153,7 @@ impl FuncTranslator { fn translate_end_unreachable(&mut self, _frame: ControlFrameKind) -> Result<(), Error> { debug_assert!(!self.stack.is_control_empty()); // We reset `last_instr` out of caution in case there is a control flow boundary. - self.instrs.reset_last_instr(); + self.instrs.try_encode_staged(); Ok(()) } @@ -1173,7 +1185,7 @@ impl FuncTranslator { for preserved in self.stack.preserve_locals(local_idx) { let result = self.layout.temp_to_slot(preserved)?; let value = self.layout.local_to_slot(local_idx)?; - self.instrs.push_instr( + self.instrs.encode( Op::copy(result, value), consume_fuel_instr, FuelCostsProvider::base, @@ -1212,25 +1224,39 @@ impl FuncTranslator { new_result: LocalIdx, old_result: Operand, ) -> Result { - let result = self.layout.local_to_slot(new_result)?; + let Some(mut staged) = self.instrs.peek_staged() else { + // Case: cannot replace result without staged operator. + return Ok(false); + }; + let new_result = self.layout.local_to_slot(new_result)?; + debug_assert!(matches!( + self.layout.stack_space(new_result), + StackSpace::Local + )); let old_result = match old_result { - Operand::Immediate(_) => { - // Case: cannot replace immediate value result. + Operand::Temp(old_result) => self.layout.temp_to_slot(old_result)?, + Operand::Local(_) | Operand::Immediate(_) => { + // Case immediate: cannot replace immediate value result. + // Case local: cannot replace local with another local due to observable behavior. return Ok(false); } - Operand::Local(_) => { - // Case: cannot replace local with another local due to observable behavior. - return Ok(false); - } - Operand::Temp(operand) => self.layout.temp_to_slot(operand)?, }; - self.instrs - .try_replace_result(result, old_result, &self.layout) + let Some(staged_result) = staged.result_mut() else { + // Case: staged has no result and thus cannot have its result changed. + return Ok(false); + }; + if *staged_result != old_result { + // Case: staged result does not match `old_result` and thus is not available for mutation. + return Ok(false); + } + *staged_result = new_result; + self.instrs.encode_staged_as(staged)?; + Ok(true) } /// Encodes an unconditional Wasm `branch` instruction. - fn encode_br(&mut self, label: LabelRef) -> Result { - let instr = self.instrs.next_instr(); + fn encode_br(&mut self, label: LabelRef) -> Result, Error> { + let instr = self.instrs.next_pos(); let offset = self.labels.try_resolve_label(label, instr)?; let br_instr = self.push_instr(Op::branch(offset), FuelCostsProvider::base)?; Ok(br_instr) @@ -1275,7 +1301,7 @@ impl FuncTranslator { } } }; - let instr = self.instrs.next_instr(); + let instr = self.instrs.next_pos(); let offset = self.labels.try_resolve_label(label, instr)?; let instr = match branch_eqz { true => Op::branch_i32_eq_si(condition, 0, offset), @@ -1292,7 +1318,7 @@ impl FuncTranslator { label: LabelRef, negate: bool, ) -> Result { - let Some(last_instr) = self.instrs.last_instr() else { + let Some(last_instr) = self.instrs.peek_staged() else { // Case: cannot fuse without a known last instruction return Ok(false); }; @@ -1316,10 +1342,7 @@ impl FuncTranslator { // Case: not possible to perform fusion with last instruction return Ok(false); }; - assert!( - self.instrs.try_replace_instr(origin, fused_instr)?, - "op-code fusion must suceed at this point", - ); + self.instrs.replace_staged(fused_instr)?; Ok(true) } @@ -1333,7 +1356,7 @@ impl FuncTranslator { /// - Returns `Ok(None)`, otherwise. fn try_make_fused_branch_cmp_instr( &mut self, - instr: OpPos, + instr: Pos, condition: TempOperand, label: LabelRef, negate: bool, @@ -1501,7 +1524,7 @@ impl FuncTranslator { } Operand::Temp(input) => { debug_assert_eq!(input.ty(), ::TY); - self.stack.push_temp(::TY, None)?; + self.stack.push_temp(::TY)?; } Operand::Immediate(input) => { let input: T = input.val().into(); @@ -1920,14 +1943,7 @@ impl FuncTranslator { }; let true_val = self.copy_if_immediate(true_val)?; let false_val = self.copy_if_immediate(false_val)?; - if !self.instrs.try_fuse_select( - ty, - condition, - &self.layout, - &mut self.stack, - true_val, - false_val, - )? { + if !self.try_fuse_select(ty, condition, true_val, false_val)? { self.push_instr_with_result( ty, |result| Op::select_i32_eq_ssi(result, condition, 0_i32, false_val, true_val), @@ -1937,6 +1953,50 @@ impl FuncTranslator { Ok(()) } + /// Tries to fuse a compare instruction with a Wasm `select` instruction. + /// + /// # Returns + /// + /// - Returns `Some` if fusion was successful. + /// - Returns `None` if fusion could not be applied. + pub fn try_fuse_select( + &mut self, + ty: ValType, + condition: Slot, + true_val: Slot, + false_val: Slot, + ) -> Result { + let Some(staged) = self.instrs.peek_staged() else { + // If there is no last instruction there is no comparison instruction to negate. + return Ok(false); + }; + let Some(staged_result) = staged.result_ref().copied() else { + // All negatable instructions have a single result register. + return Ok(false); + }; + if matches!(self.layout.stack_space(staged_result), StackSpace::Local) { + // The operator stores its result into a local variable which + // is an observable side effect which we are not allowed to mutate. + return Ok(false); + } + if staged_result != condition { + // The result of the last instruction and the select's `condition` + // are not equal thus indicating that we cannot fuse the instructions. + return Ok(false); + } + let CmpSelectFusion::Applied(fused_select) = + staged.try_into_cmp_select_instr(true_val, false_val, || { + let select_result = self.stack.push_temp(ty)?; + let select_result = self.layout.temp_to_slot(select_result)?; + Ok(select_result) + })? + else { + return Ok(false); + }; + self.instrs.replace_staged(fused_select)?; + Ok(true) + } + /// Tries to fuse a Wasm `i32.eqz` (or `i32.eq` with 0 `rhs` value) instruction. /// /// Returns @@ -1978,7 +2038,7 @@ impl FuncTranslator { // Case: cannot fuse with non-zero `rhs` return Ok(false); } - let Some(last_instr) = self.instrs.last_instr() else { + let Some(last_instr) = self.instrs.peek_staged() else { // Case: cannot fuse without registered last instruction return Ok(false); }; @@ -2012,16 +2072,15 @@ impl FuncTranslator { }; // Need to push back `lhs` but with its type adjusted to be `i32` // since that's the return type of `i{32,64}.{eqz,eq,ne}`. - let result_idx = self.stack.push_temp(ValType::I32, lhs.instr())?; + let result_idx = self.stack.push_temp(ValType::I32)?; // Need to replace `cmp` instruction result register since it might // have been misaligned if `lhs` originally referred to the zero operand. let new_result = self.layout.temp_to_slot(result_idx)?; let Some(negated) = negated.update_result_slot(new_result) else { unreachable!("`negated` has been asserted as `cmp` instruction"); }; - if !self.instrs.try_replace_instr(last_instr, negated)? { - unreachable!("`negated` has been asserted to be `last_instr`"); - } + let fuel_pos = self.stack.consume_fuel_instr(); + self.instrs.replace_staged(negated)?; Ok(true) } @@ -2302,8 +2361,8 @@ impl FuncTranslator { let rhs_hi = self.copy_if_immediate(rhs_hi)?; let lhs_lo = self.copy_if_immediate(lhs_lo)?; let lhs_hi = self.copy_if_immediate(lhs_hi)?; - let result_lo = self.stack.push_temp(ValType::I64, None)?; - let result_hi = self.stack.push_temp(ValType::I64, None)?; + let result_lo = self.stack.push_temp(ValType::I64)?; + let result_hi = self.stack.push_temp(ValType::I64)?; let result_lo = self.layout.temp_to_slot(result_lo)?; let result_hi = self.layout.temp_to_slot(result_hi)?; let Ok(results) = >::new(SlotSpan::new(result_lo)) else { @@ -2357,8 +2416,8 @@ impl FuncTranslator { (lhs, rhs) } }; - let result0 = self.stack.push_temp(ValType::I64, None)?; - let _result1 = self.stack.push_temp(ValType::I64, None)?; + let result0 = self.stack.push_temp(ValType::I64)?; + let _result1 = self.stack.push_temp(ValType::I64)?; let result0 = self.layout.temp_to_slot(result0)?; let Ok(results) = >::new(SlotSpan::new(result0)) else { return Err(Error::from(TranslationError::AllocatedTooManySlots)); diff --git a/crates/wasmi/src/engine/translator/func/stack/control.rs b/crates/wasmi/src/engine/translator/func/stack/control.rs index 139263a7e8..d655e79461 100644 --- a/crates/wasmi/src/engine/translator/func/stack/control.rs +++ b/crates/wasmi/src/engine/translator/func/stack/control.rs @@ -1,9 +1,10 @@ use super::{Operand, Reset}; use crate::{ engine::{ - translator::{labels::LabelRef, utils::OpPos}, + translator::func::{labels::LabelRef, Pos}, BlockType, }, + ir, Engine, }; use alloc::vec::{Drain, Vec}; @@ -43,7 +44,7 @@ pub struct ControlStack { /// fuel consumption instruction since this information is accessed commonly. /// /// [`Op`]: crate::ir::Op - consume_fuel_instr: Option, + consume_fuel_instr: Option>, /// Special operand stack to memorize operands for `else` control frames. else_operands: ElseOperands, /// This is `true` if an `if` with else providers was just popped from the stack. @@ -101,7 +102,7 @@ impl ControlStack { /// /// Returns `None` otherwise. #[inline] - pub fn consume_fuel_instr(&self) -> Option { + pub fn consume_fuel_instr(&self) -> Option> { debug_assert!(!self.is_empty()); self.consume_fuel_instr } @@ -128,7 +129,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) { debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(BlockControlFrame { @@ -147,7 +148,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) { debug_assert!(!self.orphaned_else_operands); self.frames.push(ControlFrame::from(LoopControlFrame { @@ -166,7 +167,7 @@ impl ControlStack { ty: BlockType, height: usize, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, reachability: IfReachability, else_operands: impl IntoIterator, ) { @@ -191,7 +192,7 @@ impl ControlStack { pub fn push_else( &mut self, if_frame: IfControlFrame, - consume_fuel: Option, + consume_fuel: Option>, is_end_of_then_reachable: bool, ) { debug_assert!(!self.orphaned_else_operands); @@ -305,7 +306,7 @@ impl<'a> ControlFrameBase for ControlFrameMut<'a> { self.0.len_branch_params(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.0.consume_fuel_instr() } } @@ -418,7 +419,7 @@ pub trait ControlFrameBase { /// Returns a reference to the [`Op::ConsumeFuel`] of `self`. /// /// Returns `None` if fuel metering is disabled. - fn consume_fuel_instr(&self) -> Option; + fn consume_fuel_instr(&self) -> Option>; } impl ControlFrameBase for ControlFrame { @@ -496,7 +497,7 @@ impl ControlFrameBase for ControlFrame { } } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { match self { ControlFrame::Block(frame) => frame.consume_fuel_instr(), ControlFrame::Loop(frame) => frame.consume_fuel_instr(), @@ -523,7 +524,7 @@ pub struct BlockControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`BlockControlFrame`]. label: LabelRef, } @@ -553,7 +554,7 @@ impl ControlFrameBase for BlockControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } @@ -572,7 +573,7 @@ pub struct LoopControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`LoopControlFrame`]. label: LabelRef, } @@ -602,7 +603,7 @@ impl ControlFrameBase for LoopControlFrame { self.ty.len_params(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } @@ -621,7 +622,7 @@ pub struct IfControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`IfControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -672,7 +673,7 @@ impl ControlFrameBase for IfControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } @@ -716,7 +717,7 @@ pub struct ElseControlFrame { /// # Note /// /// This is `Some` if fuel metering is enabled and `None` otherwise. - consume_fuel: Option, + consume_fuel: Option>, /// The label used to branch to the [`ElseControlFrame`]. label: LabelRef, /// The reachability of the `then` and `else` blocks. @@ -807,7 +808,7 @@ impl ControlFrameBase for ElseControlFrame { self.ty.len_results(engine) } - fn consume_fuel_instr(&self) -> Option { + fn consume_fuel_instr(&self) -> Option> { self.consume_fuel } } diff --git a/crates/wasmi/src/engine/translator/func/stack/mod.rs b/crates/wasmi/src/engine/translator/func/stack/mod.rs index f9d7350df4..a4e0299c6d 100644 --- a/crates/wasmi/src/engine/translator/func/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func/stack/mod.rs @@ -30,13 +30,10 @@ use super::{Reset, ReusableAllocations}; use crate::{ core::TypedVal, engine::{ - translator::{ - func::{stack::operands::PeekedOperands, LocalIdx}, - labels::LabelRef, - utils::OpPos, - }, + translator::func::{labels::LabelRef, stack::operands::PeekedOperands, LocalIdx, Pos}, BlockType, }, + ir, Engine, Error, ValType, @@ -160,7 +157,7 @@ impl Stack { &mut self, ty: BlockType, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -201,7 +198,7 @@ impl Stack { &mut self, ty: BlockType, label: LabelRef, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -230,7 +227,7 @@ impl Stack { ty: BlockType, label: LabelRef, reachability: IfReachability, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(!self.controls.is_empty()); debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); @@ -262,7 +259,7 @@ impl Stack { &mut self, if_frame: IfControlFrame, is_end_of_then_reachable: bool, - consume_fuel: Option, + consume_fuel: Option>, ) -> Result<(), Error> { debug_assert!(self.is_fuel_metering_enabled() == consume_fuel.is_some()); self.push_else_operands(&if_frame)?; @@ -361,8 +358,8 @@ impl Stack { /// /// If too many operands have been pushed onto the [`Stack`]. #[inline] - pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { - self.operands.push_temp(ty, instr) + pub fn push_temp(&mut self, ty: ValType) -> Result { + self.operands.push_temp(ty, None) } /// Pushes an immediate `value` on the [`Stack`]. @@ -523,7 +520,7 @@ impl Stack { /// /// Returns `None` otherwise. #[inline] - pub fn consume_fuel_instr(&self) -> Option { + pub fn consume_fuel_instr(&self) -> Option> { self.controls.consume_fuel_instr() } } diff --git a/crates/wasmi/src/engine/translator/func/stack/operand.rs b/crates/wasmi/src/engine/translator/func/stack/operand.rs index 819ea422d6..de2cae6188 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operand.rs @@ -1,5 +1,5 @@ use super::{LocalIdx, OperandIdx, StackOperand}; -use crate::{core::TypedVal, engine::translator::utils::OpPos, ValType}; +use crate::{core::TypedVal, engine::translator::func::encoder::BytePos, ValType}; #[cfg(doc)] use super::Stack; @@ -47,7 +47,7 @@ impl Operand { } /// Creates a temporary [`Operand`]. - pub(super) fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { + pub(super) fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { Self::Temp(TempOperand { operand_index, ty, @@ -126,7 +126,7 @@ pub struct TempOperand { /// The type of the temporary. ty: ValType, /// The instruction which created this [`TempOperand`] as its result if any. - instr: Option, + instr: Option, } impl From for Operand { @@ -147,7 +147,7 @@ impl TempOperand { } /// Returns the instruction whcih created this [`TempOperand`] as its result if any. - pub fn instr(&self) -> Option { + pub fn instr(&self) -> Option { self.instr } } diff --git a/crates/wasmi/src/engine/translator/func/stack/operands.rs b/crates/wasmi/src/engine/translator/func/stack/operands.rs index f873862378..6ded544d1c 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operands.rs @@ -1,5 +1,5 @@ use super::{LocalIdx, LocalsHead, Operand, Reset}; -use crate::{core::TypedVal, engine::translator::utils::OpPos, Error, ValType}; +use crate::{core::TypedVal, engine::translator::func::encoder::BytePos, Error, ValType}; use alloc::vec::Vec; use core::{num::NonZero, slice}; @@ -48,7 +48,7 @@ pub enum StackOperand { /// The type of the temporary value. ty: ValType, /// The instruction which has this [`StackOperand`] as result if any. - instr: Option, + instr: Option, }, /// An immediate value on the [`OperandStack`]. Immediate { @@ -183,7 +183,7 @@ impl OperandStack { /// /// If too many operands have been pushed onto the [`OperandStack`]. #[inline] - pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { + pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { let idx = self.next_index(); self.operands.push(StackOperand::Temp { ty, instr }); self.update_max_stack_height(); diff --git a/crates/wasmi/src/engine/translator/func/visit.rs b/crates/wasmi/src/engine/translator/func/visit.rs index 94f8c147ef..68e46923ef 100644 --- a/crates/wasmi/src/engine/translator/func/visit.rs +++ b/crates/wasmi/src/engine/translator/func/visit.rs @@ -125,11 +125,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let consume_fuel = self.stack.consume_fuel_instr(); self.move_operands_to_temp(usize::from(len_params), consume_fuel)?; self.pin_label(continue_label); - let consume_fuel = self.instrs.push_consume_fuel_instr()?; + let consume_fuel = self.instrs.encode_consume_fuel()?; self.stack .push_loop(block_ty, continue_label, consume_fuel)?; // Need to reset `last_instr` because a loop header is a control flow boundary. - self.instrs.reset_last_instr(); + self.instrs.try_encode_staged(); Ok(()) } @@ -159,7 +159,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { let else_label = self.labels.new_label(); self.encode_br_eqz(condition, else_label)?; let reachability = IfReachability::Both { else_label }; - let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + let consume_fuel_instr = self.instrs.encode_consume_fuel()?; (reachability, consume_fuel_instr) } }; @@ -193,11 +193,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } // Start of `else` block: self.labels - .pin_label(else_label, self.instrs.next_instr()) + .pin_label(else_label, self.instrs.next_pos()) .unwrap(); - self.instrs.reset_last_instr(); + self.instrs.try_encode_staged(); } - let consume_fuel_instr = self.instrs.push_consume_fuel_instr()?; + let consume_fuel_instr = self.instrs.encode_consume_fuel()?; self.reachable = frame.is_else_reachable(); self.stack .push_else(frame, is_end_of_then_reachable, consume_fuel_instr)?; @@ -277,7 +277,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator { } self.encode_br(label)?; self.labels - .pin_label(skip_label, self.instrs.next_instr()) + .pin_label(skip_label, self.instrs.next_pos()) .unwrap(); Ok(()) } @@ -1824,11 +1824,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { self.stack.push_local(input.local_index(), ValType::I64)?; self.visit_i64_eqz() } - Operand::Temp(input) => { + Operand::Temp(_) => { // Note: `funcref` and `externref` both serialize to `UntypedValue` // as `u64` so we can use `i64.eqz` translation for `ref.is_null` // via reinterpretation of the value's type. - self.stack.push_temp(ValType::I64, input.instr())?; + self.stack.push_temp(ValType::I64)?; self.visit_i64_eqz() } Operand::Immediate(input) => { diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 180b255464..303b1fada1 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -4,7 +4,6 @@ mod comparator; mod driver; mod error; mod func; -mod labels; mod relink_result; mod utils; diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index 7b1714f2f9..e6152fb9a3 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -161,22 +161,6 @@ impl BumpFuelConsumption for Op { } } -/// A reference to an encoded [`Op`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct OpPos(usize); - -impl From for OpPos { - fn from(index: usize) -> Self { - Self(index) - } -} - -impl From for usize { - fn from(pos: OpPos) -> Self { - pos.0 - } -} - /// Types that can be converted into bits. pub trait ToBits { /// The output bits type of [`ToBits`]. From d61fb01c9a00aa806132dc4892e0f18969b3bc39 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 27 Sep 2025 11:12:07 +0200 Subject: [PATCH 120/423] add OpEncoder::encode_param API --- .../src/engine/translator/func/encoder.rs | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/wasmi/src/engine/translator/func/encoder.rs b/crates/wasmi/src/engine/translator/func/encoder.rs index bd51e23504..b6948e0a88 100644 --- a/crates/wasmi/src/engine/translator/func/encoder.rs +++ b/crates/wasmi/src/engine/translator/func/encoder.rs @@ -355,6 +355,22 @@ impl OpEncoder { Ok(pos) } + /// Encodes the given `param` of type `T` to the [`OpEncoder`]. + /// + /// # Note + /// + /// This is usually used to encode parameters of certain variable width [`Op`]s + /// such as for the encoding of Wasm's `br_table`. + pub fn encode_param(&mut self, param: T) -> Result<(), Error> + where + T: ir::Encode, + { + debug_assert!(self.staged.is_none()); + self.encode_impl(param)?; + debug_assert!(self.ops.take_temp().is_none()); + Ok(()) + } + /// Encodes an [`Op::ConsumeFuel`] operator to `self`. /// /// # Note @@ -402,7 +418,10 @@ impl OpEncoder { /// /// - Encodes `last` [`Op`] prior to `op` if `last` is `Some`. /// - After this call `last` will yield `None`. - fn encode_impl(&mut self, op: Op) -> Result, Error> { + fn encode_impl(&mut self, op: T) -> Result, Error> + where + T: ir::Encode, + { let pos = self.ops.next_pos(); op.encode(&mut self.ops)?; Ok(Pos::from(pos)) From ba76edfe23bb684281b552f086c7e6dd1b85a998 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 27 Sep 2025 13:05:01 +0200 Subject: [PATCH 121/423] remove StackOperand::Temp::instr field --- .../src/engine/translator/func/stack/mod.rs | 2 +- .../src/engine/translator/func/stack/operand.rs | 17 +++-------------- .../engine/translator/func/stack/operands.rs | 10 ++++------ 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/stack/mod.rs b/crates/wasmi/src/engine/translator/func/stack/mod.rs index a4e0299c6d..208e0870c8 100644 --- a/crates/wasmi/src/engine/translator/func/stack/mod.rs +++ b/crates/wasmi/src/engine/translator/func/stack/mod.rs @@ -359,7 +359,7 @@ impl Stack { /// If too many operands have been pushed onto the [`Stack`]. #[inline] pub fn push_temp(&mut self, ty: ValType) -> Result { - self.operands.push_temp(ty, None) + self.operands.push_temp(ty) } /// Pushes an immediate `value` on the [`Stack`]. diff --git a/crates/wasmi/src/engine/translator/func/stack/operand.rs b/crates/wasmi/src/engine/translator/func/stack/operand.rs index de2cae6188..f2e24de632 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operand.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operand.rs @@ -22,7 +22,7 @@ impl Operand { StackOperand::Local { local_index, ty, .. } => Self::local(index, local_index, ty), - StackOperand::Temp { ty, instr } => Self::temp(index, ty, instr), + StackOperand::Temp { ty } => Self::temp(index, ty), StackOperand::Immediate { val } => Self::immediate(index, val), } } @@ -47,12 +47,8 @@ impl Operand { } /// Creates a temporary [`Operand`]. - pub(super) fn temp(operand_index: OperandIdx, ty: ValType, instr: Option) -> Self { - Self::Temp(TempOperand { - operand_index, - ty, - instr, - }) + pub(super) fn temp(operand_index: OperandIdx, ty: ValType) -> Self { + Self::Temp(TempOperand { operand_index, ty }) } /// Creates an immediate [`Operand`]. @@ -125,8 +121,6 @@ pub struct TempOperand { operand_index: OperandIdx, /// The type of the temporary. ty: ValType, - /// The instruction which created this [`TempOperand`] as its result if any. - instr: Option, } impl From for Operand { @@ -145,11 +139,6 @@ impl TempOperand { pub fn ty(&self) -> ValType { self.ty } - - /// Returns the instruction whcih created this [`TempOperand`] as its result if any. - pub fn instr(&self) -> Option { - self.instr - } } /// An immediate value on the [`Stack`]. diff --git a/crates/wasmi/src/engine/translator/func/stack/operands.rs b/crates/wasmi/src/engine/translator/func/stack/operands.rs index 6ded544d1c..809ed15bd3 100644 --- a/crates/wasmi/src/engine/translator/func/stack/operands.rs +++ b/crates/wasmi/src/engine/translator/func/stack/operands.rs @@ -47,8 +47,6 @@ pub enum StackOperand { Temp { /// The type of the temporary value. ty: ValType, - /// The instruction which has this [`StackOperand`] as result if any. - instr: Option, }, /// An immediate value on the [`OperandStack`]. Immediate { @@ -147,7 +145,7 @@ impl OperandStack { pub fn push_operand(&mut self, operand: Operand) -> Result { match operand { Operand::Local(operand) => self.push_local(operand.local_index(), operand.ty()), - Operand::Temp(operand) => self.push_temp(operand.ty(), operand.instr()), + Operand::Temp(operand) => self.push_temp(operand.ty()), Operand::Immediate(operand) => self.push_immediate(operand.val()), } } @@ -183,9 +181,9 @@ impl OperandStack { /// /// If too many operands have been pushed onto the [`OperandStack`]. #[inline] - pub fn push_temp(&mut self, ty: ValType, instr: Option) -> Result { + pub fn push_temp(&mut self, ty: ValType) -> Result { let idx = self.next_index(); - self.operands.push(StackOperand::Temp { ty, instr }); + self.operands.push(StackOperand::Temp { ty }); self.update_max_stack_height(); Ok(idx) } @@ -290,7 +288,7 @@ impl OperandStack { let operand = self.get_at(index); let ty = operand.ty(); self.try_unlink_local(operand); - self.operands[usize::from(index)] = StackOperand::Temp { ty, instr: None }; + self.operands[usize::from(index)] = StackOperand::Temp { ty }; operand } From effb2367f6ede175bb47b56c0de2bd6f2bef2a48 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 27 Sep 2025 13:05:36 +0200 Subject: [PATCH 122/423] refactor try_fuse_branch_cmp implementation --- .../wasmi/src/engine/translator/func/mod.rs | 71 ++++++------------- 1 file changed, 20 insertions(+), 51 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index ad22a2d5df..64e44e17e8 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -1318,7 +1318,7 @@ impl FuncTranslator { label: LabelRef, negate: bool, ) -> Result { - let Some(last_instr) = self.instrs.peek_staged() else { + let Some(staged_op) = self.instrs.peek_staged() else { // Case: cannot fuse without a known last instruction return Ok(false); }; @@ -1328,68 +1328,37 @@ impl FuncTranslator { // - immediates cannot be the result of a previous instruction. return Ok(false); }; - let Some(origin) = condition.instr() else { - // Case: cannot fuse temporary operands without origin instruction - return Ok(false); - }; - if last_instr != origin { - // Case: cannot fuse if last instruction does not match origin instruction - return Ok(false); - } debug_assert!(matches!(condition.ty(), ValType::I32 | ValType::I64)); - let fused_instr = self.try_make_fused_branch_cmp_instr(origin, condition, label, negate)?; - let Some(fused_instr) = fused_instr else { - // Case: not possible to perform fusion with last instruction + let Some(cmp_result) = staged_op.result_ref().copied() else { + // Note: `cmp` operators must have a result. return Ok(false); }; - self.instrs.replace_staged(fused_instr)?; - Ok(true) - } - - /// Try to return a fused cmp+branch [`Op`] from the given parameters. - /// - /// - /// # Note - /// - /// - The `instr` parameter refers to the to-be-fused cmp instruction. - /// - Returns `Ok(Some)` if cmp+branch fusion was successful. - /// - Returns `Ok(None)`, otherwise. - fn try_make_fused_branch_cmp_instr( - &mut self, - instr: Pos, - condition: TempOperand, - label: LabelRef, - negate: bool, - ) -> Result, Error> { - let cmp_instr = *self.instrs.get(instr); - let Some(result) = cmp_instr.compare_result() else { - // Note: cannot fuse non-cmp instructions or cmp-instructions without result. - return Ok(None); - }; - if matches!(self.layout.stack_space(result), StackSpace::Local) { - // Note: cannot fuse cmp instructions with observable semantics. - return Ok(None); + if matches!(self.layout.stack_space(cmp_result), StackSpace::Local) { + // Note: local variable results have observable behavior which must not change. + return Ok(false); } - if result != self.layout.temp_to_slot(condition)? { + let br_condition = self.layout.temp_to_slot(condition)?; + if cmp_result != br_condition { // Note: cannot fuse cmp instruction with a result that differs - // from the condition operand. - return Ok(None); + // from the branch condition operand. + return Ok(false); } - let cmp_instr = match negate { - false => cmp_instr, - true => match cmp_instr.negate_cmp_instr() { + let cmp_op = match negate { + false => staged_op, + true => match staged_op.negate_cmp_instr() { Some(negated) => negated, None => { - // Note: cannot negate cmp instruction, thus not possible to fuse. - return Ok(None); + // Note: cannot negate staged [`Op`], thus it is not a `cmp` operator and thus not fusable. + return Ok(false); } }, }; let offset = self.labels.try_resolve_label(label, instr)?; - let fused = cmp_instr - .try_into_cmp_branch_instr(offset) - .expect("cmp+branch fusion must succeed"); - Ok(Some(fused)) + let Some(fused_cmp_branch) = cmp_op.try_into_cmp_branch_instr(offset) else { + return Ok(false); + }; + self.instrs.replace_staged(fused_cmp_branch)?; + Ok(true) } /// Generically translates a `call` or `return_call` Wasm operator. From 80a814052f807afa6e2c2418273a02b0df5d4237 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 27 Sep 2025 13:05:52 +0200 Subject: [PATCH 123/423] refactor fuse_commutative_cmp_with implementation --- crates/wasmi/src/engine/translator/func/mod.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 64e44e17e8..60a313b43f 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -2007,7 +2007,7 @@ impl FuncTranslator { // Case: cannot fuse with non-zero `rhs` return Ok(false); } - let Some(last_instr) = self.instrs.peek_staged() else { + let Some(staged) = self.instrs.peek_staged() else { // Case: cannot fuse without registered last instruction return Ok(false); }; @@ -2017,17 +2017,8 @@ impl FuncTranslator { // - immediates cannot be the result of a previous instruction. return Ok(false); }; - let Some(origin) = lhs.instr() else { - // Case: `lhs` has no origin instruciton, thus not possible to fuse. - return Ok(false); - }; - if origin != last_instr { - // Case: `lhs`'s origin instruction does not match the last instruction - return Ok(false); - } let lhs_reg = self.layout.temp_to_slot(lhs)?; - let last_instruction = self.instrs.get(last_instr); - let Some(result) = last_instruction.compare_result() else { + let Some(result) = staged.result_ref().copied() else { // Case: cannot fuse non-cmp instructions return Ok(false); }; @@ -2035,7 +2026,7 @@ impl FuncTranslator { // Case: the `cmp` instruction does not feed into the `eqz` and cannot be fused return Ok(false); } - let Some(negated) = try_fuse(last_instruction) else { + let Some(negated) = try_fuse(&staged) else { // Case: the `cmp` instruction cannot be negated return Ok(false); }; @@ -2048,7 +2039,6 @@ impl FuncTranslator { let Some(negated) = negated.update_result_slot(new_result) else { unreachable!("`negated` has been asserted as `cmp` instruction"); }; - let fuel_pos = self.stack.consume_fuel_instr(); self.instrs.replace_staged(negated)?; Ok(true) } From 8a4ec4ea52106cef3327db7e4dc4ca64b04f0a2d Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 27 Sep 2025 15:05:11 +0200 Subject: [PATCH 124/423] use Pos in LabelRegistry API --- .../src/engine/translator/func/labels.rs | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/labels.rs b/crates/wasmi/src/engine/translator/func/labels.rs index 9204c0cf2a..1e58873a5a 100644 --- a/crates/wasmi/src/engine/translator/func/labels.rs +++ b/crates/wasmi/src/engine/translator/func/labels.rs @@ -1,6 +1,9 @@ use crate::{ - engine::{translator::func::encoder::BytePos, TranslationError}, - ir::BranchOffset, + engine::{ + translator::func::encoder::{BytePos, Pos}, + TranslationError, + }, + ir::{BranchOffset, Op}, Error, }; use alloc::vec::Vec; @@ -12,8 +15,8 @@ use core::{ /// A label during the Wasmi compilation process. #[derive(Debug, Copy, Clone)] pub enum Label { - /// The label has already been pinned to a particular [`OpPos`]. - Pinned(BytePos), + /// The label has already been pinned to a particular [`Pos`]. + Pinned(Pos), /// The label is still unpinned. Unpinned, } @@ -45,12 +48,12 @@ pub struct LabelUser { /// The label in use by the user. label: LabelRef, /// The reference to the using instruction. - user: BytePos, + user: Pos, } impl LabelUser { /// Creates a new [`LabelUser`]. - pub fn new(label: LabelRef, user: BytePos) -> Self { + pub fn new(label: LabelRef, user: Pos) -> Self { Self { label, user } } } @@ -59,7 +62,7 @@ impl LabelUser { #[derive(Debug, Copy, Clone)] pub enum LabelError { /// When trying to pin an already pinned [`Label`]. - AlreadyPinned { label: LabelRef, pinned_to: BytePos }, + AlreadyPinned { label: LabelRef, pinned_to: Pos }, /// When trying to resolve an unpinned [`Label`]. Unpinned { label: LabelRef }, } @@ -117,7 +120,7 @@ impl LabelRegistry { /// # Errors /// /// If the `label` has already been pinned to some other [`OpPos`]. - pub fn pin_label(&mut self, label: LabelRef, instr: BytePos) -> Result<(), LabelError> { + pub fn pin_label(&mut self, label: LabelRef, instr: Pos) -> Result<(), LabelError> { match self.get_label_mut(label) { Label::Pinned(pinned) => Err(LabelError::AlreadyPinned { label, @@ -131,7 +134,7 @@ impl LabelRegistry { } /// Pins the `label` to the given `instr` if unpinned. - pub fn try_pin_label(&mut self, label: LabelRef, instr: BytePos) { + pub fn try_pin_label(&mut self, label: LabelRef, instr: Pos) { if let unpinned @ Label::Unpinned = self.get_label_mut(label) { *unpinned = Label::Pinned(instr) } @@ -142,10 +145,10 @@ impl LabelRegistry { /// # Errors /// /// If the resulting [`BranchOffset`] is out of bounds. - pub fn trace_branch_offset(src: BytePos, dst: BytePos) -> Result { - fn trace_offset32(src: BytePos, dst: BytePos) -> Option { - let src = isize::try_from(usize::from(src)).ok()?; - let dst = isize::try_from(usize::from(dst)).ok()?; + pub fn trace_branch_offset(src: Pos, dst: Pos) -> Result { + fn trace_offset32(src: Pos, dst: Pos) -> Option { + let src = isize::try_from(usize::from(BytePos::from(src))).ok()?; + let dst = isize::try_from(usize::from(BytePos::from(dst))).ok()?; let offset = dst.checked_sub(src)?; i32::try_from(offset).ok() } @@ -165,7 +168,7 @@ impl LabelRegistry { pub fn try_resolve_label( &mut self, label: LabelRef, - user: BytePos, + user: Pos, ) -> Result { let offset = match *self.get_label(label) { Label::Pinned(target) => Self::trace_branch_offset(user, target)?, @@ -182,7 +185,7 @@ impl LabelRegistry { /// # Errors /// /// If the `label` is unpinned. - fn resolve_label(&self, label: LabelRef) -> Result { + fn resolve_label(&self, label: LabelRef) -> Result, LabelError> { match self.get_label(label) { Label::Pinned(instr) => Ok(*instr), Label::Unpinned => Err(LabelError::Unpinned { label }), @@ -216,7 +219,7 @@ pub struct ResolvedUserIter<'a> { } impl Iterator for ResolvedUserIter<'_> { - type Item = (BytePos, Result); + type Item = (Pos, Result); fn next(&mut self) -> Option { let next = self.users.next()?; From 6533e951e902892ca23b2ed91ccf2070e04081aa Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sat, 27 Sep 2025 15:05:20 +0200 Subject: [PATCH 125/423] fix some warnings --- crates/wasmi/src/engine/translator/func/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/mod.rs b/crates/wasmi/src/engine/translator/func/mod.rs index 60a313b43f..812a56357f 100644 --- a/crates/wasmi/src/engine/translator/func/mod.rs +++ b/crates/wasmi/src/engine/translator/func/mod.rs @@ -43,7 +43,6 @@ use crate::{ translator::{ comparator::{ CmpSelectFusion, - CompareResult as _, LogicalizeCmpInstr, NegateCmpInstr, TryIntoCmpBranchInstr as _, @@ -848,7 +847,7 @@ impl FuncTranslator { panic!("out of bounds `br_table` target does not fit `usize`: {target:?}"); }; let mut frame = self.stack.peek_control_mut(depth).control_frame(); - let offset = self + let _offset = self .labels .try_resolve_label(frame.label(), self.instrs.next_pos())?; // self.instrs.push_param(Op::branch(offset)); // TODO: finish encoding impl From d45b1e592287ceb85d5762085a3f3a4384e7ae66 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 28 Sep 2025 10:56:32 +0200 Subject: [PATCH 126/423] improve LabelError code --- crates/wasmi/src/engine/translator/func/labels.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/labels.rs b/crates/wasmi/src/engine/translator/func/labels.rs index 1e58873a5a..07f0b65b3b 100644 --- a/crates/wasmi/src/engine/translator/func/labels.rs +++ b/crates/wasmi/src/engine/translator/func/labels.rs @@ -8,6 +8,7 @@ use crate::{ }; use alloc::vec::Vec; use core::{ + error::Error as CoreError, fmt::{self, Display}, slice::Iter as SliceIter, }; @@ -67,18 +68,18 @@ pub enum LabelError { Unpinned { label: LabelRef }, } -impl core::error::Error for LabelError {} +impl CoreError for LabelError {} impl Display for LabelError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - LabelError::AlreadyPinned { label, pinned_to } => { + Self::AlreadyPinned { label, pinned_to } => { write!( f, "trying to pin already pinned label {label:?} (pinned to {pinned_to:?})" ) } - LabelError::Unpinned { label } => { + Self::Unpinned { label } => { write!(f, "trying to resolve unpinned label: {label:?}") } } From ea3497ddb6123e4d5ab78f486130f6ebaa968977 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 28 Sep 2025 10:57:44 +0200 Subject: [PATCH 127/423] use usize for LabelRef --- .../src/engine/translator/func/labels.rs | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/labels.rs b/crates/wasmi/src/engine/translator/func/labels.rs index 07f0b65b3b..f5e9cb89c9 100644 --- a/crates/wasmi/src/engine/translator/func/labels.rs +++ b/crates/wasmi/src/engine/translator/func/labels.rs @@ -24,13 +24,19 @@ pub enum Label { /// A reference to an [`Label`]. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct LabelRef(u32); +pub struct LabelRef(usize); -impl LabelRef { - /// Returns the `usize` value of the [`LabelRef`]. +impl From for LabelRef { #[inline] - fn into_usize(self) -> usize { - self.0 as usize + fn from(value: usize) -> Self { + Self(value) + } +} + +impl From for usize { + #[inline] + fn from(value: LabelRef) -> Self { + value.0 } } @@ -95,25 +101,21 @@ impl LabelRegistry { /// Allocates a new unpinned [`Label`]. pub fn new_label(&mut self) -> LabelRef { - let index: u32 = self - .labels - .len() - .try_into() - .unwrap_or_else(|err| panic!("cannot have more than u32::MAX label refs: {err}")); + let index = self.labels.len(); self.labels.push(Label::Unpinned); - LabelRef(index) + LabelRef::from(index) } /// Returns a shared reference to the underlying [`Label`]. #[inline] fn get_label(&self, label: LabelRef) -> &Label { - &self.labels[label.into_usize()] + &self.labels[usize::from(label)] } /// Returns an exclusive reference to the underlying [`Label`]. #[inline] fn get_label_mut(&mut self, label: LabelRef) -> &mut Label { - &mut self.labels[label.into_usize()] + &mut self.labels[usize::from(label)] } /// Pins the `label` to the given `instr`. From 0204e6b7ebf0672254d05be92b5b8591c98be868 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 28 Sep 2025 10:58:10 +0200 Subject: [PATCH 128/423] add `offset: Pos` field to LabelUser --- .../wasmi/src/engine/translator/func/labels.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/wasmi/src/engine/translator/func/labels.rs b/crates/wasmi/src/engine/translator/func/labels.rs index f5e9cb89c9..849a737c77 100644 --- a/crates/wasmi/src/engine/translator/func/labels.rs +++ b/crates/wasmi/src/engine/translator/func/labels.rs @@ -52,16 +52,22 @@ pub struct LabelRegistry { /// A user of a label. #[derive(Debug)] pub struct LabelUser { - /// The label in use by the user. - label: LabelRef, + /// The branch `target` label in use by the `user`. + target: LabelRef, /// The reference to the using instruction. user: Pos, + /// The [`BranchOffset`] of `user` that needs to be updated once `target` has been resolved. + offset: Pos, } impl LabelUser { /// Creates a new [`LabelUser`]. - pub fn new(label: LabelRef, user: Pos) -> Self { - Self { label, user } + pub fn new(target: LabelRef, user: Pos, offset: Pos) -> Self { + Self { + target, + user, + offset, + } } } @@ -229,7 +235,7 @@ impl Iterator for ResolvedUserIter<'_> { let src = next.user; let dst = self .registry - .resolve_label(next.label) + .resolve_label(next.target) .unwrap_or_else(|err| panic!("failed to resolve user: {err}")); let offset = LabelRegistry::trace_branch_offset(src, dst); Some((src, offset)) From 05f2eb2996f83cf5704c4851db6bfd3a6d1cde6b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Sun, 28 Sep 2025 10:59:27 +0200 Subject: [PATCH 129/423] add docs to fields --- crates/wasmi/src/engine/translator/func/labels.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/wasmi/src/engine/translator/func/labels.rs b/crates/wasmi/src/engine/translator/func/labels.rs index 849a737c77..488021eca3 100644 --- a/crates/wasmi/src/engine/translator/func/labels.rs +++ b/crates/wasmi/src/engine/translator/func/labels.rs @@ -45,7 +45,9 @@ impl From for usize { /// Allows to allocate new labels pin them and resolve pinned ones. #[derive(Debug, Default)] pub struct LabelRegistry { + /// All registered labels, pinned or unpinned. labels: Vec