diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 082d7113da7d..ddd9191c2898 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -101,6 +101,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { mut args: ArgsAccumulator, ) -> CodegenResult<(u32, Option)> { let is_fastcall = call_conv == CallConv::WindowsFastcall; + let is_tail = call_conv == CallConv::Tail; let mut next_gpr = 0; let mut next_vreg = 0; @@ -181,6 +182,11 @@ impl ABIMachineSpec for X64ABIMachineSpec { // This is consistent with LLVM's behavior, and is needed for // some uses of Cranelift (e.g., the rustc backend). // + // - Otherwise, if the calling convention is Tail, we behave as in + // the previous case, even if `enable_llvm_abi_extensions` is not + // set in the flags: This is a custom calling convention defined + // by Cranelift, LLVM doesn't know about it. + // // - Otherwise, both SysV and Fastcall specify behavior (use of // vector register, a register pair, or passing by reference // depending on the case), but for simplicity, we will just panic if @@ -194,6 +200,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { if param.value_type.bits() > 64 && !(param.value_type.is_vector() || param.value_type.is_float()) && !flags.enable_llvm_abi_extensions() + && !is_tail { panic!( "i128 args/return values not supported unless LLVM ABI extensions are enabled" diff --git a/crates/c-api/src/val.rs b/crates/c-api/src/val.rs index efc53cf52968..c99b7cd41f66 100644 --- a/crates/c-api/src/val.rs +++ b/crates/c-api/src/val.rs @@ -98,6 +98,7 @@ impl wasm_val_t { Val::ExternRef(_) => crate::abort("creating a wasm_val_t from an externref"), Val::ExnRef(_) => crate::abort("creating a wasm_val_t from an exnref"), Val::V128(_) => crate::abort("creating a wasm_val_t from a v128"), + Val::ContRef(_) => crate::abort("creating a wasm_val_t from a contref"), } } @@ -259,6 +260,7 @@ impl wasmtime_val_t { v128: val.as_u128().to_le_bytes(), }, }, + Val::ContRef(_) => crate::abort("contrefs not yet supported in C API (#10248)"), } } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 3871f47b923d..877ecc14adbe 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1,4 +1,5 @@ mod gc; +pub(crate) mod stack_switching; use crate::compiler::Compiler; use crate::translate::{ @@ -171,6 +172,16 @@ pub struct FuncEnvironment<'module_environment> { /// always present even if this is a "leaf" function, as we have to call /// into the host to trap when signal handlers are disabled. pub(crate) stack_limit_at_function_entry: Option, + + /// Used by the stack switching feature. If set, we have a allocated a + /// slot on this function's stack to be used for the + /// current stack's `handler_list` field. + stack_switching_handler_list_buffer: Option, + + /// Used by the stack switching feature. If set, we have a allocated a + /// slot on this function's stack to be used for the + /// current continuation's `values` field. + stack_switching_values_buffer: Option, } impl<'module_environment> FuncEnvironment<'module_environment> { @@ -224,6 +235,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> { translation, stack_limit_at_function_entry: None, + + stack_switching_handler_list_buffer: None, + stack_switching_values_buffer: None, } } @@ -1958,8 +1972,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { return CheckIndirectCallTypeSignature::StaticTrap; } - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. - // Engine-indexed types don't show up until runtime and it's a Wasm // validation error to perform a call through a non-function table, // so these cases are dynamically not reachable. @@ -1977,6 +1989,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { | WasmHeapType::Exn | WasmHeapType::ConcreteExn(_) | WasmHeapType::NoExn + | WasmHeapType::Cont + | WasmHeapType::ConcreteCont(_) + | WasmHeapType::NoCont | WasmHeapType::None => { unreachable!() } @@ -2267,7 +2282,9 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm let needs_stack_map = match wasm_ty.top() { WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => true, WasmHeapTopType::Func => false, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + // TODO(#10248) Once continuations can be stored on the GC heap, we + // will need stack maps for continuation objects. + WasmHeapTopType::Cont => false, }; (ty, needs_stack_map) } @@ -2320,22 +2337,32 @@ impl FuncEnvironment<'_> { let mut pos = builder.cursor(); let table = self.table(table_index); let ty = table.ref_type.heap_type; - let grow = if ty.is_vmgcref_type() { - gc::builtins::table_grow_gc_ref(self, &mut pos.func)? - } else { - debug_assert_eq!(ty.top(), WasmHeapTopType::Func); - self.builtin_functions.table_grow_func_ref(&mut pos.func) - }; - let (table_vmctx, defined_table_index) = self.table_vmctx_and_defined_index(&mut pos, table_index); - let index_type = table.idx_type; let delta = self.cast_index_to_i64(&mut pos, delta, index_type); - let call_inst = pos - .ins() - .call(grow, &[table_vmctx, defined_table_index, delta, init_value]); - let result = pos.func.dfg.first_result(call_inst); + + let mut args: SmallVec<[_; 6]> = smallvec![table_vmctx, defined_table_index, delta]; + let grow = match ty.top() { + WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => { + args.push(init_value); + gc::builtins::table_grow_gc_ref(self, pos.func)? + } + WasmHeapTopType::Func => { + args.push(init_value); + self.builtin_functions.table_grow_func_ref(pos.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut pos, init_value); + args.extend_from_slice(&[contref, revision]); + stack_switching::builtins::table_grow_cont_obj(self, pos.func)? + } + }; + + let call_inst = pos.ins().call(grow, &args); + let result = builder.func.dfg.first_result(call_inst); + Ok(self.convert_pointer_to_index_type(builder.cursor(), result, index_type, false)) } @@ -2367,7 +2394,15 @@ impl FuncEnvironment<'_> { } // Continuation types. - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); + Ok(builder.ins().load( + stack_switching::fatpointer::fatpointer_type(self), + flags, + elem_addr, + 0, + )) + } } } @@ -2415,7 +2450,11 @@ impl FuncEnvironment<'_> { } // Continuation types. - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); + builder.ins().store(flags, value, elem_addr, 0); + Ok(()) + } } } @@ -2429,21 +2468,31 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let mut pos = builder.cursor(); let table = self.table(table_index); - let index_type = table.idx_type; - let dst = self.cast_index_to_i64(&mut pos, dst, index_type); - let len = self.cast_index_to_i64(&mut pos, len, index_type); let ty = table.ref_type.heap_type; - let libcall = if ty.is_vmgcref_type() { - gc::builtins::table_fill_gc_ref(self, &mut pos.func)? - } else { - debug_assert_eq!(ty.top(), WasmHeapTopType::Func); - self.builtin_functions.table_fill_func_ref(&mut pos.func) - }; - + let dst = self.cast_index_to_i64(&mut pos, dst, table.idx_type); + let len = self.cast_index_to_i64(&mut pos, len, table.idx_type); let (table_vmctx, table_index) = self.table_vmctx_and_defined_index(&mut pos, table_index); - pos.ins() - .call(libcall, &[table_vmctx, table_index, dst, val, len]); + let mut args: SmallVec<[_; 6]> = smallvec![table_vmctx, table_index, dst]; + let libcall = match ty.top() { + WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => { + args.push(val); + gc::builtins::table_fill_gc_ref(self, &mut pos.func)? + } + WasmHeapTopType::Func => { + args.push(val); + self.builtin_functions.table_fill_func_ref(&mut pos.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut pos, val); + args.extend_from_slice(&[contref, revision]); + stack_switching::builtins::table_fill_cont_obj(self, &mut pos.func)? + } + }; + + args.push(len); + builder.ins().call(libcall, &args); Ok(()) } @@ -2795,7 +2844,10 @@ impl FuncEnvironment<'_> { WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => { pos.ins().iconst(types::I32, 0) } - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + let zero = pos.ins().iconst(self.pointer_type(), 0); + stack_switching::fatpointer::construct(self, &mut pos, zero, zero) + } }) } @@ -2811,9 +2863,18 @@ impl FuncEnvironment<'_> { return Ok(pos.ins().iconst(ir::types::I32, 0)); } - let byte_is_null = - pos.ins() - .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0); + let byte_is_null = match ty.heap_type.top() { + WasmHeapTopType::Cont => { + let (_revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut pos, value); + pos.ins() + .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, contref, 0) + } + _ => pos + .ins() + .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0), + }; + Ok(pos.ins().uextend(ir::types::I32, byte_is_null)) } @@ -3556,6 +3617,118 @@ impl FuncEnvironment<'_> { self.isa.triple().architecture == target_lexicon::Architecture::X86_64 } + pub fn translate_cont_bind( + &mut self, + builder: &mut FunctionBuilder<'_>, + contobj: ir::Value, + args: &[ir::Value], + ) -> ir::Value { + stack_switching::instructions::translate_cont_bind(self, builder, contobj, args) + } + + pub fn translate_cont_new( + &mut self, + builder: &mut FunctionBuilder<'_>, + func: ir::Value, + arg_types: &[WasmValType], + return_types: &[WasmValType], + ) -> WasmResult { + stack_switching::instructions::translate_cont_new( + self, + builder, + func, + arg_types, + return_types, + ) + } + + pub fn translate_resume( + &mut self, + builder: &mut FunctionBuilder<'_>, + type_index: u32, + contobj: ir::Value, + resume_args: &[ir::Value], + resumetable: &[(u32, Option)], + ) -> WasmResult> { + stack_switching::instructions::translate_resume( + self, + builder, + type_index, + contobj, + resume_args, + resumetable, + ) + } + + pub fn translate_suspend( + &mut self, + builder: &mut FunctionBuilder<'_>, + tag_index: u32, + suspend_args: &[ir::Value], + tag_return_types: &[ir::Type], + ) -> Vec { + stack_switching::instructions::translate_suspend( + self, + builder, + tag_index, + suspend_args, + tag_return_types, + ) + } + + /// Translates switch instructions. + pub fn translate_switch( + &mut self, + builder: &mut FunctionBuilder, + tag_index: u32, + contobj: ir::Value, + switch_args: &[ir::Value], + return_types: &[ir::Type], + ) -> WasmResult> { + stack_switching::instructions::translate_switch( + self, + builder, + tag_index, + contobj, + switch_args, + return_types, + ) + } + + pub fn continuation_arguments(&self, index: TypeIndex) -> &[WasmValType] { + let idx = self.module.types[index].unwrap_module_type_index(); + self.types[self.types[idx] + .unwrap_cont() + .clone() + .unwrap_module_type_index()] + .unwrap_func() + .params() + } + + pub fn continuation_returns(&self, index: TypeIndex) -> &[WasmValType] { + let idx = self.module.types[index].unwrap_module_type_index(); + self.types[self.types[idx] + .unwrap_cont() + .clone() + .unwrap_module_type_index()] + .unwrap_func() + .returns() + } + + pub fn tag_params(&self, tag_index: TagIndex) -> &[WasmValType] { + let idx = self.module.tags[tag_index].signature; + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .params() + } + + pub fn tag_returns(&self, tag_index: TagIndex) -> &[WasmValType] { + let idx = self.module.tags[tag_index].signature; + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .returns() + } + pub fn use_x86_blendv_for_relaxed_laneselect(&self, ty: Type) -> bool { self.isa.has_x86_blendv_lowering(ty) } @@ -4138,17 +4311,3 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { IndexType::I64 => I64, } } - -/// TODO(10248) This is removed in the next stack switching PR. It stops the -/// compiler from complaining about the stack switching libcalls being dead -/// code. -#[cfg(feature = "stack-switching")] -#[allow( - dead_code, - reason = "Dummy function to suppress more dead code warnings" -)] -pub fn use_stack_switching_libcalls() { - let _ = BuiltinFunctions::cont_new; - let _ = BuiltinFunctions::table_grow_cont_obj; - let _ = BuiltinFunctions::table_fill_cont_obj; -} diff --git a/crates/cranelift/src/func_environ/stack_switching/control_effect.rs b/crates/cranelift/src/func_environ/stack_switching/control_effect.rs new file mode 100644 index 000000000000..558fdfb87043 --- /dev/null +++ b/crates/cranelift/src/func_environ/stack_switching/control_effect.rs @@ -0,0 +1,65 @@ +use cranelift_codegen::ir; +use cranelift_codegen::ir::InstBuilder; +use cranelift_codegen::ir::types::{I32, I64}; +use cranelift_frontend::FunctionBuilder; + +/// Universal control effect. This structure encodes return signal, +/// resume signal, suspension signal, and handler index into a +/// u64 value. This instance is used at compile time. There is a runtime +/// counterpart in `continuations/src/lib.rs`. +/// We convert to and from u64 as follows: The low 32 bits of the u64 are the +/// discriminant, the high 32 bits are the handler_index (if `Suspend`) +#[derive(Clone, Copy)] +pub struct ControlEffect(ir::Value); + +impl ControlEffect { + // Returns the discriminant + pub fn signal(&self, builder: &mut FunctionBuilder) -> ir::Value { + builder.ins().ushr_imm(self.0, 32) + } + + pub fn from_u64(val: ir::Value) -> Self { + Self(val) + } + + pub fn to_u64(&self) -> ir::Value { + self.0 + } + + pub fn encode_resume(builder: &mut FunctionBuilder) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(wasmtime_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + + Self(val) + } + + pub fn encode_switch(builder: &mut FunctionBuilder) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + + Self(val) + } + + pub fn encode_suspend(builder: &mut FunctionBuilder, handler_index: ir::Value) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(wasmtime_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + let handler_index = builder.ins().uextend(I64, handler_index); + let val = builder.ins().bor(val, handler_index); + + Self(val) + } + + /// Returns the payload of the `Suspend` variant + pub fn handler_index(self, builder: &mut FunctionBuilder) -> ir::Value { + builder.ins().ireduce(I32, self.0) + } +} diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs new file mode 100644 index 000000000000..e2fe365acee5 --- /dev/null +++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs @@ -0,0 +1,55 @@ +use cranelift_codegen::ir; +use cranelift_codegen::ir::InstBuilder; + +/// Returns the Cranelift type used to represent all of the following: +/// - wasm values of type `(ref null $ct)` and `(ref $ct)` +/// - equivalently: runtime values of type `Option` and `VMContObj` +/// Note that a `VMContObj` is a fat pointer consisting of a pointer to +/// `VMContRef` and a pointer-sized revision counter. We represent this as 2 words +/// (pointer and usize). +pub fn fatpointer_type(env: &crate::func_environ::FuncEnvironment) -> ir::Type { + let ptr_bits = env.pointer_type().bits(); + ir::Type::int((2 * ptr_bits).try_into().unwrap()).unwrap() +} + +/// Turns a (possibly null) reference to a continuation object into a tuple +/// (revision, contref_ptr). If `contobj` denotes a wasm null reference, the +/// contref_ptr part will be a null pointer. +pub(crate) fn deconstruct<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + pos: &mut cranelift_codegen::cursor::FuncCursor, + contobj: ir::Value, +) -> (ir::Value, ir::Value) { + debug_assert_eq!(pos.func.dfg.value_type(contobj), fatpointer_type(env)); + let ptr_ty = env.pointer_type(); + let ptr_bits = ptr_ty.bits(); + + let contref = pos.ins().ireduce(ptr_ty, contobj); + let shifted = pos.ins().ushr_imm(contobj, i64::from(ptr_bits)); + let revision_counter = pos.ins().ireduce(ptr_ty, shifted); + + (revision_counter, contref) +} + +/// Constructs a continuation object from a given contref and revision pointer. +/// The contref_addr may be 0, to indicate that we want to build a wasm null reference. +pub(crate) fn construct<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + pos: &mut cranelift_codegen::cursor::FuncCursor, + revision_counter: ir::Value, + contref_addr: ir::Value, +) -> ir::Value { + let ptr_ty = env.pointer_type(); + let ptr_bits = ptr_ty.bits(); + let fat_ptr_ty = fatpointer_type(env); + + debug_assert_eq!(pos.func.dfg.value_type(contref_addr), ptr_ty); + debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ptr_ty); + + let contref_addr = pos.ins().uextend(fat_ptr_ty, contref_addr); + let revision_counter = pos.ins().uextend(fat_ptr_ty, revision_counter); + let shifted_counter = pos.ins().ishl_imm(revision_counter, i64::from(ptr_bits)); + let contobj = pos.ins().bor(shifted_counter, contref_addr); + + contobj +} diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs new file mode 100644 index 000000000000..383da04f93e7 --- /dev/null +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -0,0 +1,1917 @@ +use cranelift_codegen::ir::BlockArg; +use itertools::{Either, Itertools}; + +use cranelift_codegen::ir::condcodes::*; +use cranelift_codegen::ir::types::*; +use cranelift_codegen::ir::{self, MemFlags}; +use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData}; +use cranelift_frontend::FunctionBuilder; +use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported}; + +fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult { + match (triple.architecture, triple.operating_system) { + (target_lexicon::Architecture::X86_64, target_lexicon::OperatingSystem::Linux) => Ok(24), + _ => Err(wasm_unsupported!( + "stack switching not supported on {triple}" + )), + } +} + +use super::control_effect::ControlEffect; +use super::fatpointer; + +/// This module contains compile-time counterparts to types defined elsewhere. +pub(crate) mod stack_switching_helpers { + use core::marker::PhantomData; + use cranelift_codegen::ir; + use cranelift_codegen::ir::InstBuilder; + use cranelift_codegen::ir::condcodes::IntCC; + use cranelift_codegen::ir::types::*; + use cranelift_codegen::ir::{StackSlot, StackSlotKind::*}; + use cranelift_frontend::FunctionBuilder; + use wasmtime_environ::PtrSize; + + /// Provides information about the layout of a type when it is used as an + /// element in a host array. This is used for `VMHostArrayRef`. + pub(crate) trait VMHostArrayEntry { + /// Returns `(align, size)` in bytes. + fn vmhostarray_entry_layout(p: &P) -> (u8, u32); + } + + impl VMHostArrayEntry for u128 { + fn vmhostarray_entry_layout(_p: &P) -> (u8, u32) { + (16, 16) + } + } + + impl VMHostArrayEntry for *mut T { + fn vmhostarray_entry_layout(p: &P) -> (u8, u32) { + (p.size(), p.size().into()) + } + } + + #[derive(Copy, Clone)] + pub struct VMContRef { + pub address: ir::Value, + } + + #[derive(Copy, Clone)] + pub struct VMHostArrayRef { + /// Address of the VMHostArray we are referencing + address: ir::Value, + + /// The type parameter T is never used in the fields above. We still + /// want to have it for consistency with + /// `wasmtime_environ::Vector` and to use it in the associated + /// functions. + phantom: PhantomData, + } + + pub type VMPayloads = VMHostArrayRef; + + // Actually a vector of *mut VMTagDefinition + pub type VMHandlerList = VMHostArrayRef<*mut u8>; + + /// Compile-time representation of wasmtime_environ::VMStackChain, + /// consisting of two `ir::Value`s. + pub struct VMStackChain { + discriminant: ir::Value, + payload: ir::Value, + } + + pub struct VMCommonStackInformation { + pub address: ir::Value, + } + + /// Compile-time representation of `crate::runtime::vm::stack::VMContinuationStack`. + pub struct VMContinuationStack { + /// This is NOT the "top of stack" address of the stack itself. In line + /// with how the (runtime) `FiberStack` type works, this is a pointer to + /// the TOS address. + tos_ptr: ir::Value, + } + + impl VMContRef { + pub fn new(address: ir::Value) -> VMContRef { + VMContRef { address } + } + + pub fn args<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMPayloads { + let offset: i64 = env.offsets.ptr.vmcontref_args().into(); + let address = builder.ins().iadd_imm(self.address, offset); + VMPayloads::new(address) + } + + pub fn values<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMPayloads { + let offset: i64 = env.offsets.ptr.vmcontref_values().into(); + let address = builder.ins().iadd_imm(self.address, offset); + VMPayloads::new(address) + } + + pub fn common_stack_information<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMCommonStackInformation { + let offset: i64 = env.offsets.ptr.vmcontref_common_stack_information().into(); + let address = builder.ins().iadd_imm(self.address, offset); + VMCommonStackInformation { address } + } + + /// Stores the parent of this continuation, which may either be another + /// continuation or the initial stack. It is therefore represented as a + /// `VMStackChain` element. + pub fn set_parent_stack_chain<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + new_stack_chain: &VMStackChain, + ) { + let offset = env.offsets.ptr.vmcontref_parent_chain().into(); + new_stack_chain.store(env, builder, self.address, offset) + } + + /// Loads the parent of this continuation, which may either be another + /// continuation or the initial stack. It is therefore represented as a + /// `VMStackChain` element. + pub fn get_parent_stack_chain<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMStackChain { + let offset = env.offsets.ptr.vmcontref_parent_chain().into(); + VMStackChain::load(env, builder, self.address, offset, env.pointer_type()) + } + + pub fn set_last_ancestor<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + last_ancestor: ir::Value, + ) { + let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into(); + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .store(mem_flags, last_ancestor, self.address, offset); + } + + pub fn get_last_ancestor<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into(); + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .load(env.pointer_type(), mem_flags, self.address, offset) + } + + /// Gets the revision counter the a given continuation + /// reference. + pub fn get_revision<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + let offset: i32 = env.offsets.ptr.vmcontref_revision().into(); + let revision = builder.ins().load(I64, mem_flags, self.address, offset); + revision + } + + /// Sets the revision counter on the given continuation + /// reference to `revision + 1`. + + pub fn incr_revision<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + revision: ir::Value, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + let offset: i32 = env.offsets.ptr.vmcontref_revision().into(); + let revision_plus1 = builder.ins().iadd_imm(revision, 1); + builder + .ins() + .store(mem_flags, revision_plus1, self.address, offset); + revision_plus1 + } + + pub fn get_fiber_stack<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMContinuationStack { + // The top of stack field is stored at offset 0 of the `FiberStack`. + let offset: i64 = env.offsets.ptr.vmcontref_stack().into(); + let fiber_stack_top_of_stack_ptr = builder.ins().iadd_imm(self.address, offset); + VMContinuationStack::new(fiber_stack_top_of_stack_ptr) + } + } + + impl VMHostArrayRef { + pub(crate) fn new(address: ir::Value) -> Self { + Self { + address, + phantom: PhantomData::default(), + } + } + + fn get(&self, builder: &mut FunctionBuilder, ty: ir::Type, offset: i32) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + builder.ins().load(ty, mem_flags, self.address, offset) + } + + fn set(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) { + debug_assert_eq!( + builder.func.dfg.value_type(value), + Type::int_with_byte_size(u16::try_from(core::mem::size_of::()).unwrap()) + .unwrap() + ); + let mem_flags = ir::MemFlags::trusted(); + builder.ins().store(mem_flags, value, self.address, offset); + } + + pub fn get_data<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = env.offsets.ptr.vmhostarray_data().into(); + self.get(builder, env.pointer_type(), offset) + } + + pub fn get_length<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Array length is stored as u32. + let offset = env.offsets.ptr.vmhostarray_length().into(); + self.get(builder, I32, offset) + } + + fn set_length<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + length: ir::Value, + ) { + // Array length is stored as u32. + let offset = env.offsets.ptr.vmhostarray_length().into(); + self.set::(builder, offset, length); + } + + fn set_capacity<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + capacity: ir::Value, + ) { + // Array capacity is stored as u32. + let offset = env.offsets.ptr.vmhostarray_capacity().into(); + self.set::(builder, offset, capacity); + } + + fn set_data<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + data: ir::Value, + ) { + debug_assert_eq!(builder.func.dfg.value_type(data), env.pointer_type()); + let offset: i32 = env.offsets.ptr.vmhostarray_data().into(); + let mem_flags = ir::MemFlags::trusted(); + builder.ins().store(mem_flags, data, self.address, offset); + } + + /// Returns pointer to next empty slot in data buffer and marks the + /// subsequent `arg_count` slots as occupied. + pub fn occupy_next_slots<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + arg_count: i32, + ) -> ir::Value { + let data = self.get_data(env, builder); + let original_length = self.get_length(env, builder); + let new_length = builder + .ins() + .iadd_imm(original_length, i64::from(arg_count)); + self.set_length(env, builder, new_length); + + let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); + let original_length = builder.ins().uextend(I64, original_length); + let byte_offset = builder + .ins() + .imul_imm(original_length, i64::from(entry_size)); + builder.ins().iadd(data, byte_offset) + } + + pub fn allocate_or_reuse_stack_slot<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + required_capacity: u32, + existing_slot: Option, + ) -> StackSlot { + let (align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); + let required_size = required_capacity * entry_size; + + match existing_slot { + Some(slot) if builder.func.sized_stack_slots[slot].size >= required_size => { + let slot_data = &builder.func.sized_stack_slots[slot]; + debug_assert!(align <= slot_data.align_shift); + debug_assert_eq!(slot_data.kind, ExplicitSlot); + let existing_capacity = slot_data.size / entry_size; + + let capacity_value = builder.ins().iconst(I32, i64::from(existing_capacity)); + let existing_data = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, existing_data); + + slot + } + _ => { + let capacity_value = builder.ins().iconst(I32, i64::from(required_capacity)); + let slot_size = ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + required_size, + align, + ); + let slot = builder.create_sized_stack_slot(slot_size); + let new_data = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, new_data); + + slot + } + } + } + + /// Loads n entries from this Vector object, where n is the length of + /// `load_types`, which also gives the types of the values to load. + /// Loading starts at index 0 of the Vector object. + pub fn load_data_entries<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + load_types: &[ir::Type], + ) -> Vec { + let memflags = ir::MemFlags::trusted(); + + let data_start_pointer = self.get_data(env, builder); + let mut values = vec![]; + let mut offset = 0; + let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); + for valtype in load_types { + let val = builder + .ins() + .load(*valtype, memflags, data_start_pointer, offset); + values.push(val); + offset += i32::try_from(entry_size).unwrap(); + } + values + } + + /// Stores the given `values` in this Vector object, beginning at + /// index 0. This expects the Vector object to be empty (i.e., current + /// length is 0), and to be of sufficient capacity to store |`values`| + /// entries. + pub fn store_data_entries<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + values: &[ir::Value], + ) { + let store_count = builder + .ins() + .iconst(I32, i64::try_from(values.len()).unwrap()); + + let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); + + debug_assert!(values.iter().all(|val| { + let ty = builder.func.dfg.value_type(*val); + let size = ty.bytes(); + size <= entry_size + })); + + let memflags = ir::MemFlags::trusted(); + + let data_start_pointer = self.get_data(env, builder); + + let mut offset = 0; + for value in values { + builder + .ins() + .store(memflags, *value, data_start_pointer, offset); + offset += i32::try_from(entry_size).unwrap(); + } + + self.set_length(env, builder, store_count); + } + + pub fn clear<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + discard_buffer: bool, + ) { + let zero32 = builder.ins().iconst(I32, 0); + self.set_length(env, builder, zero32); + + if discard_buffer { + let zero32 = builder.ins().iconst(I32, 0); + self.set_capacity(env, builder, zero32); + + let zero_ptr = builder.ins().iconst(env.pointer_type(), 0); + self.set_data(env, builder, zero_ptr); + } + } + } + + impl VMStackChain { + /// Creates a `Self` corressponding to `VMStackChain::Continuation(contref)`. + pub fn from_continuation<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + contref: ir::Value, + ) -> VMStackChain { + debug_assert_eq!( + env.offsets.ptr.size_of_vmstack_chain(), + 2 * env.offsets.ptr.size() + ); + let discriminant = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap()); + VMStackChain { + discriminant, + payload: contref, + } + } + + /// Creates a `Self` corressponding to `VMStackChain::Absent`. + pub fn absent<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMStackChain { + debug_assert_eq!( + env.offsets.ptr.size_of_vmstack_chain(), + 2 * env.offsets.ptr.size() + ); + let discriminant = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap()); + let zero_filler = builder.ins().iconst(env.pointer_type(), 0i64); + VMStackChain { + discriminant, + payload: zero_filler, + } + } + + pub fn is_initial_stack<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().icmp_imm( + IntCC::Equal, + self.discriminant, + i64::try_from(wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT).unwrap(), + ) + } + + /// Return the two raw `ir::Value`s that represent this VMStackChain. + pub fn to_raw_parts(&self) -> [ir::Value; 2] { + [self.discriminant, self.payload] + } + + /// Construct a `Self` from two raw `ir::Value`s. + pub fn from_raw_parts(raw_data: [ir::Value; 2]) -> VMStackChain { + VMStackChain { + discriminant: raw_data[0], + payload: raw_data[1], + } + } + + /// Load a `VMStackChain` object from the given address. + pub fn load<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + pointer: ir::Value, + initial_offset: i32, + pointer_type: ir::Type, + ) -> VMStackChain { + let memflags = ir::MemFlags::trusted(); + let mut offset = initial_offset; + let mut data = vec![]; + for _ in 0..2 { + data.push(builder.ins().load(pointer_type, memflags, pointer, offset)); + offset += i32::try_from(pointer_type.bytes()).unwrap(); + } + let data = <[ir::Value; 2]>::try_from(data).unwrap(); + Self::from_raw_parts(data) + } + + /// Store this `VMStackChain` object at the given address. + pub fn store<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + target_pointer: ir::Value, + initial_offset: i32, + ) { + let memflags = ir::MemFlags::trusted(); + let mut offset = initial_offset; + let data = self.to_raw_parts(); + + for value in data { + debug_assert_eq!(builder.func.dfg.value_type(value), env.pointer_type()); + builder.ins().store(memflags, value, target_pointer, offset); + offset += i32::try_from(env.pointer_type().bytes()).unwrap(); + } + } + + /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`. + pub fn unchecked_get_continuation(&self) -> ir::Value { + self.payload + } + + /// Must only be called if `self` represents a `InitialStack` or + /// `Continuation` variant. Returns a pointer to the associated + /// `CommonStackInformation` object. + pub fn get_common_stack_information<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, + ) -> VMCommonStackInformation { + // `self` corresponds to a VMStackChain::InitialStack or + // VMStackChain::Continuation. + // In both cases, the payload is a pointer. + let address = self.payload; + + // `obj` is now a pointer to the beginning of either + // 1. A `VMContRef` struct (in the case of a + // VMStackChain::Continuation) + // 2. A CommonStackInformation struct (in the case of + // VMStackChain::InitialStack) + // + // Since a `VMContRef` starts with an (inlined) CommonStackInformation + // object at offset 0, we actually have in both cases that `ptr` is + // now the address of the beginning of a VMStackLimits object. + debug_assert_eq!(env.offsets.ptr.vmcontref_common_stack_information(), 0); + VMCommonStackInformation { address } + } + } + + impl VMCommonStackInformation { + fn get_state_ptr<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset: i64 = env.offsets.ptr.vmcommon_stack_information_state().into(); + + builder.ins().iadd_imm(self.address, offset) + } + + fn get_stack_limits_ptr<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset: i64 = env.offsets.ptr.vmcommon_stack_information_limits().into(); + + builder.ins().iadd_imm(self.address, offset) + } + + fn load_state<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + let state_ptr = self.get_state_ptr(env, builder); + + builder.ins().load(I32, mem_flags, state_ptr, 0) + } + + fn set_state_no_payload<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + discriminant: u32, + ) { + let discriminant = builder.ins().iconst(I32, i64::from(discriminant)); + let mem_flags = ir::MemFlags::trusted(); + let state_ptr = self.get_state_ptr(env, builder); + + builder.ins().store(mem_flags, discriminant, state_ptr, 0); + } + + pub fn set_state_running<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_parent<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_returned<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_suspended<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + /// Checks whether the `VMStackState` reflects that the stack has ever been + /// active (instead of just having been allocated, but never resumed). + pub fn was_invoked<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let actual_state = self.load_state(env, builder); + let allocated = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT; + builder + .ins() + .icmp_imm(IntCC::NotEqual, actual_state, i64::from(allocated)) + } + + pub fn get_handler_list<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMHandlerList { + let offset: i64 = env.offsets.ptr.vmcommon_stack_information_handlers().into(); + let address = builder.ins().iadd_imm(self.address, offset); + VMHandlerList::new(address) + } + + pub fn get_first_switch_handler_index<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Field first_switch_handler_index has type u32 + let memflags = ir::MemFlags::trusted(); + let offset: i32 = env + .offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + .into(); + builder.ins().load(I32, memflags, self.address, offset) + } + + pub fn set_first_switch_handler_index<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + value: ir::Value, + ) { + // Field first_switch_handler_index has type u32 + let memflags = ir::MemFlags::trusted(); + let offset: i32 = env + .offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + .into(); + builder.ins().store(memflags, value, self.address, offset); + } + + /// Sets `last_wasm_entry_sp` and `stack_limit` fields in + /// `VMRuntimelimits` using the values from the `VMStackLimits` of this + /// object. + pub fn write_limits_to_vmcontext<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmruntime_limits_ptr: ir::Value, + ) { + let stack_limits_ptr = self.get_stack_limits_ptr(env, builder); + + let memflags = ir::MemFlags::trusted(); + + let mut copy_to_vm_runtime_limits = |our_offset, their_offset| { + let our_value = builder.ins().load( + env.pointer_type(), + memflags, + stack_limits_ptr, + i32::from(our_offset), + ); + builder.ins().store( + memflags, + our_value, + vmruntime_limits_ptr, + i32::from(their_offset), + ); + }; + + let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap(); + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit(); + let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp(); + copy_to_vm_runtime_limits( + stack_limit_offset, + pointer_size.vmstore_context_stack_limit(), + ); + copy_to_vm_runtime_limits( + last_wasm_entry_fp_offset, + pointer_size.vmstore_context_last_wasm_entry_fp(), + ); + } + + /// Overwrites the `last_wasm_entry_fp` field of the `VMStackLimits` + /// object in the `VMStackLimits` of this object by loading the corresponding + /// field from the `VMRuntimeLimits`. + /// If `load_stack_limit` is true, we do the same for the `stack_limit` + /// field. + pub fn load_limits_from_vmcontext<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmruntime_limits_ptr: ir::Value, + load_stack_limit: bool, + ) { + let stack_limits_ptr = self.get_stack_limits_ptr(env, builder); + + let memflags = ir::MemFlags::trusted(); + let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap(); + + let mut copy = |runtime_limits_offset, stack_limits_offset| { + let from_vm_runtime_limits = builder.ins().load( + env.pointer_type(), + memflags, + vmruntime_limits_ptr, + runtime_limits_offset, + ); + builder.ins().store( + memflags, + from_vm_runtime_limits, + stack_limits_ptr, + stack_limits_offset, + ); + }; + + let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp(); + copy( + pointer_size.vmstore_context_last_wasm_entry_fp(), + last_wasm_entry_fp_offset, + ); + + if load_stack_limit { + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit(); + copy( + pointer_size.vmstore_context_stack_limit(), + stack_limit_offset, + ); + } + } + } + + impl VMContinuationStack { + /// The parameter is NOT the "top of stack" address of the stack itself. In line + /// with how the (runtime) `FiberStack` type works, this is a pointer to + /// the TOS address. + pub fn new(tos_ptr: ir::Value) -> Self { + Self { tos_ptr } + } + + fn load_top_of_stack<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .load(env.pointer_type(), mem_flags, self.tos_ptr, 0) + } + + /// Returns address of the control context stored in the stack memory, + /// as used by stack_switch instructions. + pub fn load_control_context<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let tos = self.load_top_of_stack(env, builder); + // Control context begins 24 bytes below top of stack (see unix.rs) + builder.ins().iadd_imm(tos, -0x18) + } + } +} + +use helpers::VMStackChain; +use stack_switching_helpers as helpers; + +/// Stores the given arguments in the appropriate `VMPayloads` object in the +/// continuation. If the continuation was never invoked, use the `args` object. +/// Otherwise, use the `values` object. +pub(crate) fn vmcontref_store_payloads<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + values: &[ir::Value], + contref: ir::Value, +) { + let count = + i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32"); + if values.len() > 0 { + let use_args_block = builder.create_block(); + let use_payloads_block = builder.create_block(); + let store_data_block = builder.create_block(); + builder.append_block_param(store_data_block, env.pointer_type()); + + let co = helpers::VMContRef::new(contref); + let csi = co.common_stack_information(env, builder); + let was_invoked = csi.was_invoked(env, builder); + builder + .ins() + .brif(was_invoked, use_payloads_block, &[], use_args_block, &[]); + + { + builder.switch_to_block(use_args_block); + builder.seal_block(use_args_block); + + let args = co.args(env, builder); + let ptr = args.occupy_next_slots(env, builder, count); + + builder + .ins() + .jump(store_data_block, &[BlockArg::Value(ptr)]); + } + + { + builder.switch_to_block(use_payloads_block); + builder.seal_block(use_payloads_block); + + let payloads = co.values(env, builder); + + // This also checks that the buffer is large enough to hold + // `values.len()` more elements. + let ptr = payloads.occupy_next_slots(env, builder, count); + builder + .ins() + .jump(store_data_block, &[BlockArg::Value(ptr)]); + } + + { + builder.switch_to_block(store_data_block); + builder.seal_block(store_data_block); + + let ptr = builder.block_params(store_data_block)[0]; + + // Store the values. + let memflags = ir::MemFlags::trusted(); + let mut offset = 0; + for value in values { + builder.ins().store(memflags, *value, ptr, offset); + offset += i32::from(env.offsets.ptr.maximum_value_size()); + } + } + } +} + +pub(crate) fn tag_address<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + index: u32, +) -> ir::Value { + let vmctx = env.vmctx_val(&mut builder.cursor()); + let tag_index = wasmtime_environ::TagIndex::from_u32(index); + let pointer_type = env.pointer_type(); + if let Some(def_index) = env.module.defined_tag_index(tag_index) { + let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap(); + builder.ins().iadd_imm(vmctx, i64::from(offset)) + } else { + let offset = i32::try_from(env.offsets.vmctx_vmtag_import_from(tag_index)).unwrap(); + builder.ins().load( + pointer_type, + ir::MemFlags::trusted().with_readonly(), + vmctx, + ir::immediates::Offset32::new(offset), + ) + } +} + +/// Returns the stack chain saved in the given `VMContext`. Note that the +/// head of the list is the actively running stack (initial stack or +/// continuation). +pub fn vmctx_load_stack_chain<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, +) -> VMStackChain { + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into(); + + // First we need to get the `VMStoreContext`. + let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); + let vm_store_context = builder.ins().load( + env.pointer_type(), + MemFlags::trusted(), + vmctx, + vm_store_context_offset, + ); + + VMStackChain::load( + env, + builder, + vm_store_context, + stack_chain_offset, + env.pointer_type(), + ) +} + +/// Stores the given stack chain saved in the `VMContext`, overwriting the +/// exsiting one. +pub fn vmctx_store_stack_chain<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, + stack_chain: &VMStackChain, +) { + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into(); + + // First we need to get the `VMStoreContext`. + let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); + let vm_store_context = builder.ins().load( + env.pointer_type(), + MemFlags::trusted(), + vmctx, + vm_store_context_offset, + ); + + stack_chain.store(env, builder, vm_store_context, stack_chain_offset) +} + +/// Similar to `vmctx_store_stack_chain`, but instead of storing an arbitrary +/// `VMStackChain`, stores VMStackChain::Continuation(contref)`. +pub fn vmctx_set_active_continuation<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, + contref: ir::Value, +) { + let chain = VMStackChain::from_continuation(env, builder, contref); + vmctx_store_stack_chain(env, builder, vmctx, &chain) +} + +pub fn vmctx_load_vm_runtime_limits_ptr<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, +) -> ir::Value { + let pointer_type = env.pointer_type(); + let offset = i32::from(env.offsets.ptr.vmctx_store_context()); + + // The *pointer* to the VMRuntimeLimits does not change within the + // same function, allowing us to set the `read_only` flag. + let flags = ir::MemFlags::trusted().with_readonly(); + + builder.ins().load(pointer_type, flags, vmctx, offset) +} + +/// This function generates code that searches for a handler for `tag_address`, +/// which must be a `*mut VMTagDefinition`. The search walks up the chain of +/// continuations beginning at `start`. +/// +/// The flag `search_suspend_handlers` determines whether we search for a +/// suspend or switch handler. Concretely, this influences which part of each +/// handler list we will search. +/// +/// We trap if no handler was found. +/// +/// The returned values are: +/// 1. The stack (continuation or initial stack, represented as a VMStackChain) in +/// whose handler list we found the tag (i.e., the stack that performed the +/// resume instruction that installed handler for the tag). +/// 2. The continuation whose parent is the stack mentioned in 1. +/// 3. The index of the handler in the handler list. +/// +/// In pseudo-code, the generated code's behavior can be expressed as +/// follows: +/// +/// chain_link = start +/// while !chain_link.is_initial_stack() { +/// contref = chain_link.get_contref() +/// parent_link = contref.parent +/// parent_csi = parent_link.get_common_stack_information(); +/// handlers = parent_csi.handlers; +/// (begin_range, end_range) = if search_suspend_handlers { +/// (0, parent_csi.first_switch_handler_index) +/// } else { +/// (parent_csi.first_switch_handler_index, handlers.length) +/// }; +/// for index in begin_range..end_range { +/// if handlers[index] == tag_address { +/// goto on_match(contref, index) +/// } +/// } +/// chain_link = parent_link +/// } +/// trap(unhandled_tag) +/// +/// on_match(conref : VMContRef, handler_index : u32) +/// ... execution continues here here ... +/// +fn search_handler<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + start: &helpers::VMStackChain, + tag_address: ir::Value, + search_suspend_handlers: bool, +) -> (VMStackChain, ir::Value, ir::Value) { + let handle_link = builder.create_block(); + let begin_search_handler_list = builder.create_block(); + let try_index = builder.create_block(); + let compare_tags = builder.create_block(); + let on_match = builder.create_block(); + let on_no_match = builder.create_block(); + let block_args = start.to_raw_parts().map(|v| BlockArg::Value(v)); + + // Terminate previous block: + builder.ins().jump(handle_link, &block_args); + + // Block handle_link + let chain_link = { + builder.append_block_param(handle_link, env.pointer_type()); + builder.append_block_param(handle_link, env.pointer_type()); + builder.switch_to_block(handle_link); + + let raw_parts = builder.block_params(handle_link); + let chain_link = helpers::VMStackChain::from_raw_parts([raw_parts[0], raw_parts[1]]); + let is_initial_stack = chain_link.is_initial_stack(env, builder); + builder.ins().brif( + is_initial_stack, + on_no_match, + &[], + begin_search_handler_list, + &[], + ); + chain_link + }; + + // Block begin_search_handler_list + let (contref, parent_link, handler_list_data_ptr, end_range) = { + builder.switch_to_block(begin_search_handler_list); + let contref = chain_link.unchecked_get_continuation(); + let contref = helpers::VMContRef::new(contref); + + let parent_link = contref.get_parent_stack_chain(env, builder); + let parent_csi = parent_link.get_common_stack_information(env, builder); + + let handlers = parent_csi.get_handler_list(env, builder); + let handler_list_data_ptr = handlers.get_data(env, builder); + + let first_switch_handler_index = parent_csi.get_first_switch_handler_index(env, builder); + + // Note that these indices are inclusive-exclusive, i.e. [begin_range, end_range). + let (begin_range, end_range) = if search_suspend_handlers { + let zero = builder.ins().iconst(I32, 0); + (zero, first_switch_handler_index) + } else { + let length = handlers.get_length(env, builder); + (first_switch_handler_index, length) + }; + + builder + .ins() + .jump(try_index, &[BlockArg::Value(begin_range)]); + + (contref, parent_link, handler_list_data_ptr, end_range) + }; + + // Block try_index + let index = { + builder.append_block_param(try_index, I32); + builder.switch_to_block(try_index); + let index = builder.block_params(try_index)[0]; + + let in_bounds = builder + .ins() + .icmp(IntCC::UnsignedLessThan, index, end_range); + let block_args = parent_link.to_raw_parts().map(|v| BlockArg::Value(v)); + builder + .ins() + .brif(in_bounds, compare_tags, &[], handle_link, &block_args); + index + }; + + // Block compare_tags + { + builder.switch_to_block(compare_tags); + + let base = handler_list_data_ptr; + let entry_size = env.pointer_type().bytes(); + let offset = builder.ins().imul_imm(index, i64::from(entry_size)); + let offset = builder.ins().uextend(I64, offset); + let entry_address = builder.ins().iadd(base, offset); + + let memflags = ir::MemFlags::trusted(); + + let handled_tag = builder + .ins() + .load(env.pointer_type(), memflags, entry_address, 0); + + let tags_match = builder.ins().icmp(IntCC::Equal, handled_tag, tag_address); + let incremented_index = builder.ins().iadd_imm(index, 1); + builder.ins().brif( + tags_match, + on_match, + &[], + try_index, + &[BlockArg::Value(incremented_index)], + ); + } + + // Block on_no_match + { + builder.switch_to_block(on_no_match); + builder.set_cold_block(on_no_match); + builder.ins().trap(crate::TRAP_UNHANDLED_TAG); + } + + builder.seal_block(handle_link); + builder.seal_block(begin_search_handler_list); + builder.seal_block(try_index); + builder.seal_block(compare_tags); + builder.seal_block(on_match); + builder.seal_block(on_no_match); + + // final block: on_match + builder.switch_to_block(on_match); + + (parent_link, contref.address, index) +} + +pub(crate) fn translate_cont_bind<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + contobj: ir::Value, + args: &[ir::Value], +) -> ir::Value { + let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj); + + // The typing rules for cont.bind allow a null reference to be passed to it. + builder.ins().trapz(contref, crate::TRAP_NULL_REFERENCE); + + let mut vmcontref = helpers::VMContRef::new(contref); + let revision = vmcontref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, witness, revision); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + + vmcontref_store_payloads(env, builder, args, contref); + + let revision = vmcontref.incr_revision(env, builder, revision); + let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref); + contobj +} + +pub(crate) fn translate_cont_new<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + func: ir::Value, + arg_types: &[WasmValType], + return_types: &[WasmValType], +) -> WasmResult { + // The typing rules for cont.new allow a null reference to be passed to it. + builder.ins().trapz(func, crate::TRAP_NULL_REFERENCE); + + let nargs = builder + .ins() + .iconst(I32, i64::try_from(arg_types.len()).unwrap()); + let nreturns = builder + .ins() + .iconst(I32, i64::try_from(return_types.len()).unwrap()); + + let cont_new_func = super::builtins::cont_new(env, &mut builder.func)?; + let vmctx = env.vmctx_val(&mut builder.cursor()); + let call_inst = builder + .ins() + .call(cont_new_func, &[vmctx, func, nargs, nreturns]); + let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap(); + + let tag = helpers::VMContRef::new(contref).get_revision(env, builder); + let contobj = fatpointer::construct(env, &mut builder.cursor(), tag, contref); + Ok(contobj) +} + +pub(crate) fn translate_resume<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + type_index: u32, + resume_contobj: ir::Value, + resume_args: &[ir::Value], + resumetable: &[(u32, Option)], +) -> WasmResult> { + // The resume instruction is the most involved instruction to + // compile as it is responsible for both continuation application + // and control tag dispatch. + // + // Here we translate a resume instruction into several basic + // blocks as follows: + // + // previous block + // | + // | + // resume_block + // / \ + // / \ + // | | + // return_block | + // suspend block + // | + // dispatch block + // + // * resume_block handles continuation arguments and performs + // actual stack switch. On ordinary return from resume, it jumps + // to the `return_block`, whereas on suspension it jumps to the + // `suspend_block`. + // * suspend_block is used on suspension, jumps onward to + // `dispatch_block`. + // * dispatch_block uses a jump table to dispatch to actual + // user-defined handler blocks, based on the handler index + // provided on suspension. Note that we do not jump to the + // handler blocks directly. Instead, each handler block has a + // corresponding premable block, which we jump to in order to + // reach a particular handler block. The preamble block prepares + // the arguments and continuation object to be passed to the + // actual handler block. + // + let resume_block = builder.create_block(); + let return_block = builder.create_block(); + let suspend_block = builder.create_block(); + let dispatch_block = builder.create_block(); + + let vmctx = env.vmctx_val(&mut builder.cursor()); + + // Split the resumetable into suspend handlers (each represented by the tag + // index and handler block) and the switch handlers (represented just by the + // tag index). Note that we currently don't remove duplicate tags. + let (suspend_handlers, switch_tags): (Vec<(u32, Block)>, Vec) = resumetable + .iter() + .partition_map(|(tag_index, block_opt)| match block_opt { + Some(block) => Either::Left((*tag_index, *block)), + None => Either::Right(*tag_index), + }); + + // Technically, there is no need to have a dedicated resume block, we could + // just put all of its contents into the current block. + builder.ins().jump(resume_block, &[]); + + // Resume block: actually resume the continuation chain ending at `resume_contref`. + let (resume_result, vm_runtime_limits_ptr, original_stack_chain, new_stack_chain) = { + builder.switch_to_block(resume_block); + builder.seal_block(resume_block); + + let (witness, resume_contref) = + fatpointer::deconstruct(env, &mut builder.cursor(), resume_contobj); + + // The typing rules for resume allow a null reference to be passed to it. + builder + .ins() + .trapz(resume_contref, crate::TRAP_NULL_REFERENCE); + + let mut vmcontref = helpers::VMContRef::new(resume_contref); + + let revision = vmcontref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + let _next_revision = vmcontref.incr_revision(env, builder, revision); + + if resume_args.len() > 0 { + // We store the arguments in the `VMContRef` to be resumed. + vmcontref_store_payloads(env, builder, resume_args, resume_contref); + } + + // Splice together stack chains: + // Connect the end of the chain starting at `resume_contref` to the currently active chain. + let mut last_ancestor = helpers::VMContRef::new(vmcontref.get_last_ancestor(env, builder)); + + // Make the currently running continuation (if any) the parent of the one we are about to resume. + let original_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + last_ancestor.set_parent_stack_chain(env, builder, &original_stack_chain); + + // Just for consistency: `vmcontref` is about to get state Running, so let's zero out its last_ancestor field. + let zero = builder.ins().iconst(env.pointer_type(), 0); + vmcontref.set_last_ancestor(env, builder, zero); + + // We mark `resume_contref` as the currently running one + vmctx_set_active_continuation(env, builder, vmctx, resume_contref); + + // Note that the resume_contref libcall a few lines further below + // manipulates the stack limits as follows: + // 1. Copy stack_limit, last_wasm_entry_sp and last_wasm_exit* values from + // VMRuntimeLimits into the currently active continuation (i.e., the + // one that will become the parent of the to-be-resumed one) + // + // 2. Copy `stack_limit` and `last_wasm_entry_sp` in the + // `VMStackLimits` of `resume_contref` into the `VMRuntimeLimits`. + // + // See the comment on `wasmtime_environ::VMStackChain` for a + // description of the invariants that we maintain for the various stack + // limits. + + // `resume_contref` is now active, and its parent is suspended. + let resume_contref = helpers::VMContRef::new(resume_contref); + let resume_csi = resume_contref.common_stack_information(env, builder); + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + resume_csi.set_state_running(env, builder); + parent_csi.set_state_parent(env, builder); + + // We update the `VMStackLimits` of the parent of the continuation to be resumed + // as well as the `VMRuntimeLimits`. + // See the comment on `wasmtime_environ::VMStackChain` for a description + // of the invariants that we maintain for the various stack limits. + let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx); + parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true); + resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + // Install handlers in (soon to be) parent's VMHandlerList: + // Let the i-th handler clause be (on $tag $block). + // Then the i-th entry of the VMHandlerList will be the address of $tag. + let handler_list = parent_csi.get_handler_list(env, builder); + + if resumetable.len() > 0 { + // Total number of handlers (suspend and switch). + let handler_count = u32::try_from(resumetable.len()).unwrap(); + // Populate the Array's data ptr with a pointer to a sufficiently + // large area on this stack. + env.stack_switching_handler_list_buffer = + Some(handler_list.allocate_or_reuse_stack_slot( + env, + builder, + handler_count, + env.stack_switching_handler_list_buffer, + )); + + let suspend_handler_count = suspend_handlers.len(); + + // All handlers, represented by the indices of the tags they handle. + // All the suspend handlers come first, followed by all the switch handlers. + let all_handlers = suspend_handlers + .iter() + .map(|(tag_index, _block)| *tag_index) + .chain(switch_tags); + + // Translate all tag indices to tag addresses (i.e., the corresponding *mut VMTagDefinition). + let all_tag_addresses: Vec = all_handlers + .map(|tag_index| tag_address(env, builder, tag_index)) + .collect(); + + // Store all tag addresess in the handler list. + handler_list.store_data_entries(env, builder, &all_tag_addresses); + + // To enable distinguishing switch and suspend handlers when searching the handler list: + // Store at which index the switch handlers start. + let first_switch_handler_index = builder + .ins() + .iconst(I32, i64::try_from(suspend_handler_count).unwrap()); + parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index); + } + + let resume_payload = ControlEffect::encode_resume(builder).to_u64(); + + // Note that the control context we use for switching is not the one in + // (the stack of) resume_contref, but in (the stack of) last_ancestor! + let fiber_stack = last_ancestor.get_fiber_stack(env, builder); + let control_context_ptr = fiber_stack.load_control_context(env, builder); + + let result = + builder + .ins() + .stack_switch(control_context_ptr, control_context_ptr, resume_payload); + + // At this point we know nothing about the continuation that just + // suspended or returned. In particular, it does not have to be what we + // called `resume_contref` earlier on. We must reload the information + // about the now active continuation from the VMContext. + let new_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + + // Now the parent contref (or initial stack) is active again + vmctx_store_stack_chain(env, builder, vmctx, &original_stack_chain); + parent_csi.set_state_running(env, builder); + + // Just for consistency: Clear the handler list. + handler_list.clear(env, builder, true); + parent_csi.set_first_switch_handler_index(env, builder, zero); + + // Extract the result and signal bit. + let result = ControlEffect::from_u64(result); + let signal = result.signal(builder); + + // Jump to the return block if the result signal is 0, otherwise jump to + // the suspend block. + builder + .ins() + .brif(signal, suspend_block, &[], return_block, &[]); + + ( + result, + vm_runtime_limits_ptr, + original_stack_chain, + new_stack_chain, + ) + }; + + // The suspend block: Only used when we suspended, not for returns. + // Here we extract the index of the handler to use. + let (handler_index, suspended_contref, suspended_contobj) = { + builder.switch_to_block(suspend_block); + builder.seal_block(suspend_block); + + let suspended_continuation = new_stack_chain.unchecked_get_continuation(); + let mut suspended_continuation = helpers::VMContRef::new(suspended_continuation); + let suspended_csi = suspended_continuation.common_stack_information(env, builder); + + // Note that at the suspend site, we already + // 1. Set the state of suspended_continuation to Suspended + // 2. Set suspended_continuation.last_ancestor + // 3. Broke the continuation chain at suspended_continuation.last_ancestor + + // We store parts of the VMRuntimeLimits into the continuation that just suspended. + suspended_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false); + + // Afterwards (!), restore parts of the VMRuntimeLimits from the + // parent of the suspended continuation (which is now active). + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + // Extract the handler index + let handler_index = resume_result.handler_index(builder); + + let revision = suspended_continuation.get_revision(env, builder); + let suspended_contobj = fatpointer::construct( + env, + &mut builder.cursor(), + revision, + suspended_continuation.address, + ); + + // We need to terminate this block before being allowed to switch to + // another one. + builder.ins().jump(dispatch_block, &[]); + + (handler_index, suspended_continuation, suspended_contobj) + }; + + // For technical reasons, the jump table needs to have a default + // block. In our case, it should be unreachable, since the handler + // index we dispatch on should correspond to a an actual handler + // block in the jump table. + let jt_default_block = builder.create_block(); + { + builder.switch_to_block(jt_default_block); + builder.set_cold_block(jt_default_block); + + builder.ins().trap(crate::TRAP_UNREACHABLE); + } + + // We create a preamble block for each of the actual handler blocks: It + // reads the necessary arguments and passes them to the actual handler + // block, together with the continuation object. + let target_preamble_blocks = { + let mut preamble_blocks = vec![]; + + for &(handle_tag, target_block) in &suspend_handlers { + let preamble_block = builder.create_block(); + preamble_blocks.push(preamble_block); + builder.switch_to_block(preamble_block); + + let param_types = env.tag_params(TagIndex::from_u32(handle_tag)); + let param_types: Vec = param_types + .iter() + .map(|wty| crate::value_type(env.isa(), *wty)) + .collect(); + + let values = suspended_contref.values(env, builder); + let mut suspend_args: Vec = values + .load_data_entries(env, builder, ¶m_types) + .into_iter() + .map(|v| BlockArg::Value(v)) + .collect(); + + // At the suspend site, we store the suspend args in the the + // `values` buffer of the VMContRef that was active at the time that + // the suspend instruction was performed. + suspend_args.push(BlockArg::Value(suspended_contobj)); + + // We clear the suspend args. This is mostly for consistency. Note + // that we don't zero out the data buffer, we still need it for the + + values.clear(env, builder, false); + + builder.ins().jump(target_block, &suspend_args); + } + + preamble_blocks + }; + + // Dispatch block. All it does is jump to the right premable block based on + // the handler index. + { + builder.switch_to_block(dispatch_block); + builder.seal_block(dispatch_block); + + let default_bc = builder.func.dfg.block_call(jt_default_block, &[]); + + let adapter_bcs: Vec = target_preamble_blocks + .iter() + .map(|b| builder.func.dfg.block_call(*b, &[])) + .collect(); + + let jt_data = JumpTableData::new(default_bc, &adapter_bcs); + let jt = builder.create_jump_table(jt_data); + + builder.ins().br_table(handler_index, jt); + + for preamble_block in target_preamble_blocks { + builder.seal_block(preamble_block); + } + builder.seal_block(jt_default_block); + } + + // Return block: Jumped to by resume block if continuation + // returned normally. + { + builder.switch_to_block(return_block); + builder.seal_block(return_block); + + // If we got a return signal, a continuation must have been running. + let returned_contref = new_stack_chain.unchecked_get_continuation(); + let returned_contref = helpers::VMContRef::new(returned_contref); + + // Restore parts of the VMRuntimeLimits from the parent of the + // returned continuation (which is now active). + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + let returned_csi = returned_contref.common_stack_information(env, builder); + returned_csi.set_state_returned(env, builder); + + // Load the values returned by the continuation. + let return_types: Vec<_> = env + .continuation_returns(TypeIndex::from_u32(type_index)) + .iter() + .map(|ty| crate::value_type(env.isa(), *ty)) + .collect(); + let payloads = returned_contref.args(env, builder); + let return_values = payloads.load_data_entries(env, builder, &return_types); + payloads.clear(env, builder, true); + + Ok(return_values) + } +} + +pub(crate) fn translate_suspend<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + tag_index: u32, + suspend_args: &[ir::Value], + tag_return_types: &[ir::Type], +) -> Vec { + let tag_addr = tag_address(env, builder, tag_index); + + let vmctx = env.vmctx_val(&mut builder.cursor()); + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + + let (_, end_of_chain_contref, handler_index) = + search_handler(env, builder, &active_stack_chain, tag_addr, true); + + // If we get here, the search_handler logic succeeded (i.e., did not trap). + // Thus, there is at least one parent, so we are not on the initial stack. + // Can therefore extract continuation directly. + let active_contref = active_stack_chain.unchecked_get_continuation(); + let active_contref = helpers::VMContRef::new(active_contref); + let mut end_of_chain_contref = helpers::VMContRef::new(end_of_chain_contref); + + active_contref.set_last_ancestor(env, builder, end_of_chain_contref.address); + + // In the active_contref's `values` buffer, stack-allocate enough room so that we can + // later store the following: + // 1. The suspend arguments + // 2. Afterwards, the tag return values + let values = active_contref.values(env, builder); + let required_capacity = + u32::try_from(std::cmp::max(suspend_args.len(), tag_return_types.len())) + .expect("Number of stack switching payloads should fit in u32"); + + if required_capacity > 0 { + env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + env, + builder, + required_capacity, + env.stack_switching_values_buffer, + )); + } + + if suspend_args.len() > 0 { + values.store_data_entries(env, builder, suspend_args); + } + + // Set current continuation to suspended and break up handler chain. + let active_contref_csi = active_contref.common_stack_information(env, builder); + active_contref_csi.set_state_suspended(env, builder); + let absent_chain_link = VMStackChain::absent(env, builder); + end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link); + + let suspend_payload = ControlEffect::encode_suspend(builder, handler_index).to_u64(); + + // Note that the control context we use for switching is the one + // at the end of the chain, not the one in active_contref! + // This also means that stack_switch saves the information about + // the current stack in the control context located in the stack + // of end_of_chain_contref. + let fiber_stack = end_of_chain_contref.get_fiber_stack(env, builder); + let control_context_ptr = fiber_stack.load_control_context(env, builder); + + builder + .ins() + .stack_switch(control_context_ptr, control_context_ptr, suspend_payload); + + // The return values of the suspend instruction are the tag return values, saved in the `args` buffer. + let values = active_contref.values(env, builder); + let return_values = values.load_data_entries(env, builder, tag_return_types); + // We effectively consume the values and discard the stack allocated buffer. + values.clear(env, builder, true); + + return_values +} + +pub(crate) fn translate_switch<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + tag_index: u32, + switchee_contobj: ir::Value, + switch_args: &[ir::Value], + return_types: &[ir::Type], +) -> WasmResult> { + let vmctx = env.vmctx_val(&mut builder.cursor()); + + // Check and increment revision on switchee continuation object (i.e., the + // one being switched to). Logically, the switchee continuation extends from + // `switchee_contref` to `switchee_contref.last_ancestor` (i.e., the end of + // the parent chain starting at `switchee_contref`). + let switchee_contref = { + let (witness, target_contref) = + fatpointer::deconstruct(env, &mut builder.cursor(), switchee_contobj); + + // The typing rules for switch allow a null reference to be passed to it. + builder + .ins() + .trapz(target_contref, crate::TRAP_NULL_REFERENCE); + + let mut target_contref = helpers::VMContRef::new(target_contref); + + let revision = target_contref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + let _next_revision = target_contref.incr_revision(env, builder, revision); + target_contref + }; + + // We create the "switcher continuation" (i.e., the one executing switch) + // from the current execution context: Logically, it extends from the + // continuation reference executing `switch` (subsequently called + // `switcher_contref`) to the immediate child (called + // `switcher_contref_last_ancestor`) of the stack with the corresponding + // handler (saved in `handler_stack_chain`). + let ( + switcher_contref, + switcher_contobj, + switcher_contref_last_ancestor, + handler_stack_chain, + vm_runtime_limits_ptr, + ) = { + let tag_addr = tag_address(env, builder, tag_index); + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + let (handler_stack_chain, last_ancestor, _handler_index) = + search_handler(env, builder, &active_stack_chain, tag_addr, false); + let mut last_ancestor = helpers::VMContRef::new(last_ancestor); + + // If we get here, the search_handler logic succeeded (i.e., did not trap). + // Thus, there is at least one parent, so we are not on the initial stack. + // Can therefore extract continuation directly. + let switcher_contref = active_stack_chain.unchecked_get_continuation(); + let mut switcher_contref = helpers::VMContRef::new(switcher_contref); + + switcher_contref.set_last_ancestor(env, builder, last_ancestor.address); + + // In the switcher_contref's `values` buffer, stack-allocate enough room so that we can + // later store `tag_return_types.len()` when resuming the continuation. + let values = switcher_contref.values(env, builder); + let required_capacity = u32::try_from(return_types.len()).unwrap(); + if required_capacity > 0 { + env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + env, + builder, + required_capacity, + env.stack_switching_values_buffer, + )); + } + + let switcher_contref_csi = switcher_contref.common_stack_information(env, builder); + switcher_contref_csi.set_state_suspended(env, builder); + // We break off `switcher_contref` from the chain of active + // continuations, by separating the link between `last_ancestor` and its + // parent stack. + let absent = VMStackChain::absent(env, builder); + last_ancestor.set_parent_stack_chain(env, builder, &absent); + + // Load current runtime limits from `VMContext` and store in the + // switcher continuation. + let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx); + switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false); + + let revision = switcher_contref.get_revision(env, builder); + let new_contobj = fatpointer::construct( + env, + &mut builder.cursor(), + revision, + switcher_contref.address, + ); + + ( + switcher_contref, + new_contobj, + last_ancestor, + handler_stack_chain, + vm_runtime_limits_ptr, + ) + }; + + // Prepare switchee continuation: + // - Store "ordinary" switch arguments as well as the contobj just + // synthesized from the current context (i.e., `switcher_contobj`) in the + // switchee continuation's payload buffer. + // - Splice switchee's continuation chain with handler stack to form new + // overall chain of active continuations. + let (switchee_contref_csi, switchee_contref_last_ancestor) = { + let mut combined_payloads = switch_args.to_vec(); + combined_payloads.push(switcher_contobj); + vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address); + + let switchee_contref_csi = switchee_contref.common_stack_information(env, builder); + switchee_contref_csi.set_state_running(env, builder); + + let switchee_contref_last_ancestor = switchee_contref.get_last_ancestor(env, builder); + let mut switchee_contref_last_ancestor = + helpers::VMContRef::new(switchee_contref_last_ancestor); + + switchee_contref_last_ancestor.set_parent_stack_chain(env, builder, &handler_stack_chain); + + (switchee_contref_csi, switchee_contref_last_ancestor) + }; + + // Update VMContext/Store: Update active continuation and `VMRuntimeLimits`. + { + vmctx_set_active_continuation(env, builder, vmctx, switchee_contref.address); + + switchee_contref_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + } + + // Perform actual stack switch + { + let switcher_last_ancestor_fs = + switcher_contref_last_ancestor.get_fiber_stack(env, builder); + let switcher_last_ancestor_cc = + switcher_last_ancestor_fs.load_control_context(env, builder); + + let switchee_last_ancestor_fs = + switchee_contref_last_ancestor.get_fiber_stack(env, builder); + let switchee_last_ancestor_cc = + switchee_last_ancestor_fs.load_control_context(env, builder); + + // The stack switch involves the following control contexts (e.g., IP, + // SP, FP, ...): + // - `switchee_last_ancestor_cc` contains the information to continue + // execution in the switchee/target continuation. + // - `switcher_last_ancestor_cc` contains the information about how to + // continue execution once we suspend/return to the stack with the + // switch handler. + // + // In total, the following needs to happen: + // 1. Load control context at `switchee_last_ancestor_cc` to perform + // stack switch. + // 2. Move control context at `switcher_last_ancestor_cc` over to + // `switchee_last_ancestor_cc`. + // 3. Upon actual switch, save current control context at + // `switcher_last_ancestor_cc`. + // + // We implement this as follows: + // 1. We copy `switchee_last_ancestor_cc` to a temporary area on the + // stack (`tmp_control_context`). + // 2. We copy `switcher_last_ancestor_cc` over to + // `switchee_last_ancestor_cc`. + // 3. We invoke the stack switch instruction such that it reads from the + // temporary area, and writes to `switcher_last_ancestor_cc`. + // + // Note that the temporary area is only accessed once by the + // `stack_switch` instruction emitted later in this block, meaning that we + // don't have to worry about its lifetime. + // + // NOTE(frank-emrich) The implementation below results in one stack slot + // being created per switch instruction, even though multiple switch + // instructions in the same function could safely re-use the same stack + // slot. Thus, we could implement logic for sharing the stack slot by + // adding an appropriate field to `FuncEnvironment`. + // + // NOTE(frank-emrich) We could avoid the copying to a temporary area by + // making `stack_switch` do all of the necessary moving itself. However, + // that would be a rather ad-hoc change to how the instruction uses the + // two pointers given to it. + + let cctx_size = control_context_size(env.isa().triple())?; + let slot_size = ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + u32::from(cctx_size), + u8::try_from(env.pointer_type().bytes()).unwrap(), + ); + let slot = builder.create_sized_stack_slot(slot_size); + let tmp_control_context = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + let flags = MemFlags::trusted(); + let mut offset: i32 = 0; + while offset < i32::from(cctx_size) { + // switchee_last_ancestor_cc -> tmp control context + let tmp1 = + builder + .ins() + .load(env.pointer_type(), flags, switchee_last_ancestor_cc, offset); + builder + .ins() + .store(flags, tmp1, tmp_control_context, offset); + + // switcher_last_ancestor_cc -> switchee_last_ancestor_cc + let tmp2 = + builder + .ins() + .load(env.pointer_type(), flags, switcher_last_ancestor_cc, offset); + builder + .ins() + .store(flags, tmp2, switchee_last_ancestor_cc, offset); + + offset += i32::try_from(env.pointer_type().bytes()).unwrap(); + } + + let switch_payload = ControlEffect::encode_switch(builder).to_u64(); + + let _result = builder.ins().stack_switch( + switcher_last_ancestor_cc, + tmp_control_context, + switch_payload, + ); + } + + // After switching back to the original stack: Load return values, they are + // stored on the switcher continuation. + let return_values = { + let payloads = switcher_contref.values(env, builder); + let return_values = payloads.load_data_entries(env, builder, return_types); + // We consume the values and discard the buffer (allocated on this stack) + payloads.clear(env, builder, true); + return_values + }; + + Ok(return_values) +} diff --git a/crates/cranelift/src/func_environ/stack_switching/mod.rs b/crates/cranelift/src/func_environ/stack_switching/mod.rs new file mode 100644 index 000000000000..25381f727942 --- /dev/null +++ b/crates/cranelift/src/func_environ/stack_switching/mod.rs @@ -0,0 +1,37 @@ +mod control_effect; +pub(crate) mod fatpointer; +pub(crate) mod instructions; + +pub(crate) mod builtins { + macro_rules! define_builtin_accessors { + ( $( $name:ident , )* ) => { + $( + #[inline] + pub fn $name( + func_env: &mut crate::func_environ::FuncEnvironment<'_>, + func: &mut crate::ir::Function, + ) -> wasmtime_environ::WasmResult { + #[cfg(feature = "stack-switching")] + { + return Ok(func_env.builtin_functions.$name(func)); + } + + #[cfg(not(feature = "stack-switching"))] + { + let _ = (func, func_env); + return Err(wasmtime_environ::wasm_unsupported!( + "support for Wasm Stack Switching disabled at compile time because the `stack-switching` cargo \ + feature was not enabled" + )); + } + } + )* + }; + } + + define_builtin_accessors! { + cont_new, + table_grow_cont_obj, + table_fill_cont_obj, + } +} diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index b25fc8921d47..021c042ebd47 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -213,10 +213,9 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => ir::types::I32, - WasmHeapTopType::Cont => - // TODO(10248) This is added in a follow-up PR - { - unimplemented!("codegen for stack switching types not implemented, yet") + WasmHeapTopType::Cont => { + // VMContObj is 2 * pointer_size (pointer + usize revision) + ir::Type::int((2 * pointer_type.bits()).try_into().unwrap()).unwrap() } } } @@ -376,6 +375,11 @@ impl BuiltinFunctionSignatures { AbiParam::new(ir::types::I8) } + #[cfg(feature = "stack-switching")] + fn size(&self) -> AbiParam { + AbiParam::new(self.pointer_type) + } + fn wasm_signature(&self, builtin: BuiltinFunctionIndex) -> Signature { let mut _cur = 0; macro_rules! iter { diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 359278a2df72..1e15bec0c3af 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -89,7 +89,7 @@ use cranelift_codegen::ir::{BlockArg, types::*}; use cranelift_codegen::packed_option::ReservedValue; use cranelift_frontend::{FunctionBuilder, Variable}; use itertools::Itertools; -use smallvec::SmallVec; +use smallvec::{SmallVec, ToSmallVec}; use std::collections::{HashMap, hash_map}; use std::vec::Vec; use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources}; @@ -2942,35 +2942,101 @@ pub fn translate_operator( // representation, so we don't actually need to do anything. } - Operator::ContNew { cont_type_index: _ } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + Operator::ContNew { cont_type_index } => { + let cont_type_index = TypeIndex::from_u32(*cont_type_index); + let arg_types: SmallVec<[_; 8]> = environ + .continuation_arguments(cont_type_index) + .to_smallvec(); + let result_types: SmallVec<[_; 8]> = + environ.continuation_returns(cont_type_index).to_smallvec(); + let r = stack.pop1(); + let contobj = environ.translate_cont_new(builder, r, &arg_types, &result_types)?; + stack.push1(contobj); } Operator::ContBind { - argument_index: _, - result_index: _, + argument_index, + result_index, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + let src_types = environ.continuation_arguments(TypeIndex::from_u32(*argument_index)); + let dst_arity = environ + .continuation_arguments(TypeIndex::from_u32(*result_index)) + .len(); + let arg_count = src_types.len() - dst_arity; + + let arg_types = &src_types[0..arg_count]; + for arg_type in arg_types { + // We can't bind GC objects using cont.bind at the moment: We + // don't have the necessary infrastructure to traverse the + // buffers used by cont.bind when looking for GC roots. Thus, + // this crude check ensures that these buffers can never contain + // GC roots to begin with. + if arg_type.is_vmgcref_type_and_not_i31() { + return Err(wasmtime_environ::WasmError::Unsupported( + "cont.bind does not support GC types at the moment".into(), + )); + } + } + + let (original_contobj, args) = stack.peekn(arg_count + 1).split_last().unwrap(); + + let new_contobj = environ.translate_cont_bind(builder, *original_contobj, args); + + stack.popn(arg_count + 1); + stack.push1(new_contobj); } - Operator::Suspend { tag_index: _ } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + Operator::Suspend { tag_index } => { + let tag_index = TagIndex::from_u32(*tag_index); + let param_types = environ.tag_params(tag_index).to_vec(); + let return_types: SmallVec<[_; 8]> = environ + .tag_returns(tag_index) + .iter() + .map(|ty| crate::value_type(environ.isa(), *ty)) + .collect(); + + let params = stack.peekn(param_types.len()); + let param_count = params.len(); + + let return_values = + environ.translate_suspend(builder, tag_index.as_u32(), params, &return_types); + + stack.popn(param_count); + stack.pushn(&return_values); } Operator::Resume { - cont_type_index: _, - resume_table: _, + cont_type_index, + resume_table: wasm_resume_table, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + // We translate the block indices in the wasm resume_table to actual Blocks. + let mut clif_resume_table = vec![]; + for handle in &wasm_resume_table.handlers { + match handle { + wasmparser::Handle::OnLabel { tag, label } => { + let i = stack.control_stack.len() - 1 - (*label as usize); + let frame = &mut stack.control_stack[i]; + // This is side-effecting! + frame.set_branched_to_exit(); + clif_resume_table.push((*tag, Some(frame.br_destination()))); + } + wasmparser::Handle::OnSwitch { tag } => { + clif_resume_table.push((*tag, None)); + } + } + } + + let cont_type_index = TypeIndex::from_u32(*cont_type_index); + let arity = environ.continuation_arguments(cont_type_index).len(); + let (contobj, call_args) = stack.peekn(arity + 1).split_last().unwrap(); + + let cont_return_vals = environ.translate_resume( + builder, + cont_type_index.as_u32(), + *contobj, + call_args, + &clif_resume_table, + )?; + + stack.popn(arity + 1); // arguments + continuation + stack.pushn(&cont_return_vals); } Operator::ResumeThrow { cont_type_index: _, @@ -2983,13 +3049,51 @@ pub fn translate_operator( )); } Operator::Switch { - cont_type_index: _, - tag_index: _, + cont_type_index, + tag_index, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + // Arguments of the continuation we are going to switch to + let continuation_argument_types: SmallVec<[_; 8]> = environ + .continuation_arguments(TypeIndex::from_u32(*cont_type_index)) + .to_smallvec(); + // Arity includes the continuation argument + let arity = continuation_argument_types.len(); + let (contobj, switch_args) = stack.peekn(arity).split_last().unwrap(); + + // Type of the continuation we are going to create by suspending the + // currently running stack + let current_continuation_type = continuation_argument_types.last().unwrap(); + let current_continuation_type = current_continuation_type.unwrap_ref_type(); + + // Argument types of current_continuation_type. These will in turn + // be the types of the arguments we receive when someone switches + // back to this switch instruction + let current_continuation_arg_types: SmallVec<[_; 8]> = + match current_continuation_type.heap_type { + WasmHeapType::ConcreteCont(index) => { + let mti = index + .as_module_type_index() + .expect("Only supporting module type indices on switch for now"); + + environ + .continuation_arguments(TypeIndex::from_u32(mti.as_u32())) + .iter() + .map(|ty| crate::value_type(environ.isa(), *ty)) + .collect() + } + _ => panic!("Invalid type on switch"), + }; + + let switch_return_values = environ.translate_switch( + builder, + *tag_index, + *contobj, + switch_args, + ¤t_continuation_arg_types, + )?; + + stack.popn(arity); + stack.pushn(&switch_return_values) } Operator::GlobalAtomicGet { .. } diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index 299ed239e2c3..65f0225db05a 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -233,12 +233,12 @@ macro_rules! foreach_builtin_function { // denote the continuation being `None`, `init_contref` // may be 0. #[cfg(feature = "stack-switching")] - table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer; + table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: size) -> pointer; // `value_contref` and `value_revision` together encode // the Option, as in previous libcall. #[cfg(feature = "stack-switching")] - table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool; + table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: size, len: u64) -> bool; // Return the instance ID for a given vmctx. #[cfg(feature = "gc")] diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs index 92104ad181d1..d9e771b02fd6 100644 --- a/crates/environ/src/gc.rs +++ b/crates/environ/src/gc.rs @@ -40,15 +40,10 @@ pub const VM_GC_HEADER_TYPE_INDEX_OFFSET: u32 = 4; /// Get the byte size of the given Wasm type when it is stored inside the GC /// heap. pub fn byte_size_of_wasm_ty_in_gc_heap(ty: &WasmStorageType) -> u32 { - use crate::{WasmHeapType::*, WasmRefType}; match ty { WasmStorageType::I8 => 1, WasmStorageType::I16 => 2, WasmStorageType::Val(ty) => match ty { - WasmValType::Ref(WasmRefType { - nullable: _, - heap_type: ConcreteCont(_) | Cont, - }) => unimplemented!("Stack switching feature not compatbile with GC, yet"), WasmValType::I32 | WasmValType::F32 | WasmValType::Ref(_) => 4, WasmValType::I64 | WasmValType::F64 => 8, WasmValType::V128 => 16, diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 1b12ed9a6139..71fb7b045da1 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -161,6 +161,12 @@ pub trait PtrSize { 4 } + /// This is the size of the largest value type (i.e. a V128). + #[inline] + fn maximum_value_size(&self) -> u8 { + self.size_of_vmglobal_definition() + } + // Offsets within `VMStoreContext` /// Return the offset of the `fuel_consumed` field of `VMStoreContext` @@ -333,6 +339,28 @@ pub trait PtrSize { .unwrap() } + // Offsets within `VMContObj` + + /// Return the offset of `VMContObj::contref` + fn vmcontobj_contref(&self) -> u8 { + 0 + } + + /// Return the offset of `VMContObj::revision` + fn vmcontobj_revision(&self) -> u8 { + self.size() + } + + /// Return the size of `VMContObj`. + fn size_of_vmcontobj(&self) -> u8 { + u8::try_from(align( + u32::from(self.vmcontobj_revision()) + + u32::try_from(core::mem::size_of::()).unwrap(), + u32::from(self.size()), + )) + .unwrap() + } + // Offsets within `VMContRef` /// Return the offset of `VMContRef::common_stack_information`. @@ -362,7 +390,7 @@ pub trait PtrSize { /// Return the offset of `VMContRef::stack`. fn vmcontref_stack(&self) -> u8 { - self.vmcontref_revision() + 8 + self.vmcontref_revision() + self.size() } /// Return the offset of `VMContRef::args`. diff --git a/crates/fuzzing/src/generators/value.rs b/crates/fuzzing/src/generators/value.rs index 697982a12013..685c655400f1 100644 --- a/crates/fuzzing/src/generators/value.rs +++ b/crates/fuzzing/src/generators/value.rs @@ -18,6 +18,7 @@ pub enum DiffValue { ExternRef { null: bool }, AnyRef { null: bool }, ExnRef { null: bool }, + ContRef { null: bool }, } impl DiffValue { @@ -32,6 +33,7 @@ impl DiffValue { DiffValue::ExternRef { .. } => DiffValueType::ExternRef, DiffValue::AnyRef { .. } => DiffValueType::AnyRef, DiffValue::ExnRef { .. } => DiffValueType::ExnRef, + DiffValue::ContRef { .. } => DiffValueType::ContRef, } } @@ -189,6 +191,7 @@ impl DiffValue { ExternRef => DiffValue::ExternRef { null: true }, AnyRef => DiffValue::AnyRef { null: true }, ExnRef => DiffValue::ExnRef { null: true }, + ContRef => DiffValue::ContRef { null: true }, }; arbitrary::Result::Ok(val) } @@ -236,6 +239,7 @@ impl Hash for DiffValue { DiffValue::FuncRef { null } => null.hash(state), DiffValue::AnyRef { null } => null.hash(state), DiffValue::ExnRef { null } => null.hash(state), + DiffValue::ContRef { null } => null.hash(state), } } } @@ -273,6 +277,7 @@ impl PartialEq for DiffValue { (Self::ExternRef { null: a }, Self::ExternRef { null: b }) => a == b, (Self::AnyRef { null: a }, Self::AnyRef { null: b }) => a == b, (Self::ExnRef { null: a }, Self::ExnRef { null: b }) => a == b, + (Self::ContRef { null: a }, Self::ContRef { null: b }) => a == b, _ => false, } } @@ -291,6 +296,7 @@ pub enum DiffValueType { ExternRef, AnyRef, ExnRef, + ContRef, } impl TryFrom for DiffValueType { @@ -310,6 +316,7 @@ impl TryFrom for DiffValueType { (true, HeapType::I31) => Ok(Self::AnyRef), (true, HeapType::None) => Ok(Self::AnyRef), (true, HeapType::Exn) => Ok(Self::ExnRef), + (true, HeapType::Cont) => Ok(Self::ContRef), _ => Err("non-null reference types are not supported yet"), }, } diff --git a/crates/fuzzing/src/oracles/diff_spec.rs b/crates/fuzzing/src/oracles/diff_spec.rs index c5b9edbafc38..85fab4b5d7a7 100644 --- a/crates/fuzzing/src/oracles/diff_spec.rs +++ b/crates/fuzzing/src/oracles/diff_spec.rs @@ -107,7 +107,8 @@ impl From<&DiffValue> for SpecValue { DiffValue::FuncRef { .. } | DiffValue::ExternRef { .. } | DiffValue::AnyRef { .. } - | DiffValue::ExnRef { .. } => { + | DiffValue::ExnRef { .. } + | DiffValue::ContRef { .. } => { unimplemented!() } } diff --git a/crates/fuzzing/src/oracles/diff_v8.rs b/crates/fuzzing/src/oracles/diff_v8.rs index 7e0ed12e21ec..eaf612a4be2c 100644 --- a/crates/fuzzing/src/oracles/diff_v8.rs +++ b/crates/fuzzing/src/oracles/diff_v8.rs @@ -191,6 +191,7 @@ impl DiffInstance for V8Instance { DiffValue::V128(_) => return Ok(None), DiffValue::AnyRef { .. } => unimplemented!(), DiffValue::ExnRef { .. } => unimplemented!(), + DiffValue::ContRef { .. } => unimplemented!(), }); } // JS doesn't support v128 return values @@ -316,6 +317,7 @@ fn get_diff_value( DiffValueType::AnyRef => unimplemented!(), DiffValueType::ExnRef => unimplemented!(), DiffValueType::V128 => unreachable!(), + DiffValueType::ContRef => unimplemented!(), } } diff --git a/crates/fuzzing/src/oracles/diff_wasmi.rs b/crates/fuzzing/src/oracles/diff_wasmi.rs index c591c218cdf8..dbed7f2f2700 100644 --- a/crates/fuzzing/src/oracles/diff_wasmi.rs +++ b/crates/fuzzing/src/oracles/diff_wasmi.rs @@ -195,6 +195,7 @@ impl From<&DiffValue> for wasmi::Val { } DiffValue::AnyRef { .. } => unimplemented!(), DiffValue::ExnRef { .. } => unimplemented!(), + DiffValue::ContRef { .. } => unimplemented!(), } } } diff --git a/crates/fuzzing/src/oracles/diff_wasmtime.rs b/crates/fuzzing/src/oracles/diff_wasmtime.rs index 53c54b73a239..f600005164b4 100644 --- a/crates/fuzzing/src/oracles/diff_wasmtime.rs +++ b/crates/fuzzing/src/oracles/diff_wasmtime.rs @@ -227,6 +227,10 @@ impl From<&DiffValue> for Val { assert!(null); Val::ExnRef(None) } + DiffValue::ContRef { null } => { + assert!(null); + Val::ExnRef(None) + } } } } @@ -243,6 +247,7 @@ impl From for DiffValue { Val::FuncRef(r) => DiffValue::FuncRef { null: r.is_none() }, Val::AnyRef(r) => DiffValue::AnyRef { null: r.is_none() }, Val::ExnRef(e) => DiffValue::ExnRef { null: e.is_none() }, + Val::ContRef(c) => DiffValue::ContRef { null: c.is_none() }, } } } diff --git a/crates/wasmtime/src/runtime/coredump.rs b/crates/wasmtime/src/runtime/coredump.rs index 517c2fbb241a..ac557b7f602d 100644 --- a/crates/wasmtime/src/runtime/coredump.rs +++ b/crates/wasmtime/src/runtime/coredump.rs @@ -210,6 +210,12 @@ impl WasmCoreDump { ty: wasm_encoder::AbstractHeapType::Exn, }) } + Val::ContRef(_) => { + wasm_encoder::ConstExpr::ref_null(wasm_encoder::HeapType::Abstract { + shared: false, + ty: wasm_encoder::AbstractHeapType::Cont, + }) + } }; globals.global( wasm_encoder::GlobalType { diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs index ce87e2616592..2223c114334c 100644 --- a/crates/wasmtime/src/runtime/externals/global.rs +++ b/crates/wasmtime/src/runtime/externals/global.rs @@ -274,6 +274,16 @@ impl Global { let new = new.as_ref(); definition.write_gc_ref(&mut store, new); } + Val::ContRef(None) => { + // Allow null continuation references for globals - these are just placeholders + definition.write_gc_ref(&mut store, None); + } + Val::ContRef(Some(_)) => { + // TODO(#10248): Implement non-null global continuation reference handling + return Err(anyhow::anyhow!( + "setting non-null continuation references in globals not yet supported" + )); + } } } Ok(()) diff --git a/crates/wasmtime/src/runtime/trampoline/global.rs b/crates/wasmtime/src/runtime/trampoline/global.rs index 2fad1d81aa80..5ef0a6ddf2cc 100644 --- a/crates/wasmtime/src/runtime/trampoline/global.rs +++ b/crates/wasmtime/src/runtime/trampoline/global.rs @@ -70,6 +70,16 @@ pub fn generate_global_export( let new = new.as_ref(); global.write_gc_ref(&mut store, new); } + Val::ContRef(None) => { + // Allow null continuation references for trampoline globals - these are just placeholders + global.write_gc_ref(&mut store, None); + } + Val::ContRef(Some(_)) => { + // TODO(#10248): Implement non-null trampoline continuation reference handling + return Err(anyhow::anyhow!( + "non-null continuation references in trampoline globals not yet supported" + )); + } } } diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs index e572b2dfb157..15aa627b8bd4 100644 --- a/crates/wasmtime/src/runtime/values.rs +++ b/crates/wasmtime/src/runtime/values.rs @@ -8,6 +8,13 @@ use wasmtime_environ::WasmHeapTopType; pub use crate::runtime::vm::ValRaw; +/// A stub implementation for continuation references. +/// +/// This is a placeholder until continuation objects are fully integrated +/// with the GC system (see #10248). +#[derive(Debug, Clone, Copy)] +pub struct ContRef; + /// Possible runtime values that a WebAssembly module can either consume or /// produce. /// @@ -50,6 +57,12 @@ pub enum Val { /// An exception reference. ExnRef(Option>), + + /// A continuation reference. + /// + /// Note: This is currently a stub implementation as continuation objects + /// are not yet fully integrated with the GC system. See #10248. + ContRef(Option), } macro_rules! accessors { @@ -118,7 +131,7 @@ impl Val { WasmHeapTopType::Extern => Val::ExternRef(None), WasmHeapTopType::Any => Val::AnyRef(None), WasmHeapTopType::Exn => Val::ExnRef(None), - WasmHeapTopType::Cont => todo!(), // FIXME(#10248) + WasmHeapTopType::Cont => Val::ContRef(None), } } @@ -177,6 +190,12 @@ impl Val { Val::AnyRef(Some(a)) => ValType::Ref(RefType::new(false, a._ty(store)?)), Val::ExnRef(None) => ValType::NULLEXNREF, Val::ExnRef(Some(e)) => ValType::Ref(RefType::new(false, e._ty(store)?.into())), + Val::ContRef(_) => { + // TODO(#10248): Return proper continuation reference type when available + return Err(anyhow::anyhow!( + "continuation references not yet supported in embedder API" + )); + } }) } @@ -216,7 +235,8 @@ impl Val { | (Val::FuncRef(_), _) | (Val::ExternRef(_), _) | (Val::AnyRef(_), _) - | (Val::ExnRef(_), _) => false, + | (Val::ExnRef(_), _) + | (Val::ContRef(_), _) => false, }) } @@ -268,6 +288,12 @@ impl Val { Some(f) => f.to_raw(store), None => ptr::null_mut(), })), + Val::ContRef(_) => { + // TODO(#10248): Implement proper continuation reference to_raw conversion + Err(anyhow::anyhow!( + "continuation references not yet supported in to_raw conversion" + )) + } } } @@ -363,6 +389,7 @@ impl Val { Val::AnyRef(a) => Some(Ref::Any(a)), Val::ExnRef(e) => Some(Ref::Exn(e)), Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => None, + Val::ContRef(_) => None, // TODO(#10248): Return proper Ref::Cont when available } } @@ -509,6 +536,9 @@ impl Val { // particular store, so they're always considered as "yes I came // from that store", Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => true, + + // Continuation references are not yet associated with stores + Val::ContRef(_) => true, // TODO(#10248): Proper store association when implemented } } } diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs index 04982d3c49b7..5d6f525f8a5f 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs @@ -276,6 +276,12 @@ impl VMArrayRef { .gc_object_data(self.as_gc_ref()) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement array continuation reference element handling + return Err(anyhow::anyhow!( + "setting continuation references in array elements not yet supported" + )); + } } Ok(()) } @@ -383,6 +389,12 @@ impl VMArrayRef { .gc_object_data(self.as_gc_ref()) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement array continuation reference init handling + return Err(anyhow::anyhow!( + "initializing continuation references in array elements not yet supported" + )); + } } Ok(()) } diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs index 364faf1c0194..a8ecfea074e4 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs @@ -233,6 +233,12 @@ impl VMStructRef { .gc_object_data(self.as_gc_ref()) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement struct continuation reference field handling + return Err(anyhow::anyhow!( + "setting continuation references in struct fields not yet supported" + )); + } } Ok(()) } @@ -386,6 +392,12 @@ pub(crate) fn initialize_field_impl( .gc_object_data(gc_ref) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement struct continuation reference field init handling + return Err(anyhow::anyhow!( + "initializing continuation references in struct fields not yet supported" + )); + } } Ok(()) } diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 395c22b52a78..8180ccf89ff8 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -157,6 +157,7 @@ pub mod raw { (@ty f64x2) => (f64x2); (@ty bool) => (bool); (@ty pointer) => (*mut u8); + (@ty size) => (usize); } wasmtime_environ::foreach_builtin_function!(libcall); @@ -339,7 +340,7 @@ unsafe fn table_grow_cont_obj( // The following two values together form the initial Option. // A None value is indicated by the pointer being null. init_value_contref: *mut u8, - init_value_revision: u64, + init_value_revision: usize, ) -> Result> { let defined_table_index = DefinedTableIndex::from_u32(defined_table_index); let element = unsafe { VMContObj::from_raw_parts(init_value_contref, init_value_revision) }; @@ -416,7 +417,7 @@ unsafe fn table_fill_cont_obj( table_index: u32, dst: u64, value_contref: *mut u8, - value_revision: u64, + value_revision: usize, len: u64, ) -> Result<()> { let instance = store.instance_mut(instance); diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 2a63d0baa35f..32c2c275339a 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -39,19 +39,15 @@ pub use stack::*; /// For performance reasons, the VMContRef at the bottom of this chain /// (i.e., the one pointed to by the VMContObj) has a pointer to the /// other end of the chain (i.e., its last ancestor). -// FIXME(frank-emrich) Does this actually need to be 16-byte aligned any -// more? Now that we use I128 on the Cranelift side (see -// [wasmtime_cranelift::stack_switching::fatpointer::pointer_type]), it -// should be fine to use the natural alignment of the type. -#[repr(C, align(16))] +#[repr(C)] #[derive(Debug, Clone, Copy)] pub struct VMContObj { - pub revision: u64, pub contref: NonNull, + pub revision: usize, } impl VMContObj { - pub fn new(contref: NonNull, revision: u64) -> Self { + pub fn new(contref: NonNull, revision: usize) -> Self { Self { contref, revision } } @@ -63,7 +59,7 @@ impl VMContObj { /// /// Behavior will be undefined if a pointer to data that is not a /// VMContRef is provided. - pub unsafe fn from_raw_parts(contref: *mut u8, revision: u64) -> Option { + pub unsafe fn from_raw_parts(contref: *mut u8, revision: usize) -> Option { NonNull::new(contref.cast::()).map(|contref| Self::new(contref, revision)) } } @@ -208,7 +204,7 @@ pub struct VMContRef { pub last_ancestor: *mut VMContRef, /// Revision counter. - pub revision: u64, + pub revision: usize, /// The underlying stack. pub stack: VMContinuationStack, @@ -627,6 +623,24 @@ mod tests { ); } + #[test] + fn check_vm_contobj_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMContObj, contref), + usize::from(offsets.ptr.vmcontobj_contref()) + ); + assert_eq!( + offset_of!(VMContObj, revision), + usize::from(offsets.ptr.vmcontobj_revision()) + ); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmcontobj()) + ) + } + #[test] fn check_vm_contref_offsets() { let module = Module::new(); diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 818b08940873..220e973082bc 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -1056,6 +1056,7 @@ macro_rules! define_builtin_array { (@ty f64x2) => (f64x2); (@ty bool) => (bool); (@ty pointer) => (*mut u8); + (@ty size) => (usize); (@ty vmctx) => (NonNull); } diff --git a/crates/wasmtime/src/runtime/wave/core.rs b/crates/wasmtime/src/runtime/wave/core.rs index 338ce4b2d888..fd89da83d7b9 100644 --- a/crates/wasmtime/src/runtime/wave/core.rs +++ b/crates/wasmtime/src/runtime/wave/core.rs @@ -40,6 +40,7 @@ impl WasmValue for crate::Val { Self::ExternRef(_) => WasmTypeKind::Unsupported, Self::AnyRef(_) => WasmTypeKind::Unsupported, Self::ExnRef(_) => WasmTypeKind::Unsupported, + Self::ContRef(_) => WasmTypeKind::Unsupported, } } diff --git a/src/commands/run.rs b/src/commands/run.rs index d42e573b4c05..e3fd86d27b82 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -746,6 +746,8 @@ impl RunCommand { Val::AnyRef(Some(_)) => println!(""), Val::ExnRef(None) => println!(""), Val::ExnRef(Some(_)) => println!(""), + Val::ContRef(None) => println!(""), + Val::ContRef(Some(_)) => println!(""), } } diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.wat new file mode 100644 index 000000000000..76afed2d11e7 --- /dev/null +++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat @@ -0,0 +1,311 @@ +;;! target = "x86_64-unknown-linux-gnu" +;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y" +;;! test = "optimize" + +(module + (type $ft (func)) + (tag $t (param i32)) + (type $ct (cont $ft)) + + (func $countdown + (local $i i32) + (local.set $i (i32.const 10)) + (loop $loop + ;; suspend and pass countdown to our cosnumer + (suspend $t (local.get $i)) + ;; decrement i; break if we're at 0 + (local.tee $i (i32.sub (local.get $i) (i32.const 1))) + (br_if $loop) + ) + ) + (elem declare func $countdown) + + (func (export "main") + (local $c (ref $ct)) + (local.set $c (cont.new $ct (ref.func $countdown))) + (loop $loop + (block $on_gen (result i32 (ref $ct)) + (resume $ct (on $t $on_gen) (local.get $c)) + ;; no more data, return + (return) + ) + ;; stack contains [i32 (ref $ct)] + (local.set $c) + (drop) ;; could print here + (br $loop) + ) + ) +) + +;; function u0:0(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 16, align = 65536 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003c v3 = iconst.i32 10 +;; v61 = iconst.i64 120 +;; @0044 v30 = stack_addr.i64 ss0 +;; v59 = iconst.i64 16 +;; v60 = iconst.i64 0 +;; v57 = iconst.i64 80 +;; v56 = iconst.i64 -24 +;; v70 = iconst.i64 0x0002_0000_0000 +;; @0040 jump block2(v3) ; v3 = 10 +;; +;; block2(v4: i32): +;; @0044 v8 = load.i64 notrap aligned v0+8 +;; @0044 v9 = load.i64 notrap aligned v8+64 +;; @0044 v10 = load.i64 notrap aligned v8+72 +;; v65 = iconst.i64 1 +;; v64 = iconst.i64 24 +;; @003a v2 = iconst.i32 0 +;; @0044 jump block4(v9, v10, v4) +;; +;; block4(v11: i64, v12: i64, v52: i32): +;; v73 = iconst.i64 1 +;; v74 = icmp eq v11, v73 ; v73 = 1 +;; @0044 trapnz v74, user22 +;; @0044 jump block5 +;; +;; block5: +;; @0044 v14 = load.i64 notrap aligned v12+48 +;; @0044 v15 = load.i64 notrap aligned v12+56 +;; v75 = iconst.i64 24 +;; v76 = iadd v15, v75 ; v75 = 24 +;; @0044 v17 = load.i64 notrap aligned v76+8 +;; @0044 v18 = load.i32 notrap aligned v15+40 +;; v77 = iconst.i32 0 +;; v67 = iconst.i32 3 +;; v66 = iconst.i64 48 +;; @0044 v6 = iadd.i64 v0, v66 ; v66 = 48 +;; v62 = iconst.i32 1 +;; @0044 jump block6(v77) ; v77 = 0 +;; +;; block6(v20: i32): +;; @0044 v21 = icmp ult v20, v18 +;; @0044 brif v21, block7, block4(v14, v15, v52) +;; +;; block7: +;; v78 = iconst.i32 3 +;; v79 = ishl.i32 v20, v78 ; v78 = 3 +;; @0044 v23 = uextend.i64 v79 +;; @0044 v24 = iadd.i64 v17, v23 +;; @0044 v25 = load.i64 notrap aligned v24 +;; v80 = iadd.i64 v0, v66 ; v66 = 48 +;; v81 = icmp eq v25, v80 +;; v82 = iconst.i32 1 +;; v83 = iadd.i32 v20, v82 ; v82 = 1 +;; @0044 brif v81, block8, block6(v83) +;; +;; block8: +;; @0044 store.i64 notrap aligned v12, v10+64 +;; v84 = iconst.i32 1 +;; v85 = iconst.i64 120 +;; v86 = iadd.i64 v10, v85 ; v85 = 120 +;; @0044 store notrap aligned v84, v86+4 ; v84 = 1 +;; @0044 store.i64 notrap aligned v30, v86+8 +;; @0044 store.i32 notrap aligned v4, v30 +;; @0044 store notrap aligned v84, v86 ; v84 = 1 +;; v87 = iconst.i32 3 +;; v88 = iconst.i64 16 +;; v89 = iadd.i64 v10, v88 ; v88 = 16 +;; @0044 store notrap aligned v87, v89 ; v87 = 3 +;; v90 = iconst.i64 0 +;; @0044 store notrap aligned v90, v12+48 ; v90 = 0 +;; @0044 store notrap aligned v90, v12+56 ; v90 = 0 +;; v91 = iconst.i64 80 +;; v92 = iadd.i64 v12, v91 ; v91 = 80 +;; @0044 v43 = load.i64 notrap aligned v92 +;; v93 = iconst.i64 -24 +;; v94 = iadd v43, v93 ; v93 = -24 +;; @0044 v40 = uextend.i64 v20 +;; v95 = iconst.i64 0x0002_0000_0000 +;; v96 = bor v40, v95 ; v95 = 0x0002_0000_0000 +;; @0044 v45 = stack_switch v94, v94, v96 +;; @0044 v47 = load.i64 notrap aligned v86+8 +;; v97 = iconst.i32 0 +;; @0044 store notrap aligned v97, v86 ; v97 = 0 +;; @0044 store notrap aligned v97, v86+4 ; v97 = 0 +;; @0044 store notrap aligned v90, v86+8 ; v90 = 0 +;; v98 = isub.i32 v52, v84 ; v84 = 1 +;; @004d brif v98, block2(v98), block10 +;; +;; block10: +;; @004f jump block3 +;; +;; block3: +;; @0050 jump block1 +;; +;; block1: +;; @0050 return +;; } +;; +;; function u0:1(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 8, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1610612736:7 sig0 +;; fn1 = colocated u1610612736:52 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0056 v2 = iconst.i32 0 +;; @0056 v4 = call fn0(v0, v2) ; v2 = 0 +;; @0058 trapz v4, user16 +;; @0058 v8 = call fn1(v0, v4, v2, v2) ; v2 = 0, v2 = 0 +;; @0058 v9 = load.i64 notrap aligned v8+72 +;; @0058 v11 = uextend.i128 v9 +;; v119 = iconst.i64 64 +;; v121 = ishl v11, v119 ; v119 = 64 +;; @0058 v10 = uextend.i128 v8 +;; @0058 v13 = bor v121, v10 +;; v116 = iconst.i64 1 +;; @0062 v28 = iconst.i64 0 +;; @0062 v29 = iconst.i64 2 +;; @0062 v32 = iconst.i32 1 +;; v114 = iconst.i64 16 +;; @0062 v34 = iconst.i32 2 +;; v110 = iconst.i64 24 +;; @0062 v45 = stack_addr.i64 ss0 +;; v109 = iconst.i64 48 +;; @0062 v47 = iadd v0, v109 ; v109 = 48 +;; v107 = iconst.i64 80 +;; v106 = iconst.i64 -24 +;; v125 = iconst.i64 0x0001_0000_0000 +;; v108 = iconst.i64 32 +;; @005c jump block2(v13) +;; +;; block2(v16: i128): +;; @0062 jump block5 +;; +;; block5: +;; @0062 v18 = ireduce.i64 v16 +;; @0062 trapz v18, user16 +;; @0062 v21 = load.i64 notrap aligned v18+72 +;; v128 = iconst.i64 64 +;; v129 = ushr.i128 v16, v128 ; v128 = 64 +;; @0062 v20 = ireduce.i64 v129 +;; @0062 v22 = icmp eq v21, v20 +;; @0062 trapz v22, user23 +;; v130 = iconst.i64 1 +;; v131 = iadd v21, v130 ; v130 = 1 +;; @0062 store notrap aligned v131, v18+72 +;; @0062 v24 = load.i64 notrap aligned v18+64 +;; @0062 v25 = load.i64 notrap aligned v0+8 +;; @0062 v26 = load.i64 notrap aligned v25+64 +;; @0062 v27 = load.i64 notrap aligned v25+72 +;; @0062 store notrap aligned v26, v24+48 +;; @0062 store notrap aligned v27, v24+56 +;; v132 = iconst.i64 0 +;; @0062 store notrap aligned v132, v18+64 ; v132 = 0 +;; @0062 v30 = load.i64 notrap aligned v0+8 +;; v133 = iconst.i64 2 +;; @0062 store notrap aligned v133, v30+64 ; v133 = 2 +;; @0062 store notrap aligned v18, v30+72 +;; v134 = iconst.i32 1 +;; v135 = iconst.i64 16 +;; v136 = iadd v18, v135 ; v135 = 16 +;; @0062 store notrap aligned v134, v136 ; v134 = 1 +;; v137 = iconst.i32 2 +;; v138 = iadd v27, v135 ; v135 = 16 +;; @0062 store notrap aligned v137, v138 ; v137 = 2 +;; @0062 v36 = load.i64 notrap aligned readonly v0+8 +;; @0062 v38 = load.i64 notrap aligned v36+56 +;; @0062 store notrap aligned v38, v27+8 +;; @0062 v39 = load.i64 notrap aligned v36+16 +;; @0062 store notrap aligned v39, v27 +;; @0062 v41 = load.i64 notrap aligned v18 +;; @0062 store notrap aligned v41, v36+16 +;; @0062 v42 = load.i64 notrap aligned v18+8 +;; @0062 store notrap aligned v42, v36+56 +;; v139 = iconst.i64 24 +;; v140 = iadd v27, v139 ; v139 = 24 +;; @0062 store notrap aligned v134, v140+4 ; v134 = 1 +;; @0062 store.i64 notrap aligned v45, v140+8 +;; v141 = iadd.i64 v0, v109 ; v109 = 48 +;; @0062 store notrap aligned v141, v45 +;; @0062 store notrap aligned v134, v140 ; v134 = 1 +;; @0062 store notrap aligned v134, v27+40 ; v134 = 1 +;; v142 = iconst.i64 80 +;; v143 = iadd v24, v142 ; v142 = 80 +;; @0062 v54 = load.i64 notrap aligned v143 +;; v144 = iconst.i64 -24 +;; v145 = iadd v54, v144 ; v144 = -24 +;; v146 = iconst.i64 0x0001_0000_0000 +;; @0062 v56 = stack_switch v145, v145, v146 ; v146 = 0x0001_0000_0000 +;; @0062 v57 = load.i64 notrap aligned v0+8 +;; @0062 v58 = load.i64 notrap aligned v57+64 +;; @0062 v59 = load.i64 notrap aligned v57+72 +;; @0062 store notrap aligned v26, v57+64 +;; @0062 store notrap aligned v27, v57+72 +;; @0062 store notrap aligned v134, v138 ; v134 = 1 +;; v147 = iconst.i32 0 +;; @0062 store notrap aligned v147, v140 ; v147 = 0 +;; @0062 store notrap aligned v147, v140+4 ; v147 = 0 +;; @0062 store notrap aligned v132, v140+8 ; v132 = 0 +;; @0062 store notrap aligned v132, v27+40 ; v132 = 0 +;; v148 = iconst.i64 32 +;; v149 = ushr v56, v148 ; v148 = 32 +;; @0062 brif v149, block7, block6 +;; +;; block7: +;; @0062 v69 = load.i64 notrap aligned v36+56 +;; @0062 store notrap aligned v69, v59+8 +;; @0062 v71 = load.i64 notrap aligned v27 +;; @0062 store notrap aligned v71, v36+16 +;; @0062 v72 = load.i64 notrap aligned v27+8 +;; @0062 store notrap aligned v72, v36+56 +;; @0062 v74 = load.i64 notrap aligned v59+72 +;; @0062 jump block8 +;; +;; block9 cold: +;; @0062 trap user11 +;; +;; block10: +;; v98 = iconst.i64 120 +;; @0062 v79 = iadd.i64 v59, v98 ; v98 = 120 +;; @0062 v80 = load.i64 notrap aligned v79+8 +;; @0062 v81 = load.i32 notrap aligned v80 +;; v154 = iconst.i32 0 +;; @0062 store notrap aligned v154, v79 ; v154 = 0 +;; @0062 jump block4 +;; +;; block8: +;; @0062 v73 = ireduce.i32 v56 +;; @0062 br_table v73, block9, [block10] +;; +;; block6: +;; @0062 v84 = load.i64 notrap aligned v27 +;; @0062 store notrap aligned v84, v36+16 +;; @0062 v85 = load.i64 notrap aligned v27+8 +;; @0062 store notrap aligned v85, v36+56 +;; @0062 v87 = iconst.i32 4 +;; v150 = iconst.i64 16 +;; v151 = iadd.i64 v59, v150 ; v150 = 16 +;; @0062 store notrap aligned v87, v151 ; v87 = 4 +;; v94 = iconst.i64 104 +;; @0062 v89 = iadd.i64 v59, v94 ; v94 = 104 +;; @0062 v90 = load.i64 notrap aligned v89+8 +;; v152 = iconst.i32 0 +;; @0062 store notrap aligned v152, v89 ; v152 = 0 +;; @0062 store notrap aligned v152, v89+4 ; v152 = 0 +;; v153 = iconst.i64 0 +;; @0062 store notrap aligned v153, v89+8 ; v153 = 0 +;; @0068 return +;; +;; block4: +;; @0062 v76 = uextend.i128 v74 +;; v155 = iconst.i64 64 +;; v156 = ishl v76, v155 ; v155 = 64 +;; @0062 v75 = uextend.i128 v59 +;; @0062 v78 = bor v156, v75 +;; @006d jump block2(v78) +;; } diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.wat new file mode 100644 index 000000000000..bfe8a715b1ce --- /dev/null +++ b/tests/disas/stack-switching/resume-suspend.wat @@ -0,0 +1,263 @@ +;;! target = "x86_64-unknown-linux-gnu" +;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y" +;;! test = "optimize" + +(module + (type $ft (func)) + (tag $t (type $ft)) + (type $ct (cont $ft)) + + (func $target (suspend $t)) + (elem declare func $target) + + (func (export "minimal_suspend") + (local $k (ref null $ct)) + (local.set $k (cont.new $ct (ref.func $target))) + (block $h (result (ref null $ct)) + (resume $ct (on $t $h) (local.get $k)) + ;; continuation suspended back... + (ref.null $ct) + ) + (drop) + ) +) + +;; function u0:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003b v5 = load.i64 notrap aligned v0+8 +;; @003b v6 = load.i64 notrap aligned v5+64 +;; @003b v7 = load.i64 notrap aligned v5+72 +;; v54 = iconst.i64 1 +;; v53 = iconst.i64 24 +;; @003b v16 = iconst.i32 0 +;; @003b jump block2(v6, v7) +;; +;; block2(v8: i64, v9: i64): +;; v62 = iconst.i64 1 +;; v63 = icmp eq v8, v62 ; v62 = 1 +;; @003b trapnz v63, user22 +;; @003b jump block3 +;; +;; block3: +;; @003b v11 = load.i64 notrap aligned v9+48 +;; @003b v12 = load.i64 notrap aligned v9+56 +;; v64 = iconst.i64 24 +;; v65 = iadd v12, v64 ; v64 = 24 +;; @003b v14 = load.i64 notrap aligned v65+8 +;; @003b v15 = load.i32 notrap aligned v12+40 +;; v66 = iconst.i32 0 +;; v56 = iconst.i32 3 +;; v55 = iconst.i64 48 +;; @003b v3 = iadd.i64 v0, v55 ; v55 = 48 +;; v51 = iconst.i32 1 +;; @003b jump block4(v66) ; v66 = 0 +;; +;; block4(v17: i32): +;; @003b v18 = icmp ult v17, v15 +;; @003b brif v18, block5, block2(v11, v12) +;; +;; block5: +;; v67 = iconst.i32 3 +;; v68 = ishl.i32 v17, v67 ; v67 = 3 +;; @003b v20 = uextend.i64 v68 +;; @003b v21 = iadd.i64 v14, v20 +;; @003b v22 = load.i64 notrap aligned v21 +;; v69 = iadd.i64 v0, v55 ; v55 = 48 +;; v70 = icmp eq v22, v69 +;; v71 = iconst.i32 1 +;; v72 = iadd.i32 v17, v71 ; v71 = 1 +;; @003b brif v70, block6, block4(v72) +;; +;; block6: +;; @003b store.i64 notrap aligned v9, v7+64 +;; v73 = iconst.i32 3 +;; v48 = iconst.i64 16 +;; @003b v28 = iadd.i64 v7, v48 ; v48 = 16 +;; @003b store notrap aligned v73, v28 ; v73 = 3 +;; v49 = iconst.i64 0 +;; @003b store notrap aligned v49, v9+48 ; v49 = 0 +;; @003b store notrap aligned v49, v9+56 ; v49 = 0 +;; v46 = iconst.i64 80 +;; @003b v35 = iadd.i64 v9, v46 ; v46 = 80 +;; @003b v36 = load.i64 notrap aligned v35 +;; v45 = iconst.i64 -24 +;; @003b v37 = iadd v36, v45 ; v45 = -24 +;; @003b v33 = uextend.i64 v17 +;; v59 = iconst.i64 0x0002_0000_0000 +;; v60 = bor v33, v59 ; v59 = 0x0002_0000_0000 +;; @003b v38 = stack_switch v37, v37, v60 +;; v50 = iconst.i64 120 +;; @003b v25 = iadd.i64 v7, v50 ; v50 = 120 +;; @003b v40 = load.i64 notrap aligned v25+8 +;; v74 = iconst.i32 0 +;; @003b store notrap aligned v74, v25 ; v74 = 0 +;; @003b store notrap aligned v74, v25+4 ; v74 = 0 +;; @003b store notrap aligned v49, v25+8 ; v49 = 0 +;; @003d jump block1 +;; +;; block1: +;; @003d return +;; } +;; +;; function u0:1(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 8, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1610612736:7 sig0 +;; fn1 = colocated u1610612736:52 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0043 v7 = iconst.i32 0 +;; @0043 v9 = call fn0(v0, v7) ; v7 = 0 +;; @0045 trapz v9, user16 +;; @0045 v13 = call fn1(v0, v9, v7, v7) ; v7 = 0, v7 = 0 +;; @0045 v14 = load.i64 notrap aligned v13+72 +;; @004e jump block3 +;; +;; block3: +;; @0045 v16 = uextend.i128 v14 +;; v130 = iconst.i64 64 +;; v134 = ishl v16, v130 ; v130 = 64 +;; v136 = ireduce.i64 v134 +;; v138 = bor v136, v13 +;; @004e trapz v138, user16 +;; @004e v24 = load.i64 notrap aligned v138+72 +;; @0045 v15 = uextend.i128 v13 +;; @0045 v18 = bor v134, v15 +;; v140 = ushr v18, v130 ; v130 = 64 +;; @004e v23 = ireduce.i64 v140 +;; @004e v25 = icmp eq v24, v23 +;; @004e trapz v25, user23 +;; v125 = iconst.i64 1 +;; @004e v26 = iadd v24, v125 ; v125 = 1 +;; @004e store notrap aligned v26, v138+72 +;; @004e v27 = load.i64 notrap aligned v138+64 +;; @004e v28 = load.i64 notrap aligned v0+8 +;; @004e v29 = load.i64 notrap aligned v28+64 +;; @004e v30 = load.i64 notrap aligned v28+72 +;; @004e store notrap aligned v29, v27+48 +;; @004e store notrap aligned v30, v27+56 +;; @0040 v2 = iconst.i64 0 +;; @004e store notrap aligned v2, v138+64 ; v2 = 0 +;; @004e v33 = load.i64 notrap aligned v0+8 +;; @004e v32 = iconst.i64 2 +;; @004e store notrap aligned v32, v33+64 ; v32 = 2 +;; @004e store notrap aligned v138, v33+72 +;; @004e v35 = iconst.i32 1 +;; v123 = iconst.i64 16 +;; @004e v36 = iadd v138, v123 ; v123 = 16 +;; @004e store notrap aligned v35, v36 ; v35 = 1 +;; @004e v37 = iconst.i32 2 +;; @004e v38 = iadd v30, v123 ; v123 = 16 +;; @004e store notrap aligned v37, v38 ; v37 = 2 +;; @004e v39 = load.i64 notrap aligned readonly v0+8 +;; @004e v41 = load.i64 notrap aligned v39+56 +;; @004e store notrap aligned v41, v30+8 +;; @004e v42 = load.i64 notrap aligned v39+16 +;; @004e store notrap aligned v42, v30 +;; @004e v44 = load.i64 notrap aligned v138 +;; @004e store notrap aligned v44, v39+16 +;; @004e v45 = load.i64 notrap aligned v138+8 +;; @004e store notrap aligned v45, v39+56 +;; v119 = iconst.i64 24 +;; @004e v46 = iadd v30, v119 ; v119 = 24 +;; @004e store notrap aligned v35, v46+4 ; v35 = 1 +;; @004e v48 = stack_addr.i64 ss0 +;; @004e store notrap aligned v48, v46+8 +;; v118 = iconst.i64 48 +;; @004e v50 = iadd.i64 v0, v118 ; v118 = 48 +;; @004e store notrap aligned v50, v48 +;; @004e store notrap aligned v35, v46 ; v35 = 1 +;; @004e store notrap aligned v35, v30+40 ; v35 = 1 +;; v116 = iconst.i64 80 +;; @004e v56 = iadd v27, v116 ; v116 = 80 +;; @004e v57 = load.i64 notrap aligned v56 +;; v115 = iconst.i64 -24 +;; @004e v58 = iadd v57, v115 ; v115 = -24 +;; v142 = iconst.i64 0x0001_0000_0000 +;; @004e v59 = stack_switch v58, v58, v142 ; v142 = 0x0001_0000_0000 +;; @004e v60 = load.i64 notrap aligned v0+8 +;; @004e v61 = load.i64 notrap aligned v60+64 +;; @004e v62 = load.i64 notrap aligned v60+72 +;; @004e store notrap aligned v29, v60+64 +;; @004e store notrap aligned v30, v60+72 +;; @004e store notrap aligned v35, v38 ; v35 = 1 +;; v145 = iconst.i32 0 +;; @004e store notrap aligned v145, v46 ; v145 = 0 +;; @004e store notrap aligned v145, v46+4 ; v145 = 0 +;; @004e store notrap aligned v2, v46+8 ; v2 = 0 +;; @004e store notrap aligned v2, v30+40 ; v2 = 0 +;; v117 = iconst.i64 32 +;; @004e v69 = ushr v59, v117 ; v117 = 32 +;; @004e brif v69, block5, block4 +;; +;; block5: +;; @004e v72 = load.i64 notrap aligned v39+56 +;; @004e store notrap aligned v72, v62+8 +;; @004e v74 = load.i64 notrap aligned v30 +;; @004e store notrap aligned v74, v39+16 +;; @004e v75 = load.i64 notrap aligned v30+8 +;; @004e store notrap aligned v75, v39+56 +;; @004e v77 = load.i64 notrap aligned v62+72 +;; @004e jump block6 +;; +;; block7 cold: +;; @004e trap user11 +;; +;; block8: +;; v107 = iconst.i64 120 +;; @004e v82 = iadd.i64 v62, v107 ; v107 = 120 +;; @004e v83 = load.i64 notrap aligned v82+8 +;; v153 = iconst.i32 0 +;; @004e store notrap aligned v153, v82 ; v153 = 0 +;; @004e v79 = uextend.i128 v77 +;; v154 = iconst.i64 64 +;; v155 = ishl v79, v154 ; v154 = 64 +;; @004e v78 = uextend.i128 v62 +;; @004e v81 = bor v155, v78 +;; @004e jump block2(v81) +;; +;; block6: +;; @004e v76 = ireduce.i32 v59 +;; @004e br_table v76, block7, [block8] +;; +;; block4: +;; @004e v86 = load.i64 notrap aligned v30 +;; @004e store notrap aligned v86, v39+16 +;; @004e v87 = load.i64 notrap aligned v30+8 +;; @004e store notrap aligned v87, v39+56 +;; @004e v89 = iconst.i32 4 +;; v146 = iconst.i64 16 +;; v147 = iadd.i64 v62, v146 ; v146 = 16 +;; @004e store notrap aligned v89, v147 ; v89 = 4 +;; v103 = iconst.i64 104 +;; @004e v91 = iadd.i64 v62, v103 ; v103 = 104 +;; @004e v92 = load.i64 notrap aligned v91+8 +;; v148 = iconst.i32 0 +;; @004e store notrap aligned v148, v91 ; v148 = 0 +;; @004e store notrap aligned v148, v91+4 ; v148 = 0 +;; v149 = iconst.i64 0 +;; @004e store notrap aligned v149, v91+8 ; v149 = 0 +;; v150 = uextend.i128 v149 ; v149 = 0 +;; v151 = iconst.i64 64 +;; v152 = ishl v150, v151 ; v151 = 64 +;; @0040 v6 = bor v152, v150 +;; @0056 jump block2(v6) +;; +;; block2(v19: i128): +;; @0058 jump block1 +;; +;; block1: +;; @0058 return +;; } diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.wat new file mode 100644 index 000000000000..75c0458dfba3 --- /dev/null +++ b/tests/disas/stack-switching/symmetric-switch.wat @@ -0,0 +1,418 @@ +;;! target = "x86_64-unknown-linux-gnu" +;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y" + +(module + (type $fta (func)) + (type $cta (cont $fta)) + + (type $ftb (func (param (ref $cta)))) + (type $ctb (cont $ftb)) + + (tag $yield) + + (func $task_a (type $fta) + (cont.new $ctb (ref.func $task_b)) + (switch $ctb $yield) + ) + + (func $task_b (type $ftb)) + + (elem declare func $task_a $task_b) + + (func (export "entry") + (cont.new $cta (ref.func $task_a)) + (resume $cta (on $yield switch)) + ) +) + +;; function u0:0(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 24, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1610612736:7 sig0 +;; fn1 = colocated u1610612736:52 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003a v2 = iconst.i32 1 +;; @003a v4 = call fn0(v0, v2) ; v2 = 1 +;; @003c trapz v4, user16 +;; @003c v5 = iconst.i32 1 +;; @003c v6 = iconst.i32 0 +;; @003c v8 = call fn1(v0, v4, v5, v6) ; v5 = 1, v6 = 0 +;; @003c v9 = load.i64 notrap aligned v8+72 +;; @003c v10 = uextend.i128 v8 +;; @003c v11 = uextend.i128 v9 +;; v138 = iconst.i64 64 +;; v139 = uextend.i128 v138 ; v138 = 64 +;; @003c v12 = ishl v11, v139 +;; @003c v13 = bor v12, v10 +;; @003e v15 = ireduce.i64 v13 +;; v136 = iconst.i64 64 +;; v137 = uextend.i128 v136 ; v136 = 64 +;; @003e v16 = ushr v13, v137 +;; @003e v17 = ireduce.i64 v16 +;; @003e trapz v15, user16 +;; @003e v18 = load.i64 notrap aligned v15+72 +;; @003e v19 = icmp eq v18, v17 +;; @003e trapz v19, user23 +;; v135 = iconst.i64 1 +;; @003e v20 = iadd v18, v135 ; v135 = 1 +;; @003e store notrap aligned v20, v15+72 +;; v134 = iconst.i64 48 +;; @003e v22 = iadd v0, v134 ; v134 = 48 +;; @003e v23 = load.i64 notrap aligned v0+8 +;; @003e v24 = load.i64 notrap aligned v23+64 +;; @003e v25 = load.i64 notrap aligned v23+72 +;; @003e jump block2(v24, v25) +;; +;; block2(v26: i64, v27: i64): +;; v133 = iconst.i64 1 +;; @003e v28 = icmp eq v26, v133 ; v133 = 1 +;; @003e trapnz v28, user22 +;; @003e jump block3 +;; +;; block3: +;; @003e v29 = load.i64 notrap aligned v27+48 +;; @003e v30 = load.i64 notrap aligned v27+56 +;; v132 = iconst.i64 24 +;; @003e v31 = iadd v30, v132 ; v132 = 24 +;; @003e v32 = load.i64 notrap aligned v31+8 +;; @003e v33 = load.i32 notrap aligned v30+40 +;; @003e v34 = load.i32 notrap aligned v31 +;; @003e jump block4(v33) +;; +;; block4(v35: i32): +;; @003e v36 = icmp ult v35, v34 +;; @003e brif v36, block5, block2(v29, v30) +;; +;; block5: +;; v131 = iconst.i32 8 +;; @003e v37 = imul.i32 v35, v131 ; v131 = 8 +;; @003e v38 = uextend.i64 v37 +;; @003e v39 = iadd.i64 v32, v38 +;; @003e v40 = load.i64 notrap aligned v39 +;; @003e v41 = icmp eq v40, v22 +;; v130 = iconst.i32 1 +;; @003e v42 = iadd.i32 v35, v130 ; v130 = 1 +;; @003e brif v41, block6, block4(v42) +;; +;; block6: +;; @003e store.i64 notrap aligned v27, v25+64 +;; v129 = iconst.i64 120 +;; @003e v43 = iadd.i64 v25, v129 ; v129 = 120 +;; v128 = iconst.i64 0 +;; @003e v44 = iadd.i64 v25, v128 ; v128 = 0 +;; @003e v45 = iconst.i32 3 +;; v127 = iconst.i64 16 +;; @003e v46 = iadd v44, v127 ; v127 = 16 +;; @003e store notrap aligned v45, v46 ; v45 = 3 +;; @003e v47 = iconst.i64 0 +;; @003e v48 = iconst.i64 0 +;; @003e store notrap aligned v47, v27+48 ; v47 = 0 +;; @003e store notrap aligned v48, v27+56 ; v48 = 0 +;; @003e v49 = load.i64 notrap aligned readonly v0+8 +;; v126 = iconst.i64 0 +;; @003e v50 = iadd v44, v126 ; v126 = 0 +;; @003e v51 = load.i64 notrap aligned v49+56 +;; @003e store notrap aligned v51, v50+8 +;; @003e v52 = load.i64 notrap aligned v25+72 +;; @003e v53 = uextend.i128 v25 +;; @003e v54 = uextend.i128 v52 +;; v124 = iconst.i64 64 +;; v125 = uextend.i128 v124 ; v124 = 64 +;; @003e v55 = ishl v54, v125 +;; @003e v56 = bor v55, v53 +;; v123 = iconst.i64 0 +;; @003e v58 = iadd.i64 v15, v123 ; v123 = 0 +;; v122 = iconst.i64 16 +;; @003e v59 = iadd v58, v122 ; v122 = 16 +;; @003e v60 = load.i32 notrap aligned v59 +;; v121 = iconst.i32 0 +;; @003e v61 = icmp ne v60, v121 ; v121 = 0 +;; @003e brif v61, block9, block8 +;; +;; block8: +;; v120 = iconst.i64 104 +;; @003e v62 = iadd.i64 v15, v120 ; v120 = 104 +;; @003e v63 = load.i64 notrap aligned v62+8 +;; @003e v64 = load.i32 notrap aligned v62 +;; v119 = iconst.i32 1 +;; @003e v65 = iadd v64, v119 ; v119 = 1 +;; @003e store notrap aligned v65, v62 +;; @003e v66 = uextend.i64 v64 +;; v118 = iconst.i64 16 +;; @003e v67 = imul v66, v118 ; v118 = 16 +;; @003e v68 = iadd v63, v67 +;; @003e jump block10(v68) +;; +;; block9: +;; v117 = iconst.i64 120 +;; @003e v69 = iadd.i64 v15, v117 ; v117 = 120 +;; @003e v70 = load.i64 notrap aligned v69+8 +;; @003e v71 = load.i32 notrap aligned v69 +;; v116 = iconst.i32 1 +;; @003e v72 = iadd v71, v116 ; v116 = 1 +;; @003e store notrap aligned v72, v69 +;; @003e v73 = uextend.i64 v71 +;; v115 = iconst.i64 16 +;; @003e v74 = imul v73, v115 ; v115 = 16 +;; @003e v75 = iadd v70, v74 +;; @003e jump block10(v75) +;; +;; block10(v57: i64): +;; @003e store.i128 notrap aligned v56, v57 +;; v114 = iconst.i64 0 +;; @003e v76 = iadd.i64 v15, v114 ; v114 = 0 +;; @003e v77 = iconst.i32 1 +;; v113 = iconst.i64 16 +;; @003e v78 = iadd v76, v113 ; v113 = 16 +;; @003e store notrap aligned v77, v78 ; v77 = 1 +;; @003e v79 = load.i64 notrap aligned v15+64 +;; @003e store.i64 notrap aligned v29, v79+48 +;; @003e store.i64 notrap aligned v30, v79+56 +;; @003e v80 = iconst.i64 2 +;; @003e v81 = load.i64 notrap aligned v0+8 +;; @003e store notrap aligned v80, v81+64 ; v80 = 2 +;; @003e store.i64 notrap aligned v15, v81+72 +;; v112 = iconst.i64 0 +;; @003e v82 = iadd v76, v112 ; v112 = 0 +;; @003e v83 = load.i64 notrap aligned v82 +;; @003e store notrap aligned v83, v49+16 +;; @003e v84 = load.i64 notrap aligned v82+8 +;; @003e store notrap aligned v84, v49+56 +;; v111 = iconst.i64 80 +;; @003e v85 = iadd.i64 v27, v111 ; v111 = 80 +;; @003e v86 = load.i64 notrap aligned v85 +;; v110 = iconst.i64 -24 +;; @003e v87 = iadd v86, v110 ; v110 = -24 +;; v109 = iconst.i64 80 +;; @003e v88 = iadd v79, v109 ; v109 = 80 +;; @003e v89 = load.i64 notrap aligned v88 +;; v108 = iconst.i64 -24 +;; @003e v90 = iadd v89, v108 ; v108 = -24 +;; @003e v91 = stack_addr.i64 ss0 +;; @003e v92 = load.i64 notrap aligned v90 +;; @003e store notrap aligned v92, v91 +;; @003e v93 = load.i64 notrap aligned v87 +;; @003e store notrap aligned v93, v90 +;; @003e v94 = load.i64 notrap aligned v90+8 +;; @003e store notrap aligned v94, v91+8 +;; @003e v95 = load.i64 notrap aligned v87+8 +;; @003e store notrap aligned v95, v90+8 +;; @003e v96 = load.i64 notrap aligned v90+16 +;; @003e store notrap aligned v96, v91+16 +;; @003e v97 = load.i64 notrap aligned v87+16 +;; @003e store notrap aligned v97, v90+16 +;; @003e v98 = iconst.i64 3 +;; v107 = iconst.i64 32 +;; @003e v99 = ishl v98, v107 ; v98 = 3, v107 = 32 +;; @003e v100 = stack_switch v87, v91, v99 +;; v106 = iconst.i64 120 +;; @003e v101 = iadd.i64 v25, v106 ; v106 = 120 +;; @003e v102 = load.i64 notrap aligned v101+8 +;; @003e v103 = iconst.i32 0 +;; @003e store notrap aligned v103, v101 ; v103 = 0 +;; @003e v104 = iconst.i32 0 +;; @003e store notrap aligned v104, v101+4 ; v104 = 0 +;; @003e v105 = iconst.i64 0 +;; @003e store notrap aligned v105, v101+8 ; v105 = 0 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return +;; } +;; +;; function u0:1(i64 vmctx, i64, i128) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i128): +;; @0044 jump block1 +;; +;; block1: +;; @0044 return +;; } +;; +;; function u0:2(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 8, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1610612736:7 sig0 +;; fn1 = colocated u1610612736:52 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0047 v2 = iconst.i32 0 +;; @0047 v4 = call fn0(v0, v2) ; v2 = 0 +;; @0049 trapz v4, user16 +;; @0049 v5 = iconst.i32 0 +;; @0049 v6 = iconst.i32 0 +;; @0049 v8 = call fn1(v0, v4, v5, v6) ; v5 = 0, v6 = 0 +;; @0049 v9 = load.i64 notrap aligned v8+72 +;; @0049 v10 = uextend.i128 v8 +;; @0049 v11 = uextend.i128 v9 +;; v111 = iconst.i64 64 +;; v112 = uextend.i128 v111 ; v111 = 64 +;; @0049 v12 = ishl v11, v112 +;; @0049 v13 = bor v12, v10 +;; @004b jump block2 +;; +;; block2: +;; @004b v15 = ireduce.i64 v13 +;; v109 = iconst.i64 64 +;; v110 = uextend.i128 v109 ; v109 = 64 +;; @004b v16 = ushr.i128 v13, v110 +;; @004b v17 = ireduce.i64 v16 +;; @004b trapz v15, user16 +;; @004b v18 = load.i64 notrap aligned v15+72 +;; @004b v19 = icmp eq v18, v17 +;; @004b trapz v19, user23 +;; v108 = iconst.i64 1 +;; @004b v20 = iadd v18, v108 ; v108 = 1 +;; @004b store notrap aligned v20, v15+72 +;; @004b v21 = load.i64 notrap aligned v15+64 +;; @004b v22 = load.i64 notrap aligned v0+8 +;; @004b v23 = load.i64 notrap aligned v22+64 +;; @004b v24 = load.i64 notrap aligned v22+72 +;; @004b store notrap aligned v23, v21+48 +;; @004b store notrap aligned v24, v21+56 +;; @004b v25 = iconst.i64 0 +;; @004b store notrap aligned v25, v15+64 ; v25 = 0 +;; @004b v26 = iconst.i64 2 +;; @004b v27 = load.i64 notrap aligned v0+8 +;; @004b store notrap aligned v26, v27+64 ; v26 = 2 +;; @004b store notrap aligned v15, v27+72 +;; v107 = iconst.i64 0 +;; @004b v28 = iadd v15, v107 ; v107 = 0 +;; @004b v29 = iconst.i32 1 +;; v106 = iconst.i64 16 +;; @004b v30 = iadd v28, v106 ; v106 = 16 +;; @004b store notrap aligned v29, v30 ; v29 = 1 +;; @004b v31 = iconst.i32 2 +;; v105 = iconst.i64 16 +;; @004b v32 = iadd v24, v105 ; v105 = 16 +;; @004b store notrap aligned v31, v32 ; v31 = 2 +;; @004b v33 = load.i64 notrap aligned readonly v0+8 +;; v104 = iconst.i64 0 +;; @004b v34 = iadd v24, v104 ; v104 = 0 +;; @004b v35 = load.i64 notrap aligned v33+56 +;; @004b store notrap aligned v35, v34+8 +;; @004b v36 = load.i64 notrap aligned v33+16 +;; @004b store notrap aligned v36, v34 +;; v103 = iconst.i64 0 +;; @004b v37 = iadd v28, v103 ; v103 = 0 +;; @004b v38 = load.i64 notrap aligned v37 +;; @004b store notrap aligned v38, v33+16 +;; @004b v39 = load.i64 notrap aligned v37+8 +;; @004b store notrap aligned v39, v33+56 +;; v102 = iconst.i64 24 +;; @004b v40 = iadd v24, v102 ; v102 = 24 +;; @004b v41 = iconst.i32 1 +;; @004b v42 = stack_addr.i64 ss0 +;; @004b store notrap aligned v41, v40+4 ; v41 = 1 +;; @004b store notrap aligned v42, v40+8 +;; v101 = iconst.i64 48 +;; @004b v44 = iadd.i64 v0, v101 ; v101 = 48 +;; @004b v45 = iconst.i32 1 +;; @004b v46 = load.i64 notrap aligned v40+8 +;; @004b store notrap aligned v44, v46 +;; @004b store notrap aligned v45, v40 ; v45 = 1 +;; @004b v47 = iconst.i32 0 +;; @004b store notrap aligned v47, v24+40 ; v47 = 0 +;; @004b v48 = iconst.i64 1 +;; v100 = iconst.i64 32 +;; @004b v49 = ishl v48, v100 ; v48 = 1, v100 = 32 +;; v99 = iconst.i64 80 +;; @004b v50 = iadd v21, v99 ; v99 = 80 +;; @004b v51 = load.i64 notrap aligned v50 +;; v98 = iconst.i64 -24 +;; @004b v52 = iadd v51, v98 ; v98 = -24 +;; @004b v53 = stack_switch v52, v52, v49 +;; @004b v54 = load.i64 notrap aligned v0+8 +;; @004b v55 = load.i64 notrap aligned v54+64 +;; @004b v56 = load.i64 notrap aligned v54+72 +;; @004b v57 = load.i64 notrap aligned v0+8 +;; @004b store notrap aligned v23, v57+64 +;; @004b store notrap aligned v24, v57+72 +;; @004b v58 = iconst.i32 1 +;; v97 = iconst.i64 16 +;; @004b v59 = iadd v24, v97 ; v97 = 16 +;; @004b store notrap aligned v58, v59 ; v58 = 1 +;; @004b v60 = iconst.i32 0 +;; @004b store notrap aligned v60, v40 ; v60 = 0 +;; @004b v61 = iconst.i32 0 +;; @004b store notrap aligned v61, v40+4 ; v61 = 0 +;; @004b v62 = iconst.i64 0 +;; @004b store notrap aligned v62, v40+8 ; v62 = 0 +;; @004b store notrap aligned v25, v24+40 ; v25 = 0 +;; v96 = iconst.i64 32 +;; @004b v63 = ushr v53, v96 ; v96 = 32 +;; @004b brif v63, block4, block3 +;; +;; block4: +;; v95 = iconst.i64 0 +;; @004b v64 = iadd.i64 v56, v95 ; v95 = 0 +;; v94 = iconst.i64 0 +;; @004b v65 = iadd v64, v94 ; v94 = 0 +;; @004b v66 = load.i64 notrap aligned v33+56 +;; @004b store notrap aligned v66, v65+8 +;; v93 = iconst.i64 0 +;; @004b v67 = iadd.i64 v24, v93 ; v93 = 0 +;; @004b v68 = load.i64 notrap aligned v67 +;; @004b store notrap aligned v68, v33+16 +;; @004b v69 = load.i64 notrap aligned v67+8 +;; @004b store notrap aligned v69, v33+56 +;; @004b v70 = ireduce.i32 v53 +;; @004b v71 = load.i64 notrap aligned v56+72 +;; @004b v72 = uextend.i128 v56 +;; @004b v73 = uextend.i128 v71 +;; v91 = iconst.i64 64 +;; v92 = uextend.i128 v91 ; v91 = 64 +;; @004b v74 = ishl v73, v92 +;; @004b v75 = bor v74, v72 +;; @004b jump block5 +;; +;; block6 cold: +;; @004b trap user11 +;; +;; block5: +;; @004b br_table v70, block6, [] +;; +;; block3: +;; v90 = iconst.i64 0 +;; @004b v76 = iadd.i64 v24, v90 ; v90 = 0 +;; @004b v77 = load.i64 notrap aligned v76 +;; @004b store notrap aligned v77, v33+16 +;; @004b v78 = load.i64 notrap aligned v76+8 +;; @004b store notrap aligned v78, v33+56 +;; v89 = iconst.i64 0 +;; @004b v79 = iadd.i64 v56, v89 ; v89 = 0 +;; @004b v80 = iconst.i32 4 +;; v88 = iconst.i64 16 +;; @004b v81 = iadd v79, v88 ; v88 = 16 +;; @004b store notrap aligned v80, v81 ; v80 = 4 +;; v87 = iconst.i64 104 +;; @004b v82 = iadd.i64 v56, v87 ; v87 = 104 +;; @004b v83 = load.i64 notrap aligned v82+8 +;; @004b v84 = iconst.i32 0 +;; @004b store notrap aligned v84, v82 ; v84 = 0 +;; @004b v85 = iconst.i32 0 +;; @004b store notrap aligned v85, v82+4 ; v85 = 0 +;; @004b v86 = iconst.i64 0 +;; @004b store notrap aligned v86, v82+8 ; v86 = 0 +;; @0050 jump block1 +;; +;; block1: +;; @0050 return +;; }