From 41d7ba2d2eb396cc8d7c92a3fab324840dbbe4fc Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Tue, 4 Mar 2025 11:30:25 -0700 Subject: [PATCH 1/2] add component-model-async/{fused|futures|streams}.wast tests This is another piece of #9582 which I'm splitting out to make review easier. The fused.wast test exercises fused adapter generation for various flavors of intercomponent async->async, async->sync, and sync->async calls. The futures.wast and streams.wast tests exercise the various intrinsics (e.g. `stream.read`, `future.close_writable`, etc.) involving `future`s and `stream`s. The remaining changes fill in some TODOs to make the tests pass, plus plumbing for a few intrinsics which aren't needed for these tests but which set the foundation for future tests. Signed-off-by: Joel Dice --- crates/cranelift/src/compiler/component.rs | 816 +++++++++++++++--- crates/environ/src/component.rs | 61 ++ crates/environ/src/component/dfg.rs | 48 +- crates/environ/src/component/info.rs | 40 +- crates/environ/src/component/translate.rs | 17 +- .../environ/src/component/translate/adapt.rs | 6 + .../environ/src/component/translate/inline.rs | 53 +- crates/environ/src/component/types.rs | 21 +- crates/environ/src/component/types_builder.rs | 42 +- crates/environ/src/fact.rs | 247 +++++- crates/environ/src/fact/signature.rs | 174 ++++ crates/environ/src/fact/trampoline.rs | 660 ++++++++++++-- .../fuzzing/src/generators/component_types.rs | 29 +- .../src/runtime/component/concurrent.rs | 781 ++++++++++++++++- .../src/runtime/component/func/host.rs | 2 + crates/wasmtime/src/runtime/component/mod.rs | 5 +- crates/wasmtime/src/runtime/store.rs | 7 + crates/wasmtime/src/runtime/vm.rs | 7 +- crates/wasmtime/src/runtime/vm/component.rs | 34 + .../src/runtime/vm/component/libcalls.rs | 649 +++++++++++++- crates/wasmtime/src/runtime/vm/interpreter.rs | 4 +- .../component-model-async/fused.wast | 247 ++++++ .../component-model-async/futures.wast | 90 ++ .../component-model-async/streams.wast | 90 ++ 24 files changed, 3818 insertions(+), 312 deletions(-) create mode 100644 tests/misc_testsuite/component-model-async/fused.wast create mode 100644 tests/misc_testsuite/component-model-async/futures.wast create mode 100644 tests/misc_testsuite/component-model-async/streams.wast diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index a88d26d22c4c..ae312558008f 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -3,12 +3,14 @@ use crate::{compiler::Compiler, TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT}; use anyhow::Result; use cranelift_codegen::ir::condcodes::IntCC; -use cranelift_codegen::ir::{self, InstBuilder, MemFlags}; +use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value}; use cranelift_codegen::isa::{CallConv, TargetIsa}; use cranelift_frontend::FunctionBuilder; use std::any::Any; use wasmtime_environ::component::*; -use wasmtime_environ::{HostCall, ModuleInternedTypeIndex, PtrSize, Tunables, WasmValType}; +use wasmtime_environ::{ + HostCall, ModuleInternedTypeIndex, PtrSize, Tunables, WasmFuncType, WasmValType, +}; struct TrampolineCompiler<'a> { compiler: &'a Compiler, @@ -94,166 +96,230 @@ impl<'a> TrampolineCompiler<'a> { Trampoline::AlwaysTrap => { self.translate_always_trap(); } + Trampoline::ResourceNew(ty) => self.translate_resource_new(*ty), + Trampoline::ResourceRep(ty) => self.translate_resource_rep(*ty), + Trampoline::ResourceDrop(ty) => self.translate_resource_drop(*ty), Trampoline::TaskBackpressure { instance } => { - _ = instance; - todo!() + self.translate_task_backpressure_call(*instance) } - Trampoline::TaskReturn => todo!(), + Trampoline::TaskReturn { results } => self.translate_task_return_call(*results), Trampoline::TaskWait { instance, async_, memory, } => { - _ = (instance, async_, memory); - todo!() + self.translate_task_wait_or_poll_call(*instance, *async_, *memory, host::task_wait) } Trampoline::TaskPoll { instance, async_, memory, } => { - _ = (instance, async_, memory); - todo!() - } - Trampoline::TaskYield { async_ } => { - _ = async_; - todo!() - } - Trampoline::SubtaskDrop { instance } => { - _ = instance; - todo!() - } - Trampoline::StreamNew { ty } => { - _ = ty; - todo!() + self.translate_task_wait_or_poll_call(*instance, *async_, *memory, host::task_poll) } - Trampoline::StreamRead { ty, options } => { - _ = (ty, options); - todo!() + Trampoline::TaskYield { async_ } => self.translate_task_yield_call(*async_), + Trampoline::SubtaskDrop { instance } => self.translate_subtask_drop_call(*instance), + Trampoline::StreamNew { ty } => self.translate_future_or_stream_call( + &[ty.as_u32()], + None, + host::stream_new, + ir::types::I64, + ), + Trampoline::StreamRead { + ty, + err_ctx_ty, + options, + } => { + let tys = &[ty.as_u32(), err_ctx_ty.as_u32()]; + if let Some(info) = self.flat_stream_element_info(*ty) { + self.translate_flat_stream_call(tys, options, host::flat_stream_read, &info) + } else { + self.translate_future_or_stream_call( + tys, + Some(options), + host::stream_read, + ir::types::I64, + ) + } } Trampoline::StreamWrite { ty, options } => { - _ = (ty, options); - todo!() + let tys = &[ty.as_u32()]; + if let Some(info) = self.flat_stream_element_info(*ty) { + self.translate_flat_stream_call(tys, options, host::flat_stream_write, &info) + } else { + self.translate_future_or_stream_call( + tys, + Some(options), + host::stream_write, + ir::types::I64, + ) + } } Trampoline::StreamCancelRead { ty, async_ } => { - _ = (ty, async_); - todo!() + self.translate_cancel_call(ty.as_u32(), *async_, host::stream_cancel_read) } Trampoline::StreamCancelWrite { ty, async_ } => { - _ = (ty, async_); - todo!() - } - Trampoline::StreamCloseReadable { ty } => { - _ = ty; - todo!() - } - Trampoline::StreamCloseWritable { ty } => { - _ = ty; - todo!() - } - Trampoline::FutureNew { ty } => { - _ = ty; - todo!() - } - Trampoline::FutureRead { ty, options } => { - _ = (ty, options); - todo!() - } - Trampoline::FutureWrite { ty, options } => { - _ = (ty, options); - todo!() + self.translate_cancel_call(ty.as_u32(), *async_, host::stream_cancel_write) } + Trampoline::StreamCloseReadable { ty } => self.translate_future_or_stream_call( + &[ty.as_u32()], + None, + host::stream_close_readable, + ir::types::I8, + ), + Trampoline::StreamCloseWritable { ty, err_ctx_ty } => self + .translate_future_or_stream_call( + &[ty.as_u32(), err_ctx_ty.as_u32()], + None, + host::stream_close_writable, + ir::types::I8, + ), + Trampoline::FutureNew { ty } => self.translate_future_or_stream_call( + &[ty.as_u32()], + None, + host::future_new, + ir::types::I64, + ), + Trampoline::FutureRead { + ty, + err_ctx_ty, + options, + } => self.translate_future_or_stream_call( + &[ty.as_u32(), err_ctx_ty.as_u32()], + Some(&options), + host::future_read, + ir::types::I64, + ), + Trampoline::FutureWrite { ty, options } => self.translate_future_or_stream_call( + &[ty.as_u32()], + Some(options), + host::future_write, + ir::types::I64, + ), Trampoline::FutureCancelRead { ty, async_ } => { - _ = (ty, async_); - todo!() + self.translate_cancel_call(ty.as_u32(), *async_, host::future_cancel_read) } Trampoline::FutureCancelWrite { ty, async_ } => { - _ = (ty, async_); - todo!() - } - Trampoline::FutureCloseReadable { ty } => { - _ = ty; - todo!() - } - Trampoline::FutureCloseWritable { ty } => { - _ = ty; - todo!() - } - Trampoline::ErrorContextNew { ty, options } => { - _ = (ty, options); - todo!() + self.translate_cancel_call(ty.as_u32(), *async_, host::future_cancel_write) } - Trampoline::ErrorContextDebugMessage { ty, options } => { - _ = (ty, options); - todo!() - } - Trampoline::ErrorContextDrop { ty } => { - _ = ty; - todo!() - } - Trampoline::ResourceNew(ty) => self.translate_resource_new(*ty), - Trampoline::ResourceRep(ty) => self.translate_resource_rep(*ty), - Trampoline::ResourceDrop(ty) => self.translate_resource_drop(*ty), + Trampoline::FutureCloseReadable { ty } => self.translate_future_or_stream_call( + &[ty.as_u32()], + None, + host::future_close_readable, + ir::types::I8, + ), + Trampoline::FutureCloseWritable { ty, err_ctx_ty } => self + .translate_future_or_stream_call( + &[ty.as_u32(), err_ctx_ty.as_u32()], + None, + host::future_close_writable, + ir::types::I8, + ), + Trampoline::ErrorContextNew { ty, options } => self.translate_error_context_call( + *ty, + options, + host::error_context_new, + ir::types::I64, + ), + Trampoline::ErrorContextDebugMessage { ty, options } => self + .translate_error_context_call( + *ty, + options, + host::error_context_debug_message, + ir::types::I8, + ), + Trampoline::ErrorContextDrop { ty } => self.translate_error_context_drop_call(*ty), Trampoline::ResourceTransferOwn => { - self.translate_resource_libcall(host::resource_transfer_own, |me, rets| { - rets[0] = me.raise_if_resource_trapped(rets[0]); + self.translate_host_libcall(host::resource_transfer_own, |me, rets| { + rets[0] = me.raise_if_negative_one(rets[0]); }) } Trampoline::ResourceTransferBorrow => { - self.translate_resource_libcall(host::resource_transfer_borrow, |me, rets| { - rets[0] = me.raise_if_resource_trapped(rets[0]); + self.translate_host_libcall(host::resource_transfer_borrow, |me, rets| { + rets[0] = me.raise_if_negative_one(rets[0]); }) } Trampoline::ResourceEnterCall => { - self.translate_resource_libcall(host::resource_enter_call, |_, _| {}) + self.translate_host_libcall(host::resource_enter_call, |_, _| {}) } Trampoline::ResourceExitCall => { - self.translate_resource_libcall(host::resource_exit_call, |me, rets| { + self.translate_host_libcall(host::resource_exit_call, |me, rets| { me.raise_if_host_trapped(rets.pop().unwrap()); }) } - Trampoline::AsyncEnterCall => todo!(), + Trampoline::SyncEnterCall => self.translate_sync_enter(), + Trampoline::SyncExitCall { callback } => self.translate_sync_exit(*callback), + Trampoline::AsyncEnterCall => { + self.translate_async_enter_or_exit(host::async_enter, None, ir::types::I8) + } Trampoline::AsyncExitCall { callback, post_return, - } => { - _ = (callback, post_return); - todo!() - } + } => self.translate_async_enter_or_exit( + host::async_exit, + Some((*callback, *post_return)), + ir::types::I64, + ), Trampoline::FutureTransfer => { - _ = host::future_transfer; - todo!() + self.translate_host_libcall(host::future_transfer, |me, rets| { + rets[0] = me.raise_if_negative_one(rets[0]); + }) } Trampoline::StreamTransfer => { - _ = host::stream_transfer; - todo!() + self.translate_host_libcall(host::stream_transfer, |me, rets| { + rets[0] = me.raise_if_negative_one(rets[0]); + }) } Trampoline::ErrorContextTransfer => { - _ = host::error_context_transfer; - todo!() + self.translate_host_libcall(host::error_context_transfer, |me, rets| { + rets[0] = me.raise_if_negative_one(rets[0]); + }) } } } - fn translate_lower_import( - &mut self, - index: LoweredIndex, - options: &CanonicalOptions, - lower_ty: TypeFuncIndex, - ) { + fn flat_stream_element_info(&self, ty: TypeStreamTableIndex) -> Option { + let payload = self.types[self.types[ty].ty].payload; + match payload { + None => Some(CanonicalAbiInfo { + align32: 1, + align64: 1, + flat_count: None, + size32: 0, + size64: 0, + }), + Some( + payload @ (InterfaceType::Bool + | InterfaceType::S8 + | InterfaceType::U8 + | InterfaceType::S16 + | InterfaceType::U16 + | InterfaceType::S32 + | InterfaceType::U32 + | InterfaceType::S64 + | InterfaceType::U64 + | InterfaceType::Float32 + | InterfaceType::Float64 + | InterfaceType::Char), + ) => Some(self.types.canonical_abi(&payload).clone()), + // TODO: Recursively check for other "flat" types (i.e. those without pointers or handles), + // e.g. `record`s, `variant`s, etc. which contain only flat types. + _ => None, + } + } + + fn store_wasm_arguments(&mut self, args: &[Value]) -> (Value, Value) { let pointer_type = self.isa.pointer_type(); - let args = self.builder.func.dfg.block_params(self.block0).to_vec(); - let vmctx = args[0]; - let wasm_func_ty = self.types[self.signature].unwrap_func(); + let wasm_func_ty = &self.types[self.signature].unwrap_func(); // Start off by spilling all the wasm arguments into a stack slot to be // passed to the host function. - let (values_vec_ptr, values_vec_len) = match self.abi { + match self.abi { Abi::Wasm => { let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args( wasm_func_ty, &mut self.builder, - &args[2..], + args, ); let len = self.builder.ins().iconst(pointer_type, i64::from(len)); (ptr, len) @@ -262,7 +328,337 @@ impl<'a> TrampolineCompiler<'a> { let params = self.builder.func.dfg.block_params(self.block0); (params[2], params[3]) } - }; + } + } + + fn translate_intrinsic_libcall( + &mut self, + vmctx: ir::Value, + get_libcall: fn( + &dyn TargetIsa, + &mut ir::Function, + ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + args: &[ir::Value], + result: ir::types::Type, + ) { + match self.abi { + Abi::Wasm => {} + + Abi::Array => { + // TODO: A guest could hypothetically export the same intrinsic + // it imported, allowing the host to call it directly. We need + // to support that here (except for `sync-enter`, `sync-exit`, + // `async-enter`, and `async-exit`, which are only ever called + // from FACT-generated Wasm code and never exported). + self.builder.ins().trap(TRAP_INTERNAL_ASSERT); + return; + } + } + + let call = self.call_libcall(vmctx, get_libcall, args); + + if result == ir::types::I64 { + let result = self.builder.func.dfg.inst_results(call)[0]; + let result = self.raise_if_negative_one(result); + self.abi_store_results(&[result]); + } else { + if result != ir::types::I8 { + todo!("support additional intrinsic return types") + } + let succeeded = self.builder.func.dfg.inst_results(call)[0]; + self.raise_if_host_trapped(succeeded); + self.builder.ins().return_(&[]); + } + } + + fn translate_task_return_call(&mut self, results: TypeTupleIndex) { + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + + let (values_vec_ptr, values_vec_len) = self.store_wasm_arguments(&args[2..]); + + let ty = self + .builder + .ins() + .iconst(ir::types::I32, i64::from(results.as_u32())); + + self.translate_intrinsic_libcall( + vmctx, + host::task_return, + &[vmctx, ty, values_vec_ptr, values_vec_len], + ir::types::I8, + ); + } + + fn translate_sync_enter(&mut self) { + match self.abi { + Abi::Wasm => {} + + Abi::Array => { + // This code can only be called from (FACT-generated) Wasm, so + // we don't need to support the array ABI. + self.builder.ins().trap(TRAP_INTERNAL_ASSERT); + return; + } + } + + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + + let pointer_type = self.isa.pointer_type(); + let wasm_func_ty = &self.types[self.signature].unwrap_func(); + + let param_offset = 5; + let spill_offset = param_offset + 2; + + let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args( + &WasmFuncType::new( + wasm_func_ty + .params() + .iter() + .skip(param_offset) + .copied() + .collect(), + Box::new([]), + ), + &mut self.builder, + &args[spill_offset..], + ); + let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len)); + + let mut callee_args = vec![vmctx]; + + // remaining non-Wasm parameters + callee_args.extend(args[2..spill_offset].iter().copied()); + + callee_args.push(values_vec_ptr); + callee_args.push(values_vec_len); + + self.translate_intrinsic_libcall(vmctx, host::sync_enter, &callee_args, ir::types::I8); + } + + fn translate_sync_exit(&mut self, callback: Option) { + match self.abi { + Abi::Wasm => {} + + Abi::Array => { + // This code can only be called from (FACT-generated) Wasm, so + // we don't need to support the array ABI. + self.builder.ins().trap(TRAP_INTERNAL_ASSERT); + return; + } + } + + let pointer_type = self.isa.pointer_type(); + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + let wasm_func_ty = &self.types[self.signature].unwrap_func(); + + let mut callee_args = vec![ + vmctx, + if let Some(callback) = callback { + self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_callback(callback)).unwrap(), + ) + } else { + self.builder.ins().iconst(pointer_type, 0) + }, + ]; + + // remaining non-Wasm parameters + callee_args.extend(args[2..].iter().copied()); + + let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args( + &WasmFuncType::new( + Box::new([]), + wasm_func_ty.returns().iter().copied().collect(), + ), + &mut self.builder, + &[], + ); + let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len)); + + callee_args.push(values_vec_ptr); + callee_args.push(values_vec_len); + + let call = self.call_libcall(vmctx, host::sync_exit, &callee_args); + + let succeeded = self.builder.func.dfg.inst_results(call)[0]; + self.raise_if_host_trapped(succeeded); + // After the host function has returned the results are loaded from + // `values_vec_ptr` and then returned. + let results = self.compiler.load_values_from_array( + wasm_func_ty.returns(), + &mut self.builder, + values_vec_ptr, + values_vec_len, + ); + self.builder.ins().return_(&results); + } + + fn translate_async_enter_or_exit( + &mut self, + get_libcall: fn( + &dyn TargetIsa, + &mut ir::Function, + ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + callback_and_post_return: Option<( + Option, + Option, + )>, + result: ir::types::Type, + ) { + match self.abi { + Abi::Wasm => {} + + Abi::Array => { + // This code can only be called from (FACT-generated) Wasm, so + // we don't need to support the array ABI. + self.builder.ins().trap(TRAP_INTERNAL_ASSERT); + return; + } + } + + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + + let mut callee_args = vec![vmctx]; + + if let Some((callback, post_return)) = callback_and_post_return { + let pointer_type = self.isa.pointer_type(); + + // callback: *mut VMFuncRef + if let Some(callback) = callback { + callee_args.push(self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_callback(callback)).unwrap(), + )); + } else { + callee_args.push(self.builder.ins().iconst(pointer_type, 0)); + } + + // post_return: *mut VMFuncRef + if let Some(post_return) = post_return { + callee_args.push(self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_post_return(post_return)).unwrap(), + )); + } else { + callee_args.push(self.builder.ins().iconst(pointer_type, 0)); + } + } + + // remaining parameters + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, result); + } + + fn translate_task_backpressure_call(&mut self, caller_instance: RuntimeComponentInstanceIndex) { + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + + let mut callee_args = vec![ + vmctx, + self.builder + .ins() + .iconst(ir::types::I32, i64::from(caller_instance.as_u32())), + ]; + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall( + vmctx, + host::task_backpressure, + &callee_args, + ir::types::I8, + ); + } + + fn translate_task_wait_or_poll_call( + &mut self, + caller_instance: RuntimeComponentInstanceIndex, + async_: bool, + memory: RuntimeMemoryIndex, + get_libcall: fn( + &dyn TargetIsa, + &mut ir::Function, + ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + ) { + let pointer_type = self.isa.pointer_type(); + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + + let mut callee_args = vec![ + vmctx, + self.builder + .ins() + .iconst(ir::types::I32, i64::from(caller_instance.as_u32())), + self.builder + .ins() + .iconst(ir::types::I8, if async_ { 1 } else { 0 }), + self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_memory(memory)).unwrap(), + ), + ]; + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, ir::types::I64); + } + + fn translate_task_yield_call(&mut self, async_: bool) { + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + + let callee_args = [ + vmctx, + self.builder + .ins() + .iconst(ir::types::I8, if async_ { 1 } else { 0 }), + ]; + + self.translate_intrinsic_libcall(vmctx, host::task_yield, &callee_args, ir::types::I8); + } + + fn translate_subtask_drop_call(&mut self, caller_instance: RuntimeComponentInstanceIndex) { + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + + let mut callee_args = vec![ + vmctx, + self.builder + .ins() + .iconst(ir::types::I32, i64::from(caller_instance.as_u32())), + ]; + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall(vmctx, host::subtask_drop, &callee_args, ir::types::I8); + } + + fn translate_lower_import( + &mut self, + index: LoweredIndex, + options: &CanonicalOptions, + lower_ty: TypeFuncIndex, + ) { + let pointer_type = self.isa.pointer_type(); + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + let wasm_func_ty = self.types[self.signature].unwrap_func(); + + let (values_vec_ptr, values_vec_len) = self.store_wasm_arguments(&args[2..]); // Below this will incrementally build both the signature of the host // function we're calling as well as the list of arguments since the @@ -274,12 +670,14 @@ impl<'a> TrampolineCompiler<'a> { instance, memory, realloc, + callback, post_return, string_encoding, - callback: _, async_, } = *options; + assert!(callback.is_none()); + // vmctx: *mut VMComponentContext host_sig.params.push(ir::AbiParam::new(pointer_type)); callee_args.push(vmctx); @@ -301,6 +699,14 @@ impl<'a> TrampolineCompiler<'a> { .iconst(ir::types::I32, i64::from(lower_ty.as_u32())), ); + // caller_instance: RuntimeComponentInstanceIndex + host_sig.params.push(ir::AbiParam::new(ir::types::I32)); + callee_args.push( + self.builder + .ins() + .iconst(ir::types::I32, i64::from(instance.as_u32())), + ); + // flags: *mut VMGlobalDefinition host_sig.params.push(ir::AbiParam::new(pointer_type)); callee_args.push( @@ -457,7 +863,7 @@ impl<'a> TrampolineCompiler<'a> { ); let call = self.call_libcall(vmctx, host::resource_new32, &host_args); let result = self.builder.func.dfg.inst_results(call)[0]; - let result = self.raise_if_resource_trapped(result); + let result = self.raise_if_negative_one(result); self.abi_store_results(&[result]); } @@ -486,7 +892,7 @@ impl<'a> TrampolineCompiler<'a> { ); let call = self.call_libcall(vmctx, host::resource_rep32, &host_args); let result = self.builder.func.dfg.inst_results(call)[0]; - let result = self.raise_if_resource_trapped(result); + let result = self.raise_if_negative_one(result); self.abi_store_results(&[result]); } @@ -677,7 +1083,7 @@ impl<'a> TrampolineCompiler<'a> { /// /// Only intended for simple trampolines and effectively acts as a bridge /// from the wasm abi to host. - fn translate_resource_libcall( + fn translate_host_libcall( &mut self, get_libcall: fn( &dyn TargetIsa, @@ -707,6 +1113,194 @@ impl<'a> TrampolineCompiler<'a> { self.builder.ins().return_(&results); } + fn translate_cancel_call( + &mut self, + ty: u32, + async_: bool, + get_libcall: fn( + &dyn TargetIsa, + &mut ir::Function, + ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + ) { + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + let mut callee_args = vec![ + vmctx, + self.builder.ins().iconst(ir::types::I32, i64::from(ty)), + self.builder + .ins() + .iconst(ir::types::I8, if async_ { 1 } else { 0 }), + ]; + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, ir::types::I64); + } + + fn translate_future_or_stream_call( + &mut self, + tys: &[u32], + options: Option<&CanonicalOptions>, + get_libcall: fn( + &dyn TargetIsa, + &mut ir::Function, + ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + result: ir::types::Type, + ) { + let pointer_type = self.isa.pointer_type(); + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + let mut callee_args = vec![vmctx]; + + if let Some(options) = options { + // memory: *mut VMMemoryDefinition + callee_args.push(self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_memory(options.memory.unwrap())).unwrap(), + )); + + // realloc: *mut VMFuncRef + callee_args.push(match options.realloc { + Some(idx) => self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), + ), + None => self.builder.ins().iconst(pointer_type, 0), + }); + + // string_encoding: StringEncoding + callee_args.push( + self.builder + .ins() + .iconst(ir::types::I8, i64::from(options.string_encoding as u8)), + ); + } + + for ty in tys { + callee_args.push(self.builder.ins().iconst(ir::types::I32, i64::from(*ty))); + } + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, result); + } + + fn translate_flat_stream_call( + &mut self, + tys: &[u32], + options: &CanonicalOptions, + get_libcall: fn( + &dyn TargetIsa, + &mut ir::Function, + ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + info: &CanonicalAbiInfo, + ) { + let pointer_type = self.isa.pointer_type(); + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + let mut callee_args = vec![ + vmctx, + self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_memory(options.memory.unwrap())).unwrap(), + ), + match options.realloc { + Some(idx) => self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), + ), + None => self.builder.ins().iconst(pointer_type, 0), + }, + ]; + for ty in tys { + callee_args.push(self.builder.ins().iconst(ir::types::I32, i64::from(*ty))); + } + + callee_args.extend([ + self.builder + .ins() + .iconst(ir::types::I32, i64::from(info.size32)), + self.builder + .ins() + .iconst(ir::types::I32, i64::from(info.align32)), + ]); + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, ir::types::I64); + } + + fn translate_error_context_call( + &mut self, + ty: TypeComponentLocalErrorContextTableIndex, + options: &CanonicalOptions, + get_libcall: fn( + &dyn TargetIsa, + &mut ir::Function, + ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + result: ir::types::Type, + ) { + let pointer_type = self.isa.pointer_type(); + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + let mut callee_args = vec![ + vmctx, + self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_memory(options.memory.unwrap())).unwrap(), + ), + match options.realloc { + Some(idx) => self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), + ), + None => self.builder.ins().iconst(pointer_type, 0), + }, + self.builder + .ins() + .iconst(ir::types::I8, i64::from(options.string_encoding as u8)), + self.builder + .ins() + .iconst(ir::types::I32, i64::from(ty.as_u32())), + ]; + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, result); + } + + fn translate_error_context_drop_call(&mut self, ty: TypeComponentLocalErrorContextTableIndex) { + let args = self.builder.func.dfg.block_params(self.block0).to_vec(); + let vmctx = args[0]; + let mut callee_args = vec![ + vmctx, + self.builder + .ins() + .iconst(ir::types::I32, i64::from(ty.as_u32())), + ]; + + callee_args.extend(args[2..].iter().copied()); + + self.translate_intrinsic_libcall( + vmctx, + host::error_context_drop, + &callee_args, + ir::types::I8, + ); + } + /// Loads a host function pointer for a libcall stored at the `offset` /// provided in the libcalls array. /// @@ -799,7 +1393,7 @@ impl<'a> TrampolineCompiler<'a> { self.raise_if_host_trapped(succeeded); } - fn raise_if_resource_trapped(&mut self, ret: ir::Value) -> ir::Value { + fn raise_if_negative_one(&mut self, ret: ir::Value) -> ir::Value { let minus_one = self.builder.ins().iconst(ir::types::I64, -1); let succeeded = self.builder.ins().icmp(IntCC::NotEqual, ret, minus_one); self.raise_if_host_trapped(succeeded); diff --git a/crates/environ/src/component.rs b/crates/environ/src/component.rs index 6988e4db7431..f19d2a69fdf0 100644 --- a/crates/environ/src/component.rs +++ b/crates/environ/src/component.rs @@ -83,8 +83,69 @@ macro_rules! foreach_builtin_component_function { resource_enter_call(vmctx: vmctx); resource_exit_call(vmctx: vmctx) -> bool; + #[cfg(feature = "component-model-async")] + task_backpressure(vmctx: vmctx, caller_instance: u32, enabled: u32) -> bool; + #[cfg(feature = "component-model-async")] + task_return(vmctx: vmctx, ty: u32, storage: ptr_u8, storage_len: size) -> bool; + #[cfg(feature = "component-model-async")] + task_wait(vmctx: vmctx, caller_instance: u32, async_: u8, memory: ptr_u8, payload: u32) -> u64; + #[cfg(feature = "component-model-async")] + task_poll(vmctx: vmctx, caller_instance: u32, async_: u8, memory: ptr_u8, payload: u32) -> u64; + #[cfg(feature = "component-model-async")] + task_yield(vmctx: vmctx, async_: u8) -> bool; + #[cfg(feature = "component-model-async")] + subtask_drop(vmctx: vmctx, caller_instance: u32, task_id: u32) -> bool; + #[cfg(feature = "component-model-async")] + sync_enter(vmctx: vmctx, start: ptr_u8, return_: ptr_u8, caller_instance: u32, task_return_type: u32, result_count: u32, storage: ptr_u8, storage_len: size) -> bool; + #[cfg(feature = "component-model-async")] + sync_exit(vmctx: vmctx, callback: ptr_u8, caller_instance: u32, callee: ptr_u8, callee_instance: u32, param_count: u32, storage: ptr_u8, storage_len: size) -> bool; + #[cfg(feature = "component-model-async")] + async_enter(vmctx: vmctx, start: ptr_u8, return_: ptr_u8, caller_instance: u32, task_return_type: u32, params: u32, results: u32) -> bool; + #[cfg(feature = "component-model-async")] + async_exit(vmctx: vmctx, callback: ptr_u8, post_return: ptr_u8, caller_instance: u32, callee: ptr_u8, callee_instance: u32, param_count: u32, result_count: u32, flags: u32) -> u64; + #[cfg(feature = "component-model-async")] + future_new(vmctx: vmctx, ty: u32) -> u64; + #[cfg(feature = "component-model-async")] + future_write(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, string_encoding: u8, ty: u32, future: u32, address: u32) -> u64; + #[cfg(feature = "component-model-async")] + future_read(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, string_encoding: u8, ty: u32, err_ctx_ty: u32, future: u32, address: u32) -> u64; + #[cfg(feature = "component-model-async")] + future_cancel_write(vmctx: vmctx, ty: u32, async_: u8, writer: u32) -> u64; + #[cfg(feature = "component-model-async")] + future_cancel_read(vmctx: vmctx, ty: u32, async_: u8, reader: u32) -> u64; + #[cfg(feature = "component-model-async")] + future_close_writable(vmctx: vmctx, ty: u32, err_ctx_ty: u32, writer: u32, error: u32) -> bool; + #[cfg(feature = "component-model-async")] + future_close_readable(vmctx: vmctx, ty: u32, reader: u32) -> bool; + #[cfg(feature = "component-model-async")] + stream_new(vmctx: vmctx, ty: u32) -> u64; + #[cfg(feature = "component-model-async")] + stream_write(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, string_encoding: u8, ty: u32, stream: u32, address: u32, count: u32) -> u64; + #[cfg(feature = "component-model-async")] + stream_read(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, string_encoding: u8, ty: u32, err_ctx_ty: u32, stream: u32, address: u32, count: u32) -> u64; + #[cfg(feature = "component-model-async")] + stream_cancel_write(vmctx: vmctx, ty: u32, async_: u8, writer: u32) -> u64; + #[cfg(feature = "component-model-async")] + stream_cancel_read(vmctx: vmctx, ty: u32, async_: u8, reader: u32) -> u64; + #[cfg(feature = "component-model-async")] + stream_close_writable(vmctx: vmctx, ty: u32, err_ctx_ty: u32, writer: u32, error: u32) -> bool; + #[cfg(feature = "component-model-async")] + stream_close_readable(vmctx: vmctx, ty: u32, reader: u32) -> bool; + #[cfg(feature = "component-model-async")] + flat_stream_write(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, ty: u32, payload_size: u32, payload_align: u32, stream: u32, address: u32, count: u32) -> u64; + #[cfg(feature = "component-model-async")] + flat_stream_read(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, ty: u32, err_ctx_ty: u32, payload_size: u32, payload_align: u32, stream: u32, address: u32, count: u32) -> u64; + #[cfg(feature = "component-model-async")] + error_context_new(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, string_encoding: u8, ty: u32, debug_msg_address: u32, debug_msg_len: u32) -> u64; + #[cfg(feature = "component-model-async")] + error_context_debug_message(vmctx: vmctx, memory: ptr_u8, realloc: ptr_u8, string_encoding: u8, ty: u32, err_ctx_handle: u32, debug_msg_address: u32) -> bool; + #[cfg(feature = "component-model-async")] + error_context_drop(vmctx: vmctx, ty: u32, err_ctx_handle: u32) -> bool; + #[cfg(feature = "component-model-async")] future_transfer(vmctx: vmctx, src_idx: u32, src_table: u32, dst_table: u32) -> u64; + #[cfg(feature = "component-model-async")] stream_transfer(vmctx: vmctx, src_idx: u32, src_table: u32, dst_table: u32) -> u64; + #[cfg(feature = "component-model-async")] error_context_transfer(vmctx: vmctx, src_idx: u32, src_table: u32, dst_table: u32) -> u64; trap(vmctx: vmctx, code: u8); diff --git a/crates/environ/src/component/dfg.rs b/crates/environ/src/component/dfg.rs index 010e74edabb6..1a20f7a8541d 100644 --- a/crates/environ/src/component/dfg.rs +++ b/crates/environ/src/component/dfg.rs @@ -286,7 +286,9 @@ pub enum Trampoline { TaskBackpressure { instance: RuntimeComponentInstanceIndex, }, - TaskReturn, + TaskReturn { + results: TypeTupleIndex, + }, TaskWait { instance: RuntimeComponentInstanceIndex, async_: bool, @@ -308,6 +310,7 @@ pub enum Trampoline { }, StreamRead { ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, options: CanonicalOptions, }, StreamWrite { @@ -327,12 +330,14 @@ pub enum Trampoline { }, StreamCloseWritable { ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, }, FutureNew { ty: TypeFutureTableIndex, }, FutureRead { ty: TypeFutureTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, options: CanonicalOptions, }, FutureWrite { @@ -352,6 +357,7 @@ pub enum Trampoline { }, FutureCloseWritable { ty: TypeFutureTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, }, ErrorContextNew { ty: TypeComponentLocalErrorContextTableIndex, @@ -368,6 +374,10 @@ pub enum Trampoline { ResourceTransferBorrow, ResourceEnterCall, ResourceExitCall, + SyncEnterCall, + SyncExitCall { + callback: Option, + }, AsyncEnterCall, AsyncExitCall { callback: Option, @@ -765,7 +775,9 @@ impl LinearizeDfg<'_> { Trampoline::TaskBackpressure { instance } => info::Trampoline::TaskBackpressure { instance: *instance, }, - Trampoline::TaskReturn => info::Trampoline::TaskReturn, + Trampoline::TaskReturn { results } => { + info::Trampoline::TaskReturn { results: *results } + } Trampoline::TaskWait { instance, async_, @@ -789,8 +801,13 @@ impl LinearizeDfg<'_> { instance: *instance, }, Trampoline::StreamNew { ty } => info::Trampoline::StreamNew { ty: *ty }, - Trampoline::StreamRead { ty, options } => info::Trampoline::StreamRead { + Trampoline::StreamRead { + ty, + err_ctx_ty, + options, + } => info::Trampoline::StreamRead { ty: *ty, + err_ctx_ty: *err_ctx_ty, options: self.options(options), }, Trampoline::StreamWrite { ty, options } => info::Trampoline::StreamWrite { @@ -808,12 +825,20 @@ impl LinearizeDfg<'_> { Trampoline::StreamCloseReadable { ty } => { info::Trampoline::StreamCloseReadable { ty: *ty } } - Trampoline::StreamCloseWritable { ty } => { - info::Trampoline::StreamCloseWritable { ty: *ty } + Trampoline::StreamCloseWritable { ty, err_ctx_ty } => { + info::Trampoline::StreamCloseWritable { + ty: *ty, + err_ctx_ty: *err_ctx_ty, + } } Trampoline::FutureNew { ty } => info::Trampoline::FutureNew { ty: *ty }, - Trampoline::FutureRead { ty, options } => info::Trampoline::FutureRead { + Trampoline::FutureRead { + ty, + err_ctx_ty, + options, + } => info::Trampoline::FutureRead { ty: *ty, + err_ctx_ty: *err_ctx_ty, options: self.options(options), }, Trampoline::FutureWrite { ty, options } => info::Trampoline::FutureWrite { @@ -831,8 +856,11 @@ impl LinearizeDfg<'_> { Trampoline::FutureCloseReadable { ty } => { info::Trampoline::FutureCloseReadable { ty: *ty } } - Trampoline::FutureCloseWritable { ty } => { - info::Trampoline::FutureCloseWritable { ty: *ty } + Trampoline::FutureCloseWritable { ty, err_ctx_ty } => { + info::Trampoline::FutureCloseWritable { + ty: *ty, + err_ctx_ty: *err_ctx_ty, + } } Trampoline::ErrorContextNew { ty, options } => info::Trampoline::ErrorContextNew { ty: *ty, @@ -849,6 +877,10 @@ impl LinearizeDfg<'_> { Trampoline::ResourceTransferBorrow => info::Trampoline::ResourceTransferBorrow, Trampoline::ResourceEnterCall => info::Trampoline::ResourceEnterCall, Trampoline::ResourceExitCall => info::Trampoline::ResourceExitCall, + Trampoline::SyncEnterCall => info::Trampoline::SyncEnterCall, + Trampoline::SyncExitCall { callback } => info::Trampoline::SyncExitCall { + callback: callback.map(|v| self.runtime_callback(v)), + }, Trampoline::AsyncEnterCall => info::Trampoline::AsyncEnterCall, Trampoline::AsyncExitCall { callback, diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs index d4da14e5d6c1..6bc07660504f 100644 --- a/crates/environ/src/component/info.rs +++ b/crates/environ/src/component/info.rs @@ -689,7 +689,10 @@ pub enum Trampoline { /// A `task.return` intrinsic, which returns a result to the caller of a /// lifted export function. This allows the callee to continue executing /// after returning a result. - TaskReturn, + TaskReturn { + /// Tuple representing the result types this intrinsic accepts. + results: TypeTupleIndex, + }, /// A `task.wait` intrinsic, which waits for at least one outstanding async /// task/stream/future to make progress, returning the first such event. @@ -738,6 +741,10 @@ pub enum Trampoline { StreamRead { /// The table index for the specific `stream` type and caller instance. ty: TypeStreamTableIndex, + + /// The table index for the `error-context` type in the caller instance. + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + /// Any options (e.g. string encoding) to use when storing values to /// memory. options: CanonicalOptions, @@ -784,6 +791,9 @@ pub enum Trampoline { StreamCloseWritable { /// The table index for the specific `stream` type and caller instance. ty: TypeStreamTableIndex, + + /// The table index for the `error-context` type in the caller instance. + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, }, /// A `future.new` intrinsic to create a new `future` handle of the @@ -797,6 +807,10 @@ pub enum Trampoline { FutureRead { /// The table index for the specific `future` type and caller instance. ty: TypeFutureTableIndex, + + /// The table index for the `error-context` type in the caller instance. + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + /// Any options (e.g. string encoding) to use when storing values to /// memory. options: CanonicalOptions, @@ -843,6 +857,9 @@ pub enum Trampoline { FutureCloseWritable { /// The table index for the specific `future` type and caller instance. ty: TypeFutureTableIndex, + + /// The table index for the `error-context` type in the caller instance. + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, }, /// A `error-context.new` intrinsic to create a new `error-context` with a @@ -890,12 +907,23 @@ pub enum Trampoline { /// Same as `ResourceEnterCall` except for when exiting a call. ResourceExitCall, - /// An intrinsic used by FACT-generated modules to begin a call to an + /// An intrinsic used by FACT-generated modules to begin a call involving a + /// sync-lowered import and async-lifted export. + SyncEnterCall, + + /// An intrinsic used by FACT-generated modules to complete a call involving + /// a sync-lowered import and async-lifted export. + SyncExitCall { + /// The callee's callback function, if any. + callback: Option, + }, + + /// An intrinsic used by FACT-generated modules to begin a call involving an /// async-lowered import function. AsyncEnterCall, - /// An intrinsic used by FACT-generated modules to complete a call to an - /// async-lowered import function. + /// An intrinsic used by FACT-generated modules to complete a call involving + /// an async-lowered import function. /// /// Note that `AsyncEnterCall` and `AsyncExitCall` could theoretically be /// combined into a single `AsyncCall` intrinsic, but we separate them to @@ -956,7 +984,7 @@ impl Trampoline { ResourceRep(i) => format!("component-resource-rep[{}]", i.as_u32()), ResourceDrop(i) => format!("component-resource-drop[{}]", i.as_u32()), TaskBackpressure { .. } => format!("task-backpressure"), - TaskReturn => format!("task-return"), + TaskReturn { .. } => format!("task-return"), TaskWait { .. } => format!("task-wait"), TaskPoll { .. } => format!("task-poll"), TaskYield { .. } => format!("task-yield"), @@ -982,6 +1010,8 @@ impl Trampoline { ResourceTransferBorrow => format!("component-resource-transfer-borrow"), ResourceEnterCall => format!("component-resource-enter-call"), ResourceExitCall => format!("component-resource-exit-call"), + SyncEnterCall => format!("component-sync-enter-call"), + SyncExitCall { .. } => format!("component-sync-exit-call"), AsyncEnterCall => format!("component-async-enter-call"), AsyncExitCall { .. } => format!("component-async-exit-call"), FutureTransfer => format!("future-transfer"), diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs index 7ff4263bd504..3e9d7a687e69 100644 --- a/crates/environ/src/component/translate.rs +++ b/crates/environ/src/component/translate.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use std::mem; use wasmparser::component_types::{ AliasableResourceId, ComponentCoreModuleTypeId, ComponentDefinedTypeId, ComponentEntityType, - ComponentFuncTypeId, ComponentInstanceTypeId, + ComponentFuncTypeId, ComponentInstanceTypeId, ComponentValType, }; use wasmparser::types::Types; use wasmparser::{Chunk, ComponentImportName, Encoding, Parser, Payload, Validator}; @@ -193,6 +193,7 @@ enum LocalInitializer<'data> { }, TaskReturn { func: ModuleInternedTypeIndex, + result: Option, }, TaskWait { func: ModuleInternedTypeIndex, @@ -632,10 +633,18 @@ impl<'a, 'data> Translator<'a, 'data> { core_func_index += 1; LocalInitializer::TaskBackpressure { func: core_type } } - wasmparser::CanonicalFunction::TaskReturn { .. } => { - let core_type = self.core_func_signature(core_func_index)?; + wasmparser::CanonicalFunction::TaskReturn { result } => { + let result = result.map(|ty| match ty { + wasmparser::ComponentValType::Primitive(ty) => { + ComponentValType::Primitive(ty) + } + wasmparser::ComponentValType::Type(ty) => { + ComponentValType::Type(types.component_defined_type_at(ty)) + } + }); + let func = self.core_func_signature(core_func_index)?; core_func_index += 1; - LocalInitializer::TaskReturn { func: core_type } + LocalInitializer::TaskReturn { func, result } } wasmparser::CanonicalFunction::TaskWait { async_, memory } => { let func = self.core_func_signature(core_func_index)?; diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rs index a2d7020691ef..9286305d7cb8 100644 --- a/crates/environ/src/component/translate/adapt.rs +++ b/crates/environ/src/component/translate/adapt.rs @@ -304,6 +304,12 @@ fn fact_import_to_core_def( } fact::Import::ResourceEnterCall => simple_intrinsic(dfg::Trampoline::ResourceEnterCall), fact::Import::ResourceExitCall => simple_intrinsic(dfg::Trampoline::ResourceExitCall), + fact::Import::SyncEnterCall => simple_intrinsic(dfg::Trampoline::SyncEnterCall), + fact::Import::SyncExitCall { callback } => { + simple_intrinsic(dfg::Trampoline::SyncExitCall { + callback: callback.clone().map(|v| dfg.callbacks.push(v)), + }) + } fact::Import::AsyncEnterCall => simple_intrinsic(dfg::Trampoline::AsyncEnterCall), fact::Import::AsyncExitCall { callback, diff --git a/crates/environ/src/component/translate/inline.rs b/crates/environ/src/component/translate/inline.rs index aa2ca6f61e87..c0c0417c57d3 100644 --- a/crates/environ/src/component/translate/inline.rs +++ b/crates/environ/src/component/translate/inline.rs @@ -679,11 +679,16 @@ impl<'a> Inliner<'a> { )); frame.funcs.push(dfg::CoreDef::Trampoline(index)); } - TaskReturn { func } => { + TaskReturn { func, result } => { + let results = result + .iter() + .map(|ty| types.valtype(frame.translation.types_ref(), ty)) + .collect::>()?; + let results = types.new_tuple_type(results); let index = self .result .trampolines - .push((*func, dfg::Trampoline::TaskReturn)); + .push((*func, dfg::Trampoline::TaskReturn { results })); frame.funcs.push(dfg::CoreDef::Trampoline(index)); } TaskWait { @@ -754,12 +759,17 @@ impl<'a> Inliner<'a> { else { unreachable!() }; + let err_ctx_ty = types.error_context_table_type()?; let options = self.adapter_options(frame, types, options); let options = self.canonical_options(options); - let index = self - .result - .trampolines - .push((*func, dfg::Trampoline::StreamRead { ty, options })); + let index = self.result.trampolines.push(( + *func, + dfg::Trampoline::StreamRead { + ty, + err_ctx_ty, + options, + }, + )); frame.funcs.push(dfg::CoreDef::Trampoline(index)); } StreamWrite { ty, func, options } => { @@ -824,10 +834,11 @@ impl<'a> Inliner<'a> { else { unreachable!() }; - let index = self - .result - .trampolines - .push((*func, dfg::Trampoline::StreamCloseWritable { ty })); + let err_ctx_ty = types.error_context_table_type()?; + let index = self.result.trampolines.push(( + *func, + dfg::Trampoline::StreamCloseWritable { ty, err_ctx_ty }, + )); frame.funcs.push(dfg::CoreDef::Trampoline(index)); } FutureNew { ty, func } => { @@ -848,12 +859,17 @@ impl<'a> Inliner<'a> { else { unreachable!() }; + let err_ctx_ty = types.error_context_table_type()?; let options = self.adapter_options(frame, types, options); let options = self.canonical_options(options); - let index = self - .result - .trampolines - .push((*func, dfg::Trampoline::FutureRead { ty, options })); + let index = self.result.trampolines.push(( + *func, + dfg::Trampoline::FutureRead { + ty, + err_ctx_ty, + options, + }, + )); frame.funcs.push(dfg::CoreDef::Trampoline(index)); } FutureWrite { ty, func, options } => { @@ -918,10 +934,11 @@ impl<'a> Inliner<'a> { else { unreachable!() }; - let index = self - .result - .trampolines - .push((*func, dfg::Trampoline::FutureCloseWritable { ty })); + let err_ctx_ty = types.error_context_table_type()?; + let index = self.result.trampolines.push(( + *func, + dfg::Trampoline::FutureCloseWritable { ty, err_ctx_ty }, + )); frame.funcs.push(dfg::CoreDef::Trampoline(index)); } ErrorContextNew { func, options } => { diff --git a/crates/environ/src/component/types.rs b/crates/environ/src/component/types.rs index 7ea3bf7311ca..a0be404da8d1 100644 --- a/crates/environ/src/component/types.rs +++ b/crates/environ/src/component/types.rs @@ -91,13 +91,16 @@ indices! { pub struct TypeListIndex(u32); /// Index pointing to a future type in the component model. pub struct TypeFutureIndex(u32); + /// Index pointing to a future table within a component. /// /// This is analogous to `TypeResourceTableIndex` in that it tracks /// ownership of futures within each (sub)component instance. pub struct TypeFutureTableIndex(u32); + /// Index pointing to a stream type in the component model. pub struct TypeStreamIndex(u32); + /// Index pointing to a stream table within a component. /// /// This is analogous to `TypeResourceTableIndex` in that it tracks @@ -117,9 +120,6 @@ indices! { /// not just a subcomponent. pub struct TypeComponentGlobalErrorContextTableIndex(u32); - /// Index pointing to an interned `task.return` type within a component. - pub struct TypeTaskReturnIndex(u32); - /// Index pointing to a resource table within a component. /// /// This is a Wasmtime-specific type index which isn't part of the component @@ -277,7 +277,6 @@ pub struct ComponentTypes { pub(super) stream_tables: PrimaryMap, pub(super) error_context_tables: PrimaryMap, - pub(super) task_returns: PrimaryMap, } impl TypeTrace for ComponentTypes { @@ -538,20 +537,6 @@ pub struct TypeFunc { pub params: TypeTupleIndex, /// Results of the function represented as a tuple. pub results: TypeTupleIndex, - /// Expected core func type for memory32 `task.return` calls for this function. - pub task_return_type32: TypeTaskReturnIndex, - /// Expected core func type for memory64 `task.return` calls for this function. - pub task_return_type64: TypeTaskReturnIndex, -} - -/// A core type representing the expected `task.return` signature for a -/// component function. -#[derive(Serialize, Deserialize, Clone, Hash, Eq, PartialEq, Debug)] -pub struct TypeTaskReturn { - /// Core type parameters for the signature. - /// - /// Note that `task.return` never returns results. - pub params: Vec, } /// All possible interface types that values can have. diff --git a/crates/environ/src/component/types_builder.rs b/crates/environ/src/component/types_builder.rs index 82ccaa2c1340..0128ff27b4ff 100644 --- a/crates/environ/src/component/types_builder.rs +++ b/crates/environ/src/component/types_builder.rs @@ -50,7 +50,6 @@ pub struct ComponentTypesBuilder { future_tables: HashMap, stream_tables: HashMap, error_context_tables: HashMap, - task_returns: HashMap, component_types: ComponentTypes, module_types: ModuleTypesBuilder, @@ -109,7 +108,6 @@ impl ComponentTypesBuilder { future_tables: HashMap::default(), stream_tables: HashMap::default(), error_context_tables: HashMap::default(), - task_returns: HashMap::default(), component_types: ComponentTypes::default(), type_info: TypeInformationCache::default(), resources: ResourcesBuilder::default(), @@ -248,22 +246,10 @@ impl ComponentTypesBuilder { .collect::>()?; let params = self.new_tuple_type(params); let results = self.new_tuple_type(results); - let (task_return_type32, task_return_type64) = - if let Some(types) = self.flat_types(&InterfaceType::Tuple(results)) { - (types.memory32.to_vec(), types.memory64.to_vec()) - } else { - (vec![FlatType::I32], vec![FlatType::I64]) - }; let ty = TypeFunc { param_names, params, results, - task_return_type32: self.add_task_return_type(TypeTaskReturn { - params: task_return_type32, - }), - task_return_type64: self.add_task_return_type(TypeTaskReturn { - params: task_return_type64, - }), }; Ok(self.add_func_type(ty)) } @@ -443,7 +429,16 @@ impl ComponentTypesBuilder { Ok(ret) } - fn valtype(&mut self, types: TypesRef<'_>, ty: &ComponentValType) -> Result { + /// Retrieve Wasmtime's type representation of the `error-context` type. + pub fn error_context_type(&mut self) -> Result { + self.error_context_table_type() + } + + pub(crate) fn valtype( + &mut self, + types: TypesRef<'_>, + ty: &ComponentValType, + ) -> Result { assert_eq!(types.id(), self.module_types.validator_id()); match ty { ComponentValType::Primitive(p) => Ok(p.into()), @@ -509,7 +504,7 @@ impl ComponentTypesBuilder { Ok(self.new_tuple_type(types)) } - fn new_tuple_type(&mut self, types: Box<[InterfaceType]>) -> TypeTupleIndex { + pub(crate) fn new_tuple_type(&mut self, types: Box<[InterfaceType]>) -> TypeTupleIndex { let abi = CanonicalAbiInfo::record( types .iter() @@ -699,21 +694,6 @@ impl ComponentTypesBuilder { ) } - /// Interns a new task return type within this type information. - pub fn add_task_return_type(&mut self, ty: TypeTaskReturn) -> TypeTaskReturnIndex { - intern( - &mut self.task_returns, - &mut self.component_types.task_returns, - ty, - ) - } - - /// Gets a previously interned task return type within this type - /// information, if any. - pub fn get_task_return_type(&self, ty: &TypeTaskReturn) -> Option { - self.task_returns.get(ty).copied() - } - /// Returns the canonical ABI information about the specified type. pub fn canonical_abi(&self, ty: &InterfaceType) -> &CanonicalAbiInfo { self.component_types.canonical_abi(ty) diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rs index 8e7232dce29a..7d25884e803b 100644 --- a/crates/environ/src/fact.rs +++ b/crates/environ/src/fact.rs @@ -21,7 +21,7 @@ use crate::component::dfg::CoreDef; use crate::component::{ Adapter, AdapterOptions as AdapterOptionsDfg, ComponentTypesBuilder, FlatType, InterfaceType, - StringEncoding, Transcode, TypeFuncIndex, + RuntimeComponentInstanceIndex, StringEncoding, Transcode, TypeFuncIndex, }; use crate::fact::transcode::Transcoder; use crate::prelude::*; @@ -36,6 +36,13 @@ mod trampoline; mod transcode; mod traps; +/// Bit flag for indicating async-lifted exports +/// +/// This flag may be passed to the `async-exit` built-in function (which is +/// called from both async->async and async->sync adapters) to indicate that the +/// callee is an async-lifted export. +pub const EXIT_FLAG_ASYNC_CALLEE: i32 = 1 << 0; + /// Representation of an adapter module. pub struct Module<'a> { /// Whether or not debug code is inserted into the adapters themselves. @@ -65,6 +72,23 @@ pub struct Module<'a> { imported_resource_enter_call: Option, imported_resource_exit_call: Option, + // Cached versions of imported trampolines for working with the async ABI. + imported_async_enter_call: Option, + imported_async_exit_call: Option, + + // Cached versions of imported trampolines for fusing sync-lowered imports + // with async-lifted exports. These are `HashMap`s (using the adapter + // function name) because the signatures of the trampolines vary depending + // on the signature of the adapter function we're generating code for. + imported_sync_enter_call: HashMap, + imported_sync_exit_call: HashMap, + + // Cached versions of imported trampolines for working with `stream`s, + // `future`s, and `error-context`s. + imported_future_transfer: Option, + imported_stream_transfer: Option, + imported_error_context_transfer: Option, + // Current status of index spaces from the imports generated so far. imported_funcs: PrimaryMap>, imported_memories: PrimaryMap, @@ -73,6 +97,8 @@ pub struct Module<'a> { funcs: PrimaryMap, helper_funcs: HashMap, helper_worklist: Vec<(FunctionId, Helper)>, + + exports: Vec<(u32, String)>, } struct AdapterData { @@ -95,6 +121,7 @@ struct AdapterData { /// These options are typically unique per-adapter and generally aren't needed /// when translating recursive types within an adapter. struct AdapterOptions { + instance: RuntimeComponentInstanceIndex, /// The ascribed type of this adapter. ty: TypeFuncIndex, /// The global that represents the instance flags for where this adapter @@ -189,6 +216,14 @@ impl<'a> Module<'a> { imported_resource_transfer_borrow: None, imported_resource_enter_call: None, imported_resource_exit_call: None, + imported_async_enter_call: None, + imported_async_exit_call: None, + imported_future_transfer: None, + imported_stream_transfer: None, + imported_error_context_transfer: None, + imported_sync_enter_call: HashMap::new(), + imported_sync_exit_call: HashMap::new(), + exports: Vec::new(), } } @@ -311,6 +346,7 @@ impl<'a> Module<'a> { }); AdapterOptions { + instance: *instance, ty, flags, post_return: None, @@ -407,9 +443,30 @@ impl<'a> Module<'a> { results: &[ValType], import: Import, get: impl Fn(&mut Self) -> &mut Option, + ) -> FuncIndex { + self.import_simple_get_and_set( + module, + name, + params, + results, + import, + |me| *get(me), + |me, v| *get(me) = Some(v), + ) + } + + fn import_simple_get_and_set( + &mut self, + module: &str, + name: &str, + params: &[ValType], + results: &[ValType], + import: Import, + get: impl Fn(&mut Self) -> Option, + set: impl Fn(&mut Self, FuncIndex), ) -> FuncIndex { if let Some(idx) = get(self) { - return *idx; + return idx; } let ty = self.core_types.function(params, results); let ty = EntityType::Function(ty); @@ -417,10 +474,172 @@ impl<'a> Module<'a> { self.imports.push(import); let idx = self.imported_funcs.push(None); - *get(self) = Some(idx); + set(self, idx); idx } + /// Import a host built-in function to set up a subtask for a sync-lowered + /// import call to an async-lifted export. + /// + /// Given that the callee may exert backpressure before the host can copy + /// the parameters, the adapter must use this function to set up the subtask + /// and stash the parameters as part of that subtask until any backpressure + /// has cleared. + fn import_sync_enter_call(&mut self, suffix: &str, params: &[ValType]) -> FuncIndex { + self.import_simple_get_and_set( + "sync", + &format!("[enter-call]{suffix}"), + &[ + ValType::FUNCREF, + ValType::FUNCREF, + ValType::I32, + ValType::I32, + ValType::I32, + ] + .into_iter() + .chain(params.iter().copied()) + .collect::>(), + &[], + Import::SyncEnterCall, + |me| me.imported_sync_enter_call.get(suffix).copied(), + |me, v| { + assert!(me + .imported_sync_enter_call + .insert(suffix.to_owned(), v) + .is_none()) + }, + ) + } + + /// Import a host built-in function to start a subtask for a sync-lowered + /// import call to an async-lifted export. + /// + /// This call with block until the subtask has produced result(s) via the + /// `task.return` intrinsic. + /// + /// Note that this could potentially be combined with the `sync-enter` + /// built-in into a single built-in function that does both jobs. However, + /// we've kept them separate to allow a future optimization where the caller + /// calls the callee directly rather than using `sync-exit` to have the host + /// do it. + fn import_sync_exit_call( + &mut self, + suffix: &str, + callback: Option, + results: &[ValType], + ) -> FuncIndex { + self.import_simple_get_and_set( + "sync", + &format!("[exit-call]{suffix}"), + &[ValType::I32, ValType::FUNCREF, ValType::I32, ValType::I32], + results, + Import::SyncExitCall { + callback: callback + .map(|callback| self.imported_funcs.get(callback).unwrap().clone().unwrap()), + }, + |me| me.imported_sync_exit_call.get(suffix).copied(), + |me, v| { + assert!(me + .imported_sync_exit_call + .insert(suffix.to_owned(), v) + .is_none()) + }, + ) + } + + /// Import a host built-in function to set up a subtask for an async-lowered + /// import call to an async- or sync-lifted export. + fn import_async_enter_call(&mut self) -> FuncIndex { + self.import_simple( + "async", + "enter-call", + &[ + ValType::FUNCREF, + ValType::FUNCREF, + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ], + &[], + Import::AsyncEnterCall, + |me| &mut me.imported_async_enter_call, + ) + } + + /// Import a host built-in function to start a subtask for an async-lowered + /// import call to an async- or sync-lifted export. + /// + /// Note that this could potentially be combined with the `async-enter` + /// built-in into a single built-in function that does both jobs. However, + /// we've kept them separate to allow a future optimization where the caller + /// calls the callee directly rather than using `async-exit` to have the + /// host do it. + fn import_async_exit_call( + &mut self, + callback: Option, + post_return: Option, + ) -> FuncIndex { + self.import_simple( + "async", + "exit-call", + &[ + ValType::I32, + ValType::FUNCREF, + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ], + &[ValType::I32], + Import::AsyncExitCall { + callback: callback + .map(|callback| self.imported_funcs.get(callback).unwrap().clone().unwrap()), + post_return: post_return.map(|post_return| { + self.imported_funcs + .get(post_return) + .unwrap() + .clone() + .unwrap() + }), + }, + |me| &mut me.imported_async_exit_call, + ) + } + + fn import_future_transfer(&mut self) -> FuncIndex { + self.import_simple( + "future", + "transfer", + &[ValType::I32; 3], + &[ValType::I32], + Import::FutureTransfer, + |me| &mut me.imported_future_transfer, + ) + } + + fn import_stream_transfer(&mut self) -> FuncIndex { + self.import_simple( + "stream", + "transfer", + &[ValType::I32; 3], + &[ValType::I32], + Import::StreamTransfer, + |me| &mut me.imported_stream_transfer, + ) + } + + fn import_error_context_transfer(&mut self) -> FuncIndex { + self.import_simple( + "error-context", + "transfer", + &[ValType::I32; 3], + &[ValType::I32], + Import::ErrorContextTransfer, + |me| &mut me.imported_error_context_transfer, + ) + } + fn import_resource_transfer_own(&mut self) -> FuncIndex { self.import_simple( "resource", @@ -496,6 +715,9 @@ impl<'a> Module<'a> { exports.export(name, ExportKind::Func, idx.as_u32()); } } + for (idx, name) in &self.exports { + exports.export(name, ExportKind::Func, *idx); + } // With all functions numbered the fragments of the body of each // function can be assigned into one final adapter function. @@ -528,6 +750,9 @@ impl<'a> Module<'a> { Body::Call(id) => { Instruction::Call(id_to_index[*id].as_u32()).encode(&mut body); } + Body::RefFunc(id) => { + Instruction::RefFunc(id_to_index[*id].as_u32()).encode(&mut body); + } } } code.raw(&body); @@ -585,11 +810,20 @@ pub enum Import { /// Tears down a previous entry and handles checking borrow-related /// metadata. ResourceExitCall, - /// An intrinsic used by FACT-generated modules to begin a call to an + /// An intrinsic used by FACT-generated modules to begin a call involving a + /// sync-lowered import and async-lifted export. + SyncEnterCall, + /// An intrinsic used by FACT-generated modules to complete a call involving + /// a sync-lowered import and async-lifted export. + SyncExitCall { + /// The callee's callback function, if any. + callback: Option, + }, + /// An intrinsic used by FACT-generated modules to begin a call involving an /// async-lowered import function. AsyncEnterCall, - /// An intrinsic used by FACT-generated modules to complete a call to an - /// async-lowered import function. + /// An intrinsic used by FACT-generated modules to complete a call involving + /// an async-lowered import function. AsyncExitCall { /// The callee's callback function, if any. callback: Option, @@ -704,6 +938,7 @@ struct Function { enum Body { Raw(Vec, Vec<(usize, traps::Trap)>), Call(FunctionId), + RefFunc(FunctionId), } impl Function { diff --git a/crates/environ/src/fact/signature.rs b/crates/environ/src/fact/signature.rs index 328ec085e359..9affe4297d7c 100644 --- a/crates/environ/src/fact/signature.rs +++ b/crates/environ/src/fact/signature.rs @@ -26,6 +26,22 @@ impl ComponentTypesBuilder { let ty = &self[options.ty]; let ptr_ty = options.options.ptr(); + // The async lower ABI is always `(param i32 i32) (result i32)` (for + // wasm32, anyway), regardless of the component-level signature. + // + // The first param is a pointer to linear memory where the parameters have + // been stored by the caller, the second param is a pointer to linear + // memory where the results should be stored by the callee, and the + // result is a status code optionally ORed with a subtask ID. + if let (Context::Lower, true) = (&context, options.options.async_) { + return Signature { + params: vec![ptr_ty; 2], + results: vec![ValType::I32], + }; + } + + // If we're lifting async or sync, or if we're lowering sync, we can + // pass up to `MAX_FLAT_PARAMS` via the stack. let mut params = match self.flatten_types( &options.options, MAX_FLAT_PARAMS, @@ -37,6 +53,28 @@ impl ComponentTypesBuilder { } }; + // If we're lifting async with a callback, the result is an `i32` status + // code, optionally ORed with a guest task identifier, and the result + // will be returned via `task.return`. + // + // If we're lifting async without a callback, then there's no need to return + // anything here since the result will be returned via `task.return` and the + // guest will use `task.wait` rather than return a status code in order to suspend + // itself, if necessary. + if options.options.async_ { + return Signature { + params, + results: if options.options.callback.is_some() { + vec![ptr_ty] + } else { + Vec::new() + }, + }; + } + + // If we've reached this point, we're either lifting or lowering sync, + // in which case the guest will return up to `MAX_FLAT_RESULTS` via the + // stack or spill to linear memory otherwise. let results = match self.flatten_types( &options.options, MAX_FLAT_RESULTS, @@ -62,6 +100,142 @@ impl ComponentTypesBuilder { Signature { params, results } } + /// Generates the signature for a function to be exported by the adapter + /// module and called by the host to lift the parameters from the caller and + /// lower them to the callee. + /// + /// This allows the host to delay copying the parameters until the callee + /// signals readiness by clearing its backpressure flag. + /// + /// Note that this function uses multi-value return to return up to + /// `MAX_FLAT_PARAMS` _results_ via the stack, allowing the host to pass + /// them directly to the callee with no additional effort. + pub(super) fn async_start_signature( + &self, + lower: &AdapterOptions, + lift: &AdapterOptions, + ) -> Signature { + let lower_ty = &self[lower.ty]; + let lower_ptr_ty = lower.options.ptr(); + let params = if lower.options.async_ { + vec![lower_ptr_ty] + } else { + match self.flatten_types( + &lower.options, + MAX_FLAT_PARAMS, + self[lower_ty.params].types.iter().copied(), + ) { + Some(list) => list, + None => { + vec![lower_ptr_ty] + } + } + }; + + let lift_ty = &self[lift.ty]; + let lift_ptr_ty = lift.options.ptr(); + let results = match self.flatten_types( + &lift.options, + // Both sync- and async-lifted functions accept up to this many core + // parameters via the stack. The host will call the `async-start` + // function (possibly after a backpressure delay), which will + // _return_ that many values (using a multi-value return, if + // necessary); the host will then pass them directly to the callee. + MAX_FLAT_PARAMS, + self[lift_ty.params].types.iter().copied(), + ) { + Some(list) => list, + None => { + vec![lift_ptr_ty] + } + }; + + Signature { params, results } + } + + pub(super) fn flatten_lowering_types( + &self, + options: &Options, + tys: impl IntoIterator, + ) -> Option> { + if options.async_ { + // When lowering an async function, we always spill parameters to + // linear memory. + None + } else { + self.flatten_types(options, MAX_FLAT_RESULTS, tys) + } + } + + pub(super) fn flatten_lifting_types( + &self, + options: &Options, + tys: impl IntoIterator, + ) -> Option> { + self.flatten_types( + options, + if options.async_ { + // Async functions return results by calling `task.return`, + // which accepts up to `MAX_FLAT_PARAMS` parameters via the + // stack. + MAX_FLAT_PARAMS + } else { + // Sync functions return results directly (at least until we add + // a `always-task-return` canonical option) and so are limited + // to returning up to `MAX_FLAT_RESULTS` results via the stack. + MAX_FLAT_RESULTS + }, + tys, + ) + } + + /// Generates the signature for a function to be exported by the adapter + /// module and called by the host to lift the results from the callee and + /// lower them to the caller. + /// + /// Given that async-lifted exports return their results via the + /// `task.return` intrinsic, the host will need to copy the results from + /// callee to caller when that intrinsic is called rather than when the + /// callee task fully completes (which may happen much later). + pub(super) fn async_return_signature( + &self, + lower: &AdapterOptions, + lift: &AdapterOptions, + ) -> Signature { + let lift_ty = &self[lift.ty]; + let lift_ptr_ty = lift.options.ptr(); + let mut params = match self + .flatten_lifting_types(&lift.options, self[lift_ty.results].types.iter().copied()) + { + Some(list) => list, + None => { + vec![lift_ptr_ty] + } + }; + + let lower_ty = &self[lower.ty]; + let results = if lower.options.async_ { + // Add return pointer + params.push(lift_ptr_ty); + Vec::new() + } else { + match self.flatten_types( + &lower.options, + MAX_FLAT_RESULTS, + self[lower_ty.results].types.iter().copied(), + ) { + Some(list) => list, + None => { + // Add return pointer + params.push(lift_ptr_ty); + Vec::new() + } + } + }; + + Signature { params, results } + } + /// Pushes the flat version of a list of component types into a final result /// list. pub(super) fn flatten_types( diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs index b96ac5875e07..0735fab52f89 100644 --- a/crates/environ/src/fact/trampoline.rs +++ b/crates/environ/src/fact/trampoline.rs @@ -17,9 +17,10 @@ use crate::component::{ CanonicalAbiInfo, ComponentTypesBuilder, FixedEncoding as FE, FlatType, InterfaceType, - StringEncoding, Transcode, TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, - TypeRecordIndex, TypeResourceTableIndex, TypeResultIndex, TypeTupleIndex, TypeVariantIndex, - VariantInfo, FLAG_MAY_ENTER, FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, + StringEncoding, Transcode, TypeComponentLocalErrorContextTableIndex, TypeEnumIndex, + TypeFlagsIndex, TypeFutureTableIndex, TypeListIndex, TypeOptionIndex, TypeRecordIndex, + TypeResourceTableIndex, TypeResultIndex, TypeStreamTableIndex, TypeTupleIndex, + TypeVariantIndex, VariantInfo, FLAG_MAY_ENTER, FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, }; use crate::fact::signature::Signature; use crate::fact::transcode::Transcoder; @@ -80,50 +81,176 @@ struct Compiler<'a, 'b> { } pub(super) fn compile(module: &mut Module<'_>, adapter: &AdapterData) { + fn compiler<'a, 'b>( + module: &'b mut Module<'a>, + adapter: &AdapterData, + ) -> (Compiler<'a, 'b>, Signature, Signature) { + let lower_sig = module.types.signature(&adapter.lower, Context::Lower); + let lift_sig = module.types.signature(&adapter.lift, Context::Lift); + let ty = module + .core_types + .function(&lower_sig.params, &lower_sig.results); + let result = module + .funcs + .push(Function::new(Some(adapter.name.clone()), ty)); + + // If this type signature contains any borrowed resources then invocations + // of enter/exit call for resource-related metadata tracking must be used. + // It shouldn't matter whether the lower/lift signature is used here as both + // should return the same answer. + let emit_resource_call = module.types.contains_borrow_resource(&adapter.lower); + assert_eq!( + emit_resource_call, + module.types.contains_borrow_resource(&adapter.lift) + ); + + ( + Compiler::new( + module, + result, + lower_sig.params.len() as u32, + emit_resource_call, + ), + lower_sig, + lift_sig, + ) + } + + // This closure compiles a function to be exported to the host which host to + // lift the parameters from the caller and lower them to the callee. + // + // This allows the host to delay copying the parameters until the callee + // signals readiness by clearing its backpressure flag. + let async_start_adapter = |module: &mut Module| { + let sig = module + .types + .async_start_signature(&adapter.lower, &adapter.lift); + let ty = module.core_types.function(&sig.params, &sig.results); + let result = module.funcs.push(Function::new( + Some(format!("[async-start]{}", adapter.name)), + ty, + )); + + Compiler::new(module, result, sig.params.len() as u32, false) + .compile_async_start_adapter(adapter, &sig); + + result + }; + + // This closure compiles a function to be exported by the adapter module and + // called by the host to lift the results from the callee and lower them to + // the caller. + // + // Given that async-lifted exports return their results via the + // `task.return` intrinsic, the host will need to copy the results from + // callee to caller when that intrinsic is called rather than when the + // callee task fully completes (which may happen much later). + let async_return_adapter = |module: &mut Module| { + let sig = module + .types + .async_return_signature(&adapter.lower, &adapter.lift); + let ty = module.core_types.function(&sig.params, &sig.results); + let result = module.funcs.push(Function::new( + Some(format!("[async-return]{}", adapter.name)), + ty, + )); + + Compiler::new(module, result, sig.params.len() as u32, false) + .compile_async_return_adapter(adapter, &sig); + + result + }; + match (adapter.lower.options.async_, adapter.lift.options.async_) { - (false, false) => {} + (false, false) => { + // We can adapt sync->sync case with only minimal use of intrinsics, + // e.g. resource enter and exit calls as needed. + let (compiler, lower_sig, lift_sig) = compiler(module, adapter); + compiler.compile_sync_to_sync_adapter(adapter, &lower_sig, &lift_sig) + } (true, true) => { - todo!() + // In the async->async case, we must compile a couple of helper functions: + // + // - `async-start`: copies the parameters from the caller to the callee + // - `async-return`: copies the result from the callee to the caller + // + // Unlike synchronous calls, the above operations are asynchronous + // and subject to backpressure. If the callee is not yet ready to + // handle a new call, the `async-start` function will not be called + // immediately. Instead, control will return to the caller, + // allowing it to do other work while waiting for this call to make + // progress. Once the callee indicates it is ready, `async-start` + // will be called, and sometime later (possibly after various task + // switch events), when the callee has produced a result, it will + // call `async-return` via the `task.return` intrinsic, at which + // point a `STATUS_RETURNED` event will be delivered to the caller. + let start = async_start_adapter(module); + let return_ = async_return_adapter(module); + let (compiler, _, lift_sig) = compiler(module, adapter); + compiler.compile_async_to_async_adapter( + adapter, + start, + return_, + i32::try_from(lift_sig.params.len()).unwrap(), + ); } (false, true) => { - todo!() + // Like the async->async case above, for the sync->async case we + // also need `async-start` and `async-return` helper functions to + // allow the callee to asynchronously "pull" the parameters and + // "push" the results when it is ready. + // + // However, since the caller is using the synchronous ABI, the + // parameters may have been passed via the stack rather than linear + // memory. In that case, we pass them to the host to store in a + // task-local location temporarily in the case of backpressure. + // Similarly, the host will also temporarily store the results that + // the callee provides to `async-return` until it is ready to resume + // the caller. + let start = async_start_adapter(module); + let return_ = async_return_adapter(module); + let (compiler, lower_sig, lift_sig) = compiler(module, adapter); + compiler.compile_sync_to_async_adapter( + adapter, + start, + return_, + i32::try_from(lift_sig.params.len()).unwrap(), + &lower_sig, + ); } (true, false) => { - todo!() - } - } - - let lower_sig = module.types.signature(&adapter.lower, Context::Lower); - let lift_sig = module.types.signature(&adapter.lift, Context::Lift); - let ty = module - .core_types - .function(&lower_sig.params, &lower_sig.results); - let result = module - .funcs - .push(Function::new(Some(adapter.name.clone()), ty)); - - // If this type signature contains any borrowed resources then invocations - // of enter/exit call for resource-related metadata tracking must be used. - // It shouldn't matter whether the lower/lift signature is used here as both - // should return the same answer. - let emit_resource_call = module.types.contains_borrow_resource(&adapter.lower); - assert_eq!( - emit_resource_call, - module.types.contains_borrow_resource(&adapter.lift) - ); - - Compiler { - types: module.types, - module, - code: Vec::new(), - nlocals: lower_sig.params.len() as u32, - free_locals: HashMap::new(), - traps: Vec::new(), - result, - fuel: INITIAL_FUEL, - emit_resource_call, + // As with the async->async and sync->async cases above, for the + // async->sync case we use `async-start` and `async-return` helper + // functions. Here, those functions allow the host to enforce + // backpressure in the case where the callee instance already has + // another synchronous call in progress, in which case we can't + // start a new one until the current one (and any others already + // waiting in line behind it) has completed. + // + // In the case of backpressure, we'll return control to the caller + // immediately so it can do other work. Later, once the callee is + // ready, the host will call the `async-start` function to retrieve + // the parameters and pass them to the callee. At that point, the + // callee may block on a host call, at which point the host will + // suspend the fiber it is running on and allow the caller (or any + // other ready instance) to run concurrently with the blocked + // callee. Once the callee finally returns, the host will call the + // `async-return` function to write the result to the caller's + // linear memory and deliver a `STATUS_RETURNED` event to the + // caller. + let lift_sig = module.types.signature(&adapter.lift, Context::Lift); + let start = async_start_adapter(module); + let return_ = async_return_adapter(module); + let (compiler, ..) = compiler(module, adapter); + compiler.compile_async_to_sync_adapter( + adapter, + start, + return_, + i32::try_from(lift_sig.params.len()).unwrap(), + i32::try_from(lift_sig.results.len()).unwrap(), + ); + } } - .compile_adapter(adapter, &lower_sig, &lift_sig) } /// Compiles a helper function as specified by the `Helper` configuration. @@ -256,8 +383,309 @@ struct Memory<'a> { offset: u32, } -impl Compiler<'_, '_> { - fn compile_adapter( +impl<'a, 'b> Compiler<'a, 'b> { + fn new( + module: &'b mut Module<'a>, + result: FunctionId, + nlocals: u32, + emit_resource_call: bool, + ) -> Self { + Self { + types: module.types, + module, + result, + code: Vec::new(), + nlocals, + free_locals: HashMap::new(), + traps: Vec::new(), + fuel: INITIAL_FUEL, + emit_resource_call, + } + } + + /// Compile an adapter function supporting an async-lowered import to an + /// async-lifted export. + /// + /// This uses a pair of `async-enter` and `async-exit` built-in functions to + /// set up and start a subtask, respectively. `async-enter` accepts `start` + /// and `return_` functions which copy the parameters and results, + /// respectively; the host will call the former when the callee has cleared + /// its backpressure flag and the latter when the callee has called + /// `task.return`. + fn compile_async_to_async_adapter( + mut self, + adapter: &AdapterData, + start: FunctionId, + return_: FunctionId, + param_count: i32, + ) { + let enter = self.module.import_async_enter_call(); + let exit = self + .module + .import_async_exit_call(adapter.lift.options.callback, None); + + self.flush_code(); + self.module.funcs[self.result] + .body + .push(Body::RefFunc(start)); + self.module.funcs[self.result] + .body + .push(Body::RefFunc(return_)); + self.instruction(I32Const( + i32::try_from(adapter.lower.instance.as_u32()).unwrap(), + )); + self.instruction(I32Const( + i32::try_from(self.types[adapter.lift.ty].results.as_u32()).unwrap(), + )); + // Async-lowered imports pass params and receive results via linear + // memory, and those pointers are in the the first and second params to + // this adapter. We pass them on to the host so it can store them in + // the subtask for later use. + self.instruction(LocalGet(0)); + self.instruction(LocalGet(1)); + self.instruction(Call(enter.as_u32())); + + // TODO: As an optimization, consider checking the backpressure flag on + // the callee instance and, if it's unset _and_ the callee uses a + // callback, translate the params and call the callee function directly + // here (and make sure `exit` knows _not_ to call it in that case). + + // We export this function so we can pass a funcref to the host. + // + // TODO: Use a declarative element segment instead of exporting this. + self.module.exports.push(( + adapter.callee.as_u32(), + format!("[adapter-callee]{}", adapter.name), + )); + + self.instruction(I32Const( + i32::try_from(adapter.lower.instance.as_u32()).unwrap(), + )); + self.instruction(RefFunc(adapter.callee.as_u32())); + self.instruction(I32Const( + i32::try_from(adapter.lift.instance.as_u32()).unwrap(), + )); + self.instruction(I32Const(param_count)); + // The result count for an async callee is either one (if there's a + // callback) or zero (if there's no callback). We conservatively use + // one here to ensure the host provides room for the result, if any. + self.instruction(I32Const(1)); + self.instruction(I32Const(super::EXIT_FLAG_ASYNC_CALLEE)); + self.instruction(Call(exit.as_u32())); + + self.finish() + } + + /// Compile an adapter function supporting a sync-lowered import to an + /// async-lifted export. + /// + /// This uses a pair of `sync-enter` and `sync-exit` built-in functions to + /// set up and start a subtask, respectively. `sync-enter` accepts `start` + /// and `return_` functions which copy the parameters and results, + /// respectively; the host will call the former when the callee has cleared + /// its backpressure flag and the latter when the callee has called + /// `task.return`. + fn compile_sync_to_async_adapter( + mut self, + adapter: &AdapterData, + start: FunctionId, + return_: FunctionId, + lift_param_count: i32, + lower_sig: &Signature, + ) { + let enter = self + .module + .import_sync_enter_call(&adapter.name, &lower_sig.params); + let exit = self.module.import_sync_exit_call( + &adapter.name, + adapter.lift.options.callback, + &lower_sig.results, + ); + + self.flush_code(); + self.module.funcs[self.result] + .body + .push(Body::RefFunc(start)); + self.module.funcs[self.result] + .body + .push(Body::RefFunc(return_)); + self.instruction(I32Const( + i32::try_from(adapter.lower.instance.as_u32()).unwrap(), + )); + self.instruction(I32Const( + i32::try_from(self.types[adapter.lift.ty].results.as_u32()).unwrap(), + )); + self.instruction(I32Const( + i32::try_from( + self.types + .flatten_types( + &adapter.lower.options, + usize::MAX, + self.types[self.types[adapter.lower.ty].results] + .types + .iter() + .copied(), + ) + .map(|v| v.len()) + .unwrap_or(usize::try_from(i32::MAX).unwrap()), + ) + .unwrap(), + )); + + for index in 0..lower_sig.params.len() { + self.instruction(LocalGet(u32::try_from(index).unwrap())); + } + + self.instruction(Call(enter.as_u32())); + + // TODO: As an optimization, consider checking the backpressure flag on + // the callee instance and, if it's unset _and_ the callee uses a + // callback, translate the params and call the callee function directly + // here (and make sure `exit` knows _not_ to call it in that case). + + // We export this function so we can pass a funcref to the host. + // + // TODO: Use a declarative element segment instead of exporting this. + self.module.exports.push(( + adapter.callee.as_u32(), + format!("[adapter-callee]{}", adapter.name), + )); + + self.instruction(I32Const( + i32::try_from(adapter.lower.instance.as_u32()).unwrap(), + )); + self.instruction(RefFunc(adapter.callee.as_u32())); + self.instruction(I32Const( + i32::try_from(adapter.lift.instance.as_u32()).unwrap(), + )); + self.instruction(I32Const(lift_param_count)); + self.instruction(Call(exit.as_u32())); + + self.finish() + } + + /// Compile an adapter function supporting an async-lowered import to a + /// sync-lifted export. + /// + /// This uses a pair of `async-enter` and `async-exit` built-in functions to + /// set up and start a subtask, respectively. `async-enter` accepts `start` + /// and `return_` functions which copy the parameters and results, + /// respectively; the host will call the former when the callee has cleared + /// its backpressure flag and the latter when the callee has returned its + /// result(s). + fn compile_async_to_sync_adapter( + mut self, + adapter: &AdapterData, + start: FunctionId, + return_: FunctionId, + param_count: i32, + result_count: i32, + ) { + let enter = self.module.import_async_enter_call(); + let exit = self + .module + .import_async_exit_call(None, adapter.lift.post_return); + + self.flush_code(); + self.module.funcs[self.result] + .body + .push(Body::RefFunc(start)); + self.module.funcs[self.result] + .body + .push(Body::RefFunc(return_)); + self.instruction(I32Const( + i32::try_from(adapter.lower.instance.as_u32()).unwrap(), + )); + self.instruction(I32Const( + i32::try_from(self.types[adapter.lift.ty].results.as_u32()).unwrap(), + )); + self.instruction(LocalGet(0)); + self.instruction(LocalGet(1)); + self.instruction(Call(enter.as_u32())); + + // We export this function so we can pass a funcref to the host. + // + // TODO: Use a declarative element segment instead of exporting this. + self.module.exports.push(( + adapter.callee.as_u32(), + format!("[adapter-callee]{}", adapter.name), + )); + + self.instruction(I32Const( + i32::try_from(adapter.lower.instance.as_u32()).unwrap(), + )); + self.instruction(RefFunc(adapter.callee.as_u32())); + self.instruction(I32Const( + i32::try_from(adapter.lift.instance.as_u32()).unwrap(), + )); + self.instruction(I32Const(param_count)); + self.instruction(I32Const(result_count)); + self.instruction(I32Const(0)); + self.instruction(Call(exit.as_u32())); + + self.finish() + } + + /// Compiles a function to be exported to the host which host to lift the + /// parameters from the caller and lower them to the callee. + /// + /// This allows the host to delay copying the parameters until the callee + /// signals readiness by clearing its backpressure flag. + fn compile_async_start_adapter(mut self, adapter: &AdapterData, sig: &Signature) { + let param_locals = sig + .params + .iter() + .enumerate() + .map(|(i, ty)| (i as u32, *ty)) + .collect::>(); + + self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, false); + self.translate_params(adapter, ¶m_locals); + self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, true); + + self.finish(); + } + + /// Compiles a function to be exported by the adapter module and called by + /// the host to lift the results from the callee and lower them to the + /// caller. + /// + /// Given that async-lifted exports return their results via the + /// `task.return` intrinsic, the host will need to copy the results from + /// callee to caller when that intrinsic is called rather than when the + /// callee task fully completes (which may happen much later). + fn compile_async_return_adapter(mut self, adapter: &AdapterData, sig: &Signature) { + let param_locals = sig + .params + .iter() + .enumerate() + .map(|(i, ty)| (i as u32, *ty)) + .collect::>(); + + self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, false); + // Note that we pass `param_locals` as _both_ the `param_locals` and + // `result_locals` parameters to `translate_results`. That's because + // the _parameters_ to `task.return` are actually the _results_ that the + // caller is waiting for. + // + // Additionally, the host will append a return + // pointer to the end of that list before calling this adapter's + // `async-return` function if the results exceed `MAX_FLAT_RESULTS` or + // the import is lowered async, in which case `translate_results` will + // use that pointer to store the results. + self.translate_results(adapter, ¶m_locals, ¶m_locals); + self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, true); + + self.finish() + } + + /// Compile an adapter function supporting a sync-lowered import to a + /// sync-lifted export. + /// + /// Unlike calls involving async-lowered imports or async-lifted exports, + /// this adapter need not involve host built-ins except possibly for + /// resource bookkeeping. + fn compile_sync_to_sync_adapter( mut self, adapter: &AdapterData, lower_sig: &Signature, @@ -375,9 +803,12 @@ impl Compiler<'_, '_> { // TODO: handle subtyping assert_eq!(src_tys.len(), dst_tys.len()); - let src_flat = + let src_flat = if adapter.lower.options.async_ { + None + } else { self.types - .flatten_types(lower_opts, MAX_FLAT_PARAMS, src_tys.iter().copied()); + .flatten_types(lower_opts, MAX_FLAT_PARAMS, src_tys.iter().copied()) + }; let dst_flat = self.types .flatten_types(lift_opts, MAX_FLAT_PARAMS, dst_tys.iter().copied()); @@ -404,14 +835,15 @@ impl Compiler<'_, '_> { let dst = if let Some(flat) = &dst_flat { Destination::Stack(flat, lift_opts) } else { - // If there are too many parameters then space is allocated in the - // destination module for the parameters via its `realloc` function. let abi = CanonicalAbiInfo::record(dst_tys.iter().map(|t| self.types.canonical_abi(t))); let (size, align) = if lift_opts.memory64 { (abi.size64, abi.align64) } else { (abi.size32, abi.align32) }; + + // If there are too many parameters then space is allocated in the + // destination module for the parameters via its `realloc` function. let size = MallocSize::Const(size); Destination::Memory(self.malloc(lift_opts, size, align)) }; @@ -456,12 +888,12 @@ impl Compiler<'_, '_> { let lift_opts = &adapter.lift.options; let lower_opts = &adapter.lower.options; - let src_flat = - self.types - .flatten_types(lift_opts, MAX_FLAT_RESULTS, src_tys.iter().copied()); - let dst_flat = - self.types - .flatten_types(lower_opts, MAX_FLAT_RESULTS, dst_tys.iter().copied()); + let src_flat = self + .types + .flatten_lifting_types(lift_opts, src_tys.iter().copied()); + let dst_flat = self + .types + .flatten_lowering_types(lower_opts, dst_tys.iter().copied()); let src = if src_flat.is_some() { Source::Stack(Stack { @@ -478,7 +910,14 @@ impl Compiler<'_, '_> { .map(|t| self.types.align(lift_opts, t)) .max() .unwrap_or(1); - assert_eq!(result_locals.len(), 1); + assert_eq!( + result_locals.len(), + if lower_opts.async_ || lift_opts.async_ { + 2 + } else { + 1 + } + ); let (addr, ty) = result_locals[0]; assert_eq!(ty, lift_opts.ptr()); Source::Memory(self.memory_operand(lift_opts, TempLocal::new(addr, ty), align)) @@ -600,13 +1039,11 @@ impl Compiler<'_, '_> { InterfaceType::Option(_) | InterfaceType::Result(_) => 2, // TODO(#6696) - something nonzero, is 1 right? - InterfaceType::Own(_) | InterfaceType::Borrow(_) => 1, - - InterfaceType::Future(_) + InterfaceType::Own(_) + | InterfaceType::Borrow(_) + | InterfaceType::Future(_) | InterfaceType::Stream(_) - | InterfaceType::ErrorContext(_) => { - todo!() - } + | InterfaceType::ErrorContext(_) => 1, }; match self.fuel.checked_sub(cost) { @@ -641,10 +1078,10 @@ impl Compiler<'_, '_> { InterfaceType::Result(t) => self.translate_result(*t, src, dst_ty, dst), InterfaceType::Own(t) => self.translate_own(*t, src, dst_ty, dst), InterfaceType::Borrow(t) => self.translate_borrow(*t, src, dst_ty, dst), - InterfaceType::Future(_) - | InterfaceType::Stream(_) - | InterfaceType::ErrorContext(_) => { - todo!() + InterfaceType::Future(t) => self.translate_future(*t, src, dst_ty, dst), + InterfaceType::Stream(t) => self.translate_stream(*t, src, dst_ty, dst), + InterfaceType::ErrorContext(t) => { + self.translate_error_context(*t, src, dst_ty, dst) } } } @@ -1149,13 +1586,13 @@ impl Compiler<'_, '_> { // the number of code units in the destination). There is no return // value from the transcode function since the encoding should always // work on the first pass. - fn string_copy<'a>( + fn string_copy<'c>( &mut self, src: &WasmString<'_>, src_enc: FE, - dst_opts: &'a Options, + dst_opts: &'c Options, dst_enc: FE, - ) -> WasmString<'a> { + ) -> WasmString<'c> { assert!(dst_enc.width() >= src_enc.width()); self.validate_string_length(src, dst_enc); @@ -1244,12 +1681,12 @@ impl Compiler<'_, '_> { // and dst ptr/len and return how many code units were consumed on both // sides. The amount of code units consumed in the source dictates which // branches are taken in this conversion. - fn string_deflate_to_utf8<'a>( + fn string_deflate_to_utf8<'c>( &mut self, src: &WasmString<'_>, src_enc: FE, - dst_opts: &'a Options, - ) -> WasmString<'a> { + dst_opts: &'c Options, + ) -> WasmString<'c> { self.validate_string_length(src, src_enc); // Optimistically assume that the code unit length of the source is @@ -1425,11 +1862,11 @@ impl Compiler<'_, '_> { // destination should always be big enough to hold the result of the // transcode and so the result of the host function is how many code // units were written to the destination. - fn string_utf8_to_utf16<'a>( + fn string_utf8_to_utf16<'c>( &mut self, src: &WasmString<'_>, - dst_opts: &'a Options, - ) -> WasmString<'a> { + dst_opts: &'c Options, + ) -> WasmString<'c> { self.validate_string_length(src, FE::Utf16); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); @@ -1494,11 +1931,11 @@ impl Compiler<'_, '_> { // string. If the upper bit is set then utf16 was used and the // conversion is done. If the upper bit is not set then latin1 was used // and a downsizing needs to happen. - fn string_compact_utf16_to_compact<'a>( + fn string_compact_utf16_to_compact<'c>( &mut self, src: &WasmString<'_>, - dst_opts: &'a Options, - ) -> WasmString<'a> { + dst_opts: &'c Options, + ) -> WasmString<'c> { self.validate_string_length(src, FE::Utf16); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); @@ -1568,12 +2005,12 @@ impl Compiler<'_, '_> { // failure a larger buffer is allocated for utf16 and then utf16 is // encoded in-place into the buffer. After either latin1 or utf16 the // buffer is then resized to fit the final string allocation. - fn string_to_compact<'a>( + fn string_to_compact<'c>( &mut self, src: &WasmString<'_>, src_enc: FE, - dst_opts: &'a Options, - ) -> WasmString<'a> { + dst_opts: &'c Options, + ) -> WasmString<'c> { self.validate_string_length(src, src_enc); self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr()); let dst_len = self.local_tee_new_tmp(dst_opts.ptr()); @@ -2344,13 +2781,13 @@ impl Compiler<'_, '_> { ); } - fn convert_variant<'a>( + fn convert_variant<'c>( &mut self, src: &Source<'_>, src_info: &VariantInfo, dst: &Destination, dst_info: &VariantInfo, - src_cases: impl ExactSizeIterator>, + src_cases: impl ExactSizeIterator>, ) { // The outermost block is special since it has the result type of the // translation here. That will depend on the `dst`. @@ -2472,6 +2909,51 @@ impl Compiler<'_, '_> { } } + fn translate_future( + &mut self, + src_ty: TypeFutureTableIndex, + src: &Source<'_>, + dst_ty: &InterfaceType, + dst: &Destination, + ) { + let dst_ty = match dst_ty { + InterfaceType::Future(t) => *t, + _ => panic!("expected a `Future`"), + }; + let transfer = self.module.import_future_transfer(); + self.translate_handle(src_ty.as_u32(), src, dst_ty.as_u32(), dst, transfer); + } + + fn translate_stream( + &mut self, + src_ty: TypeStreamTableIndex, + src: &Source<'_>, + dst_ty: &InterfaceType, + dst: &Destination, + ) { + let dst_ty = match dst_ty { + InterfaceType::Stream(t) => *t, + _ => panic!("expected a `Stream`"), + }; + let transfer = self.module.import_stream_transfer(); + self.translate_handle(src_ty.as_u32(), src, dst_ty.as_u32(), dst, transfer); + } + + fn translate_error_context( + &mut self, + src_ty: TypeComponentLocalErrorContextTableIndex, + src: &Source<'_>, + dst_ty: &InterfaceType, + dst: &Destination, + ) { + let dst_ty = match dst_ty { + InterfaceType::ErrorContext(t) => *t, + _ => panic!("expected an `ErrorContext`"), + }; + let transfer = self.module.import_error_context_transfer(); + self.translate_handle(src_ty.as_u32(), src, dst_ty.as_u32(), dst, transfer); + } + fn translate_own( &mut self, src_ty: TypeResourceTableIndex, @@ -2484,7 +2966,7 @@ impl Compiler<'_, '_> { _ => panic!("expected an `Own`"), }; let transfer = self.module.import_resource_transfer_own(); - self.translate_resource(src_ty, src, dst_ty, dst, transfer); + self.translate_handle(src_ty.as_u32(), src, dst_ty.as_u32(), dst, transfer); } fn translate_borrow( @@ -2500,7 +2982,7 @@ impl Compiler<'_, '_> { }; let transfer = self.module.import_resource_transfer_borrow(); - self.translate_resource(src_ty, src, dst_ty, dst, transfer); + self.translate_handle(src_ty.as_u32(), src, dst_ty.as_u32(), dst, transfer); } /// Translates the index `src`, which resides in the table `src_ty`, into @@ -2510,11 +2992,11 @@ impl Compiler<'_, '_> { /// cranelift-generated trampoline to satisfy this import will call. The /// `transfer` function is an imported function which takes the src, src_ty, /// and dst_ty, and returns the dst index. - fn translate_resource( + fn translate_handle( &mut self, - src_ty: TypeResourceTableIndex, + src_ty: u32, src: &Source<'_>, - dst_ty: TypeResourceTableIndex, + dst_ty: u32, dst: &Destination, transfer: FuncIndex, ) { @@ -2523,8 +3005,8 @@ impl Compiler<'_, '_> { Source::Memory(mem) => self.i32_load(mem), Source::Stack(stack) => self.stack_get(stack, ValType::I32), } - self.instruction(I32Const(src_ty.as_u32() as i32)); - self.instruction(I32Const(dst_ty.as_u32() as i32)); + self.instruction(I32Const(src_ty as i32)); + self.instruction(I32Const(dst_ty as i32)); self.instruction(Call(transfer.as_u32())); match dst { Destination::Memory(mem) => self.i32_store(mem), @@ -2597,7 +3079,7 @@ impl Compiler<'_, '_> { self.instruction(End); } - fn malloc<'a>(&mut self, opts: &'a Options, size: MallocSize, align: u32) -> Memory<'a> { + fn malloc<'c>(&mut self, opts: &'c Options, size: MallocSize, align: u32) -> Memory<'c> { let realloc = opts.realloc.unwrap(); self.ptr_uconst(opts, 0); self.ptr_uconst(opts, 0); @@ -2611,7 +3093,7 @@ impl Compiler<'_, '_> { self.memory_operand(opts, addr, align) } - fn memory_operand<'a>(&mut self, opts: &'a Options, addr: TempLocal, align: u32) -> Memory<'a> { + fn memory_operand<'c>(&mut self, opts: &'c Options, addr: TempLocal, align: u32) -> Memory<'c> { let ret = Memory { addr, offset: 0, diff --git a/crates/fuzzing/src/generators/component_types.rs b/crates/fuzzing/src/generators/component_types.rs index 9db7d46c8585..6c868849c04e 100644 --- a/crates/fuzzing/src/generators/component_types.rs +++ b/crates/fuzzing/src/generators/component_types.rs @@ -109,7 +109,9 @@ pub fn arbitrary_val(ty: &component::Type, input: &mut Unstructured) -> arbitrar ), // Resources aren't fuzzed at this time. - Type::Own(_) | Type::Borrow(_) => unreachable!(), + Type::Own(_) | Type::Borrow(_) => { + unreachable!() + } }) } @@ -120,8 +122,25 @@ pub fn static_api_test<'a, P, R>( declarations: &Declarations, ) -> arbitrary::Result<()> where - P: ComponentNamedList + Lift + Lower + Clone + PartialEq + Debug + Arbitrary<'a> + 'static, - R: ComponentNamedList + Lift + Lower + Clone + PartialEq + Debug + Arbitrary<'a> + 'static, + P: ComponentNamedList + + Lift + + Lower + + Clone + + PartialEq + + Debug + + Arbitrary<'a> + + Send + + 'static, + R: ComponentNamedList + + Lift + + Lower + + Clone + + PartialEq + + Debug + + Arbitrary<'a> + + Send + + Sync + + 'static, { crate::init_fuzzing(); @@ -138,7 +157,7 @@ where .root() .func_wrap( IMPORT_FUNCTION, - |cx: StoreContextMut<'_, Box>, params: P| { + |cx: StoreContextMut<'_, Box>, params: P| { log::trace!("received parameters {params:?}"); let data: &(P, R) = cx.data().downcast_ref().unwrap(); let (expected_params, result) = data; @@ -148,7 +167,7 @@ where }, ) .unwrap(); - let mut store: Store> = Store::new(&engine, Box::new(())); + let mut store: Store> = Store::new(&engine, Box::new(())); let instance = linker.instantiate(&mut store, &component).unwrap(); let func = instance .get_typed_func::(&mut store, EXPORT_FUNCTION) diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rs index 92f2b37d5186..4e7577b660e7 100644 --- a/crates/wasmtime/src/runtime/component/concurrent.rs +++ b/crates/wasmtime/src/runtime/component/concurrent.rs @@ -1,8 +1,16 @@ use { - crate::AsContextMut, + crate::{ + store::StoreInner, + vm::{component::ComponentInstance, VMFuncRef, VMMemoryDefinition}, + AsContextMut, ValRaw, + }, anyhow::Result, futures::{stream::FuturesUnordered, FutureExt}, - std::{boxed::Box, future::Future, pin::Pin}, + std::{boxed::Box, future::Future, mem::MaybeUninit, pin::Pin}, + wasmtime_environ::component::{ + RuntimeComponentInstanceIndex, TypeComponentLocalErrorContextTableIndex, + TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex, + }, }; pub use futures_and_streams::{ErrorContext, FutureReader, StreamReader}; @@ -72,3 +80,772 @@ impl PromisesUnordered { todo!() } } + +/// Trait representing component model ABI async intrinsics and fused adapter +/// helper functions. +pub unsafe trait VMComponentAsyncStore { + /// The `task.backpressure` intrinsic. + fn task_backpressure( + &mut self, + caller_instance: RuntimeComponentInstanceIndex, + enabled: u32, + ) -> Result<()>; + + /// The `task.return` intrinsic. + fn task_return( + &mut self, + instance: &mut ComponentInstance, + ty: TypeTupleIndex, + storage: *mut ValRaw, + storage_len: usize, + ) -> Result<()>; + + /// The `task.wait` intrinsic. + fn task_wait( + &mut self, + instance: &mut ComponentInstance, + caller_instance: RuntimeComponentInstanceIndex, + async_: bool, + memory: *mut VMMemoryDefinition, + payload: u32, + ) -> Result; + + /// The `task.poll` intrinsic. + fn task_poll( + &mut self, + instance: &mut ComponentInstance, + caller_instance: RuntimeComponentInstanceIndex, + async_: bool, + memory: *mut VMMemoryDefinition, + payload: u32, + ) -> Result; + + /// The `task.yield` intrinsic. + fn task_yield(&mut self, instance: &mut ComponentInstance, async_: bool) -> Result<()>; + + /// The `subtask.drop` intrinsic. + fn subtask_drop( + &mut self, + instance: &mut ComponentInstance, + caller_instance: RuntimeComponentInstanceIndex, + task_id: u32, + ) -> Result<()>; + + /// A helper function for fused adapter modules involving calls where the + /// caller is sync-lowered but the callee is async-lifted. + fn sync_enter( + &mut self, + start: *mut VMFuncRef, + return_: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + task_return_type: TypeTupleIndex, + result_count: u32, + storage: *mut ValRaw, + storage_len: usize, + ) -> Result<()>; + + /// A helper function for fused adapter modules involving calls where the + /// caller is sync-lowered but the callee is async-lifted. + fn sync_exit( + &mut self, + instance: &mut ComponentInstance, + callback: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + callee: *mut VMFuncRef, + callee_instance: RuntimeComponentInstanceIndex, + param_count: u32, + storage: *mut MaybeUninit, + storage_len: usize, + ) -> Result<()>; + + /// A helper function for fused adapter modules involving calls where the + /// caller is async-lowered. + fn async_enter( + &mut self, + start: *mut VMFuncRef, + return_: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + task_return_type: TypeTupleIndex, + params: u32, + results: u32, + ) -> Result<()>; + + /// A helper function for fused adapter modules involving calls where the + /// caller is async-lowered. + fn async_exit( + &mut self, + instance: &mut ComponentInstance, + callback: *mut VMFuncRef, + post_return: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + callee: *mut VMFuncRef, + callee_instance: RuntimeComponentInstanceIndex, + param_count: u32, + result_count: u32, + flags: u32, + ) -> Result; + + /// The `future.new` intrinsic. + fn future_new( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + ) -> Result; + + /// The `future.write` intrinsic. + fn future_write( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeFutureTableIndex, + future: u32, + address: u32, + ) -> Result; + + /// The `future.read` intrinsic. + fn future_read( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeFutureTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + future: u32, + address: u32, + ) -> Result; + + /// The `future.cancel-write` intrinsic. + fn future_cancel_write( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + async_: bool, + writer: u32, + ) -> Result; + + /// The `future.cancel-read` intrinsic. + fn future_cancel_read( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + async_: bool, + reader: u32, + ) -> Result; + + /// The `future.close-writable` intrinsic. + fn future_close_writable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + writer: u32, + error: u32, + ) -> Result<()>; + + /// The `future.close-readable` intrinsic. + fn future_close_readable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + reader: u32, + ) -> Result<()>; + + /// The `stream.new` intrinsic. + fn stream_new( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + ) -> Result; + + /// The `stream.write` intrinsic. + fn stream_write( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeStreamTableIndex, + stream: u32, + address: u32, + count: u32, + ) -> Result; + + /// The `stream.read` intrinsic. + fn stream_read( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + stream: u32, + address: u32, + count: u32, + ) -> Result; + + /// The `stream.cancel-write` intrinsic. + fn stream_cancel_write( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + async_: bool, + writer: u32, + ) -> Result; + + /// The `stream.cancel-read` intrinsic. + fn stream_cancel_read( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + async_: bool, + reader: u32, + ) -> Result; + + /// The `stream.close-writable` intrinsic. + fn stream_close_writable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + writer: u32, + error: u32, + ) -> Result<()>; + + /// The `stream.close-readable` intrinsic. + fn stream_close_readable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + reader: u32, + ) -> Result<()>; + + /// The "fast-path" implementation of the `stream.write` intrinsic for + /// "flat" (i.e. memcpy-able) payloads. + fn flat_stream_write( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + ty: TypeStreamTableIndex, + payload_size: u32, + payload_align: u32, + stream: u32, + address: u32, + count: u32, + ) -> Result; + + /// The "fast-path" implementation of the `stream.read` intrinsic for "flat" + /// (i.e. memcpy-able) payloads. + fn flat_stream_read( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + payload_size: u32, + payload_align: u32, + stream: u32, + address: u32, + count: u32, + ) -> Result; + + /// The `error-context.new` intrinsic. + fn error_context_new( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeComponentLocalErrorContextTableIndex, + debug_msg_address: u32, + debug_msg_len: u32, + ) -> Result; + + /// The `error-context.debug-message` intrinsic. + fn error_context_debug_message( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeComponentLocalErrorContextTableIndex, + err_ctx_handle: u32, + debug_msg_address: u32, + ) -> Result<()>; + + /// The `error-context.drop` intrinsic. + fn error_context_drop( + &mut self, + instance: &mut ComponentInstance, + ty: TypeComponentLocalErrorContextTableIndex, + err_ctx_handle: u32, + ) -> Result<()>; +} + +unsafe impl VMComponentAsyncStore for StoreInner { + fn task_backpressure( + &mut self, + caller_instance: RuntimeComponentInstanceIndex, + enabled: u32, + ) -> Result<()> { + _ = (caller_instance, enabled); + todo!() + } + + fn task_return( + &mut self, + instance: &mut ComponentInstance, + ty: TypeTupleIndex, + storage: *mut ValRaw, + storage_len: usize, + ) -> Result<()> { + _ = (instance, ty, storage, storage_len); + todo!() + } + + fn task_wait( + &mut self, + instance: &mut ComponentInstance, + caller_instance: RuntimeComponentInstanceIndex, + async_: bool, + memory: *mut VMMemoryDefinition, + payload: u32, + ) -> Result { + _ = (instance, caller_instance, async_, memory, payload); + todo!() + } + + fn task_poll( + &mut self, + instance: &mut ComponentInstance, + caller_instance: RuntimeComponentInstanceIndex, + async_: bool, + memory: *mut VMMemoryDefinition, + payload: u32, + ) -> Result { + _ = (instance, caller_instance, async_, memory, payload); + todo!() + } + + fn task_yield(&mut self, instance: &mut ComponentInstance, async_: bool) -> Result<()> { + _ = (instance, async_); + todo!() + } + + fn subtask_drop( + &mut self, + instance: &mut ComponentInstance, + caller_instance: RuntimeComponentInstanceIndex, + task_id: u32, + ) -> Result<()> { + _ = (instance, caller_instance, task_id); + todo!() + } + + fn sync_enter( + &mut self, + start: *mut VMFuncRef, + return_: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + task_return_type: TypeTupleIndex, + result_count: u32, + storage: *mut ValRaw, + storage_len: usize, + ) -> Result<()> { + _ = ( + start, + return_, + caller_instance, + task_return_type, + result_count, + storage, + storage_len, + ); + todo!() + } + + fn sync_exit( + &mut self, + instance: &mut ComponentInstance, + callback: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + callee: *mut VMFuncRef, + callee_instance: RuntimeComponentInstanceIndex, + param_count: u32, + storage: *mut MaybeUninit, + storage_len: usize, + ) -> Result<()> { + _ = ( + instance, + callback, + caller_instance, + callee, + callee_instance, + param_count, + storage, + storage_len, + ); + todo!() + } + + fn async_enter( + &mut self, + start: *mut VMFuncRef, + return_: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + task_return_type: TypeTupleIndex, + params: u32, + results: u32, + ) -> Result<()> { + _ = ( + start, + return_, + caller_instance, + task_return_type, + params, + results, + ); + todo!() + } + + fn async_exit( + &mut self, + instance: &mut ComponentInstance, + callback: *mut VMFuncRef, + post_return: *mut VMFuncRef, + caller_instance: RuntimeComponentInstanceIndex, + callee: *mut VMFuncRef, + callee_instance: RuntimeComponentInstanceIndex, + param_count: u32, + result_count: u32, + flags: u32, + ) -> Result { + _ = ( + instance, + callback, + post_return, + caller_instance, + callee, + callee_instance, + param_count, + result_count, + flags, + ); + todo!() + } + + fn future_new( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + ) -> Result { + _ = (instance, ty); + todo!() + } + + fn future_write( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeFutureTableIndex, + future: u32, + address: u32, + ) -> Result { + _ = ( + instance, + memory, + realloc, + string_encoding, + ty, + future, + address, + ); + todo!() + } + + fn future_read( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeFutureTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + future: u32, + address: u32, + ) -> Result { + _ = ( + instance, + memory, + realloc, + string_encoding, + ty, + err_ctx_ty, + future, + address, + ); + todo!() + } + + fn future_cancel_write( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + async_: bool, + writer: u32, + ) -> Result { + _ = (instance, ty, async_, writer); + todo!() + } + + fn future_cancel_read( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + async_: bool, + reader: u32, + ) -> Result { + _ = (instance, ty, async_, reader); + todo!() + } + + fn future_close_writable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + writer: u32, + error: u32, + ) -> Result<()> { + _ = (instance, ty, err_ctx_ty, writer, error); + todo!() + } + + fn future_close_readable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeFutureTableIndex, + reader: u32, + ) -> Result<()> { + _ = (instance, ty, reader); + todo!() + } + + fn stream_new( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + ) -> Result { + _ = (instance, ty); + todo!() + } + + fn stream_write( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeStreamTableIndex, + stream: u32, + address: u32, + count: u32, + ) -> Result { + _ = ( + instance, + memory, + realloc, + string_encoding, + ty, + stream, + address, + count, + ); + todo!() + } + + fn stream_read( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + stream: u32, + address: u32, + count: u32, + ) -> Result { + _ = ( + instance, + memory, + realloc, + string_encoding, + ty, + err_ctx_ty, + stream, + address, + count, + ); + todo!() + } + + fn stream_cancel_write( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + async_: bool, + writer: u32, + ) -> Result { + _ = (instance, ty, async_, writer); + todo!() + } + + fn stream_cancel_read( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + async_: bool, + reader: u32, + ) -> Result { + _ = (instance, ty, async_, reader); + todo!() + } + + fn stream_close_writable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + writer: u32, + error: u32, + ) -> Result<()> { + _ = (instance, ty, err_ctx_ty, writer, error); + todo!() + } + + fn stream_close_readable( + &mut self, + instance: &mut ComponentInstance, + ty: TypeStreamTableIndex, + reader: u32, + ) -> Result<()> { + _ = (instance, ty, reader); + todo!() + } + + fn flat_stream_write( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + ty: TypeStreamTableIndex, + payload_size: u32, + payload_align: u32, + stream: u32, + address: u32, + count: u32, + ) -> Result { + _ = ( + instance, + memory, + realloc, + ty, + payload_size, + payload_align, + stream, + address, + count, + ); + todo!() + } + + fn flat_stream_read( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + ty: TypeStreamTableIndex, + err_ctx_ty: TypeComponentLocalErrorContextTableIndex, + payload_size: u32, + payload_align: u32, + stream: u32, + address: u32, + count: u32, + ) -> Result { + _ = ( + instance, + memory, + realloc, + ty, + err_ctx_ty, + payload_size, + payload_align, + stream, + address, + count, + ); + todo!() + } + + fn error_context_new( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeComponentLocalErrorContextTableIndex, + debug_msg_address: u32, + debug_msg_len: u32, + ) -> Result { + _ = ( + instance, + memory, + realloc, + string_encoding, + ty, + debug_msg_address, + debug_msg_len, + ); + todo!() + } + + fn error_context_debug_message( + &mut self, + instance: &mut ComponentInstance, + memory: *mut VMMemoryDefinition, + realloc: *mut VMFuncRef, + string_encoding: u8, + ty: TypeComponentLocalErrorContextTableIndex, + err_ctx_handle: u32, + debug_msg_address: u32, + ) -> Result<()> { + _ = ( + instance, + memory, + realloc, + string_encoding, + ty, + err_ctx_handle, + debug_msg_address, + ); + todo!() + } + + fn error_context_drop( + &mut self, + instance: &mut ComponentInstance, + ty: TypeComponentLocalErrorContextTableIndex, + err_ctx_handle: u32, + ) -> Result<()> { + _ = (instance, ty, err_ctx_handle); + todo!() + } +} diff --git a/crates/wasmtime/src/runtime/component/func/host.rs b/crates/wasmtime/src/runtime/component/func/host.rs index 77e91e892b35..1e8bf946a081 100644 --- a/crates/wasmtime/src/runtime/component/func/host.rs +++ b/crates/wasmtime/src/runtime/component/func/host.rs @@ -48,6 +48,7 @@ impl HostFunc { cx: NonNull, data: NonNull, ty: u32, + _caller_instance: u32, flags: NonNull, memory: *mut VMMemoryDefinition, realloc: *mut VMFuncRef, @@ -443,6 +444,7 @@ extern "C" fn dynamic_entrypoint( cx: NonNull, data: NonNull, ty: u32, + _caller_instance: u32, flags: NonNull, memory: *mut VMMemoryDefinition, realloc: *mut VMFuncRef, diff --git a/crates/wasmtime/src/runtime/component/mod.rs b/crates/wasmtime/src/runtime/component/mod.rs index 1dd845522c6c..ab9f4bf7a66c 100644 --- a/crates/wasmtime/src/runtime/component/mod.rs +++ b/crates/wasmtime/src/runtime/component/mod.rs @@ -115,7 +115,9 @@ pub mod types; mod values; pub use self::component::{Component, ComponentExportIndex}; #[cfg(feature = "component-model-async")] -pub use self::concurrent::{ErrorContext, FutureReader, Promise, PromisesUnordered, StreamReader}; +pub use self::concurrent::{ + ErrorContext, FutureReader, Promise, PromisesUnordered, StreamReader, VMComponentAsyncStore, +}; pub use self::func::{ ComponentNamedList, ComponentType, Func, Lift, Lower, TypedFunc, WasmList, WasmStr, }; @@ -149,6 +151,7 @@ pub mod __internal { pub use alloc::string::String; pub use alloc::vec::Vec; pub use anyhow; + pub use core::cell::RefCell; pub use core::mem::transmute; #[cfg(feature = "async")] pub use trait_variant::make as trait_variant_make; diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 634bea850234..559abf7ac410 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1888,6 +1888,13 @@ at https://bytecodealliance.org/security. } unsafe impl crate::runtime::vm::VMStore for StoreInner { + #[cfg(feature = "component-model-async")] + fn component_async_store( + &mut self, + ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore { + self + } + fn store_opaque(&self) -> &StoreOpaque { &self.inner } diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 0607b9f5566d..189f93dc86a5 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -203,6 +203,11 @@ pub unsafe trait VMStore { /// Metadata required for resources for the component model. #[cfg(feature = "component-model")] fn component_calls(&mut self) -> &mut component::CallContexts; + + #[cfg(feature = "component-model-async")] + fn component_async_store( + &mut self, + ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore; } impl Deref for dyn VMStore + '_ { @@ -235,7 +240,7 @@ impl DerefMut for dyn VMStore + '_ { /// usage of `Instance` and `ComponentInstance` for example. #[derive(Copy, Clone)] #[repr(transparent)] -struct VMStoreRawPtr(NonNull); +struct VMStoreRawPtr(pub NonNull); // SAFETY: this is the purpose of `VMStoreRawPtr`, see docs above about safe // usage. diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs index 2d4aef8938d3..4c9d83b69404 100644 --- a/crates/wasmtime/src/runtime/vm/component.rs +++ b/crates/wasmtime/src/runtime/vm/component.rs @@ -114,6 +114,7 @@ pub type VMLoweringCallee = extern "C" fn( vmctx: NonNull, data: NonNull, ty: u32, + caller_instance: u32, flags: NonNull, opt_memory: *mut VMMemoryDefinition, opt_realloc: *mut VMFuncRef, @@ -664,6 +665,39 @@ impl ComponentInstance { pub(crate) fn resource_exit_call(&mut self) -> Result<()> { self.resource_tables().exit_call() } + + #[cfg(feature = "component-model-async")] + pub(crate) fn future_transfer( + &mut self, + src_idx: u32, + src: TypeFutureTableIndex, + dst: TypeFutureTableIndex, + ) -> Result { + _ = (src_idx, src, dst); + todo!() + } + + #[cfg(feature = "component-model-async")] + pub(crate) fn stream_transfer( + &mut self, + src_idx: u32, + src: TypeStreamTableIndex, + dst: TypeStreamTableIndex, + ) -> Result { + _ = (src_idx, src, dst); + todo!() + } + + #[cfg(feature = "component-model-async")] + pub(crate) fn error_context_transfer( + &mut self, + src_idx: u32, + src: TypeComponentLocalErrorContextTableIndex, + dst: TypeComponentLocalErrorContextTableIndex, + ) -> Result { + _ = (src_idx, src, dst); + todo!() + } } impl VMComponentContext { diff --git a/crates/wasmtime/src/runtime/vm/component/libcalls.rs b/crates/wasmtime/src/runtime/vm/component/libcalls.rs index 4932a479fa2f..9705249697ba 100644 --- a/crates/wasmtime/src/runtime/vm/component/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/component/libcalls.rs @@ -69,7 +69,7 @@ mod trampolines { macro_rules! shims { ( $( - $( #[$attr:meta] )* + $( #[cfg($attr:meta)] )? $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?; )* ) => ( @@ -77,12 +77,19 @@ mod trampolines { pub unsafe extern "C" fn $name( $($pname : signature!(@ty $param),)* ) $( -> signature!(@ty $result))? { - $(shims!(@validate_param $pname $param);)* + $(#[cfg($attr)])? + { + $(shims!(@validate_param $pname $param);)* - let ret = crate::runtime::vm::traphandlers::catch_unwind_and_record_trap(|| { - shims!(@invoke $name() $($pname)*) - }); - shims!(@convert_ret ret $($pname: $param)*) + let ret = crate::runtime::vm::traphandlers::catch_unwind_and_record_trap(|| { + shims!(@invoke $name() $($pname)*) + }); + shims!(@convert_ret ret $($pname: $param)*) + } + $( + #[cfg(not($attr))] + unreachable!(); + )? } )* ); @@ -572,32 +579,650 @@ unsafe fn trap(_vmctx: NonNull, code: u8) -> Result, + caller_instance: u32, + enabled: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .task_backpressure( + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32( + caller_instance, + ), + enabled, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn task_return( + vmctx: NonNull, + ty: u32, + storage: *mut u8, + storage_len: usize, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().task_return( + instance, + wasmtime_environ::component::TypeTupleIndex::from_u32(ty), + storage.cast::(), + storage_len, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn task_wait( + vmctx: NonNull, + caller_instance: u32, + async_: u8, + memory: *mut u8, + payload: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().task_wait( + instance, + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(caller_instance), + async_ != 0, + memory.cast::(), + payload, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn task_poll( + vmctx: NonNull, + caller_instance: u32, + async_: u8, + memory: *mut u8, + payload: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().task_poll( + instance, + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(caller_instance), + async_ != 0, + memory.cast::(), + payload, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn task_yield(vmctx: NonNull, async_: u8) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .task_yield(instance, async_ != 0) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn subtask_drop( + vmctx: NonNull, + caller_instance: u32, + task_id: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().subtask_drop( + instance, + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(caller_instance), + task_id, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn sync_enter( + vmctx: NonNull, + start: *mut u8, + return_: *mut u8, + caller_instance: u32, + task_return_type: u32, + result_count: u32, + storage: *mut u8, + storage_len: usize, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().sync_enter( + start.cast::(), + return_.cast::(), + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(caller_instance), + wasmtime_environ::component::TypeTupleIndex::from_u32(task_return_type), + result_count, + storage.cast::(), + storage_len, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn sync_exit( + vmctx: NonNull, + callback: *mut u8, + caller_instance: u32, + callee: *mut u8, + callee_instance: u32, + param_count: u32, + storage: *mut u8, + storage_len: usize, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().sync_exit( + instance, + callback.cast::(), + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(caller_instance), + callee.cast::(), + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(callee_instance), + param_count, + storage.cast::>(), + storage_len, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn async_enter( + vmctx: NonNull, + start: *mut u8, + return_: *mut u8, + caller_instance: u32, + task_return_type: u32, + params: u32, + results: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().async_enter( + start.cast::(), + return_.cast::(), + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(caller_instance), + wasmtime_environ::component::TypeTupleIndex::from_u32(task_return_type), + params, + results, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn async_exit( + vmctx: NonNull, + callback: *mut u8, + post_return: *mut u8, + caller_instance: u32, + callee: *mut u8, + callee_instance: u32, + param_count: u32, + result_count: u32, + flags: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().async_exit( + instance, + callback.cast::(), + post_return.cast::(), + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(caller_instance), + callee.cast::(), + wasmtime_environ::component::RuntimeComponentInstanceIndex::from_u32(callee_instance), + param_count, + result_count, + flags, + ) + }) +} + +#[cfg(feature = "component-model-async")] unsafe fn future_transfer( vmctx: NonNull, src_idx: u32, src_table: u32, dst_table: u32, ) -> Result { - _ = (vmctx, src_idx, src_table, dst_table); - todo!() + let src_table = wasmtime_environ::component::TypeFutureTableIndex::from_u32(src_table); + let dst_table = wasmtime_environ::component::TypeFutureTableIndex::from_u32(dst_table); + ComponentInstance::from_vmctx(vmctx, |instance| { + instance.future_transfer(src_idx, src_table, dst_table) + }) } +#[cfg(feature = "component-model-async")] unsafe fn stream_transfer( vmctx: NonNull, src_idx: u32, src_table: u32, dst_table: u32, ) -> Result { - _ = (vmctx, src_idx, src_table, dst_table); - todo!() + let src_table = wasmtime_environ::component::TypeStreamTableIndex::from_u32(src_table); + let dst_table = wasmtime_environ::component::TypeStreamTableIndex::from_u32(dst_table); + ComponentInstance::from_vmctx(vmctx, |instance| { + instance.stream_transfer(src_idx, src_table, dst_table) + }) } +#[cfg(feature = "component-model-async")] unsafe fn error_context_transfer( vmctx: NonNull, src_idx: u32, src_table: u32, dst_table: u32, ) -> Result { - _ = (vmctx, src_idx, src_table, dst_table); - todo!() + let src_table = + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32(src_table); + let dst_table = + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32(dst_table); + ComponentInstance::from_vmctx(vmctx, |instance| { + instance.error_context_transfer(src_idx, src_table, dst_table) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn future_new(vmctx: NonNull, ty: u32) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().future_new( + instance, + wasmtime_environ::component::TypeFutureTableIndex::from_u32(ty), + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn future_write( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + string_encoding: u8, + ty: u32, + future: u32, + address: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().future_write( + instance, + memory.cast::(), + realloc.cast::(), + string_encoding, + wasmtime_environ::component::TypeFutureTableIndex::from_u32(ty), + future, + address, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn future_read( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + string_encoding: u8, + ty: u32, + err_ctx_ty: u32, + future: u32, + address: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().future_read( + instance, + memory.cast::(), + realloc.cast::(), + string_encoding, + wasmtime_environ::component::TypeFutureTableIndex::from_u32(ty), + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32( + err_ctx_ty, + ), + future, + address, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn future_cancel_write( + vmctx: NonNull, + ty: u32, + async_: u8, + writer: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .future_cancel_write( + instance, + wasmtime_environ::component::TypeFutureTableIndex::from_u32(ty), + async_ != 0, + writer, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn future_cancel_read( + vmctx: NonNull, + ty: u32, + async_: u8, + reader: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .future_cancel_read( + instance, + wasmtime_environ::component::TypeFutureTableIndex::from_u32(ty), + async_ != 0, + reader, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn future_close_writable( + vmctx: NonNull, + ty: u32, + err_ctx_ty: u32, + writer: u32, + error: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .future_close_writable( + instance, + wasmtime_environ::component::TypeFutureTableIndex::from_u32(ty), + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32( + err_ctx_ty, + ), + writer, + error, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn future_close_readable( + vmctx: NonNull, + ty: u32, + reader: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .future_close_readable( + instance, + wasmtime_environ::component::TypeFutureTableIndex::from_u32(ty), + reader, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn stream_new(vmctx: NonNull, ty: u32) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().stream_new( + instance, + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn stream_write( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + string_encoding: u8, + ty: u32, + stream: u32, + address: u32, + count: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().stream_write( + instance, + memory.cast::(), + realloc.cast::(), + string_encoding, + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + stream, + address, + count, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn stream_read( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + string_encoding: u8, + ty: u32, + err_ctx_ty: u32, + stream: u32, + address: u32, + count: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()).component_async_store().stream_read( + instance, + memory.cast::(), + realloc.cast::(), + string_encoding, + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32( + err_ctx_ty, + ), + stream, + address, + count, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn stream_cancel_write( + vmctx: NonNull, + ty: u32, + async_: u8, + writer: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .stream_cancel_write( + instance, + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + async_ != 0, + writer, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn stream_cancel_read( + vmctx: NonNull, + ty: u32, + async_: u8, + reader: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .stream_cancel_read( + instance, + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + async_ != 0, + reader, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn stream_close_writable( + vmctx: NonNull, + ty: u32, + err_ctx_ty: u32, + writer: u32, + error: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .stream_close_writable( + instance, + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32( + err_ctx_ty, + ), + writer, + error, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn stream_close_readable( + vmctx: NonNull, + ty: u32, + reader: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .stream_close_readable( + instance, + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + reader, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn flat_stream_write( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + ty: u32, + payload_size: u32, + payload_align: u32, + stream: u32, + address: u32, + count: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .flat_stream_write( + instance, + memory.cast::(), + realloc.cast::(), + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + payload_size, + payload_align, + stream, + address, + count, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn flat_stream_read( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + ty: u32, + err_ctx_ty: u32, + payload_size: u32, + payload_align: u32, + stream: u32, + address: u32, + count: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .flat_stream_read( + instance, + memory.cast::(), + realloc.cast::(), + wasmtime_environ::component::TypeStreamTableIndex::from_u32(ty), + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32( + err_ctx_ty, + ), + payload_size, + payload_align, + stream, + address, + count, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn error_context_new( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + string_encoding: u8, + ty: u32, + debug_msg_address: u32, + debug_msg_len: u32, +) -> Result { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .error_context_new( + instance, + memory.cast::(), + realloc.cast::(), + string_encoding, + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32(ty), + debug_msg_address, + debug_msg_len, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn error_context_debug_message( + vmctx: NonNull, + memory: *mut u8, + realloc: *mut u8, + string_encoding: u8, + ty: u32, + err_ctx_handle: u32, + debug_msg_address: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .error_context_debug_message( + instance, + memory.cast::(), + realloc.cast::(), + string_encoding, + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32(ty), + err_ctx_handle, + debug_msg_address, + ) + }) +} + +#[cfg(feature = "component-model-async")] +unsafe fn error_context_drop( + vmctx: NonNull, + ty: u32, + err_ctx_handle: u32, +) -> Result<()> { + ComponentInstance::from_vmctx(vmctx, |instance| { + (*instance.store()) + .component_async_store() + .error_context_drop( + instance, + wasmtime_environ::component::TypeComponentLocalErrorContextTableIndex::from_u32(ty), + err_ctx_handle, + ) + }) } diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs index 740640e8d3b2..5a82b5bd10bb 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -379,16 +379,18 @@ impl InterpreterRef<'_> { use wasmtime_environ::component::ComponentBuiltinFunctionIndex; if id == const { HostCall::ComponentLowerImport.index() } { - call!(@host VMLoweringCallee(nonnull, nonnull, u32, nonnull, ptr, ptr, u8, u8, nonnull, size) -> bool); + call!(@host VMLoweringCallee(nonnull, nonnull, u32, u32, nonnull, ptr, ptr, u8, u8, nonnull, size) -> bool); } macro_rules! component { ( $( + $( #[cfg($attr:meta)] )? $name:ident($($pname:ident: $param:ident ),* ) $(-> $result:ident)?; )* ) => { $( + $( #[cfg($attr)] )? if id == const { HostCall::ComponentBuiltin(ComponentBuiltinFunctionIndex::$name()).index() } { call!(@builtin($($param),*) $(-> $result)?); } diff --git a/tests/misc_testsuite/component-model-async/fused.wast b/tests/misc_testsuite/component-model-async/fused.wast new file mode 100644 index 000000000000..3179c358f340 --- /dev/null +++ b/tests/misc_testsuite/component-model-async/fused.wast @@ -0,0 +1,247 @@ +;;! component_model_async = true +;;! reference_types = true +;;! gc_types = true +;;! multi_memory = true + +;; async lower -> async lift without callback +(component + (component $lifter + (core module $m + (import "" "task.return" (func $task-return (param i32))) + (func (export "foo") (param i32) (call $task-return (local.get 0))) + ) + (core func $task-return (canon task.return (result u32))) + (core instance $i (instantiate $m + (with "" (instance (export "task.return" (func $task-return)))) + )) + + (func (export "foo") (param "p1" u32) (result u32) + (canon lift (core func $i "foo") async) + ) + ) + + (component $lowerer + (import "a" (func $foo (param "p1" u32) (result u32))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $foo (canon lower (func $foo) async (memory $libc "memory"))) + (core module $m + (import "libc" "memory" (memory 1)) + (import "" "foo" (func $foo (param i32 i32) (result i32))) + (func (export "run") + block + (i32.store offset=0 (i32.const 1200) (i32.const 42)) + (call $foo (i32.const 1200) (i32.const 1204)) + (i32.eq (i32.load offset=0 (i32.const 1204)) (i32.const 42)) + br_if 0 + unreachable + end + ) + ) + (core instance $i (instantiate $m + (with "libc" (instance $libc)) + (with "" (instance (export "foo" (func $foo)))) + )) + (func (export "run") (canon lift (core func $i "run"))) + ) + + (instance $lifter (instantiate $lifter)) + (instance $lowerer (instantiate $lowerer (with "a" (func $lifter "foo")))) + (func (export "run") (alias export $lowerer "run")) +) + +;; TODO: this requires async support in `wasmtime-wast`: +;;(assert_return (invoke "run")) + +;; async lower -> async lift with callback +(component + (component $lifter + (core module $m + (import "" "task.return" (func $task-return (param i32))) + (func (export "callback") (param i32 i32 i32 i32) (result i32) unreachable) + (func (export "foo") (param i32) (result i32) + (call $task-return (local.get 0)) + i32.const 0 + ) + ) + (core func $task-return (canon task.return (result u32))) + (core instance $i (instantiate $m + (with "" (instance (export "task.return" (func $task-return)))) + )) + + (func (export "foo") (param "p1" u32) (result u32) + (canon lift (core func $i "foo") async (callback (func $i "callback"))) + ) + ) + + (component $lowerer + (import "a" (func $foo (param "p1" u32) (result u32))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $foo (canon lower (func $foo) async (memory $libc "memory"))) + (core module $m + (import "libc" "memory" (memory 1)) + (import "" "foo" (func $foo (param i32 i32) (result i32))) + (func (export "run") + block + (i32.store offset=0 (i32.const 1200) (i32.const 42)) + (call $foo (i32.const 1200) (i32.const 1204)) + (i32.eq (i32.load offset=0 (i32.const 1204)) (i32.const 42)) + br_if 0 + unreachable + end + ) + ) + (core instance $i (instantiate $m + (with "libc" (instance $libc)) + (with "" (instance (export "foo" (func $foo)))) + )) + (func (export "run") (canon lift (core func $i "run"))) + ) + + (instance $lifter (instantiate $lifter)) + (instance $lowerer (instantiate $lowerer (with "a" (func $lifter "foo")))) + (func (export "run") (alias export $lowerer "run")) +) + +;; TODO: this requires async support in `wasmtime-wast`: +;;(assert_return (invoke "run")) + +;; async lower -> sync lift +(component + (component $lifter + (core module $m + (func (export "foo") (param i32) (result i32) + local.get 0 + ) + ) + (core instance $i (instantiate $m)) + (func (export "foo") (param "p1" u32) (result u32) + (canon lift (core func $i "foo")) + ) + ) + + (component $lowerer + (import "a" (func $foo (param "p1" u32) (result u32))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $foo (canon lower (func $foo) async (memory $libc "memory"))) + (core module $m + (import "libc" "memory" (memory 1)) + (import "" "foo" (func $foo (param i32 i32) (result i32))) + (func (export "run") + block + (i32.store offset=0 (i32.const 1200) (i32.const 42)) + (call $foo (i32.const 1200) (i32.const 1204)) + (i32.eq (i32.load offset=0 (i32.const 1204)) (i32.const 42)) + br_if 0 + unreachable + end + ) + ) + (core instance $i (instantiate $m + (with "libc" (instance $libc)) + (with "" (instance (export "foo" (func $foo)))) + )) + (func (export "run") (canon lift (core func $i "run"))) + ) + + (instance $lifter (instantiate $lifter)) + (instance $lowerer (instantiate $lowerer (with "a" (func $lifter "foo")))) + (func (export "run") (alias export $lowerer "run")) +) + +;; TODO: this requires async support in `wasmtime-wast`: +;;(assert_return (invoke "run")) + +;; sync lower -> async lift without callback +(component + (component $lifter + (core module $m + (import "" "task.return" (func $task-return (param i32))) + (func (export "foo") (param i32) (call $task-return (local.get 0))) + ) + (core func $task-return (canon task.return (result u32))) + (core instance $i (instantiate $m + (with "" (instance (export "task.return" (func $task-return)))) + )) + + (func (export "foo") (param "p1" u32) (result u32) + (canon lift (core func $i "foo") async) + ) + ) + + (component $lowerer + (import "a" (func $foo (param "p1" u32) (result u32))) + (core func $foo (canon lower (func $foo))) + (core module $m + (import "" "foo" (func $foo (param i32) (result i32))) + (func (export "run") + block + (i32.eq (call $foo (i32.const 42)) (i32.const 42)) + br_if 0 + unreachable + end + ) + ) + (core instance $i (instantiate $m + (with "" (instance (export "foo" (func $foo)))) + )) + (func (export "run") (canon lift (core func $i "run"))) + ) + + (instance $lifter (instantiate $lifter)) + (instance $lowerer (instantiate $lowerer (with "a" (func $lifter "foo")))) + (func (export "run") (alias export $lowerer "run")) +) + +;; TODO: this requires async support in `wasmtime-wast`: +;;(assert_return (invoke "run")) + +;; sync lower -> async lift with callback +(component + (component $lifter + (core module $m + (import "" "task.return" (func $task-return (param i32))) + (func (export "callback") (param i32 i32 i32 i32) (result i32) unreachable) + (func (export "foo") (param i32) (result i32) + (call $task-return (local.get 0)) + i32.const 0 + ) + ) + (core func $task-return (canon task.return (result u32))) + (core instance $i (instantiate $m + (with "" (instance (export "task.return" (func $task-return)))) + )) + + (func (export "foo") (param "p1" u32) (result u32) + (canon lift (core func $i "foo") async (callback (func $i "callback"))) + ) + ) + + (component $lowerer + (import "a" (func $foo (param "p1" u32) (result u32))) + (core func $foo (canon lower (func $foo))) + (core module $m + (import "" "foo" (func $foo (param i32) (result i32))) + (func (export "run") + block + (i32.eq (call $foo (i32.const 42)) (i32.const 42)) + br_if 0 + unreachable + end + ) + ) + (core instance $i (instantiate $m + (with "" (instance (export "foo" (func $foo)))) + )) + (func (export "run") (canon lift (core func $i "run"))) + ) + + (instance $lifter (instantiate $lifter)) + (instance $lowerer (instantiate $lowerer (with "a" (func $lifter "foo")))) + (func (export "run") (alias export $lowerer "run")) +) + +;; TODO: this requires async support in `wasmtime-wast`: +;;(assert_return (invoke "run")) diff --git a/tests/misc_testsuite/component-model-async/futures.wast b/tests/misc_testsuite/component-model-async/futures.wast new file mode 100644 index 000000000000..f1e4d4d5b940 --- /dev/null +++ b/tests/misc_testsuite/component-model-async/futures.wast @@ -0,0 +1,90 @@ +;;! component_model_async = true + +;; future.new +(component + (core module $m + (import "" "future.new" (func $future-new (result i32))) + ) + (type $future-type (future u8)) + (core func $future-new (canon future.new $future-type)) + (core instance $i (instantiate $m (with "" (instance (export "future.new" (func $future-new)))))) +) + +;; future.read +(component + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core module $m + (import "" "future.read" (func $future-read (param i32 i32) (result i32))) + ) + (type $future-type (future u8)) + (core func $future-read (canon future.read $future-type async (memory $libc "memory"))) + (core instance $i (instantiate $m (with "" (instance (export "future.read" (func $future-read)))))) +) + +;; future.read; with realloc +(component + (core module $libc + (func (export "realloc") (param i32 i32 i32 i32) (result i32) unreachable) + (memory (export "memory") 1) + ) + (core instance $libc (instantiate $libc)) + (core module $m + (import "" "future.read" (func $future-read (param i32 i32) (result i32))) + ) + (type $future-type (future string)) + (core func $future-read (canon future.read $future-type async (memory $libc "memory") (realloc (func $libc "realloc")))) + (core instance $i (instantiate $m (with "" (instance (export "future.read" (func $future-read)))))) +) + +;; future.write +(component + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core module $m + (import "" "future.write" (func $future-write (param i32 i32) (result i32))) + ) + (type $future-type (future u8)) + (core func $future-write (canon future.write $future-type async (memory $libc "memory"))) + (core instance $i (instantiate $m (with "" (instance (export "future.write" (func $future-write)))))) +) + +;; future.cancel-read +(component + (core module $m + (import "" "future.cancel-read" (func $future-cancel-read (param i32) (result i32))) + ) + (type $future-type (future u8)) + (core func $future-cancel-read (canon future.cancel-read $future-type async)) + (core instance $i (instantiate $m (with "" (instance (export "future.cancel-read" (func $future-cancel-read)))))) +) + +;; future.cancel-write +(component + (core module $m + (import "" "future.cancel-write" (func $future-cancel-write (param i32) (result i32))) + ) + (type $future-type (future u8)) + (core func $future-cancel-write (canon future.cancel-write $future-type async)) + (core instance $i (instantiate $m (with "" (instance (export "future.cancel-write" (func $future-cancel-write)))))) +) + +;; future.close-readable +(component + (core module $m + (import "" "future.close-readable" (func $future-close-readable (param i32))) + ) + (type $future-type (future u8)) + (core func $future-close-readable (canon future.close-readable $future-type)) + (core instance $i (instantiate $m (with "" (instance (export "future.close-readable" (func $future-close-readable)))))) +) + +;; future.close-writable +(component + (core module $m + (import "" "future.close-writable" (func $future-close-writable (param i32 i32))) + ) + (type $future-type (future u8)) + (core func $future-close-writable (canon future.close-writable $future-type)) + (core instance $i (instantiate $m (with "" (instance (export "future.close-writable" (func $future-close-writable)))))) +) diff --git a/tests/misc_testsuite/component-model-async/streams.wast b/tests/misc_testsuite/component-model-async/streams.wast new file mode 100644 index 000000000000..790ddec7e5f8 --- /dev/null +++ b/tests/misc_testsuite/component-model-async/streams.wast @@ -0,0 +1,90 @@ +;;! component_model_async = true + +;; stream.new +(component + (core module $m + (import "" "stream.new" (func $stream-new (result i32))) + ) + (type $stream-type (stream u8)) + (core func $stream-new (canon stream.new $stream-type)) + (core instance $i (instantiate $m (with "" (instance (export "stream.new" (func $stream-new)))))) +) + +;; stream.read +(component + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core module $m + (import "" "stream.read" (func $stream-read (param i32 i32 i32) (result i32))) + ) + (type $stream-type (stream u8)) + (core func $stream-read (canon stream.read $stream-type async (memory $libc "memory"))) + (core instance $i (instantiate $m (with "" (instance (export "stream.read" (func $stream-read)))))) +) + +;; stream.read; with realloc +(component + (core module $libc + (func (export "realloc") (param i32 i32 i32 i32) (result i32) unreachable) + (memory (export "memory") 1) + ) + (core instance $libc (instantiate $libc)) + (core module $m + (import "" "stream.read" (func $stream-read (param i32 i32 i32) (result i32))) + ) + (type $stream-type (stream string)) + (core func $stream-read (canon stream.read $stream-type async (memory $libc "memory") (realloc (func $libc "realloc")))) + (core instance $i (instantiate $m (with "" (instance (export "stream.read" (func $stream-read)))))) +) + +;; stream.write +(component + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core module $m + (import "" "stream.write" (func $stream-write (param i32 i32 i32) (result i32))) + ) + (type $stream-type (stream u8)) + (core func $stream-write (canon stream.write $stream-type async (memory $libc "memory"))) + (core instance $i (instantiate $m (with "" (instance (export "stream.write" (func $stream-write)))))) +) + +;; stream.cancel-read +(component + (core module $m + (import "" "stream.cancel-read" (func $stream-cancel-read (param i32) (result i32))) + ) + (type $stream-type (stream u8)) + (core func $stream-cancel-read (canon stream.cancel-read $stream-type async)) + (core instance $i (instantiate $m (with "" (instance (export "stream.cancel-read" (func $stream-cancel-read)))))) +) + +;; stream.cancel-write +(component + (core module $m + (import "" "stream.cancel-write" (func $stream-cancel-write (param i32) (result i32))) + ) + (type $stream-type (stream u8)) + (core func $stream-cancel-write (canon stream.cancel-write $stream-type async)) + (core instance $i (instantiate $m (with "" (instance (export "stream.cancel-write" (func $stream-cancel-write)))))) +) + +;; stream.close-readable +(component + (core module $m + (import "" "stream.close-readable" (func $stream-close-readable (param i32))) + ) + (type $stream-type (stream u8)) + (core func $stream-close-readable (canon stream.close-readable $stream-type)) + (core instance $i (instantiate $m (with "" (instance (export "stream.close-readable" (func $stream-close-readable)))))) +) + +;; stream.close-writable +(component + (core module $m + (import "" "stream.close-writable" (func $stream-close-writable (param i32 i32))) + ) + (type $stream-type (stream u8)) + (core func $stream-close-writable (canon stream.close-writable $stream-type)) + (core instance $i (instantiate $m (with "" (instance (export "stream.close-writable" (func $stream-close-writable)))))) +) From 32f9b359f649f1f84b54ae68a81ddcc689f72cc8 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Wed, 5 Mar 2025 09:28:48 -0700 Subject: [PATCH 2/2] address review feedback Signed-off-by: Joel Dice --- crates/cranelift/src/compiler/component.rs | 402 ++++++++---------- crates/environ/src/component/types.rs | 2 +- crates/environ/src/fact.rs | 28 +- .../fuzzing/src/generators/component_types.rs | 29 +- crates/wasmtime/src/runtime/vm/component.rs | 4 +- 5 files changed, 208 insertions(+), 257 deletions(-) diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index ae312558008f..24a197fd2507 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -8,8 +8,9 @@ use cranelift_codegen::isa::{CallConv, TargetIsa}; use cranelift_frontend::FunctionBuilder; use std::any::Any; use wasmtime_environ::component::*; +use wasmtime_environ::fact::SYNC_ENTER_FIXED_PARAMS; use wasmtime_environ::{ - HostCall, ModuleInternedTypeIndex, PtrSize, Tunables, WasmFuncType, WasmValType, + HostCall, ModuleInternedTypeIndex, PtrSize, TrapSentinel, Tunables, WasmFuncType, WasmValType, }; struct TrampolineCompiler<'a> { @@ -31,6 +32,9 @@ enum Abi { Array, } +type GetLibcallFn = + fn(&dyn TargetIsa, &mut ir::Function) -> (ir::SigRef, ComponentBuiltinFunctionIndex); + impl<'a> TrampolineCompiler<'a> { fn new( compiler: &'a Compiler, @@ -123,7 +127,7 @@ impl<'a> TrampolineCompiler<'a> { &[ty.as_u32()], None, host::stream_new, - ir::types::I64, + TrapSentinel::NegativeOne, ), Trampoline::StreamRead { ty, @@ -131,27 +135,27 @@ impl<'a> TrampolineCompiler<'a> { options, } => { let tys = &[ty.as_u32(), err_ctx_ty.as_u32()]; - if let Some(info) = self.flat_stream_element_info(*ty) { + if let Some(info) = self.flat_stream_element_info(*ty).cloned() { self.translate_flat_stream_call(tys, options, host::flat_stream_read, &info) } else { self.translate_future_or_stream_call( tys, Some(options), host::stream_read, - ir::types::I64, + TrapSentinel::NegativeOne, ) } } Trampoline::StreamWrite { ty, options } => { let tys = &[ty.as_u32()]; - if let Some(info) = self.flat_stream_element_info(*ty) { + if let Some(info) = self.flat_stream_element_info(*ty).cloned() { self.translate_flat_stream_call(tys, options, host::flat_stream_write, &info) } else { self.translate_future_or_stream_call( tys, Some(options), host::stream_write, - ir::types::I64, + TrapSentinel::NegativeOne, ) } } @@ -165,20 +169,20 @@ impl<'a> TrampolineCompiler<'a> { &[ty.as_u32()], None, host::stream_close_readable, - ir::types::I8, + TrapSentinel::Falsy, ), Trampoline::StreamCloseWritable { ty, err_ctx_ty } => self .translate_future_or_stream_call( &[ty.as_u32(), err_ctx_ty.as_u32()], None, host::stream_close_writable, - ir::types::I8, + TrapSentinel::Falsy, ), Trampoline::FutureNew { ty } => self.translate_future_or_stream_call( &[ty.as_u32()], None, host::future_new, - ir::types::I64, + TrapSentinel::NegativeOne, ), Trampoline::FutureRead { ty, @@ -188,13 +192,13 @@ impl<'a> TrampolineCompiler<'a> { &[ty.as_u32(), err_ctx_ty.as_u32()], Some(&options), host::future_read, - ir::types::I64, + TrapSentinel::NegativeOne, ), Trampoline::FutureWrite { ty, options } => self.translate_future_or_stream_call( &[ty.as_u32()], Some(options), host::future_write, - ir::types::I64, + TrapSentinel::NegativeOne, ), Trampoline::FutureCancelRead { ty, async_ } => { self.translate_cancel_call(ty.as_u32(), *async_, host::future_cancel_read) @@ -206,27 +210,27 @@ impl<'a> TrampolineCompiler<'a> { &[ty.as_u32()], None, host::future_close_readable, - ir::types::I8, + TrapSentinel::Falsy, ), Trampoline::FutureCloseWritable { ty, err_ctx_ty } => self .translate_future_or_stream_call( &[ty.as_u32(), err_ctx_ty.as_u32()], None, host::future_close_writable, - ir::types::I8, + TrapSentinel::Falsy, ), Trampoline::ErrorContextNew { ty, options } => self.translate_error_context_call( *ty, options, host::error_context_new, - ir::types::I64, + TrapSentinel::NegativeOne, ), Trampoline::ErrorContextDebugMessage { ty, options } => self .translate_error_context_call( *ty, options, host::error_context_debug_message, - ir::types::I8, + TrapSentinel::Falsy, ), Trampoline::ErrorContextDrop { ty } => self.translate_error_context_drop_call(*ty), Trampoline::ResourceTransferOwn => { @@ -250,7 +254,7 @@ impl<'a> TrampolineCompiler<'a> { Trampoline::SyncEnterCall => self.translate_sync_enter(), Trampoline::SyncExitCall { callback } => self.translate_sync_exit(*callback), Trampoline::AsyncEnterCall => { - self.translate_async_enter_or_exit(host::async_enter, None, ir::types::I8) + self.translate_async_enter_or_exit(host::async_enter, None, TrapSentinel::Falsy) } Trampoline::AsyncExitCall { callback, @@ -258,7 +262,7 @@ impl<'a> TrampolineCompiler<'a> { } => self.translate_async_enter_or_exit( host::async_exit, Some((*callback, *post_return)), - ir::types::I64, + TrapSentinel::NegativeOne, ), Trampoline::FutureTransfer => { self.translate_host_libcall(host::future_transfer, |me, rets| { @@ -278,16 +282,17 @@ impl<'a> TrampolineCompiler<'a> { } } - fn flat_stream_element_info(&self, ty: TypeStreamTableIndex) -> Option { + /// Determine whether the specified type can be optimized as a stream + /// payload by lifting and lowering with a simple `memcpy`. + /// + /// Any type containing only "flat", primitive data (i.e. no pointers or + /// handles) should qualify for this optimization, but it's also okay to + /// conservatively return `None` here; the fallback slow path will always + /// work -- it just won't be as efficient. + fn flat_stream_element_info(&self, ty: TypeStreamTableIndex) -> Option<&CanonicalAbiInfo> { let payload = self.types[self.types[ty].ty].payload; match payload { - None => Some(CanonicalAbiInfo { - align32: 1, - align64: 1, - flat_count: None, - size32: 0, - size64: 0, - }), + None => Some(&CanonicalAbiInfo::ZERO), Some( payload @ (InterfaceType::Bool | InterfaceType::S8 @@ -301,7 +306,7 @@ impl<'a> TrampolineCompiler<'a> { | InterfaceType::Float32 | InterfaceType::Float64 | InterfaceType::Char), - ) => Some(self.types.canonical_abi(&payload).clone()), + ) => Some(self.types.canonical_abi(&payload)), // TODO: Recursively check for other "flat" types (i.e. those without pointers or handles), // e.g. `record`s, `variant`s, etc. which contain only flat types. _ => None, @@ -334,12 +339,9 @@ impl<'a> TrampolineCompiler<'a> { fn translate_intrinsic_libcall( &mut self, vmctx: ir::Value, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + get_libcall: GetLibcallFn, args: &[ir::Value], - result: ir::types::Type, + sentinel: TrapSentinel, ) { match self.abi { Abi::Wasm => {} @@ -350,6 +352,8 @@ impl<'a> TrampolineCompiler<'a> { // to support that here (except for `sync-enter`, `sync-exit`, // `async-enter`, and `async-exit`, which are only ever called // from FACT-generated Wasm code and never exported). + // + // https://github.com/bytecodealliance/wasmtime/issues/10143 self.builder.ins().trap(TRAP_INTERNAL_ASSERT); return; } @@ -357,17 +361,17 @@ impl<'a> TrampolineCompiler<'a> { let call = self.call_libcall(vmctx, get_libcall, args); - if result == ir::types::I64 { - let result = self.builder.func.dfg.inst_results(call)[0]; - let result = self.raise_if_negative_one(result); - self.abi_store_results(&[result]); - } else { - if result != ir::types::I8 { - todo!("support additional intrinsic return types") + let result = self.builder.func.dfg.inst_results(call)[0]; + match sentinel { + TrapSentinel::NegativeOne => { + let result = self.raise_if_negative_one(result); + self.abi_store_results(&[result]); + } + TrapSentinel::Falsy => { + self.raise_if_host_trapped(result); + self.builder.ins().return_(&[]); } - let succeeded = self.builder.func.dfg.inst_results(call)[0]; - self.raise_if_host_trapped(succeeded); - self.builder.ins().return_(&[]); + _ => todo!("support additional return types if/when necessary"), } } @@ -386,7 +390,7 @@ impl<'a> TrampolineCompiler<'a> { vmctx, host::task_return, &[vmctx, ty, values_vec_ptr, values_vec_len], - ir::types::I8, + TrapSentinel::Falsy, ); } @@ -408,7 +412,7 @@ impl<'a> TrampolineCompiler<'a> { let pointer_type = self.isa.pointer_type(); let wasm_func_ty = &self.types[self.signature].unwrap_func(); - let param_offset = 5; + let param_offset = SYNC_ENTER_FIXED_PARAMS.len(); let spill_offset = param_offset + 2; let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args( @@ -434,7 +438,12 @@ impl<'a> TrampolineCompiler<'a> { callee_args.push(values_vec_ptr); callee_args.push(values_vec_len); - self.translate_intrinsic_libcall(vmctx, host::sync_enter, &callee_args, ir::types::I8); + self.translate_intrinsic_libcall( + vmctx, + host::sync_enter, + &callee_args, + TrapSentinel::Falsy, + ); } fn translate_sync_exit(&mut self, callback: Option) { @@ -454,19 +463,7 @@ impl<'a> TrampolineCompiler<'a> { let vmctx = args[0]; let wasm_func_ty = &self.types[self.signature].unwrap_func(); - let mut callee_args = vec![ - vmctx, - if let Some(callback) = callback { - self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_callback(callback)).unwrap(), - ) - } else { - self.builder.ins().iconst(pointer_type, 0) - }, - ]; + let mut callee_args = vec![vmctx, self.load_callback(vmctx, callback)]; // remaining non-Wasm parameters callee_args.extend(args[2..].iter().copied()); @@ -501,15 +498,12 @@ impl<'a> TrampolineCompiler<'a> { fn translate_async_enter_or_exit( &mut self, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + get_libcall: GetLibcallFn, callback_and_post_return: Option<( Option, Option, )>, - result: ir::types::Type, + sentinel: TrapSentinel, ) { match self.abi { Abi::Wasm => {} @@ -528,37 +522,16 @@ impl<'a> TrampolineCompiler<'a> { let mut callee_args = vec![vmctx]; if let Some((callback, post_return)) = callback_and_post_return { - let pointer_type = self.isa.pointer_type(); - // callback: *mut VMFuncRef - if let Some(callback) = callback { - callee_args.push(self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_callback(callback)).unwrap(), - )); - } else { - callee_args.push(self.builder.ins().iconst(pointer_type, 0)); - } - + callee_args.push(self.load_callback(vmctx, callback)); // post_return: *mut VMFuncRef - if let Some(post_return) = post_return { - callee_args.push(self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_post_return(post_return)).unwrap(), - )); - } else { - callee_args.push(self.builder.ins().iconst(pointer_type, 0)); - } + callee_args.push(self.load_post_return(vmctx, post_return)); } // remaining parameters callee_args.extend(args[2..].iter().copied()); - self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, result); + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, sentinel); } fn translate_task_backpressure_call(&mut self, caller_instance: RuntimeComponentInstanceIndex) { @@ -578,7 +551,7 @@ impl<'a> TrampolineCompiler<'a> { vmctx, host::task_backpressure, &callee_args, - ir::types::I8, + TrapSentinel::Falsy, ); } @@ -587,12 +560,8 @@ impl<'a> TrampolineCompiler<'a> { caller_instance: RuntimeComponentInstanceIndex, async_: bool, memory: RuntimeMemoryIndex, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + get_libcall: GetLibcallFn, ) { - let pointer_type = self.isa.pointer_type(); let args = self.builder.func.dfg.block_params(self.block0).to_vec(); let vmctx = args[0]; @@ -604,17 +573,17 @@ impl<'a> TrampolineCompiler<'a> { self.builder .ins() .iconst(ir::types::I8, if async_ { 1 } else { 0 }), - self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_memory(memory)).unwrap(), - ), + self.load_memory(vmctx, memory), ]; callee_args.extend(args[2..].iter().copied()); - self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, ir::types::I64); + self.translate_intrinsic_libcall( + vmctx, + get_libcall, + &callee_args, + TrapSentinel::NegativeOne, + ); } fn translate_task_yield_call(&mut self, async_: bool) { @@ -628,7 +597,12 @@ impl<'a> TrampolineCompiler<'a> { .iconst(ir::types::I8, if async_ { 1 } else { 0 }), ]; - self.translate_intrinsic_libcall(vmctx, host::task_yield, &callee_args, ir::types::I8); + self.translate_intrinsic_libcall( + vmctx, + host::task_yield, + &callee_args, + TrapSentinel::Falsy, + ); } fn translate_subtask_drop_call(&mut self, caller_instance: RuntimeComponentInstanceIndex) { @@ -644,7 +618,12 @@ impl<'a> TrampolineCompiler<'a> { callee_args.extend(args[2..].iter().copied()); - self.translate_intrinsic_libcall(vmctx, host::subtask_drop, &callee_args, ir::types::I8); + self.translate_intrinsic_libcall( + vmctx, + host::subtask_drop, + &callee_args, + TrapSentinel::Falsy, + ); } fn translate_lower_import( @@ -718,26 +697,13 @@ impl<'a> TrampolineCompiler<'a> { // memory: *mut VMMemoryDefinition host_sig.params.push(ir::AbiParam::new(pointer_type)); callee_args.push(match memory { - Some(idx) => self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_memory(idx)).unwrap(), - ), + Some(idx) => self.load_memory(vmctx, idx), None => self.builder.ins().iconst(pointer_type, 0), }); // realloc: *mut VMFuncRef host_sig.params.push(ir::AbiParam::new(pointer_type)); - callee_args.push(match realloc { - Some(idx) => self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), - ), - None => self.builder.ins().iconst(pointer_type, 0), - }); + callee_args.push(self.load_realloc(vmctx, realloc)); // A post-return option is only valid on `canon.lift`'d functions so no // valid component should have this specified for a lowering which this @@ -746,11 +712,7 @@ impl<'a> TrampolineCompiler<'a> { // string_encoding: StringEncoding host_sig.params.push(ir::AbiParam::new(ir::types::I8)); - callee_args.push( - self.builder - .ins() - .iconst(ir::types::I8, i64::from(string_encoding as u8)), - ); + callee_args.push(self.string_encoding(string_encoding)); // async_: bool host_sig.params.push(ir::AbiParam::new(ir::types::I8)); @@ -1085,10 +1047,7 @@ impl<'a> TrampolineCompiler<'a> { /// from the wasm abi to host. fn translate_host_libcall( &mut self, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + get_libcall: GetLibcallFn, handle_results: fn(&mut Self, &mut Vec), ) { match self.abi { @@ -1113,15 +1072,7 @@ impl<'a> TrampolineCompiler<'a> { self.builder.ins().return_(&results); } - fn translate_cancel_call( - &mut self, - ty: u32, - async_: bool, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), - ) { + fn translate_cancel_call(&mut self, ty: u32, async_: bool, get_libcall: GetLibcallFn) { let args = self.builder.func.dfg.block_params(self.block0).to_vec(); let vmctx = args[0]; let mut callee_args = vec![ @@ -1134,50 +1085,98 @@ impl<'a> TrampolineCompiler<'a> { callee_args.extend(args[2..].iter().copied()); - self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, ir::types::I64); + self.translate_intrinsic_libcall( + vmctx, + get_libcall, + &callee_args, + TrapSentinel::NegativeOne, + ); + } + + fn load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value { + self.builder.ins().load( + self.isa.pointer_type(), + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_memory(memory)).unwrap(), + ) + } + + fn load_realloc( + &mut self, + vmctx: ir::Value, + realloc: Option, + ) -> ir::Value { + let pointer_type = self.isa.pointer_type(); + match realloc { + Some(idx) => self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), + ), + None => self.builder.ins().iconst(pointer_type, 0), + } + } + + fn load_callback( + &mut self, + vmctx: ir::Value, + callback: Option, + ) -> ir::Value { + let pointer_type = self.isa.pointer_type(); + match callback { + Some(idx) => self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_callback(idx)).unwrap(), + ), + None => self.builder.ins().iconst(pointer_type, 0), + } + } + + fn load_post_return( + &mut self, + vmctx: ir::Value, + post_return: Option, + ) -> ir::Value { + let pointer_type = self.isa.pointer_type(); + match post_return { + Some(idx) => self.builder.ins().load( + pointer_type, + MemFlags::trusted(), + vmctx, + i32::try_from(self.offsets.runtime_post_return(idx)).unwrap(), + ), + None => self.builder.ins().iconst(pointer_type, 0), + } + } + + fn string_encoding(&mut self, string_encoding: StringEncoding) -> ir::Value { + self.builder + .ins() + .iconst(ir::types::I8, i64::from(string_encoding as u8)) } fn translate_future_or_stream_call( &mut self, tys: &[u32], options: Option<&CanonicalOptions>, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), - result: ir::types::Type, + get_libcall: GetLibcallFn, + sentinel: TrapSentinel, ) { - let pointer_type = self.isa.pointer_type(); let args = self.builder.func.dfg.block_params(self.block0).to_vec(); let vmctx = args[0]; let mut callee_args = vec![vmctx]; if let Some(options) = options { // memory: *mut VMMemoryDefinition - callee_args.push(self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_memory(options.memory.unwrap())).unwrap(), - )); - + callee_args.push(self.load_memory(vmctx, options.memory.unwrap())); // realloc: *mut VMFuncRef - callee_args.push(match options.realloc { - Some(idx) => self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), - ), - None => self.builder.ins().iconst(pointer_type, 0), - }); - + callee_args.push(self.load_realloc(vmctx, options.realloc)); // string_encoding: StringEncoding - callee_args.push( - self.builder - .ins() - .iconst(ir::types::I8, i64::from(options.string_encoding as u8)), - ); + callee_args.push(self.string_encoding(options.string_encoding)) } for ty in tys { @@ -1186,39 +1185,22 @@ impl<'a> TrampolineCompiler<'a> { callee_args.extend(args[2..].iter().copied()); - self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, result); + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, sentinel); } fn translate_flat_stream_call( &mut self, tys: &[u32], options: &CanonicalOptions, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + get_libcall: GetLibcallFn, info: &CanonicalAbiInfo, ) { - let pointer_type = self.isa.pointer_type(); let args = self.builder.func.dfg.block_params(self.block0).to_vec(); let vmctx = args[0]; let mut callee_args = vec![ vmctx, - self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_memory(options.memory.unwrap())).unwrap(), - ), - match options.realloc { - Some(idx) => self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), - ), - None => self.builder.ins().iconst(pointer_type, 0), - }, + self.load_memory(vmctx, options.memory.unwrap()), + self.load_realloc(vmctx, options.realloc), ]; for ty in tys { callee_args.push(self.builder.ins().iconst(ir::types::I32, i64::from(*ty))); @@ -1235,42 +1217,28 @@ impl<'a> TrampolineCompiler<'a> { callee_args.extend(args[2..].iter().copied()); - self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, ir::types::I64); + self.translate_intrinsic_libcall( + vmctx, + get_libcall, + &callee_args, + TrapSentinel::NegativeOne, + ); } fn translate_error_context_call( &mut self, ty: TypeComponentLocalErrorContextTableIndex, options: &CanonicalOptions, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), - result: ir::types::Type, + get_libcall: GetLibcallFn, + sentinel: TrapSentinel, ) { - let pointer_type = self.isa.pointer_type(); let args = self.builder.func.dfg.block_params(self.block0).to_vec(); let vmctx = args[0]; let mut callee_args = vec![ vmctx, - self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_memory(options.memory.unwrap())).unwrap(), - ), - match options.realloc { - Some(idx) => self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_realloc(idx)).unwrap(), - ), - None => self.builder.ins().iconst(pointer_type, 0), - }, - self.builder - .ins() - .iconst(ir::types::I8, i64::from(options.string_encoding as u8)), + self.load_memory(vmctx, options.memory.unwrap()), + self.load_realloc(vmctx, options.realloc), + self.string_encoding(options.string_encoding), self.builder .ins() .iconst(ir::types::I32, i64::from(ty.as_u32())), @@ -1278,7 +1246,7 @@ impl<'a> TrampolineCompiler<'a> { callee_args.extend(args[2..].iter().copied()); - self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, result); + self.translate_intrinsic_libcall(vmctx, get_libcall, &callee_args, sentinel); } fn translate_error_context_drop_call(&mut self, ty: TypeComponentLocalErrorContextTableIndex) { @@ -1297,7 +1265,7 @@ impl<'a> TrampolineCompiler<'a> { vmctx, host::error_context_drop, &callee_args, - ir::types::I8, + TrapSentinel::Falsy, ); } @@ -1403,10 +1371,7 @@ impl<'a> TrampolineCompiler<'a> { fn call_libcall( &mut self, vmctx: ir::Value, - get_libcall: fn( - &dyn TargetIsa, - &mut ir::Function, - ) -> (ir::SigRef, ComponentBuiltinFunctionIndex), + get_libcall: GetLibcallFn, args: &[ir::Value], ) -> ir::Inst { let (host_sig, index) = get_libcall(self.isa, &mut self.builder.func); @@ -1653,12 +1618,7 @@ impl TrampolineCompiler<'_> { fn load_runtime_memory_base(&mut self, vmctx: ir::Value, mem: RuntimeMemoryIndex) -> ir::Value { let pointer_type = self.isa.pointer_type(); - let from_vmmemory_definition = self.builder.ins().load( - pointer_type, - MemFlags::trusted(), - vmctx, - i32::try_from(self.offsets.runtime_memory(mem)).unwrap(), - ); + let from_vmmemory_definition = self.load_memory(vmctx, mem); self.builder.ins().load( pointer_type, MemFlags::trusted(), diff --git a/crates/environ/src/component/types.rs b/crates/environ/src/component/types.rs index a0be404da8d1..2915bcd680d2 100644 --- a/crates/environ/src/component/types.rs +++ b/crates/environ/src/component/types.rs @@ -644,7 +644,7 @@ const fn max(a: u32, b: u32) -> u32 { impl CanonicalAbiInfo { /// ABI information for zero-sized types. - const ZERO: CanonicalAbiInfo = CanonicalAbiInfo { + pub const ZERO: CanonicalAbiInfo = CanonicalAbiInfo { size32: 0, align32: 1, size64: 0, diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rs index 7d25884e803b..d5d692579d6c 100644 --- a/crates/environ/src/fact.rs +++ b/crates/environ/src/fact.rs @@ -43,6 +43,19 @@ mod traps; /// callee is an async-lifted export. pub const EXIT_FLAG_ASYNC_CALLEE: i32 = 1 << 0; +/// Fixed parameter types for the `sync-enter` built-in function. +/// +/// Note that `sync-enter` also takes a variable number of parameters in +/// addition to these, determined by the signature of the function for which +/// we're generating an adapter. +pub static SYNC_ENTER_FIXED_PARAMS: &[ValType] = &[ + ValType::FUNCREF, + ValType::FUNCREF, + ValType::I32, + ValType::I32, + ValType::I32, +]; + /// Representation of an adapter module. pub struct Module<'a> { /// Whether or not debug code is inserted into the adapters themselves. @@ -489,16 +502,11 @@ impl<'a> Module<'a> { self.import_simple_get_and_set( "sync", &format!("[enter-call]{suffix}"), - &[ - ValType::FUNCREF, - ValType::FUNCREF, - ValType::I32, - ValType::I32, - ValType::I32, - ] - .into_iter() - .chain(params.iter().copied()) - .collect::>(), + &SYNC_ENTER_FIXED_PARAMS + .iter() + .copied() + .chain(params.iter().copied()) + .collect::>(), &[], Import::SyncEnterCall, |me| me.imported_sync_enter_call.get(suffix).copied(), diff --git a/crates/fuzzing/src/generators/component_types.rs b/crates/fuzzing/src/generators/component_types.rs index 6c868849c04e..9db7d46c8585 100644 --- a/crates/fuzzing/src/generators/component_types.rs +++ b/crates/fuzzing/src/generators/component_types.rs @@ -109,9 +109,7 @@ pub fn arbitrary_val(ty: &component::Type, input: &mut Unstructured) -> arbitrar ), // Resources aren't fuzzed at this time. - Type::Own(_) | Type::Borrow(_) => { - unreachable!() - } + Type::Own(_) | Type::Borrow(_) => unreachable!(), }) } @@ -122,25 +120,8 @@ pub fn static_api_test<'a, P, R>( declarations: &Declarations, ) -> arbitrary::Result<()> where - P: ComponentNamedList - + Lift - + Lower - + Clone - + PartialEq - + Debug - + Arbitrary<'a> - + Send - + 'static, - R: ComponentNamedList - + Lift - + Lower - + Clone - + PartialEq - + Debug - + Arbitrary<'a> - + Send - + Sync - + 'static, + P: ComponentNamedList + Lift + Lower + Clone + PartialEq + Debug + Arbitrary<'a> + 'static, + R: ComponentNamedList + Lift + Lower + Clone + PartialEq + Debug + Arbitrary<'a> + 'static, { crate::init_fuzzing(); @@ -157,7 +138,7 @@ where .root() .func_wrap( IMPORT_FUNCTION, - |cx: StoreContextMut<'_, Box>, params: P| { + |cx: StoreContextMut<'_, Box>, params: P| { log::trace!("received parameters {params:?}"); let data: &(P, R) = cx.data().downcast_ref().unwrap(); let (expected_params, result) = data; @@ -167,7 +148,7 @@ where }, ) .unwrap(); - let mut store: Store> = Store::new(&engine, Box::new(())); + let mut store: Store> = Store::new(&engine, Box::new(())); let instance = linker.instantiate(&mut store, &component).unwrap(); let func = instance .get_typed_func::(&mut store, EXPORT_FUNCTION) diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs index 4c9d83b69404..c3ace55dc8e9 100644 --- a/crates/wasmtime/src/runtime/vm/component.rs +++ b/crates/wasmtime/src/runtime/vm/component.rs @@ -86,6 +86,8 @@ pub struct ComponentInstance { /// which this function pointer was registered. /// * `ty` - the type index, relative to the tables in `vmctx`, that is the /// type of the function being called. +/// * `caller_instance` - the `RuntimeComponentInstanceIndex` representing the +/// caller component instance, used to track the owner of an async host task. /// * `flags` - the component flags for may_enter/leave corresponding to the /// component instance that the lowering happened within. /// * `opt_memory` - this nullable pointer represents the memory configuration @@ -106,7 +108,7 @@ pub struct ComponentInstance { /// or not. On failure this function records trap information in TLS which /// should be suitable for reading later. // -// FIXME: 9 arguments is probably too many. The `data` through `string-encoding` +// FIXME: 11 arguments is probably too many. The `data` through `string-encoding` // parameters should probably get packaged up into the `VMComponentContext`. // Needs benchmarking one way or another though to figure out what the best // balance is here.