diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 37f87cc21511..69589363c941 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -196,7 +196,6 @@ cache = ["dep:wasmtime-cache", "std"] # `async fn` and calling functions asynchronously. async = [ "dep:wasmtime-fiber", - "dep:async-trait", "wasmtime-component-macro?/async", "runtime", ] @@ -261,6 +260,7 @@ runtime = [ "dep:windows-sys", "pulley-interpreter/interp", "dep:wasmtime-unwinder", + "dep:async-trait", ] # Enable support for garbage collection-related things. diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rs index b60d43eab21c..6f78c47aef02 100644 --- a/crates/wasmtime/src/runtime/component/instance.rs +++ b/crates/wasmtime/src/runtime/component/instance.rs @@ -634,7 +634,7 @@ impl<'a> Instantiator<'a> { } } - fn run(&mut self, store: &mut StoreContextMut<'_, T>) -> Result<()> { + async fn run(&mut self, store: &mut StoreContextMut<'_, T>) -> Result<()> { let env_component = self.component.env_component(); // Before all initializers are processed configure all destructors for @@ -714,7 +714,7 @@ impl<'a> Instantiator<'a> { // if required. let i = unsafe { - crate::Instance::new_started_impl(store, module, imports.as_ref())? + crate::Instance::new_started(store, module, imports.as_ref()).await? }; self.instance_mut(store.0).push_instance_id(i.id()); } @@ -991,7 +991,7 @@ impl InstancePre { !store.as_context().async_support(), "must use async instantiation when async support is enabled" ); - self.instantiate_impl(store) + vm::assert_ready(self._instantiate(store)) } /// Performs the instantiation process into the store specified. /// @@ -999,29 +999,18 @@ impl InstancePre { // // TODO: needs more docs #[cfg(feature = "async")] - pub async fn instantiate_async( - &self, - mut store: impl AsContextMut, - ) -> Result - where - T: Send, - { - let mut store = store.as_context_mut(); - assert!( - store.0.async_support(), - "must use sync instantiation when async support is disabled" - ); - store.on_fiber(|store| self.instantiate_impl(store)).await? + pub async fn instantiate_async(&self, store: impl AsContextMut) -> Result { + self._instantiate(store).await } - fn instantiate_impl(&self, mut store: impl AsContextMut) -> Result { + async fn _instantiate(&self, mut store: impl AsContextMut) -> Result { let mut store = store.as_context_mut(); store .engine() .allocator() .increment_component_instance_count()?; let mut instantiator = Instantiator::new(&self.component, store.0, &self.imports); - instantiator.run(&mut store).map_err(|e| { + instantiator.run(&mut store).await.map_err(|e| { store .engine() .allocator() diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs index 30a8ec5615e6..789560c8f214 100644 --- a/crates/wasmtime/src/runtime/externals/table.rs +++ b/crates/wasmtime/src/runtime/externals/table.rs @@ -94,7 +94,8 @@ impl Table { /// # } /// ``` pub fn new(mut store: impl AsContextMut, ty: TableType, init: Ref) -> Result { - Table::_new(store.as_context_mut().0, ty, init) + vm::one_poll(Table::_new(store.as_context_mut().0, ty, init)) + .expect("must use `new_async` when async resource limiters are in use") } /// Async variant of [`Table::new`]. You must use this variant with @@ -111,18 +112,11 @@ impl Table { ty: TableType, init: Ref, ) -> Result
{ - let mut store = store.as_context_mut(); - assert!( - store.0.async_support(), - "cannot use `new_async` without enabling async support on the config" - ); - store - .on_fiber(|store| Table::_new(store.0, ty, init)) - .await? + Table::_new(store.as_context_mut().0, ty, init).await } - fn _new(store: &mut StoreOpaque, ty: TableType, init: Ref) -> Result
{ - let table = generate_table_export(store, &ty)?; + async fn _new(store: &mut StoreOpaque, ty: TableType, init: Ref) -> Result
{ + let table = generate_table_export(store, &ty).await?; table._fill(store, 0, init, ty.minimum())?; Ok(table) } diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs index bc7adff0e862..d92e4e2145c1 100644 --- a/crates/wasmtime/src/runtime/instance.rs +++ b/crates/wasmtime/src/runtime/instance.rs @@ -117,7 +117,8 @@ impl Instance { // Note that the unsafety here should be satisfied by the call to // `typecheck_externs` above which satisfies the condition that all // the imports are valid for this module. - unsafe { Instance::new_started(&mut store, module, imports.as_ref()) } + assert!(!store.0.async_support()); + vm::assert_ready(unsafe { Instance::new_started(&mut store, module, imports.as_ref()) }) } /// Same as [`Instance::new`], except for usage in [asynchronous stores]. @@ -200,7 +201,7 @@ impl Instance { let mut store = store.as_context_mut(); let imports = Instance::typecheck_externs(store.0, module, imports)?; // See `new` for notes on this unsafety - unsafe { Instance::new_started_async(&mut store, module, imports.as_ref()).await } + unsafe { Instance::new_started(&mut store, module, imports.as_ref()).await } } fn typecheck_externs( @@ -242,62 +243,31 @@ impl Instance { /// Internal function to create an instance and run the start function. /// /// This function's unsafety is the same as `Instance::new_raw`. - pub(crate) unsafe fn new_started( - store: &mut StoreContextMut<'_, T>, - module: &Module, - imports: Imports<'_>, - ) -> Result { - assert!( - !store.0.async_support(), - "must use async instantiation when async support is enabled", - ); - - // SAFETY: the safety contract of `new_started_impl` is the same as this - // function. - unsafe { Self::new_started_impl(store, module, imports) } - } - - /// Internal function to create an instance and run the start function. - /// - /// ONLY CALL THIS IF YOU HAVE ALREADY CHECKED FOR ASYNCNESS AND HANDLED - /// THE FIBER NONSENSE - pub(crate) unsafe fn new_started_impl( + pub(crate) async unsafe fn new_started( store: &mut StoreContextMut<'_, T>, module: &Module, imports: Imports<'_>, ) -> Result { // SAFETY: the safety contract of `new_raw` is the same as this // function. - let (instance, start) = unsafe { Instance::new_raw(store.0, module, imports)? }; + let (instance, start) = unsafe { Instance::new_raw(store.0, module, imports).await? }; if let Some(start) = start { - instance.start_raw(store, start)?; + if store.0.async_support() { + #[cfg(feature = "async")] + { + store + .on_fiber(|store| instance.start_raw(store, start)) + .await??; + } + #[cfg(not(feature = "async"))] + unreachable!(); + } else { + instance.start_raw(store, start)?; + } } Ok(instance) } - /// Internal function to create an instance and run the start function. - /// - /// This function's unsafety is the same as `Instance::new_raw`. - #[cfg(feature = "async")] - async unsafe fn new_started_async( - store: &mut StoreContextMut<'_, T>, - module: &Module, - imports: Imports<'_>, - ) -> Result { - assert!( - store.0.async_support(), - "must use sync instantiation when async support is disabled", - ); - - store - .on_fiber(|store| { - // SAFETY: the unsafe contract of `new_started_impl` is the same - // as this function. - unsafe { Self::new_started_impl(store, module, imports) } - }) - .await? - } - /// Internal function to create an instance which doesn't have its `start` /// function run yet. /// @@ -313,7 +283,7 @@ impl Instance { /// This method is unsafe because it does not type-check the `imports` /// provided. The `imports` provided must be suitable for the module /// provided as well. - unsafe fn new_raw( + async unsafe fn new_raw( store: &mut StoreOpaque, module: &Module, imports: Imports<'_>, @@ -325,7 +295,7 @@ impl Instance { // Allocate the GC heap, if necessary. if module.env_module().needs_gc_heap { - store.ensure_gc_store()?; + store.ensure_gc_store().await?; } let compiled_module = module.compiled_module(); @@ -341,11 +311,13 @@ impl Instance { // SAFETY: this module, by construction, was already validated within // the store. let id = unsafe { - store.allocate_instance( - AllocateInstanceKind::Module(module_id), - &ModuleRuntimeInfo::Module(module.clone()), - imports, - )? + store + .allocate_instance( + AllocateInstanceKind::Module(module_id), + &ModuleRuntimeInfo::Module(module.clone()), + imports, + ) + .await? }; // Additionally, before we start doing fallible instantiation, we @@ -377,24 +349,7 @@ impl Instance { .features() .contains(WasmFeatures::BULK_MEMORY); - if store.async_support() { - #[cfg(feature = "async")] - store.block_on(|store| { - let module = compiled_module.module().clone(); - Box::pin( - async move { vm::initialize_instance(store, id, &module, bulk_memory).await }, - ) - })??; - #[cfg(not(feature = "async"))] - unreachable!(); - } else { - vm::assert_ready(vm::initialize_instance( - store, - id, - compiled_module.module(), - bulk_memory, - ))?; - } + vm::initialize_instance(store, id, compiled_module.module(), bulk_memory).await?; Ok((instance, compiled_module.module().start_func)) } @@ -905,7 +860,10 @@ impl InstancePre { // This unsafety should be handled by the type-checking performed by the // constructor of `InstancePre` to assert that all the imports we're passing // in match the module we're instantiating. - unsafe { Instance::new_started(&mut store, &self.module, imports.as_ref()) } + assert!(!store.0.async_support()); + vm::assert_ready(unsafe { + Instance::new_started(&mut store, &self.module, imports.as_ref()) + }) } /// Creates a new instance, running the start function asynchronously @@ -935,7 +893,7 @@ impl InstancePre { // This unsafety should be handled by the type-checking performed by the // constructor of `InstancePre` to assert that all the imports we're passing // in match the module we're instantiating. - unsafe { Instance::new_started_async(&mut store, &self.module, imports.as_ref()).await } + unsafe { Instance::new_started(&mut store, &self.module, imports.as_ref()).await } } } diff --git a/crates/wasmtime/src/runtime/memory.rs b/crates/wasmtime/src/runtime/memory.rs index d593bd35bedc..c2ab4d4164c9 100644 --- a/crates/wasmtime/src/runtime/memory.rs +++ b/crates/wasmtime/src/runtime/memory.rs @@ -260,7 +260,8 @@ impl Memory { /// # } /// ``` pub fn new(mut store: impl AsContextMut, ty: MemoryType) -> Result { - Self::_new(store.as_context_mut().0, ty) + vm::one_poll(Self::_new(store.as_context_mut().0, ty)) + .expect("must use `new_async` when async resource limiters are in use") } /// Async variant of [`Memory::new`]. You must use this variant with @@ -273,17 +274,12 @@ impl Memory { /// [`Store`](`crate::Store`). #[cfg(feature = "async")] pub async fn new_async(mut store: impl AsContextMut, ty: MemoryType) -> Result { - let mut store = store.as_context_mut(); - assert!( - store.0.async_support(), - "cannot use `new_async` without enabling async support on the config" - ); - store.on_fiber(|store| Self::_new(store.0, ty)).await? + Self::_new(store.as_context_mut().0, ty).await } /// Helper function for attaching the memory to a "frankenstein" instance - fn _new(store: &mut StoreOpaque, ty: MemoryType) -> Result { - generate_memory_export(store, &ty, None) + async fn _new(store: &mut StoreOpaque, ty: MemoryType) -> Result { + generate_memory_export(store, &ty, None).await } /// Returns the underlying type of this memory. @@ -1005,7 +1001,10 @@ impl SharedMemory { /// Construct a single-memory instance to provide a way to import /// [`SharedMemory`] into other modules. pub(crate) fn vmimport(&self, store: &mut StoreOpaque) -> crate::runtime::vm::VMMemoryImport { - generate_memory_export(store, &self.ty(), Some(&self.vm)) + // Note `vm::assert_ready` shouldn't panic here because this isn't + // actually allocating any new memory so resource limiting shouldn't + // kick in. + vm::assert_ready(generate_memory_export(store, &self.ty(), Some(&self.vm))) .unwrap() .vmimport(store) } diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index d7c99f8cb3ca..60719641f3ce 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -93,8 +93,7 @@ use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::{ self, GcStore, Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, Interpreter, InterpreterRef, ModuleRuntimeInfo, OnDemandInstanceAllocator, SendSyncPtr, - SignalHandler, StoreBox, StorePtr, Unwind, VMContext, VMFuncRef, VMGcRef, VMStore, - VMStoreContext, + SignalHandler, StoreBox, Unwind, VMContext, VMFuncRef, VMGcRef, VMStore, VMStoreContext, }; use crate::trampoline::VMHostGlobalContext; use crate::{Engine, Module, Trap, Val, ValRaw, module::ModuleRegistry}; @@ -470,6 +469,20 @@ pub struct StoreOpaque { executor: Executor, } +/// Self-pointer to `StoreInner` from within a `StoreOpaque` which is chiefly +/// used to copy into instances during instantiation. +/// +/// FIXME: ideally this type would get deleted and Wasmtime's reliance on it +/// would go away. +struct StorePtr(Option>); + +// We can't make `VMStore: Send + Sync` because that requires making all of +// Wastime's internals generic over the `Store`'s `T`. So instead, we take care +// in the whole VM layer to only use the `VMStore` in ways that are `Send`- and +// `Sync`-safe and we have to have these unsafe impls. +unsafe impl Send for StorePtr {} +unsafe impl Sync for StorePtr {} + /// Executor state within `StoreOpaque`. /// /// Effectively stores Pulley interpreter state and handles conditional support @@ -646,7 +659,7 @@ impl Store { fuel_reserve: 0, fuel_yield_interval: None, store_data, - traitobj: StorePtr::empty(), + traitobj: StorePtr(None), default_caller_vmctx: SendSyncPtr::new(NonNull::dangling()), hostcall_val_storage: Vec::new(), wasm_val_raw_storage: Vec::new(), @@ -670,7 +683,7 @@ impl Store { data: ManuallyDrop::new(data), }); - inner.traitobj = StorePtr::new(NonNull::from(&mut *inner)); + inner.traitobj = StorePtr(Some(NonNull::from(&mut *inner))); // Wasmtime uses the callee argument to host functions to learn about // the original pointer to the `Store` itself, allowing it to @@ -688,15 +701,17 @@ impl Store { .unwrap(); unsafe { - let id = inner - .allocate_instance( - AllocateInstanceKind::Dummy { - allocator: &allocator, - }, - &shim, - Default::default(), - ) - .expect("failed to allocate default callee"); + // Note that this dummy instance doesn't allocate tables or memories + // so it won't have an async await point meaning that it should be + // ok to assert the future is always ready. + let id = vm::assert_ready(inner.allocate_instance( + AllocateInstanceKind::Dummy { + allocator: &allocator, + }, + &shim, + Default::default(), + )) + .expect("failed to allocate default callee"); let default_caller_vmctx = inner.instance(id).vmctx(); inner.default_caller_vmctx = default_caller_vmctx.into(); } @@ -1487,15 +1502,15 @@ impl StoreOpaque { /// `ResourceLimiterAsync` which means that this should only be executed /// in a fiber context at this time. #[inline] - pub(crate) fn ensure_gc_store(&mut self) -> Result<&mut GcStore> { + pub(crate) async fn ensure_gc_store(&mut self) -> Result<&mut GcStore> { if self.gc_store.is_some() { return Ok(self.gc_store.as_mut().unwrap()); } - self.allocate_gc_store() + self.allocate_gc_store().await } #[inline(never)] - fn allocate_gc_store(&mut self) -> Result<&mut GcStore> { + async fn allocate_gc_store(&mut self) -> Result<&mut GcStore> { log::trace!("allocating GC heap for store {:?}", self.id()); assert!(self.gc_store.is_none()); @@ -1505,19 +1520,24 @@ impl StoreOpaque { ); assert_eq!(self.vm_store_context.gc_heap.current_length(), 0); - let vmstore = self.traitobj(); - let gc_store = allocate_gc_store(self.engine(), vmstore, self.get_pkey())?; + let gc_store = allocate_gc_store(self).await?; self.vm_store_context.gc_heap = gc_store.vmmemory_definition(); return Ok(self.gc_store.insert(gc_store)); #[cfg(feature = "gc")] - fn allocate_gc_store( - engine: &Engine, - vmstore: NonNull, - pkey: Option, - ) -> Result { + async fn allocate_gc_store(store: &mut StoreOpaque) -> Result { use wasmtime_environ::packed_option::ReservedValue; + // FIXME(#11409) this is not a sound widening borrow + let (mut limiter, store) = unsafe { + store + .traitobj() + .as_mut() + .resource_limiter_and_store_opaque() + }; + + let engine = store.engine(); + let mem_ty = engine.tunables().gc_heap_memory_type(); ensure!( engine.features().gc_types(), "cannot allocate a GC store when GC is disabled at configuration time" @@ -1530,19 +1550,14 @@ impl StoreOpaque { wasmtime_environ::Module::default(), )), imports: vm::Imports::default(), - store: StorePtr::new(vmstore), - #[cfg(feature = "wmemcheck")] - wmemcheck: false, - pkey, - tunables: engine.tunables(), + store, + limiter: limiter.as_mut(), }; - let mem_ty = engine.tunables().gc_heap_memory_type(); - let tunables = engine.tunables(); - let (mem_alloc_index, mem) = - engine - .allocator() - .allocate_memory(&mut request, &mem_ty, tunables, None)?; + let (mem_alloc_index, mem) = engine + .allocator() + .allocate_memory(&mut request, &mem_ty, None) + .await?; // Then, allocate the actual GC heap, passing in that memory // storage. @@ -1558,11 +1573,7 @@ impl StoreOpaque { } #[cfg(not(feature = "gc"))] - fn allocate_gc_store( - _engine: &Engine, - _vmstore: NonNull, - _pkey: Option, - ) -> Result { + async fn allocate_gc_store(_: &mut StoreOpaque) -> Result { bail!("cannot allocate a GC store: the `gc` feature was disabled at compile time") } } @@ -1942,7 +1953,7 @@ impl StoreOpaque { #[inline] pub fn traitobj(&self) -> NonNull { - self.traitobj.as_raw().unwrap() + self.traitobj.0.unwrap() } /// Takes the cached `Vec` stored internally across hostcalls to get @@ -2089,6 +2100,7 @@ at https://bytecodealliance.org/security. /// Retrieve the store's protection key. #[inline] + #[cfg(feature = "pooling-allocator")] pub(crate) fn get_pkey(&self) -> Option { self.pkey } @@ -2208,7 +2220,7 @@ at https://bytecodealliance.org/security. /// /// The `imports` provided must be correctly sized/typed for the module /// being allocated. - pub(crate) unsafe fn allocate_instance( + pub(crate) async unsafe fn allocate_instance( &mut self, kind: AllocateInstanceKind<'_>, runtime_info: &ModuleRuntimeInfo, @@ -2223,16 +2235,17 @@ at https://bytecodealliance.org/security. // SAFETY: this function's own contract is the same as // `allocate_module`, namely the imports provided are valid. let handle = unsafe { - allocator.allocate_module(InstanceAllocationRequest { - id, - runtime_info, - imports, - store: StorePtr::new(self.traitobj()), - #[cfg(feature = "wmemcheck")] - wmemcheck: self.engine().config().wmemcheck, - pkey: self.get_pkey(), - tunables: self.engine().tunables(), - })? + // FIXME(#11409) this is not a sound widening borrow + let (mut limiter, store) = self.traitobj().as_mut().resource_limiter_and_store_opaque(); + allocator + .allocate_module(InstanceAllocationRequest { + id, + runtime_info, + imports, + store, + limiter: limiter.as_mut(), + }) + .await? }; let actual = match kind { @@ -2314,82 +2327,6 @@ unsafe impl VMStore for StoreInner { ) } - fn memory_growing( - &mut self, - current: usize, - desired: usize, - maximum: Option, - ) -> Result { - match self.limiter { - Some(ResourceLimiterInner::Sync(ref mut limiter)) => { - limiter(&mut self.data).memory_growing(current, desired, maximum) - } - #[cfg(feature = "async")] - Some(ResourceLimiterInner::Async(_)) => self.block_on(|store| { - let limiter = match &mut store.0.limiter { - Some(ResourceLimiterInner::Async(limiter)) => limiter, - _ => unreachable!(), - }; - limiter(&mut store.0.data).memory_growing(current, desired, maximum) - })?, - None => Ok(true), - } - } - - fn memory_grow_failed(&mut self, error: anyhow::Error) -> Result<()> { - match self.limiter { - Some(ResourceLimiterInner::Sync(ref mut limiter)) => { - limiter(&mut self.data).memory_grow_failed(error) - } - #[cfg(feature = "async")] - Some(ResourceLimiterInner::Async(ref mut limiter)) => { - limiter(&mut self.data).memory_grow_failed(error) - } - None => { - log::debug!("ignoring memory growth failure error: {error:?}"); - Ok(()) - } - } - } - - fn table_growing( - &mut self, - current: usize, - desired: usize, - maximum: Option, - ) -> Result { - match self.limiter { - Some(ResourceLimiterInner::Sync(ref mut limiter)) => { - limiter(&mut self.data).table_growing(current, desired, maximum) - } - #[cfg(feature = "async")] - Some(ResourceLimiterInner::Async(_)) => self.block_on(|store| { - let limiter = match &mut store.0.limiter { - Some(ResourceLimiterInner::Async(limiter)) => limiter, - _ => unreachable!(), - }; - limiter(&mut store.0.data).table_growing(current, desired, maximum) - })?, - None => Ok(true), - } - } - - fn table_grow_failed(&mut self, error: anyhow::Error) -> Result<()> { - match self.limiter { - Some(ResourceLimiterInner::Sync(ref mut limiter)) => { - limiter(&mut self.data).table_grow_failed(error) - } - #[cfg(feature = "async")] - Some(ResourceLimiterInner::Async(ref mut limiter)) => { - limiter(&mut self.data).table_grow_failed(error) - } - None => { - log::debug!("ignoring table growth failure: {error:?}"); - Ok(()) - } - } - } - fn out_of_gas(&mut self) -> Result<()> { if !self.refuel() { return Err(Trap::OutOfFuel.into()); diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs index 1947b164a370..1ca8613a98f5 100644 --- a/crates/wasmtime/src/runtime/store/gc.rs +++ b/crates/wasmtime/src/runtime/store/gc.rs @@ -179,7 +179,7 @@ impl StoreOpaque { !self.async_support(), "use the `*_async` versions of methods when async is configured" ); - self.ensure_gc_store()?; + vm::assert_ready(self.ensure_gc_store())?; match alloc_func(self, value) { Ok(x) => Ok(x), Err(e) => match e.downcast::>() { @@ -208,7 +208,7 @@ impl StoreOpaque { where T: Send + Sync + 'static, { - self.ensure_gc_store()?; + self.ensure_gc_store().await?; match alloc_func(self, value) { Ok(x) => Ok(x), Err(e) => match e.downcast::>() { diff --git a/crates/wasmtime/src/runtime/trampoline.rs b/crates/wasmtime/src/runtime/trampoline.rs index c3b7e50f9d8a..58b821b6443d 100644 --- a/crates/wasmtime/src/runtime/trampoline.rs +++ b/crates/wasmtime/src/runtime/trampoline.rs @@ -19,21 +19,21 @@ use crate::store::StoreOpaque; use crate::{MemoryType, TableType, TagType}; use wasmtime_environ::{MemoryIndex, TableIndex, TagIndex}; -pub fn generate_memory_export( +pub async fn generate_memory_export( store: &mut StoreOpaque, m: &MemoryType, preallocation: Option<&SharedMemory>, ) -> Result { let id = store.id(); - let instance = create_memory(store, m, preallocation)?; + let instance = create_memory(store, m, preallocation).await?; Ok(store .instance_mut(instance) .get_exported_memory(id, MemoryIndex::from_u32(0))) } -pub fn generate_table_export(store: &mut StoreOpaque, t: &TableType) -> Result { +pub async fn generate_table_export(store: &mut StoreOpaque, t: &TableType) -> Result { let id = store.id(); - let instance = create_table(store, t)?; + let instance = create_table(store, t).await?; Ok(store .instance_mut(instance) .get_exported_table(id, TableIndex::from_u32(0))) diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rs index 471de1bcbac8..caae5a753654 100644 --- a/crates/wasmtime/src/runtime/trampoline/memory.rs +++ b/crates/wasmtime/src/runtime/trampoline/memory.rs @@ -24,7 +24,7 @@ use wasmtime_environ::{ /// This separate instance is necessary because Wasm objects in Wasmtime must be /// attached to instances (versus the store, e.g.) and some objects exist /// outside: a host-provided memory import, shared memory. -pub fn create_memory( +pub async fn create_memory( store: &mut StoreOpaque, memory_ty: &MemoryType, preallocation: Option<&SharedMemory>, @@ -52,13 +52,15 @@ pub fn create_memory( ondemand: OnDemandInstanceAllocator::default(), }; unsafe { - store.allocate_instance( - AllocateInstanceKind::Dummy { - allocator: &allocator, - }, - &ModuleRuntimeInfo::bare(Arc::new(module)), - Default::default(), - ) + store + .allocate_instance( + AllocateInstanceKind::Dummy { + allocator: &allocator, + }, + &ModuleRuntimeInfo::bare(Arc::new(module)), + Default::default(), + ) + .await } } @@ -122,6 +124,7 @@ struct SingleMemoryInstance<'a> { ondemand: OnDemandInstanceAllocator, } +#[async_trait::async_trait] unsafe impl InstanceAllocator for SingleMemoryInstance<'_> { #[cfg(feature = "component-model")] fn validate_component<'a>( @@ -165,11 +168,10 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> { self.ondemand.decrement_core_instance_count(); } - fn allocate_memory( + async fn allocate_memory( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Memory, - tunables: &Tunables, memory_index: Option, ) -> Result<(MemoryAllocationIndex, Memory)> { if cfg!(debug_assertions) { @@ -184,9 +186,11 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> { MemoryAllocationIndex::default(), shared_memory.clone().as_memory(), )), - None => self - .ondemand - .allocate_memory(request, ty, tunables, memory_index), + None => { + self.ondemand + .allocate_memory(request, ty, memory_index) + .await + } } } @@ -202,14 +206,13 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> { } } - fn allocate_table( + async fn allocate_table( &self, - req: &mut InstanceAllocationRequest, + req: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Table, - tunables: &Tunables, table_index: DefinedTableIndex, ) -> Result<(TableAllocationIndex, Table)> { - self.ondemand.allocate_table(req, ty, tunables, table_index) + self.ondemand.allocate_table(req, ty, table_index).await } unsafe fn deallocate_table( diff --git a/crates/wasmtime/src/runtime/trampoline/table.rs b/crates/wasmtime/src/runtime/trampoline/table.rs index 5b7528143b25..2efd313d9e28 100644 --- a/crates/wasmtime/src/runtime/trampoline/table.rs +++ b/crates/wasmtime/src/runtime/trampoline/table.rs @@ -5,7 +5,7 @@ use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque}; use alloc::sync::Arc; use wasmtime_environ::{EntityIndex, Module, TypeTrace}; -pub fn create_table(store: &mut StoreOpaque, table: &TableType) -> Result { +pub async fn create_table(store: &mut StoreOpaque, table: &TableType) -> Result { let mut module = Module::new(); let wasmtime_table = *table.wasmtime_table(); @@ -29,15 +29,17 @@ pub fn create_table(store: &mut StoreOpaque, table: &TableType) -> Result Result { let allocator = OnDemandInstanceAllocator::new(store.engine().config().mem_creator.clone(), 0, false); let module = Arc::new(module); - store.allocate_instance( + + // Note that `assert_ready` should be valid here because this module + // doesn't allocate tables or memories meaning it shouldn't need an + // await point. + vm::assert_ready(store.allocate_instance( AllocateInstanceKind::Dummy { allocator: &allocator, }, &ModuleRuntimeInfo::bare_with_registered_type(module, Some(func_ty)), imports, - ) + )) } } diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 57ff167daf49..2492cd71b1f6 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -99,8 +99,7 @@ pub use crate::runtime::vm::gc::*; pub use crate::runtime::vm::imports::Imports; pub use crate::runtime::vm::instance::{ GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, - MemoryAllocationIndex, OnDemandInstanceAllocator, StorePtr, TableAllocationIndex, - initialize_instance, + MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, initialize_instance, }; #[cfg(feature = "pooling-allocator")] pub use crate::runtime::vm::instance::{ @@ -209,36 +208,6 @@ pub unsafe trait VMStore: 'static { &mut self, ) -> (Option>, &mut StoreOpaque); - /// Callback invoked to allow the store's resource limiter to reject a - /// memory grow operation. - fn memory_growing( - &mut self, - current: usize, - desired: usize, - maximum: Option, - ) -> Result; - - /// Callback invoked to notify the store's resource limiter that a memory - /// grow operation has failed. - /// - /// Note that this is not invoked if `memory_growing` returns an error. - fn memory_grow_failed(&mut self, error: Error) -> Result<()>; - - /// Callback invoked to allow the store's resource limiter to reject a - /// table grow operation. - fn table_growing( - &mut self, - current: usize, - desired: usize, - maximum: Option, - ) -> Result; - - /// Callback invoked to notify the store's resource limiter that a table - /// grow operation has failed. - /// - /// Note that this is not invoked if `table_growing` returns an error. - fn table_grow_failed(&mut self, error: Error) -> Result<()>; - /// Callback invoked whenever fuel runs out by a wasm instance. If an error /// is returned that's raised as a trap. Otherwise wasm execution will /// continue as normal. diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs index f1e8d87561ab..a7ba3625e38a 100644 --- a/crates/wasmtime/src/runtime/vm/instance.rs +++ b/crates/wasmtime/src/runtime/vm/instance.rs @@ -174,7 +174,7 @@ impl Instance { dropped_data, #[cfg(feature = "wmemcheck")] wmemcheck_state: { - if req.wmemcheck { + if req.store.engine().config().wmemcheck { let size = memory_tys .iter() .next() @@ -487,32 +487,22 @@ impl Instance { unsafe { self.vmctx_plus_offset_mut(offset) } } - pub(crate) unsafe fn set_store(mut self: Pin<&mut Self>, store: Option>) { + pub(crate) unsafe fn set_store(mut self: Pin<&mut Self>, store: &StoreOpaque) { // FIXME: should be more targeted ideally with the `unsafe` than just // throwing this entire function in a large `unsafe` block. unsafe { - *self.as_mut().store_mut() = store.map(VMStoreRawPtr); - if let Some(mut store) = store { - let store = store.as_mut(); - self.vm_store_context() - .write(Some(store.vm_store_context_ptr().into())); - #[cfg(target_has_atomic = "64")] - { - *self.as_mut().epoch_ptr() = - Some(NonNull::from(store.engine().epoch_counter()).into()); - } + *self.as_mut().store_mut() = Some(VMStoreRawPtr(store.traitobj())); + self.vm_store_context() + .write(Some(store.vm_store_context_ptr().into())); + #[cfg(target_has_atomic = "64")] + { + *self.as_mut().epoch_ptr() = + Some(NonNull::from(store.engine().epoch_counter()).into()); + } - if self.env_module().needs_gc_heap { - self.as_mut().set_gc_heap(Some(store.unwrap_gc_store())); - } else { - self.as_mut().set_gc_heap(None); - } + if self.env_module().needs_gc_heap { + self.as_mut().set_gc_heap(Some(store.unwrap_gc_store())); } else { - self.vm_store_context().write(None); - #[cfg(target_has_atomic = "64")] - { - *self.as_mut().epoch_ptr() = None; - } self.as_mut().set_gc_heap(None); } } @@ -1246,7 +1236,7 @@ impl Instance { mut self: Pin<&mut Self>, module: &Module, offsets: &VMOffsets, - store: StorePtr, + store: &StoreOpaque, imports: Imports, ) { assert!(ptr::eq(module, self.env_module().as_ref())); @@ -1260,7 +1250,7 @@ impl Instance { // SAFETY: it's up to the caller to provide a valid store pointer here. unsafe { - self.as_mut().set_store(store.as_raw()); + self.as_mut().set_store(store); } // Initialize shared types diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs index 8e01045458ac..988dae1fe0b7 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs @@ -5,15 +5,13 @@ use crate::runtime::vm::instance::{Instance, InstanceHandle}; use crate::runtime::vm::memory::Memory; use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::table::Table; -use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo, VMStore}; -use crate::store::{InstanceId, StoreOpaque}; +use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo}; +use crate::store::{InstanceId, StoreOpaque, StoreResourceLimiter}; use crate::{OpaqueRootScope, Val}; -use core::ptr::NonNull; use core::{mem, ptr}; use wasmtime_environ::{ DefinedMemoryIndex, DefinedTableIndex, HostPtr, InitMemory, MemoryInitialization, - MemoryInitializer, Module, PrimaryMap, SizeOverflow, TableInitialValue, Trap, Tunables, - VMOffsets, + MemoryInitializer, Module, PrimaryMap, SizeOverflow, TableInitialValue, Trap, VMOffsets, }; #[cfg(feature = "gc")] @@ -52,79 +50,11 @@ pub struct InstanceAllocationRequest<'a> { /// The imports to use for the instantiation. pub imports: Imports<'a>, - /// A pointer to the "store" for this instance to be allocated. The store - /// correlates with the `Store` in wasmtime itself, and lots of contextual - /// information about the execution of wasm can be learned through the - /// store. - /// - /// Note that this is a raw pointer and has a static lifetime, both of which - /// are a bit of a lie. This is done purely so a store can learn about - /// itself when it gets called as a host function, and additionally so this - /// runtime can access internals as necessary (such as the - /// VMExternRefActivationsTable or the resource limiter methods). - /// - /// Note that this ends up being a self-pointer to the instance when stored. - /// The reason is that the instance itself is then stored within the store. - /// We use a number of `PhantomPinned` declarations to indicate this to the - /// compiler. More info on this in `wasmtime/src/store.rs` - pub store: StorePtr, - - /// Indicates '--wmemcheck' flag. - #[cfg(feature = "wmemcheck")] - pub wmemcheck: bool, - - /// Request that the instance's memories be protected by a specific - /// protection key. - #[cfg_attr( - not(feature = "pooling-allocator"), - expect( - dead_code, - reason = "easier to keep this field than remove it, not perf-critical to remove" - ) - )] - pub pkey: Option, - - /// Tunable configuration options the engine is using. - pub tunables: &'a Tunables, -} - -/// A pointer to a Store. This Option<*mut dyn Store> is wrapped in a struct -/// so that the function to create a &mut dyn Store is a method on a member of -/// InstanceAllocationRequest, rather than on a &mut InstanceAllocationRequest -/// itself, because several use-sites require a split mut borrow on the -/// InstanceAllocationRequest. -pub struct StorePtr(Option>); - -// We can't make `VMStore: Send + Sync` because that requires making all of -// Wastime's internals generic over the `Store`'s `T`. So instead, we take care -// in the whole VM layer to only use the `VMStore` in ways that are `Send`- and -// `Sync`-safe and we have to have these unsafe impls. -unsafe impl Send for StorePtr {} -unsafe impl Sync for StorePtr {} - -impl StorePtr { - /// A pointer to no Store. - pub fn empty() -> Self { - Self(None) - } + /// The store that this instance is being allocated into. + pub store: &'a StoreOpaque, - /// A pointer to a Store. - pub fn new(ptr: NonNull) -> Self { - Self(Some(ptr)) - } - - /// The raw contents of this struct - pub fn as_raw(&self) -> Option> { - self.0 - } - - /// Use the StorePtr as a mut ref to the Store. - /// - /// Safety: must not be used outside the original lifetime of the borrow. - pub(crate) unsafe fn get(&mut self) -> Option<&mut dyn VMStore> { - let ptr = unsafe { self.0?.as_mut() }; - Some(ptr) - } + /// The store's resource limiter, if configured by the embedder. + pub limiter: Option<&'a mut StoreResourceLimiter<'a>>, } /// The index of a memory allocation within an `InstanceAllocator`. @@ -197,6 +127,7 @@ impl GcHeapAllocationIndex { /// /// This trait is unsafe as it requires knowledge of Wasmtime's runtime /// internals to implement correctly. +#[async_trait::async_trait] pub unsafe trait InstanceAllocator: Send + Sync { /// Validate whether a component (including all of its contained core /// modules) is allocatable by this instance allocator. @@ -253,11 +184,10 @@ pub unsafe trait InstanceAllocator: Send + Sync { fn decrement_core_instance_count(&self); /// Allocate a memory for an instance. - fn allocate_memory( + async fn allocate_memory( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Memory, - tunables: &Tunables, memory_index: Option, ) -> Result<(MemoryAllocationIndex, Memory)>; @@ -276,11 +206,10 @@ pub unsafe trait InstanceAllocator: Send + Sync { ); /// Allocate a table for an instance. - fn allocate_table( + async fn allocate_table( &self, - req: &mut InstanceAllocationRequest, + req: &mut InstanceAllocationRequest<'_>, table: &wasmtime_environ::Table, - tunables: &Tunables, table_index: DefinedTableIndex, ) -> Result<(TableAllocationIndex, Table)>; @@ -373,9 +302,9 @@ impl dyn InstanceAllocator + '_ { /// /// The `request` provided must be valid, e.g. the imports within are /// correctly sized/typed for the instance being created. - pub(crate) unsafe fn allocate_module( + pub(crate) async unsafe fn allocate_module( &self, - mut request: InstanceAllocationRequest, + mut request: InstanceAllocationRequest<'_>, ) -> Result { let module = request.runtime_info.env_module(); @@ -396,8 +325,10 @@ impl dyn InstanceAllocator + '_ { allocator: self, }; - self.allocate_memories(&mut request, &mut guard.memories)?; - self.allocate_tables(&mut request, &mut guard.tables)?; + self.allocate_memories(&mut request, &mut guard.memories) + .await?; + self.allocate_tables(&mut request, &mut guard.tables) + .await?; guard.run_deallocate = false; // SAFETY: memories/tables were just allocated from the store within // `request` and this function's own contract requires that the @@ -455,9 +386,9 @@ impl dyn InstanceAllocator + '_ { /// Allocate the memories for the given instance allocation request, pushing /// them into `memories`. - fn allocate_memories( + async fn allocate_memories( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, memories: &mut PrimaryMap, ) -> Result<()> { let module = request.runtime_info.env_module(); @@ -472,7 +403,9 @@ impl dyn InstanceAllocator + '_ { .defined_memory_index(memory_index) .expect("should be a defined memory since we skipped imported ones"); - let memory = self.allocate_memory(request, ty, request.tunables, Some(memory_index))?; + let memory = self + .allocate_memory(request, ty, Some(memory_index)) + .await?; memories.push(memory); } @@ -506,9 +439,9 @@ impl dyn InstanceAllocator + '_ { /// Allocate tables for the given instance allocation request, pushing them /// into `tables`. - fn allocate_tables( + async fn allocate_tables( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, tables: &mut PrimaryMap, ) -> Result<()> { let module = request.runtime_info.env_module(); @@ -523,7 +456,7 @@ impl dyn InstanceAllocator + '_ { .defined_table_index(index) .expect("should be a defined table since we skipped imported ones"); - let table = self.allocate_table(request, table, request.tunables, def_index)?; + let table = self.allocate_table(request, table, def_index).await?; tables.push(table); } diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs index 26aa362de1a0..9dab461c8c70 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs @@ -8,9 +8,7 @@ use crate::runtime::vm::memory::{DefaultMemoryCreator, Memory}; use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::table::Table; use alloc::sync::Arc; -use wasmtime_environ::{ - DefinedMemoryIndex, DefinedTableIndex, HostPtr, Module, Tunables, VMOffsets, -}; +use wasmtime_environ::{DefinedMemoryIndex, DefinedTableIndex, HostPtr, Module, VMOffsets}; #[cfg(feature = "gc")] use crate::runtime::vm::{GcHeap, GcHeapAllocationIndex, GcRuntime}; @@ -76,6 +74,7 @@ impl Default for OnDemandInstanceAllocator { } } +#[async_trait::async_trait] unsafe impl InstanceAllocator for OnDemandInstanceAllocator { #[cfg(feature = "component-model")] fn validate_component<'a>( @@ -110,11 +109,10 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { fn decrement_core_instance_count(&self) {} - fn allocate_memory( + async fn allocate_memory( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Memory, - tunables: &Tunables, memory_index: Option, ) -> Result<(MemoryAllocationIndex, Memory)> { let creator = self @@ -131,16 +129,12 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { let allocation_index = MemoryAllocationIndex::default(); let memory = Memory::new_dynamic( ty, - tunables, + request.store.engine().tunables(), creator, - unsafe { - request - .store - .get() - .expect("if module has memory plans, store is not empty") - }, image, - )?; + request.limiter.as_deref_mut(), + ) + .await?; Ok((allocation_index, memory)) } @@ -154,20 +148,19 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { // Normal destructors do all the necessary clean up. } - fn allocate_table( + async fn allocate_table( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Table, - tunables: &Tunables, _table_index: DefinedTableIndex, ) -> Result<(TableAllocationIndex, Table)> { let allocation_index = TableAllocationIndex::default(); - let table = Table::new_dynamic(ty, tunables, unsafe { - request - .store - .get() - .expect("if module has table plans, store is not empty") - })?; + let table = Table::new_dynamic( + ty, + request.store.engine().tunables(), + request.limiter.as_deref_mut(), + ) + .await?; Ok((allocation_index, table)) } diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs index 42db185408c6..52a199a2efe9 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs @@ -475,6 +475,7 @@ impl PoolingInstanceAllocator { /// Execute `f` and if it returns `Err(PoolConcurrencyLimitError)`, then try /// flushing the decommit queue. If flushing the queue freed up slots, then /// try running `f` again. + #[cfg(feature = "async")] fn with_flush_and_retry(&self, mut f: impl FnMut() -> Result) -> Result { f().or_else(|e| { if e.is::() { @@ -523,6 +524,7 @@ impl PoolingInstanceAllocator { } } +#[async_trait::async_trait] unsafe impl InstanceAllocator for PoolingInstanceAllocator { #[cfg(feature = "component-model")] fn validate_component<'a>( @@ -644,14 +646,29 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { self.live_core_instances.fetch_sub(1, Ordering::AcqRel); } - fn allocate_memory( + async fn allocate_memory( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Memory, - tunables: &Tunables, memory_index: Option, ) -> Result<(MemoryAllocationIndex, Memory)> { - self.with_flush_and_retry(|| self.memories.allocate(request, ty, tunables, memory_index)) + // FIXME(rust-lang/rust#145127) this should ideally use a version of + // `with_flush_and_retry` but adapted for async closures instead of only + // sync closures. Right now that won't compile though so this is the + // manually expanded version of the method. + let e = match self.memories.allocate(request, ty, memory_index).await { + Ok(result) => return Ok(result), + Err(e) => e, + }; + + if e.is::() { + let queue = self.decommit_queue.lock().unwrap(); + if self.flush_decommit_queue(queue) { + return self.memories.allocate(request, ty, memory_index).await; + } + } + + Err(e) } unsafe fn deallocate_memory( @@ -689,14 +706,27 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { self.merge_or_flush(queue); } - fn allocate_table( + async fn allocate_table( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Table, - tunables: &Tunables, _table_index: DefinedTableIndex, ) -> Result<(super::TableAllocationIndex, Table)> { - self.with_flush_and_retry(|| self.tables.allocate(request, ty, tunables)) + // FIXME: see `allocate_memory` above for comments about duplication + // with `with_flush_and_retry`. + let e = match self.tables.allocate(request, ty).await { + Ok(result) => return Ok(result), + Err(e) => e, + }; + + if e.is::() { + let queue = self.decommit_queue.lock().unwrap(); + if self.flush_decommit_queue(queue) { + return self.tables.allocate(request, ty).await; + } + } + + Err(e) } unsafe fn deallocate_table( diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs index 9f0a3a341e6f..5a298bbad695 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs @@ -315,14 +315,14 @@ impl MemoryPool { } /// Allocate a single memory for the given instance allocation request. - pub fn allocate( + pub async fn allocate( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Memory, - tunables: &Tunables, memory_index: Option, ) -> Result<(MemoryAllocationIndex, Memory)> { - let stripe_index = if let Some(pkey) = &request.pkey { + let tunables = request.store.engine().tunables(); + let stripe_index = if let Some(pkey) = request.store.get_pkey() { pkey.as_stripe() } else { debug_assert!(self.stripes.len() < 2); @@ -397,8 +397,9 @@ impl MemoryPool { MemoryBase::Mmap(base), base_capacity.byte_count(), slot, - unsafe { &mut *request.store.get().unwrap() }, - )?; + request.limiter.as_deref_mut(), + ) + .await?; guard.active = false; return Ok((allocation_index, memory)); diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs index 37effa9fbe19..aa02e6355d36 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs @@ -9,7 +9,7 @@ use crate::runtime::vm::{ }; use crate::{prelude::*, vm::HostAlignedByteCount}; use std::ptr::NonNull; -use wasmtime_environ::{Module, Tunables}; +use wasmtime_environ::Module; /// Represents a pool of WebAssembly tables. /// @@ -130,12 +130,12 @@ impl TablePool { } /// Allocate a single table for the given instance allocation request. - pub fn allocate( + pub async fn allocate( &self, - request: &mut InstanceAllocationRequest, + request: &mut InstanceAllocationRequest<'_>, ty: &wasmtime_environ::Table, - tunables: &Tunables, ) -> Result<(TableAllocationIndex, Table)> { + let tunables = request.store.engine().tunables(); let allocation_index = self .index_allocator .alloc() @@ -161,8 +161,9 @@ impl TablePool { ty, tunables, SendSyncPtr::new(ptr), - &mut *request.store.get().unwrap(), - )? + request.limiter.as_deref_mut(), + ) + .await? }; guard.active = false; return Ok((allocation_index, table)); diff --git a/crates/wasmtime/src/runtime/vm/memory.rs b/crates/wasmtime/src/runtime/vm/memory.rs index 99032acf2f82..6a774f45d8a5 100644 --- a/crates/wasmtime/src/runtime/vm/memory.rs +++ b/crates/wasmtime/src/runtime/vm/memory.rs @@ -79,7 +79,7 @@ use crate::runtime::store::StoreResourceLimiter; use crate::runtime::vm::vmcontext::VMMemoryDefinition; #[cfg(has_virtual_memory)] use crate::runtime::vm::{HostAlignedByteCount, MmapOffset}; -use crate::runtime::vm::{MemoryImage, MemoryImageSlot, SendSyncPtr, VMStore}; +use crate::runtime::vm::{MemoryImage, MemoryImageSlot, SendSyncPtr}; use alloc::sync::Arc; use core::{ops::Range, ptr::NonNull}; use wasmtime_environ::Tunables; @@ -231,14 +231,14 @@ pub enum Memory { impl Memory { /// Create a new dynamic (movable) memory instance for the specified plan. - pub fn new_dynamic( + pub async fn new_dynamic( ty: &wasmtime_environ::Memory, tunables: &Tunables, creator: &dyn RuntimeMemoryCreator, - store: &mut dyn VMStore, memory_image: Option<&Arc>, + limiter: Option<&mut StoreResourceLimiter<'_>>, ) -> Result { - let (minimum, maximum) = Self::limit_new(ty, Some(store))?; + let (minimum, maximum) = Self::limit_new(ty, limiter).await?; let allocation = creator.new_memory(ty, tunables, minimum, maximum)?; let memory = LocalMemory::new(ty, tunables, allocation, memory_image)?; @@ -251,15 +251,15 @@ impl Memory { /// Create a new static (immovable) memory instance for the specified plan. #[cfg(feature = "pooling-allocator")] - pub fn new_static( + pub async fn new_static( ty: &wasmtime_environ::Memory, tunables: &Tunables, base: MemoryBase, base_capacity: usize, memory_image: MemoryImageSlot, - store: &mut dyn VMStore, + limiter: Option<&mut StoreResourceLimiter<'_>>, ) -> Result { - let (minimum, maximum) = Self::limit_new(ty, Some(store))?; + let (minimum, maximum) = Self::limit_new(ty, limiter).await?; let pooled_memory = StaticMemory::new(base, base_capacity, minimum, maximum)?; let allocation = Box::new(pooled_memory); @@ -286,9 +286,9 @@ impl Memory { /// /// Returns a tuple of the minimum size, optional maximum size, and log(page /// size) of the memory, all in bytes. - pub(crate) fn limit_new( + pub(crate) async fn limit_new( ty: &wasmtime_environ::Memory, - store: Option<&mut dyn VMStore>, + limiter: Option<&mut StoreResourceLimiter<'_>>, ) -> Result<(usize, Option)> { let page_size = usize::try_from(ty.page_size()).unwrap(); @@ -331,8 +331,11 @@ impl Memory { // `minimum` calculation overflowed. This means that the `minimum` we're // informing the limiter is lossy and may not be 100% accurate, but for // now the expected uses of limiter means that's ok. - if let Some(store) = store { - if !store.memory_growing(0, minimum.unwrap_or(absolute_max), maximum)? { + if let Some(limiter) = limiter { + if !limiter + .memory_growing(0, minimum.unwrap_or(absolute_max), maximum) + .await? + { bail!( "memory minimum size of {} pages exceeds memory limits", ty.limits.min diff --git a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs index aab6068ad899..a1abe2d64491 100644 --- a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs +++ b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs @@ -31,7 +31,9 @@ struct SharedMemoryInner { impl SharedMemory { /// Construct a new [`SharedMemory`]. pub fn new(ty: &wasmtime_environ::Memory, tunables: &Tunables) -> Result { - let (minimum_bytes, maximum_bytes) = Memory::limit_new(ty, None)?; + // Note that without a limiter being passed to `limit_new` this + // `assert_ready` should never panic. + let (minimum_bytes, maximum_bytes) = vm::assert_ready(Memory::limit_new(ty, None))?; let mmap_memory = MmapMemory::new(ty, tunables, minimum_bytes, maximum_bytes)?; Self::wrap( ty, diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index 1405d4d1b7db..8b3f84f4692c 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -6,7 +6,7 @@ use crate::prelude::*; use crate::runtime::store::StoreResourceLimiter; use crate::runtime::vm::stack_switching::VMContObj; use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition}; -use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VMStore, VmPtr}; +use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VmPtr}; use core::alloc::Layout; use core::mem; use core::ops::Range; @@ -324,12 +324,12 @@ unsafe fn alloc_dynamic_table_elements(len: usize) -> Result>> impl Table { /// Create a new dynamic (movable) table instance for the specified table plan. - pub fn new_dynamic( + pub async fn new_dynamic( ty: &wasmtime_environ::Table, tunables: &Tunables, - store: &mut dyn VMStore, + limiter: Option<&mut StoreResourceLimiter<'_>>, ) -> Result { - let (minimum, maximum) = Self::limit_new(ty, store)?; + let (minimum, maximum) = Self::limit_new(ty, limiter).await?; match wasm_to_table_type(ty.ref_type) { TableElementType::Func => Ok(Self::from(DynamicFuncTable { elements: unsafe { alloc_dynamic_table_elements(minimum)? }, @@ -348,13 +348,13 @@ impl Table { } /// Create a new static (immovable) table instance for the specified table plan. - pub unsafe fn new_static( + pub async unsafe fn new_static( ty: &wasmtime_environ::Table, tunables: &Tunables, data: SendSyncPtr<[u8]>, - store: &mut dyn VMStore, + limiter: Option<&mut StoreResourceLimiter<'_>>, ) -> Result { - let (minimum, maximum) = Self::limit_new(ty, store)?; + let (minimum, maximum) = Self::limit_new(ty, limiter).await?; let size = minimum; let max = maximum.unwrap_or(usize::MAX); @@ -435,9 +435,9 @@ impl Table { // Calls the `store`'s limiter to optionally prevent the table from being created. // // Returns the minimum and maximum size of the table if the table can be created. - fn limit_new( + async fn limit_new( ty: &wasmtime_environ::Table, - store: &mut dyn VMStore, + limiter: Option<&mut StoreResourceLimiter<'_>>, ) -> Result<(usize, Option)> { // No matter how the table limits are specified // The table size is limited by the host's pointer size @@ -458,11 +458,16 @@ impl Table { }; // Inform the store's limiter what's about to happen. - if !store.table_growing(0, minimum.unwrap_or(absolute_max), maximum)? { - bail!( - "table minimum size of {} elements exceeds table limits", - ty.limits.min - ); + if let Some(limiter) = limiter { + if !limiter + .table_growing(0, minimum.unwrap_or(absolute_max), maximum) + .await? + { + bail!( + "table minimum size of {} elements exceeds table limits", + ty.limits.min + ); + } } // At this point we need to actually handle overflows, so bail out with diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs index c2ad007bd16f..14f9833a4e52 100644 --- a/tests/all/pooling_allocator.rs +++ b/tests/all/pooling_allocator.rs @@ -938,6 +938,9 @@ async fn total_stacks_limit() -> Result<()> { (func (export "run") call $yield ) + + (func $empty) + (start $empty) ) "#, )?;