Make const-expr evaluation async (#11468)

* Make const-expr evaluation `async`

This commit is extracted from #11430 to accurately reflect how
const-expr evaluation is an async operation due to GC pauses that may
happen. The changes in this commit are:

* Const-expr evaluation is, at its core, now an `async` function.
* To leverage this new `async`-ness all internal operations are switched
  from `*_maybe_async` to `*_async` meaning all the `*_maybe_async`
  methods can be removed.
* Some libcalls using `*_maybe_async` are switch to using `*_async` plus
  the `block_on!` utility to help jettison more `*_maybe_async` methods.
* Instance initialization is now an `async` function. This is
  temporarily handled with `block_on` during instance initialization to
  avoid propagating the `async`-ness further upwards. This `block_on`
  will get deleted in future refactorings.
* Const-expr evaluation has been refactored slightly to enable having a
  fast path in global initialization which skips an `await` point
  entirely, achieving performance-parity in benchmarks prior to this commit.

This ended up fixing a niche issue with GC where if a wasm execution was
suspended during `table.init`, for example, during a const-expr
evaluation triggering a GC then if the wasm execution was cancelled it
would panic the host. This panic was because the GC operation returned
`Result` but it was `unwrap`'d as part of the const-expr evaluation
which can fail not only to invalid-ness but also due to "computation is
cancelled" traps.

* Fix configured build

* Undo rebase mistake

Author: alexcrichton

crates/wasmtime/src/runtime/gc/enabled/arrayref.rs

@@ -353,7 +353,6 @@ impl ArrayRef {
         Self::_new_async(store.as_context_mut().0, allocator, elem, len).await
     }
 
-    #[cfg(feature = "async")]
     pub(crate) async fn _new_async(
         store: &mut StoreOpaque,
         allocator: &ArrayRefPre,
@@ -367,25 +366,6 @@ impl ArrayRef {
             .await
     }
 
-    /// Like `ArrayRef::new` but when async is configured must only ever be
-    /// called from on a fiber stack.
-    pub(crate) unsafe fn new_maybe_async(
-        store: &mut StoreOpaque,
-        allocator: &ArrayRefPre,
-        elem: &Val,
-        len: u32,
-    ) -> Result<Rooted<ArrayRef>> {
-        // Type check the initial element value against the element type.
-        elem.ensure_matches_ty(store, allocator.ty.element_type().unpack())
-            .context("element type mismatch")?;
-
-        unsafe {
-            store.retry_after_gc_maybe_async((), |store, ()| {
-                Self::new_from_iter(store, allocator, RepeatN(elem, len))
-            })
-        }
-    }
-
     /// Allocate a new array of the given elements.
     ///
     /// Does not attempt a GC on OOM; leaves that to callers.
@@ -531,7 +511,6 @@ impl ArrayRef {
         Self::_new_fixed_async(store.as_context_mut().0, allocator, elems).await
     }
 
-    #[cfg(feature = "async")]
     pub(crate) async fn _new_fixed_async(
         store: &mut StoreOpaque,
         allocator: &ArrayRefPre,
@@ -544,21 +523,6 @@ impl ArrayRef {
             .await
     }
 
-    /// Like `ArrayRef::new_fixed[_async]` but it is the caller's responsibility
-    /// to ensure that when async is enabled, this is only called from on a
-    /// fiber stack.
-    pub(crate) unsafe fn new_fixed_maybe_async(
-        store: &mut StoreOpaque,
-        allocator: &ArrayRefPre,
-        elems: &[Val],
-    ) -> Result<Rooted<ArrayRef>> {
-        unsafe {
-            store.retry_after_gc_maybe_async((), |store, ()| {
-                Self::new_from_iter(store, allocator, elems.iter())
-            })
-        }
-    }
-
     #[inline]
     pub(crate) fn comes_from_same_store(&self, store: &StoreOpaque) -> bool {
         self.inner.comes_from_same_store(store)

crates/wasmtime/src/runtime/gc/enabled/structref.rs

@@ -269,10 +269,6 @@ impl StructRef {
     ///
     /// # Panics
     ///
-    /// Panics if your engine is not configured for async; use
-    /// [`StructRef::new`][crate::StructRef::new] to perform synchronous
-    /// allocation instead.
-    ///
     /// Panics if the allocator, or any of the field values, is not associated
     /// with the given store.
     #[cfg(feature = "async")]
@@ -284,16 +280,11 @@ impl StructRef {
         Self::_new_async(store.as_context_mut().0, allocator, fields).await
     }
 
-    #[cfg(feature = "async")]
     pub(crate) async fn _new_async(
         store: &mut StoreOpaque,
         allocator: &StructRefPre,
         fields: &[Val],
     ) -> Result<Rooted<StructRef>> {
-        assert!(
-            store.async_support(),
-            "use `StructRef::new` with synchronous stores"
-        );
         Self::type_check_fields(store, allocator, fields)?;
         store
             .retry_after_gc_async((), |store, ()| {
@@ -302,21 +293,6 @@ impl StructRef {
             .await
     }
 
-    /// Like `Self::new` but caller's must ensure that if the store is
-    /// configured for async, this is only ever called from on a fiber stack.
-    pub(crate) unsafe fn new_maybe_async(
-        store: &mut StoreOpaque,
-        allocator: &StructRefPre,
-        fields: &[Val],
-    ) -> Result<Rooted<StructRef>> {
-        Self::type_check_fields(store, allocator, fields)?;
-        unsafe {
-            store.retry_after_gc_maybe_async((), |store, ()| {
-                Self::new_unchecked(store, allocator, fields)
-            })
-        }
-    }
-
     /// Type check the field values before allocating a new struct.
     fn type_check_fields(
         store: &mut StoreOpaque,

crates/wasmtime/src/runtime/instance.rs

@@ -377,7 +377,24 @@ impl Instance {
             .features()
             .contains(WasmFeatures::BULK_MEMORY);
 
-        vm::initialize_instance(store, id, compiled_module.module(), bulk_memory)?;
+        if store.async_support() {
+            #[cfg(feature = "async")]
+            store.block_on(|store| {
+                let module = compiled_module.module().clone();
+                Box::pin(
+                    async move { vm::initialize_instance(store, id, &module, bulk_memory).await },
+                )
+            })??;
+            #[cfg(not(feature = "async"))]
+            unreachable!();
+        } else {
+            vm::assert_ready(vm::initialize_instance(
+                store,
+                id,
+                compiled_module.module(),
+                bulk_memory,
+            ))?;
+        }
 
         Ok((instance, compiled_module.module().start_func))
     }

crates/wasmtime/src/runtime/store/gc.rs

@@ -198,56 +198,6 @@ impl StoreOpaque {
         }
     }
 
-    /// Like `retry_after_gc` but async yielding (if necessary) is transparent.
-    ///
-    /// # Safety
-    ///
-    /// When async is enabled, it is the caller's responsibility to ensure that
-    /// this is called on a fiber stack.
-    pub(crate) unsafe fn retry_after_gc_maybe_async<T, U>(
-        &mut self,
-        value: T,
-        alloc_func: impl Fn(&mut Self, T) -> Result<U>,
-    ) -> Result<U>
-    where
-        T: Send + Sync + 'static,
-    {
-        self.ensure_gc_store()?;
-        match alloc_func(self, value) {
-            Ok(x) => Ok(x),
-            Err(e) => match e.downcast::<crate::GcHeapOutOfMemory<T>>() {
-                Ok(oom) => {
-                    let (value, oom) = oom.take_inner();
-                    // Note it's the caller's responsibility to ensure that
-                    // this is on a fiber stack if necessary.
-                    //
-                    // SAFETY: FIXME(#11409)
-                    unsafe {
-                        if self.async_support() {
-                            #[cfg(feature = "async")]
-                            self.block_on(|store| {
-                                Box::pin(
-                                    store.gc_unsafe_get_limiter(None, Some(oom.bytes_needed())),
-                                )
-                            })?;
-                            #[cfg(not(feature = "async"))]
-                            unreachable!();
-                        } else {
-                            vm::assert_ready(
-                                self.gc_unsafe_get_limiter(None, Some(oom.bytes_needed())),
-                            );
-                        }
-                    }
-                    alloc_func(self, value)
-                }
-                Err(e) => Err(e),
-            },
-        }
-    }
-}
-
-#[cfg(feature = "async")]
-impl StoreOpaque {
     /// Attempt an allocation, if it fails due to GC OOM, then do a GC and
     /// retry.
     pub(crate) async fn retry_after_gc_async<T, U>(
@@ -258,10 +208,6 @@ impl StoreOpaque {
     where
         T: Send + Sync + 'static,
     {
-        assert!(
-            self.async_support(),
-            "you must configure async to use the `*_async` versions of methods"
-        );
         self.ensure_gc_store()?;
         match alloc_func(self, value) {
             Ok(x) => Ok(x),