async/stream/future support for wasmtime-wit-bindgen

I've split this out of bytecodealliance#9582 to make review easier.

This patch adds async/stream/future/error-context support to the host binding
generator, along with placeholder type and function definitions in the
`wasmtime` crate which the generated bindings can refer to.  See
https://github.com/dicej/rfcs/blob/component-async/accepted/component-model-async.md#componentbindgen-updates
for the design and rationale.

Note that I've added temporary `[patch.crates-io]` overrides in Cargo.toml until
bytecodealliance/wit-bindgen#1130 and
bytecodealliance/wasm-tools#1978 have been released.

Also note that we emit a `T: 'static` bound for `AsContextMut<Data = T>` when
generating bindings with `concurrent_imports: true`.  This is only because
`rustc` insists that the closure we're passing to
`LinkerInstance::func_wrap_concurrent` captures the lifetime of `T` despite my
best efforts to convince it otherwise.  Alex and I suspect this is a limitation
in the compiler, and I asked about it on the rust-lang Zulip, but we haven't
been able to determine a workaround so far.

Signed-off-by: Joel Dice <[email protected]>

Author: dicej

Cargo.lock

@@ -564,3 +564,21 @@ opt-level = 's'
 inherits = "release"
 codegen-units = 1
 lto = true
+
+# TODO: Remove these patches once wit-bindgen 0.38 and wasm-tools 1.224 are available:
+[patch.crates-io]
+wit-bindgen = { git = "https://github.com/dicej/wit-bindgen", branch = "feat/stream-unit" }
+wit-bindgen-rt = { git = "https://github.com/dicej/wit-bindgen", branch = "feat/stream-unit" }
+wit-bindgen-rust-macro = { git = "https://github.com/dicej/wit-bindgen", branch = "feat/stream-unit" }
+wasmparser = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wasm-metadata = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wat = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wast = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wasmprinter = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wasm-encoder = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wasm-smith = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wasm-mutate = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wit-parser = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wit-component = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wasm-wave = { git = "https://github.com/bytecodealliance/wasm-tools" }
+

Cargo.toml

@@ -29,7 +29,8 @@ wasmtime-wit-bindgen = { workspace = true }
 wit-parser = { workspace = true }
 
 [dev-dependencies]
-wasmtime = { path = '../wasmtime', features = ['component-model'] }
+wasmtime = { path = '../wasmtime', features = ['component-model', 'component-model-async'] }
+wasmtime-wit-bindgen = { workspace = true, features = ['component-model-async'] }
 component-macro-test-helpers = { path = 'test-helpers' }
 tracing = { workspace = true }
 # For use with the custom attributes test
@@ -41,3 +42,4 @@ similar = { workspace = true }
 [features]
 async = []
 std = ['wasmtime-wit-bindgen/std']
+component-model-async = ['std', 'async', 'wasmtime-wit-bindgen/component-model-async']

crates/component-macro/Cargo.toml

@@ -1,14 +1,15 @@
 use proc_macro2::{Span, TokenStream};
 use quote::ToTokens;
-use std::collections::HashMap;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::env;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
 use syn::parse::{Error, Parse, ParseStream, Result};
 use syn::punctuated::Punctuated;
 use syn::{braced, token, Token};
-use wasmtime_wit_bindgen::{AsyncConfig, Opts, Ownership, TrappableError, TrappableImports};
+use wasmtime_wit_bindgen::{
+    AsyncConfig, CallStyle, Opts, Ownership, TrappableError, TrappableImports,
+};
 use wit_parser::{PackageId, Resolve, UnresolvedPackageGroup, WorldId};
 
 pub struct Config {
@@ -20,13 +21,22 @@ pub struct Config {
 }
 
 pub fn expand(input: &Config) -> Result<TokenStream> {
-    if !cfg!(feature = "async") && input.opts.async_.maybe_async() {
+    if let (CallStyle::Async | CallStyle::Concurrent, false) =
+        (input.opts.call_style(), cfg!(feature = "async"))
+    {
         return Err(Error::new(
             Span::call_site(),
             "cannot enable async bindings unless `async` crate feature is active",
         ));
     }
 
+    if input.opts.concurrent_imports && !cfg!(feature = "component-model-async") {
+        return Err(Error::new(
+            Span::call_site(),
+            "cannot enable `concurrent_imports` option unless `component-model-async` crate feature is active",
+        ));
+    }
+
     let mut src = match input.opts.generate(&input.resolve, input.world) {
         Ok(s) => s,
         Err(e) => return Err(Error::new(Span::call_site(), e.to_string())),
@@ -40,7 +50,10 @@ pub fn expand(input: &Config) -> Result<TokenStream> {
     // place a formatted version of the expanded code into a file. This file
     // will then show up in rustc error messages for any codegen issues and can
     // be inspected manually.
-    if input.include_generated_code_from_file || std::env::var("WASMTIME_DEBUG_BINDGEN").is_ok() {
+    if input.include_generated_code_from_file
+        || input.opts.debug
+        || std::env::var("WASMTIME_DEBUG_BINDGEN").is_ok()
+    {
         static INVOCATION: AtomicUsize = AtomicUsize::new(0);
         let root = Path::new(env!("DEBUG_OUTPUT_DIR"));
         let world_name = &input.resolve.worlds[input.world].name;
@@ -107,13 +120,16 @@ impl Parse for Config {
                     }
                     Opt::Tracing(val) => opts.tracing = val,
                     Opt::VerboseTracing(val) => opts.verbose_tracing = val,
+                    Opt::Debug(val) => opts.debug = val,
                     Opt::Async(val, span) => {
                         if async_configured {
                             return Err(Error::new(span, "cannot specify second async config"));
                         }
                         async_configured = true;
                         opts.async_ = val;
                     }
+                    Opt::ConcurrentImports(val) => opts.concurrent_imports = val,
+                    Opt::ConcurrentExports(val) => opts.concurrent_exports = val,
                     Opt::TrappableErrorType(val) => opts.trappable_error_type = val,
                     Opt::TrappableImports(val) => opts.trappable_imports = val,
                     Opt::Ownership(val) => opts.ownership = val,
@@ -138,7 +154,7 @@ impl Parse for Config {
                                 "cannot specify a world with `interfaces`",
                             ));
                         }
-                        world = Some("interfaces".to_string());
+                        world = Some("wasmtime:component-macro-synthesized/interfaces".to_string());
 
                         opts.only_interfaces = true;
                     }
@@ -281,6 +297,9 @@ mod kw {
     syn::custom_keyword!(require_store_data_send);
     syn::custom_keyword!(wasmtime_crate);
     syn::custom_keyword!(include_generated_code_from_file);
+    syn::custom_keyword!(concurrent_imports);
+    syn::custom_keyword!(concurrent_exports);
+    syn::custom_keyword!(debug);
 }
 
 enum Opt {
@@ -301,12 +320,19 @@ enum Opt {
     RequireStoreDataSend(bool),
     WasmtimeCrate(syn::Path),
     IncludeGeneratedCodeFromFile(bool),
+    ConcurrentImports(bool),
+    ConcurrentExports(bool),
+    Debug(bool),
 }
 
 impl Parse for Opt {
     fn parse(input: ParseStream<'_>) -> Result<Self> {
         let l = input.lookahead1();
-        if l.peek(kw::path) {
+        if l.peek(kw::debug) {
+            input.parse::<kw::debug>()?;
+            input.parse::<Token![:]>()?;
+            Ok(Opt::Debug(input.parse::<syn::LitBool>()?.value))
+        } else if l.peek(kw::path) {
             input.parse::<kw::path>()?;
             input.parse::<Token![:]>()?;
 
@@ -380,6 +406,14 @@ impl Parse for Opt {
                     span,
                 ))
             }
+        } else if l.peek(kw::concurrent_imports) {
+            input.parse::<kw::concurrent_imports>()?;
+            input.parse::<Token![:]>()?;
+            Ok(Opt::ConcurrentImports(input.parse::<syn::LitBool>()?.value))
+        } else if l.peek(kw::concurrent_exports) {
+            input.parse::<kw::concurrent_exports>()?;
+            input.parse::<Token![:]>()?;
+            Ok(Opt::ConcurrentExports(input.parse::<syn::LitBool>()?.value))
         } else if l.peek(kw::ownership) {
             input.parse::<kw::ownership>()?;
             input.parse::<Token![:]>()?;

crates/component-macro/src/bindgen.rs

@@ -12,6 +12,14 @@ macro_rules! gentest {
                     async: true,
                 });
             }
+            mod concurrent {
+                wasmtime::component::bindgen!({
+                    path: $path,
+                    async: true,
+                    concurrent_imports: true,
+                    concurrent_exports: true,
+                });
+            }
             mod tracing {
                 wasmtime::component::bindgen!({
                     path: $path,

crates/component-macro/tests/codegen.rs

@@ -15,6 +15,14 @@ macro_rules! genexpand {
             stringify: true,
         }))?;
 
+        process_expanded($path, "_concurrent", wasmtime::component::bindgen!({
+            path: $path,
+            async: true,
+            concurrent_imports: true,
+            concurrent_exports: true,
+            stringify: true,
+        }))?;
+
         process_expanded($path, "_tracing_async", wasmtime::component::bindgen!({
             path: $path,
             async: true,