Implement merging semver-compatible interfaces in imports (#1815)

* Implement merging semver-compatible interfaces in imports

This commit is the first half and the meat of the implementation
of #1774 where, by default, WIT processing tools will now merge world
imports when possible based on semver versions. The precise shape of how
this is done has gone through a few iterations and this is the end
result I've ended up settling on. This should handle all the various
cases we're interested in and continue to produce valid worlds within a
`Resolve` (with the help of `elaborate_world` added in #1800). CLI
tooling has been updated with flags to configure this behavior but the
behavior is now enabled by default.

* Add broken test case for wit-component

* Match wasm/component import names based on versions

This commit implements the final bit of #1774 where the `wit-component`
crate will now match imports based on semver version in addition to the
previous exact-name matching that was performed previously.

* Don't panic on duplicate shims in wit-component

This is now possible with import version matching so refactor the
internal data structures to better support insertion of possibly
duplicate shims. Most of wit-component was already ready for this
refactoring, it was just the initial generation of shims that needed to
be reorganized slightly.

* wip

* Elaborate all worlds after semver merging

Any interface could be modified, so elaborate all of them to fixup anything
that needs new imports.

* Only merge once at the end, not continuously

This commit updates how the semver-merging bits of `Resolve` work by
moving it towards the end of the encoding process rather than
once-per-world-merged. That helps both deduplicate work and fix some
asserts I'm seeing that are tripping if a `Resolve` is import-merged and
then merged again elsewhere.

This additionally simplifies some APIs because the boolean of what to do
isn't threaded to quite so many locations.

* Fix test

* Update interface deps of exports too

Dependencies might depend on replaced interfaces so the dependency edges
there need to be updated in the same manner as imports.

* Fix fuzzer build

Also ensure to at least try to test the new merging function.

Author: alexcrichton

crates/wit-component/src/encoding.rs

@@ -1222,6 +1222,11 @@ pub struct Linker {
     ///
     /// If `None`, use `DEFAULT_STACK_SIZE_BYTES`.
     stack_size: Option<u32>,
+
+    /// This affects how when to WIT worlds are merged together, for example
+    /// from two different libraries, whether their imports are unified when the
+    /// semver version ranges for interface allow it.
+    merge_imports_based_on_semver: Option<bool>,
 }
 
 impl Linker {
@@ -1269,6 +1274,16 @@ impl Linker {
         self
     }
 
+    /// This affects how when to WIT worlds are merged together, for example
+    /// from two different libraries, whether their imports are unified when the
+    /// semver version ranges for interface allow it.
+    ///
+    /// This is enabled by default.
+    pub fn merge_imports_based_on_semver(mut self, merge: bool) -> Self {
+        self.merge_imports_based_on_semver = Some(merge);
+        self
+    }
+
     /// Encode the component and return the bytes
     pub fn encode(mut self) -> Result<Vec<u8>> {
         if self.use_built_in_libdl {
@@ -1415,9 +1430,11 @@ impl Linker {
             self.stack_size.unwrap_or(DEFAULT_STACK_SIZE_BYTES),
         );
 
-        let mut encoder = ComponentEncoder::default()
-            .validate(self.validate)
-            .module(&env_module)?;
+        let mut encoder = ComponentEncoder::default().validate(self.validate);
+        if let Some(merge) = self.merge_imports_based_on_semver {
+            encoder = encoder.merge_imports_based_on_semver(merge);
+        };
+        encoder = encoder.module(&env_module)?;
 
         for (name, module) in &self.adapters {
             encoder = encoder.adapter(name, module)?;

crates/wit-component/src/linking.rs

@@ -44,14 +44,14 @@
 use crate::validation::BARE_FUNC_MODULE_NAME;
 use crate::{DecodedWasm, StringEncoding};
 use anyhow::{bail, Context, Result};
-use indexmap::IndexMap;
+use indexmap::{IndexMap, IndexSet};
 use std::borrow::Cow;
 use wasm_encoder::{
     ComponentBuilder, ComponentExportKind, ComponentType, ComponentTypeRef, CustomSection,
 };
 use wasm_metadata::Producers;
 use wasmparser::{BinaryReader, Encoding, Parser, Payload};
-use wit_parser::{Package, PackageName, Resolve, World, WorldId, WorldItem};
+use wit_parser::{Package, PackageName, Resolve, World, WorldId, WorldItem, WorldKey};
 
 const CURRENT_VERSION: u8 = 0x04;
 const CUSTOM_SECTION_NAME: &str = "wit-component-encoding";
@@ -298,7 +298,10 @@ impl Bindgen {
     ///
     /// Note that at this time there's no support for changing string encodings
     /// between metadata.
-    pub fn merge(&mut self, other: Bindgen) -> Result<WorldId> {
+    ///
+    /// This function returns the set of exports that the main world of
+    /// `other` added to the world in `self`.
+    pub fn merge(&mut self, other: Bindgen) -> Result<IndexSet<WorldKey>> {
         let Bindgen {
             resolve,
             world,
@@ -315,6 +318,7 @@ impl Bindgen {
             .merge(resolve)
             .context("failed to merge WIT package sets together")?
             .map_world(world, None)?;
+        let exports = self.resolve.worlds[world].exports.keys().cloned().collect();
         self.resolve
             .merge_worlds(world, self.world)
             .context("failed to merge worlds from two documents")?;
@@ -349,7 +353,7 @@ impl Bindgen {
             }
         }
 
-        Ok(world)
+        Ok(exports)
     }
 }
 

crates/wit-component/src/metadata.rs

@@ -414,14 +414,80 @@ impl ImportMap {
         resolve: &Resolve,
         items: &IndexMap<WorldKey, WorldItem>,
     ) -> Result<(WorldKey, InterfaceId)> {
-        let key = world_key(resolve, module);
-        match items.get(&key) {
-            Some(WorldItem::Interface { id, .. }) => Ok((key, *id)),
-            Some(WorldItem::Function(_) | WorldItem::Type(_)) => {
-                bail!("import `{module}` is not an interface")
+        // First see if this is a bare name
+        let bare_name = WorldKey::Name(module.to_string());
+        if let Some(WorldItem::Interface { id, .. }) = items.get(&bare_name) {
+            return Ok((bare_name, *id));
+        }
+
+        // ... and if this isn't a bare name then it's time to do some parsing
+        // related to interfaces, versions, and such. First up the `module` name
+        // is parsed as a normal component name from `wasmparser` to see if it's
+        // of the "interface kind". If it's not then that means the above match
+        // should have been a hit but it wasn't, so an error is returned.
+        let kebab_name = ComponentName::new(module, 0);
+        let name = match kebab_name.as_ref().map(|k| k.kind()) {
+            Ok(ComponentNameKind::Interface(name)) => name,
+            _ => bail!("module requires an import interface named `{module}`"),
+        };
+
+        // Prioritize an exact match based on versions, so try that first.
+        let pkgname = PackageName {
+            namespace: name.namespace().to_string(),
+            name: name.package().to_string(),
+            version: name.version(),
+        };
+        if let Some(pkg) = resolve.package_names.get(&pkgname) {
+            if let Some(id) = resolve.packages[*pkg]
+                .interfaces
+                .get(name.interface().as_str())
+            {
+                let key = WorldKey::Interface(*id);
+                if items.contains_key(&key) {
+                    return Ok((key, *id));
+                }
+            }
+        }
+
+        // If an exact match wasn't found then instead search for the first
+        // match based on versions. This means that a core wasm import for
+        // "1.2.3" might end up matching an interface at "1.2.4", for example.
+        // (or "1.2.2", depending on what's available).
+        for (key, _) in items {
+            let id = match key {
+                WorldKey::Interface(id) => *id,
+                WorldKey::Name(_) => continue,
+            };
+            // Make sure the interface names match
+            let interface = &resolve.interfaces[id];
+            if interface.name.as_ref().unwrap() != name.interface().as_str() {
+                continue;
+            }
+
+            // Make sure the package name (without version) matches
+            let pkg = &resolve.packages[interface.package.unwrap()];
+            if pkg.name.namespace != pkgname.namespace || pkg.name.name != pkgname.name {
+                continue;
+            }
+
+            let module_version = match name.version() {
+                Some(version) => version,
+                None => continue,
+            };
+            let pkg_version = match &pkg.name.version {
+                Some(version) => version,
+                None => continue,
+            };
+
+            // Test if the two semver versions are compatible
+            let module_compat = PackageName::version_compat_track(&module_version);
+            let pkg_compat = PackageName::version_compat_track(pkg_version);
+            if module_compat == pkg_compat {
+                return Ok((key.clone(), id));
             }
-            None => bail!("module requires an import interface named `{module}`"),
         }
+
+        bail!("module requires an import interface named `{module}`")
     }
 
     fn classify_import_with_library(
@@ -907,29 +973,6 @@ pub fn validate_adapter_module(
     Ok(ret)
 }
 
-fn world_key(resolve: &Resolve, name: &str) -> WorldKey {
-    let kebab_name = ComponentName::new(name, 0);
-    let (pkgname, interface) = match kebab_name.as_ref().map(|k| k.kind()) {
-        Ok(ComponentNameKind::Interface(name)) => {
-            let pkgname = PackageName {
-                namespace: name.namespace().to_string(),
-                name: name.package().to_string(),
-                version: name.version(),
-            };
-            (pkgname, name.interface().as_str())
-        }
-        _ => return WorldKey::Name(name.to_string()),
-    };
-    match resolve
-        .package_names
-        .get(&pkgname)
-        .and_then(|p| resolve.packages[*p].interfaces.get(interface))
-    {
-        Some(id) => WorldKey::Interface(*id),
-        None => WorldKey::Name(name.to_string()),
-    }
-}
-
 fn valid_resource_drop(
     func_name: &str,
     ty: &FuncType,

crates/wit-component/src/validation.rs

@@ -0,0 +1,10 @@
+(module
+  (import "a:b/[email protected]" "[constructor]r" (func (result i32)))
+  (import "a:b/[email protected]" "[resource-drop]r" (func (param i32)))
+  (import "a:b/[email protected]" "[method]r.x" (func (param i32)))
+  (import "a:b/[email protected]" "x" (func (param i32 i32)))
+  (import "a:b/[email protected]" "y" (func))
+
+  (func (export "f"))
+)
+

crates/wit-component/tests/components/merge-import-versions-with-adapter/adapt-old.wat

@@ -0,0 +1,19 @@
+package foo:foo;
+
+world adapt-old {
+  import a:b/c@0.1.1;
+}
+
+package a:b@0.1.1 {
+  interface c {
+    x: func(x: string);
+    y: func();
+
+    resource r {
+      constructor();
+
+      x: func();
+    }
+  }
+}
+

crates/wit-component/tests/components/merge-import-versions-with-adapter/adapt-old.wit

@@ -0,0 +1,104 @@
+(component
+  (type (;0;)
+    (instance
+      (export (;0;) "r" (type (sub resource)))
+      (type (;1;) (own 0))
+      (type (;2;) (func (result 1)))
+      (export (;0;) "[constructor]r" (func (type 2)))
+      (type (;3;) (func (param "x" string)))
+      (export (;1;) "x" (func (type 3)))
+    )
+  )
+  (import "a:b/[email protected]" (instance (;0;) (type 0)))
+  (core module (;0;)
+    (type (;0;) (func))
+    (type (;1;) (func (result i32)))
+    (type (;2;) (func (param i32)))
+    (type (;3;) (func (param i32 i32)))
+    (import "old" "f" (func (;0;) (type 0)))
+    (import "a:b/[email protected]" "[constructor]r" (func (;1;) (type 1)))
+    (import "a:b/[email protected]" "[resource-drop]r" (func (;2;) (type 2)))
+    (import "a:b/[email protected]" "x" (func (;3;) (type 3)))
+    (memory (;0;) 1)
+    (export "memory" (memory 0))
+    (@producers
+      (processed-by "wit-component" "$CARGO_PKG_VERSION")
+      (processed-by "my-fake-bindgen" "123.45")
+    )
+  )
+  (core module (;1;)
+    (type (;0;) (func))
+    (export "f" (func 0))
+    (func (;0;) (type 0))
+  )
+  (core module (;2;)
+    (type (;0;) (func))
+    (type (;1;) (func (param i32 i32)))
+    (table (;0;) 2 2 funcref)
+    (export "0" (func $adapt-old-f))
+    (export "1" (func $indirect-a:b/[email protected]))
+    (export "$imports" (table 0))
+    (func $adapt-old-f (;0;) (type 0)
+      i32.const 0
+      call_indirect (type 0)
+    )
+    (func $indirect-a:b/[email protected] (;1;) (type 1) (param i32 i32)
+      local.get 0
+      local.get 1
+      i32.const 1
+      call_indirect (type 1)
+    )
+    (@producers
+      (processed-by "wit-component" "$CARGO_PKG_VERSION")
+    )
+  )
+  (core module (;3;)
+    (type (;0;) (func))
+    (type (;1;) (func (param i32 i32)))
+    (import "" "0" (func (;0;) (type 0)))
+    (import "" "1" (func (;1;) (type 1)))
+    (import "" "$imports" (table (;0;) 2 2 funcref))
+    (elem (;0;) (i32.const 0) func 0 1)
+    (@producers
+      (processed-by "wit-component" "$CARGO_PKG_VERSION")
+    )
+  )
+  (core instance (;0;) (instantiate 2))
+  (alias core export 0 "0" (core func (;0;)))
+  (core instance (;1;)
+    (export "f" (func 0))
+  )
+  (alias export 0 "[constructor]r" (func (;0;)))
+  (core func (;1;) (canon lower (func 0)))
+  (alias export 0 "r" (type (;1;)))
+  (core func (;2;) (canon resource.drop 1))
+  (alias core export 0 "1" (core func (;3;)))
+  (core instance (;2;)
+    (export "[constructor]r" (func 1))
+    (export "[resource-drop]r" (func 2))
+    (export "x" (func 3))
+  )
+  (core instance (;3;) (instantiate 0
+      (with "old" (instance 1))
+      (with "a:b/[email protected]" (instance 2))
+    )
+  )
+  (alias core export 3 "memory" (core memory (;0;)))
+  (core instance (;4;) (instantiate 1))
+  (alias core export 0 "$imports" (core table (;0;)))
+  (alias core export 4 "f" (core func (;4;)))
+  (alias export 0 "x" (func (;1;)))
+  (core func (;5;) (canon lower (func 1) (memory 0) string-encoding=utf8))
+  (core instance (;5;)
+    (export "$imports" (table 0))
+    (export "0" (func 4))
+    (export "1" (func 5))
+  )
+  (core instance (;6;) (instantiate 3
+      (with "" (instance 5))
+    )
+  )
+  (@producers
+    (processed-by "wit-component" "$CARGO_PKG_VERSION")
+  )
+)

crates/wit-component/tests/components/merge-import-versions-with-adapter/component.wat

@@ -0,0 +1,5 @@
+package root:component;
+
+world root {
+  import a:b/[email protected];
+}

crates/wit-component/tests/components/merge-import-versions-with-adapter/component.wit.print

@@ -0,0 +1,9 @@
+(module
+  (import "old" "f" (func))
+
+  (import "a:b/[email protected]" "[constructor]r" (func (result i32)))
+  (import "a:b/[email protected]" "[resource-drop]r" (func (param i32)))
+  (import "a:b/[email protected]" "x" (func (param i32 i32)))
+
+  (memory (export "memory") 1)
+)

crates/wit-component/tests/components/merge-import-versions-with-adapter/module.wat

@@ -0,0 +1,14 @@
+package foo:foo;
+
+world module {
+  import a:b/c@0.1.0;
+}
+
+package a:b@0.1.0 {
+  interface c {
+    resource r {
+      constructor();
+    }
+    x: func(x: string);
+  }
+}