Support wit bindgens adding producer information via component-type metadata (#930)

* wasm-metadata: refactor to make it easier to manipulate producers sections

previously the only api for updating a producers section was with
AddMetadata, this allows you to rewrite a wasm using that or just a
method on your own Producers

* wit-component: add facility for bindgen (metadata::encode) to plumb through additional producers info

* ComponentBuilder keeps track of a Producers section added to a
component
* metadata::encode gets an additional argument for an optional Producers
* ComponentEncoder::module decodes producers out of type metadata section, and
rewrites them into the core module

* fix

* fix fuzz target

Author: Pat Hickey

crates/wasm-metadata/src/lib.rs

@@ -1,5 +1,6 @@
 use std::mem;
 use wasm_encoder::*;
+use wasm_metadata::Producers;
 
 /// Helper type used when encoding a component to have helpers that
 /// simultaneously encode an item while returning its corresponding index in the
@@ -29,11 +30,18 @@ pub struct ComponentBuilder {
     instances: u32,
     types: u32,
     components: u32,
+
+    producers: Producers,
 }
 
 impl ComponentBuilder {
     pub fn finish(mut self) -> Vec<u8> {
-        self.component.section(&crate::producer_section());
+        // Ideally the Default for self.producers would be crate::base_producers,
+        // but in lieu we stick it in now:
+        let mut base = crate::base_producers();
+        base.merge(&self.producers);
+        // Write producers section as last section:
+        self.component.section(&base.section());
         self.flush();
         self.component.finish()
     }
@@ -197,6 +205,10 @@ impl ComponentBuilder {
             .instantiate(component_index, args);
         inc(&mut self.instances)
     }
+
+    pub fn add_producers(&mut self, producers: &Producers) {
+        self.producers.merge(producers)
+    }
 }
 
 // Helper macro to generate methods on `ComponentBuilder` to get specific

crates/wit-component/src/builder.rs

@@ -89,7 +89,7 @@ use wit_parser::{
 const INDIRECT_TABLE_NAME: &str = "$imports";
 
 mod wit;
-pub use wit::encode;
+pub use wit::{encode, encode_component};
 
 mod types;
 use types::{InstanceTypeEncoder, RootTypeEncoder, ValtypeEncoder};
@@ -982,14 +982,14 @@ impl<'a> EncodingState<'a> {
         shim.section(&tables);
         shim.section(&exports);
         shim.section(&code);
-        shim.section(&crate::producer_section());
+        shim.section(&crate::base_producers().section());
         shim.section(&names);
 
         let mut fixups = Module::default();
         fixups.section(&types);
         fixups.section(&imports_section);
         fixups.section(&elements);
-        fixups.section(&crate::producer_section());
+        fixups.section(&crate::base_producers().section());
         let mut names = NameSection::new();
         names.module("wit-component:fixups");
         fixups.section(&names);
@@ -1351,10 +1351,16 @@ impl ComponentEncoder {
     /// Set the core module to encode as a component.
     /// This method will also parse any component type information stored in custom sections
     /// inside the module, and add them as the interface, imports, and exports.
+    /// It will also add any producers information inside the component type information to the
+    /// core module.
     pub fn module(mut self, module: &[u8]) -> Result<Self> {
         let (wasm, metadata) = metadata::decode(module)?;
-        self.module = wasm;
         self.metadata.merge(metadata)?;
+        self.module = if let Some(producers) = &self.metadata.producers {
+            producers.add_to_wasm(&wasm)?
+        } else {
+            wasm
+        };
         Ok(self)
     }
 

crates/wit-component/src/encoding.rs

@@ -27,13 +27,19 @@ use wit_parser::*;
 /// The binary returned can be [`decode`d](crate::decode) to recover the WIT
 /// package provided.
 pub fn encode(resolve: &Resolve, package: PackageId) -> Result<Vec<u8>> {
+    Ok(encode_component(resolve, package)?.finish())
+}
+
+/// Exactly like `encode`, except gives an unfinished `ComponentBuilder` in case you need
+/// to append anything else before finishing.
+pub fn encode_component(resolve: &Resolve, package: PackageId) -> Result<ComponentBuilder> {
     let mut encoder = Encoder {
         component: ComponentBuilder::default(),
         resolve,
         package,
     };
     encoder.run()?;
-    Ok(encoder.component.finish())
+    Ok(encoder.component)
 }
 
 struct Encoder<'a> {

crates/wit-component/src/encoding/wit.rs

@@ -77,8 +77,8 @@ impl From<StringEncoding> for wasm_encoder::CanonicalOption {
 
 /// A producer section to be added to all modules and components synthesized by
 /// this crate
-pub(crate) fn producer_section() -> wasm_encoder::ProducersSection {
+pub(crate) fn base_producers() -> wasm_metadata::Producers {
     let mut producer = wasm_metadata::Producers::empty();
     producer.add("processed-by", "wit-component", env!("CARGO_PKG_VERSION"));
-    producer.section()
+    producer
 }

crates/wit-component/src/lib.rs

@@ -37,6 +37,7 @@ use crate::{DecodedWasm, StringEncoding};
 use anyhow::{bail, Context, Result};
 use indexmap::IndexMap;
 use wasm_encoder::Encode;
+use wasm_metadata::Producers;
 use wasmparser::BinaryReader;
 use wit_parser::{Document, Package, Resolve, World, WorldId, WorldItem};
 
@@ -53,6 +54,8 @@ pub struct Bindgen {
     pub world: WorldId,
     /// Metadata about this specific module that was bound.
     pub metadata: ModuleMetadata,
+    /// Producer information about tools used to produce this specific module.
+    pub producers: Option<Producers>,
 }
 
 impl Default for Bindgen {
@@ -82,6 +85,7 @@ impl Default for Bindgen {
             resolve,
             world,
             metadata: ModuleMetadata::default(),
+            producers: None,
         }
     }
 }
@@ -143,7 +147,12 @@ pub fn decode(wasm: &[u8]) -> Result<(Vec<u8>, Bindgen)> {
 /// into the final core wasm binary. The core wasm binary is later fed
 /// through `wit-component` to produce the actual component where this returned
 /// section will be decoded.
-pub fn encode(resolve: &Resolve, world: WorldId, encoding: StringEncoding) -> Result<Vec<u8>> {
+pub fn encode(
+    resolve: &Resolve,
+    world: WorldId,
+    encoding: StringEncoding,
+    producers: Option<&Producers>,
+) -> Result<Vec<u8>> {
     let world = &resolve.worlds[world];
     let doc = &resolve.documents[world.document];
     let pkg = &resolve.packages[doc.package.unwrap()];
@@ -167,7 +176,13 @@ pub fn encode(resolve: &Resolve, world: WorldId, encoding: StringEncoding) -> Re
     pkg.name.encode(&mut ret);
     doc.name.encode(&mut ret);
     world.name.encode(&mut ret);
-    ret.extend(crate::encoding::encode(resolve, doc.package.unwrap())?);
+    // This appends a wasm binary encoded Component to the ret:
+    let mut component_builder = crate::encoding::encode_component(resolve, doc.package.unwrap())?;
+
+    if let Some(p) = producers {
+        component_builder.add_producers(p);
+    }
+    ret.extend(component_builder.finish());
     Ok(ret)
 }
 
@@ -195,10 +210,12 @@ impl Bindgen {
         let doc = resolve.packages[pkg].documents[doc_name];
         let world = resolve.documents[doc].worlds[world_name];
         let metadata = ModuleMetadata::new(&resolve, world, encoding);
+        let producers = wasm_metadata::Producers::from_wasm(&data[reader.original_position()..])?;
         Ok(Bindgen {
             resolve,
             world,
             metadata,
+            producers,
         })
     }
 
@@ -220,6 +237,7 @@ impl Bindgen {
                     import_encodings,
                     export_encodings,
                 },
+            producers,
         } = other;
 
         let world = self.resolve.merge(resolve).worlds[world.index()];
@@ -249,6 +267,13 @@ impl Bindgen {
                 }
             }
         }
+        if let Some(producers) = producers {
+            if let Some(mine) = &mut self.producers {
+                mine.merge(&producers);
+            } else {
+                self.producers = Some(producers);
+            }
+        }
 
         Ok(())
     }

crates/wit-component/src/metadata.rs

@@ -63,6 +63,7 @@ fn component_encoding_via_flags() -> Result<()> {
                 continue;
             }
         };
+
         let wat = wasmprinter::print_bytes(&bytes)?;
         assert_output(&wat, &component_path)?;
         let (doc, resolve) = match wit_component::decode("component", &bytes)? {
@@ -71,6 +72,34 @@ fn component_encoding_via_flags() -> Result<()> {
         };
         let wit = DocumentPrinter::default().print(&resolve, doc)?;
         assert_output(&wit, &component_wit_path)?;
+
+        // Check that the producer data got piped through properly
+        let metadata = wasm_metadata::Metadata::from_binary(&bytes)?;
+        match metadata {
+            // Depends on the ComponentEncoder always putting the first module as the 0th child:
+            wasm_metadata::Metadata::Component { children, .. } => match children[0].as_ref() {
+                wasm_metadata::Metadata::Module { producers, .. } => {
+                    let producers = producers.as_ref().expect("child module has producers");
+                    let processed_by = producers
+                        .get("processed-by")
+                        .expect("child has processed-by section");
+                    assert_eq!(
+                        processed_by
+                            .get("wit-component")
+                            .expect("wit-component producer present"),
+                        env!("CARGO_PKG_VERSION")
+                    );
+                    assert_eq!(
+                        processed_by
+                            .get("my-fake-bindgen")
+                            .expect("added bindgen field present"),
+                        "123.45"
+                    );
+                }
+                _ => panic!("expected child to be a module"),
+            },
+            _ => panic!("expected top level metadata of component"),
+        }
     }
 
     Ok(())
@@ -100,7 +129,14 @@ fn read_core_module(path: &Path) -> Result<Vec<u8>> {
         &Default::default(),
     )?;
     let world = resolve.select_world(pkg, None)?;
-    let encoded = wit_component::metadata::encode(&resolve, world, StringEncoding::UTF8)?;
+
+    // Add this producer data to the wit-component metadata so we can make sure it gets through the
+    // translation:
+    let mut producers = wasm_metadata::Producers::empty();
+    producers.add("processed-by", "my-fake-bindgen", "123.45");
+
+    let encoded =
+        wit_component::metadata::encode(&resolve, world, StringEncoding::UTF8, Some(&producers))?;
 
     let section = wasm_encoder::CustomSection {
         name: "component-type",

crates/wit-component/tests/components.rs

@@ -54,7 +54,8 @@ fuzz_target!(|data: &[u8]| {
 
     for (id, _world) in resolve.worlds.iter() {
         let mut dummy = wit_component::dummy_module(&resolve, id);
-        let metadata = wit_component::metadata::encode(&resolve, id, StringEncoding::UTF8).unwrap();
+        let metadata =
+            wit_component::metadata::encode(&resolve, id, StringEncoding::UTF8, None).unwrap();
         let section = CustomSection {
             name: "component-type",
             data: &metadata,

fuzz/fuzz_targets/roundtrip-wit.rs

@@ -186,6 +186,7 @@ impl EmbedOpts {
             &resolve,
             world,
             self.encoding.unwrap_or(StringEncoding::UTF8),
+            None,
         )?;
 
         let section = wasm_encoder::CustomSection {