wasm-smith: Implement support for generating GC instructions (#1382)

* wasm-smith: Implement support for generating GC instructions

Found and fixed a bunch of bugs in the validator, encoder, and the generator
itself. Since then, this has been fuzzing for about 18 hours without finding any
new bugs.

* review feedback

Author: fitzgen

crates/fuzz-stats/src/bin/failed-instantiations.rs

@@ -103,7 +103,7 @@ impl State {
         config.allow_start_export = false;
 
         // Wasmtime doesn't support this proposal yet.
-        config.tail_call_enabled = false;
+        config.gc_enabled = false;
 
         let mut wasm = wasm_smith::Module::new(config, &mut u)?;
         wasm.ensure_termination(10_000);

crates/wasm-encoder/src/core/code.rs

@@ -1614,8 +1614,8 @@ impl Encode for Instruction<'_> {
                 sink.push(0x18);
                 let cast_flags =
                     (from_ref_type.nullable as u8) | ((to_ref_type.nullable as u8) << 1);
-                relative_depth.encode(sink);
                 sink.push(cast_flags);
+                relative_depth.encode(sink);
                 from_ref_type.heap_type.encode(sink);
                 to_ref_type.heap_type.encode(sink);
             }
@@ -1628,8 +1628,8 @@ impl Encode for Instruction<'_> {
                 sink.push(0x19);
                 let cast_flags =
                     (from_ref_type.nullable as u8) | ((to_ref_type.nullable as u8) << 1);
-                relative_depth.encode(sink);
                 sink.push(cast_flags);
+                relative_depth.encode(sink);
                 from_ref_type.heap_type.encode(sink);
                 to_ref_type.heap_type.encode(sink);
             }

crates/wasm-encoder/src/core/types.rs

@@ -102,7 +102,7 @@ impl TryFrom<wasmparser::FuncType> for FuncType {
 }
 
 /// Represents a type of an array in a WebAssembly module.
-#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub struct ArrayType(pub FieldType);
 
 #[cfg(feature = "wasmparser")]
@@ -178,6 +178,21 @@ impl TryFrom<wasmparser::StorageType> for StorageType {
     }
 }
 
+impl StorageType {
+    /// Is this storage type defaultable?
+    pub fn is_defaultable(&self) -> bool {
+        self.unpack().is_defaultable()
+    }
+
+    /// Unpack this storage type into a value type.
+    pub fn unpack(&self) -> ValType {
+        match self {
+            StorageType::I8 | StorageType::I16 => ValType::I32,
+            StorageType::Val(v) => *v,
+        }
+    }
+}
+
 /// The type of a core WebAssembly value.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Ord, PartialOrd)]
 pub enum ValType {
@@ -216,6 +231,32 @@ impl TryFrom<wasmparser::ValType> for ValType {
     }
 }
 
+impl ValType {
+    /// Is this a numeric value type?
+    pub fn is_numeric(&self) -> bool {
+        match self {
+            ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 => true,
+            ValType::V128 | ValType::Ref(_) => false,
+        }
+    }
+
+    /// Is this a vector type?
+    pub fn is_vector(&self) -> bool {
+        match self {
+            ValType::V128 => true,
+            ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::Ref(_) => false,
+        }
+    }
+
+    /// Is this a reference type?
+    pub fn is_reference(&self) -> bool {
+        match self {
+            ValType::Ref(_) => true,
+            ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::V128 => false,
+        }
+    }
+}
+
 impl FuncType {
     /// Creates a new [`FuncType`] from the given `params` and `results`.
     pub fn new<P, R>(params: P, results: R) -> Self
@@ -257,6 +298,14 @@ impl ValType {
     pub const EXTERNREF: ValType = ValType::Ref(RefType::EXTERNREF);
     /// Alias for the `exnref` type in WebAssembly
     pub const EXNREF: ValType = ValType::Ref(RefType::EXNREF);
+
+    /// Is this value defaultable?
+    pub fn is_defaultable(&self) -> bool {
+        match self {
+            ValType::Ref(r) => r.nullable,
+            ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::V128 => true,
+        }
+    }
 }
 
 impl Encode for StorageType {

crates/wasm-smith/src/core.rs

@@ -249,13 +249,50 @@ pub(crate) struct SubType {
     pub(crate) composite_type: CompositeType,
 }
 
+impl SubType {
+    fn unwrap_struct(&self) -> &StructType {
+        self.composite_type.unwrap_struct()
+    }
+
+    fn unwrap_func(&self) -> &Rc<FuncType> {
+        self.composite_type.unwrap_func()
+    }
+
+    fn unwrap_array(&self) -> &ArrayType {
+        self.composite_type.unwrap_array()
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub(crate) enum CompositeType {
     Array(ArrayType),
     Func(Rc<FuncType>),
     Struct(StructType),
 }
 
+impl CompositeType {
+    fn unwrap_struct(&self) -> &StructType {
+        match self {
+            CompositeType::Struct(s) => s,
+            _ => panic!("not a struct"),
+        }
+    }
+
+    fn unwrap_func(&self) -> &Rc<FuncType> {
+        match self {
+            CompositeType::Func(f) => f,
+            _ => panic!("not a func"),
+        }
+    }
+
+    fn unwrap_array(&self) -> &ArrayType {
+        match self {
+            CompositeType::Array(a) => a,
+            _ => panic!("not an array"),
+        }
+    }
+}
+
 /// A function signature.
 #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
 pub(crate) struct FuncType {
@@ -392,6 +429,96 @@ impl Module {
         Ok(())
     }
 
+    #[inline]
+    fn val_type_is_sub_type(&self, a: ValType, b: ValType) -> bool {
+        match (a, b) {
+            (a, b) if a == b => true,
+            (ValType::Ref(a), ValType::Ref(b)) => self.ref_type_is_sub_type(a, b),
+            _ => false,
+        }
+    }
+
+    /// Is `a` a subtype of `b`?
+    fn ref_type_is_sub_type(&self, a: RefType, b: RefType) -> bool {
+        if a == b {
+            return true;
+        }
+
+        if a.nullable && !b.nullable {
+            return false;
+        }
+
+        self.heap_type_is_sub_type(a.heap_type, b.heap_type)
+    }
+
+    fn heap_type_is_sub_type(&self, a: HeapType, b: HeapType) -> bool {
+        use HeapType as HT;
+        match (a, b) {
+            (a, b) if a == b => true,
+
+            (HT::Eq | HT::I31 | HT::Struct | HT::Array | HT::None, HT::Any) => true,
+            (HT::I31 | HT::Struct | HT::Array | HT::None, HT::Eq) => true,
+            (HT::NoExtern, HT::Extern) => true,
+            (HT::NoFunc, HT::Func) => true,
+            (HT::None, HT::I31 | HT::Array | HT::Struct) => true,
+
+            (HT::Concrete(a), HT::Eq | HT::Any) => matches!(
+                self.ty(a).composite_type,
+                CompositeType::Array(_) | CompositeType::Struct(_)
+            ),
+
+            (HT::Concrete(a), HT::Struct) => {
+                matches!(self.ty(a).composite_type, CompositeType::Struct(_))
+            }
+
+            (HT::Concrete(a), HT::Array) => {
+                matches!(self.ty(a).composite_type, CompositeType::Array(_))
+            }
+
+            (HT::Concrete(a), HT::Func) => {
+                matches!(self.ty(a).composite_type, CompositeType::Func(_))
+            }
+
+            (HT::Concrete(mut a), HT::Concrete(b)) => loop {
+                if a == b {
+                    return true;
+                }
+                if let Some(supertype) = self.ty(a).supertype {
+                    a = supertype;
+                } else {
+                    return false;
+                }
+            },
+
+            (HT::None, HT::Concrete(b)) => matches!(
+                self.ty(b).composite_type,
+                CompositeType::Array(_) | CompositeType::Struct(_)
+            ),
+
+            (HT::NoFunc, HT::Concrete(b)) => {
+                matches!(self.ty(b).composite_type, CompositeType::Func(_))
+            }
+
+            // Nothing else matches. (Avoid full wildcard matches so that
+            // adding/modifying variants is easier in the future.)
+            (HT::Concrete(_), _)
+            | (HT::Func, _)
+            | (HT::Extern, _)
+            | (HT::Any, _)
+            | (HT::None, _)
+            | (HT::NoExtern, _)
+            | (HT::NoFunc, _)
+            | (HT::Eq, _)
+            | (HT::Struct, _)
+            | (HT::Array, _)
+            | (HT::I31, _) => false,
+
+            // TODO: `exn` probably will be its own type hierarchy and will
+            // probably get `noexn` as well.
+            (HT::Exn, _) => false,
+        }
+    }
+
     fn arbitrary_types(&mut self, u: &mut Unstructured) -> Result<()> {
         assert!(self.config.min_types <= self.config.max_types);
         while self.types.len() < self.config.min_types {
@@ -601,28 +728,17 @@ impl Module {
         }
     }
 
-    fn arbitrary_matching_ref_type(
-        &mut self,
-        u: &mut Unstructured,
-        ty: RefType,
-    ) -> Result<RefType> {
+    fn arbitrary_matching_ref_type(&self, u: &mut Unstructured, ty: RefType) -> Result<RefType> {
         Ok(RefType {
-            // TODO: For now, only create allow nullable reference
-            // types. Eventually we should support non-nullable reference types,
-            // but this means that we will also need to recognize when it is
-            // impossible to create an instance of the reference (eg `(ref
-            // nofunc)` has no instances, and self-referential types that
-            // contain a non-null self-reference are also impossible to create).
-            nullable: true,
+            nullable: ty.nullable,
             heap_type: self.arbitrary_matching_heap_type(u, ty.heap_type)?,
         })
     }
 
-    fn arbitrary_matching_heap_type(
-        &mut self,
-        u: &mut Unstructured,
-        ty: HeapType,
-    ) -> Result<HeapType> {
+    fn arbitrary_matching_heap_type(&self, u: &mut Unstructured, ty: HeapType) -> Result<HeapType> {
+        if !self.config.gc_enabled {
+            return Ok(ty);
+        }
         use HeapType as HT;
         let mut choices = vec![ty];
         match ty {
@@ -716,7 +832,7 @@ impl Module {
     }
 
     fn arbitrary_super_type_of_ref_type(
-        &mut self,
+        &self,
         u: &mut Unstructured,
         ty: RefType,
     ) -> Result<RefType> {
@@ -733,10 +849,13 @@ impl Module {
     }
 
     fn arbitrary_super_type_of_heap_type(
-        &mut self,
+        &self,
         u: &mut Unstructured,
         ty: HeapType,
     ) -> Result<HeapType> {
+        if !self.config.gc_enabled {
+            return Ok(ty);
+        }
         use HeapType as HT;
         let mut choices = vec![ty];
         match ty {
@@ -860,6 +979,45 @@ impl Module {
         }
     }
 
+    fn arbitrary_ref_type(&self, u: &mut Unstructured) -> Result<RefType> {
+        Ok(RefType {
+            nullable: true,
+            heap_type: self.arbitrary_heap_type(u)?,
+        })
+    }
+
+    fn arbitrary_heap_type(&self, u: &mut Unstructured) -> Result<HeapType> {
+        assert!(self.config.reference_types_enabled);
+
+        if self.config.gc_enabled && !self.types.is_empty() && u.arbitrary()? {
+            let type_ref_limit = u32::try_from(self.types.len()).unwrap();
+            let idx = u.int_in_range(0..=type_ref_limit)?;
+            return Ok(HeapType::Concrete(idx));
+        }
+
+        let mut choices = vec![HeapType::Func, HeapType::Extern];
+        if self.config.exceptions_enabled {
+            choices.push(HeapType::Exn);
+        }
+        if self.config.gc_enabled {
+            choices.extend(
+                [
+                    HeapType::Any,
+                    HeapType::None,
+                    HeapType::NoExtern,
+                    HeapType::NoFunc,
+                    HeapType::Eq,
+                    HeapType::Struct,
+                    HeapType::Array,
+                    HeapType::I31,
+                ]
+                .iter()
+                .copied(),
+            );
+        }
+        u.choose(&choices).copied()
+    }
+
     fn arbitrary_func_type(
         &mut self,
         u: &mut Unstructured,
@@ -2212,6 +2370,7 @@ flags! {
         Table,
         Memory,
         Control,
+        Aggregate,
     }
 }