Fix HeapStoreOptimization on struct.new traps due to null descriptors (#7761)

Fixes #7759

Author: kripken

scripts/test/fuzzing.py

@@ -132,6 +132,7 @@
     'remove-unused-brs-desc.wast',
     'vacuum-desc.wast',
     'j2cl-merge-itables-desc.wast',
+    'heap-store-optimization-desc.wast',
     # TODO: fix split_wast() on tricky escaping situations like a string ending
     #       in \\" (the " is not escaped - there is an escaped \ before it)
     'string-lifting-section.wast',

src/passes/HeapStoreOptimization.cpp

@@ -248,6 +248,16 @@ struct HeapStoreOptimization
       }
     }
 
+    // We also cannot reorder if the struct.new itself has effects (which it can
+    // in the case of a descriptor) that interact with X' (from the comment
+    // above), as e.g. a struct.new trap happens before effects in X', but after
+    // the optimization X' would happen first.
+    ShallowEffectAnalyzer structNewEffects(
+      getPassOptions(), *getModule(), new_);
+    if (structNewEffects.invalidates(setValueEffects)) {
+      return false;
+    }
+
     // We must also be careful of branches out from the value that skip the
     // local.set, see below.
     if (canSkipLocalSet(set, setValueEffects, localSet)) {

test/lit/passes/heap-store-optimization-desc.wast

@@ -0,0 +1,79 @@
+;; NOTE: Assertions have been generated by update_lit_checks.py and should not be edited.
+;; RUN: wasm-opt %s --heap-store-optimization -all -S -o - \
+;; RUN:   | filecheck %s
+
+(module
+  (rec
+    ;; CHECK:      (rec
+    ;; CHECK-NEXT:  (type $struct (descriptor $desc (struct (field (mut i32)))))
+    (type $struct (sub final (descriptor $desc (struct (field (mut i32))))))
+    ;; CHECK:       (type $desc (sub (describes $struct (struct))))
+    (type $desc (sub (describes $struct (struct))))
+  )
+
+  ;; CHECK:      (import "" "" (func $effect (type $2)))
+  (import "" "" (func $effect))
+
+  ;; CHECK:      (func $no-reorder (type $2)
+  ;; CHECK-NEXT:  (local $struct (ref $struct))
+  ;; CHECK-NEXT:  (local.set $struct
+  ;; CHECK-NEXT:   (struct.new_default $struct
+  ;; CHECK-NEXT:    (ref.null none)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (struct.set $struct 0
+  ;; CHECK-NEXT:   (local.get $struct)
+  ;; CHECK-NEXT:   (block $block (result i32)
+  ;; CHECK-NEXT:    (call $effect)
+  ;; CHECK-NEXT:    (i32.const 0)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT: )
+  (func $no-reorder
+    (local $struct (ref $struct))
+    (local.set $struct
+      ;; This traps on the null descriptor, so we cannot optimize the struct.set
+      ;; into the struct.new (which would call $effect before the trap).
+      (struct.new_default $struct
+        (ref.null none)
+      )
+    )
+    (struct.set $struct 0
+      (local.get $struct)
+      (block $block (result i32)
+        (call $effect)
+        (i32.const 0)
+      )
+    )
+  )
+
+  ;; CHECK:      (func $yes-reorder (type $2)
+  ;; CHECK-NEXT:  (local $struct (ref $struct))
+  ;; CHECK-NEXT:  (local.set $struct
+  ;; CHECK-NEXT:   (struct.new $struct
+  ;; CHECK-NEXT:    (block $block (result i32)
+  ;; CHECK-NEXT:     (call $effect)
+  ;; CHECK-NEXT:     (i32.const 0)
+  ;; CHECK-NEXT:    )
+  ;; CHECK-NEXT:    (struct.new_default $desc)
+  ;; CHECK-NEXT:   )
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (nop)
+  ;; CHECK-NEXT: )
+  (func $yes-reorder
+    (local $struct (ref $struct))
+    ;; As above, but now the descriptor does not trap, so we optimize.
+    (local.set $struct
+      (struct.new_default $struct
+        (struct.new $desc)
+      )
+    )
+    (struct.set $struct 0
+      (local.get $struct)
+      (block $block (result i32)
+        (call $effect)
+        (i32.const 0)
+      )
+    )
+  )
+)