[Wasm GC] Skip types with subtypes in SignatureRefining (#5544)

For now just skip them, to avoid problems. In the future we should look
into modifying their children, when possible.

Fixes #5463

Author: kripken

src/passes/SignatureRefining.cpp

@@ -30,6 +30,7 @@
 #include "ir/find_all.h"
 #include "ir/lubs.h"
 #include "ir/module-utils.h"
+#include "ir/subtypes.h"
 #include "ir/type-updating.h"
 #include "ir/utils.h"
 #include "pass.h"
@@ -139,6 +140,16 @@ struct SignatureRefining : public Pass {
       allInfo[exportedFunc->type].canModify = false;
     }
 
+    // For now, do not optimize types that have subtypes. When we modify such a
+    // type we need to modify subtypes as well, similar to the analysis in
+    // TypeRefining, and perhaps we can unify this pass with that. TODO
+    SubTypes subTypes(*module);
+    for (auto& [type, info] : allInfo) {
+      if (!subTypes.getStrictSubTypes(type).empty()) {
+        info.canModify = false;
+      }
+    }
+
     bool refinedResults = false;
 
     // Compute optimal LUBs.

test/lit/passes/signature-refining.wast

@@ -758,3 +758,25 @@
     )
   )
 )
+
+;; If we refine a type, that may require changes to its subtypes. For now, we
+;; skip such optimizations. TODO
+(module
+  (rec
+    ;; CHECK:      (type $A (func (param (ref null $B)) (result (ref null $A))))
+    (type $A (func         (param (ref null $B)) (result (ref null $A))))
+    ;; CHECK:      (type $B (func_subtype (param (ref null $A)) (result (ref null $B)) $A))
+    (type $B (func_subtype (param (ref null $A)) (result (ref null $B)) $A))
+  )
+
+  ;; CHECK:      (elem declare func $func)
+
+  ;; CHECK:      (func $func (type $A) (param $0 (ref null $B)) (result (ref null $A))
+  ;; CHECK-NEXT:  (ref.func $func)
+  ;; CHECK-NEXT: )
+  (func $func (type $A) (param $0 (ref null $B)) (result (ref null $A))
+    ;; This result is non-nullable, and we could refine type $A accordingly. But
+    ;; if we did that, we'd need to refine $B as well.
+    (ref.func $func)
+  )
+)