[vm,dyn_modules] Fix canonicalization of types when serializing bytecode

Avoid caching generic types C<T1,...,Tn> in the object table if
any of type arguments T1,...,Tn are not cacheable.
Otherwise, bytecode reader might create function type
'C<Y0> Function<Y0>()' where cached type 'C<Y0>' references parameter
type Y0 which belongs to a wrong owner, such as another function type
'C<Y0> Function<Y0>(int)'.

Also, fix equivalence of default values of function type parameters
in case matching 'null' (meaning all dynamic) and '[dynamic, ..., dynamic]'.
This may happen during canonicalization before '[dynamic, ..., dynamic]'
type arguments are replaced with 'null'.

TEST=ci

Change-Id: If81736fb18a62fd16fd93e59436c8cd7ccce94ff
Cq-Include-Trybots: luci.dart.try:vm-aot-dyn-linux-debug-x64-try,vm-aot-dyn-linux-product-x64-try,vm-dyn-linux-debug-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/437522
Reviewed-by: Slava Egorov <[email protected]>
Commit-Queue: Alexander Markov <[email protected]>

Author: alexmarkov

pkg/dart2bytecode/lib/object_table.dart

@@ -650,6 +650,9 @@ class _GenericTypeHandle extends _TypeHandle {
   _GenericTypeHandle(this.class_, this.typeArgs, Nullability nullability)
       : super(TypeTag.kGenericType, nullability);
 
+  @override
+  bool get isCacheable => typeArgs?.isCacheable ?? true;
+
   @override
   void writeContents(BufferedWriter writer) {
     writer.writePackedObject(class_);

runtime/vm/object.cc

@@ -7449,8 +7449,10 @@ bool TypeArguments::IsSubvectorEquivalent(
     return true;
   }
   if (kind == TypeEquality::kCanonical) {
-    if (IsNull() || other.IsNull()) {
-      return false;
+    if (IsNull()) {
+      return other.IsRaw(from_index, len);
+    } else if (other.IsNull()) {
+      return IsRaw(from_index, len);
     }
     if (Length() != other.Length()) {
       return false;
@@ -10597,16 +10599,12 @@ bool FunctionType::HasSameTypeParametersAndBounds(
             TypeArguments::Handle(zone, type_params.defaults());
         const TypeArguments& other_defaults =
             TypeArguments::Handle(zone, other_type_params.defaults());
-        if (defaults.IsNull()) {
-          if (!other_defaults.IsNull()) {
-            TRACE_TYPE_CHECKS_VERBOSE(
-                "   - result: false (mismatch in defaults)\n");
-            return false;
-          }
-        } else if (!defaults.IsEquivalent(other_defaults, kind,
-                                          function_type_equivalence)) {
+        if (!defaults.IsEquivalent(other_defaults, kind,
+                                   function_type_equivalence)) {
           TRACE_TYPE_CHECKS_VERBOSE(
-              "   - result: false (default types are not equivalent)\n");
+              "   - result: false (default types are not equivalent,"
+              " %s - %s)\n",
+              defaults.ToCString(), other_defaults.ToCString());
           return false;
         }
       }

runtime/vm/object.h

@@ -8839,14 +8839,20 @@ class TypeArguments : public Instance {
 
   // Check if the vectors are equal (they may be null).
   bool Equals(const TypeArguments& other) const {
-    return IsSubvectorEquivalent(other, 0, IsNull() ? 0 : Length(),
+    if (ptr() == other.ptr()) {
+      return true;
+    }
+    return IsSubvectorEquivalent(other, 0, IsNull() ? other.Length() : Length(),
                                  TypeEquality::kCanonical);
   }
 
   bool IsEquivalent(
       const TypeArguments& other,
       TypeEquality kind,
       FunctionTypeMapping* function_type_equivalence = nullptr) const {
+    if (ptr() == other.ptr()) {
+      return true;
+    }
     // Make a null vector a vector of dynamic as long as the other vector.
     return IsSubvectorEquivalent(other, 0, IsNull() ? other.Length() : Length(),
                                  kind, function_type_equivalence);