SILOptimizer: Fancier 'too complex' check to prevent runaway specialization

We weren't looking at the length of an opaque archetype's type parameter,
which could lead to unbounded growth in the number of emitted specializations.

Fixes rdar://problem/121867690.

Author: slavapestov

lib/SILOptimizer/Utils/Generics.cpp

@@ -82,133 +82,59 @@ static const unsigned TypeDepthThreshold = 50;
 /// Set the width threshold rather high, because some projects uses very wide
 /// tuples to model fixed size arrays.
 static const unsigned TypeWidthThreshold = 2000;
+/// Max length of an opaque archetype's type parameter.
+static const unsigned TypeLengthThreshold = 10;
 
 /// Compute the width and the depth of a type.
 /// We compute both, because some pathological test-cases result in very
 /// wide types and some others result in very deep types. It is important
 /// to bail as soon as we hit the threshold on any of both dimensions to
 /// prevent compiler hangs and crashes.
-static std::pair<unsigned, unsigned> getTypeDepthAndWidth(Type t) {
-  unsigned Depth = 0;
-  unsigned Width = 0;
-  if (auto *BGT = t->getAs<BoundGenericType>()) {
-    auto *NTD = BGT->getNominalOrBoundGenericNominal();
-    if (NTD) {
-      auto StoredProperties = NTD->getStoredProperties();
-      Width += StoredProperties.size();
-    }
-    ++Depth;
-    unsigned MaxTypeDepth = 0;
-    auto GenericArgs = BGT->getGenericArgs();
-    for (auto Ty : GenericArgs) {
-      unsigned TypeWidth;
-      unsigned TypeDepth;
-      std::tie(TypeDepth, TypeWidth) = getTypeDepthAndWidth(Ty);
-      if (TypeDepth > MaxTypeDepth)
-        MaxTypeDepth = TypeDepth;
-      Width += TypeWidth;
-    }
-    Depth += MaxTypeDepth;
-    return std::make_pair(Depth, Width);
-  }
-
-  if (auto *TupleTy = t->getAs<TupleType>()) {
-    Width += TupleTy->getNumElements();
-    ++Depth;
-    unsigned MaxTypeDepth = 0;
-    auto ElementTypes = TupleTy->getElementTypes();
-    for (auto Ty : ElementTypes) {
-      unsigned TypeWidth;
-      unsigned TypeDepth;
-      std::tie(TypeDepth, TypeWidth) = getTypeDepthAndWidth(Ty);
-      if (TypeDepth > MaxTypeDepth)
-        MaxTypeDepth = TypeDepth;
-      Width += TypeWidth;
-    }
-    Depth += MaxTypeDepth;
-    return std::make_pair(Depth, Width);
-  }
-
-  if (auto *FnTy = t->getAs<SILFunctionType>()) {
-    ++Depth;
-    unsigned MaxTypeDepth = 0;
-    auto Params = FnTy->getParameters();
-    Width += Params.size();
-    for (auto Param : Params) {
-      unsigned TypeWidth;
-      unsigned TypeDepth;
-      std::tie(TypeDepth, TypeWidth) =
-        getTypeDepthAndWidth(Param.getInterfaceType());
-      if (TypeDepth > MaxTypeDepth)
-        MaxTypeDepth = TypeDepth;
-      Width += TypeWidth;
-    }
-    auto Results = FnTy->getResults();
-    Width += Results.size();
-    for (auto Result : Results) {
-      unsigned TypeWidth;
-      unsigned TypeDepth;
-      std::tie(TypeDepth, TypeWidth) =
-        getTypeDepthAndWidth(Result.getInterfaceType());
-      if (TypeDepth > MaxTypeDepth)
-        MaxTypeDepth = TypeDepth;
-      Width += TypeWidth;
-    }
-    if (FnTy->hasErrorResult()) {
-      Width += 1;
-      unsigned TypeWidth;
-      unsigned TypeDepth;
-      std::tie(TypeDepth, TypeWidth) =
-          getTypeDepthAndWidth(FnTy->getErrorResult().getInterfaceType());
-      if (TypeDepth > MaxTypeDepth)
-        MaxTypeDepth = TypeDepth;
-      Width += TypeWidth;
-    }
-    Depth += MaxTypeDepth;
-    return std::make_pair(Depth, Width);
-  }
-
-  if (auto *FnTy = t->getAs<FunctionType>()) {
-    ++Depth;
-    unsigned MaxTypeDepth = 0;
-    auto Params = FnTy->getParams();
-    Width += Params.size();
-    for (auto &Param : Params) {
-      unsigned TypeWidth;
-      unsigned TypeDepth;
-      std::tie(TypeDepth, TypeWidth) = getTypeDepthAndWidth(Param.getParameterType());
-      if (TypeDepth > MaxTypeDepth)
-        MaxTypeDepth = TypeDepth;
-      Width += TypeWidth;
-    }
-    unsigned TypeWidth;
-    unsigned TypeDepth;
-    std::tie(TypeDepth, TypeWidth) = getTypeDepthAndWidth(FnTy->getResult());
-    if (TypeDepth > MaxTypeDepth)
-      MaxTypeDepth = TypeDepth;
-    Width += TypeWidth;
-    Depth += MaxTypeDepth;
-    return std::make_pair(Depth, Width);
-  }
-
-  if (auto *MT = t->getAs<MetatypeType>()) {
-    Depth += 1;
-    unsigned TypeWidth;
-    unsigned TypeDepth;
-    std::tie(TypeDepth, TypeWidth) = getTypeDepthAndWidth(MT->getInstanceType());
-    Width += TypeWidth;
-    Depth += TypeDepth;
-    return std::make_pair(Depth, Width);
-  }
-
-  return std::make_pair(Depth, Width);
-}
-
 static bool isTypeTooComplex(Type t) {
-  unsigned TypeWidth;
-  unsigned TypeDepth;
-  std::tie(TypeDepth, TypeWidth) = getTypeDepthAndWidth(t);
-  return TypeWidth >= TypeWidthThreshold || TypeDepth >= TypeDepthThreshold;
+  struct Walker : TypeWalker {
+    unsigned Depth = 0;
+    unsigned MaxDepth = 0;
+    unsigned MaxWidth = 0;
+    unsigned MaxLength = 0;
+
+    Action walkToTypePre(Type ty) override {
+      // The TypeWalker won't visit the interface type encapsulated by the
+      // archetype, so we do it directly to measure its length.
+      if (auto *opaqueArchetypeTy = ty->getAs<OpaqueTypeArchetypeType>()) {
+        auto interfaceTy = opaqueArchetypeTy->getInterfaceType();
+
+        unsigned length = 0;
+        while (auto memberTy = interfaceTy->getAs<DependentMemberType>()) {
+          ++length;
+          interfaceTy = memberTy->getBase();
+        }
+        assert(interfaceTy->is<GenericTypeParamType>());
+
+        if (length > MaxLength)
+          MaxLength = length;
+      }
+
+      ++Depth;
+      MaxDepth = std::max(Depth, MaxDepth);
+
+      ++MaxWidth;
+
+      return Action::Continue;
+    }
+
+    Action walkToTypePost(Type ty) override {
+      --Depth;
+
+      return Action::Continue;
+    }
+  };
+
+  Walker walker;
+  t.walk(walker);
+
+  return (walker.MaxWidth >= TypeWidthThreshold ||
+          walker.MaxDepth >= TypeDepthThreshold ||
+          walker.MaxLength >= TypeLengthThreshold);
 }
 
 namespace {

test/SILOptimizer/specialize_opaque_type_archetypes_loop.swift

@@ -0,0 +1,22 @@
+// RUN: %target-swift-frontend -emit-sil -O %s -disable-availability-checking
+
+// This should not hang forever.
+
+public func foo() -> some P {
+  return S()
+}
+
+public protocol P {
+  associatedtype A: P
+  func p() -> A
+}
+
+public struct S: P {
+  public func p() -> some P { return self }
+}
+
+public func bar<T: P>(_ t: T, _ n: Int) {
+  if n > 0 { bar(t.p(), n - 1) }
+}
+
+bar(foo(), 1000)