[sil-generic-specializer] Avoid unlimited generic specialization of very deeply nested bound generic types

The generic specialized would get out of control in certain cases and would not stop generating specializations of generic functions until it runs out of memory after a while.

rdar://21260480

Author: swiftix

lib/SILOptimizer/Utils/Generics.cpp

@@ -19,6 +19,29 @@
 
 using namespace swift;
 
+// Max depth of a bound generic which can be processed by the generic
+// specializer.
+// E.g. the depth of Array<Array<Array<T>>> is 3.
+// No specializations will be produced, if any of generic parameters contains
+// a bound generic type with the depth higher than this threshold 
+static const unsigned BoundGenericDepthThreshold = 50;
+
+static unsigned getBoundGenericDepth(Type t) {
+  unsigned Depth = 0;
+  if (auto BGT = t->getAs<BoundGenericType>()) {
+    Depth++;
+    auto GenericArgs = BGT->getGenericArgs();
+    unsigned MaxGenericArgDepth = 0;
+    for (auto GenericArg : GenericArgs) {
+      auto ArgDepth = getBoundGenericDepth(GenericArg);
+      if (ArgDepth > MaxGenericArgDepth)
+        MaxGenericArgDepth = ArgDepth;
+    }
+    Depth += MaxGenericArgDepth;
+  }
+  return Depth;
+}
+
 // =============================================================================
 // ReabstractionInfo
 // =============================================================================
@@ -50,6 +73,19 @@ ReabstractionInfo::ReabstractionInfo(SILFunction *OrigF,
     DEBUG(llvm::dbgs() << "    Cannot specialize with dynamic self.\n");
     return;
   }
+
+  // Check if the substitution contains any generic types that are too deep.
+  // If this is the case, bail to avoid the explosion in the number of 
+  // generated specializations.
+  for (auto Sub : ParamSubs) {
+    auto Replacement = Sub.getReplacement();
+    if (Replacement.findIf([](Type ty) -> bool {
+          return getBoundGenericDepth(ty) >= BoundGenericDepthThreshold;
+        })) {
+      return;
+    }
+  }
+
   SILModule &M = OrigF->getModule();
   Module *SM = M.getSwiftModule();
 

test/SILOptimizer/specialize_deep_generics.swift

@@ -0,0 +1,34 @@
+// RUN: %target-swift-frontend -O -emit-sil %s | FileCheck %s
+
+// Check that this Church Numerals inspired example does not hang
+// a compiler in the generic specializer.
+//
+// rdar://problem/21260480
+
+protocol Nat {
+  init()
+  // static stored properties are not supported in generic structs.
+  var val : Int32 {get}
+}
+
+struct Zero : Nat { var val : Int32 = 0 }
+
+struct PlusOne<X : Nat> : Nat {
+  var val : Int32 = X().val + 1
+}
+
+// Compiler used to keep performing the generic specialization of
+// computeNat for increasingly deeply nested bound generic types
+// like  PlusOne<PlusOne<....<PlusOne<Zero>>>
+func computeNat<T: Nat>(_ v : Int32, _ t: T) -> Int32 {
+  if v == 0 {
+    return t.val
+  }
+  return computeNat(v - 1, PlusOne<T>())
+}
+
+// CHECK-LABEL: sil @_TF24specialize_deep_generics14testComputeNatFT_Vs5Int32
+public func testComputeNat() -> Int32 {
+ return computeNat(8, Zero())
+}
+