[ConstraintSystem] Strengthen dependent member type checks while inferring bindings

Look through specifier (inout, l-value) and optional types while
checking for presence of dependent member types to avoid inferring
incorrect bindings (which could lead to infinite recursion in the
solver).

Resolves: SR-13856
Resolves: rdar://problem/71383770

Author: xedin

lib/Sema/CSBindings.cpp

@@ -865,7 +865,9 @@ ConstraintSystem::getPotentialBindingForRelationalConstraint(
 
   // If the type we'd be binding to is a dependent member, don't try to
   // resolve this type variable yet.
-  if (type->is<DependentMemberType>()) {
+  if (type->getWithoutSpecifierType()
+          ->lookThroughAllOptionalTypes()
+          ->is<DependentMemberType>()) {
     SmallVector<TypeVariableType *, 4> referencedVars;
     type->getTypeVariables(referencedVars);
 
@@ -1210,15 +1212,19 @@ Optional<Type> ConstraintSystem::checkTypeOfBinding(TypeVariableType *typeVar,
     }
   }
 
-  // If the type is a type variable itself, don't permit the binding.
-  if (type->getRValueType()->is<TypeVariableType>())
-    return None;
+  {
+    auto objType = type->getWithoutSpecifierType();
 
-  // Don't bind to a dependent member type, even if it's currently
-  // wrapped in any number of optionals, because binding producer
-  // might unwrap and try to attempt it directly later.
-  if (type->lookThroughAllOptionalTypes()->is<DependentMemberType>())
-    return None;
+    // If the type is a type variable itself, don't permit the binding.
+    if (objType->is<TypeVariableType>())
+      return None;
+
+    // Don't bind to a dependent member type, even if it's currently
+    // wrapped in any number of optionals, because binding producer
+    // might unwrap and try to attempt it directly later.
+    if (objType->lookThroughAllOptionalTypes()->is<DependentMemberType>())
+      return None;
+  }
 
   // Okay, allow the binding (with the simplified type).
   return type;

validation-test/Sema/type_checker_crashers_fixed/sr13856.swift

@@ -0,0 +1,42 @@
+// RUN: %target-typecheck-verify-swift
+
+protocol OptionalType {
+  associatedtype Wrapped
+  var wrapped: Wrapped? { get }
+}
+
+protocol DefaultsBridge {
+  associatedtype T
+}
+
+protocol DefaultsSerializable {
+  typealias T = Bridge.T
+  associatedtype Bridge: DefaultsBridge
+}
+
+protocol DefaultsKeyStore {}
+
+struct DefaultsKeys: DefaultsKeyStore {}
+struct DefaultsKey<ValueType> {}
+
+@dynamicMemberLookup
+struct DefaultsAdapter<KeyStore: DefaultsKeyStore> {
+  @available(*, unavailable)
+  subscript(dynamicMember member: String) -> Never {
+    fatalError()
+  }
+
+  subscript<T: DefaultsSerializable>(keyPath: KeyPath<KeyStore, DefaultsKey<T>>) -> T.T where T.T == T {
+    get { fatalError() }
+    set { fatalError() }
+  }
+
+  subscript<T: DefaultsSerializable>(dynamicMember keyPath: KeyPath<KeyStore, DefaultsKey<T>>) -> T.T where T: OptionalType, T.T == T {
+    get { fatalError() }
+    set { fatalError() }
+  }
+}
+
+
+var Defaults = DefaultsAdapter<DefaultsKeys>()
+Defaults[\.missingKey] = "" // expected-error {{}}