Sema: Fix some crashes with invalid nesting of extensions and protocols

Make sure we consistently use getParentForLookup() and not getParent()
when looking at generic DeclContexts. This is because an extension or
protocol that is nested inside of another generic context must never
inherit generic parameters from the parent context.

We already had this invariant enforced in some places, but now that we
do it more consistently we can fix more crashes of this kind.

Fixes <rdar://problem/58813746>, <https://bugs.swift.org/browse/SR-13004>.

Author: slavapestov

lib/AST/DeclContext.cpp

@@ -301,13 +301,9 @@ bool DeclContext::isGenericContext() const {
       if (auto GC = decl->getAsGenericContext()) {
         if (GC->getGenericParams())
           return true;
-
-        // Extensions do not capture outer generic parameters.
-        if (isa<ExtensionDecl>(decl))
-          break;
       }
     }
-  } while ((dc = dc->getParent()));
+  } while ((dc = dc->getParentForLookup()));
 
   return false;
 }

lib/Sema/ConstraintSystem.cpp

@@ -676,8 +676,11 @@ Type ConstraintSystem::openUnboundGenericType(
     for (auto pair : subs) {
       auto found = replacements.find(
         cast<GenericTypeParamType>(pair.first));
-      assert(found != replacements.end() &&
-             "Missing generic parameter?");
+      if (found == replacements.end()) {
+        // Can happen with invalid generic code.
+        continue;
+      }
+
       addConstraint(ConstraintKind::Bind, found->second, pair.second,
                     locator);
     }

lib/Sema/TypeCheckDeclPrimary.cpp

@@ -1782,19 +1782,19 @@ class DeclChecker : public DeclVisitor<DeclChecker> {
       return;
     }
 
-    // We don't support nested types in generics yet.
-    if (NTD->isGenericContext()) {
-      auto DC = NTD->getDeclContext();
-      if (auto proto = DC->getSelfProtocolDecl()) {
-        if (DC->getExtendedProtocolDecl()) {
-          NTD->diagnose(diag::unsupported_type_nested_in_protocol_extension,
-                        NTD->getName(), proto->getName());
-        } else {
-          NTD->diagnose(diag::unsupported_type_nested_in_protocol,
-                        NTD->getName(), proto->getName());
-        }
+    // We don't support nested types in protocols.
+    if (auto proto = DC->getSelfProtocolDecl()) {
+      if (DC->getExtendedProtocolDecl()) {
+        NTD->diagnose(diag::unsupported_type_nested_in_protocol_extension,
+                      NTD->getName(), proto->getName());
+      } else {
+        NTD->diagnose(diag::unsupported_type_nested_in_protocol,
+                      NTD->getName(), proto->getName());
       }
+    }
 
+    // We don't support nested types in generic functions yet.
+    if (NTD->isGenericContext()) {
       if (DC->isLocalContext() && DC->isGenericContext()) {
         // A local generic context is a generic function.
         if (auto AFD = dyn_cast<AbstractFunctionDecl>(DC)) {

lib/Sema/TypeCheckGeneric.cpp

@@ -646,10 +646,10 @@ GenericSignatureRequest::evaluate(Evaluator &evaluator,
       if (auto subscript = dyn_cast<SubscriptDecl>(accessor->getStorage()))
          return subscript->getGenericSignature();
 
-    return GC->getParent()->getGenericSignatureOfContext();
+    return GC->getParentForLookup()->getGenericSignatureOfContext();
   }
 
-  auto parentSig = GC->getParent()->getGenericSignatureOfContext();
+  auto parentSig = GC->getParentForLookup()->getGenericSignatureOfContext();
   SmallVector<TypeLoc, 2> inferenceSources;
   SmallVector<Requirement, 2> sameTypeReqs;
   if (auto VD = dyn_cast_or_null<ValueDecl>(GC->getAsDecl())) {
@@ -731,19 +731,10 @@ GenericSignatureRequest::evaluate(Evaluator &evaluator,
 
     // Allow parameters to be equated with concrete types.
     allowConcreteGenericParams = true;
-    // Extensions must occur at the top level, they have no
-    // (valid) parent signature.
-    parentSig = nullptr;
+
     inferenceSources.emplace_back(nullptr, extInterfaceType);
   }
 
-  // EGREGIOUS HACK: The GSB cannot handle the addition of parent signatures
-  // from malformed decls in many cases.  Check the invalid bit and null out the
-  // parent signature.
-  if (auto *DD = GC->getParent()->getAsDecl()) {
-    parentSig = DD->isInvalid() ? nullptr : parentSig;
-  }
-  
   return TypeChecker::checkGenericSignature(
       GC, GC, parentSig,
       allowConcreteGenericParams,

validation-test/compiler_crashers_2_fixed/rdar58813746.swift

@@ -0,0 +1,13 @@
+// RUN: not %target-swift-frontend -typecheck %s
+
+struct CommandContext<Command> {
+  init() {
+    let input = [1,2,3].filter {
+
+extension CommandContext {
+  struct Options {
+    subscript<T>(path: T) -> Int {
+      fatalError()
+    }
+  }
+}

validation-test/compiler_crashers_2_fixed/sr13004.swift

@@ -0,0 +1,8 @@
+// RUN: not %target-swift-frontend -typecheck %s
+
+func verifyDecoder<Decoder>() {
+
+extension ByteToMessageDecoderVerifier {
+
+class RecordingChannel {
+  func readInbound<T>(foo bar: Int) {}