Fix two issues with typealiases in protocol extensions (#20654)

- The GenericSignatureBuilder assumed it didn't have to look in
  protocol extensions to resolve member types.

- Serialization was incorrectly filtering out such typealiases when
  trying to resolve a cross-module reference to one.

rdar://problem/46103190
(cherry picked from commit 1f25ea5)

Author: jrose-apple

lib/AST/GenericSignatureBuilder.cpp

@@ -2031,9 +2031,7 @@ TypeDecl *EquivalenceClass::lookupNestedType(
     ProtocolDecl *proto = conforms.first;
 
     // Look for an associated type and/or concrete type with this name.
-    auto flags = OptionSet<NominalTypeDecl::LookupDirectFlags>();
-    flags |= NominalTypeDecl::LookupDirectFlags::IgnoreNewExtensions;
-    for (auto member : proto->lookupDirect(name, flags)) {
+    for (auto member : proto->lookupDirect(name)) {
       // If this is an associated type, record whether it is the best
       // associated type we've seen thus far.
       if (auto assocType = dyn_cast<AssociatedTypeDecl>(member)) {

lib/Serialization/Deserialization.cpp

@@ -1414,9 +1414,9 @@ ModuleFile::resolveCrossReference(ModuleDecl *baseModule, uint32_t pathLen) {
         IdentifierID IID;
         IdentifierID privateDiscriminator;
         bool importedFromClang = false;
+        bool inProtocolExt = false;
         XRefTypePathPieceLayout::readRecord(scratch, IID, privateDiscriminator,
-                                            /*inProtocolExt*/None,
-                                            importedFromClang);
+                                            inProtocolExt, importedFromClang);
         if (privateDiscriminator)
           goto giveUpFastPath;
 
@@ -1446,9 +1446,8 @@ ModuleFile::resolveCrossReference(ModuleDecl *baseModule, uint32_t pathLen) {
         if (nestedType) {
           SmallVector<ValueDecl *, 1> singleValueBuffer{nestedType};
           filterValues(/*expectedTy*/Type(), extensionModule, genericSig,
-                       /*isType*/true, /*inProtocolExt*/false,
-                       importedFromClang, /*isStatic*/false, /*ctorInit*/None,
-                       singleValueBuffer);
+                       /*isType*/true, inProtocolExt, importedFromClang,
+                       /*isStatic*/false, /*ctorInit*/None, singleValueBuffer);
           if (!singleValueBuffer.empty()) {
             values.assign({nestedType});
             ++NumNestedTypeShortcuts;

lib/Serialization/Serialization.cpp

@@ -1894,10 +1894,12 @@ void Serializer::writeCrossReference(const DeclContext *DC, uint32_t pathLen) {
       discriminator = containingFile->getDiscriminatorForPrivateValue(generic);
     }
 
+    bool isProtocolExt = DC->getParent()->getExtendedProtocolDecl();
+
     XRefTypePathPieceLayout::emitRecord(Out, ScratchRecord, abbrCode,
                                         addDeclBaseNameRef(generic->getName()),
                                         addDeclBaseNameRef(discriminator),
-                                        /*inProtocolExtension*/false,
+                                        isProtocolExt,
                                         generic->hasClangNode());
     break;
   }

test/multifile/typealias/one-module/Inputs/library.swift

@@ -5,3 +5,9 @@ public enum Result<T, U>
 }
 
 public typealias GenericResult<T> = Result<T, Error>
+
+public protocol Rdar46103190 {}
+extension Rdar46103190 {
+  public typealias Alias = String
+  public typealias Map<K: Hashable> = [K: Self]
+}

test/multifile/typealias/one-module/main.swift

@@ -5,3 +5,18 @@
 
 func testFunction<T>(withCompletion completion: (Result<T, Error>) -> Void) { }
 testFunction { (result: GenericResult<Int>) in }
+
+extension Rdar46103190 {
+  public typealias AnotherAlias = Self.Alias
+  public typealias StringMap = Map<String>
+}
+
+typealias Rdar46103190Alias<R: Rdar46103190> = R.Map<String>
+
+struct Rdar46103190Impl: Rdar46103190 {}
+
+func test46103190() {
+  let _: String = Rdar46103190Impl.AnotherAlias()
+  let _: [String: Rdar46103190Impl] = Rdar46103190Impl.StringMap()
+  let _: [String: Rdar46103190Impl] = Rdar46103190Alias()
+}

test/multifile/typealias/two-modules/Inputs/library.swift

@@ -5,3 +5,9 @@ public enum Result<T, U>
 }
 
 public typealias GenericResult<T> = Result<T, Error>
+
+public protocol Rdar46103190 {}
+extension Rdar46103190 {
+  public typealias Alias = String
+  public typealias Map<K: Hashable> = [K: Self]
+}

test/multifile/typealias/two-modules/main.swift

@@ -14,3 +14,18 @@ import enum library.Result
 
 func testFunction<T>(withCompletion completion: (Result<T, Error>) -> Void) { }
 testFunction { (result: GenericResult<Int>) in }
+
+extension Rdar46103190 {
+  public typealias AnotherAlias = Self.Alias
+  public typealias StringMap = Map<String>
+}
+
+typealias Rdar46103190Alias<R: Rdar46103190> = R.Map<String>
+
+struct Rdar46103190Impl: Rdar46103190 {}
+
+func test46103190() {
+  let _: String = Rdar46103190Impl.AnotherAlias()
+  let _: [String: Rdar46103190Impl] = Rdar46103190Impl.StringMap()
+  let _: [String: Rdar46103190Impl] = Rdar46103190Alias()
+}