Sema: Improve the infinite opaque return type check

Now look through other opaque return types that appear in the
underlying type. This catches various forms of recursion that
otherwise would cause a SILGen or SILOptimizer crash.

- Fixes rdar://82992151.

Author: slavapestov

lib/Sema/MiscDiagnostics.cpp

@@ -27,6 +27,7 @@
 #include "swift/AST/DiagnosticsSema.h"
 #include "swift/AST/ExistentialLayout.h"
 #include "swift/AST/Expr.h"
+#include "swift/AST/InFlightSubstitution.h"
 #include "swift/AST/NameLookup.h"
 #include "swift/AST/NameLookupRequests.h"
 #include "swift/AST/Pattern.h"
@@ -3670,12 +3671,33 @@ class OpaqueUnderlyingTypeChecker : public ASTWalker {
 
     // The underlying type can't be defined recursively
     // in terms of the opaque type itself.
-    auto opaqueTypeInContext = Implementation->mapTypeIntoContext(
-        OpaqueDecl->getDeclaredInterfaceType());
     for (auto genericParam : OpaqueDecl->getOpaqueGenericParams()) {
       auto underlyingType = Type(genericParam).subst(substitutions);
-      auto isSelfReferencing = underlyingType.findIf(
-          [&](Type t) -> bool { return t->isEqual(opaqueTypeInContext); });
+
+      // Look through underlying types of other opaque archetypes known to
+      // us. This is not something the type checker is allowed to do in
+      // general, since the intent is that the underlying type is completely
+      // hidden from view at the type system level. However, here we're
+      // trying to catch recursive underlying types before we proceed to
+      // SIL, so we specifically want to erase opaque archetypes just
+      // for the purpose of this check.
+      ReplaceOpaqueTypesWithUnderlyingTypes replacer(
+          OpaqueDecl->getDeclContext(),
+          ResilienceExpansion::Maximal,
+          /*isWholeModuleContext=*/false);
+      InFlightSubstitution IFS(replacer, replacer,
+                               SubstFlags::SubstituteOpaqueArchetypes |
+                               SubstFlags::PreservePackExpansionLevel);
+      auto simplifiedUnderlyingType = underlyingType.subst(IFS);
+
+      auto isSelfReferencing =
+          (IFS.wasLimitReached() ||
+           simplifiedUnderlyingType.findIf([&](Type t) -> bool {
+             if (auto *other = t->getAs<OpaqueTypeArchetypeType>()) {
+               return other->getDecl() == OpaqueDecl;
+             }
+             return false;
+           }));
 
       if (isSelfReferencing) {
         Ctx.Diags.diagnose(std::get<0>(candidate)->getLoc(),

test/Generics/infinite_opaque_result_type.swift

@@ -0,0 +1,73 @@
+// RUN: %target-typecheck-verify-swift
+
+func concrete1() -> some Any {
+// expected-error@-1 {{function declares an opaque return type, but has no return statements in its body from which to infer an underlying type}}
+  return concrete1()
+}
+
+func concrete2() -> some Any {
+  return [concrete2()]  // expected-error {{function opaque return type was inferred as '[some Any]', which defines the opaque type in terms of itself}}
+}
+
+
+func concrete1a() -> some Any {
+  return concrete1b()  // expected-error {{function opaque return type was inferred as 'some Any', which defines the opaque type in terms of itself}}
+}
+
+func concrete1b() -> some Any {
+  return concrete1a()
+}
+
+
+func concrete2a() -> some Any {
+  return [concrete2b()]  // expected-error {{function opaque return type was inferred as '[some Any]', which defines the opaque type in terms of itself}}
+}
+
+func concrete2b() -> some Any {
+  return [concrete2a()]
+}
+
+
+func generic1<T>(_ t: T) -> some Any {
+// expected-error@-1 {{function declares an opaque return type, but has no return statements in its body from which to infer an underlying type}}
+  return generic1(t)
+}
+
+func generic2<T>(_ t: T) -> some Any {
+  return [generic2(t)]  // expected-error {{function opaque return type was inferred as '[some Any]', which defines the opaque type in terms of itself}}
+}
+
+
+func generic1a<T>(_ t: T) -> some Any {
+  return generic1b(t)  // expected-error {{function opaque return type was inferred as 'some Any', which defines the opaque type in terms of itself}}
+}
+
+func generic1b<T>(_ t: T) -> some Any {
+  return generic1a(t)
+}
+
+
+func generic2a<T>(_ t: T) -> some Any {
+  return [generic2b(t)]  // expected-error {{function opaque return type was inferred as '[some Any]', which defines the opaque type in terms of itself}}
+}
+
+func generic2b<T>(_ t: T) -> some Any {
+  return [generic2a(t)]
+}
+
+
+func generic3a<T>(_ t: T) -> some Any {
+  return [generic3b(t)]  // expected-error {{function opaque return type was inferred as '[some Any]', which defines the opaque type in terms of itself}}
+}
+
+func generic3b<T>(_ t: T) -> some Any {
+  return [generic3a([t])]
+}
+
+func very_wide1() -> some Any {
+  return (very_wide2(), very_wide2())  // expected-error {{function opaque return type was inferred as '(some Any, some Any)', which defines the opaque type in terms of itself}}
+}
+
+func very_wide2() -> some Any {
+  return (very_wide1(), very_wide1())
+}

test/type/opaque.swift

@@ -1,4 +1,4 @@
-// RUN: %target-swift-frontend -disable-availability-checking -typecheck -verify %s
+// RUN: %target-swift-frontend -target %target-swift-5.1-abi-triple -typecheck -verify %s
 
 protocol P {
   func paul()
@@ -205,13 +205,6 @@ struct MismatchedReturnTypesSubscript {
   }
 }
 
-func jan() -> some P {
-  return [marcia(), marcia(), marcia()]
-}
-func marcia() -> some P {
-  return [marcia(), marcia(), marcia()] // expected-error{{defines the opaque type in terms of itself}} {{documentation-file=opaque-type-inference}}
-}
-
 protocol R {
   associatedtype S: P, Q // expected-note*{{}}