Sema: Ban uncallable protocol member operators

Member operators of concrete nominal types must declare at least
one parameter with that type, like

```
struct S {
  static func +(lhs: S, rhs: Int) -> S {}
}
```

For protocol member operators, we would look for a parameter of type
`Self`, or an existential type `any P`. While the latter was
consistent with the concrete nominal type case, it was actually
wrong because then the resulting interface type does not give the
type checker any way to bind the `Self` type parameter.

There were two existing test cases that now produce errors, which I
believe is now correct. While this is technically a source break,
because these bogus operators seemingly cannot be witnessed or called,
such a protocol probably had no conforming types.

Fixes swiftlang#73201.

Author: slavapestov

lib/Sema/TypeCheckDecl.cpp

@@ -2034,48 +2034,59 @@ FunctionOperatorRequest::evaluate(Evaluator &evaluator, FuncDecl *FD) const {
   return op.get();
 }
 
-bool swift::isMemberOperator(FuncDecl *decl, Type type) {
+/// This means two things:
+/// - If selfTy is null, 'decl' is assumed to be a member of a nominal type
+///   or extension. We check if its a valid member operator.
+/// - Otherwise, 'decl' is a member or top-level operator. We check if it
+///   is a suitable witness for the given conforming type.
+bool swift::isMemberOperator(FuncDecl *decl, Type selfTy) {
   // Check that member operators reference the type of 'Self'.
   if (decl->isInvalid())
     return true;
 
   auto *DC = decl->getDeclContext();
 
   auto selfNominal = DC->getSelfNominalTypeDecl();
+  assert(selfNominal || selfTy);
 
-  // Check the parameters for a reference to 'Self'.
+  // Is the operator a member of a protocol or protocol extension?
   bool isProtocol = isa_and_nonnull<ProtocolDecl>(selfNominal);
+
+  // Is the operator a member of a tuple extension?
   bool isTuple = isa_and_nonnull<BuiltinTupleDecl>(selfNominal);
 
+  // Check the parameters for a reference to 'Self'.
   for (auto param : *decl->getParameters()) {
     // Look through a metatype reference, if there is one.
     auto paramType = param->getInterfaceType()->getMetatypeInstanceType();
 
+    if (isProtocol || isTuple) {
+      // For a member of a protocol or tuple extension, is it the 'Self'
+      // type parameter?
+      if (paramType->isEqual(DC->getSelfInterfaceType()))
+        return true;
+
+      continue;
+    }
+
+    // We have a member operator of a concrete nominal type, or a global operator.
     auto nominal = paramType->getAnyNominal();
-    if (type.isNull()) {
-      // Is it the same nominal type?
-      if (selfNominal && nominal == selfNominal)
+
+    if (selfTy.isNull()) {
+      // We're validating a member operator.
+
+      // Does the parameter have the right nominal type?
+      if (nominal == selfNominal)
         return true;
     } else {
-      // Is it the same nominal type? Or a generic (which may or may not match)?
-      if (paramType->is<GenericTypeParamType>() ||
-          nominal == type->getAnyNominal())
-        return true;
-    }
+      // We're checking a conformance and this operator is a candidate witness.
 
-    if (isProtocol) {
-      // FIXME: Source compatibility hack for Swift 5. The compiler
-      // accepts member operators on protocols with existential
-      // type arguments. We should consider banning this in Swift 6.
-      if (auto existential = paramType->getAs<ExistentialType>()) {
-        if (selfNominal == existential->getConstraintType()->getAnyNominal())
-          return true;
-      }
-    }
+      // Does the parameter have the right nominal type for the conformance?
+      if (nominal == selfTy->getAnyNominal())
+        return true;
 
-    if (isProtocol || isTuple) {
-      // For a protocol or tuple extension, is it the 'Self' type parameter?
-      if (paramType->isEqual(DC->getSelfInterfaceType()))
+      // Otherwise, we might also have a match if the top-level operator is generic.
+      if (paramType->is<GenericTypeParamType>())
         return true;
     }
   }

test/api-digester/Inputs/cake.swift

@@ -85,7 +85,7 @@ public protocol PSub: PSuper {
 public let GlobalVar = 1
 
 public extension P1 {
-  static func +(lhs: P1, rhs: P1) -> P1 { return lhs }
+  static func +(lhs: Self, rhs: Self) -> Self { return lhs }
 }
 
 infix operator ..*..

test/api-digester/Outputs/cake-abi.json

@@ -54,26 +54,23 @@
             "children": [
               {
                 "kind": "TypeNominal",
-                "name": "P1",
-                "printedName": "any cake.P1",
-                "usr": "s:4cake2P1P"
+                "name": "GenericTypeParam",
+                "printedName": "τ_0_0"
               },
               {
                 "kind": "TypeNominal",
-                "name": "P1",
-                "printedName": "any cake.P1",
-                "usr": "s:4cake2P1P"
+                "name": "GenericTypeParam",
+                "printedName": "τ_0_0"
               },
               {
                 "kind": "TypeNominal",
-                "name": "P1",
-                "printedName": "any cake.P1",
-                "usr": "s:4cake2P1P"
+                "name": "GenericTypeParam",
+                "printedName": "τ_0_0"
               }
             ],
             "declKind": "Func",
-            "usr": "s:4cake2P1PAAE1poiyAaB_pAaB_p_AaB_ptFZ",
-            "mangledName": "$s4cake2P1PAAE1poiyAaB_pAaB_p_AaB_ptFZ",
+            "usr": "s:4cake2P1PAAE1poiyxx_xtFZ",
+            "mangledName": "$s4cake2P1PAAE1poiyxx_xtFZ",
             "moduleName": "cake",
             "genericSig": "<τ_0_0 where τ_0_0 : cake.P1>",
             "sugared_genericSig": "<Self where Self : cake.P1>",

test/api-digester/Outputs/cake.json

@@ -54,26 +54,23 @@
             "children": [
               {
                 "kind": "TypeNominal",
-                "name": "P1",
-                "printedName": "any cake.P1",
-                "usr": "s:4cake2P1P"
+                "name": "GenericTypeParam",
+                "printedName": "Self"
               },
               {
                 "kind": "TypeNominal",
-                "name": "P1",
-                "printedName": "any cake.P1",
-                "usr": "s:4cake2P1P"
+                "name": "GenericTypeParam",
+                "printedName": "Self"
               },
               {
                 "kind": "TypeNominal",
-                "name": "P1",
-                "printedName": "any cake.P1",
-                "usr": "s:4cake2P1P"
+                "name": "GenericTypeParam",
+                "printedName": "Self"
               }
             ],
             "declKind": "Func",
-            "usr": "s:4cake2P1PAAE1poiyAaB_pAaB_p_AaB_ptFZ",
-            "mangledName": "$s4cake2P1PAAE1poiyAaB_pAaB_p_AaB_ptFZ",
+            "usr": "s:4cake2P1PAAE1poiyxx_xtFZ",
+            "mangledName": "$s4cake2P1PAAE1poiyxx_xtFZ",
             "moduleName": "cake",
             "genericSig": "<Self where Self : cake.P1>",
             "static": true,

test/decl/func/operator.swift

@@ -380,13 +380,15 @@ extension P2 {
 }
 
 protocol P3 {
-  // Okay: refers to P3
+  // Not allowed: there's no way to infer 'Self' from this interface type
   static func %%%(lhs: P3, rhs: Unrelated) -> Unrelated
+  // expected-error@-1 {{member operator '%%%' of protocol 'P3' must have at least one argument of type 'Self'}}
 }
 
 extension P3 {
-  // Okay: refers to P3
+  // Not allowed: there's no way to infer 'Self' from this interface type
   static func %%%%(lhs: P3, rhs: Unrelated) -> Unrelated { }
+  // expected-error@-1 {{member operator '%%%%' of protocol 'P3' must have at least one argument of type 'Self'}}
 }
 
 // rdar://problem/27940842 - recovery with a non-static '=='.