[match-case] fix matching against typing.Callable and Protocol types. (#19471)

- Fixes #14014
- Partially addresses #19470

Added extra logic in `checker.py:conditional_types` function to deal
with structural types such as `typing.Callable` or protocols.

## new tests

- `testMatchClassPatternCallable`: tests `case Callable() as fn` usage
- `testMatchClassPatternProtocol`: tests `case Proto()` usage, where
`Proto` is a Protocol
- `testMatchClassPatternCallbackProtocol`: tests `case Proto()` usage,
where `Proto` is a Callback-Protocol
- `testGenericAliasIsinstanceUnreachable`: derived from a mypy-primer
failure in mesonbuild. Tests that `isinstance(x, Proto)` can produce
unreachable error.
- `testGenericAliasRedundantExprCompoundIfExpr`: derived from a CI
failure of `python runtest.py self` of an earlier version of this PR.

## modified tests

- `testOverloadOnProtocol` added annotations to overload implementation,
which wasn't getting checked. Added missing return. Fixed return type in
second branch.

---------

Co-authored-by: Stanislav Terliakov <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ivan Levkivskyi <[email protected]>

Author: 4 people

mypy/checker.py

@@ -8174,11 +8174,15 @@ def conditional_types(
 ) -> tuple[Type | None, Type | None]:
     """Takes in the current type and a proposed type of an expression.
 
-    Returns a 2-tuple: The first element is the proposed type, if the expression
-    can be the proposed type. The second element is the type it would hold
-    if it was not the proposed type, if any. UninhabitedType means unreachable.
-    None means no new information can be inferred. If default is set it is returned
-    instead."""
+    Returns a 2-tuple:
+        The first element is the proposed type, if the expression can be the proposed type.
+            (or default, if default is set and the expression is a subtype of the proposed type).
+        The second element is the type it would hold if it was not the proposed type, if any.
+            (or default, if default is set and the expression is not a subtype of the proposed type).
+
+        UninhabitedType means unreachable.
+        None means no new information can be inferred.
+    """
     if proposed_type_ranges:
         if len(proposed_type_ranges) == 1:
             target = proposed_type_ranges[0].item
@@ -8190,14 +8194,25 @@ def conditional_types(
                 current_type = try_expanding_sum_type_to_union(current_type, enum_name)
         proposed_items = [type_range.item for type_range in proposed_type_ranges]
         proposed_type = make_simplified_union(proposed_items)
-        if isinstance(proposed_type, AnyType):
+        if isinstance(get_proper_type(current_type), AnyType):
+            return proposed_type, current_type
+        elif isinstance(proposed_type, AnyType):
             # We don't really know much about the proposed type, so we shouldn't
             # attempt to narrow anything. Instead, we broaden the expr to Any to
             # avoid false positives
             return proposed_type, default
-        elif not any(
-            type_range.is_upper_bound for type_range in proposed_type_ranges
-        ) and is_proper_subtype(current_type, proposed_type, ignore_promotions=True):
+        elif not any(type_range.is_upper_bound for type_range in proposed_type_ranges) and (
+            # concrete subtypes
+            is_proper_subtype(current_type, proposed_type, ignore_promotions=True)
+            # structural subtypes
+            or (
+                (
+                    isinstance(proposed_type, CallableType)
+                    or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol)
+                )
+                and is_subtype(current_type, proposed_type, ignore_promotions=True)
+            )
+        ):
             # Expression is always of one of the types in proposed_type_ranges
             return default, UninhabitedType()
         elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True):

mypy/checkpattern.py

@@ -51,6 +51,7 @@
     UninhabitedType,
     UnionType,
     UnpackType,
+    callable_with_ellipsis,
     find_unpack_in_list,
     get_proper_type,
     split_with_prefix_and_suffix,
@@ -546,6 +547,15 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType:
             return self.early_non_match()
         elif isinstance(p_typ, FunctionLike) and p_typ.is_type_obj():
             typ = fill_typevars_with_any(p_typ.type_object())
+        elif (
+            isinstance(type_info, Var)
+            and type_info.type is not None
+            and type_info.fullname == "typing.Callable"
+        ):
+            # Create a `Callable[..., Any]`
+            fallback = self.chk.named_type("builtins.function")
+            any_type = AnyType(TypeOfAny.unannotated)
+            typ = callable_with_ellipsis(any_type, ret_type=any_type, fallback=fallback)
         elif not isinstance(p_typ, AnyType):
             self.msg.fail(
                 message_registry.CLASS_PATTERN_TYPE_REQUIRED.format(

test-data/unit/check-generic-alias.test

@@ -149,6 +149,38 @@ t23: collections.abc.ValuesView[str]
 # reveal_type(t23)  # Nx Revealed type is "collections.abc.ValuesView[builtins.str]"
 [builtins fixtures/tuple.pyi]
 
+[case testGenericAliasIsinstanceUnreachable]
+# flags: --warn-unreachable --python-version 3.10
+from collections.abc import Iterable
+
+class A: ...
+
+def test(dependencies: list[A] | None) -> None:
+    if dependencies is None:
+        dependencies = []
+    elif not isinstance(dependencies, Iterable):
+        dependencies = [dependencies]  # E: Statement is unreachable
+
+[builtins fixtures/isinstancelist.pyi]
+[typing fixtures/typing-full.pyi]
+
+[case testGenericAliasRedundantExprCompoundIfExpr]
+# flags: --warn-unreachable --enable-error-code=redundant-expr --python-version 3.10
+
+from typing import Any, reveal_type
+from collections.abc import Iterable
+
+def test_example(x: Iterable[Any]) -> None:
+    if isinstance(x, Iterable) and not isinstance(x, str):  # E: Left operand of "and" is always true
+        reveal_type(x)  # N: Revealed type is "typing.Iterable[Any]"
+
+def test_counterexample(x: Any) -> None:
+    if isinstance(x, Iterable) and not isinstance(x, str):
+        reveal_type(x)  # N: Revealed type is "typing.Iterable[Any]"
+
+[builtins fixtures/isinstancelist.pyi]
+[typing fixtures/typing-full.pyi]
+
 
 [case testGenericBuiltinTupleTyping]
 from typing import Tuple

test-data/unit/check-protocols.test

@@ -1506,11 +1506,12 @@ class C: pass
 def f(x: P1) -> int: ...
 @overload
 def f(x: P2) -> str: ...
-def f(x):
+def f(x: object) -> object:
     if isinstance(x, P1):
         return P1.attr1
     if isinstance(x, P2): # E: Only @runtime_checkable protocols can be used with instance and class checks
-        return P1.attr2
+        return P2.attr2
+    return None
 
 reveal_type(f(C1())) # N: Revealed type is "builtins.int"
 reveal_type(f(C2())) # N: Revealed type is "builtins.str"

test-data/unit/check-python310.test

@@ -11,11 +11,30 @@ match m:
 -- Literal Pattern --
 
 [case testMatchLiteralPatternNarrows]
+# flags: --warn-unreachable
 m: object
 
 match m:
     case 1:
         reveal_type(m)  # N: Revealed type is "Literal[1]"
+    case 2:
+        reveal_type(m)  # N: Revealed type is "Literal[2]"
+    case other:
+        reveal_type(other)  # N: Revealed type is "builtins.object"
+
+[case testMatchLiteralPatternNarrows2]
+# flags: --warn-unreachable
+from typing import Any
+
+m: Any
+
+match m:
+    case 1:
+        reveal_type(m)  # N: Revealed type is "Literal[1]"
+    case 2:
+        reveal_type(m)  # N: Revealed type is "Literal[2]"
+    case other:
+        reveal_type(other)  # N: Revealed type is "Any"
 
 [case testMatchLiteralPatternAlreadyNarrower-skip]
 m: bool
@@ -1079,6 +1098,143 @@ match m:
     case Foo():
         pass
 
+[case testMatchClassPatternCallable]
+# flags: --warn-unreachable
+from typing import Callable, Any
+
+class FnImpl:
+    def __call__(self, x: object, /) -> int: ...
+
+def test_any(x: Any) -> None:
+    match x:
+        case Callable() as fn:
+            reveal_type(fn)  # N: Revealed type is "def (*Any, **Any) -> Any"
+        case other:
+            reveal_type(other)  # N: Revealed type is "Any"
+
+def test_object(x: object) -> None:
+    match x:
+        case Callable() as fn:
+            reveal_type(fn)  # N: Revealed type is "def (*Any, **Any) -> Any"
+        case other:
+            reveal_type(other)  # N: Revealed type is "builtins.object"
+
+def test_impl(x: FnImpl) -> None:
+    match x:
+        case Callable() as fn:
+            reveal_type(fn)  # N: Revealed type is "__main__.FnImpl"
+        case other:
+            reveal_type(other)  # E: Statement is unreachable
+
+def test_callable(x: Callable[[object], int]) -> None:
+    match x:
+        case Callable() as fn:
+            reveal_type(fn)  # N: Revealed type is "def (builtins.object) -> builtins.int"
+        case other:
+            reveal_type(other)  # E: Statement is unreachable
+
+[case testMatchClassPatternCallbackProtocol]
+# flags: --warn-unreachable
+from typing import Any, Callable
+from typing_extensions import Protocol, runtime_checkable
+
+@runtime_checkable
+class FnProto(Protocol):
+    def __call__(self, x: int, /) -> object: ...
+
+class FnImpl:
+    def __call__(self, x: object, /) -> int: ...
+
+def test_any(x: Any) -> None:
+    match x:
+        case FnProto() as fn:
+            reveal_type(fn)  # N: Revealed type is "__main__.FnProto"
+        case other:
+            reveal_type(other)  # N: Revealed type is "Any"
+
+def test_object(x: object) -> None:
+    match x:
+        case FnProto() as fn:
+            reveal_type(fn)  # N: Revealed type is "__main__.FnProto"
+        case other:
+            reveal_type(other)  # N: Revealed type is "builtins.object"
+
+def test_impl(x: FnImpl) -> None:
+    match x:
+        case FnProto() as fn:
+            reveal_type(fn)  # N: Revealed type is "__main__.FnImpl"
+        case other:
+            reveal_type(other)  # E: Statement is unreachable
+
+def test_callable(x: Callable[[object], int]) -> None:
+    match x:
+        case FnProto() as fn:
+            reveal_type(fn)  # N: Revealed type is "def (builtins.object) -> builtins.int"
+        case other:
+            reveal_type(other)  # E: Statement is unreachable
+
+[builtins fixtures/dict.pyi]
+
+[case testMatchClassPatternAnyCallableProtocol]
+# flags: --warn-unreachable
+from typing import Any, Callable
+from typing_extensions import Protocol, runtime_checkable
+
+@runtime_checkable
+class AnyCallable(Protocol):
+    def __call__(self, *args: Any, **kwargs: Any) -> Any: ...
+
+class FnImpl:
+    def __call__(self, x: object, /) -> int: ...
+
+def test_object(x: object) -> None:
+    match x:
+        case AnyCallable() as fn:
+            reveal_type(fn)  # N: Revealed type is "__main__.AnyCallable"
+        case other:
+            reveal_type(other)  # N: Revealed type is "builtins.object"
+
+def test_impl(x: FnImpl) -> None:
+    match x:
+        case AnyCallable() as fn:
+            reveal_type(fn)  # N: Revealed type is "__main__.FnImpl"
+        case other:
+            reveal_type(other)  # E: Statement is unreachable
+
+def test_callable(x: Callable[[object], int]) -> None:
+    match x:
+        case AnyCallable() as fn:
+            reveal_type(fn)  # N: Revealed type is "def (builtins.object) -> builtins.int"
+        case other:
+            reveal_type(other)  # E: Statement is unreachable
+
+[builtins fixtures/dict.pyi]
+
+
+[case testMatchClassPatternProtocol]
+from typing import Any
+from typing_extensions import Protocol, runtime_checkable
+
+@runtime_checkable
+class Proto(Protocol):
+    def foo(self, x: int, /) -> object: ...
+
+class Impl:
+    def foo(self, x: object, /) -> int: ...
+
+def test_object(x: object) -> None:
+    match x:
+        case Proto() as y:
+            reveal_type(y)  # N: Revealed type is "__main__.Proto"
+
+def test_impl(x: Impl) -> None:
+    match x:
+        case Proto() as y:
+            reveal_type(y)  # N: Revealed type is "__main__.Impl"
+
+[builtins fixtures/dict.pyi]
+
+
 [case testMatchClassPatternNestedGenerics]
 # From cpython test_patma.py
 x = [[{0: 0}]]