Use empty context as fallback for return statements (python#19767)

Fixes python#16924
Fixes python#15886

Mypy uses external type context first, this can cause bad type inference
in return statements (see example in test case added), usually we
recommend a workaround to users like replacing:
```python
    return foo(x)
```
with
```python
    y = foo(x)
    return y
```
But this is a bit ugly, and more importantly we can essentially
automatically try this workaround. This is what this PR adds. I checked
performance impact, and don't see any (but for some reason noise level
on my desktop is much higher now).

Author: ilevkivskyi

mypy/checker.py

@@ -90,6 +90,7 @@
     ContinueStmt,
     Decorator,
     DelStmt,
+    DictExpr,
     EllipsisExpr,
     Expression,
     ExpressionStmt,
@@ -124,6 +125,7 @@
     RaiseStmt,
     RefExpr,
     ReturnStmt,
+    SetExpr,
     StarExpr,
     Statement,
     StrExpr,
@@ -4859,6 +4861,42 @@ def visit_return_stmt(self, s: ReturnStmt) -> None:
         self.check_return_stmt(s)
         self.binder.unreachable()
 
+    def infer_context_dependent(
+        self, expr: Expression, type_ctx: Type, allow_none_func_call: bool
+    ) -> ProperType:
+        """Infer type of an expression with fallback to empty type context."""
+        with self.msg.filter_errors(
+            filter_errors=True, filter_deprecated=True, save_filtered_errors=True
+        ) as msg:
+            with self.local_type_map as type_map:
+                typ = get_proper_type(
+                    self.expr_checker.accept(
+                        expr, type_ctx, allow_none_return=allow_none_func_call
+                    )
+                )
+        if not msg.has_new_errors():
+            self.store_types(type_map)
+            return typ
+
+        # If there are errors with the original type context, try re-inferring in empty context.
+        original_messages = msg.filtered_errors()
+        original_type_map = type_map
+        with self.msg.filter_errors(
+            filter_errors=True, filter_deprecated=True, save_filtered_errors=True
+        ) as msg:
+            with self.local_type_map as type_map:
+                alt_typ = get_proper_type(
+                    self.expr_checker.accept(expr, None, allow_none_return=allow_none_func_call)
+                )
+        if not msg.has_new_errors() and is_subtype(alt_typ, type_ctx):
+            self.store_types(type_map)
+            return alt_typ
+
+        # If empty fallback didn't work, use results from the original type context.
+        self.msg.add_errors(original_messages)
+        self.store_types(original_type_map)
+        return typ
+
     def check_return_stmt(self, s: ReturnStmt) -> None:
         defn = self.scope.current_function()
         if defn is not None:
@@ -4891,11 +4929,18 @@ def check_return_stmt(self, s: ReturnStmt) -> None:
                 allow_none_func_call = is_lambda or declared_none_return or declared_any_return
 
                 # Return with a value.
-                typ = get_proper_type(
-                    self.expr_checker.accept(
-                        s.expr, return_type, allow_none_return=allow_none_func_call
+                if isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)):
+                    # For expressions that (strongly) depend on type context (i.e. those that
+                    # are handled like a function call), we allow fallback to empty type context
+                    # in case of errors, this improves user experience in some cases,
+                    # see e.g. testReturnFallbackInference.
+                    typ = self.infer_context_dependent(s.expr, return_type, allow_none_func_call)
+                else:
+                    typ = get_proper_type(
+                        self.expr_checker.accept(
+                            s.expr, return_type, allow_none_return=allow_none_func_call
+                        )
                     )
-                )
                 # Treat NotImplemented as having type Any, consistent with its
                 # definition in typeshed prior to python/typeshed#4222.
                 if (

mypy/errors.py

@@ -206,7 +206,8 @@ def on_error(self, file: str, info: ErrorInfo) -> bool:
         """
         if info.code == codes.DEPRECATED:
             # Deprecated is not a type error, so it is handled on opt-in basis here.
-            return self._filter_deprecated
+            if not self._filter_deprecated:
+                return False
 
         self._has_new_errors = True
         if isinstance(self._filter, bool):

test-data/unit/check-inference-context.test

@@ -1540,3 +1540,45 @@ def f(x: dict[str, Union[str, None, int]]) -> None:
 def g(x: Optional[dict[str, Any]], s: Optional[str]) -> None:
     f(x or {'x': s})
 [builtins fixtures/dict.pyi]
+
+[case testReturnFallbackInferenceTuple]
+from typing import TypeVar, Union
+
+T = TypeVar("T")
+def foo(x: list[T]) -> tuple[T, ...]: ...
+
+def bar(x: list[int]) -> tuple[Union[str, int], ...]:
+    return foo(x)
+
+def bar2(x: list[int]) -> tuple[Union[str, int], ...]:
+    y = foo(x)
+    return y
+[builtins fixtures/tuple.pyi]
+
+[case testReturnFallbackInferenceUnion]
+from typing import Generic, TypeVar, Union
+
+T = TypeVar("T")
+
+class Cls(Generic[T]):
+    pass
+
+def inner(c: Cls[T]) -> Union[T, int]:
+    return 1
+
+def outer(c: Cls[T]) -> Union[T, int]:
+    return inner(c)
+
+[case testReturnFallbackInferenceAsync]
+from typing import Generic, TypeVar, Optional
+
+T = TypeVar("T")
+
+class Cls(Generic[T]):
+    pass
+
+async def inner(c: Cls[T]) -> Optional[T]:
+    return None
+
+async def outer(c: Cls[T]) -> Optional[T]:
+    return await inner(c)

test-data/unit/pythoneval.test

@@ -2188,3 +2188,19 @@ reveal_type([*map(str, x)])
 [out]
 _testUnpackIteratorBuiltins.py:4: note: Revealed type is "builtins.list[builtins.int]"
 _testUnpackIteratorBuiltins.py:5: note: Revealed type is "builtins.list[builtins.str]"
+
+[case testReturnFallbackInferenceDict]
+# Requires full dict stubs.
+from typing import Dict, Mapping, TypeVar, Union
+
+K = TypeVar("K")
+V = TypeVar("V")
+K2 = TypeVar("K2")
+V2 = TypeVar("V2")
+
+def func(one: Dict[K, V], two: Mapping[K2, V2]) -> Dict[Union[K, K2], Union[V, V2]]:
+    ...
+
+def caller(arg1: Mapping[K, V], arg2: Mapping[K2, V2]) -> Dict[Union[K, K2], Union[V, V2]]:
+    _arg1 = arg1 if isinstance(arg1, dict) else dict(arg1)
+    return func(_arg1, arg2)