Merge branch 'python:master' into master

Author: bzoracler

docs/source/command_line.rst

@@ -166,6 +166,10 @@ imports.
 
     For more details, see :ref:`ignore-missing-imports`.
 
+.. option:: --follow-untyped-imports
+
+    This flag makes mypy analyze imports without stubs or a py.typed marker.
+
 .. option:: --follow-imports {normal,silent,skip,error}
 
     This flag adjusts how mypy follows imported modules that were not
@@ -537,12 +541,12 @@ potentially problematic or redundant in some way.
 
         This limitation will be removed in future releases of mypy.
 
-.. option:: --report-deprecated-as-error
+.. option:: --report-deprecated-as-note
 
-    By default, mypy emits notes if your code imports or uses deprecated
-    features.    This flag converts such notes to errors, causing mypy to
-    eventually finish with a non-zero exit code.  Features are considered
-    deprecated when decorated with ``warnings.deprecated``.
+    If error code ``deprecated`` is enabled, mypy emits errors if your code
+    imports or uses deprecated features. This flag converts such errors to
+    notes, causing mypy to eventually finish with a zero exit code. Features
+    are considered deprecated when decorated with ``warnings.deprecated``.
 
 .. _miscellaneous-strictness-flags:
 

docs/source/config_file.rst

@@ -315,6 +315,18 @@ section of the command line docs.
     match the name of the *imported* module, not the module containing the
     import statement.
 
+.. confval:: follow_untyped_imports
+
+    :type: boolean
+    :default: False
+
+    Typechecks imports from modules that do not have stubs or a py.typed marker.
+
+    If this option is used in a per-module section, the module name should
+    match the name of the *imported* module, not the module containing the
+    import statement. Note that scanning all unannotated modules might
+    significantly increase the runtime of your mypy calls.
+
 .. confval:: follow_imports
 
     :type: string

docs/source/error_code_list2.rst

@@ -236,13 +236,13 @@ incorrect control flow or conditional checks that are accidentally always true o
 Check that imported or used feature is deprecated [deprecated]
 --------------------------------------------------------------
 
-By default, mypy generates a note if your code imports a deprecated feature explicitly with a
+If you use :option:`--enable-error-code deprecated <mypy --enable-error-code>`,
+mypy generates an error if your code imports a deprecated feature explicitly with a
 ``from mod import depr`` statement or uses a deprecated feature imported otherwise or defined
 locally.  Features are considered deprecated when decorated with ``warnings.deprecated``, as
-specified in `PEP 702 <https://peps.python.org/pep-0702>`_.  You can silence single notes via
-``# type: ignore[deprecated]`` or turn off this check completely via ``--disable-error-code=deprecated``.
-Use the :option:`--report-deprecated-as-error <mypy --report-deprecated-as-error>` option for
-more strictness, which turns all such notes into errors.
+specified in `PEP 702 <https://peps.python.org/pep-0702>`_.
+Use the :option:`--report-deprecated-as-note <mypy --report-deprecated-as-note>` option to
+turn all such errors into notes.
 
 .. note::
 

docs/source/extending_mypy.rst

@@ -245,4 +245,4 @@ Mypy ships ``mypy.plugins.proper_plugin`` plugin which can be useful
 for plugin authors, since it finds missing ``get_proper_type()`` calls,
 which is a pretty common mistake.
 
-It is recommended to enable it is a part of your plugin's CI.
+It is recommended to enable it as a part of your plugin's CI.

docs/source/running_mypy.rst

@@ -321,6 +321,12 @@ not catch errors in its use.
     recommend avoiding ``--ignore-missing-imports`` if possible: it's equivalent
     to adding a ``# type: ignore`` to all unresolved imports in your codebase.
 
+4.  To make mypy typecheck imports from modules without stubs or a py.typed
+    marker, you can set the :option:`--follow-untyped-imports <mypy --follow-untyped-imports>`
+    command line flag or set the :confval:`follow_untyped_imports` config file option to True,
+    either in the global section of your mypy config file, or individually on a
+    per-module basis.
+
 
 Library stubs not installed
 ---------------------------

mypy/binder.py

@@ -2,7 +2,7 @@
 
 from collections import defaultdict
 from contextlib import contextmanager
-from typing import DefaultDict, Iterator, List, Optional, Tuple, Union, cast
+from typing import DefaultDict, Iterator, List, NamedTuple, Optional, Tuple, Union
 from typing_extensions import TypeAlias as _TypeAlias
 
 from mypy.erasetype import remove_instance_last_known_values
@@ -30,6 +30,11 @@
 BindableExpression: _TypeAlias = Union[IndexExpr, MemberExpr, NameExpr]
 
 
+class CurrentType(NamedTuple):
+    type: Type
+    from_assignment: bool
+
+
 class Frame:
     """A Frame represents a specific point in the execution of a program.
     It carries information about the current types of expressions at
@@ -44,7 +49,7 @@ class Frame:
 
     def __init__(self, id: int, conditional_frame: bool = False) -> None:
         self.id = id
-        self.types: dict[Key, Type] = {}
+        self.types: dict[Key, CurrentType] = {}
         self.unreachable = False
         self.conditional_frame = conditional_frame
         self.suppress_unreachable_warnings = False
@@ -132,18 +137,18 @@ def push_frame(self, conditional_frame: bool = False) -> Frame:
         self.options_on_return.append([])
         return f
 
-    def _put(self, key: Key, type: Type, index: int = -1) -> None:
-        self.frames[index].types[key] = type
+    def _put(self, key: Key, type: Type, from_assignment: bool, index: int = -1) -> None:
+        self.frames[index].types[key] = CurrentType(type, from_assignment)
 
-    def _get(self, key: Key, index: int = -1) -> Type | None:
+    def _get(self, key: Key, index: int = -1) -> CurrentType | None:
         if index < 0:
             index += len(self.frames)
         for i in range(index, -1, -1):
             if key in self.frames[i].types:
                 return self.frames[i].types[key]
         return None
 
-    def put(self, expr: Expression, typ: Type) -> None:
+    def put(self, expr: Expression, typ: Type, *, from_assignment: bool = True) -> None:
         if not isinstance(expr, (IndexExpr, MemberExpr, NameExpr)):
             return
         if not literal(expr):
@@ -153,7 +158,7 @@ def put(self, expr: Expression, typ: Type) -> None:
         if key not in self.declarations:
             self.declarations[key] = get_declaration(expr)
             self._add_dependencies(key)
-        self._put(key, typ)
+        self._put(key, typ, from_assignment)
 
     def unreachable(self) -> None:
         self.frames[-1].unreachable = True
@@ -164,7 +169,10 @@ def suppress_unreachable_warnings(self) -> None:
     def get(self, expr: Expression) -> Type | None:
         key = literal_hash(expr)
         assert key is not None, "Internal error: binder tried to get non-literal"
-        return self._get(key)
+        found = self._get(key)
+        if found is None:
+            return None
+        return found.type
 
     def is_unreachable(self) -> bool:
         # TODO: Copy the value of unreachable into new frames to avoid
@@ -193,7 +201,7 @@ def update_from_options(self, frames: list[Frame]) -> bool:
         If a key is declared as AnyType, only update it if all the
         options are the same.
         """
-
+        all_reachable = all(not f.unreachable for f in frames)
         frames = [f for f in frames if not f.unreachable]
         changed = False
         keys = {key for f in frames for key in f.types}
@@ -207,17 +215,30 @@ def update_from_options(self, frames: list[Frame]) -> bool:
                 # know anything about key in at least one possible frame.
                 continue
 
-            type = resulting_values[0]
-            assert type is not None
+            if all_reachable and all(
+                x is not None and not x.from_assignment for x in resulting_values
+            ):
+                # Do not synthesize a new type if we encountered a conditional block
+                # (if, while or match-case) without assignments.
+                # See check-isinstance.test::testNoneCheckDoesNotMakeTypeVarOptional
+                # This is a safe assumption: the fact that we checked something with `is`
+                # or `isinstance` does not change the type of the value.
+                continue
+
+            current_type = resulting_values[0]
+            assert current_type is not None
+            type = current_type.type
             declaration_type = get_proper_type(self.declarations.get(key))
             if isinstance(declaration_type, AnyType):
                 # At this point resulting values can't contain None, see continue above
-                if not all(is_same_type(type, cast(Type, t)) for t in resulting_values[1:]):
+                if not all(
+                    t is not None and is_same_type(type, t.type) for t in resulting_values[1:]
+                ):
                     type = AnyType(TypeOfAny.from_another_any, source_any=declaration_type)
             else:
                 for other in resulting_values[1:]:
                     assert other is not None
-                    type = join_simple(self.declarations[key], type, other)
+                    type = join_simple(self.declarations[key], type, other.type)
                     # Try simplifying resulting type for unions involving variadic tuples.
                     # Technically, everything is still valid without this step, but if we do
                     # not do this, this may create long unions after exiting an if check like:
@@ -236,8 +257,8 @@ def update_from_options(self, frames: list[Frame]) -> bool:
                         )
                         if simplified == self.declarations[key]:
                             type = simplified
-            if current_value is None or not is_same_type(type, current_value):
-                self._put(key, type)
+            if current_value is None or not is_same_type(type, current_value[0]):
+                self._put(key, type, from_assignment=True)
                 changed = True
 
         self.frames[-1].unreachable = not frames
@@ -374,7 +395,9 @@ def most_recent_enclosing_type(self, expr: BindableExpression, type: Type) -> Ty
         key = literal_hash(expr)
         assert key is not None
         enclosers = [get_declaration(expr)] + [
-            f.types[key] for f in self.frames if key in f.types and is_subtype(type, f.types[key])
+            f.types[key].type
+            for f in self.frames
+            if key in f.types and is_subtype(type, f.types[key][0])
         ]
         return enclosers[-1]
 

mypy/checker.py

@@ -4725,11 +4725,11 @@ def visit_if_stmt(self, s: IfStmt) -> None:
 
                 # XXX Issue a warning if condition is always False?
                 with self.binder.frame_context(can_skip=True, fall_through=2):
-                    self.push_type_map(if_map)
+                    self.push_type_map(if_map, from_assignment=False)
                     self.accept(b)
 
                 # XXX Issue a warning if condition is always True?
-                self.push_type_map(else_map)
+                self.push_type_map(else_map, from_assignment=False)
 
             with self.binder.frame_context(can_skip=False, fall_through=2):
                 if s.else_body:
@@ -5310,18 +5310,21 @@ def visit_match_stmt(self, s: MatchStmt) -> None:
                     if b.is_unreachable or isinstance(
                         get_proper_type(pattern_type.type), UninhabitedType
                     ):
-                        self.push_type_map(None)
+                        self.push_type_map(None, from_assignment=False)
                         else_map: TypeMap = {}
                     else:
                         pattern_map, else_map = conditional_types_to_typemaps(
                             named_subject, pattern_type.type, pattern_type.rest_type
                         )
                         self.remove_capture_conflicts(pattern_type.captures, inferred_types)
-                        self.push_type_map(pattern_map)
+                        self.push_type_map(pattern_map, from_assignment=False)
                         if pattern_map:
                             for expr, typ in pattern_map.items():
-                                self.push_type_map(self._get_recursive_sub_patterns_map(expr, typ))
-                        self.push_type_map(pattern_type.captures)
+                                self.push_type_map(
+                                    self._get_recursive_sub_patterns_map(expr, typ),
+                                    from_assignment=False,
+                                )
+                        self.push_type_map(pattern_type.captures, from_assignment=False)
                     if g is not None:
                         with self.binder.frame_context(can_skip=False, fall_through=3):
                             gt = get_proper_type(self.expr_checker.accept(g))
@@ -5347,11 +5350,11 @@ def visit_match_stmt(self, s: MatchStmt) -> None:
                                                 continue
                                             type_map[named_subject] = type_map[expr]
 
-                            self.push_type_map(guard_map)
+                            self.push_type_map(guard_map, from_assignment=False)
                             self.accept(b)
                     else:
                         self.accept(b)
-                self.push_type_map(else_map)
+                self.push_type_map(else_map, from_assignment=False)
 
             # This is needed due to a quirk in frame_context. Without it types will stay narrowed
             # after the match.
@@ -6274,10 +6277,6 @@ def has_no_custom_eq_checks(t: Type) -> bool:
                 coerce_only_in_literal_context,
             )
 
-        # Strictly speaking, we should also skip this check if the objects in the expr
-        # chain have custom __eq__ or __ne__ methods. But we (maybe optimistically)
-        # assume nobody would actually create a custom objects that considers itself
-        # equal to None.
         if if_map == {} and else_map == {}:
             if_map, else_map = self.refine_away_none_in_comparison(
                 operands, operand_types, expr_indices, narrowable_operand_index_to_hash.keys()
@@ -6602,25 +6601,36 @@ def refine_away_none_in_comparison(
         For more details about what the different arguments mean, see the
         docstring of 'refine_identity_comparison_expression' up above.
         """
+
         non_optional_types = []
         for i in chain_indices:
             typ = operand_types[i]
             if not is_overlapping_none(typ):
                 non_optional_types.append(typ)
 
-        # Make sure we have a mixture of optional and non-optional types.
-        if len(non_optional_types) == 0 or len(non_optional_types) == len(chain_indices):
-            return {}, {}
+        if_map, else_map = {}, {}
 
-        if_map = {}
-        for i in narrowable_operand_indices:
-            expr_type = operand_types[i]
-            if not is_overlapping_none(expr_type):
-                continue
-            if any(is_overlapping_erased_types(expr_type, t) for t in non_optional_types):
-                if_map[operands[i]] = remove_optional(expr_type)
+        if not non_optional_types or (len(non_optional_types) != len(chain_indices)):
 
-        return if_map, {}
+            # Narrow e.g. `Optional[A] == "x"` or `Optional[A] is "x"` to `A` (which may be
+            # convenient but is strictly not type-safe):
+            for i in narrowable_operand_indices:
+                expr_type = operand_types[i]
+                if not is_overlapping_none(expr_type):
+                    continue
+                if any(is_overlapping_erased_types(expr_type, t) for t in non_optional_types):
+                    if_map[operands[i]] = remove_optional(expr_type)
+
+            # Narrow e.g. `Optional[A] != None` to `A` (which is stricter than the above step and
+            # so type-safe but less convenient, because e.g. `Optional[A] == None` still results
+            # in `Optional[A]`):
+            if any(isinstance(get_proper_type(ot), NoneType) for ot in operand_types):
+                for i in narrowable_operand_indices:
+                    expr_type = operand_types[i]
+                    if is_overlapping_none(expr_type):
+                        else_map[operands[i]] = remove_optional(expr_type)
+
+        return if_map, else_map
 
     def is_len_of_tuple(self, expr: Expression) -> bool:
         """Is this expression a `len(x)` call where x is a tuple or union of tuples?"""
@@ -7365,12 +7375,12 @@ def iterable_item_type(
     def function_type(self, func: FuncBase) -> FunctionLike:
         return function_type(func, self.named_type("builtins.function"))
 
-    def push_type_map(self, type_map: TypeMap) -> None:
+    def push_type_map(self, type_map: TypeMap, *, from_assignment: bool = True) -> None:
         if type_map is None:
             self.binder.unreachable()
         else:
             for expr, type in type_map.items():
-                self.binder.put(expr, type)
+                self.binder.put(expr, type, from_assignment=from_assignment)
 
     def infer_issubclass_maps(self, node: CallExpr, expr: Expression) -> tuple[TypeMap, TypeMap]:
         """Infer type restrictions for an expression in issubclass call."""
@@ -7686,7 +7696,7 @@ def warn_deprecated(self, node: SymbolNode | None, context: Context) -> None:
             and ((deprecated := node.deprecated) is not None)
             and not self.is_typeshed_stub
         ):
-            warn = self.msg.fail if self.options.report_deprecated_as_error else self.msg.note
+            warn = self.msg.note if self.options.report_deprecated_as_note else self.msg.fail
             warn(deprecated, context, code=codes.DEPRECATED)
 
 
@@ -7743,9 +7753,7 @@ def conditional_types(
         ) and is_proper_subtype(current_type, proposed_type, ignore_promotions=True):
             # Expression is always of one of the types in proposed_type_ranges
             return default, UninhabitedType()
-        elif not is_overlapping_types(
-            current_type, proposed_type, prohibit_none_typevar_overlap=True, ignore_promotions=True
-        ):
+        elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True):
             # Expression is never of any type in proposed_type_ranges
             return UninhabitedType(), default
         else: