Merge branch 'master' into f-string-folding

Author: BobTheBuidler

.github/workflows/mypy_primer.yml

@@ -67,6 +67,7 @@ jobs:
             --debug \
             --additional-flags="--debug-serialize" \
             --output concise \
+            --mypy-install-librt \
             | tee diff_${{ matrix.shard-index }}.txt
           ) || [ $? -eq 1 ]
       - if: ${{ matrix.shard-index == 0 }}

docs/source/command_line.rst

@@ -372,7 +372,7 @@ definitions or calls.
 
 .. option:: --untyped-calls-exclude
 
-    This flag allows to selectively disable :option:`--disallow-untyped-calls`
+    This flag allows one to selectively disable :option:`--disallow-untyped-calls`
     for functions and methods defined in specific packages, modules, or classes.
     Note that each exclude entry acts as a prefix. For example (assuming there
     are no type annotations for ``third_party_lib`` available):
@@ -562,7 +562,7 @@ potentially problematic or redundant in some way.
 
 .. option:: --deprecated-calls-exclude
 
-    This flag allows to selectively disable :ref:`deprecated<code-deprecated>` warnings
+    This flag allows one to selectively disable :ref:`deprecated<code-deprecated>` warnings
     for functions and methods defined in specific packages, modules, or classes.
     Note that each exclude entry acts as a prefix. For example (assuming ``foo.A.func`` is deprecated):
 

docs/source/error_code_list.rst

@@ -1032,8 +1032,8 @@ Warn about top level await expressions [top-level-await]
 This error code is separate from the general ``[syntax]`` errors, because in
 some environments (e.g. IPython) a top level ``await`` is allowed. In such
 environments a user may want to use ``--disable-error-code=top-level-await``,
-that allows to still have errors for other improper uses of ``await``, for
-example:
+which allows one to still have errors for other improper uses of ``await``,
+for example:
 
 .. code-block:: python
 

docs/source/mypy_daemon.rst

@@ -252,16 +252,16 @@ command.
 Statically inspect expressions
 ******************************
 
-The daemon allows to get declared or inferred type of an expression (or other
+The daemon allows one to get the declared or inferred type of an expression (or other
 information about an expression, such as known attributes or definition location)
-using ``dmypy inspect LOCATION`` command. The location of the expression should be
+using the ``dmypy inspect LOCATION`` command. The location of the expression should be
 specified in the format ``path/to/file.py:line:column[:end_line:end_column]``.
 Both line and column are 1-based. Both start and end position are inclusive.
 These rules match how mypy prints the error location in error messages.
 
 If a span is given (i.e. all 4 numbers), then only an exactly matching expression
 is inspected. If only a position is given (i.e. 2 numbers, line and column), mypy
-will inspect all *expressions*, that include this position, starting from the
+will inspect all expressions that include this position, starting from the
 innermost one.
 
 Consider this Python code snippet:

mypy-requirements.txt

@@ -4,3 +4,4 @@ typing_extensions>=4.6.0
 mypy_extensions>=1.0.0
 pathspec>=0.9.0
 tomli>=1.1.0; python_version<'3.11'
+librt>=0.1.0

mypy/build.py

@@ -1546,7 +1546,7 @@ def write_cache(
     source_hash: str,
     ignore_all: bool,
     manager: BuildManager,
-) -> tuple[str, tuple[dict[str, Any], str, str] | None]:
+) -> tuple[str, tuple[dict[str, Any], str] | None]:
     """Write cache files for a module.
 
     Note that this mypy's behavior is still correct when any given
@@ -1568,7 +1568,7 @@ def write_cache(
 
     Returns:
       A tuple containing the interface hash and inner tuple with cache meta JSON
-      that should be written and paths to cache files (inner tuple may be None,
+      that should be written and path to cache file (inner tuple may be None,
       if the cache data could not be written).
     """
     metastore = manager.metastore
@@ -1662,12 +1662,10 @@ def write_cache(
         "ignore_all": ignore_all,
         "plugin_data": plugin_data,
     }
-    return interface_hash, (meta, meta_json, data_json)
+    return interface_hash, (meta, meta_json)
 
 
-def write_cache_meta(
-    meta: dict[str, Any], manager: BuildManager, meta_json: str, data_json: str
-) -> CacheMeta:
+def write_cache_meta(meta: dict[str, Any], manager: BuildManager, meta_json: str) -> None:
     # Write meta cache file
     metastore = manager.metastore
     meta_str = json_dumps(meta, manager.options.debug_cache)
@@ -1677,8 +1675,6 @@ def write_cache_meta(
         # The next run will simply find the cache entry out of date.
         manager.log(f"Error writing meta JSON file {meta_json}")
 
-    return cache_meta_from_dict(meta, data_json)
-
 
 """Dependency manager.
 
@@ -1864,9 +1860,6 @@ class State:
     # List of (path, line number) tuples giving context for import
     import_context: list[tuple[str, int]]
 
-    # The State from which this module was imported, if any
-    caller_state: State | None = None
-
     # If caller_state is set, the line number in the caller where the import occurred
     caller_line = 0
 
@@ -1917,7 +1910,6 @@ def __init__(
         self.manager = manager
         State.order_counter += 1
         self.order = State.order_counter
-        self.caller_state = caller_state
         self.caller_line = caller_line
         if caller_state:
             self.import_context = caller_state.import_context.copy()
@@ -2008,11 +2000,6 @@ def __init__(
             self.parse_file(temporary=temporary)
             self.compute_dependencies()
 
-    @property
-    def xmeta(self) -> CacheMeta:
-        assert self.meta, "missing meta on allegedly fresh module"
-        return self.meta
-
     def add_ancestors(self) -> None:
         if self.path is not None:
             _, name = os.path.split(self.path)
@@ -2479,7 +2466,7 @@ def valid_references(self) -> set[str]:
 
         return valid_refs
 
-    def write_cache(self) -> tuple[dict[str, Any], str, str] | None:
+    def write_cache(self) -> tuple[dict[str, Any], str] | None:
         assert self.tree is not None, "Internal error: method must be called on parsed file only"
         # We don't support writing cache files in fine-grained incremental mode.
         if (
@@ -3477,18 +3464,17 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No
     for id in stale:
         meta_tuple = meta_tuples[id]
         if meta_tuple is None:
-            graph[id].meta = None
             continue
-        meta, meta_json, data_json = meta_tuple
+        meta, meta_json = meta_tuple
         meta["dep_hashes"] = {
             dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph
         }
         meta["error_lines"] = errors_by_id.get(id, [])
-        graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json)
+        write_cache_meta(meta, manager, meta_json)
 
 
 def sorted_components(
-    graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_ALL
+    graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_INDIRECT
 ) -> list[AbstractSet[str]]:
     """Return the graph's SCCs, topologically sorted by dependencies.
 

mypy/cache.py

@@ -1,65 +1,22 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
-from typing import TYPE_CHECKING, Final
+from typing import Final
 
 from mypy_extensions import u8
-
-try:
-    from native_internal import (
-        Buffer as Buffer,
-        read_bool as read_bool,
-        read_float as read_float,
-        read_int as read_int,
-        read_str as read_str,
-        read_tag as read_tag,
-        write_bool as write_bool,
-        write_float as write_float,
-        write_int as write_int,
-        write_str as write_str,
-        write_tag as write_tag,
-    )
-except ImportError:
-    # TODO: temporary, remove this after we publish mypy-native on PyPI.
-    if not TYPE_CHECKING:
-
-        class Buffer:
-            def __init__(self, source: bytes = b"") -> None:
-                raise NotImplementedError
-
-            def getvalue(self) -> bytes:
-                raise NotImplementedError
-
-        def read_int(data: Buffer) -> int:
-            raise NotImplementedError
-
-        def write_int(data: Buffer, value: int) -> None:
-            raise NotImplementedError
-
-        def read_tag(data: Buffer) -> u8:
-            raise NotImplementedError
-
-        def write_tag(data: Buffer, value: u8) -> None:
-            raise NotImplementedError
-
-        def read_str(data: Buffer) -> str:
-            raise NotImplementedError
-
-        def write_str(data: Buffer, value: str) -> None:
-            raise NotImplementedError
-
-        def read_bool(data: Buffer) -> bool:
-            raise NotImplementedError
-
-        def write_bool(data: Buffer, value: bool) -> None:
-            raise NotImplementedError
-
-        def read_float(data: Buffer) -> float:
-            raise NotImplementedError
-
-        def write_float(data: Buffer, value: float) -> None:
-            raise NotImplementedError
-
+from native_internal import (
+    Buffer as Buffer,
+    read_bool as read_bool,
+    read_float as read_float,
+    read_int as read_int,
+    read_str as read_str,
+    read_tag as read_tag,
+    write_bool as write_bool,
+    write_float as write_float,
+    write_int as write_int,
+    write_str as write_str,
+    write_tag as write_tag,
+)
 
 # Always use this type alias to refer to type tags.
 Tag = u8

mypy/checkexpr.py

@@ -18,6 +18,7 @@
 from mypy.checker_shared import ExpressionCheckerSharedApi
 from mypy.checkmember import analyze_member_access, has_operator
 from mypy.checkstrformat import StringFormatterChecker
+from mypy.constant_fold import constant_fold_expr
 from mypy.erasetype import erase_type, remove_instance_last_known_values, replace_meta_vars
 from mypy.errors import ErrorInfo, ErrorWatcher, report_internal_error
 from mypy.expandtype import (
@@ -656,11 +657,12 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) ->
         return ret_type
 
     def check_str_format_call(self, e: CallExpr) -> None:
-        """More precise type checking for str.format() calls on literals."""
+        """More precise type checking for str.format() calls on literals and folded constants."""
         assert isinstance(e.callee, MemberExpr)
         format_value = None
-        if isinstance(e.callee.expr, StrExpr):
-            format_value = e.callee.expr.value
+        folded_callee_expr = constant_fold_expr(e.callee.expr, "<unused>")
+        if isinstance(folded_callee_expr, str):
+            format_value = folded_callee_expr
         elif self.chk.has_type(e.callee.expr):
             typ = get_proper_type(self.chk.lookup_type(e.callee.expr))
             if (

mypy/semanal.py

@@ -5928,8 +5928,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None:
                 if isinstance(sym.node, PlaceholderNode):
                     self.process_placeholder(expr.name, "attribute", expr)
                     return
-                if sym.node is not None:
-                    self.record_imported_symbol(sym.node)
+                self.record_imported_symbol(sym)
                 expr.kind = sym.kind
                 expr.fullname = sym.fullname or ""
                 expr.node = sym.node
@@ -5960,7 +5959,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None:
             if type_info:
                 n = type_info.names.get(expr.name)
                 if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)):
-                    self.record_imported_symbol(n.node)
+                    self.record_imported_symbol(n)
                     expr.kind = n.kind
                     expr.fullname = n.fullname or ""
                     expr.node = n.node
@@ -6282,30 +6281,48 @@ def lookup(
         self, name: str, ctx: Context, suppress_errors: bool = False
     ) -> SymbolTableNode | None:
         node = self._lookup(name, ctx, suppress_errors)
-        if node is not None and node.node is not None:
+        if node is not None:
             # This call is unfortunate from performance point of view, but
             # needed for rare cases like e.g. testIncrementalChangingAlias.
-            self.record_imported_symbol(node.node)
+            self.record_imported_symbol(node)
         return node
 
-    def record_imported_symbol(self, node: SymbolNode) -> None:
+    def record_imported_symbol(self, sym: SymbolTableNode) -> None:
         """If the symbol was not defined in current module, add its module to module_refs."""
-        if not node.fullname:
+        if sym.kind == LDEF or sym.node is None:
             return
+        node = sym.node
+        if isinstance(node, PlaceholderNode) or not node.fullname:
+            # This node is not ready yet.
+            return
+        if node.fullname.startswith(("builtins.", "typing.")):
+            # Skip dependencies on builtins/typing.
+            return
+        # Modules, classes, and type aliases store defining module directly.
         if isinstance(node, MypyFile):
             fullname = node.fullname
         elif isinstance(node, TypeInfo):
             fullname = node.module_name
         elif isinstance(node, TypeAlias):
             fullname = node.module
-        elif isinstance(node, (Var, FuncDef, OverloadedFuncDef)) and node.info:
-            fullname = node.info.module_name
+        elif isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)):
+            # For functions/variables infer defining module from enclosing class.
+            info = node.var.info if isinstance(node, Decorator) else node.info
+            if info:
+                fullname = info.module_name
+            else:
+                # global function/variable
+                fullname = node.fullname.rsplit(".", maxsplit=1)[0]
         else:
-            fullname = node.fullname.rsplit(".")[0]
+            # Some nodes (currently only TypeVarLikeExpr subclasses) don't store
+            # module fullname explicitly, infer it from the node fullname iteratively.
+            # TODO: this is not 100% robust for type variables nested within a class
+            # with a name that matches name of a submodule.
+            fullname = node.fullname.rsplit(".", maxsplit=1)[0]
             if fullname == self.cur_mod_id:
                 return
             while "." in fullname and fullname not in self.modules:
-                fullname = fullname.rsplit(".")[0]
+                fullname = fullname.rsplit(".", maxsplit=1)[0]
         if fullname != self.cur_mod_id:
             self.cur_mod_node.module_refs.add(fullname)
 
@@ -6519,8 +6536,8 @@ def lookup_qualified(
                         self.name_not_defined(name, ctx, namespace=namespace)
                     return None
                 sym = nextsym
-        if sym is not None and sym.node is not None:
-            self.record_imported_symbol(sym.node)
+        if sym is not None:
+            self.record_imported_symbol(sym)
         return sym
 
     def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None:

mypy/semanal_shared.py

@@ -303,7 +303,9 @@ def calculate_tuple_fallback(typ: TupleType) -> None:
             ):
                 items.append(unpacked_type.args[0])
             else:
-                raise NotImplementedError
+                # This is called before semanal_typeargs.py fixes broken unpacks,
+                # where the error should also be generated.
+                items.append(AnyType(TypeOfAny.from_error))
         else:
             items.append(item)
     fallback.args = (make_simplified_union(items),)