Merge branch 'master' into constant-fold-checkers-for-c-type

Author: BobTheBuidler

docs/source/common_issues.rst

@@ -731,7 +731,7 @@ This example demonstrates both safe and unsafe overrides:
 
     class NarrowerReturn(A):
         # A more specific return type is fine
-        def test(self, t: Sequence[int]) -> List[str]:  # OK
+        def test(self, t: Sequence[int]) -> list[str]:  # OK
             ...
 
     class GeneralizedReturn(A):
@@ -746,7 +746,7 @@ not necessary:
 .. code-block:: python
 
     class NarrowerArgument(A):
-        def test(self, t: List[int]) -> Sequence[str]:  # type: ignore[override]
+        def test(self, t: list[int]) -> Sequence[str]:  # type: ignore[override]
             ...
 
 .. _unreachable:

docs/source/error_code_list2.rst

@@ -676,3 +676,26 @@ Example:
                 print("red")
             case _:
                 print("other")
+
+.. _code-untyped-decorator:
+
+Error if an untyped decorator makes a typed function effectively untyped [untyped-decorator]
+--------------------------------------------------------------------------------------------
+
+If enabled with :option:`--disallow-untyped-decorators <mypy --disallow-untyped-decorators>`
+mypy generates an error if a typed function is wrapped by an untyped decorator
+(as this would effectively remove the benefits of typing the function).
+
+Example:
+
+.. code-block:: python
+
+        def printing_decorator(func):
+            def wrapper(*args, **kwds):
+                print("Calling", func)
+                return func(*args, **kwds)
+            return wrapper
+        # A decorated function.
+        @printing_decorator  # E: Untyped decorator makes function "add_forty_two" untyped  [untyped-decorator]
+        def add_forty_two(value: int) -> int:
+            return value + 42

docs/source/stubtest.rst

@@ -99,8 +99,75 @@ to mypy build errors". In this case, you will need to mitigate those errors
 before stubtest will run. Despite potential overlap in errors here, stubtest is
 not intended as a substitute for running mypy directly.
 
+Allowlist
+*********
+
 If you wish to ignore some of stubtest's complaints, stubtest supports a
-pretty handy allowlist system.
+pretty handy :option:`--allowlist` system.
+
+Let's say that you have this python module called ``ex``:
+
+.. code-block:: python
+
+   try:
+       import optional_expensive_dep
+   except ImportError:
+       optional_expensive_dep = None
+
+   first = 1
+   if optional_expensive_dep:
+       second = 2
+
+Let's say that you can't install ``optional_expensive_dep`` in CI for some reason,
+but you still want to include ``second: int`` in the stub file:
+
+.. code-block:: python
+
+    first: int
+    second: int
+
+In this case stubtest will correctly complain:
+
+.. code-block:: shell
+
+   error: ex.second is not present at runtime
+   Stub: in file /.../ex.pyi:2
+   builtins.int
+   Runtime:
+   MISSING
+
+   Found 1 error (checked 1 module)
+
+To fix this, you can add an ``allowlist`` entry:
+
+.. code-block:: ini
+
+   # Allowlist entries in `allowlist.txt` file:
+
+   # Does not exist if `optional_expensive_dep` is not installed:
+   ex.second
+
+And now when running stubtest with ``--allowlist=allowlist.txt``,
+no errors will be generated anymore.
+
+Allowlists also support regular expressions,
+which can be useful to ignore many similar errors at once.
+They can also be useful for suppressing stubtest errors that occur sometimes,
+but not on every CI run. For example, if some CI workers have
+``optional_expensive_dep`` installed, stubtest might complain with this message
+on those workers if you had the ``ex.second`` allowlist entry:
+
+.. code-block:: ini
+
+   note: unused allowlist entry ex.second
+   Found 1 error (checked 1 module)
+
+Changing ``ex.second`` to be ``(ex\.second)?`` will make this error optional,
+meaning that stubtest will pass whether or not a CI runner
+has``optional_expensive_dep`` installed.
+
+CLI
+***
 
 The rest of this section documents the command line interface of stubtest.
 
@@ -119,15 +186,15 @@ The rest of this section documents the command line interface of stubtest.
 .. option:: --allowlist FILE
 
     Use file as an allowlist. Can be passed multiple times to combine multiple
-    allowlists. Allowlists can be created with --generate-allowlist. Allowlists
-    support regular expressions.
+    allowlists. Allowlists can be created with :option:`--generate-allowlist`.
+    Allowlists support regular expressions.
 
     The presence of an entry in the allowlist means stubtest will not generate
     any errors for the corresponding definition.
 
 .. option:: --generate-allowlist
 
-    Print an allowlist (to stdout) to be used with --allowlist
+    Print an allowlist (to stdout) to be used with :option:`--allowlist`.
 
     When introducing stubtest to an existing project, this is an easy way to
     silence all existing errors.
@@ -141,17 +208,17 @@ The rest of this section documents the command line interface of stubtest.
 
     Note if an allowlist entry is a regex that matches the empty string,
     stubtest will never consider it unused. For example, to get
-    `--ignore-unused-allowlist` behaviour for a single allowlist entry like
+    ``--ignore-unused-allowlist`` behaviour for a single allowlist entry like
     ``foo.bar`` you could add an allowlist entry ``(foo\.bar)?``.
     This can be useful when an error only occurs on a specific platform.
 
 .. option:: --mypy-config-file FILE
 
-    Use specified mypy config file to determine mypy plugins and mypy path
+    Use specified mypy config *file* to determine mypy plugins and mypy path
 
 .. option:: --custom-typeshed-dir DIR
 
-    Use the custom typeshed in DIR
+    Use the custom typeshed in *DIR*
 
 .. option:: --check-typeshed
 

mypy/build.py

@@ -143,6 +143,10 @@ def __init__(self, manager: BuildManager, graph: Graph) -> None:
         self.errors: list[str] = []  # Filled in by build if desired
 
 
+def build_error(msg: str) -> NoReturn:
+    raise CompileError([f"mypy: error: {msg}"])
+
+
 def build(
     sources: list[BuildSource],
     options: Options,
@@ -241,6 +245,9 @@ def _build(
     errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read))
     plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins)
 
+    # Validate error codes after plugins are loaded.
+    options.process_error_codes(error_callback=build_error)
+
     # Add catch-all .gitignore to cache dir if we created it
     cache_dir_existed = os.path.isdir(options.cache_dir)
 

mypy/cache.py

@@ -1,65 +1,22 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
-from typing import TYPE_CHECKING, Final
+from typing import Final
 
 from mypy_extensions import u8
-
-try:
-    from native_internal import (
-        Buffer as Buffer,
-        read_bool as read_bool,
-        read_float as read_float,
-        read_int as read_int,
-        read_str as read_str,
-        read_tag as read_tag,
-        write_bool as write_bool,
-        write_float as write_float,
-        write_int as write_int,
-        write_str as write_str,
-        write_tag as write_tag,
-    )
-except ImportError:
-    # TODO: temporary, remove this after we publish mypy-native on PyPI.
-    if not TYPE_CHECKING:
-
-        class Buffer:
-            def __init__(self, source: bytes = b"") -> None:
-                raise NotImplementedError
-
-            def getvalue(self) -> bytes:
-                raise NotImplementedError
-
-        def read_int(data: Buffer) -> int:
-            raise NotImplementedError
-
-        def write_int(data: Buffer, value: int) -> None:
-            raise NotImplementedError
-
-        def read_tag(data: Buffer) -> u8:
-            raise NotImplementedError
-
-        def write_tag(data: Buffer, value: u8) -> None:
-            raise NotImplementedError
-
-        def read_str(data: Buffer) -> str:
-            raise NotImplementedError
-
-        def write_str(data: Buffer, value: str) -> None:
-            raise NotImplementedError
-
-        def read_bool(data: Buffer) -> bool:
-            raise NotImplementedError
-
-        def write_bool(data: Buffer, value: bool) -> None:
-            raise NotImplementedError
-
-        def read_float(data: Buffer) -> float:
-            raise NotImplementedError
-
-        def write_float(data: Buffer, value: float) -> None:
-            raise NotImplementedError
-
+from native_internal import (
+    Buffer as Buffer,
+    read_bool as read_bool,
+    read_float as read_float,
+    read_int as read_int,
+    read_str as read_str,
+    read_tag as read_tag,
+    write_bool as write_bool,
+    write_float as write_float,
+    write_int as write_int,
+    write_str as write_str,
+    write_tag as write_tag,
+)
 
 # Always use this type alias to refer to type tags.
 Tag = u8

mypy/checker.py

@@ -1801,7 +1801,8 @@ def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None:
                 "but must return a subtype of",
             )
         elif not isinstance(
-            get_proper_type(bound_type.ret_type), (AnyType, Instance, TupleType, UninhabitedType)
+            get_proper_type(bound_type.ret_type),
+            (AnyType, Instance, TupleType, UninhabitedType, LiteralType),
         ):
             self.fail(
                 message_registry.NON_INSTANCE_NEW_TYPE.format(

mypy/checkexpr.py

@@ -18,6 +18,7 @@
 from mypy.checker_shared import ExpressionCheckerSharedApi
 from mypy.checkmember import analyze_member_access, has_operator
 from mypy.checkstrformat import StringFormatterChecker
+from mypy.constant_fold import constant_fold_expr
 from mypy.erasetype import erase_type, remove_instance_last_known_values, replace_meta_vars
 from mypy.errors import ErrorInfo, ErrorWatcher, report_internal_error
 from mypy.expandtype import (
@@ -656,11 +657,12 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) ->
         return ret_type
 
     def check_str_format_call(self, e: CallExpr) -> None:
-        """More precise type checking for str.format() calls on literals."""
+        """More precise type checking for str.format() calls on literals and folded constants."""
         assert isinstance(e.callee, MemberExpr)
         format_value = None
-        if isinstance(e.callee.expr, StrExpr):
-            format_value = e.callee.expr.value
+        folded_callee_expr = constant_fold_expr(e.callee.expr, "<unused>")
+        if isinstance(folded_callee_expr, str):
+            format_value = folded_callee_expr
         elif self.chk.has_type(e.callee.expr):
             typ = get_proper_type(self.chk.lookup_type(e.callee.expr))
             if (

mypy/checkmember.py

@@ -410,6 +410,8 @@ def analyze_type_callable_member_access(name: str, typ: FunctionLike, mx: Member
         ret_type = tuple_fallback(ret_type)
     if isinstance(ret_type, TypedDictType):
         ret_type = ret_type.fallback
+    if isinstance(ret_type, LiteralType):
+        ret_type = ret_type.fallback
     if isinstance(ret_type, Instance):
         if not mx.is_operator:
             # When Python sees an operator (eg `3 == 4`), it automatically translates that

mypy/config_parser.py

@@ -8,8 +8,6 @@
 import sys
 from io import StringIO
 
-from mypy.errorcodes import error_codes
-
 if sys.version_info >= (3, 11):
     import tomllib
 else:
@@ -87,15 +85,6 @@ def complain(x: object, additional_info: str = "") -> Never:
         complain(v)
 
 
-def validate_codes(codes: list[str]) -> list[str]:
-    invalid_codes = set(codes) - set(error_codes.keys())
-    if invalid_codes:
-        raise argparse.ArgumentTypeError(
-            f"Invalid error code(s): {', '.join(sorted(invalid_codes))}"
-        )
-    return codes
-
-
 def validate_package_allow_list(allow_list: list[str]) -> list[str]:
     for p in allow_list:
         msg = f"Invalid allow list entry: {p}"
@@ -209,8 +198,8 @@ def split_commas(value: str) -> list[str]:
         [p.strip() for p in split_commas(s)]
     ),
     "enable_incomplete_feature": lambda s: [p.strip() for p in split_commas(s)],
-    "disable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]),
-    "enable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]),
+    "disable_error_code": lambda s: [p.strip() for p in split_commas(s)],
+    "enable_error_code": lambda s: [p.strip() for p in split_commas(s)],
     "package_root": lambda s: [p.strip() for p in split_commas(s)],
     "cache_dir": expand_path,
     "python_executable": expand_path,
@@ -234,8 +223,8 @@ def split_commas(value: str) -> list[str]:
         "always_false": try_split,
         "untyped_calls_exclude": lambda s: validate_package_allow_list(try_split(s)),
         "enable_incomplete_feature": try_split,
-        "disable_error_code": lambda s: validate_codes(try_split(s)),
-        "enable_error_code": lambda s: validate_codes(try_split(s)),
+        "disable_error_code": lambda s: try_split(s),
+        "enable_error_code": lambda s: try_split(s),
         "package_root": try_split,
         "exclude": str_or_array_as_list,
         "packages": try_split,

mypy/errorcodes.py

@@ -302,6 +302,10 @@ def __hash__(self) -> int:
     sub_code_of=MISC,
 )
 
+UNTYPED_DECORATOR: Final = ErrorCode(
+    "untyped-decorator", "Error if an untyped decorator makes a typed function untyped", "General"
+)
+
 NARROWED_TYPE_NOT_SUBTYPE: Final = ErrorCode(
     "narrowed-type-not-subtype",
     "Warn if a TypeIs function's narrowed type is not a subtype of the original type",