[PEP 747] Recognize TypeForm[T] type and values (#9773)

docs/source/error_code_list.rst

@@ -1286,6 +1286,65 @@ type must be a subtype of the original type::
     def g(x: object) -> TypeIs[str]:  # OK
         ...
 
+.. _code-maybe-unrecognized-str-typeform:
+
+String appears in a context which expects a TypeForm [maybe-unrecognized-str-typeform]
+--------------------------------------------------------------------------------------
+
+TypeForm literals may contain string annotations:
+
+.. code-block:: python
+
+   typx1: TypeForm = str | None
+   typx2: TypeForm = 'str | None'  # OK
+   typx3: TypeForm = 'str' | None  # OK
+
+However TypeForm literals containing a string annotation can only be recognized
+by mypy in the following locations:
+
+.. code-block:: python
+
+   typx_var: TypeForm = 'str | None'  # assignment r-value
+
+   def func(typx_param: TypeForm) -> TypeForm:
+       return 'str | None'  # returned expression
+
+   func('str | None')  # callable's argument
+
+If you try to use a string annotation in some other location
+which expects a TypeForm, the string value will always be treated as a ``str``
+even if a ``TypeForm`` would be more appropriate and this error code
+will be generated:
+
+.. code-block:: python
+
+   # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize.  [maybe-unrecognized-str-typeform]
+   # Error: List item 0 has incompatible type "str"; expected "TypeForm[Any]"  [list-item]
+   list_of_typx: list[TypeForm] = ['str | None', float]
+
+Fix the error by surrounding the entire type with ``TypeForm(...)``:
+
+.. code-block:: python
+
+   list_of_typx: list[TypeForm] = [TypeForm('str | None'), float]  # OK
+
+Similarly, if you try to use a string literal in a location which expects a
+TypeForm, this error code will be generated:
+
+.. code-block:: python
+
+   dict_of_typx = {'str_or_none': TypeForm(str | None)}
+   # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize.  [maybe-unrecognized-str-typeform]
+   list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']]
+
+Fix the error by adding ``# type: ignore[maybe-unrecognized-str-typeform]``
+to the line with the string literal:
+
+.. code-block:: python
+
+   dict_of_typx = {'str_or_none': TypeForm(str | None)}
+   list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']]  # type: ignore[maybe-unrecognized-str-typeform]
+
 .. _code-misc:
 
 Miscellaneous checks [misc]

misc/analyze_typeform_stats.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Analyze TypeForm parsing efficiency from mypy build stats.
+
+Usage:
+    python3 analyze_typeform_stats.py '<mypy_output_with_stats>'
+    python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py
+
+Example output:
+    TypeForm Expression Parsing Statistics:
+    ==================================================
+    Total calls to SA.try_parse_as_type_expression: 14,555
+    Quick rejections (no full parse): 14,255
+    Full parses attempted: 300
+    - Successful: 248
+    - Failed: 52
+
+    Efficiency Metrics:
+    - Quick rejection rate: 97.9%
+    - Full parse rate: 2.1%
+    - Full parse success rate: 82.7%
+    - Overall success rate: 1.7%
+
+    Performance Implications:
+    - Expensive failed full parses: 52 (0.4% of all calls)
+
+See also:
+    - mypy/semanal.py: SemanticAnalyzer.try_parse_as_type_expression()
+    - mypy/semanal.py: DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES
+"""
+
+import re
+import sys
+
+
+def analyze_stats(output: str) -> None:
+    """Parse mypy stats output and calculate TypeForm parsing efficiency."""
+
+    # Extract the three counters
+    total_match = re.search(r"type_expression_parse_count:\s*(\d+)", output)
+    success_match = re.search(r"type_expression_full_parse_success_count:\s*(\d+)", output)
+    failure_match = re.search(r"type_expression_full_parse_failure_count:\s*(\d+)", output)
+
+    if not (total_match and success_match and failure_match):
+        print("Error: Could not find all required counters in output")
+        return
+
+    total = int(total_match.group(1))
+    successes = int(success_match.group(1))
+    failures = int(failure_match.group(1))
+
+    full_parses = successes + failures
+
+    print("TypeForm Expression Parsing Statistics:")
+    print("=" * 50)
+    print(f"Total calls to SA.try_parse_as_type_expression: {total:,}")
+    print(f"Quick rejections (no full parse): {total - full_parses:,}")
+    print(f"Full parses attempted: {full_parses:,}")
+    print(f"  - Successful: {successes:,}")
+    print(f"  - Failed: {failures:,}")
+    if total > 0:
+        print()
+        print("Efficiency Metrics:")
+        print(f"  - Quick rejection rate: {((total - full_parses) / total * 100):.1f}%")
+        print(f"  - Full parse rate: {(full_parses / total * 100):.1f}%")
+        print(f"  - Full parse success rate: {(successes / full_parses * 100):.1f}%")
+        print(f"  - Overall success rate: {(successes / total * 100):.1f}%")
+        print()
+        print("Performance Implications:")
+        print(
+            f"  - Expensive failed full parses: {failures:,} ({(failures / total * 100):.1f}% of all calls)"
+        )
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        # Read from stdin
+        output = sys.stdin.read()
+    elif len(sys.argv) == 2:
+        # Read from command line argument
+        output = sys.argv[1]
+    else:
+        print("Usage: python3 analyze_typeform_stats.py [mypy_output_with_stats]")
+        print("Examples:")
+        print(
+            "  python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py"
+        )
+        print("  python3 analyze_typeform_stats.py 'output_string'")
+        sys.exit(1)
+
+    analyze_stats(output)

mypy/checker.py

@@ -140,6 +140,7 @@
 from mypy.scope import Scope
 from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name
 from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS
+from mypy.semanal_shared import SemanticAnalyzerCoreInterface
 from mypy.sharedparse import BINARY_MAGIC_METHODS
 from mypy.state import state
 from mypy.subtypes import (
@@ -309,6 +310,8 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi):
 
     tscope: Scope
     scope: CheckerScope
+    # Innermost enclosing type
+    type: TypeInfo | None
     # Stack of function return types
     return_types: list[Type]
     # Flags; true for dynamically typed functions
@@ -386,6 +389,7 @@ def __init__(
         self.scope = CheckerScope(tree)
         self.binder = ConditionalTypeBinder(options)
         self.globals = tree.names
+        self.type = None
         self.return_types = []
         self.dynamic_funcs = []
         self.partial_types = []
@@ -2611,7 +2615,11 @@ def visit_class_def(self, defn: ClassDef) -> None:
         for base in typ.mro[1:]:
             if base.is_final:
                 self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn)
-        with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True):
+        with (
+            self.tscope.class_scope(defn.info),
+            self.enter_partial_types(is_class=True),
+            self.enter_class(defn.info),
+        ):
             old_binder = self.binder
             self.binder = ConditionalTypeBinder(self.options)
             with self.binder.top_frame_context():
@@ -2679,6 +2687,15 @@ def visit_class_def(self, defn: ClassDef) -> None:
             self.check_enum(defn)
         infer_class_variances(defn.info)
 
+    @contextmanager
+    def enter_class(self, type: TypeInfo) -> Iterator[None]:
+        original_type = self.type
+        self.type = type
+        try:
+            yield
+        finally:
+            self.type = original_type
+
     def check_final_deletable(self, typ: TypeInfo) -> None:
         # These checks are only for mypyc. Only perform some checks that are easier
         # to implement here than in mypyc.
@@ -7891,7 +7908,9 @@ def add_any_attribute_to_type(self, typ: Type, name: str) -> Type:
             fallback = typ.fallback.copy_with_extra_attr(name, any_type)
             return typ.copy_modified(fallback=fallback)
         if isinstance(typ, TypeType) and isinstance(typ.item, Instance):
-            return TypeType.make_normalized(self.add_any_attribute_to_type(typ.item, name))
+            return TypeType.make_normalized(
+                self.add_any_attribute_to_type(typ.item, name), is_type_form=typ.is_type_form
+            )
         if isinstance(typ, TypeVarType):
             return typ.copy_modified(
                 upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name),
@@ -8019,6 +8038,97 @@ def visit_global_decl(self, o: GlobalDecl, /) -> None:
         return None
 
 
+class TypeCheckerAsSemanticAnalyzer(SemanticAnalyzerCoreInterface):
+    """
+    Adapts TypeChecker to the SemanticAnalyzerCoreInterface,
+    allowing most type expressions to be parsed during the TypeChecker pass.
+
+    See ExpressionChecker.try_parse_as_type_expression() to understand how this
+    class is used.
+    """
+
+    _chk: TypeChecker
+    _names: dict[str, SymbolTableNode]
+    did_fail: bool
+
+    def __init__(self, chk: TypeChecker, names: dict[str, SymbolTableNode]) -> None:
+        self._chk = chk
+        self._names = names
+        self.did_fail = False
+
+    def lookup_qualified(
+        self, name: str, ctx: Context, suppress_errors: bool = False
+    ) -> SymbolTableNode | None:
+        sym = self._names.get(name)
+        # All names being looked up should have been previously gathered,
+        # even if the related SymbolTableNode does not refer to a valid SymbolNode
+        assert sym is not None, name
+        return sym
+
+    def lookup_fully_qualified(self, fullname: str, /) -> SymbolTableNode:
+        ret = self.lookup_fully_qualified_or_none(fullname)
+        assert ret is not None, fullname
+        return ret
+
+    def lookup_fully_qualified_or_none(self, fullname: str, /) -> SymbolTableNode | None:
+        try:
+            return self._chk.lookup_qualified(fullname)
+        except KeyError:
+            return None
+
+    def fail(
+        self,
+        msg: str,
+        ctx: Context,
+        serious: bool = False,
+        *,
+        blocker: bool = False,
+        code: ErrorCode | None = None,
+    ) -> None:
+        self.did_fail = True
+
+    def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None:
+        pass
+
+    def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool:
+        if feature not in self._chk.options.enable_incomplete_feature:
+            self.fail("__ignored__", ctx)
+            return False
+        return True
+
+    def record_incomplete_ref(self) -> None:
+        pass
+
+    def defer(self, debug_context: Context | None = None, force_progress: bool = False) -> None:
+        pass
+
+    def is_incomplete_namespace(self, fullname: str) -> bool:
+        return False
+
+    @property
+    def final_iteration(self) -> bool:
+        return True
+
+    def is_future_flag_set(self, flag: str) -> bool:
+        return self._chk.tree.is_future_flag_set(flag)
+
+    @property
+    def is_stub_file(self) -> bool:
+        return self._chk.tree.is_stub
+
+    def is_func_scope(self) -> bool:
+        # Return arbitrary value.
+        #
+        # This method is currently only used to decide whether to pair
+        # a fail() message with a note() message or not. Both of those
+        # message types are ignored.
+        return False
+
+    @property
+    def type(self) -> TypeInfo | None:
+        return self._chk.type
+
+
 class CollectArgTypeVarTypes(TypeTraverserVisitor):
     """Collects the non-nested argument types in a set."""