Merge branch 'main' into pep728

Author: JelleZijlstra

CHANGELOG.md

@@ -21,6 +21,10 @@ aliases that have a `Concatenate` special form as their argument.
   Patch by [Daraan](https://github.com/Daraan).
 - Extended the `Concatenate` backport for Python 3.8-3.10 to now accept
   `Ellipsis` as an argument. Patch by [Daraan](https://github.com/Daraan).
+- Fix backport of `get_type_hints` to reflect Python 3.11+ behavior which does not add
+  `Union[..., NoneType]` to annotations that have a `None` default value anymore.
+  This fixes wrapping of `Annotated` in an unwanted `Optional` in such cases.
+  Patch by [Daraan](https://github.com/Daraan).
 - Fix error in subscription of `Unpack` aliases causing nested Unpacks 
   to not be resolved correctly. Patch by [Daraan](https://github.com/Daraan).
 - Backport CPython PR [#124795](https://github.com/python/cpython/pull/124795):
@@ -34,6 +38,13 @@ aliases that have a `Concatenate` special form as their argument.
   Patch by [Daraan](https://github.com/Daraan).
 - Fix error on Python 3.10 when using `typing.Concatenate` and 
   `typing_extensions.Concatenate` together. Patch by [Daraan](https://github.com/Daraan).
+- Backport of CPython PR [#109544](https://github.com/python/cpython/pull/109544)
+  to reflect Python 3.13+ behavior: A value assigned to `__total__` in the class body of a
+  `TypedDict` will be overwritten by the `total` argument of the `TypedDict` constructor.
+  Patch by [Daraan](https://github.com/Daraan), backporting a CPython PR by Jelle Zijlstra.
+- Fix for Python 3.11 that now `isinstance(typing_extensions.Unpack[...], TypeVar)` 
+  evaluates to `False`, however still `True` for <3.11.
+  Patch by [Daraan](https://github.com/Daraan)
 
 # Release 4.12.2 (June 7, 2024)
 

doc/index.rst

@@ -133,7 +133,7 @@ Example usage::
    False
    >>> is_literal(get_origin(typing.Literal[42]))
    True
-   >>> is_literal(get_origin(typing_extensions.Final[42]))
+   >>> is_literal(get_origin(typing_extensions.Final[int]))
    False
 
 Python version support

pyproject.toml

@@ -1,6 +1,6 @@
 # Build system requirements.
 [build-system]
-requires = ["flit_core >=3.4,<4"]
+requires = ["flit_core >=3.11,<4"]
 build-backend = "flit_core.buildapi"
 
 # Project metadata
@@ -10,7 +10,8 @@ version = "4.12.2"
 description = "Backported and Experimental Type Hints for Python 3.8+"
 readme = "README.md"
 requires-python = ">=3.8"
-license = { text = "PSF-2.0" }
+license = "PSF-2.0"
+license-files = ["LICENSE"]
 keywords = [
     "annotations",
     "backport",
@@ -30,7 +31,6 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Environment :: Console",
     "Intended Audience :: Developers",
-    "License :: OSI Approved :: Python Software Foundation License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
@@ -93,6 +93,8 @@ ignore = [
     "UP038",
     # Not relevant here
     "RUF012",
+    "RUF022",
+    "RUF023",
 ]
 
 [tool.ruff.lint.per-file-ignores]

src/test_typing_extensions.py

@@ -1650,6 +1650,95 @@ def test_final_forward_ref(self):
         self.assertNotEqual(gth(Loop, globals())['attr'], Final[int])
         self.assertNotEqual(gth(Loop, globals())['attr'], Final)
 
+    def test_annotation_and_optional_default(self):
+        annotation = Annotated[Union[int, None], "data"]
+        NoneAlias = None
+        StrAlias = str
+        T_default = TypeVar("T_default", default=None)
+        Ts = TypeVarTuple("Ts")
+
+        cases = {
+            # annotation: expected_type_hints
+            Annotated[None, "none"] : Annotated[None, "none"],
+            annotation              : annotation,
+            Optional[int]           : Optional[int],
+            Optional[List[str]]     : Optional[List[str]],
+            Optional[annotation]     : Optional[annotation],
+            Union[str, None, str]   : Optional[str],
+            Unpack[Tuple[int, None]]: Unpack[Tuple[int, None]],
+            # Note: A starred *Ts will use typing.Unpack in 3.11+ see Issue #485
+            Unpack[Ts]              : Unpack[Ts],
+        }
+        # contains a ForwardRef, TypeVar(~prefix) or no expression
+        do_not_stringify_cases = {
+            ()          : {},  # Special-cased below to create an unannotated parameter
+            int         : int,
+            "int"       : int,
+            None        : type(None),
+            "NoneAlias" : type(None),
+            List["str"] : List[str],
+            Union[str, "str"]                  : str,
+            Union[str, None, "str"]            : Optional[str],
+            Union[str, "NoneAlias", "StrAlias"]: Optional[str],
+            Union[str, "Union[None, StrAlias]"]: Optional[str],
+            Union["annotation", T_default]     : Union[annotation, T_default],
+            Annotated["annotation", "nested"]  : Annotated[Union[int, None], "data", "nested"],
+        }
+        if TYPING_3_10_0:  # cannot construct UnionTypes before 3.10
+            do_not_stringify_cases["str | NoneAlias | StrAlias"] = str | None
+            cases[str | None] = Optional[str]
+        cases.update(do_not_stringify_cases)
+        for (annot, expected), none_default, as_str, wrap_optional in itertools.product(
+            cases.items(), (False, True), (False, True), (False, True)
+        ):
+            # Special case:
+            skip_reason = None
+            annot_unchanged = annot
+            if sys.version_info[:2] == (3, 10) and annot == "str | NoneAlias | StrAlias" and none_default:
+                # In 3.10 converts Optional[str | None] to Optional[str] which has a different repr
+                skip_reason = "UnionType not preserved in 3.10"
+            if wrap_optional:
+                if annot_unchanged == ():
+                    continue
+                annot = Optional[annot]
+                expected = {"x": Optional[expected]}
+            else:
+                expected = {"x": expected} if annot_unchanged != () else {}
+            if as_str:
+                if annot_unchanged in do_not_stringify_cases or annot_unchanged == ():
+                    continue
+                annot = str(annot)
+            with self.subTest(
+                annotation=annot,
+                as_str=as_str,
+                wrap_optional=wrap_optional,
+                none_default=none_default,
+                expected_type_hints=expected,
+            ):
+                # Create function to check
+                if annot_unchanged == ():
+                    if none_default:
+                        def func(x=None): pass
+                    else:
+                        def func(x): pass
+                elif none_default:
+                    def func(x: annot = None): pass
+                else:
+                    def func(x: annot): pass
+                type_hints = get_type_hints(func, globals(), locals(), include_extras=True)
+                # Equality
+                self.assertEqual(type_hints, expected)
+                # Hash
+                for k in type_hints.keys():
+                    self.assertEqual(hash(type_hints[k]), hash(expected[k]))
+                    # Test if UnionTypes are preserved
+                    self.assertIs(type(type_hints[k]), type(expected[k]))
+                # Repr
+                with self.subTest("Check str and repr"):
+                    if skip_reason == "UnionType not preserved in 3.10":
+                        self.skipTest(skip_reason)
+                    self.assertEqual(repr(type_hints), repr(expected))
+
 
 class GetUtilitiesTestCase(TestCase):
     def test_get_origin(self):
@@ -4158,6 +4247,37 @@ def test_total(self):
         self.assertEqual(Options.__required_keys__, frozenset())
         self.assertEqual(Options.__optional_keys__, {'log_level', 'log_path'})
 
+    def test_total_inherits_non_total(self):
+        class TD1(TypedDict, total=False):
+            a: int
+
+        self.assertIs(TD1.__total__, False)
+
+        class TD2(TD1):
+            b: str
+
+        self.assertIs(TD2.__total__, True)
+
+    def test_total_with_assigned_value(self):
+        class TD(TypedDict):
+            __total__ = "some_value"
+
+        self.assertIs(TD.__total__, True)
+
+        class TD2(TypedDict, total=True):
+            __total__ = "some_value"
+
+        self.assertIs(TD2.__total__, True)
+
+        class TD3(TypedDict, total=False):
+            __total__ = "some value"
+
+        self.assertIs(TD3.__total__, False)
+
+        TD4 = TypedDict('TD4', {'__total__': "some_value"})  # noqa: F821
+        self.assertIs(TD4.__total__, True)
+
+
     def test_optional_keys(self):
         class Point2Dor3D(Point2D, total=False):
             z: int
@@ -6156,6 +6276,12 @@ def test_equivalent_nested_variadics(self):
         self.assertEqual(nested_tuple_bare, TupleAliasTsT[Unpack[Tuple[str, int]], object])
         self.assertEqual(nested_tuple_bare, TupleAliasTsT[Unpack[Tuple[str]], Unpack[Tuple[int]], object])
 
+    @skipUnless(TYPING_3_11_0, "Needed for backport")
+    def test_type_var_inheritance(self):
+        Ts = TypeVarTuple("Ts")
+        self.assertFalse(isinstance(Unpack[Ts], TypeVar))
+        self.assertFalse(isinstance(Unpack[Ts], typing.TypeVar))
+
 
 class TypeVarTupleTests(BaseTestCase):
 

src/typing_extensions.py

@@ -1313,10 +1313,90 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
             )
         else:  # 3.8
             hint = typing.get_type_hints(obj, globalns=globalns, localns=localns)
+        if sys.version_info < (3, 11):
+            _clean_optional(obj, hint, globalns, localns)
+        if sys.version_info < (3, 9):
+            # In 3.8 eval_type does not flatten Optional[ForwardRef] correctly
+            # This will recreate and and cache Unions.
+            hint = {
+                k: (t
+                    if get_origin(t) != Union
+                    else Union[t.__args__])
+                for k, t in hint.items()
+            }
         if include_extras:
             return hint
         return {k: _strip_extras(t) for k, t in hint.items()}
 
+    _NoneType = type(None)
+
+    def _could_be_inserted_optional(t):
+        """detects Union[..., None] pattern"""
+        # 3.8+ compatible checking before _UnionGenericAlias
+        if get_origin(t) is not Union:
+            return False
+        # Assume if last argument is not None they are user defined
+        if t.__args__[-1] is not _NoneType:
+            return False
+        return True
+
+    # < 3.11
+    def _clean_optional(obj, hints, globalns=None, localns=None):
+        # reverts injected Union[..., None] cases from typing.get_type_hints
+        # when a None default value is used.
+        # see https://github.com/python/typing_extensions/issues/310
+        if not hints or isinstance(obj, type):
+            return
+        defaults = typing._get_defaults(obj)  # avoid accessing __annotations___
+        if not defaults:
+            return
+        original_hints = obj.__annotations__
+        for name, value in hints.items():
+            # Not a Union[..., None] or replacement conditions not fullfilled
+            if (not _could_be_inserted_optional(value)
+                or name not in defaults
+                or defaults[name] is not None
+            ):
+                continue
+            original_value = original_hints[name]
+            # value=NoneType should have caused a skip above but check for safety
+            if original_value is None:
+                original_value = _NoneType
+            # Forward reference
+            if isinstance(original_value, str):
+                if globalns is None:
+                    if isinstance(obj, _types.ModuleType):
+                        globalns = obj.__dict__
+                    else:
+                        nsobj = obj
+                        # Find globalns for the unwrapped object.
+                        while hasattr(nsobj, '__wrapped__'):
+                            nsobj = nsobj.__wrapped__
+                        globalns = getattr(nsobj, '__globals__', {})
+                    if localns is None:
+                        localns = globalns
+                elif localns is None:
+                    localns = globalns
+                if sys.version_info < (3, 9):
+                    original_value = ForwardRef(original_value)
+                else:
+                    original_value = ForwardRef(
+                        original_value,
+                        is_argument=not isinstance(obj, _types.ModuleType)
+                    )
+            original_evaluated = typing._eval_type(original_value, globalns, localns)
+            if sys.version_info < (3, 9) and get_origin(original_evaluated) is Union:
+                # Union[str, None, "str"] is not reduced to Union[str, None]
+                original_evaluated = Union[original_evaluated.__args__]
+            # Compare if values differ. Note that even if equal
+            # value might be cached by typing._tp_cache contrary to original_evaluated
+            if original_evaluated != value or (
+                # 3.10: ForwardRefs of UnionType might be turned into _UnionGenericAlias
+                hasattr(_types, "UnionType")
+                and isinstance(original_evaluated, _types.UnionType)
+                and not isinstance(value, _types.UnionType)
+            ):
+                hints[name] = original_evaluated
 
 # Python 3.9+ has PEP 593 (Annotated)
 if hasattr(typing, 'Annotated'):
@@ -2621,7 +2701,9 @@ def __init__(self, getitem):
             self.__doc__ = _UNPACK_DOC
 
     class _UnpackAlias(typing._GenericAlias, _root=True):
-        __class__ = typing.TypeVar
+        if sys.version_info < (3, 11):
+            # needed for compatibility with Generic[Unpack[Ts]]
+            __class__ = typing.TypeVar
 
         @property
         def __typing_unpacked_tuple_args__(self):

test-requirements.txt

@@ -1 +1 @@
-ruff==0.4.5
+ruff==0.9.6