Fix Union[..., NoneType] injection by get_type_hints if a None default value is used. (python#482)

Co-authored-by: Jelle Zijlstra <[email protected]>

Author: Daraan

CHANGELOG.md

@@ -21,6 +21,10 @@ aliases that have a `Concatenate` special form as their argument.
   Patch by [Daraan](https://github.com/Daraan).
 - Extended the `Concatenate` backport for Python 3.8-3.10 to now accept
   `Ellipsis` as an argument. Patch by [Daraan](https://github.com/Daraan).
+- Fix backport of `get_type_hints` to reflect Python 3.11+ behavior which does not add
+  `Union[..., NoneType]` to annotations that have a `None` default value anymore.
+  This fixes wrapping of `Annotated` in an unwanted `Optional` in such cases.
+  Patch by [Daraan](https://github.com/Daraan).
 - Fix error in subscription of `Unpack` aliases causing nested Unpacks 
   to not be resolved correctly. Patch by [Daraan](https://github.com/Daraan).
 - Backport CPython PR [#124795](https://github.com/python/cpython/pull/124795):

src/test_typing_extensions.py

@@ -1647,6 +1647,95 @@ def test_final_forward_ref(self):
         self.assertNotEqual(gth(Loop, globals())['attr'], Final[int])
         self.assertNotEqual(gth(Loop, globals())['attr'], Final)
 
+    def test_annotation_and_optional_default(self):
+        annotation = Annotated[Union[int, None], "data"]
+        NoneAlias = None
+        StrAlias = str
+        T_default = TypeVar("T_default", default=None)
+        Ts = TypeVarTuple("Ts")
+
+        cases = {
+            # annotation: expected_type_hints
+            Annotated[None, "none"] : Annotated[None, "none"],
+            annotation              : annotation,
+            Optional[int]           : Optional[int],
+            Optional[List[str]]     : Optional[List[str]],
+            Optional[annotation]     : Optional[annotation],
+            Union[str, None, str]   : Optional[str],
+            Unpack[Tuple[int, None]]: Unpack[Tuple[int, None]],
+            # Note: A starred *Ts will use typing.Unpack in 3.11+ see Issue #485
+            Unpack[Ts]              : Unpack[Ts],
+        }
+        # contains a ForwardRef, TypeVar(~prefix) or no expression
+        do_not_stringify_cases = {
+            ()          : {},  # Special-cased below to create an unannotated parameter
+            int         : int,
+            "int"       : int,
+            None        : type(None),
+            "NoneAlias" : type(None),
+            List["str"] : List[str],
+            Union[str, "str"]                  : str,
+            Union[str, None, "str"]            : Optional[str],
+            Union[str, "NoneAlias", "StrAlias"]: Optional[str],
+            Union[str, "Union[None, StrAlias]"]: Optional[str],
+            Union["annotation", T_default]     : Union[annotation, T_default],
+            Annotated["annotation", "nested"]  : Annotated[Union[int, None], "data", "nested"],
+        }
+        if TYPING_3_10_0:  # cannot construct UnionTypes before 3.10
+            do_not_stringify_cases["str | NoneAlias | StrAlias"] = str | None
+            cases[str | None] = Optional[str]
+        cases.update(do_not_stringify_cases)
+        for (annot, expected), none_default, as_str, wrap_optional in itertools.product(
+            cases.items(), (False, True), (False, True), (False, True)
+        ):
+            # Special case:
+            skip_reason = None
+            annot_unchanged = annot
+            if sys.version_info[:2] == (3, 10) and annot == "str | NoneAlias | StrAlias" and none_default:
+                # In 3.10 converts Optional[str | None] to Optional[str] which has a different repr
+                skip_reason = "UnionType not preserved in 3.10"
+            if wrap_optional:
+                if annot_unchanged == ():
+                    continue
+                annot = Optional[annot]
+                expected = {"x": Optional[expected]}
+            else:
+                expected = {"x": expected} if annot_unchanged != () else {}
+            if as_str:
+                if annot_unchanged in do_not_stringify_cases or annot_unchanged == ():
+                    continue
+                annot = str(annot)
+            with self.subTest(
+                annotation=annot,
+                as_str=as_str,
+                wrap_optional=wrap_optional,
+                none_default=none_default,
+                expected_type_hints=expected,
+            ):
+                # Create function to check
+                if annot_unchanged == ():
+                    if none_default:
+                        def func(x=None): pass
+                    else:
+                        def func(x): pass
+                elif none_default:
+                    def func(x: annot = None): pass
+                else:
+                    def func(x: annot): pass
+                type_hints = get_type_hints(func, globals(), locals(), include_extras=True)
+                # Equality
+                self.assertEqual(type_hints, expected)
+                # Hash
+                for k in type_hints.keys():
+                    self.assertEqual(hash(type_hints[k]), hash(expected[k]))
+                    # Test if UnionTypes are preserved
+                    self.assertIs(type(type_hints[k]), type(expected[k]))
+                # Repr
+                with self.subTest("Check str and repr"):
+                    if skip_reason == "UnionType not preserved in 3.10":
+                        self.skipTest(skip_reason)
+                    self.assertEqual(repr(type_hints), repr(expected))
+
 
 class GetUtilitiesTestCase(TestCase):
     def test_get_origin(self):

src/typing_extensions.py

@@ -1242,10 +1242,90 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
             )
         else:  # 3.8
             hint = typing.get_type_hints(obj, globalns=globalns, localns=localns)
+        if sys.version_info < (3, 11):
+            _clean_optional(obj, hint, globalns, localns)
+        if sys.version_info < (3, 9):
+            # In 3.8 eval_type does not flatten Optional[ForwardRef] correctly
+            # This will recreate and and cache Unions.
+            hint = {
+                k: (t
+                    if get_origin(t) != Union
+                    else Union[t.__args__])
+                for k, t in hint.items()
+            }
         if include_extras:
             return hint
         return {k: _strip_extras(t) for k, t in hint.items()}
 
+    _NoneType = type(None)
+
+    def _could_be_inserted_optional(t):
+        """detects Union[..., None] pattern"""
+        # 3.8+ compatible checking before _UnionGenericAlias
+        if get_origin(t) is not Union:
+            return False
+        # Assume if last argument is not None they are user defined
+        if t.__args__[-1] is not _NoneType:
+            return False
+        return True
+
+    # < 3.11
+    def _clean_optional(obj, hints, globalns=None, localns=None):
+        # reverts injected Union[..., None] cases from typing.get_type_hints
+        # when a None default value is used.
+        # see https://github.com/python/typing_extensions/issues/310
+        if not hints or isinstance(obj, type):
+            return
+        defaults = typing._get_defaults(obj)  # avoid accessing __annotations___
+        if not defaults:
+            return
+        original_hints = obj.__annotations__
+        for name, value in hints.items():
+            # Not a Union[..., None] or replacement conditions not fullfilled
+            if (not _could_be_inserted_optional(value)
+                or name not in defaults
+                or defaults[name] is not None
+            ):
+                continue
+            original_value = original_hints[name]
+            # value=NoneType should have caused a skip above but check for safety
+            if original_value is None:
+                original_value = _NoneType
+            # Forward reference
+            if isinstance(original_value, str):
+                if globalns is None:
+                    if isinstance(obj, _types.ModuleType):
+                        globalns = obj.__dict__
+                    else:
+                        nsobj = obj
+                        # Find globalns for the unwrapped object.
+                        while hasattr(nsobj, '__wrapped__'):
+                            nsobj = nsobj.__wrapped__
+                        globalns = getattr(nsobj, '__globals__', {})
+                    if localns is None:
+                        localns = globalns
+                elif localns is None:
+                    localns = globalns
+                if sys.version_info < (3, 9):
+                    original_value = ForwardRef(original_value)
+                else:
+                    original_value = ForwardRef(
+                        original_value,
+                        is_argument=not isinstance(obj, _types.ModuleType)
+                    )
+            original_evaluated = typing._eval_type(original_value, globalns, localns)
+            if sys.version_info < (3, 9) and get_origin(original_evaluated) is Union:
+                # Union[str, None, "str"] is not reduced to Union[str, None]
+                original_evaluated = Union[original_evaluated.__args__]
+            # Compare if values differ. Note that even if equal
+            # value might be cached by typing._tp_cache contrary to original_evaluated
+            if original_evaluated != value or (
+                # 3.10: ForwardRefs of UnionType might be turned into _UnionGenericAlias
+                hasattr(_types, "UnionType")
+                and isinstance(original_evaluated, _types.UnionType)
+                and not isinstance(value, _types.UnionType)
+            ):
+                hints[name] = original_evaluated
 
 # Python 3.9+ has PEP 593 (Annotated)
 if hasattr(typing, 'Annotated'):