Merge branch 'main' into ParamSpec/Generic-simple

Author: Daraan

CHANGELOG.md

@@ -29,6 +29,8 @@ aliases that have a `Concatenate` special form as their argument.
 - Fix that lists and ... could not be used for parameter expressions for `TypeAliasType`
   instances before Python 3.11.
   Patch by [Daraan](https://github.com/Daraan).
+- Fix error on Python 3.10 when using `typing.Concatenate` and 
+  `typing_extensions.Concatenate` together. Patch by [Daraan](https://github.com/Daraan).
 
 # Release 4.12.2 (June 7, 2024)
 

src/test_typing_extensions.py

@@ -5307,6 +5307,7 @@ class ProtoZ(Protocol[P]):
                 G6 = klass[int, str, T]
                 G6args = G6.__args__[0] if sys.version_info >= (3, 10) else G6.__args__
                 self.assertEqual(G6args, (int, str, T))
+                self.assertEqual(G6.__parameters__, (T,))
 
                 # P = [int]
                 G7 = klass[int]
@@ -5350,54 +5351,54 @@ class ProtoZ(Protocol[P]):
         things = "arguments" if sys.version_info >= (3, 10) else "parameters"
         for klass in Z, ProtoZ:
             with self.subTest(klass=klass.__name__):
-                G6 = klass[int, str, T]
                 G8 = klass[Concatenate[T, ...]]
+
+                H8_1 = G8[int]
+                self.assertEqual(H8_1.__parameters__, ())
+                with self.assertRaisesRegex(TypeError, "not a generic class"):
+                    H8_1[str]
+
+                H8_2 = G8[T][int]
+                self.assertEqual(H8_2.__parameters__, ())
+                with self.assertRaisesRegex(TypeError, "not a generic class"):
+                    H8_2[str]
+
                 G9 = klass[Concatenate[T, P_2]]
-                G10 = klass[int, Concatenate[str, P]]
+                self.assertEqual(G9.__parameters__, (T, P_2))
 
-                with self.subTest("Check generic substitution", klass=klass.__name__):
-                    if sys.version_info < (3, 10):
-                        self.skipTest("_ConcatenateGenericAlias not subscriptable")
-                    with self.assertRaisesRegex(TypeError, "Expected a list of types, an ellipsis, ParamSpec, or Concatenate"):
-                        G9[int, int]
+                with self.assertRaisesRegex(TypeError,
+                    "The last parameter to Concatenate should be a ParamSpec variable or ellipsis."
+                    if sys.version_info < (3, 10) else
+                    # from __typing_subst__
+                    "Expected a list of types, an ellipsis, ParamSpec, or Concatenate"
+                ):
+                    G9[int, int]
+
+                with self.assertRaisesRegex(TypeError, f"Too few {things}"):
+                    G9[int]
 
                 with self.subTest("Check list as parameter expression", klass=klass.__name__):
                     if sys.version_info < (3, 10):
                         self.skipTest("Cannot pass non-types")
                     G5 = klass[[int, str, T]]
+                    self.assertEqual(G5.__parameters__, (T,))
                     self.assertEqual(G5.__args__, ((int, str, T),))
+
                     H9 = G9[int, [T]]
+                    self.assertEqual(H9.__parameters__, (T,))
 
-                self.assertEqual(G9.__parameters__, (T, P_2))
-                with self.subTest("Check parametrization", klass=klass.__name__):
-                    if sys.version_info[:2] == (3, 10):
-                        self.skipTest("Parameter detection fails in 3.10")
-                    with self.assertRaisesRegex(TypeError, f"Too few {things}"):
-                        G9[int]  # for python 3.10 this has no parameters
-                    self.assertEqual(G6.__parameters__, (T,))
-                    if sys.version_info >= (3, 10):  # skipped above
-                        self.assertEqual(G5.__parameters__, (T,))
-                        self.assertEqual(H9.__parameters__, (T,))
-
-                with self.subTest("Check further substitution", klass=klass.__name__):
-                    if sys.version_info < (3, 10):
-                        self.skipTest("_ConcatenateGenericAlias not subscriptable")
-                    if sys.version_info[:2] == (3, 10):
-                        self.skipTest("Parameter detection fails in 3.10")
-                    if (3, 11, 0) <= sys.version_info[:3] < (3, 11, 3):
-                        self.skipTest("Wrong recursive substitution")
-                    H1 = G8[int]
-                    self.assertEqual(H1.__parameters__, ())
-                    with self.assertRaisesRegex(TypeError, "not a generic class"):
-                        H1[str]  # for python 3.11.0-3 this still has a parameter
-
-                    H2 = G8[T][int]
-                    self.assertEqual(H2.__parameters__, ())
-                    with self.assertRaisesRegex(TypeError, "not a generic class"):
-                        H2[str]  # for python 3.11.0-3 this still has a parameter
-
-                    H3 = G10[int]
-                    self.assertEqual(H3.__args__, ((int, (str, int)),))
+                # This is an invalid parameter expression but useful for testing correct subsitution
+                G10 = klass[int, Concatenate[str, P]]
+                with self.subTest("Check invalid form substitution"):
+                    self.assertEqual(G10.__parameters__, (P, ))
+                    if sys.version_info < (3, 9):
+                        self.skipTest("3.8 typing._type_subst does not support this substitution process")
+                    H10 = G10[int]
+                    if (3, 10) <= sys.version_info < (3, 11, 3):
+                        self.skipTest("3.10-3.11.2 does not substitute Concatenate here")
+                    self.assertEqual(H10.__parameters__, ())
+                    H10args = H10.__args__[0] if sys.version_info >= (3, 10) else H10.__args__
+                    self.assertEqual(H10args, (int, (str, int)))
 
     @skipUnless(TYPING_3_10_0, "ParamSpec not present before 3.10")
     def test_substitution_with_typing_variants(self):
@@ -5503,6 +5504,19 @@ class MyClass: ...
         self.assertNotEqual(d, c)
         self.assertNotEqual(d, Concatenate)
 
+    @skipUnless(TYPING_3_10_0, "Concatenate not available in <3.10")
+    def test_typing_compatibility(self):
+        P = ParamSpec('P')
+        C1 = Concatenate[int, P][typing.Concatenate[int, P]]
+        self.assertEqual(C1, Concatenate[int, int, P])
+        self.assertEqual(get_args(C1), (int, int, P))
+
+        C2 = typing.Concatenate[int, P][Concatenate[int, P]]
+        with self.subTest("typing compatibility with typing_extensions"):
+            if sys.version_info < (3, 10, 3):
+                self.skipTest("Unpacking not introduced until 3.10.3")
+            self.assertEqual(get_args(C2), (int, int, P))
+
     def test_valid_uses(self):
         P = ParamSpec('P')
         T = TypeVar('T')
@@ -5598,6 +5612,32 @@ def test_eq(self):
         self.assertEqual(hash(C4), hash(C5))
         self.assertNotEqual(C4, C6)
 
+    def test_substitution(self):
+        T = TypeVar('T')
+        P = ParamSpec('P')
+        Ts = TypeVarTuple("Ts")
+
+        C1 = Concatenate[str, T, ...]
+        self.assertEqual(C1[int], Concatenate[str, int, ...])
+
+        C2 = Concatenate[str, P]
+        self.assertEqual(C2[...], Concatenate[str, ...])
+        self.assertEqual(C2[int], (str, int))
+        U1 = Unpack[Tuple[int, str]]
+        U2 = Unpack[Ts]
+        self.assertEqual(C2[U1], (str, int, str))
+        self.assertEqual(C2[U2], (str, Unpack[Ts]))
+        self.assertEqual(C2["U2"], (str, typing.ForwardRef("U2")))
+
+        if (3, 12, 0) <= sys.version_info < (3, 12, 4):
+            with self.assertRaises(AssertionError):
+                C2[Unpack[U2]]
+        else:
+            with self.assertRaisesRegex(TypeError, "must be used with a tuple type"):
+                C2[Unpack[U2]]
+
+        C3 = Concatenate[str, T, P]
+        self.assertEqual(C3[int, [bool]], (str, int, bool))
 
     @skipUnless(TYPING_3_10_0, "Concatenate not present before 3.10")
     def test_is_param_expr(self):

src/typing_extensions.py

@@ -1765,6 +1765,23 @@ def __call__(self, *args, **kwargs):
 # 3.8-3.9
 if not hasattr(typing, 'Concatenate'):
     # Inherits from list as a workaround for Callable checks in Python < 3.9.2.
+
+    # 3.9.0-1
+    if not hasattr(typing, '_type_convert'):
+        def _type_convert(arg, module=None, *, allow_special_forms=False):
+            """For converting None to type(None), and strings to ForwardRef."""
+            if arg is None:
+                return type(None)
+            if isinstance(arg, str):
+                if sys.version_info <= (3, 9, 6):
+                    return ForwardRef(arg)
+                if sys.version_info <= (3, 9, 7):
+                    return ForwardRef(arg, module=module)
+                return ForwardRef(arg, module=module, is_class=allow_special_forms)
+            return arg
+    else:
+        _type_convert = typing._type_convert
+
     class _ConcatenateGenericAlias(list):
 
         # Trick Generic into looking into this for __parameters__.
@@ -1795,27 +1812,122 @@ def __parameters__(self):
             return tuple(
                 tp for tp in self.__args__ if isinstance(tp, (typing.TypeVar, ParamSpec))
             )
+
+        # 3.8; needed for typing._subst_tvars
+        # 3.9 used by __getitem__ below
+        def copy_with(self, params):
+            if isinstance(params[-1], _ConcatenateGenericAlias):
+                params = (*params[:-1], *params[-1].__args__)
+            elif isinstance(params[-1], (list, tuple)):
+                return (*params[:-1], *params[-1])
+            elif (not (params[-1] is ... or isinstance(params[-1], ParamSpec))):
+                raise TypeError("The last parameter to Concatenate should be a "
+                        "ParamSpec variable or ellipsis.")
+            return self.__class__(self.__origin__, params)
+
+        # 3.9; accessed during GenericAlias.__getitem__ when substituting
+        def __getitem__(self, args):
+            if self.__origin__ in (Generic, Protocol):
+                # Can't subscript Generic[...] or Protocol[...].
+                raise TypeError(f"Cannot subscript already-subscripted {self}")
+            if not self.__parameters__:
+                raise TypeError(f"{self} is not a generic class")
+
+            if not isinstance(args, tuple):
+                args = (args,)
+            args = _unpack_args(*(_type_convert(p) for p in args))
+            params = self.__parameters__
+            for param in params:
+                prepare = getattr(param, "__typing_prepare_subst__", None)
+                if prepare is not None:
+                    args = prepare(self, args)
+                # 3.8 - 3.9 & typing.ParamSpec
+                elif isinstance(param, ParamSpec):
+                    i = params.index(param)
+                    if (
+                        i == len(args)
+                        and getattr(param, '__default__', NoDefault) is not NoDefault
+                    ):
+                        args = [*args, param.__default__]
+                    if i >= len(args):
+                        raise TypeError(f"Too few arguments for {self}")
+                    # Special case for Z[[int, str, bool]] == Z[int, str, bool]
+                    if len(params) == 1 and not _is_param_expr(args[0]):
+                        assert i == 0
+                        args = (args,)
+                    elif (
+                        isinstance(args[i], list)
+                        # 3.8 - 3.9
+                        # This class inherits from list do not convert
+                        and not isinstance(args[i], _ConcatenateGenericAlias)
+                    ):
+                        args = (*args[:i], tuple(args[i]), *args[i + 1:])
+
+            alen = len(args)
+            plen = len(params)
+            if alen != plen:
+                raise TypeError(
+                    f"Too {'many' if alen > plen else 'few'} arguments for {self};"
+                    f" actual {alen}, expected {plen}"
+                )
+
+            subst = dict(zip(self.__parameters__, args))
+            # determine new args
+            new_args = []
+            for arg in self.__args__:
+                if isinstance(arg, type):
+                    new_args.append(arg)
+                    continue
+                if isinstance(arg, TypeVar):
+                    arg = subst[arg]
+                    if (
+                        (isinstance(arg, typing._GenericAlias) and _is_unpack(arg))
+                        or (
+                            hasattr(_types, "GenericAlias")
+                            and isinstance(arg, _types.GenericAlias)
+                            and getattr(arg, "__unpacked__", False)
+                        )
+                    ):
+                        raise TypeError(f"{arg} is not valid as type argument")
+
+                elif isinstance(arg,
+                    typing._GenericAlias
+                    if not hasattr(_types, "GenericAlias") else
+                    (typing._GenericAlias, _types.GenericAlias)
+                ):
+                    subparams = arg.__parameters__
+                    if subparams:
+                        subargs = tuple(subst[x] for x in subparams)
+                        arg = arg[subargs]
+                new_args.append(arg)
+            return self.copy_with(tuple(new_args))
+
 # 3.10+
 else:
     _ConcatenateGenericAlias = typing._ConcatenateGenericAlias
 
     # 3.10
     if sys.version_info < (3, 11):
-        _typing_ConcatenateGenericAlias = _ConcatenateGenericAlias
 
-        class _ConcatenateGenericAlias(_typing_ConcatenateGenericAlias, _root=True):
+        class _ConcatenateGenericAlias(typing._ConcatenateGenericAlias, _root=True):
             # needed for checks in collections.abc.Callable to accept this class
             __module__ = "typing"
 
             def copy_with(self, params):
                 if isinstance(params[-1], (list, tuple)):
                     return (*params[:-1], *params[-1])
-                if isinstance(params[-1], _ConcatenateGenericAlias):
+                if isinstance(params[-1], typing._ConcatenateGenericAlias):
                     params = (*params[:-1], *params[-1].__args__)
                 elif not (params[-1] is ... or isinstance(params[-1], ParamSpec)):
                     raise TypeError("The last parameter to Concatenate should be a "
                             "ParamSpec variable or ellipsis.")
-                return super(_typing_ConcatenateGenericAlias, self).copy_with(params)
+                return super(typing._ConcatenateGenericAlias, self).copy_with(params)
+
+            def __getitem__(self, args):
+                value = super().__getitem__(args)
+                if isinstance(value, tuple) and any(_is_unpack(t) for t in value):
+                    return tuple(_unpack_args(*(n for n in value)))
+                return value
 
 
 # 3.8-3.9.2
@@ -2496,6 +2608,17 @@ def _is_unpack(obj):
     class _UnpackAlias(typing._GenericAlias, _root=True):
         __class__ = typing.TypeVar
 
+        @property
+        def __typing_unpacked_tuple_args__(self):
+            assert self.__origin__ is Unpack
+            assert len(self.__args__) == 1
+            arg, = self.__args__
+            if isinstance(arg, typing._GenericAlias):
+                if arg.__origin__ is not tuple:
+                    raise TypeError("Unpack[...] must be used with a tuple type")
+                return arg.__args__
+            return None
+
         @property
         def __typing_is_unpacked_typevartuple__(self):
             assert self.__origin__ is Unpack
@@ -2519,21 +2642,22 @@ def _is_unpack(obj):
         return isinstance(obj, _UnpackAlias)
 
 
+def _unpack_args(*args):
+    newargs = []
+    for arg in args:
+        subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
+        if subargs is not None and (not (subargs and subargs[-1] is ...)):
+            newargs.extend(subargs)
+        else:
+            newargs.append(arg)
+    return newargs
+
+
 if _PEP_696_IMPLEMENTED:
     from typing import TypeVarTuple
 
 elif hasattr(typing, "TypeVarTuple"):  # 3.11+
 
-    def _unpack_args(*args):
-        newargs = []
-        for arg in args:
-            subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
-            if subargs is not None and not (subargs and subargs[-1] is ...):
-                newargs.extend(subargs)
-            else:
-                newargs.append(arg)
-        return newargs
-
     # Add default parameter - PEP 696
     class TypeVarTuple(metaclass=_TypeVarLikeMeta):
         """Type variable tuple."""
@@ -3024,6 +3148,7 @@ def _is_param_expr(arg):
             ),
         )
 
+
 # We have to do some monkey patching to deal with the dual nature of
 # Unpack/TypeVarTuple:
 # - We want Unpack to be a kind of TypeVar so it gets accepted in
@@ -3199,6 +3324,13 @@ def _collect_type_vars(types, typevar_types=None):
                 tvars.append(t)
             if _should_collect_from_parameters(t):
                 tvars.extend([t for t in t.__parameters__ if t not in tvars])
+            elif isinstance(t, tuple):
+                # Collect nested type_vars
+                # tuple wrapped by  _prepare_paramspec_params(cls, params)
+                for x in t:
+                    for collected in _collect_type_vars([x]):
+                        if collected not in tvars:
+                            tvars.append(collected)
         return tuple(tvars)
 
     typing._collect_type_vars = _collect_type_vars