From 27318441ed6f0b810ebcc512e0f842727f73a39c Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 14 Mar 2023 09:01:02 +0300 Subject: [PATCH 1/2] gh-102615: Fix type vars substitution of `collections.abc.Callable` and custom generics with `ParamSpec` --- Lib/_collections_abc.py | 11 ++ Lib/test/test_typing.py | 71 +++++++++- Lib/typing.py | 121 +++++++++++------- ...-03-14-10-11-46.gh-issue-102615.hgTYdd.rst | 2 + 4 files changed, 155 insertions(+), 50 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-14-10-11-46.gh-issue-102615.hgTYdd.rst diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c62233b81a5c95..9b4f2c7fa7ac33 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -498,6 +498,17 @@ def __getitem__(self, item): t_result = new_args[-1] t_args = new_args[:-1] new_args = (t_args, t_result) + + # This happens in cases like `Callable[P, T][[P, str], bool][int]`, + # we need to flatten the result. + if len(new_args) > 2: + res = [] + for new_arg in new_args: + if isinstance(new_arg, tuple): + res.extend(new_arg) + else: + res.append(new_arg) + new_args = (res[:-1], res[-1]) return _CallableGenericAlias(Callable, tuple(new_args)) def _is_param_expr(obj): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c17be6cd0bbc4a..69c06961ea9739 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -848,7 +848,6 @@ class C(Generic[*Ts]): pass ) - class UnpackTests(BaseTestCase): def test_accepts_single_type(self): @@ -1997,6 +1996,16 @@ def test_paramspec(self): self.assertEqual(repr(C2), f"{fullname}[~P, int]") self.assertEqual(repr(C2[int, str]), f"{fullname}[[int, str], int]") + # gh-102615: + C3 = C1[[P, str], bool] + self.assertEqual(C3.__parameters__, (P,)) + self.assertEqual(C3.__args__, (P, str, bool)) + + self.assertEqual(C3[int].__args__, (int, str, bool)) + self.assertEqual(C3[[int, complex]].__args__, (int, complex, str, bool)) + self.assertEqual(C3[int, complex].__args__, (int, complex, str, bool)) + self.assertEqual(C3[[]].__args__, (str, bool)) + def test_concatenate(self): Callable = self.Callable fullname = f"{Callable.__module__}.Callable" @@ -7516,16 +7525,18 @@ class Z(Generic[P]): def test_multiple_paramspecs_in_user_generics(self): P = ParamSpec("P") P2 = ParamSpec("P2") + T = TypeVar("T") - class X(Generic[P, P2]): + class X(Generic[P, P2, T]): f: Callable[P, int] g: Callable[P2, str] + t: T - G1 = X[[int, str], [bytes]] - G2 = X[[int], [str, bytes]] + G1 = X[[int, str], [bytes], bool] + G2 = X[[int], [str, bytes], bool] self.assertNotEqual(G1, G2) - self.assertEqual(G1.__args__, ((int, str), (bytes,))) - self.assertEqual(G2.__args__, ((int,), (str, bytes))) + self.assertEqual(G1.__args__, ((int, str), (bytes,), bool)) + self.assertEqual(G2.__args__, ((int,), (str, bytes), bool)) def test_typevartuple_and_paramspecs_in_user_generics(self): Ts = TypeVarTuple("Ts") @@ -7561,6 +7572,54 @@ class Y(Generic[P, *Ts]): with self.assertRaises(TypeError): Y[()] + def test_paramspec_subst(self): + # See: https://github.com/python/cpython/issues/102615 + P = ParamSpec("P") + T = TypeVar("T") + + class MyCallable(Generic[P, T]): + pass + + G = MyCallable[P, T] + self.assertEqual(G.__parameters__, (P, T)) + self.assertEqual(G.__args__, (P, T)) + + C = G[[P, str], bool] + self.assertEqual(C.__parameters__, (P,)) + self.assertEqual(C.__args__, ((P, str), bool)) + + self.assertEqual(C[int].__parameters__, ()) + self.assertEqual(C[int].__args__, ((int, str), bool)) + self.assertEqual(C[[int, complex]].__args__, ((int, complex, str), bool)) + self.assertEqual(C[[]].__args__, ((str,), bool)) + + Q = G[[int, str], T] + self.assertEqual(Q.__parameters__, (T,)) + self.assertEqual(Q[bool].__parameters__, ()) + self.assertEqual(Q[bool].__args__, ((int, str), bool)) + + # Reversed order: + class MyCallable2(Generic[T, P]): + pass + + G2 = MyCallable[T, P] + self.assertEqual(G2.__parameters__, (T, P)) + self.assertEqual(G2.__args__, (T, P)) + + C2 = G2[bool, [P, str]] + self.assertEqual(C2.__parameters__, (P,)) + self.assertEqual(C2.__args__, (bool, (P, str))) + + self.assertEqual(C2[int].__parameters__, ()) + self.assertEqual(C2[int].__args__, (bool, (int, str))) + self.assertEqual(C2[[int, complex]].__args__, (bool, (int, complex, str))) + self.assertEqual(C2[[]].__args__, (bool, (str,))) + + Q2 = G2[T, [int, str]] + self.assertEqual(Q2.__parameters__, (T,)) + self.assertEqual(Q2[bool].__parameters__, ()) + self.assertEqual(Q2[bool].__args__, (bool, (int, str))) + def test_typevartuple_and_paramspecs_in_generic_aliases(self): P = ParamSpec('P') T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index 8d40e923bb1d08..2fdb2d784dabf6 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -254,6 +254,8 @@ def _collect_parameters(args): # We don't want __parameters__ descriptor of a bare Python class. if isinstance(t, type): continue + if isinstance(t, tuple): + parameters.extend(_collect_parameters(t)) if hasattr(t, '__typing_subst__'): if t not in parameters: parameters.append(t) @@ -1440,54 +1442,85 @@ def _determine_new_args(self, args): new_args = [] for old_arg in self.__args__: + if isinstance(old_arg, tuple): + self._substitute_tuple_args(old_arg, new_args, new_arg_by_param) + else: + self._substitute_arg(old_arg, new_args, new_arg_by_param) + return tuple(new_args) + + def _substitute_tuple_args(self, old_arg, new_args, new_arg_by_param): + # This method required to make this case correct: + # + # P = ParamSpec("P") + # T = TypeVar("T") + # class MyCallable(Generic[P, T]): ... + # + # MyCallable[P, T][[P, str], bool][int] + # + # Which must be equal to: + # MyCallable[[int, str], bool] + sub_args = [] + for sub_old_arg in old_arg: + if _is_param_expr(sub_old_arg): + self._substitute_arg(sub_old_arg, sub_args, new_arg_by_param) + else: + sub_args.append(sub_old_arg) - if isinstance(old_arg, type): - new_args.append(old_arg) + # Now, unflatten the result: + res = [] + for sub_arg in sub_args: + if isinstance(sub_arg, tuple): + res.extend(sub_arg) continue + res.append(sub_arg) + new_args.append(tuple(res)) - substfunc = getattr(old_arg, '__typing_subst__', None) - if substfunc: - new_arg = substfunc(new_arg_by_param[old_arg]) - else: - subparams = getattr(old_arg, '__parameters__', ()) - if not subparams: - new_arg = old_arg - else: - subargs = [] - for x in subparams: - if isinstance(x, TypeVarTuple): - subargs.extend(new_arg_by_param[x]) - else: - subargs.append(new_arg_by_param[x]) - new_arg = old_arg[tuple(subargs)] - - if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple): - # Consider the following `Callable`. - # C = Callable[[int], str] - # Here, `C.__args__` should be (int, str) - NOT ([int], str). - # That means that if we had something like... - # P = ParamSpec('P') - # T = TypeVar('T') - # C = Callable[P, T] - # D = C[[int, str], float] - # ...we need to be careful; `new_args` should end up as - # `(int, str, float)` rather than `([int, str], float)`. - new_args.extend(new_arg) - elif _is_unpacked_typevartuple(old_arg): - # Consider the following `_GenericAlias`, `B`: - # class A(Generic[*Ts]): ... - # B = A[T, *Ts] - # If we then do: - # B[float, int, str] - # The `new_arg` corresponding to `T` will be `float`, and the - # `new_arg` corresponding to `*Ts` will be `(int, str)`. We - # should join all these types together in a flat list - # `(float, int, str)` - so again, we should `extend`. - new_args.extend(new_arg) - else: - new_args.append(new_arg) + def _substitute_arg(self, old_arg, new_args, new_arg_by_param): + if isinstance(old_arg, type): + new_args.append(old_arg) + return - return tuple(new_args) + substfunc = getattr(old_arg, '__typing_subst__', None) + if substfunc: + new_arg = substfunc(new_arg_by_param[old_arg]) + else: + subparams = getattr(old_arg, '__parameters__', ()) + if not subparams: + new_arg = old_arg + else: + subargs = [] + for x in subparams: + if isinstance(x, TypeVarTuple): + subargs.extend(new_arg_by_param[x]) + else: + subargs.append(new_arg_by_param[x]) + new_arg = old_arg[tuple(subargs)] + + if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple): + # Consider the following `Callable`. + # C = Callable[[int], str] + # Here, `C.__args__` should be (int, str) - NOT ([int], str). + # That means that if we had something like... + # P = ParamSpec('P') + # T = TypeVar('T') + # C = Callable[P, T] + # D = C[[int, str], float] + # ...we need to be careful; `new_args` should end up as + # `(int, str, float)` rather than `([int, str], float)`. + new_args.extend(new_arg) + elif _is_unpacked_typevartuple(old_arg): + # Consider the following `_GenericAlias`, `B`: + # class A(Generic[*Ts]): ... + # B = A[T, *Ts] + # If we then do: + # B[float, int, str] + # The `new_arg` corresponding to `T` will be `float`, and the + # `new_arg` corresponding to `*Ts` will be `(int, str)`. We + # should join all these types together in a flat list + # `(float, int, str)` - so again, we should `extend`. + new_args.extend(new_arg) + else: + new_args.append(new_arg) def copy_with(self, args): return self.__class__(self.__origin__, args, name=self._name, inst=self._inst, diff --git a/Misc/NEWS.d/next/Library/2023-03-14-10-11-46.gh-issue-102615.hgTYdd.rst b/Misc/NEWS.d/next/Library/2023-03-14-10-11-46.gh-issue-102615.hgTYdd.rst new file mode 100644 index 00000000000000..1a6f8c5664069c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-14-10-11-46.gh-issue-102615.hgTYdd.rst @@ -0,0 +1,2 @@ +Fix type variables substitution of :class:`collections.abc.Callable` and +custom generics with ``ParamSpec``. From a461b10a5977101c76c665f2be1a9a819f9f6cd7 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 14 Mar 2023 16:00:52 +0300 Subject: [PATCH 2/2] Address review --- Lib/_collections_abc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 9b4f2c7fa7ac33..ecf536d192d5c0 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -501,7 +501,9 @@ def __getitem__(self, item): # This happens in cases like `Callable[P, T][[P, str], bool][int]`, # we need to flatten the result. - if len(new_args) > 2: + if (len(new_args) > 2 + and self.__parameters__ + and _is_param_expr(self.__parameters__[0])): res = [] for new_arg in new_args: if isinstance(new_arg, tuple):