From 2c4a297703423a8ffc9680cc633c86ccfaae47b1 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 31 Oct 2020 17:20:23 +0800 Subject: [PATCH 01/30] Allow subclassing of GenericAlias, fix collections.abc.Callable's GenericAlias --- Lib/_collections_abc.py | 62 ++++++++++++++++++++++++++++++++++- Lib/test/test_genericalias.py | 19 +++++++++++ Objects/genericaliasobject.c | 29 ++++++++++++++-- 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 28690f8c0bdc5c..6a7cc1e01ee7d6 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -8,6 +8,7 @@ from abc import ABCMeta, abstractmethod import sys +import types GenericAlias = type(list[int]) @@ -409,6 +410,65 @@ def __subclasshook__(cls, C): return NotImplemented +class _CallableGenericAlias(GenericAlias): + """ Internal class specifically for consistency between the ``__args__`` of + ``collections.abc.Callable``'s and ``typing.Callable``'s ``GenericAlias``. + + See :issue:`42195`. + """ + def __new__(cls, *args, **kwargs): + if not isinstance(args, tuple) or len(args) != 2: + raise TypeError("Callable must be used as " + "Callable[[arg, ...], result].") + _typ, _args = args + if not isinstance(_args, tuple) or len(_args) != 2: + raise TypeError("Callable must be used as " + "Callable[[arg, ...], result].") + t_args, t_result = _args + if not isinstance(t_args, list): + raise TypeError("Callable must be used as " + "Callable[[arg, ...], result].") + + ga_args = [] + for arg in args[1]: + if isinstance(arg, list): + ga_args.extend(arg) + else: + ga_args.append(arg) + return super().__new__(cls, _typ, tuple(ga_args)) + + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + t_args = self.__args__[:-1] + t_result = self.__args__[-1] + return f"{_type_repr(self.__origin__)}" \ + f"[[{', '.join(_type_repr(a) for a in t_args)}], " \ + f"{_type_repr(t_result)}]" + + +def _type_repr(obj): + """Return the repr() of an object, special-casing types (internal helper). + + If obj is a type, we return a shorter version than the default + type.__repr__, based on the module and qualified name, which is + typically enough to uniquely identify a type. For everything + else, we fall back on repr(obj). + + Borrowed from :mod:`typing`. + """ + if isinstance(obj, type): + if obj.__module__ == 'builtins': + return obj.__qualname__ + return f'{obj.__module__}.{obj.__qualname__}' + if obj is ...: + return('...') + if isinstance(obj, types.FunctionType): + return obj.__name__ + return repr(obj) + + class Callable(metaclass=ABCMeta): __slots__ = () @@ -423,7 +483,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__call__") return NotImplemented - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(_CallableGenericAlias) ### SETS ### diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 912fb33af1a21b..efc678d9451741 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -302,6 +302,25 @@ def test_weakref(self): alias = t[int] self.assertEqual(ref(alias)(), alias) + def test_abc_callable(self): + alias = Callable[[int, str], float] + with self.subTest("Testing collections.abc.Callable's subscription"): + self.assertIs(alias.__origin__, Callable) + self.assertEqual(alias.__args__, (int, str, float)) + self.assertEqual(alias.__parameters__, ()) + + with self.subTest("Testing collections.abc.Callable's instance checks"): + self.assertIsInstance(alias, GenericAlias) + + invalid_params = ('Callable[int]', 'Callable[int, str]') + with self.subTest("Testing collections.abc.Callable's parameter " + "validation"): + for bad in invalid_params: + with self.subTest(f'Testing expression {bad}'): + with self.assertRaises(TypeError): + eval(bad) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 6102e05c165c5d..0460ee3eeca7c1 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -567,15 +567,38 @@ static PyGetSetDef ga_properties[] = { static PyObject * ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + gaobject *self; + + assert(type != NULL && type->tp_alloc != NULL); + self = (gaobject *)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + if (!_PyArg_NoKwnames("GenericAlias", kwds)) { return NULL; } if (!_PyArg_CheckPositional("GenericAlias", PyTuple_GET_SIZE(args), 2, 2)) { return NULL; } - PyObject *origin = PyTuple_GET_ITEM(args, 0); + PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); - return Py_GenericAlias(origin, arguments); + + // almost the same as Py_GenericAlias' code, but to assign to self + if (!PyTuple_Check(arguments)) { + arguments = PyTuple_Pack(1, arguments); + if (arguments == NULL) { + return NULL; + } + } + else { + Py_INCREF(arguments); + } + + Py_INCREF(origin); + self->origin = origin; + self->args = arguments; + self->parameters = NULL; + return (PyObject *) self; } static PyNumberMethods ga_as_number = { @@ -600,7 +623,7 @@ PyTypeObject Py_GenericAliasType = { .tp_hash = ga_hash, .tp_call = ga_call, .tp_getattro = ga_getattro, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, .tp_traverse = ga_traverse, .tp_richcompare = ga_richcompare, .tp_weaklistoffset = offsetof(gaobject, weakreflist), From 588d421984b46da1a07ffb14729d0ce65cc45769 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 31 Oct 2020 18:18:29 +0800 Subject: [PATCH 02/30] fix typing tests, add hash and eq methods --- Lib/_collections_abc.py | 25 ++++++++++++++++++++----- Lib/test/test_typing.py | 6 +++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 6a7cc1e01ee7d6..ba41638b3a4758 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -421,11 +421,11 @@ def __new__(cls, *args, **kwargs): raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") _typ, _args = args - if not isinstance(_args, tuple) or len(_args) != 2: + if not isinstance(_args, (tuple, type(...))) or len(_args) != 2: raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") t_args, t_result = _args - if not isinstance(t_args, list): + if not isinstance(t_args, (list, type(...))): raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") @@ -440,12 +440,27 @@ def __new__(cls, *args, **kwargs): def __init__(self, *args, **kwargs): pass + def __eq__(self, other): + if not isinstance(other, GenericAlias): + return NotImplemented + return (self.__origin__ == other.__origin__ + and self.__args__ == other.__args__) + + def __hash__(self): + return super().__hash__() + def __repr__(self): t_args = self.__args__[:-1] t_result = self.__args__[-1] - return f"{_type_repr(self.__origin__)}" \ - f"[[{', '.join(_type_repr(a) for a in t_args)}], " \ - f"{_type_repr(t_result)}]" + + orig = _type_repr(self.__origin__) + args = f"{', '.join(_type_repr(a) for a in t_args)}" + result = _type_repr(t_result) + + if not isinstance(t_args[0], type(...)): + args = f"[{args}]" + + return f"{orig}[{args}, {result}]" def _type_repr(obj): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7deba0d71b7c4f..2d148177c9c182 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1818,7 +1818,7 @@ def __call__(self): ## with self.assertRaises(TypeError): ## T2[int, str] - self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') + self.assertEqual(repr(C1[[int], int]).split('.')[-1], 'C1[[int], int]') self.assertEqual(C2.__parameters__, ()) self.assertIsInstance(C2(), collections.abc.Callable) self.assertIsSubclass(C2, collections.abc.Callable) @@ -1858,8 +1858,8 @@ class MyTup(Tuple[T, T]): ... self.assertEqual(MyTup[int]().__orig_class__, MyTup[int]) class MyCall(Callable[..., T]): def __call__(self): return None - self.assertIs(MyCall[T]().__class__, MyCall) - self.assertEqual(MyCall[T]().__orig_class__, MyCall[T]) + self.assertIs(MyCall[[T], T]().__class__, MyCall) + self.assertEqual(MyCall[[T], T]().__orig_class__, MyCall[[T], T]) class MyDict(typing.Dict[T, T]): ... self.assertIs(MyDict[int]().__class__, MyDict) self.assertEqual(MyDict[int]().__orig_class__, MyDict[int]) From 050fa13f34c2ac978df586475a29e316f9f89a2d Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 31 Oct 2020 18:41:32 +0800 Subject: [PATCH 03/30] Fix pickling --- Lib/_collections_abc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index ba41638b3a4758..21b6a0248c6a19 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -21,6 +21,8 @@ "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", "ByteString", + # To allow for pickling, not actually meant to be imported. + "_CallableGenericAlias" ] # This module has been renamed from collections.abc to _collections_abc to From f60ea8a932935d2423a4a25ce1f1c6d647dc7b26 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 31 Oct 2020 18:47:30 +0800 Subject: [PATCH 04/30] whitespace --- Objects/genericaliasobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 0460ee3eeca7c1..6427a3eeb9db9d 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -580,7 +580,7 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!_PyArg_CheckPositional("GenericAlias", PyTuple_GET_SIZE(args), 2, 2)) { return NULL; } - PyObject *origin = PyTuple_GET_ITEM(args, 0); + PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); // almost the same as Py_GenericAlias' code, but to assign to self From 2f3c6dc90b4e35885a57b30ece0f8a3f994e779a Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 31 Oct 2020 20:17:14 +0800 Subject: [PATCH 05/30] Add test specifically for bpo --- Lib/test/test_genericalias.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index efc678d9451741..a4a5fc7957976c 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -320,7 +320,11 @@ def test_abc_callable(self): with self.assertRaises(TypeError): eval(bad) - + # bpo-42195 + with self.subTest("Testing collections.abc.Callable's consistency " + "with typing.Callable"): + self.assertEquals(typing.Callable[[int, str], dict].__args__, + Callable[[int, str], dict].__args__) if __name__ == "__main__": unittest.main() From 2c4508e74d609afd268fa8871cadaf3ccf3bd0d7 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 31 Oct 2020 20:37:14 +0800 Subject: [PATCH 06/30] update error message --- Lib/_collections_abc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 21b6a0248c6a19..9b162f79667cd1 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -423,13 +423,13 @@ def __new__(cls, *args, **kwargs): raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") _typ, _args = args - if not isinstance(_args, (tuple, type(...))) or len(_args) != 2: + if not isinstance(_args, (tuple, types.EllipsisType)) or len(_args) != 2: raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") t_args, t_result = _args - if not isinstance(t_args, (list, type(...))): - raise TypeError("Callable must be used as " - "Callable[[arg, ...], result].") + if not isinstance(t_args, (list, types.EllipsisType)): + raise TypeError("Callable[args, result]: args must be a list. Got" + f" {_type_repr(t_args)}") ga_args = [] for arg in args[1]: @@ -459,7 +459,7 @@ def __repr__(self): args = f"{', '.join(_type_repr(a) for a in t_args)}" result = _type_repr(t_result) - if not isinstance(t_args[0], type(...)): + if not isinstance(t_args[0], types.EllipsisType): args = f"[{args}]" return f"{orig}[{args}, {result}]" From 19d29736bcb7b9293f521eb6763d01b0e0a1e2a9 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 31 Oct 2020 21:40:55 +0800 Subject: [PATCH 07/30] Appease test_site --- Lib/_collections_abc.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 9b162f79667cd1..0e85ad1fd11384 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -8,9 +8,12 @@ from abc import ABCMeta, abstractmethod import sys -import types GenericAlias = type(list[int]) +EllipsisType = type(...) +def _f(): pass +FunctionType = type(_f) + __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", @@ -418,21 +421,22 @@ class _CallableGenericAlias(GenericAlias): See :issue:`42195`. """ + def __new__(cls, *args, **kwargs): if not isinstance(args, tuple) or len(args) != 2: raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") _typ, _args = args - if not isinstance(_args, (tuple, types.EllipsisType)) or len(_args) != 2: + if not isinstance(_args, (tuple, EllipsisType)) or len(_args) != 2: raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") t_args, t_result = _args - if not isinstance(t_args, (list, types.EllipsisType)): + if not isinstance(t_args, (list, EllipsisType)): raise TypeError("Callable[args, result]: args must be a list. Got" f" {_type_repr(t_args)}") ga_args = [] - for arg in args[1]: + for arg in _args: if isinstance(arg, list): ga_args.extend(arg) else: @@ -459,7 +463,7 @@ def __repr__(self): args = f"{', '.join(_type_repr(a) for a in t_args)}" result = _type_repr(t_result) - if not isinstance(t_args[0], types.EllipsisType): + if not isinstance(t_args[0], EllipsisType): args = f"[{args}]" return f"{orig}[{args}, {result}]" @@ -481,7 +485,7 @@ def _type_repr(obj): return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: return('...') - if isinstance(obj, types.FunctionType): + if isinstance(obj, FunctionType): return obj.__name__ return repr(obj) From 3116c8ea356bbad159a34a9505441afd3ffa16ed Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 20 Nov 2020 00:03:07 +0800 Subject: [PATCH 08/30] Represent Callable __args__ via [tuple[args], result] --- Lib/_collections_abc.py | 62 ++++++++++++++++------------------- Lib/test/test_genericalias.py | 5 ++- Lib/test/test_typing.py | 2 +- Lib/typing.py | 15 +++++---- Objects/genericaliasobject.c | 11 ++++--- 5 files changed, 48 insertions(+), 47 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 0e85ad1fd11384..54fbbcdf4fdc39 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -25,7 +25,7 @@ def _f(): pass "Sequence", "MutableSequence", "ByteString", # To allow for pickling, not actually meant to be imported. - "_CallableGenericAlias" + "_CallableGenericAlias", ] # This module has been renamed from collections.abc to _collections_abc to @@ -418,30 +418,25 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): """ Internal class specifically for consistency between the ``__args__`` of ``collections.abc.Callable``'s and ``typing.Callable``'s ``GenericAlias``. - - See :issue:`42195`. """ def __new__(cls, *args, **kwargs): if not isinstance(args, tuple) or len(args) != 2: - raise TypeError("Callable must be used as " - "Callable[[arg, ...], result].") - _typ, _args = args - if not isinstance(_args, (tuple, EllipsisType)) or len(_args) != 2: - raise TypeError("Callable must be used as " - "Callable[[arg, ...], result].") + raise TypeError("Callable must be used as Callable[[arg, ...], result]") + origin, _args = args + if not isinstance(_args, tuple) or len(_args) != 2: + raise TypeError("Callable must be used as Callable[[arg, ...], result]") t_args, t_result = _args if not isinstance(t_args, (list, EllipsisType)): - raise TypeError("Callable[args, result]: args must be a list. Got" - f" {_type_repr(t_args)}") + raise TypeError("Callable[args, result]: args must be a list " + f"or Ellipsis. Got {_type_repr(t_args)}") + + if t_args == Ellipsis: + ga_args = _args + else: + ga_args = tuple[tuple(t_args)], t_result - ga_args = [] - for arg in _args: - if isinstance(arg, list): - ga_args.extend(arg) - else: - ga_args.append(arg) - return super().__new__(cls, _typ, tuple(ga_args)) + return super().__new__(cls, origin, ga_args) def __init__(self, *args, **kwargs): pass @@ -456,35 +451,36 @@ def __hash__(self): return super().__hash__() def __repr__(self): - t_args = self.__args__[:-1] - t_result = self.__args__[-1] + t_args = self.__args__[0] + origin = _type_repr(self.__origin__) - orig = _type_repr(self.__origin__) - args = f"{', '.join(_type_repr(a) for a in t_args)}" - result = _type_repr(t_result) + if len(self.__args__) == 2 and t_args is Ellipsis: + return super().__repr__() + if t_args.__args__ == ((),): + t_args_repr = '[]' + else: + t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' - if not isinstance(t_args[0], EllipsisType): - args = f"[{args}]" - return f"{orig}[{args}, {result}]" + return f"{origin}[{t_args_repr}, {_type_repr(self.__args__[-1])}]" def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). - If obj is a type, we return a shorter version than the default - type.__repr__, based on the module and qualified name, which is - typically enough to uniquely identify a type. For everything - else, we fall back on repr(obj). - - Borrowed from :mod:`typing`. + Borrowed from :mod:`typing` without importing since collections.abc + shouldn't depend on that module. """ + if isinstance(obj, tuple): + return f'[{", ".join(_type_repr(elem) for elem in obj)}]' + if isinstance(obj, GenericAlias): + return repr(obj) if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: - return('...') + return ('...') if isinstance(obj, FunctionType): return obj.__name__ return repr(obj) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index a4a5fc7957976c..abb42800b04301 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -62,7 +62,6 @@ class BaseTest(unittest.TestCase): Iterable, Iterator, Reversible, Container, Collection, - Callable, Mailbox, _PartialFile, ContextVar, Token, Field, @@ -306,7 +305,7 @@ def test_abc_callable(self): alias = Callable[[int, str], float] with self.subTest("Testing collections.abc.Callable's subscription"): self.assertIs(alias.__origin__, Callable) - self.assertEqual(alias.__args__, (int, str, float)) + self.assertEqual(alias.__args__, (tuple[int, str], float)) self.assertEqual(alias.__parameters__, ()) with self.subTest("Testing collections.abc.Callable's instance checks"): @@ -323,7 +322,7 @@ def test_abc_callable(self): # bpo-42195 with self.subTest("Testing collections.abc.Callable's consistency " "with typing.Callable"): - self.assertEquals(typing.Callable[[int, str], dict].__args__, + self.assertEqual(typing.Callable[[int, str], dict].__args__, Callable[[int, str], dict].__args__) if __name__ == "__main__": diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 2d148177c9c182..14d586e47bb40c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3058,7 +3058,7 @@ class C(Generic[T]): pass (int, Tuple[str, int])) self.assertEqual(get_args(typing.Dict[int, Tuple[T, T]][Optional[int]]), (int, Tuple[Optional[int], Optional[int]])) - self.assertEqual(get_args(Callable[[], T][int]), ([], int)) + self.assertEqual(get_args(Callable[[], T][int]), (tuple[()], int)) self.assertEqual(get_args(Callable[..., int]), (..., int)) self.assertEqual(get_args(Union[int, Callable[[Tuple[T, ...]], str]]), (int, Callable[[Tuple[T, ...]], str])) diff --git a/Lib/typing.py b/Lib/typing.py index d310b3dd5820dc..73f684d790c076 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -877,16 +877,21 @@ def __ror__(self, right): class _CallableGenericAlias(_GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' - if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: + t_args = self.__args__[0] + if len(self.__args__) == 2 and t_args is Ellipsis: return super().__repr__() + if t_args.__args__ == ((),): + t_args_repr = '[]' + else: + t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' return (f'typing.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'[{t_args_repr}, ' f'{_type_repr(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ if not (len(args) == 2 and args[0] is ...): - args = list(args[:-1]), args[-1] + args = list(t for t in args[0].__args__), args[-1] return operator.getitem, (Callable, args) @@ -918,7 +923,7 @@ def __getitem_inner__(self, params): return self.copy_with((_TypingEllipsis, result)) msg = "Callable[[arg, ...], result]: each arg must be a type." args = tuple(_type_check(arg, msg) for arg in args) - params = args + (result,) + params = (tuple[args], result) return self.copy_with(params) @@ -1551,8 +1556,6 @@ def get_args(tp): return (tp.__origin__,) + tp.__metadata__ if isinstance(tp, _GenericAlias): res = tp.__args__ - if tp.__origin__ is collections.abc.Callable and res[0] is not Ellipsis: - res = (list(res[:-1]), res[-1]) return res if isinstance(tp, GenericAlias): return tp.__args__ diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 6427a3eeb9db9d..74886903d75402 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -570,16 +570,18 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) gaobject *self; assert(type != NULL && type->tp_alloc != NULL); - self = (gaobject *)type->tp_alloc(type, 0); - if (self == NULL) - return NULL; - if (!_PyArg_NoKwnames("GenericAlias", kwds)) { return NULL; } if (!_PyArg_CheckPositional("GenericAlias", PyTuple_GET_SIZE(args), 2, 2)) { return NULL; } + + self = (gaobject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); @@ -598,6 +600,7 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->origin = origin; self->args = arguments; self->parameters = NULL; + self->weakreflist = NULL; return (PyObject *) self; } From f2b593a2a5e8867376c977dd94986336a552cb9d Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 20 Nov 2020 01:02:06 +0800 Subject: [PATCH 09/30] add back tests for weakref, styling nits, add news --- Lib/_collections_abc.py | 13 ++++--------- Lib/test/test_genericalias.py | 3 +++ Lib/typing.py | 6 ++---- .../2020-11-20-00-57-47.bpo-42195.HeqcpS.rst | 6 ++++++ 4 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 54fbbcdf4fdc39..b3f65e5d0db35f 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -431,10 +431,8 @@ def __new__(cls, *args, **kwargs): raise TypeError("Callable[args, result]: args must be a list " f"or Ellipsis. Got {_type_repr(t_args)}") - if t_args == Ellipsis: - ga_args = _args - else: - ga_args = tuple[tuple(t_args)], t_result + ga_args = (_args if t_args is Ellipsis + else (tuple[tuple(t_args)], t_result)) return super().__new__(cls, origin, ga_args) @@ -456,11 +454,8 @@ def __repr__(self): if len(self.__args__) == 2 and t_args is Ellipsis: return super().__repr__() - if t_args.__args__ == ((),): - t_args_repr = '[]' - else: - t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' - + t_args_repr = ('[]' if t_args.__args__ == ((),) else + f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]') return f"{origin}[{t_args_repr}, {_type_repr(self.__args__[-1])}]" diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index abb42800b04301..77b17cc94da116 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -319,6 +319,9 @@ def test_abc_callable(self): with self.assertRaises(TypeError): eval(bad) + with self.subTest("Testing collections.abc.Callable's weakref"): + self.assertEqual(ref(alias)(), alias) + # bpo-42195 with self.subTest("Testing collections.abc.Callable's consistency " "with typing.Callable"): diff --git a/Lib/typing.py b/Lib/typing.py index 73f684d790c076..8cfe7ff5d834c5 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -880,10 +880,8 @@ def __repr__(self): t_args = self.__args__[0] if len(self.__args__) == 2 and t_args is Ellipsis: return super().__repr__() - if t_args.__args__ == ((),): - t_args_repr = '[]' - else: - t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' + t_args_repr = ('[]' if t_args.__args__ == ((),) else + f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]') return (f'typing.Callable' f'[{t_args_repr}, ' f'{_type_repr(self.__args__[-1])}]') diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst new file mode 100644 index 00000000000000..880c5e2e220e2c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst @@ -0,0 +1,6 @@ +The ``__args__`` of the parameterized generics for :data:`typing.Callable` +and :class:`collections.abc.Callable` are now consistent. Said ``__args__`` +are now in the form ``(tuple[args], result)``. For example, +``typing.Callable[[int], float].__args__`` now gives ``(tuple[int], float)``. +Due to this change, :class:`types.GenericAlias` can now be subclassed. + From 93d51e4c3a1f15d04400ce401ff2eae59e8c60fc Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 20 Nov 2020 01:45:09 +0800 Subject: [PATCH 10/30] remove redundant tuple checks leftover from old code --- Lib/_collections_abc.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b3f65e5d0db35f..69ff28b04aa5d5 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -466,8 +466,6 @@ def _type_repr(obj): Borrowed from :mod:`typing` without importing since collections.abc shouldn't depend on that module. """ - if isinstance(obj, tuple): - return f'[{", ".join(_type_repr(elem) for elem in obj)}]' if isinstance(obj, GenericAlias): return repr(obj) if isinstance(obj, type): @@ -475,7 +473,7 @@ def _type_repr(obj): return obj.__qualname__ return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: - return ('...') + return '...' if isinstance(obj, FunctionType): return obj.__name__ return repr(obj) From 327e1a5bb75e4320c41ef0d3aabff2c1c407fac4 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 28 Nov 2020 13:53:20 +0800 Subject: [PATCH 11/30] Use _PosArgs instead of tuple 1. allow equality between typing._PosArgs and collections.abc._PosArgs 2. allow equality checks between genericalias subclasses 3. add support to genericalias subclasses for union type expressions --- Lib/_collections_abc.py | 53 ++++++++++++++++++++++++++--------- Lib/test/test_genericalias.py | 24 ++++++++++------ Lib/test/test_typing.py | 2 +- Lib/typing.py | 5 +++- Objects/genericaliasobject.c | 4 +-- Objects/unionobject.c | 2 +- 6 files changed, 64 insertions(+), 26 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 69ff28b04aa5d5..f3b232117f5f8f 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -24,8 +24,11 @@ def _f(): pass "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", "ByteString", - # To allow for pickling, not actually meant to be imported. + # The following classse are to allow pickling, not actually + # meant to be imported. "_CallableGenericAlias", + "_PosArgs", + "_PosArgsGenericAlias" ] # This module has been renamed from collections.abc to _collections_abc to @@ -415,6 +418,35 @@ def __subclasshook__(cls, C): return NotImplemented +class _PosArgsGenericAlias(GenericAlias): + """ Internal class specifically to represent positional arguments in + ``_CallableGenericAlias``. + """ + def __repr__(self): + return f"{__name__}._PosArgsGenericAlias" \ + f"[{', '.join(_type_repr(t) for t in self.__args__)}]" + + def __eq__(self, other): + o_cls = other.__class__ + if not (o_cls.__module__ == "typing" and o_cls.__name__ + == "_GenericAlias" or isinstance(other, GenericAlias)): + return NotImplemented + return (self.__origin__ == other.__origin__ + and self.__args__ == other.__args__) + + def __hash__(self): + return hash((self.__origin__, self.__args__)) + +# Only used for _CallableGenericAlias +class _PosArgs: + """ Internal class specifically to represent positional arguments in + ``_CallableGenericAlias``. + """ + def __class_getitem__(cls, item): + return _PosArgsGenericAlias(tuple, item) + +# _PosArgs = type("_PosArgs", (tuple, ), {}) + class _CallableGenericAlias(GenericAlias): """ Internal class specifically for consistency between the ``__args__`` of ``collections.abc.Callable``'s and ``typing.Callable``'s ``GenericAlias``. @@ -432,21 +464,10 @@ def __new__(cls, *args, **kwargs): f"or Ellipsis. Got {_type_repr(t_args)}") ga_args = (_args if t_args is Ellipsis - else (tuple[tuple(t_args)], t_result)) + else (_PosArgs[tuple(t_args)], t_result)) return super().__new__(cls, origin, ga_args) - def __init__(self, *args, **kwargs): - pass - - def __eq__(self, other): - if not isinstance(other, GenericAlias): - return NotImplemented - return (self.__origin__ == other.__origin__ - and self.__args__ == other.__args__) - - def __hash__(self): - return super().__hash__() def __repr__(self): t_args = self.__args__[0] @@ -459,6 +480,12 @@ def __repr__(self): return f"{origin}[{t_args_repr}, {_type_repr(self.__args__[-1])}]" + def __reduce__(self): + args = self.__args__ + if not (len(args) == 2 and args[0] is ...): + args = list(t for t in args[0].__args__), args[-1] + return _CallableGenericAlias, (Callable, args) + def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 77b17cc94da116..6ea13792f5f198 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -303,30 +303,38 @@ def test_weakref(self): def test_abc_callable(self): alias = Callable[[int, str], float] - with self.subTest("Testing collections.abc.Callable's subscription"): + with self.subTest("Testing subscription"): self.assertIs(alias.__origin__, Callable) - self.assertEqual(alias.__args__, (tuple[int, str], float)) + self.assertEqual(alias.__args__, (_PosArgs[int, str], float)) self.assertEqual(alias.__parameters__, ()) - with self.subTest("Testing collections.abc.Callable's instance checks"): + with self.subTest("Testing nstance checks"): self.assertIsInstance(alias, GenericAlias) invalid_params = ('Callable[int]', 'Callable[int, str]') - with self.subTest("Testing collections.abc.Callable's parameter " - "validation"): + with self.subTest("Testing parameter validation"): for bad in invalid_params: with self.subTest(f'Testing expression {bad}'): with self.assertRaises(TypeError): eval(bad) - with self.subTest("Testing collections.abc.Callable's weakref"): + with self.subTest("Testing weakref"): self.assertEqual(ref(alias)(), alias) + with self.subTest("Testing picling"): + s = pickle.dumps(alias) + loaded = pickle.loads(s) + self.assertEqual(alias.__origin__, loaded.__origin__) + self.assertEqual(alias.__args__, loaded.__args__) + self.assertEqual(alias.__parameters__, loaded.__parameters__) + # bpo-42195 with self.subTest("Testing collections.abc.Callable's consistency " "with typing.Callable"): - self.assertEqual(typing.Callable[[int, str], dict].__args__, - Callable[[int, str], dict].__args__) + c1 = typing.Callable[[int, str], dict] + c2 = Callable[[int, str], dict] + self.assertEqual(c1.__args__, c2.__args__) + self.assertEqual(hash(c1.__args__), hash(c2.__args__)) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 14d586e47bb40c..4b51790e500733 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3058,7 +3058,7 @@ class C(Generic[T]): pass (int, Tuple[str, int])) self.assertEqual(get_args(typing.Dict[int, Tuple[T, T]][Optional[int]]), (int, Tuple[Optional[int], Optional[int]])) - self.assertEqual(get_args(Callable[[], T][int]), (tuple[()], int)) + self.assertEqual(get_args(Callable[[], T][int]), (typing._PosArgs[()], int)) self.assertEqual(get_args(Callable[..., int]), (..., int)) self.assertEqual(get_args(Union[int, Callable[[Tuple[T, ...]], str]]), (int, Callable[[Tuple[T, ...]], str])) diff --git a/Lib/typing.py b/Lib/typing.py index 8cfe7ff5d834c5..4092042dc5abbb 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -114,6 +114,7 @@ 'Text', 'TYPE_CHECKING', 'TypeAlias', + '_PosArgs', # Not meant to be imported, just for pickling. ] # The pseudo-submodules 're' and 'io' are part of the public @@ -921,7 +922,7 @@ def __getitem_inner__(self, params): return self.copy_with((_TypingEllipsis, result)) msg = "Callable[[arg, ...], result]: each arg must be a type." args = tuple(_type_check(arg, msg) for arg in args) - params = (tuple[args], result) + params = (_PosArgs[args], result) return self.copy_with(params) @@ -1711,6 +1712,8 @@ class Other(Leaf): # Error reported by type checker Sized = _alias(collections.abc.Sized, 0) # Not generic. Container = _alias(collections.abc.Container, 1) Collection = _alias(collections.abc.Collection, 1) +# Only used for Callable +_PosArgs = _TupleType(tuple, -1, inst=False, name='_PosArgs') Callable = _CallableType(collections.abc.Callable, 2) Callable.__doc__ = \ """Callable type; Callable[[int], str] is a function of (int) -> str. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 74886903d75402..36a9dce6478046 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -429,8 +429,8 @@ ga_getattro(PyObject *self, PyObject *name) static PyObject * ga_richcompare(PyObject *a, PyObject *b, int op) { - if (!Py_IS_TYPE(a, &Py_GenericAliasType) || - !Py_IS_TYPE(b, &Py_GenericAliasType) || + if (!PyObject_TypeCheck(a, &Py_GenericAliasType) || + !PyObject_TypeCheck(b, &Py_GenericAliasType) || (op != Py_EQ && op != Py_NE)) { Py_RETURN_NOTIMPLEMENTED; diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 2308bfc9f2a278..7484df02dbd1af 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -296,7 +296,7 @@ is_unionable(PyObject *obj) is_new_type(obj) || is_special_form(obj) || PyType_Check(obj) || - type == &Py_GenericAliasType || + PyObject_TypeCheck(obj, &Py_GenericAliasType) || type == &_Py_UnionType); } From e971ccb7e83b80cc2868f1390edb03f38e782547 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 30 Nov 2020 02:12:07 +0800 Subject: [PATCH 12/30] Fix typo and news --- Lib/test/test_genericalias.py | 2 +- .../2020-11-20-00-57-47.bpo-42195.HeqcpS.rst | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 6ea13792f5f198..6925b51e37e1ce 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -321,7 +321,7 @@ def test_abc_callable(self): with self.subTest("Testing weakref"): self.assertEqual(ref(alias)(), alias) - with self.subTest("Testing picling"): + with self.subTest("Testing pickling"): s = pickle.dumps(alias) loaded = pickle.loads(s) self.assertEqual(alias.__origin__, loaded.__origin__) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst index 880c5e2e220e2c..5c149bb6af01e1 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst @@ -1,6 +1,7 @@ The ``__args__`` of the parameterized generics for :data:`typing.Callable` and :class:`collections.abc.Callable` are now consistent. Said ``__args__`` -are now in the form ``(tuple[args], result)``. For example, -``typing.Callable[[int], float].__args__`` now gives ``(tuple[int], float)``. -Due to this change, :class:`types.GenericAlias` can now be subclassed. +are now in the form ``(_PosArgs[args], result)``. For example, +``typing.Callable[[int], float].__args__`` now gives +``(typing._PosArgs[int], float)``. Due to this change, +:class:`types.GenericAlias` can now be subclassed. Patch by Ken Jin. From abd8b98bcd8b05c31ce4db0969f1b901c7c4e8a6 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 30 Nov 2020 13:26:31 +0800 Subject: [PATCH 13/30] Refactor C code to use less duplication --- Objects/genericaliasobject.c | 94 ++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 36a9dce6478046..5cc3dc3c41c888 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -564,11 +564,49 @@ static PyGetSetDef ga_properties[] = { {0} }; +// Helper to create inheritable or non-inheritable gaobjects +static inline gaobject * +create_ga(PyTypeObject *type, PyObject *origin, PyObject *args) { + if (!PyTuple_Check(args)) { + args = PyTuple_Pack(1, args); + if (args == NULL) { + return NULL; + } + } + else { + Py_INCREF(args); + } + + gaobject *alias; + int gc_should_track = 0; + if (type != NULL) { + assert(type->tp_alloc != NULL); + alias = (gaobject *)type->tp_alloc(type, 0); + } + else { + alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); + gc_should_track = 1; + } + + if (alias == NULL) { + Py_DECREF(args); + return NULL; + } + + Py_INCREF(origin); + alias->origin = origin; + alias->args = args; + alias->parameters = NULL; + alias->weakreflist = NULL; + if (gc_should_track) { + _PyObject_GC_TRACK(alias); + } + return alias; +} + static PyObject * ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - gaobject *self; - assert(type != NULL && type->tp_alloc != NULL); if (!_PyArg_NoKwnames("GenericAlias", kwds)) { return NULL; @@ -576,32 +614,18 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!_PyArg_CheckPositional("GenericAlias", PyTuple_GET_SIZE(args), 2, 2)) { return NULL; } - - self = (gaobject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); - // almost the same as Py_GenericAlias' code, but to assign to self - if (!PyTuple_Check(arguments)) { - arguments = PyTuple_Pack(1, arguments); - if (arguments == NULL) { - return NULL; - } - } - else { - Py_INCREF(arguments); + PyObject *self = (PyObject *)create_ga(type, origin, arguments); + if (self == NULL) { + Py_DECREF(origin); + Py_DECREF(arguments); + return NULL; } - - Py_INCREF(origin); - self->origin = origin; - self->args = arguments; - self->parameters = NULL; - self->weakreflist = NULL; - return (PyObject *) self; + + return self; } static PyNumberMethods ga_as_number = { @@ -641,27 +665,5 @@ PyTypeObject Py_GenericAliasType = { PyObject * Py_GenericAlias(PyObject *origin, PyObject *args) { - if (!PyTuple_Check(args)) { - args = PyTuple_Pack(1, args); - if (args == NULL) { - return NULL; - } - } - else { - Py_INCREF(args); - } - - gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); - if (alias == NULL) { - Py_DECREF(args); - return NULL; - } - - Py_INCREF(origin); - alias->origin = origin; - alias->args = args; - alias->parameters = NULL; - alias->weakreflist = NULL; - _PyObject_GC_TRACK(alias); - return (PyObject *)alias; + return (PyObject *)create_ga(NULL, origin, args); } From 3ddca0664b4e53d525ace1947128ed11ff6d452c Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 2 Dec 2020 00:47:59 +0800 Subject: [PATCH 14/30] Address most of Guido's reviews (tests failing on purpose) --- Lib/_collections_abc.py | 47 ++++++++++++++++------------------- Lib/collections/abc.py | 1 + Lib/test/test_genericalias.py | 3 ++- Lib/test/test_typing.py | 6 ++--- Lib/typing.py | 11 ++++---- Objects/genericaliasobject.c | 3 --- 6 files changed, 33 insertions(+), 38 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index f3b232117f5f8f..59b589eab7a75e 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -24,11 +24,6 @@ def _f(): pass "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", "ByteString", - # The following classse are to allow pickling, not actually - # meant to be imported. - "_CallableGenericAlias", - "_PosArgs", - "_PosArgsGenericAlias" ] # This module has been renamed from collections.abc to _collections_abc to @@ -422,17 +417,19 @@ class _PosArgsGenericAlias(GenericAlias): """ Internal class specifically to represent positional arguments in ``_CallableGenericAlias``. """ + __slots__ = () + def __repr__(self): return f"{__name__}._PosArgsGenericAlias" \ f"[{', '.join(_type_repr(t) for t in self.__args__)}]" def __eq__(self, other): o_cls = other.__class__ - if not (o_cls.__module__ == "typing" and o_cls.__name__ - == "_GenericAlias" or isinstance(other, GenericAlias)): - return NotImplemented - return (self.__origin__ == other.__origin__ - and self.__args__ == other.__args__) + if ((o_cls.__module__ == "typing" and o_cls.__name__ + == "_GenericAlias") or isinstance(other, GenericAlias)): + return (self.__origin__ == other.__origin__ + and self.__args__ == other.__args__) + return NotImplemented def __hash__(self): return hash((self.__origin__, self.__args__)) @@ -445,39 +442,37 @@ class _PosArgs: def __class_getitem__(cls, item): return _PosArgsGenericAlias(tuple, item) -# _PosArgs = type("_PosArgs", (tuple, ), {}) class _CallableGenericAlias(GenericAlias): """ Internal class specifically for consistency between the ``__args__`` of ``collections.abc.Callable``'s and ``typing.Callable``'s ``GenericAlias``. """ - def __new__(cls, *args, **kwargs): - if not isinstance(args, tuple) or len(args) != 2: - raise TypeError("Callable must be used as Callable[[arg, ...], result]") - origin, _args = args + def __new__(cls, origin, _args, **kwargs): if not isinstance(_args, tuple) or len(_args) != 2: raise TypeError("Callable must be used as Callable[[arg, ...], result]") t_args, t_result = _args if not isinstance(t_args, (list, EllipsisType)): - raise TypeError("Callable[args, result]: args must be a list " - f"or Ellipsis. Got {_type_repr(t_args)}") + raise TypeError(f"Callable[args, result]: args must be a list or Ellipsis. " + f"Got {_type_repr(t_args)}") - ga_args = (_args if t_args is Ellipsis - else (_PosArgs[tuple(t_args)], t_result)) + if t_args is Ellipsis: + ga_args = _args + else: + ga_args = _PosArgs[tuple(t_args)], t_result return super().__new__(cls, origin, ga_args) def __repr__(self): + if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: + return super().__repr__() t_args = self.__args__[0] + if t_args.__args__ == ((),): + t_args_repr = '[]' + else: + t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' origin = _type_repr(self.__origin__) - - if len(self.__args__) == 2 and t_args is Ellipsis: - return super().__repr__() - t_args_repr = ('[]' if t_args.__args__ == ((),) else - f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]') - return f"{origin}[{t_args_repr}, {_type_repr(self.__args__[-1])}]" def __reduce__(self): @@ -490,7 +485,7 @@ def __reduce__(self): def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). - Borrowed from :mod:`typing` without importing since collections.abc + Copied from :mod:`typing` since collections.abc shouldn't depend on that module. """ if isinstance(obj, GenericAlias): diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 891600d16bee9e..79cc4d1369708d 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,2 +1,3 @@ from _collections_abc import * from _collections_abc import __all__ +from _collections_abc import _CallableGenericAlias, _PosArgs, _PosArgsGenericAlias diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 6925b51e37e1ce..765345970c4c6f 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -6,6 +6,7 @@ defaultdict, deque, OrderedDict, Counter, UserDict, UserList ) from collections.abc import * +from collections.abc import _PosArgs from concurrent.futures import Future from concurrent.futures.thread import _WorkItem from contextlib import AbstractContextManager, AbstractAsyncContextManager @@ -308,7 +309,7 @@ def test_abc_callable(self): self.assertEqual(alias.__args__, (_PosArgs[int, str], float)) self.assertEqual(alias.__parameters__, ()) - with self.subTest("Testing nstance checks"): + with self.subTest("Testing instance checks"): self.assertIsInstance(alias, GenericAlias) invalid_params = ('Callable[int]', 'Callable[int, str]') diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 4b51790e500733..24e68e6607f871 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1818,7 +1818,7 @@ def __call__(self): ## with self.assertRaises(TypeError): ## T2[int, str] - self.assertEqual(repr(C1[[int], int]).split('.')[-1], 'C1[[int], int]') + self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') self.assertEqual(C2.__parameters__, ()) self.assertIsInstance(C2(), collections.abc.Callable) self.assertIsSubclass(C2, collections.abc.Callable) @@ -1858,8 +1858,8 @@ class MyTup(Tuple[T, T]): ... self.assertEqual(MyTup[int]().__orig_class__, MyTup[int]) class MyCall(Callable[..., T]): def __call__(self): return None - self.assertIs(MyCall[[T], T]().__class__, MyCall) - self.assertEqual(MyCall[[T], T]().__orig_class__, MyCall[[T], T]) + self.assertIs(MyCall[T]().__class__, MyCall) + self.assertEqual(MyCall[T]().__orig_class__, MyCall[T]) class MyDict(typing.Dict[T, T]): ... self.assertIs(MyDict[int]().__class__, MyDict) self.assertEqual(MyDict[int]().__orig_class__, MyDict[int]) diff --git a/Lib/typing.py b/Lib/typing.py index 4092042dc5abbb..4a9a0d127b8d04 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -114,7 +114,6 @@ 'Text', 'TYPE_CHECKING', 'TypeAlias', - '_PosArgs', # Not meant to be imported, just for pickling. ] # The pseudo-submodules 're' and 'io' are part of the public @@ -878,11 +877,13 @@ def __ror__(self, right): class _CallableGenericAlias(_GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' - t_args = self.__args__[0] - if len(self.__args__) == 2 and t_args is Ellipsis: + if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: return super().__repr__() - t_args_repr = ('[]' if t_args.__args__ == ((),) else - f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]') + t_args = self.__args__[0] + if t_args.__args__ == ((),): + t_args_repr = '[]' + else: + t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' return (f'typing.Callable' f'[{t_args_repr}, ' f'{_type_repr(self.__args__[-1])}]') diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 5cc3dc3c41c888..7c6eb3deac2b75 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -614,17 +614,14 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!_PyArg_CheckPositional("GenericAlias", PyTuple_GET_SIZE(args), 2, 2)) { return NULL; } - PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); - PyObject *self = (PyObject *)create_ga(type, origin, arguments); if (self == NULL) { Py_DECREF(origin); Py_DECREF(arguments); return NULL; } - return self; } From 1ab59c56ccf39c18f46f238369a0c2c4e9b82c23 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 2 Dec 2020 22:56:42 +0800 Subject: [PATCH 15/30] try to revert back to good old flat tuple __args__ days --- Lib/_collections_abc.py | 43 ++++++----------------------------------- Lib/collections/abc.py | 2 +- Lib/test/test_typing.py | 2 +- Lib/typing.py | 17 ++++++---------- 4 files changed, 14 insertions(+), 50 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 59b589eab7a75e..21e0402a4954f6 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -413,57 +413,26 @@ def __subclasshook__(cls, C): return NotImplemented -class _PosArgsGenericAlias(GenericAlias): - """ Internal class specifically to represent positional arguments in - ``_CallableGenericAlias``. - """ - __slots__ = () - - def __repr__(self): - return f"{__name__}._PosArgsGenericAlias" \ - f"[{', '.join(_type_repr(t) for t in self.__args__)}]" - - def __eq__(self, other): - o_cls = other.__class__ - if ((o_cls.__module__ == "typing" and o_cls.__name__ - == "_GenericAlias") or isinstance(other, GenericAlias)): - return (self.__origin__ == other.__origin__ - and self.__args__ == other.__args__) - return NotImplemented - - def __hash__(self): - return hash((self.__origin__, self.__args__)) - -# Only used for _CallableGenericAlias -class _PosArgs: - """ Internal class specifically to represent positional arguments in - ``_CallableGenericAlias``. - """ - def __class_getitem__(cls, item): - return _PosArgsGenericAlias(tuple, item) - - class _CallableGenericAlias(GenericAlias): """ Internal class specifically for consistency between the ``__args__`` of ``collections.abc.Callable``'s and ``typing.Callable``'s ``GenericAlias``. """ - - def __new__(cls, origin, _args, **kwargs): - if not isinstance(_args, tuple) or len(_args) != 2: + __slots__ = () + def __new__(cls, origin, args, **kwargs): + if not isinstance(args, tuple) or len(args) != 2: raise TypeError("Callable must be used as Callable[[arg, ...], result]") - t_args, t_result = _args + t_args, t_result = args if not isinstance(t_args, (list, EllipsisType)): raise TypeError(f"Callable[args, result]: args must be a list or Ellipsis. " f"Got {_type_repr(t_args)}") if t_args is Ellipsis: - ga_args = _args + ga_args = args else: - ga_args = _PosArgs[tuple(t_args)], t_result + ga_args = tuple(t_args) + (t_result,) return super().__new__(cls, origin, ga_args) - def __repr__(self): if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: return super().__repr__() diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 79cc4d1369708d..86ca8b8a8414b3 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ from _collections_abc import * from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias, _PosArgs, _PosArgsGenericAlias +from _collections_abc import _CallableGenericAlias diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 24e68e6607f871..7deba0d71b7c4f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3058,7 +3058,7 @@ class C(Generic[T]): pass (int, Tuple[str, int])) self.assertEqual(get_args(typing.Dict[int, Tuple[T, T]][Optional[int]]), (int, Tuple[Optional[int], Optional[int]])) - self.assertEqual(get_args(Callable[[], T][int]), (typing._PosArgs[()], int)) + self.assertEqual(get_args(Callable[[], T][int]), ([], int)) self.assertEqual(get_args(Callable[..., int]), (..., int)) self.assertEqual(get_args(Union[int, Callable[[Tuple[T, ...]], str]]), (int, Callable[[Tuple[T, ...]], str])) diff --git a/Lib/typing.py b/Lib/typing.py index 4a9a0d127b8d04..74251a4aa20dda 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -879,19 +879,14 @@ def __repr__(self): assert self._name == 'Callable' if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: return super().__repr__() - t_args = self.__args__[0] - if t_args.__args__ == ((),): - t_args_repr = '[]' - else: - t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' return (f'typing.Callable' - f'[{t_args_repr}, ' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' f'{_type_repr(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ if not (len(args) == 2 and args[0] is ...): - args = list(t for t in args[0].__args__), args[-1] + args = list(args[:-1]), args[-1] return operator.getitem, (Callable, args) @@ -911,7 +906,7 @@ def __getitem__(self, params): if not isinstance(args, list): raise TypeError(f"Callable[args, result]: args must be a list." f" Got {args}") - params = (tuple(args), result) + params = tuple(args) + (result,) return self.__getitem_inner__(params) @_tp_cache @@ -923,7 +918,7 @@ def __getitem_inner__(self, params): return self.copy_with((_TypingEllipsis, result)) msg = "Callable[[arg, ...], result]: each arg must be a type." args = tuple(_type_check(arg, msg) for arg in args) - params = (_PosArgs[args], result) + params = args + (result,) return self.copy_with(params) @@ -1556,6 +1551,8 @@ def get_args(tp): return (tp.__origin__,) + tp.__metadata__ if isinstance(tp, _GenericAlias): res = tp.__args__ + if tp.__origin__ is collections.abc.Callable and res[0] is not Ellipsis: + res = (list(res[:-1]), res[-1]) return res if isinstance(tp, GenericAlias): return tp.__args__ @@ -1713,8 +1710,6 @@ class Other(Leaf): # Error reported by type checker Sized = _alias(collections.abc.Sized, 0) # Not generic. Container = _alias(collections.abc.Container, 1) Collection = _alias(collections.abc.Collection, 1) -# Only used for Callable -_PosArgs = _TupleType(tuple, -1, inst=False, name='_PosArgs') Callable = _CallableType(collections.abc.Callable, 2) Callable.__doc__ = \ """Callable type; Callable[[int], str] is a function of (int) -> str. From ee2d2e1570d93283f56222691bc0e82befbc2d43 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 2 Dec 2020 23:26:44 +0800 Subject: [PATCH 16/30] getting even closer --- Lib/_collections_abc.py | 32 ++++++++++++++++++-------------- Lib/test/test_genericalias.py | 3 +-- Lib/typing.py | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 21e0402a4954f6..b419834f8c462c 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -13,7 +13,7 @@ EllipsisType = type(...) def _f(): pass FunctionType = type(_f) - +del _f __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", @@ -419,35 +419,39 @@ class _CallableGenericAlias(GenericAlias): """ __slots__ = () def __new__(cls, origin, args, **kwargs): + try: + return cls.__getitem_type(origin, args, **kwargs) + except TypeError as exc: + return super().__new__(cls, origin, args) + + @classmethod + def __getitem_type(cls, origin, args, **kwargs): if not isinstance(args, tuple) or len(args) != 2: - raise TypeError("Callable must be used as Callable[[arg, ...], result]") + print(args) + raise TypeError( + "Callable must be used as Callable[[arg, ...], result]") t_args, t_result = args if not isinstance(t_args, (list, EllipsisType)): - raise TypeError(f"Callable[args, result]: args must be a list or Ellipsis. " - f"Got {_type_repr(t_args)}") - + raise TypeError( + f"Callable[args, result]: args must be a list or Ellipsis. " + f"Got {_type_repr(t_args)}") if t_args is Ellipsis: ga_args = args else: ga_args = tuple(t_args) + (t_result,) - return super().__new__(cls, origin, ga_args) def __repr__(self): if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: return super().__repr__() - t_args = self.__args__[0] - if t_args.__args__ == ((),): - t_args_repr = '[]' - else: - t_args_repr = f'[{", ".join(_type_repr(a) for a in t_args.__args__)}]' - origin = _type_repr(self.__origin__) - return f"{origin}[{t_args_repr}, {_type_repr(self.__args__[-1])}]" + return (f'collections.abc.Callable' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'{_type_repr(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ if not (len(args) == 2 and args[0] is ...): - args = list(t for t in args[0].__args__), args[-1] + args = list(args[:-1]), args[-1] return _CallableGenericAlias, (Callable, args) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 765345970c4c6f..b6e3bfb5352a3e 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -6,7 +6,6 @@ defaultdict, deque, OrderedDict, Counter, UserDict, UserList ) from collections.abc import * -from collections.abc import _PosArgs from concurrent.futures import Future from concurrent.futures.thread import _WorkItem from contextlib import AbstractContextManager, AbstractAsyncContextManager @@ -306,7 +305,7 @@ def test_abc_callable(self): alias = Callable[[int, str], float] with self.subTest("Testing subscription"): self.assertIs(alias.__origin__, Callable) - self.assertEqual(alias.__args__, (_PosArgs[int, str], float)) + self.assertEqual(alias.__args__, (int, str, float)) self.assertEqual(alias.__parameters__, ()) with self.subTest("Testing instance checks"): diff --git a/Lib/typing.py b/Lib/typing.py index 74251a4aa20dda..d310b3dd5820dc 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -906,7 +906,7 @@ def __getitem__(self, params): if not isinstance(args, list): raise TypeError(f"Callable[args, result]: args must be a list." f" Got {args}") - params = tuple(args) + (result,) + params = (tuple(args), result) return self.__getitem_inner__(params) @_tp_cache From 20157382932e0e1ef17e8b7974b7a4bbdd6a1d4f Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 3 Dec 2020 00:20:48 +0800 Subject: [PATCH 17/30] finally done --- Lib/_collections_abc.py | 27 ++++++++++++++++++++------- Lib/test/test_genericalias.py | 7 ------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b419834f8c462c..1ef218b09803bd 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -418,16 +418,29 @@ class _CallableGenericAlias(GenericAlias): ``collections.abc.Callable``'s and ``typing.Callable``'s ``GenericAlias``. """ __slots__ = () - def __new__(cls, origin, args, **kwargs): - try: - return cls.__getitem_type(origin, args, **kwargs) - except TypeError as exc: - return super().__new__(cls, origin, args) + def __new__(cls, origin, args): + try: + return cls.__getitem_type(origin, args) + except TypeError: + # Fail-safe: most builtin generic collections don't validate the + # arguments passed to types.GenericAlias anyways. + # This is also because a subclass of the typing.Callable generic + # will have an __mro__ (, + # , , + # ). Subclasses wil use the __class_getitem__ of + # collections.abc.Callable before typing.Generic. As a result, + # we need to fall back on this to allow things like:: + # + # T = TypeVar('T') + # class C1(typing.Callable[[T], T]): ... + # C1[int] + # + # Otherwise that will raise a TypeError. + return GenericAlias(origin, args) @classmethod - def __getitem_type(cls, origin, args, **kwargs): + def __getitem_type(cls, origin, args): if not isinstance(args, tuple) or len(args) != 2: - print(args) raise TypeError( "Callable must be used as Callable[[arg, ...], result]") t_args, t_result = args diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index b6e3bfb5352a3e..f16206e4c689fd 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -311,13 +311,6 @@ def test_abc_callable(self): with self.subTest("Testing instance checks"): self.assertIsInstance(alias, GenericAlias) - invalid_params = ('Callable[int]', 'Callable[int, str]') - with self.subTest("Testing parameter validation"): - for bad in invalid_params: - with self.subTest(f'Testing expression {bad}'): - with self.assertRaises(TypeError): - eval(bad) - with self.subTest("Testing weakref"): self.assertEqual(ref(alias)(), alias) From 6704ffda0d8454789d261231e7c36d0627197b64 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 5 Dec 2020 00:12:50 +0800 Subject: [PATCH 18/30] Update news --- .../2020-11-20-00-57-47.bpo-42195.HeqcpS.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst index 5c149bb6af01e1..c8835457b314a7 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst @@ -1,7 +1,6 @@ The ``__args__`` of the parameterized generics for :data:`typing.Callable` and :class:`collections.abc.Callable` are now consistent. Said ``__args__`` -are now in the form ``(_PosArgs[args], result)``. For example, -``typing.Callable[[int], float].__args__`` now gives -``(typing._PosArgs[int], float)``. Due to this change, +for :class:`collections.abc.Callable` are now flattened while +:data:`typing.Callable`'s have not changed. To allow this change, :class:`types.GenericAlias` can now be subclassed. Patch by Ken Jin. From 598d29bc0ce52c78e4bc723749911ed8ec86d384 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 5 Dec 2020 13:54:08 +0800 Subject: [PATCH 19/30] Address review partially * Add tests for callable typevar substitution * Update docstrings and news Co-Authored-By: Guido van Rossum --- Lib/_collections_abc.py | 13 +++++++++---- Lib/test/test_genericalias.py | 6 ++++++ .../2020-11-20-00-57-47.bpo-42195.HeqcpS.rst | 2 +- Objects/genericaliasobject.c | 1 - 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 1ef218b09803bd..08baf546656482 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -414,8 +414,13 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): - """ Internal class specifically for consistency between the ``__args__`` of - ``collections.abc.Callable``'s and ``typing.Callable``'s ``GenericAlias``. + """ Represent `Callable[argtypes, resulttype]`. + + This sets ``__args__`` to a tuple containing the flattened``argtypes`` + followed by ``resulttype``. + + Example: ``Callable[[int, str], float]`` sets ``__args__`` to + ``(int, str, float)``. """ __slots__ = () def __new__(cls, origin, args): @@ -463,7 +468,7 @@ def __repr__(self): def __reduce__(self): args = self.__args__ - if not (len(args) == 2 and args[0] is ...): + if not (len(args) == 2 and args[0] is Ellipsis): args = list(args[:-1]), args[-1] return _CallableGenericAlias, (Callable, args) @@ -480,7 +485,7 @@ def _type_repr(obj): if obj.__module__ == 'builtins': return obj.__qualname__ return f'{obj.__module__}.{obj.__qualname__}' - if obj is ...: + if obj is Ellipsis: return '...' if isinstance(obj, FunctionType): return obj.__name__ diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index f16206e4c689fd..53b22cd835cc20 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -321,6 +321,12 @@ def test_abc_callable(self): self.assertEqual(alias.__args__, loaded.__args__) self.assertEqual(alias.__parameters__, loaded.__parameters__) + with self.subTest("Testing TypeVar substitution"): + C1 = Callable[[int, T], T] + self.assertEqual(C1[str], Callable[[int, str], str]) + C2 = Callable[[K, T], V] + self.assertEqual(C2[int, float, str], Callable[[int, float], str]) + # bpo-42195 with self.subTest("Testing collections.abc.Callable's consistency " "with typing.Callable"): diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst index c8835457b314a7..df9bfa8ddcf508 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst @@ -1,5 +1,5 @@ The ``__args__`` of the parameterized generics for :data:`typing.Callable` -and :class:`collections.abc.Callable` are now consistent. Said ``__args__`` +and :class:`collections.abc.Callable` are now consistent. The ``__args__`` for :class:`collections.abc.Callable` are now flattened while :data:`typing.Callable`'s have not changed. To allow this change, :class:`types.GenericAlias` can now be subclassed. Patch by Ken Jin. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 7c6eb3deac2b75..a0dc0221c5633f 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -564,7 +564,6 @@ static PyGetSetDef ga_properties[] = { {0} }; -// Helper to create inheritable or non-inheritable gaobjects static inline gaobject * create_ga(PyTypeObject *type, PyObject *origin, PyObject *args) { if (!PyTuple_Check(args)) { From c43ebcf9cc51d62a2135934e6dca2920670aee3f Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 5 Dec 2020 16:37:52 +0800 Subject: [PATCH 20/30] Address review fully, update news and tests, remove try-except block --- Lib/_collections_abc.py | 23 ++----------------- Lib/test/test_genericalias.py | 14 ++++++++++- Lib/test/test_typing.py | 23 +++++++++---------- .../2020-11-20-00-57-47.bpo-42195.HeqcpS.rst | 5 +++- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 08baf546656482..7bac6ff03ece7b 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -422,29 +422,10 @@ class _CallableGenericAlias(GenericAlias): Example: ``Callable[[int, str], float]`` sets ``__args__`` to ``(int, str, float)``. """ + __slots__ = () - def __new__(cls, origin, args): - try: - return cls.__getitem_type(origin, args) - except TypeError: - # Fail-safe: most builtin generic collections don't validate the - # arguments passed to types.GenericAlias anyways. - # This is also because a subclass of the typing.Callable generic - # will have an __mro__ (, - # , , - # ). Subclasses wil use the __class_getitem__ of - # collections.abc.Callable before typing.Generic. As a result, - # we need to fall back on this to allow things like:: - # - # T = TypeVar('T') - # class C1(typing.Callable[[T], T]): ... - # C1[int] - # - # Otherwise that will raise a TypeError. - return GenericAlias(origin, args) - @classmethod - def __getitem_type(cls, origin, args): + def __new__(cls, origin, args): if not isinstance(args, tuple) or len(args) != 2: raise TypeError( "Callable must be used as Callable[[arg, ...], result]") diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 53b22cd835cc20..7cd7067d23d7df 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -302,6 +302,8 @@ def test_weakref(self): self.assertEqual(ref(alias)(), alias) def test_abc_callable(self): + # A separate test is needed for Callable since it uses a subclass of + # GenericAlias. alias = Callable[[int, str], float] with self.subTest("Testing subscription"): self.assertIs(alias.__origin__, Callable) @@ -323,9 +325,19 @@ def test_abc_callable(self): with self.subTest("Testing TypeVar substitution"): C1 = Callable[[int, T], T] - self.assertEqual(C1[str], Callable[[int, str], str]) C2 = Callable[[K, T], V] + C3 = Callable[..., T] + self.assertEqual(C1[str], Callable[[int, str], str]) self.assertEqual(C2[int, float, str], Callable[[int, float], str]) + self.assertEqual(C3[int], Callable[..., int]) + + with self.subTest("Testing type erasure"): + class C1(Callable): + def __call__(self): + return None + a = C1[[int], T] + self.assertIs(a().__class__, C1) + self.assertEqual(a().__orig_class__, C1[[int], T]) # bpo-42195 with self.subTest("Testing collections.abc.Callable's consistency " diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7deba0d71b7c4f..a8458ba0042851 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1802,10 +1802,9 @@ def barfoo2(x: CT): ... def test_extended_generic_rules_subclassing(self): class T1(Tuple[T, KT]): ... class T2(Tuple[T, ...]): ... - class C1(Callable[[T], T]): ... - class C2(Callable[..., int]): - def __call__(self): - return None + class C1(typing.Container[T]): + def __contains__(self, item): + return False self.assertEqual(T1.__parameters__, (T, KT)) self.assertEqual(T1[int, str].__args__, (int, str)) @@ -1819,10 +1818,9 @@ def __call__(self): ## T2[int, str] self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') - self.assertEqual(C2.__parameters__, ()) - self.assertIsInstance(C2(), collections.abc.Callable) - self.assertIsSubclass(C2, collections.abc.Callable) - self.assertIsSubclass(C1, collections.abc.Callable) + self.assertEqual(C1.__parameters__, (T,)) + self.assertIsInstance(C1(), collections.abc.Container) + self.assertIsSubclass(C1, collections.abc.Container) self.assertIsInstance(T1(), tuple) self.assertIsSubclass(T2, tuple) with self.assertRaises(TypeError): @@ -1856,10 +1854,11 @@ def test_type_erasure_special(self): class MyTup(Tuple[T, T]): ... self.assertIs(MyTup[int]().__class__, MyTup) self.assertEqual(MyTup[int]().__orig_class__, MyTup[int]) - class MyCall(Callable[..., T]): - def __call__(self): return None - self.assertIs(MyCall[T]().__class__, MyCall) - self.assertEqual(MyCall[T]().__orig_class__, MyCall[T]) + # This isn't valid as Callable isn't a valid base class. + ## class MyCall(Callable[..., T]): + ## def __call__(self): return None + ## self.assertIs(MyCall[T]().__class__, MyCall) + ## self.assertEqual(MyCall[T]().__orig_class__, MyCall[T]) class MyDict(typing.Dict[T, T]): ... self.assertIs(MyDict[int]().__class__, MyDict) self.assertEqual(MyDict[int]().__orig_class__, MyDict[int]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst index df9bfa8ddcf508..087ddc35282187 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst @@ -2,5 +2,8 @@ The ``__args__`` of the parameterized generics for :data:`typing.Callable` and :class:`collections.abc.Callable` are now consistent. The ``__args__`` for :class:`collections.abc.Callable` are now flattened while :data:`typing.Callable`'s have not changed. To allow this change, -:class:`types.GenericAlias` can now be subclassed. Patch by Ken Jin. +:class:`types.GenericAlias` can now be subclassed and +``collections.abc.Callable``'s ``__class_getitem__`` will now return a subclass +of ``types.GenericAlias``. Tests for typing were also updated to not subclass +things like ``Callable[..., T]`` as that is not a valid base class. Patch by Ken Jin. From 37ae3a92ef5e53dcffa92dbbdd743708934c2327 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 5 Dec 2020 16:49:33 +0800 Subject: [PATCH 21/30] Borrowed references don't need decref --- Objects/genericaliasobject.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index a0dc0221c5633f..924ed7bc54feb0 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -616,11 +616,6 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); PyObject *self = (PyObject *)create_ga(type, origin, arguments); - if (self == NULL) { - Py_DECREF(origin); - Py_DECREF(arguments); - return NULL; - } return self; } From adbfcadbdcbe04be667312388c4a20869637380e Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 5 Dec 2020 17:36:30 +0800 Subject: [PATCH 22/30] improve _PyArg_NoKwnames error handling, add union and subclass tests --- Lib/test/test_genericalias.py | 11 +++++++++++ Lib/test/test_types.py | 12 +++++++----- Objects/unionobject.c | 4 ++-- Python/getargs.c | 5 ++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 7cd7067d23d7df..b77ce66020d9b4 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -301,6 +301,17 @@ def test_weakref(self): alias = t[int] self.assertEqual(ref(alias)(), alias) + def test_subclassing_types_genericalias(self): + class SubClass(GenericAlias): ... + alias = SubClass(list, int) + class Bad(GenericAlias): + def __new__(cls, *args, **kwargs): + super().__new__(cls, *args, **kwargs) + + self.assertEqual(alias, list[int]) + with self.assertRaises(SystemError): + Bad(list, int) + def test_abc_callable(self): # A separate test is needed for Callable since it uses a subclass of # GenericAlias. diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3058a02d6eeb4a..83196ad3c17436 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -717,14 +717,16 @@ def test_or_type_operator_with_genericalias(self): a = list[int] b = list[str] c = dict[float, str] + class SubClass(types.GenericAlias): ... + d = SubClass(list, float) # equivalence with typing.Union - self.assertEqual(a | b | c, typing.Union[a, b, c]) + self.assertEqual(a | b | c | d, typing.Union[a, b, c, d]) # de-duplicate - self.assertEqual(a | c | b | b | a | c, a | b | c) + self.assertEqual(a | c | b | b | a | c | d | d, a | b | c | d) # order shouldn't matter - self.assertEqual(a | b, b | a) - self.assertEqual(repr(a | b | c), - "list[int] | list[str] | dict[float, str]") + self.assertEqual(a | b | d, b | a | d) + self.assertEqual(repr(a | b | c | d), + "list[int] | list[str] | dict[float, str] | list[float]") class BadType(type): def __eq__(self, other): diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 7484df02dbd1af..32aa5078afcef4 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -237,8 +237,8 @@ dedup_and_flatten_args(PyObject* args) PyObject* i_element = PyTuple_GET_ITEM(args, i); for (Py_ssize_t j = i + 1; j < arg_length; j++) { PyObject* j_element = PyTuple_GET_ITEM(args, j); - int is_ga = Py_TYPE(i_element) == &Py_GenericAliasType && - Py_TYPE(j_element) == &Py_GenericAliasType; + int is_ga = PyObject_TypeCheck(i_element, &Py_GenericAliasType) && + PyObject_TypeCheck(j_element, &Py_GenericAliasType); // RichCompare to also deduplicate GenericAlias types (slower) is_duplicate = is_ga ? PyObject_RichCompareBool(i_element, j_element, Py_EQ) : i_element == j_element; diff --git a/Python/getargs.c b/Python/getargs.c index c85ff6d4777d2c..91b15dae0a4d8e 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2764,7 +2764,10 @@ _PyArg_NoKwnames(const char *funcname, PyObject *kwnames) return 1; } - assert(PyTuple_CheckExact(kwnames)); + if (!PyTuple_CheckExact(kwnames)) { + PyErr_BadInternalCall(); + return 0; + } if (PyTuple_GET_SIZE(kwnames) == 0) { return 1; From d1dd627b560fbb6c342cd59d3e23435736219589 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 5 Dec 2020 22:12:36 +0800 Subject: [PATCH 23/30] Don't change getargs, use _PyArg_NoKeywords instead --- Lib/test/test_genericalias.py | 4 ++-- Objects/genericaliasobject.c | 2 +- Python/getargs.c | 5 +---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index b77ce66020d9b4..d30f0560c426eb 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -309,8 +309,8 @@ def __new__(cls, *args, **kwargs): super().__new__(cls, *args, **kwargs) self.assertEqual(alias, list[int]) - with self.assertRaises(SystemError): - Bad(list, int) + with self.assertRaises(TypeError): + Bad(list, int, bad=int) def test_abc_callable(self): # A separate test is needed for Callable since it uses a subclass of diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 924ed7bc54feb0..ede88cbbc9ab01 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -607,7 +607,7 @@ static PyObject * ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { assert(type != NULL && type->tp_alloc != NULL); - if (!_PyArg_NoKwnames("GenericAlias", kwds)) { + if (!_PyArg_NoKeywords("GenericAlias", kwds)) { return NULL; } if (!_PyArg_CheckPositional("GenericAlias", PyTuple_GET_SIZE(args), 2, 2)) { diff --git a/Python/getargs.c b/Python/getargs.c index 91b15dae0a4d8e..c85ff6d4777d2c 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2764,10 +2764,7 @@ _PyArg_NoKwnames(const char *funcname, PyObject *kwnames) return 1; } - if (!PyTuple_CheckExact(kwnames)) { - PyErr_BadInternalCall(); - return 0; - } + assert(PyTuple_CheckExact(kwnames)); if (PyTuple_GET_SIZE(kwnames) == 0) { return 1; From 9f71667e787b4d180f7d867c9d69b2208da04686 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 6 Dec 2020 00:15:03 +0800 Subject: [PATCH 24/30] remove stray whitespace --- Lib/test/test_genericalias.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 76b6c05adc5ca6..5de13fe6d2f68c 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -300,7 +300,7 @@ def test_weakref(self): with self.subTest(f"Testing {tname}"): alias = t[int] self.assertEqual(ref(alias)(), alias) - + def test_no_kwargs(self): # bpo-42576 with self.assertRaises(TypeError): From a7896206ea64ee9ec781aa4a97c13cae3a71d6e9 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 6 Dec 2020 11:57:03 +0800 Subject: [PATCH 25/30] refactor C code, add deprecation warning for 3.9 --- Lib/_collections_abc.py | 18 ++++++++++-- Objects/genericaliasobject.c | 55 +++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 7bac6ff03ece7b..c94cd1c15a2b17 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -426,14 +426,28 @@ class _CallableGenericAlias(GenericAlias): __slots__ = () def __new__(cls, origin, args): + try: + return cls.__create_ga(origin, args) + except TypeError as exc: + if sys.version_info[:2] == (3, 9): + import warnings + warnings.warn(f'{str(exc)} ' + f'(This will raise TypeError in Python 3.10)', + DeprecationWarning) + return GenericAlias(origin, args) + else: + raise exc + + @classmethod + def __create_ga(cls, origin, args): if not isinstance(args, tuple) or len(args) != 2: raise TypeError( - "Callable must be used as Callable[[arg, ...], result]") + "Callable must be used as Callable[[arg, ...], result].") t_args, t_result = args if not isinstance(t_args, (list, EllipsisType)): raise TypeError( f"Callable[args, result]: args must be a list or Ellipsis. " - f"Got {_type_repr(t_args)}") + f"Got {_type_repr(t_args)} instead.") if t_args is Ellipsis: ga_args = args else: diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index ede88cbbc9ab01..4b3ea3791d4c19 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -564,43 +564,30 @@ static PyGetSetDef ga_properties[] = { {0} }; -static inline gaobject * -create_ga(PyTypeObject *type, PyObject *origin, PyObject *args) { +/* A helper function to create GenericAlias' args tuple and set its attributes. + * Returns 1 on success, 0 on failure. + */ +static inline int +setup_ga(gaobject *alias, PyObject *origin, PyObject *args) { + if (alias == NULL) { + return 0; + } if (!PyTuple_Check(args)) { args = PyTuple_Pack(1, args); if (args == NULL) { - return NULL; + return 0; } } else { Py_INCREF(args); } - gaobject *alias; - int gc_should_track = 0; - if (type != NULL) { - assert(type->tp_alloc != NULL); - alias = (gaobject *)type->tp_alloc(type, 0); - } - else { - alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); - gc_should_track = 1; - } - - if (alias == NULL) { - Py_DECREF(args); - return NULL; - } - Py_INCREF(origin); alias->origin = origin; alias->args = args; alias->parameters = NULL; alias->weakreflist = NULL; - if (gc_should_track) { - _PyObject_GC_TRACK(alias); - } - return alias; + return 1; } static PyObject * @@ -615,8 +602,15 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } PyObject *origin = PyTuple_GET_ITEM(args, 0); PyObject *arguments = PyTuple_GET_ITEM(args, 1); - PyObject *self = (PyObject *)create_ga(type, origin, arguments); - return self; + gaobject *self = (gaobject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + if (!setup_ga(self, origin, arguments)) { + type->tp_free(self); + return NULL; + } + return (PyObject *)self; } static PyNumberMethods ga_as_number = { @@ -656,5 +650,14 @@ PyTypeObject Py_GenericAliasType = { PyObject * Py_GenericAlias(PyObject *origin, PyObject *args) { - return (PyObject *)create_ga(NULL, origin, args); + gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); + if (alias == NULL) { + return NULL; + } + if (!setup_ga(alias, origin, args)) { + PyObject_GC_Del(alias); + return NULL; + } + _PyObject_GC_TRACK(alias); + return (PyObject *)alias; } From 1890b37f1b775250f1b3f2f37363e6669078e727 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 6 Dec 2020 14:08:09 +0800 Subject: [PATCH 26/30] remove redundant check in C code, and try except in __new__ --- Lib/_collections_abc.py | 12 +----------- Objects/genericaliasobject.c | 3 --- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c94cd1c15a2b17..d159767d64bf64 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -426,17 +426,7 @@ class _CallableGenericAlias(GenericAlias): __slots__ = () def __new__(cls, origin, args): - try: - return cls.__create_ga(origin, args) - except TypeError as exc: - if sys.version_info[:2] == (3, 9): - import warnings - warnings.warn(f'{str(exc)} ' - f'(This will raise TypeError in Python 3.10)', - DeprecationWarning) - return GenericAlias(origin, args) - else: - raise exc + return cls.__create_ga(origin, args) @classmethod def __create_ga(cls, origin, args): diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 4b3ea3791d4c19..7b74888db09c6d 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -569,9 +569,6 @@ static PyGetSetDef ga_properties[] = { */ static inline int setup_ga(gaobject *alias, PyObject *origin, PyObject *args) { - if (alias == NULL) { - return 0; - } if (!PyTuple_Check(args)) { args = PyTuple_Pack(1, args); if (args == NULL) { From 4e928c638bf07992dcac5737d34ecff491813c60 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 7 Dec 2020 12:31:48 +0800 Subject: [PATCH 27/30] remove check --- Objects/genericaliasobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 7b74888db09c6d..338cac08b9db96 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -590,7 +590,6 @@ setup_ga(gaobject *alias, PyObject *origin, PyObject *args) { static PyObject * ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - assert(type != NULL && type->tp_alloc != NULL); if (!_PyArg_NoKeywords("GenericAlias", kwds)) { return NULL; } From 6b11d33acb75af3229406a65155755b1c50b24ae Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 12 Dec 2020 00:49:00 +0800 Subject: [PATCH 28/30] Loosen type checks for Callable args, cast to PyObject in genericaliasobject.c --- Lib/_collections_abc.py | 13 ++++++------- Lib/test/test_typing.py | 19 +++++++++++-------- Lib/typing.py | 32 ++++++++++++++++++++------------ Objects/genericaliasobject.c | 4 ++-- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index d159767d64bf64..7c3faa64ea7f98 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -434,14 +434,13 @@ def __create_ga(cls, origin, args): raise TypeError( "Callable must be used as Callable[[arg, ...], result].") t_args, t_result = args - if not isinstance(t_args, (list, EllipsisType)): - raise TypeError( - f"Callable[args, result]: args must be a list or Ellipsis. " - f"Got {_type_repr(t_args)} instead.") - if t_args is Ellipsis: - ga_args = args - else: + if isinstance(t_args, list): ga_args = tuple(t_args) + (t_result,) + # This relaxes what t_args can be on purpose to allow things like + # PEP 612 ParamSpec. Responsibility for whether a user is using + # Callable[...] properly is deferred to static type checkers. + else: + ga_args = args return super().__new__(cls, origin, ga_args) def __repr__(self): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3dec0e77d079fa..b748eb52094df7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -446,14 +446,17 @@ def test_cannot_instantiate(self): type(c)() def test_callable_wrong_forms(self): - with self.assertRaises(TypeError): - Callable[[...], int] - with self.assertRaises(TypeError): - Callable[(), int] - with self.assertRaises(TypeError): - Callable[[()], int] - with self.assertRaises(TypeError): - Callable[[int, 1], 2] + ## These no longer raise TypeError since Callable no longer type checks + ## args. That responsiblity is deferred to static type checkers + ## to simplify implementation of PEP 612 ParamSpec. + # with self.assertRaises(TypeError): + # Callable[[...], int] + # with self.assertRaises(TypeError): + # Callable[(), int] + # with self.assertRaises(TypeError): + # Callable[[()], int] + # with self.assertRaises(TypeError): + # Callable[[int, 1], 2] with self.assertRaises(TypeError): Callable[int] diff --git a/Lib/typing.py b/Lib/typing.py index 46c54c406992f7..193e6c95177b0e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -120,6 +120,16 @@ # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. + +def _type_convert(arg): + """For converting None to type(None), and strings to ForwardRef.""" + if arg is None: + return type(None) + if isinstance(arg, str): + return ForwardRef(arg) + return arg + + def _type_check(arg, msg, is_argument=True): """Check that the argument is a type, and return it (internal helper). @@ -136,10 +146,7 @@ def _type_check(arg, msg, is_argument=True): if is_argument: invalid_generic_forms = invalid_generic_forms + (ClassVar, Final) - if arg is None: - return type(None) - if isinstance(arg, str): - return ForwardRef(arg) + arg = _type_convert(arg) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") @@ -900,13 +907,13 @@ def __getitem__(self, params): raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") args, result = params - if args is Ellipsis: - params = (Ellipsis, result) - else: - if not isinstance(args, list): - raise TypeError(f"Callable[args, result]: args must be a list." - f" Got {args}") + # This relaxes what args can be on purpose to allow things like + # PEP 612 ParamSpec. Responsibility for whether a user is using + # Callable[...] properly is deferred to static type checkers. + if isinstance(args, list): params = (tuple(args), result) + else: + params = (args, result) return self.__getitem_inner__(params) @_tp_cache @@ -916,8 +923,9 @@ def __getitem_inner__(self, params): result = _type_check(result, msg) if args is Ellipsis: return self.copy_with((_TypingEllipsis, result)) - msg = "Callable[[arg, ...], result]: each arg must be a type." - args = tuple(_type_check(arg, msg) for arg in args) + if not isinstance(args, tuple): + args = (args,) + args = tuple(_type_convert(arg) for arg in args) params = args + (result,) return self.copy_with(params) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 338cac08b9db96..756a7ce474aee9 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -603,7 +603,7 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } if (!setup_ga(self, origin, arguments)) { - type->tp_free(self); + type->tp_free((PyObject *)self); return NULL; } return (PyObject *)self; @@ -651,7 +651,7 @@ Py_GenericAlias(PyObject *origin, PyObject *args) return NULL; } if (!setup_ga(alias, origin, args)) { - PyObject_GC_Del(alias); + PyObject_GC_Del((PyObject *)alias); return NULL; } _PyObject_GC_TRACK(alias); From 4215c3b2f922a6892e5df87b8eb494a433e8d395 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 12 Dec 2020 01:02:23 +0800 Subject: [PATCH 29/30] update news to mention about removing validation in argtypes --- .../2020-11-20-00-57-47.bpo-42195.HeqcpS.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst index 087ddc35282187..ac52a008e352f5 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst @@ -5,5 +5,7 @@ for :class:`collections.abc.Callable` are now flattened while :class:`types.GenericAlias` can now be subclassed and ``collections.abc.Callable``'s ``__class_getitem__`` will now return a subclass of ``types.GenericAlias``. Tests for typing were also updated to not subclass -things like ``Callable[..., T]`` as that is not a valid base class. Patch by Ken Jin. +things like ``Callable[..., T]`` as that is not a valid base class. Finally, +both ``Callable``s no longer validate their ``argtypes``, in +``Callable[[argtypes], resulttype]`` to prepare for :pep:`612`. Patch by Ken Jin. From 585bf19168a2f293827b33c711f1875e394532a2 Mon Sep 17 00:00:00 2001 From: Fidget-Spinner <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 12 Dec 2020 12:39:52 +0800 Subject: [PATCH 30/30] remove commented out code --- Lib/test/test_typing.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b748eb52094df7..b4f5ac677aefbb 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -446,17 +446,6 @@ def test_cannot_instantiate(self): type(c)() def test_callable_wrong_forms(self): - ## These no longer raise TypeError since Callable no longer type checks - ## args. That responsiblity is deferred to static type checkers - ## to simplify implementation of PEP 612 ParamSpec. - # with self.assertRaises(TypeError): - # Callable[[...], int] - # with self.assertRaises(TypeError): - # Callable[(), int] - # with self.assertRaises(TypeError): - # Callable[[()], int] - # with self.assertRaises(TypeError): - # Callable[[int, 1], 2] with self.assertRaises(TypeError): Callable[int] @@ -1862,11 +1851,6 @@ def test_type_erasure_special(self): class MyTup(Tuple[T, T]): ... self.assertIs(MyTup[int]().__class__, MyTup) self.assertEqual(MyTup[int]().__orig_class__, MyTup[int]) - # This isn't valid as Callable isn't a valid base class. - ## class MyCall(Callable[..., T]): - ## def __call__(self): return None - ## self.assertIs(MyCall[T]().__class__, MyCall) - ## self.assertEqual(MyCall[T]().__orig_class__, MyCall[T]) class MyDict(typing.Dict[T, T]): ... self.assertIs(MyDict[int]().__class__, MyDict) self.assertEqual(MyDict[int]().__orig_class__, MyDict[int])