From 31962f36616c29da4b5d5d7c05a846e111d1032a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 24 May 2022 19:38:26 +0100 Subject: [PATCH 1/6] `multiprocessing.managers`: fix `TypeVar` usage #7928 `dict()` and `list()` just return empty dictionaries and lists (respectively) if no arguments are supplied: ```python >>> from multiprocessing.managers import SyncManager >>> with SyncManager() as s: ... print(s.dict()) ... {} ``` --- stdlib/multiprocessing/managers.pyi | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/stdlib/multiprocessing/managers.pyi b/stdlib/multiprocessing/managers.pyi index b8d5ddda0f35..e875b16e364b 100644 --- a/stdlib/multiprocessing/managers.pyi +++ b/stdlib/multiprocessing/managers.pyi @@ -5,7 +5,7 @@ from _typeshed import Self from builtins import dict as _dict, list as _list # Conflicts with method names from collections.abc import Callable, Iterable, Mapping, Sequence from types import TracebackType -from typing import Any, AnyStr, Generic, TypeVar +from typing import Any, AnyStr, Generic, TypeVar, overload from typing_extensions import TypeAlias from .connection import Connection @@ -124,8 +124,14 @@ class SyncManager(BaseManager): def Semaphore(self, value: Any = ...) -> threading.Semaphore: ... def Array(self, typecode: Any, sequence: Sequence[_T]) -> Sequence[_T]: ... def Value(self, typecode: Any, value: _T) -> ValueProxy[_T]: ... - def dict(self, sequence: Mapping[_KT, _VT] = ...) -> _dict[_KT, _VT]: ... - def list(self, sequence: Sequence[_T] = ...) -> _list[_T]: ... + @overload + def dict(self, sequence: Mapping[_KT, _VT]) -> _dict[_KT, _VT]: ... + @overload + def dict(self) -> _dict[Any, Any]: ... + @overload + def list(self, sequence: Sequence[_T]) -> _list[_T]: ... + @overload + def list(self) -> _list[Any]: ... class RemoteError(Exception): ... From 503cbfe59802f820c031bb69cb206fcc1471557b Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 25 May 2022 13:36:02 +0100 Subject: [PATCH 2/6] Substantially revise --- stdlib/builtins.pyi | 5 +- stdlib/multiprocessing/managers.pyi | 76 +++++++++++++++++++++--- tests/stubtest_allowlists/py3_common.txt | 16 +++++ 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/stdlib/builtins.pyi b/stdlib/builtins.pyi index 53ba2a8a2a94..f83b04d9cf21 100644 --- a/stdlib/builtins.pyi +++ b/stdlib/builtins.pyi @@ -871,13 +871,15 @@ class list(MutableSequence[_T], Generic[_T]): def extend(self, __iterable: Iterable[_T]) -> None: ... def pop(self, __index: SupportsIndex = ...) -> _T: ... # Signature of `list.index` should be kept in line with `collections.UserList.index()` + # and multiprocessing.managers.ListProxy.index() def index(self, __value: _T, __start: SupportsIndex = ..., __stop: SupportsIndex = ...) -> int: ... def count(self, __value: _T) -> int: ... def insert(self, __index: SupportsIndex, __object: _T) -> None: ... def remove(self, __value: _T) -> None: ... # Signature of `list.sort` should be kept inline with `collections.UserList.sort()` + # and multiprocessing.managers.ListProxy.sort() @overload - def sort(self: list[SupportsRichComparisonT], *, key: None = ..., reverse: bool = ...) -> None: ... + def sort(self: list[SupportsRichComparison], *, key: None = ..., reverse: bool = ...) -> None: ... @overload def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = ...) -> None: ... def __len__(self) -> int: ... @@ -908,6 +910,7 @@ class list(MutableSequence[_T], Generic[_T]): class dict(MutableMapping[_KT, _VT], Generic[_KT, _VT]): # __init__ should be kept roughly in line with `collections.UserDict.__init__`, which has similar semantics + # Also multiprocessing.managers.SyncManager.dict() @overload def __init__(self: dict[_KT, _VT]) -> None: ... @overload diff --git a/stdlib/multiprocessing/managers.pyi b/stdlib/multiprocessing/managers.pyi index e875b16e364b..8639d79e9ac4 100644 --- a/stdlib/multiprocessing/managers.pyi +++ b/stdlib/multiprocessing/managers.pyi @@ -1,12 +1,11 @@ import queue import sys import threading -from _typeshed import Self -from builtins import dict as _dict, list as _list # Conflicts with method names -from collections.abc import Callable, Iterable, Mapping, Sequence +from _typeshed import Self, SupportsKeysAndGetItem, SupportsRichComparison +from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence from types import TracebackType -from typing import Any, AnyStr, Generic, TypeVar, overload -from typing_extensions import TypeAlias +from typing import Any, AnyStr, ClassVar, Generic, TypeVar, overload +from typing_extensions import SupportsIndex, TypeAlias from .connection import Connection from .context import BaseContext @@ -66,6 +65,58 @@ class ValueProxy(BaseProxy, Generic[_T]): if sys.version_info >= (3, 9): def __class_getitem__(cls, item: Any) -> GenericAlias: ... +class DictProxy(BaseProxy, MutableMapping[_KT, _VT]): + __builtins__: ClassVar[dict[str, Any]] + def __len__(self) -> int: ... + def __getitem__(self, __k: _KT) -> _VT: ... + def __setitem__(self, __k: _KT, __v: _VT) -> None: ... + def __delitem__(self, __v: _KT) -> None: ... + def __iter__(self) -> Iterator[_KT]: ... + def copy(self) -> dict[_KT, _VT]: ... + @overload + def get(self, __key: _KT) -> _VT | None: ... + @overload + def get(self, __key: _KT, __default: _VT | _T) -> _VT | _T: ... + @overload + def pop(self, __key: _KT) -> _VT: ... + @overload + def pop(self, __key: _KT, __default: _VT | _T) -> _VT | _T: ... + def keys(self) -> list[_KT]: ... # type: ignore[override] + def values(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override] + def items(self) -> list[_VT]: ... # type: ignore[override] + +class BaseListProxy(BaseProxy, MutableSequence[_T]): + __builtins__: ClassVar[dict[str, Any]] + def __len__(self) -> int: ... + def __add__(self, __x: list[_T]) -> list[_T]: ... + def __delitem__(self, __i: SupportsIndex | slice) -> None: ... + @overload + def __getitem__(self, __i: SupportsIndex) -> _T: ... + @overload + def __getitem__(self, __s: slice) -> list[_T]: ... + @overload + def __setitem__(self, __i: SupportsIndex, __o: _T) -> None: ... + @overload + def __setitem__(self, __s: slice, __o: Iterable[_T]) -> None: ... + def __mul__(self, __n: SupportsIndex) -> list[_T]: ... + def __rmul__(self, __n: SupportsIndex) -> list[_T]: ... + def __reversed__(self) -> Iterator[_T]: ... + def append(self, __object: _T) -> None: ... + def extend(self, __iterable: Iterable[_T]) -> None: ... + def pop(self, __index: SupportsIndex = ...) -> _T: ... + def index(self, __value: _T, __start: SupportsIndex = ..., __stop: SupportsIndex = ...) -> int: ... + def count(self, __value: _T) -> int: ... + def insert(self, __index: SupportsIndex, __object: _T) -> None: ... + def remove(self, __value: _T) -> None: ... + @overload + def sort(self: BaseListProxy[SupportsRichComparison], *, key: None = ..., reverse: bool = ...) -> None: ... + @overload + def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = ...) -> None: ... + +class ListProxy(BaseListProxy[_T]): + def __iadd__(self: Self, __x: Iterable[_T]) -> Self: ... # type: ignore[override] + def __imul__(self: Self, __n: SupportsIndex) -> Self: ... # type: ignore[override] + # Returned by BaseManager.get_server() class Server: address: Any @@ -124,14 +175,21 @@ class SyncManager(BaseManager): def Semaphore(self, value: Any = ...) -> threading.Semaphore: ... def Array(self, typecode: Any, sequence: Sequence[_T]) -> Sequence[_T]: ... def Value(self, typecode: Any, value: _T) -> ValueProxy[_T]: ... + # Overloads are copied from builtins.dict.__init__ + @overload + def dict(self) -> DictProxy[Any, Any]: ... + @overload + def dict(self, **kwargs: _VT) -> DictProxy[str, _VT]: ... + @overload + def dict(self, __map: SupportsKeysAndGetItem[_KT, _VT], **kwargs: _VT) -> DictProxy[_KT, _VT]: ... @overload - def dict(self, sequence: Mapping[_KT, _VT]) -> _dict[_KT, _VT]: ... + def dict(self, __iterable: Iterable[tuple[_KT, _VT]], **kwargs: _VT) -> DictProxy[_KT, _VT]: ... @overload - def dict(self) -> _dict[Any, Any]: ... + def dict(self, __iterable: Iterable[list[str]]) -> DictProxy[str, str]: ... @overload - def list(self, sequence: Sequence[_T]) -> _list[_T]: ... + def list(self, sequence: Sequence[_T]) -> ListProxy[_T]: ... @overload - def list(self) -> _list[Any]: ... + def list(self) -> ListProxy[Any]: ... class RemoteError(Exception): ... diff --git a/tests/stubtest_allowlists/py3_common.txt b/tests/stubtest_allowlists/py3_common.txt index 2521c2f698e5..d5232e63cddd 100644 --- a/tests/stubtest_allowlists/py3_common.txt +++ b/tests/stubtest_allowlists/py3_common.txt @@ -384,6 +384,22 @@ builtins.super.__self__ builtins.super.__self_class__ builtins.super.__thisclass__ +# These multiprocessing proxy methods have *args, **kwargs signatures at runtime, +# But have more precise (accurate) signatures in the stub +multiprocessing.managers.BaseListProxy.__imul__ +multiprocessing.managers.BaseListProxy.__len__ +multiprocessing.managers.BaseListProxy.__reversed__ +multiprocessing.managers.BaseListProxy.reverse +multiprocessing.managers.BaseListProxy.sort +multiprocessing.managers.DictProxy.__iter__ +multiprocessing.managers.DictProxy.__len__ +multiprocessing.managers.DictProxy.clear +multiprocessing.managers.DictProxy.copy +multiprocessing.managers.DictProxy.items +multiprocessing.managers.DictProxy.keys +multiprocessing.managers.DictProxy.popitem +multiprocessing.managers.DictProxy.values + # GetSetDescriptor that always raises AttributeError builtins.OSError.characters_written # Aliases for OSError From 1b110c34190c963f55f8e78cdf01ade2a91d37ed Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 25 May 2022 13:48:21 +0100 Subject: [PATCH 3/6] Weird Python 2 holdover --- stdlib/multiprocessing/managers.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/multiprocessing/managers.pyi b/stdlib/multiprocessing/managers.pyi index 8639d79e9ac4..d1b5d57e7398 100644 --- a/stdlib/multiprocessing/managers.pyi +++ b/stdlib/multiprocessing/managers.pyi @@ -84,6 +84,8 @@ class DictProxy(BaseProxy, MutableMapping[_KT, _VT]): def keys(self) -> list[_KT]: ... # type: ignore[override] def values(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override] def items(self) -> list[_VT]: ... # type: ignore[override] + if sys.version_info < (3, 7): + def has_key(self, k: _KT) -> bool: ... class BaseListProxy(BaseProxy, MutableSequence[_T]): __builtins__: ClassVar[dict[str, Any]] From 359b6d27105534c44ce35fc23ed3611e31cae6f0 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 25 May 2022 13:51:27 +0100 Subject: [PATCH 4/6] Invariance strikes again --- stdlib/builtins.pyi | 5 ++++- stdlib/multiprocessing/managers.pyi | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/stdlib/builtins.pyi b/stdlib/builtins.pyi index f83b04d9cf21..f528d9ed1847 100644 --- a/stdlib/builtins.pyi +++ b/stdlib/builtins.pyi @@ -878,8 +878,11 @@ class list(MutableSequence[_T], Generic[_T]): def remove(self, __value: _T) -> None: ... # Signature of `list.sort` should be kept inline with `collections.UserList.sort()` # and multiprocessing.managers.ListProxy.sort() + # + # Use list[SupportsRichComparisonT] for the first overload rather than [SupportsRichComparison] + # to work around invariance @overload - def sort(self: list[SupportsRichComparison], *, key: None = ..., reverse: bool = ...) -> None: ... + def sort(self: list[SupportsRichComparisonT], *, key: None = ..., reverse: bool = ...) -> None: ... @overload def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = ...) -> None: ... def __len__(self) -> int: ... diff --git a/stdlib/multiprocessing/managers.pyi b/stdlib/multiprocessing/managers.pyi index d1b5d57e7398..61c53b062aae 100644 --- a/stdlib/multiprocessing/managers.pyi +++ b/stdlib/multiprocessing/managers.pyi @@ -1,7 +1,7 @@ import queue import sys import threading -from _typeshed import Self, SupportsKeysAndGetItem, SupportsRichComparison +from _typeshed import Self, SupportsKeysAndGetItem, SupportsRichComparisonT from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence from types import TracebackType from typing import Any, AnyStr, ClassVar, Generic, TypeVar, overload @@ -110,8 +110,10 @@ class BaseListProxy(BaseProxy, MutableSequence[_T]): def count(self, __value: _T) -> int: ... def insert(self, __index: SupportsIndex, __object: _T) -> None: ... def remove(self, __value: _T) -> None: ... + # Use BaseListProxy[SupportsRichComparisonT] for the first overload rather than [SupportsRichComparison] + # to work around invariance @overload - def sort(self: BaseListProxy[SupportsRichComparison], *, key: None = ..., reverse: bool = ...) -> None: ... + def sort(self: BaseListProxy[SupportsRichComparisonT], *, key: None = ..., reverse: bool = ...) -> None: ... @overload def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = ...) -> None: ... From 908a130ab06bde309134fb340134ce55c75597de Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 25 May 2022 13:53:43 +0100 Subject: [PATCH 5/6] Missing import --- stdlib/multiprocessing/managers.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/multiprocessing/managers.pyi b/stdlib/multiprocessing/managers.pyi index 61c53b062aae..0d59114f4622 100644 --- a/stdlib/multiprocessing/managers.pyi +++ b/stdlib/multiprocessing/managers.pyi @@ -1,7 +1,7 @@ import queue import sys import threading -from _typeshed import Self, SupportsKeysAndGetItem, SupportsRichComparisonT +from _typeshed import Self, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence from types import TracebackType from typing import Any, AnyStr, ClassVar, Generic, TypeVar, overload From 3ef32112672b4e0f838c7e819c207741bec607d9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 26 May 2022 07:32:16 +0100 Subject: [PATCH 6/6] Update stdlib/multiprocessing/managers.pyi Co-authored-by: Jelle Zijlstra --- stdlib/multiprocessing/managers.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/multiprocessing/managers.pyi b/stdlib/multiprocessing/managers.pyi index 0d59114f4622..212ffcbf5a3a 100644 --- a/stdlib/multiprocessing/managers.pyi +++ b/stdlib/multiprocessing/managers.pyi @@ -191,7 +191,7 @@ class SyncManager(BaseManager): @overload def dict(self, __iterable: Iterable[list[str]]) -> DictProxy[str, str]: ... @overload - def list(self, sequence: Sequence[_T]) -> ListProxy[_T]: ... + def list(self, __sequence: Sequence[_T]) -> ListProxy[_T]: ... @overload def list(self) -> ListProxy[Any]: ...