Skip to content

multiprocessing.managers: fix TypeVar usage #7938

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 26, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions stdlib/builtins.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -871,11 +871,16 @@ class list(MutableSequence[_T], Generic[_T]):
def extend(self, __iterable: Iterable[_T]) -> None: ...
def pop(self, __index: SupportsIndex = ...) -> _T: ...
# Signature of `list.index` should be kept in line with `collections.UserList.index()`
# and multiprocessing.managers.ListProxy.index()
def index(self, __value: _T, __start: SupportsIndex = ..., __stop: SupportsIndex = ...) -> int: ...
def count(self, __value: _T) -> int: ...
def insert(self, __index: SupportsIndex, __object: _T) -> None: ...
def remove(self, __value: _T) -> None: ...
# Signature of `list.sort` should be kept inline with `collections.UserList.sort()`
# and multiprocessing.managers.ListProxy.sort()
#
# Use list[SupportsRichComparisonT] for the first overload rather than [SupportsRichComparison]
# to work around invariance
@overload
def sort(self: list[SupportsRichComparisonT], *, key: None = ..., reverse: bool = ...) -> None: ...
@overload
Expand Down Expand Up @@ -908,6 +913,7 @@ class list(MutableSequence[_T], Generic[_T]):

class dict(MutableMapping[_KT, _VT], Generic[_KT, _VT]):
# __init__ should be kept roughly in line with `collections.UserDict.__init__`, which has similar semantics
# Also multiprocessing.managers.SyncManager.dict()
@overload
def __init__(self) -> None: ...
@overload
Expand Down
82 changes: 75 additions & 7 deletions stdlib/multiprocessing/managers.pyi
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import queue
import sys
import threading
from _typeshed import Self
from builtins import dict as _dict, list as _list # Conflicts with method names
from collections.abc import Callable, Iterable, Mapping, Sequence
from _typeshed import Self, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT
from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence
from types import TracebackType
from typing import Any, AnyStr, Generic, TypeVar
from typing_extensions import TypeAlias
from typing import Any, AnyStr, ClassVar, Generic, TypeVar, overload
from typing_extensions import SupportsIndex, TypeAlias

from .connection import Connection
from .context import BaseContext
Expand Down Expand Up @@ -66,6 +65,62 @@ class ValueProxy(BaseProxy, Generic[_T]):
if sys.version_info >= (3, 9):
def __class_getitem__(cls, item: Any) -> GenericAlias: ...

class DictProxy(BaseProxy, MutableMapping[_KT, _VT]):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also __contains__, clear, popitem, setdefault, update.

(Or are those inherited? In that case, why do we have __len__?)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All inherited from MutableMapping. MutableMapping.__len__ is an abstract method, so must be overridden here in order to make this class concrete.

__builtins__: ClassVar[dict[str, Any]]
def __len__(self) -> int: ...
def __getitem__(self, __k: _KT) -> _VT: ...
def __setitem__(self, __k: _KT, __v: _VT) -> None: ...
def __delitem__(self, __v: _KT) -> None: ...
def __iter__(self) -> Iterator[_KT]: ...
def copy(self) -> dict[_KT, _VT]: ...
@overload
def get(self, __key: _KT) -> _VT | None: ...
@overload
def get(self, __key: _KT, __default: _VT | _T) -> _VT | _T: ...
@overload
def pop(self, __key: _KT) -> _VT: ...
@overload
def pop(self, __key: _KT, __default: _VT | _T) -> _VT | _T: ...
def keys(self) -> list[_KT]: ... # type: ignore[override]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

def values(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override]
def items(self) -> list[_VT]: ... # type: ignore[override]
if sys.version_info < (3, 7):
def has_key(self, k: _KT) -> bool: ...

class BaseListProxy(BaseProxy, MutableSequence[_T]):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing __contains__ and reverse

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both inherited from MutableSequence

__builtins__: ClassVar[dict[str, Any]]
def __len__(self) -> int: ...
def __add__(self, __x: list[_T]) -> list[_T]: ...
def __delitem__(self, __i: SupportsIndex | slice) -> None: ...
@overload
def __getitem__(self, __i: SupportsIndex) -> _T: ...
@overload
def __getitem__(self, __s: slice) -> list[_T]: ...
@overload
def __setitem__(self, __i: SupportsIndex, __o: _T) -> None: ...
@overload
def __setitem__(self, __s: slice, __o: Iterable[_T]) -> None: ...
def __mul__(self, __n: SupportsIndex) -> list[_T]: ...
def __rmul__(self, __n: SupportsIndex) -> list[_T]: ...
def __reversed__(self) -> Iterator[_T]: ...
def append(self, __object: _T) -> None: ...
def extend(self, __iterable: Iterable[_T]) -> None: ...
def pop(self, __index: SupportsIndex = ...) -> _T: ...
def index(self, __value: _T, __start: SupportsIndex = ..., __stop: SupportsIndex = ...) -> int: ...
def count(self, __value: _T) -> int: ...
def insert(self, __index: SupportsIndex, __object: _T) -> None: ...
def remove(self, __value: _T) -> None: ...
# Use BaseListProxy[SupportsRichComparisonT] for the first overload rather than [SupportsRichComparison]
# to work around invariance
@overload
def sort(self: BaseListProxy[SupportsRichComparisonT], *, key: None = ..., reverse: bool = ...) -> None: ...
@overload
def sort(self, *, key: Callable[[_T], SupportsRichComparison], reverse: bool = ...) -> None: ...

class ListProxy(BaseListProxy[_T]):
def __iadd__(self: Self, __x: Iterable[_T]) -> Self: ... # type: ignore[override]
def __imul__(self: Self, __n: SupportsIndex) -> Self: ... # type: ignore[override]

# Returned by BaseManager.get_server()
class Server:
address: Any
Expand Down Expand Up @@ -124,8 +179,21 @@ class SyncManager(BaseManager):
def Semaphore(self, value: Any = ...) -> threading.Semaphore: ...
def Array(self, typecode: Any, sequence: Sequence[_T]) -> Sequence[_T]: ...
def Value(self, typecode: Any, value: _T) -> ValueProxy[_T]: ...
def dict(self, sequence: Mapping[_KT, _VT] = ...) -> _dict[_KT, _VT]: ...
def list(self, sequence: Sequence[_T] = ...) -> _list[_T]: ...
# Overloads are copied from builtins.dict.__init__
@overload
def dict(self) -> DictProxy[Any, Any]: ...
@overload
def dict(self, **kwargs: _VT) -> DictProxy[str, _VT]: ...
@overload
def dict(self, __map: SupportsKeysAndGetItem[_KT, _VT], **kwargs: _VT) -> DictProxy[_KT, _VT]: ...
@overload
def dict(self, __iterable: Iterable[tuple[_KT, _VT]], **kwargs: _VT) -> DictProxy[_KT, _VT]: ...
@overload
def dict(self, __iterable: Iterable[list[str]]) -> DictProxy[str, str]: ...
@overload
def list(self, __sequence: Sequence[_T]) -> ListProxy[_T]: ...
@overload
def list(self) -> ListProxy[Any]: ...

class RemoteError(Exception): ...

Expand Down
16 changes: 16 additions & 0 deletions tests/stubtest_allowlists/py3_common.txt
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,22 @@ builtins.super.__self__
builtins.super.__self_class__
builtins.super.__thisclass__

# These multiprocessing proxy methods have *args, **kwargs signatures at runtime,
# But have more precise (accurate) signatures in the stub
multiprocessing.managers.BaseListProxy.__imul__
multiprocessing.managers.BaseListProxy.__len__
multiprocessing.managers.BaseListProxy.__reversed__
multiprocessing.managers.BaseListProxy.reverse
multiprocessing.managers.BaseListProxy.sort
multiprocessing.managers.DictProxy.__iter__
multiprocessing.managers.DictProxy.__len__
multiprocessing.managers.DictProxy.clear
multiprocessing.managers.DictProxy.copy
multiprocessing.managers.DictProxy.items
multiprocessing.managers.DictProxy.keys
multiprocessing.managers.DictProxy.popitem
multiprocessing.managers.DictProxy.values

# GetSetDescriptor that always raises AttributeError
builtins.OSError.characters_written
# Aliases for OSError
Expand Down