From 884481d782e83c237204600a0e9a9e271c848f18 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 26 Mar 2017 14:20:52 +0200 Subject: [PATCH 001/117] Recognize protocols in semanal.py --- mypy/nodes.py | 3 ++ mypy/semanal.py | 58 ++++++++++++++++++++++++++---- test-data/unit/check-generics.test | 4 +-- test-data/unit/lib-stub/typing.pyi | 31 +++++++++++----- test-data/unit/semanal-errors.test | 2 +- 5 files changed, 79 insertions(+), 19 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 41fa7d196338..ffdb543fd6c3 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1909,7 +1909,10 @@ class is generic then it will be a type constructor of higher kind. subtypes = None # type: Set[TypeInfo] # Direct subclasses encountered so far names = None # type: SymbolTable # Names defined directly in this type is_abstract = False # Does the class have any abstract attributes? + is_protocol = False # Is this a protocol class? + runtime_protocol = False # Does this protocol support isinstance checks? abstract_attributes = None # type: List[str] + protocol_members = None # type: List[str] # Classes inheriting from Enum shadow their true members with a __getattr__, so we # have to treat them as a special case. is_enum = False diff --git a/mypy/semanal.py b/mypy/semanal.py index f8ff94dec623..23e5a5726428 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -588,6 +588,7 @@ def check_function_signature(self, fdef: FuncItem) -> None: self.fail('Type signature has too many arguments', fdef, blocker=True) def visit_class_def(self, defn: ClassDef) -> None: + is_protocol = self.detect_protocol_base(defn) self.clean_up_bases_and_infer_type_variables(defn) if self.analyze_typeddict_classdef(defn): return @@ -604,8 +605,10 @@ def visit_class_def(self, defn: ClassDef) -> None: self.analyze_base_classes(defn) self.analyze_metaclass(defn) + runtime_protocol = False for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) + if self.analyze_class_decorator(defn, decorator): + runtime_protocol = True self.enter_class(defn) @@ -613,6 +616,8 @@ def visit_class_def(self, defn: ClassDef) -> None: defn.defs.accept(self) self.calculate_abstract_status(defn.info) + defn.info.is_protocol = is_protocol + defn.info.runtime_protocol = runtime_protocol self.setup_type_promotion(defn) self.leave_class() @@ -652,8 +657,9 @@ def unbind_class_type_vars(self) -> None: if self.bound_tvars: enable_typevars(self.bound_tvars) - def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None: + def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> bool: decorator.accept(self) + return isinstance(decorator, RefExpr) and decorator.fullname == 'typing.runtime' def calculate_abstract_status(self, typ: TypeInfo) -> None: """Calculate abstract status of a class. @@ -701,12 +707,26 @@ def setup_type_promotion(self, defn: ClassDef) -> None: promote_target = self.named_type_or_none(promotions[defn.fullname]) defn.info._promote = promote_target + def detect_protocol_base(self, defn: ClassDef) -> bool: + for base_expr in defn.base_type_exprs: + try: + base = expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + continue # This will be reported later + if not isinstance(base, UnboundType): + return False + sym = self.lookup_qualified(base.name, base) + if sym is None or sym.node is None: + return False + return sym.node.fullname() == 'typing.Protocol' + return False + def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: """Remove extra base classes such as Generic and infer type vars. For example, consider this class: - . class Foo(Bar, Generic[T]): ... + class Foo(Bar, Generic[T]): ... Now we will remove Generic[T] from bases of Foo and infer that the type variable 'T' is a type argument of Foo. @@ -725,17 +745,24 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: tvars = self.analyze_typevar_declaration(base) if tvars is not None: if declared_tvars: - self.fail('Duplicate Generic in bases', defn) + self.fail('Only single Generic[...] or Protocol[...] can be in bases', defn) removed.append(i) declared_tvars.extend(tvars) + if isinstance(base, UnboundType): + sym = self.lookup_qualified(base.name, base) + if sym is not None and sym.node is not None: + if sym.node.fullname() == 'typing.Protocol' and i not in removed: + # also remove bare 'Protocol' bases + removed.append(i) all_tvars = self.get_all_bases_tvars(defn, removed) if declared_tvars: if len(self.remove_dups(declared_tvars)) < len(declared_tvars): - self.fail("Duplicate type variables in Generic[...]", defn) + self.fail("Duplicate type variables in Generic[...] or Protocol[...]", defn) declared_tvars = self.remove_dups(declared_tvars) if not set(all_tvars).issubset(set(declared_tvars)): - self.fail("If Generic[...] is present it should list all type variables", defn) + self.fail("If Generic[...] or Protocol[...] is present" + " it should list all type variables", defn) # In case of error, Generic tvars will go first declared_tvars = self.remove_dups(declared_tvars + all_tvars) else: @@ -757,7 +784,8 @@ def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeV sym = self.lookup_qualified(unbound.name, unbound) if sym is None or sym.node is None: return None - if sym.node.fullname() == 'typing.Generic': + if (sym.node.fullname() == 'typing.Generic' or + sym.node.fullname() == 'typing.Protocol' and t.args): tvars = [] # type: List[Tuple[str, TypeVarExpr]] for arg in unbound.args: tvar = self.analyze_unbound_tvar(arg) @@ -3405,12 +3433,18 @@ def visit_func_def(self, fdef: FuncDef) -> None: def visit_class_def(self, tdef: ClassDef) -> None: for type in tdef.info.bases: self.analyze(type) + if tdef.info.is_protocol: + if not isinstance(type, Instance) or not type.type.is_protocol: + if type.type.fullname() != 'builtins.object': + self.fail('All bases of a protocol must be protocols', tdef) # Recompute MRO now that we have analyzed all modules, to pick # up superclasses of bases imported from other modules in an # import loop. (Only do so if we succeeded the first time.) if tdef.info.mro: tdef.info.mro = [] # Force recomputation calculate_class_mro(tdef, self.fail_blocker) + if tdef.info.is_protocol: + add_protocol_members(tdef.info) super().visit_class_def(tdef) def visit_decorator(self, dec: Decorator) -> None: @@ -3501,6 +3535,16 @@ def builtin_type(self, name: str, args: List[Type] = None) -> Instance: return Instance(sym.node, args or []) +def add_protocol_members(typ: TypeInfo) -> None: + members = set() # type: Set[str] + if typ.mro: + for base in typ.mro[:-1]: # we skip "object" since everyone implements it + if base.is_protocol: + for name in base.names: + members.add(name) + typ.protocol_members = list(members) + + def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: if isinstance(sig, CallableType): return sig.copy_modified(arg_types=[new] + sig.arg_types[1:]) diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index e1efd0a3f559..418a8f558551 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -1080,7 +1080,7 @@ reveal_type(D[str, int]().c()) # E: Revealed type is 'builtins.str*' from typing import TypeVar, Generic T = TypeVar('T') -class A(Generic[T, T]): # E: Duplicate type variables in Generic[...] +class A(Generic[T, T]): # E: Duplicate type variables in Generic[...] or Protocol[...] pass a = A[int]() @@ -1097,7 +1097,7 @@ class A(Generic[T]): class B(Generic[T]): pass -class C(A[T], B[S], Generic[T]): # E: If Generic[...] is present it should list all type variables +class C(A[T], B[S], Generic[T]): # E: If Generic[...] or Protocol[...] is present it should list all type variables pass c = C[int, str]() diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 8d252dbddf2e..1e75564c66f9 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -18,6 +18,7 @@ NamedTuple = 0 Type = 0 no_type_check = 0 ClassVar = 0 +Protocol = 0 # Type aliases. List = 0 @@ -29,20 +30,24 @@ U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') -class Container(Generic[T]): +@runtime +class Container(Protocol[T]): @abstractmethod # Use int because bool isn't in the default test builtins def __contains__(self, arg: T) -> int: pass -class Sized: +@runtime +class Sized(Protocol): @abstractmethod def __len__(self) -> int: pass -class Iterable(Generic[T]): +@runtime +class Iterable(Protocol[T]): @abstractmethod def __iter__(self) -> 'Iterator[T]': pass -class Iterator(Iterable[T], Generic[T]): +@runtime +class Iterator(Iterable[T], Protocol): @abstractmethod def __next__(self) -> T: pass @@ -75,14 +80,16 @@ class AsyncGenerator(AsyncIterator[T], Generic[T, U]): @abstractmethod def __aiter__(self) -> 'AsyncGenerator[T, U]': pass -class Awaitable(Generic[T]): +@runtime +class Awaitable(Protocol[T]): @abstractmethod def __await__(self) -> Generator[Any, Any, T]: pass class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S]): pass -class AsyncIterable(Generic[T]): +@runtime +class AsyncIterable(Protocol[T]): @abstractmethod def __aiter__(self) -> 'AsyncIterator[T]': pass @@ -91,17 +98,23 @@ class AsyncIterator(AsyncIterable[T], Generic[T]): @abstractmethod def __anext__(self) -> Awaitable[T]: pass -class Sequence(Iterable[T], Generic[T]): +@runtime +class Sequence(Iterable[T], Protocol): @abstractmethod def __getitem__(self, n: Any) -> T: pass -class Mapping(Generic[T, U]): pass +@runtime +class Mapping(Protocol[T, U]): pass -class MutableMapping(Generic[T, U]): pass +@runtime +class MutableMapping(Protocol[T, U]): pass def NewType(name: str, tp: Type[T]) -> Callable[[T], T]: def new_type(x): return x return new_type +def runtime(cls: T) -> T: + return cls + TYPE_CHECKING = 1 diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index aa3778a26d75..ac4b691a0e2c 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -951,7 +951,7 @@ from typing import Generic, TypeVar T = TypeVar('T') S = TypeVar('S') class A(Generic[T], Generic[S]): pass \ - # E: Duplicate Generic in bases + # E: Only single Generic[...] or Protocol[...] can be in bases [out] [case testInvalidMetaclass] From c19b6d4668874aa9cffcbd72e4c595c6de339b41 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 27 Mar 2017 01:36:44 +0200 Subject: [PATCH 002/117] Add issubtype and basic constraints --- mypy/checkexpr.py | 7 +- mypy/constraints.py | 7 ++ mypy/semanal.py | 8 +- mypy/subtypes.py | 87 ++++++++++++++++++++- test-data/unit/check-async-await.test | 18 ++--- test-data/unit/check-classes.test | 14 ++-- test-data/unit/check-typeddict.test | 4 +- test-data/unit/fixtures/dict.pyi | 1 + test-data/unit/lib-stub/mypy_extensions.pyi | 1 + test-data/unit/lib-stub/typing.pyi | 3 +- 10 files changed, 122 insertions(+), 28 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ba598f166a81..4bfcc3e2835a 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2110,8 +2110,9 @@ def check_awaitable_expr(self, t: Type, ctx: Context, msg: str) -> Type: Also used by `async for` and `async with`. """ - if not self.chk.check_subtype(t, self.named_type('typing.Awaitable'), ctx, - msg, 'actual type', 'expected type'): + if not self.chk.check_subtype(t, self.chk.named_generic_type('typing.Awaitable', + [AnyType()]), + ctx, msg, 'actual type', 'expected type'): return AnyType() else: method = self.analyze_external_member_access('__await__', t, ctx) @@ -2432,6 +2433,8 @@ def overload_arg_similarity(actual: Type, formal: Type) -> int: # subtyping algorithm if type promotions are possible (e.g., int vs. float). if formal.type in actual.type.mro: return 2 + elif formal.type.is_protocol and is_subtype(actual, formal): + return 2 elif actual.type._promote and is_subtype(actual, formal): return 1 else: diff --git a/mypy/constraints.py b/mypy/constraints.py index d6e44bea857d..bbad72a0a674 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -314,6 +314,13 @@ def visit_instance(self, template: Instance) -> List[Constraint]: actual = actual.fallback if isinstance(actual, Instance): instance = actual + if template.type.is_protocol and self.direction == SUPERTYPE_OF: + for member in template.type.protocol_members: + inst = mypy.subtypes.find_member(member, instance) + temp = mypy.subtypes.find_member(member, template) + res.extend(infer_constraints(temp, inst, self.direction)) + res.extend(infer_constraints(temp, inst, neg_op(self.direction))) + return res if (self.direction == SUBTYPE_OF and template.type.has_base(instance.type.fullname())): mapped = map_instance_to_supertype(template, instance.type) diff --git a/mypy/semanal.py b/mypy/semanal.py index 23e5a5726428..c40fdb590cf6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -718,7 +718,7 @@ def detect_protocol_base(self, defn: ClassDef) -> bool: sym = self.lookup_qualified(base.name, base) if sym is None or sym.node is None: return False - return sym.node.fullname() == 'typing.Protocol' + return sym.node.fullname() in ('typing.Protocol', 'mypy_extensions.Protocol') return False def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: @@ -751,7 +751,8 @@ class Foo(Bar, Generic[T]): ... if isinstance(base, UnboundType): sym = self.lookup_qualified(base.name, base) if sym is not None and sym.node is not None: - if sym.node.fullname() == 'typing.Protocol' and i not in removed: + if (sym.node.fullname() in ('typing.Protocol', 'mypy_extensions.Protocol') and + i not in removed): # also remove bare 'Protocol' bases removed.append(i) @@ -785,7 +786,8 @@ def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeV if sym is None or sym.node is None: return None if (sym.node.fullname() == 'typing.Generic' or - sym.node.fullname() == 'typing.Protocol' and t.args): + sym.node.fullname() == 'typing.Protocol' and t.args or + sym.node.fullname() == 'mypy_extensions.Protocol' and t.args): tvars = [] # type: List[Tuple[str, TypeVarExpr]] for arg in unbound.args: tvar = self.analyze_unbound_tvar(arg) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index c98bb8e8d144..25af4135bf04 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,9 +1,10 @@ -from typing import List, Optional, Dict, Callable +from typing import List, Optional, Dict, Callable, Tuple from mypy.types import ( - Type, AnyType, UnboundType, TypeVisitor, ErrorType, FormalArgument, NoneTyp, + Type, AnyType, UnboundType, TypeVisitor, ErrorType, FormalArgument, NoneTyp, function_type, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, - ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance + ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance, + FunctionLike, ) import mypy.applytype import mypy.constraints @@ -11,10 +12,11 @@ # import mypy.solve from mypy import messages, sametypes from mypy.nodes import ( - CONTRAVARIANT, COVARIANT, + CONTRAVARIANT, COVARIANT, FuncBase, Var, Decorator, OverloadedFuncDef, ARG_POS, ARG_OPT, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, ) from mypy.maptype import map_instance_to_supertype +from mypy.expandtype import expand_type_by_instance from mypy import experiments @@ -60,6 +62,8 @@ def is_subtype(left: Type, right: Type, else: return left.accept(SubtypeVisitor(right, type_parameter_checker, ignore_pos_arg_names=ignore_pos_arg_names)) + # print(left, right, res) + # return res def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: @@ -131,6 +135,8 @@ def visit_instance(self, left: Instance) -> bool: ignore_pos_arg_names=self.ignore_pos_arg_names): return True rname = right.type.fullname() + if right.type.is_protocol: + return is_protocol_implementation(left, right) if not left.type.has_base(rname) and rname != 'builtins.object': return False @@ -278,6 +284,79 @@ def visit_type_type(self, left: TypeType) -> bool: return False +ASSUMING = [] # type: List[Tuple[Instance, Instance]] + + +def is_protocol_implementation(left: Instance, right: Instance) -> bool: + global ASSUMING + assert right.type.is_protocol + for (l, r) in ASSUMING: + if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): + return True + ASSUMING.append((left, right)) + for member in right.type.protocol_members: + supertype = find_member(member, right) + subtype = find_member(member, left) + # print(member, subtype, supertype) + if not subtype or not is_subtype(subtype, supertype): + ASSUMING.pop() + return False + return True + + +def find_member(name: str, itype: Instance) -> Optional[Type]: + info = itype.type + method = info.get_method(name) + if method: + if method.is_property: + assert isinstance(method, OverloadedFuncDef) + return find_var_type(method.items[0].var, itype) + return map_method(method, itype) + else: + node = info.get(name) + if not node: + return None + v = node.node + if isinstance(v, Decorator): + v = v.var + if isinstance(v, Var): + return find_var_type(v, itype) + if not v and name not in ['__getattr__', '__setattr__', '__getattribute__']: + for method_name in ('__getattribute__', '__getattr__'): + method = info.get_method(method_name) + if method and method.info.fullname() != 'builtins.object': + getattr_type = map_method(method, itype) + if isinstance(getattr_type, CallableType): + return getattr_type.ret_type + if itype.type.fallback_to_any: + return AnyType() + return None + + +def find_var_type(var: Var, itype: Instance) -> Type: + from mypy.checkmember import bind_self + itype = map_instance_to_supertype(itype, var.info) + typ = var.type + if typ is None: + return AnyType() + typ = expand_type_by_instance(typ, itype) + if isinstance(typ, FunctionLike): + signature = bind_self(typ) + assert isinstance(signature, CallableType) + if var.is_property: + return signature.ret_type + return signature + return typ + + +def map_method(method: FuncBase, itype: Instance) -> Type: + from mypy.checkmember import bind_self + signature = function_type(method, Instance(None, [])) + signature = bind_self(signature) + itype = map_instance_to_supertype(itype, method.info) + return expand_type_by_instance(signature, itype) + + def is_callable_subtype(left: CallableType, right: CallableType, ignore_return: bool = False, ignore_pos_arg_names: bool = False) -> bool: diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index 3c2f4c6b84ba..672bf2b408b8 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -84,7 +84,7 @@ async def f() -> int: x = await g() return x [out] -main:7: error: Incompatible types in await (actual type Generator[int, None, str], expected type "Awaitable") +main:7: error: Incompatible types in await (actual type Generator[int, None, str], expected type Awaitable[Any]) [case testAwaitIteratorError] @@ -95,7 +95,7 @@ async def f() -> int: x = await g() return x [out] -main:6: error: Incompatible types in await (actual type Iterator[Any], expected type "Awaitable") +main:6: error: Incompatible types in await (actual type Iterator[Any], expected type Awaitable[Any]) [case testAwaitArgumentError] @@ -106,7 +106,7 @@ async def f() -> int: return x [builtins fixtures/async_await.pyi] [out] -main:5: error: Incompatible types in await (actual type "int", expected type "Awaitable") +main:5: error: Incompatible types in await (actual type "int", expected type Awaitable[Any]) [case testAwaitResultError] @@ -271,7 +271,7 @@ class C: def __aenter__(self) -> int: pass async def __aexit__(self, x, y, z) -> None: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type "Awaitable") + async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any]) pass [builtins fixtures/async_await.pyi] [out] @@ -293,7 +293,7 @@ class C: async def __aenter__(self) -> int: pass def __aexit__(self, x, y, z) -> int: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type "Awaitable") + async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any]) pass [builtins fixtures/async_await.pyi] [out] @@ -615,11 +615,11 @@ def plain_host_generator() -> Generator[str, None, None]: async def plain_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type "Awaitable") + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type "Awaitable") + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) x = await other_coroutine() @coroutine @@ -636,11 +636,11 @@ def decorated_host_generator() -> Generator[str, None, None]: @coroutine async def decorated_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type "Awaitable") + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type "Awaitable") + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) x = await other_coroutine() [builtins fixtures/async_await.pyi] diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index d841394196a5..63c1912e5ac1 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -2947,19 +2947,19 @@ def f(TB: Type[B]): [case testMetaclassIterable] from typing import Iterable, Iterator -class BadMeta(type): +class ImplicitMeta(type): def __iter__(self) -> Iterator[int]: yield 1 -class Bad(metaclass=BadMeta): pass +class Implicit(metaclass=ImplicitMeta): pass -for _ in Bad: pass # E: Iterable expected +for _ in Implicit: pass -class GoodMeta(type, Iterable[int]): +class ExplicitMeta(type, Iterable[int]): def __iter__(self) -> Iterator[int]: yield 1 -class Good(metaclass=GoodMeta): pass -for _ in Good: pass -reveal_type(list(Good)) # E: Revealed type is 'builtins.list[builtins.int*]' +class Explicit(metaclass=ExplicitMeta): pass +for _ in Explicit: pass +reveal_type(list(Explicit)) # E: Revealed type is 'builtins.list[builtins.int*]' [builtins fixtures/list.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 78fade8c8258..07371723b3be 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -301,14 +301,14 @@ def as_mapping(p: Point) -> Mapping[str, str]: -- return p --[builtins fixtures/dict.pyi] -[case testCannotConvertTypedDictToDictOrMutableMapping] +[case testCannotConvertTypedDictToDict] from mypy_extensions import TypedDict from typing import Dict, MutableMapping Point = TypedDict('Point', {'x': int, 'y': int}) def as_dict(p: Point) -> Dict[str, int]: return p # E: Incompatible return value type (got "Point", expected Dict[str, int]) def as_mutable_mapping(p: Point) -> MutableMapping[str, int]: - return p # E: Incompatible return value type (got "Point", expected MutableMapping[str, int]) + return p # this is OK since homogeneous TypedDict is a structural subtype of MutableMapping [builtins fixtures/dict.pyi] [case testCanConvertTypedDictToAny] diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index dc89366c1133..2c37400b62a2 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -16,6 +16,7 @@ class dict(Iterable[KT], Mapping[KT, VT], Generic[KT, VT]): def __init__(self, **kwargs: VT) -> None: pass @overload def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: pass + def __getitem__(self, key: KT) -> VT: pass def __setitem__(self, k: KT, v: VT) -> None: pass def __iter__(self) -> Iterator[KT]: pass def update(self, a: Mapping[KT, VT]) -> None: pass diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index 6e1e3b0ed285..57f064eba684 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -6,3 +6,4 @@ T = TypeVar('T') def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: pass class NoReturn: pass +class Protocol: pass diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 1e75564c66f9..90edfb39e53f 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -104,7 +104,8 @@ class Sequence(Iterable[T], Protocol): def __getitem__(self, n: Any) -> T: pass @runtime -class Mapping(Protocol[T, U]): pass +class Mapping(Protocol[T, U]): + def __getitem__(self, key: T) -> U: pass @runtime class MutableMapping(Protocol[T, U]): pass From b01f5b08e5b083c1abaa70a94cc6914f478e1329 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 27 Mar 2017 01:45:48 +0200 Subject: [PATCH 003/117] Remove debugging code --- mypy/checkexpr.py | 2 -- mypy/subtypes.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 4bfcc3e2835a..342db3ae7280 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2433,8 +2433,6 @@ def overload_arg_similarity(actual: Type, formal: Type) -> int: # subtyping algorithm if type promotions are possible (e.g., int vs. float). if formal.type in actual.type.mro: return 2 - elif formal.type.is_protocol and is_subtype(actual, formal): - return 2 elif actual.type._promote and is_subtype(actual, formal): return 1 else: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 25af4135bf04..245a779a4f1f 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -62,8 +62,6 @@ def is_subtype(left: Type, right: Type, else: return left.accept(SubtypeVisitor(right, type_parameter_checker, ignore_pos_arg_names=ignore_pos_arg_names)) - # print(left, right, res) - # return res def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: @@ -297,7 +295,6 @@ def is_protocol_implementation(left: Instance, right: Instance) -> bool: for member in right.type.protocol_members: supertype = find_member(member, right) subtype = find_member(member, left) - # print(member, subtype, supertype) if not subtype or not is_subtype(subtype, supertype): ASSUMING.pop() return False From 16901fc5384fc18d93e8595586dfe85f67e1cc48 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 27 Mar 2017 15:51:32 +0200 Subject: [PATCH 004/117] Minor fixes: semanal, constrains, lib-stub; add few tests --- mypy/constraints.py | 9 ++- mypy/semanal.py | 7 +- mypy/subtypes.py | 2 +- mypy/test/testcheck.py | 1 + test-data/unit/check-protocols.test | 101 ++++++++++++++++++++++++++++ test-data/unit/check-typeddict.test | 2 +- test-data/unit/lib-stub/typing.pyi | 6 +- 7 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 test-data/unit/check-protocols.test diff --git a/mypy/constraints.py b/mypy/constraints.py index bbad72a0a674..0110315bb556 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -261,6 +261,9 @@ def visit_uninhabited_type(self, t: UninhabitedType) -> bool: return False +INFERRING = [] # type: List[nodes.TypeInfo] + + class ConstraintBuilderVisitor(TypeVisitor[List[Constraint]]): """Visitor class for inferring type constraints.""" @@ -308,18 +311,22 @@ def visit_type_var(self, template: TypeVarType) -> List[Constraint]: # Non-leaf types def visit_instance(self, template: Instance) -> List[Constraint]: + global INFERRING actual = self.actual res = [] # type: List[Constraint] if isinstance(actual, CallableType) and actual.fallback is not None: actual = actual.fallback if isinstance(actual, Instance): instance = actual - if template.type.is_protocol and self.direction == SUPERTYPE_OF: + if (template.type.is_protocol and self.direction == SUPERTYPE_OF and + template.type not in INFERRING): + INFERRING.append(template.type) for member in template.type.protocol_members: inst = mypy.subtypes.find_member(member, instance) temp = mypy.subtypes.find_member(member, template) res.extend(infer_constraints(temp, inst, self.direction)) res.extend(infer_constraints(temp, inst, neg_op(self.direction))) + INFERRING.pop() return res if (self.direction == SUBTYPE_OF and template.type.has_base(instance.type.fullname())): diff --git a/mypy/semanal.py b/mypy/semanal.py index c40fdb590cf6..e93e18a62b07 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -714,11 +714,12 @@ def detect_protocol_base(self, defn: ClassDef) -> bool: except TypeTranslationError: continue # This will be reported later if not isinstance(base, UnboundType): - return False + continue sym = self.lookup_qualified(base.name, base) if sym is None or sym.node is None: - return False - return sym.node.fullname() in ('typing.Protocol', 'mypy_extensions.Protocol') + continue + if sym.node.fullname() in ('typing.Protocol', 'mypy_extensions.Protocol'): + return True return False def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 245a779a4f1f..cfdb5ce7bf71 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -337,7 +337,7 @@ def find_var_type(var: Var, itype: Instance) -> Type: if typ is None: return AnyType() typ = expand_type_by_instance(typ, itype) - if isinstance(typ, FunctionLike): + if isinstance(typ, FunctionLike) and not var.is_staticmethod: signature = bind_self(typ) assert isinstance(signature, CallableType) if var.is_property: diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 645c44d93fef..44cf6a3350f6 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -71,6 +71,7 @@ 'check-generic-subtyping.test', 'check-varargs.test', 'check-newsyntax.test', + 'check-protocols.test', 'check-underscores.test', 'check-classvar.test', ] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test new file mode 100644 index 000000000000..8d18f2cd3681 --- /dev/null +++ b/test-data/unit/check-protocols.test @@ -0,0 +1,101 @@ +-- Simple protocol types +-- --------------------- + + +[case testSimpleProtocolOneMethod] +from typing import Protocol + +class P(Protocol): + def meth(self) -> None: + pass + +class B: pass + +class C: + def meth(self) -> None: + pass + +x: P + +def fun(x: P) -> None: + x.meth() + x.meth(x) # E: Too many arguments for "meth" of "P" + x.bad # E: "P" has no attribute "bad" + +x = C() +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") + +fun(C()) +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" +[out] + +[case testSimpleProtocolOneAbstractMethod] +from typing import Protocol +from abc import abstractmethod + +class P(Protocol): + @abstractmethod + def meth(self) -> None: + pass + +class B: pass + +class C: + def meth(self) -> None: + pass + +class D(B): + def meth(self) -> None: + pass + +x: P + +def fun(x: P) -> None: + x.meth() + x.meth(x) # E: Too many arguments for "meth" of "P" + x.bad # E: "P" has no attribute "bad" + +x = C() +x = D() +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") +fun(C()) +fun(D()) +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" +[out] + +[case testSimpleProtocolOneMethodOverride] +from typing import Protocol, Union + +class P(Protocol): + def meth(self) -> Union[int, str]: + pass +class SubP(P, Protocol): + def meth(self) -> int: + pass + +class B: pass +class C: + def meth(self) -> int: + pass + +x: SubP +def fun(x: SubP) -> str: + x.bad # E: "SubP" has no attribute "bad" + return x.meth() # E: Incompatible return value type (got "int", expected "str") + +x = C() +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") + +reveal_type(fun(C())) # E: Revealed type is 'builtins.str' +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" +[out] + +[case testSimpleProtocolTwoMethodsMerge] +from typing import Protocol + +[out] + +[case testSimpleProtocolTwoMethodsExtend] +from typing import Protocol + +[out] \ No newline at end of file diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 07371723b3be..9234102f9e36 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -308,7 +308,7 @@ Point = TypedDict('Point', {'x': int, 'y': int}) def as_dict(p: Point) -> Dict[str, int]: return p # E: Incompatible return value type (got "Point", expected Dict[str, int]) def as_mutable_mapping(p: Point) -> MutableMapping[str, int]: - return p # this is OK since homogeneous TypedDict is a structural subtype of MutableMapping + return p # E: Incompatible return value type (got "Point", expected MutableMapping[str, int]) [builtins fixtures/dict.pyi] [case testCanConvertTypedDictToAny] diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 90edfb39e53f..35c40af45485 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -93,7 +93,8 @@ class AsyncIterable(Protocol[T]): @abstractmethod def __aiter__(self) -> 'AsyncIterator[T]': pass -class AsyncIterator(AsyncIterable[T], Generic[T]): +@runtime +class AsyncIterator(AsyncIterable[T], Protocol): def __aiter__(self) -> 'AsyncIterator[T]': return self @abstractmethod def __anext__(self) -> Awaitable[T]: pass @@ -108,7 +109,8 @@ class Mapping(Protocol[T, U]): def __getitem__(self, key: T) -> U: pass @runtime -class MutableMapping(Protocol[T, U]): pass +class MutableMapping(Mapping[T, U], Protocol): + def __setitem__(self, k: T, v: U) -> None: pass def NewType(name: str, tp: Type[T]) -> Callable[[T], T]: def new_type(x): From c8c1247136d7fcb47be3ee15b78a64058e021e95 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 27 Mar 2017 16:01:35 +0200 Subject: [PATCH 005/117] Allow mypy_extensions.runtime, revert change to test name. --- mypy/semanal.py | 3 ++- test-data/unit/check-typeddict.test | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e93e18a62b07..7ba049793aa5 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -659,7 +659,8 @@ def unbind_class_type_vars(self) -> None: def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> bool: decorator.accept(self) - return isinstance(decorator, RefExpr) and decorator.fullname == 'typing.runtime' + return (isinstance(decorator, RefExpr) and + decorator.fullname in ('typing.runtime', 'mypy_extensions.runtime') def calculate_abstract_status(self, typ: TypeInfo) -> None: """Calculate abstract status of a class. diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 9234102f9e36..78fade8c8258 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -301,7 +301,7 @@ def as_mapping(p: Point) -> Mapping[str, str]: -- return p --[builtins fixtures/dict.pyi] -[case testCannotConvertTypedDictToDict] +[case testCannotConvertTypedDictToDictOrMutableMapping] from mypy_extensions import TypedDict from typing import Dict, MutableMapping Point = TypedDict('Point', {'x': int, 'y': int}) From f430c6586e8215dbbf66a7be2362ae4bcec3b135 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 27 Mar 2017 16:02:42 +0200 Subject: [PATCH 006/117] Fix typo --- mypy/semanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 7ba049793aa5..71ee6c8c5dee 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -660,7 +660,7 @@ def unbind_class_type_vars(self) -> None: def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> bool: decorator.accept(self) return (isinstance(decorator, RefExpr) and - decorator.fullname in ('typing.runtime', 'mypy_extensions.runtime') + decorator.fullname in ('typing.runtime', 'mypy_extensions.runtime')) def calculate_abstract_status(self, typ: TypeInfo) -> None: """Calculate abstract status of a class. From 260237a704a84ad12c20869de727450b5ff6213e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 28 Mar 2017 00:27:22 +0200 Subject: [PATCH 007/117] Enhance constraints, add join (meet seems to be already OK); minor fixes --- mypy/checkexpr.py | 2 ++ mypy/constraints.py | 14 ++++++++++++-- mypy/join.py | 10 +++++++++- mypy/sametypes.py | 4 ++-- mypy/subtypes.py | 11 +++++++---- test-data/unit/check-classes.test | 1 + test-data/unit/check-protocols.test | 26 +++++++++++++++++++++++++- 7 files changed, 58 insertions(+), 10 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 342db3ae7280..33f264c060e9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2433,6 +2433,8 @@ def overload_arg_similarity(actual: Type, formal: Type) -> int: # subtyping algorithm if type promotions are possible (e.g., int vs. float). if formal.type in actual.type.mro: return 2 + elif formal.type.is_protocol and is_subtype(actual, erasetype.erase_type(formal)): + return 2 elif actual.type._promote and is_subtype(actual, formal): return 1 else: diff --git a/mypy/constraints.py b/mypy/constraints.py index 0110315bb556..06b16a377085 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -319,13 +319,23 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if isinstance(actual, Instance): instance = actual if (template.type.is_protocol and self.direction == SUPERTYPE_OF and - template.type not in INFERRING): + template.type not in INFERRING and + mypy.subtypes.is_subtype(instance, erase_typevars(template))): INFERRING.append(template.type) for member in template.type.protocol_members: inst = mypy.subtypes.find_member(member, instance) temp = mypy.subtypes.find_member(member, template) res.extend(infer_constraints(temp, inst, self.direction)) - res.extend(infer_constraints(temp, inst, neg_op(self.direction))) + INFERRING.pop() + return res + elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and + instance.type not in INFERRING and + mypy.subtypes.is_subtype(erase_typevars(template), instance)): + INFERRING.append(instance.type) + for member in instance.type.protocol_members: + inst = mypy.subtypes.find_member(member, instance) + temp = mypy.subtypes.find_member(member, template) + res.extend(infer_constraints(temp, inst, self.direction)) INFERRING.pop() return res if (self.direction == SUBTYPE_OF and diff --git a/mypy/join.py b/mypy/join.py index 06d5416cd2ea..6e7a5a1ec467 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -151,7 +151,15 @@ def visit_type_var(self, t: TypeVarType) -> Type: def visit_instance(self, t: Instance) -> Type: if isinstance(self.s, Instance): - return join_instances(t, self.s) + nominal = join_instances(t, self.s) + structural = None # type: Instance + if t.type.is_protocol and is_subtype(self.s, t): + structural = t + if self.s.type.is_protocol and is_subtype(t, self.s): + structural = self.s + if not structural or is_better(nominal, structural): + return nominal + return structural elif isinstance(self.s, FunctionLike): return join_types(t, self.s.fallback) elif isinstance(self.s, TypeType): diff --git a/mypy/sametypes.py b/mypy/sametypes.py index d3f283d449ea..083b88e8d25c 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -74,8 +74,8 @@ def visit_erased_type(self, left: ErasedType) -> bool: # We can get here when isinstance is used inside a lambda # whose type is being inferred. In any event, we have no reason # to think that an ErasedType will end up being the same as - # any other type, even another ErasedType. - return False + # any other type, except another ErasedType (for protocols). + return isinstance(self.right, ErasedType) def visit_deleted_type(self, left: DeletedType) -> bool: return isinstance(self.right, DeletedType) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index cfdb5ce7bf71..81e0e9effc12 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -307,13 +307,16 @@ def find_member(name: str, itype: Instance) -> Optional[Type]: if method: if method.is_property: assert isinstance(method, OverloadedFuncDef) - return find_var_type(method.items[0].var, itype) + dec = method.items[0] + assert isinstance(dec, Decorator) + return find_var_type(dec.var, itype) return map_method(method, itype) else: node = info.get(name) if not node: - return None - v = node.node + v = None + else: + v = node.node if isinstance(v, Decorator): v = v.var if isinstance(v, Var): @@ -348,7 +351,7 @@ def find_var_type(var: Var, itype: Instance) -> Type: def map_method(method: FuncBase, itype: Instance) -> Type: from mypy.checkmember import bind_self - signature = function_type(method, Instance(None, [])) + signature = function_type(method, Instance(itype.type.mro[-1], [])) signature = bind_self(signature) itype = map_instance_to_supertype(itype, method.info) return expand_type_by_instance(signature, itype) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 447a7d916cc6..bf982083ec1e 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3010,6 +3010,7 @@ class ImplicitMeta(type): class Implicit(metaclass=ImplicitMeta): pass for _ in Implicit: pass +reveal_type(list(Implicit)) # E: Revealed type is 'builtins.list[builtins.int*]' class ExplicitMeta(type, Iterable[int]): def __iter__(self) -> Iterator[int]: yield 1 diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 8d18f2cd3681..ffb4f16fe4bd 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -98,4 +98,28 @@ from typing import Protocol [case testSimpleProtocolTwoMethodsExtend] from typing import Protocol -[out] \ No newline at end of file +[out] + +-- Generic protocol types +-- ---------------------- + +-- Recursive protocol types +-- ------------------------ + +-- @property, @classmethod and @staticmethod in protocol types +-- ----------------------------------------------------------- + +-- Meet and join of protocol types +-- ------------------------------- + +-- Unions of protocol types +-- ------------------------ + +-- Type[] with protocol types +-- -------------------------- + +-- isinsatnce() with @runtime protocols +-- ------------------------------------ + +-- Non-Instances and protocol types (Callable vs __call__ etc.) +-- ------------------------------------------------------------ From 7f8a5142ac1fa4c5d692389a05304cc873cecefb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 28 Mar 2017 16:32:12 +0200 Subject: [PATCH 008/117] Prohibit protocol instatiation except Type[]; start working on @runtime --- mypy/checker.py | 12 +++++---- mypy/checkexpr.py | 24 +++++++++++++++--- test-data/unit/check-abstract.test | 14 +++++------ test-data/unit/check-protocols.test | 38 +++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 16 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4fa03cdf0c7d..b38ddb3d50eb 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1210,14 +1210,16 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type else: rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue) - # Special case: only non-abstract classes can be assigned to variables - # with explicit type Type[A]. + # Special case: only non-abstract non-protocol classes can be assigned to + # variables with explicit type Type[A], where A is protocol or abstract. if (isinstance(rvalue_type, CallableType) and rvalue_type.is_type_obj() and - rvalue_type.type_object().is_abstract and + (rvalue_type.type_object().is_abstract or + rvalue_type.type_object().is_protocol) and isinstance(lvalue_type, TypeType) and isinstance(lvalue_type.item, Instance) and - lvalue_type.item.type.is_abstract): - self.fail("Can only assign non-abstract classes" + (lvalue_type.item.type.is_abstract or + lvalue_type.item.type.is_protocol)): + self.fail("Can only assign concrete classes" " to a variable of type '{}'".format(lvalue_type), rvalue) return if rvalue_type and infer_lvalue_type: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 30a75cef472e..50194e259790 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -185,6 +185,15 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: and callee_type.implicit): return self.msg.untyped_function_call(callee_type, e) ret_type = self.check_call_expr_with_callee_type(callee_type, e) + if isinstance(e.callee, RefExpr) and e.callee.fullname in ('builtins.isinstance', + 'builtins.issubclass'): + for expr in mypy.checker.flatten(e.args[1]): + tp = self.chk.type_map[expr] + if (isinstance(tp, CallableType) and tp.is_type_obj() and + tp.type_object().is_protocol and + not tp.type_object().runtime_protocol): + self.chk.fail('Only @runtime protocols can be used with' + ' instance and class checks', e) if isinstance(ret_type, UninhabitedType): self.chk.binder.unreachable() if not allow_none_return and isinstance(ret_type, NoneTyp): @@ -356,6 +365,11 @@ def check_call(self, callee: Type, args: List[Expression], self.msg.cannot_instantiate_abstract_class( callee.type_object().name(), type.abstract_attributes, context) + elif (callee.is_type_obj() and callee.type_object().is_protocol + # Exceptions for Type[...] and classmethod first argument + and not callee.from_type_type and not callee.is_classmethod_class): + self.chk.fail('Cannot instantiate protocol class "{}"' + .format(callee.type_object().fullname()), context) formal_to_actual = map_actuals_to_formals( arg_kinds, arg_names, @@ -838,13 +852,15 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, """Check the type of a single argument in a call.""" if isinstance(caller_type, DeletedType): messages.deleted_as_rvalue(caller_type, context) - # Only non-abstract class can be given where Type[...] is expected... + # Only non-abstract non-protocol class can be given where Type[...] is expected... elif (isinstance(caller_type, CallableType) and isinstance(callee_type, TypeType) and - caller_type.is_type_obj() and caller_type.type_object().is_abstract and - isinstance(callee_type.item, Instance) and callee_type.item.type.is_abstract and + caller_type.is_type_obj() and + (caller_type.type_object().is_abstract or caller_type.type_object().is_protocol) and + isinstance(callee_type.item, Instance) and + (callee_type.item.type.is_abstract or callee_type.item.type.is_protocol) and # ...except for classmethod first argument not caller_type.is_classmethod_class): - messages.fail("Only non-abstract class can be given where '{}' is expected" + messages.fail("Only concrete class can be given where '{}' is expected" .format(callee_type), context) elif not is_subtype(caller_type, callee_type): if self.chk.should_suppress_optional_error([caller_type, callee_type]): diff --git a/test-data/unit/check-abstract.test b/test-data/unit/check-abstract.test index 2de9d129436b..6a46904e79a4 100644 --- a/test-data/unit/check-abstract.test +++ b/test-data/unit/check-abstract.test @@ -174,8 +174,8 @@ def f(cls: Type[A]) -> A: def g() -> A: return A() # E: Cannot instantiate abstract class 'A' with abstract attribute 'm' -f(A) # E: Only non-abstract class can be given where 'Type[__main__.A]' is expected -f(B) # E: Only non-abstract class can be given where 'Type[__main__.A]' is expected +f(A) # E: Only concrete class can be given where 'Type[__main__.A]' is expected +f(B) # E: Only concrete class can be given where 'Type[__main__.A]' is expected f(C) # OK x: Type[B] f(x) # OK @@ -200,7 +200,7 @@ Alias = A GoodAlias = C Alias() # E: Cannot instantiate abstract class 'A' with abstract attribute 'm' GoodAlias() -f(Alias) # E: Only non-abstract class can be given where 'Type[__main__.A]' is expected +f(Alias) # E: Only concrete class can be given where 'Type[__main__.A]' is expected f(GoodAlias) [out] @@ -218,14 +218,14 @@ class C(B): var: Type[A] var() -var = A # E: Can only assign non-abstract classes to a variable of type 'Type[__main__.A]' -var = B # E: Can only assign non-abstract classes to a variable of type 'Type[__main__.A]' +var = A # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' +var = B # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' var = C # OK var_old = None # type: Type[A] # Old syntax for variable annotations var_old() -var_old = A # E: Can only assign non-abstract classes to a variable of type 'Type[__main__.A]' -var_old = B # E: Can only assign non-abstract classes to a variable of type 'Type[__main__.A]' +var_old = A # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' +var_old = B # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' var_old = C # OK [out] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index ffb4f16fe4bd..0f34f3ede3df 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1,6 +1,15 @@ -- Simple protocol types -- --------------------- +[case testCannotInstantiateProtocol] +from typing import Protocol + +class P(Protocol): + def meth(self) -> None: + pass + +P() # E: Cannot instantiate protocol class "__main__.P" +[out] [case testSimpleProtocolOneMethod] from typing import Protocol @@ -100,6 +109,9 @@ from typing import Protocol [out] +-- Semanal errors in protocol types +-- -------------------------------- + -- Generic protocol types -- ---------------------- @@ -121,5 +133,31 @@ from typing import Protocol -- isinsatnce() with @runtime protocols -- ------------------------------------ +[case testSimpleRuntimeProtocolCheck] +from typing import Protocol, runtime + +class P(Protocol): + def meth(self) -> None: + pass + +@runtime +class R(Protocol): + def meth(self) -> int: + pass + +x: object + +if isinstance(x, P): # E: Only @runtime protocols can be used with instance and class checks + reveal_type(x) # E: Revealed type is '__main__.P' + +if isinstance(x, R): + reveal_type(x) # E: Revealed type is '__main__.R' + reveal_type(x.meth()) # E: Revealed type is 'builtins.int' # TODO: this should be Any +[builtins fixtures/isinstance.pyi] +[out] + -- Non-Instances and protocol types (Callable vs __call__ etc.) -- ------------------------------------------------------------ + +-- Standard protocol types (SupportsInt, Reversible, etc.) +-- ------------------------------------------------------- From 92b71a25a910805bf609ec5f9efdfc098fa931f4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 28 Mar 2017 17:13:48 +0200 Subject: [PATCH 009/117] Add some tests --- test-data/unit/check-protocols.test | 192 +++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 0f34f3ede3df..6c6e0cb3f537 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -58,7 +58,6 @@ class D(B): pass x: P - def fun(x: P) -> None: x.meth() x.meth(x) # E: Too many arguments for "meth" of "P" @@ -86,12 +85,13 @@ class B: pass class C: def meth(self) -> int: pass - +z: P x: SubP def fun(x: SubP) -> str: x.bad # E: "SubP" has no attribute "bad" return x.meth() # E: Incompatible return value type (got "int", expected "str") +z = x x = C() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") @@ -102,11 +102,113 @@ fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" [case testSimpleProtocolTwoMethodsMerge] from typing import Protocol +class P1(Protocol): + def meth1(self) -> int: + pass +class P2(Protocol): + def meth2(self) -> str: + pass + +class C1: + def meth1(self) -> int: + pass +class C2(C1): + def meth2(self) -> str: + pass +class C: + def meth1(self) -> int: + pass + def meth2(self) -> str: + pass + +class AnotherP(Protocol): + def meth1(self) -> int: + pass + def meth2(self) -> str: + pass + +class P(P1, P2, Protocol): pass + +x: P +reveal_type(x.meth1()) # E: Revealed type is 'builtins.int' +reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' + +c: C +c1: C1 +c2: C2 +y: AnotherP + +x = c +x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P") +x = c2 +x = y +y = x [out] [case testSimpleProtocolTwoMethodsExtend] from typing import Protocol +class P1(Protocol): + def meth1(self) -> int: + pass +class P2(P1, Protocol): + def meth2(self) -> str: + pass + +class Cbad: + def meth1(self) -> int: + pass + +class C: + def meth1(self) -> int: + pass + def meth2(self) -> str: + pass + +x: P2 +reveal_type(x.meth1()) # E: Revealed type is 'builtins.int' +reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' + +x = C() # OK +x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2") +[out] + +[case testCannotAssignNormalToProtocol] +from typing import Protocol + +class P(Protocol): + def meth(self) -> int: + pass +class C: + def meth(self) -> int: + pass + +x: C +y: P +x = y # E: Incompatible types in assignment (expression has type "P", variable has type "C") +[out] + +[case testIndependentProtocolSubtyping] +from typing import Protocol + +class P1(Protocol): + def meth(self) -> int: + pass +class P2(Protocol): + def meth(self) -> int: + pass + +x1: P1 +x2: P2 + +x1 = x2 +x2 = x1 + +def f1(x: P1) -> None: pass +def f2(x: P2) -> None: pass + +f1(x2) +f2(x1) [out] -- Semanal errors in protocol types @@ -130,6 +232,92 @@ from typing import Protocol -- Type[] with protocol types -- -------------------------- +[case testInstantiationProtocolInTypeForFunctions] +from typing import Type, Protocol + +class P(Protocol): + def m(self) -> None: pass +class P1(Protocol): + def m(self) -> None: pass +class Pbad(Protocol): + def mbad(self) -> int: pass +class B(P): pass +class C: + def m(self) -> None: + pass + +def f(cls: Type[P]) -> P: + return cls() # OK +def g() -> P: + return P() # E: Cannot instantiate protocol class "__main__.P" + +f(P) # E: Only concrete class can be given where 'Type[__main__.P]' is expected +f(B) # OK +f(C) # OK +x: Type[P1] +xbad: Type[Pbad] +f(x) # OK +f(xbad) # E: Argument 1 to "f" has incompatible type Type[Pbad]; expected Type[P] +[out] + +[case testInstantiationProtocolInTypeForAliases] +from typing import Type, Protocol + +class P(Protocol): + def m(self) -> None: pass +class C: + def m(self) -> None: + pass + +def f(cls: Type[P]) -> P: + return cls() # OK + +Alias = P +GoodAlias = C +Alias() # E: Cannot instantiate protocol class "__main__.P" +GoodAlias() +f(Alias) # E: Only concrete class can be given where 'Type[__main__.P]' is expected +f(GoodAlias) +[out] + +[case testInstantiationProtocolInTypeForVariables] +from typing import Type, Protocol + +class P(Protocol): + def m(self) -> None: pass +class B(P): pass +class C: + def m(self) -> None: + pass + +var: Type[P] +var() +var = P # E: Can only assign concrete classes to a variable of type 'Type[__main__.P]' +var = B # OK +var = C # OK + +var_old = None # type: Type[P] # Old syntax for variable annotations +var_old() +var_old = P # E: Can only assign concrete classes to a variable of type 'Type[__main__.P]' +var_old = B # OK +var_old = C # OK +[out] + +[case testInstantiationProtocolInTypeForClassMethods] +from typing import Type, Protocol + +class Logger: + @staticmethod + def log(a: Type[C]): + pass +class C(Protocol): + @classmethod + def action(cls) -> None: + cls() #OK for classmethods + Logger.log(cls) #OK for classmethods +[builtins fixtures/classmethod.pyi] +[out] + -- isinsatnce() with @runtime protocols -- ------------------------------------ From 83085c7257151386be85563f4de9c8c9dad55747 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 28 Mar 2017 23:20:19 +0200 Subject: [PATCH 010/117] Restore docstring formstting --- mypy/semanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8fe378e4026b..4c7b56d3fd99 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -792,7 +792,7 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: For example, consider this class: - class Foo(Bar, Generic[T]): ... + . class Foo(Bar, Generic[T]): ... Now we will remove Generic[T] from bases of Foo and infer that the type variable 'T' is a type argument of Foo. From 41179c26363143f0d36febf71efd51715c9a08aa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 29 Mar 2017 00:15:02 +0200 Subject: [PATCH 011/117] Fix serialization; add few more tests --- mypy/nodes.py | 4 +++- test-data/unit/check-incremental.test | 20 ++++++++++++++++++++ test-data/unit/check-protocols.test | 27 +++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 3d92ed7eba6a..1828468e1d3b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1971,7 +1971,7 @@ class is generic then it will be a type constructor of higher kind. FLAGS = [ 'is_abstract', 'is_enum', 'fallback_to_any', 'is_named_tuple', - 'is_newtype' + 'is_newtype', 'is_protocol', 'runtime_protocol' ] def __init__(self, names: 'SymbolTable', defn: ClassDef, module_name: str) -> None: @@ -2123,6 +2123,7 @@ def serialize(self) -> JsonDict: 'names': self.names.serialize(self.alt_fullname or self.fullname()), 'defn': self.defn.serialize(), 'abstract_attributes': self.abstract_attributes, + 'protocol_members': self.protocol_members, 'type_vars': self.type_vars, 'bases': [b.serialize() for b in self.bases], '_promote': None if self._promote is None else self._promote.serialize(), @@ -2145,6 +2146,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': ti.alt_fullname = data['alt_fullname'] # TODO: Is there a reason to reconstruct ti.subtypes? ti.abstract_attributes = data['abstract_attributes'] + ti.protocol_members = data['protocol_members'] ti.type_vars = data['type_vars'] ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] ti._promote = (None if data['_promote'] is None diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 4ddec38618dc..ff3312d5e6b9 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1451,6 +1451,26 @@ class MyClass: [rechecked] [stale] +[case testIncrementalWorksWithBasicProtocols] +from a import P + +x: int +y: P[int] +x = y.meth() + +class C: + def meth(self) -> int: + pass + +y = C() +[file a.py] +from typing import Protocol, TypeVar + +T = TypeVar('T') +class P(Protocol[T]): + def meth(self) -> T: + pass + [case testIncrementalWorksWithNamedTuple] import foo diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 6c6e0cb3f537..e8fdcdb03ca0 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -211,8 +211,31 @@ f1(x2) f2(x1) [out] --- Semanal errors in protocol types --- -------------------------------- +[case testSemanalErrorsInProtocols] +from typing import Protocol, Generic, TypeVar, Iterable + +T = TypeVar('T') +S = TypeVar('S') + +class P1(Protocol[T, T]): # E: Duplicate type variables in Generic[...] or Protocol[...] + def meth(self) -> T: + pass + +class P2(Protocol[T], Protocol[S]): # E: Only single Generic[...] or Protocol[...] can be in bases + def meth(self) -> T: + pass + +class P3(Protocol[T], Generic[S]): # E: Only single Generic[...] or Protocol[...] can be in bases + def meth(self) -> T: + pass + +class P4(Protocol[T]): + attr: Iterable[S] # E: Invalid type "__main__.S" + +class P5(Iterable[S], Protocol[T]): # E: If Generic[...] or Protocol[...] is present it should list all type variables + def meth(self) -> T: + pass +[out] -- Generic protocol types -- ---------------------- From 51718445638286cab25b9145ab02deb124e98cb8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Mar 2017 15:19:04 +0200 Subject: [PATCH 012/117] Add notes for missing protocol members in assignment --- mypy/checker.py | 7 ++++- mypy/subtypes.py | 8 +++++ test-data/unit/check-async-await.test | 42 +++++++++++++++++++++++---- test-data/unit/check-protocols.test | 35 ++++++++++++++++++---- 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9d67cff9b060..a411e68b8909 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -38,7 +38,7 @@ from mypy.subtypes import ( is_subtype, is_equivalent, is_proper_subtype, is_more_precise, restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_subtype, - unify_generic_callable, + unify_generic_callable, get_missing_members ) from mypy.maptype import map_instance_to_supertype from mypy.typevars import fill_typevars, has_no_typevars @@ -2302,6 +2302,11 @@ def check_subtype(self, subtype: Type, supertype: Type, context: Context, if extra_info: msg += ' (' + ', '.join(extra_info) + ')' self.fail(msg, context) + if (isinstance(supertype, Instance) and supertype.type.is_protocol and + isinstance(subtype, Instance)): + self.note('{} missing following {} protocol members:' + .format(subtype, supertype), context) + self.note(', '.join(get_missing_members(subtype, supertype)), context) return False def contains_none(self, t: Type) -> bool: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 81e0e9effc12..b9a162e28ad8 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -357,6 +357,14 @@ def map_method(method: FuncBase, itype: Instance) -> Type: return expand_type_by_instance(signature, itype) +def get_missing_members(left: Instance, right: Instance) -> List[str]: + missing = [] # type: List[str] + for member in right.type.protocol_members: + if not find_member(member, left): + missing.append(member) + return missing + + def is_callable_subtype(left: CallableType, right: CallableType, ignore_return: bool = False, ignore_pos_arg_names: bool = False) -> bool: diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index 672bf2b408b8..be17c85cb154 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -85,6 +85,8 @@ async def f() -> int: return x [out] main:7: error: Incompatible types in await (actual type Generator[int, None, str], expected type Awaitable[Any]) +main:7: note: typing.Generator[builtins.int, builtins.None, builtins.str] missing following typing.Awaitable[Any] protocol members: +main:7: note: __await__ [case testAwaitIteratorError] @@ -96,6 +98,8 @@ async def f() -> int: return x [out] main:6: error: Incompatible types in await (actual type Iterator[Any], expected type Awaitable[Any]) +main:6: note: typing.Iterator[Any] missing following typing.Awaitable[Any] protocol members: +main:6: note: __await__ [case testAwaitArgumentError] @@ -107,6 +111,8 @@ async def f() -> int: [builtins fixtures/async_await.pyi] [out] main:5: error: Incompatible types in await (actual type "int", expected type Awaitable[Any]) +main:5: note: builtins.int missing following typing.Awaitable[Any] protocol members: +main:5: note: __await__ [case testAwaitResultError] @@ -150,6 +156,8 @@ async def f() -> None: [builtins fixtures/async_await.pyi] [out] main:4: error: AsyncIterable expected +main:4: note: builtins.list[builtins.int*] missing following typing.AsyncIterable[Any] protocol members: +main:4: note: __aiter__ main:4: error: List[int] has no attribute "__aiter__" [case testAsyncForTypeComments] @@ -232,12 +240,20 @@ async def wrong_iterable(obj: Iterable[int]): [out] main:18: error: AsyncIterable expected +main:18: note: typing.Iterable[builtins.int] missing following typing.AsyncIterable[Any] protocol members: +main:18: note: __aiter__ main:18: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:19: error: Iterable expected +main:19: note: __main__.asyncify[builtins.int*] missing following typing.Iterable[Any] protocol members: +main:19: note: __iter__ main:19: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? main:20: error: AsyncIterable expected +main:20: note: typing.Iterable[builtins.int] missing following typing.AsyncIterable[Any] protocol members: +main:20: note: __aiter__ main:20: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:21: error: Iterable expected +main:21: note: __main__.asyncify[builtins.int*] missing following typing.Iterable[Any] protocol members: +main:21: note: __iter__ main:21: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? [builtins fixtures/async_await.pyi] @@ -271,7 +287,9 @@ class C: def __aenter__(self) -> int: pass async def __aexit__(self, x, y, z) -> None: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any]) + async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any])\ + # N: builtins.int missing following typing.Awaitable[Any] protocol members:\ + # N: __await__ pass [builtins fixtures/async_await.pyi] [out] @@ -293,7 +311,9 @@ class C: async def __aenter__(self) -> int: pass def __aexit__(self, x, y, z) -> int: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any]) + async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any])\ + # N: builtins.int missing following typing.Awaitable[Any] protocol members:\ + # N: __await__ pass [builtins fixtures/async_await.pyi] [out] @@ -523,6 +543,8 @@ def h() -> None: [out] main:9: error: Iterable expected +main:9: note: typing.AsyncGenerator[builtins.int, builtins.None] missing following typing.Iterable[Any] protocol members: +main:9: note: __iter__ main:9: error: AsyncGenerator[int, None] has no attribute "__iter__"; maybe "__aiter__"? [case testAsyncGeneratorNoYieldFrom] @@ -615,11 +637,15 @@ def plain_host_generator() -> Generator[str, None, None]: async def plain_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any])\ + # N: typing.Generator[builtins.str, builtins.None, builtins.int] missing following typing.Awaitable[Any] protocol members:\ + # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any])\ + # N: __main__.It missing following typing.Awaitable[Any] protocol members:\ + # N: __await__ x = await other_coroutine() @coroutine @@ -636,11 +662,15 @@ def decorated_host_generator() -> Generator[str, None, None]: @coroutine async def decorated_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any])\ + # N: typing.Generator[builtins.str, builtins.None, builtins.int] missing following typing.Awaitable[Any] protocol members:\ + # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any])\ + # N: __main__.It missing following typing.Awaitable[Any] protocol members:\ + # N: __await__ x = await other_coroutine() [builtins fixtures/async_await.pyi] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index e8fdcdb03ca0..fbaf50ae5827 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -32,7 +32,9 @@ def fun(x: P) -> None: x.bad # E: "P" has no attribute "bad" x = C() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ + # N: __main__.B missing following __main__.P protocol members:\ + # N: meth fun(C()) fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" @@ -65,7 +67,9 @@ def fun(x: P) -> None: x = C() x = D() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ + # N: __main__.B missing following __main__.P protocol members:\ + # N: meth fun(C()) fun(D()) fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" @@ -93,7 +97,9 @@ def fun(x: SubP) -> str: z = x x = C() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP")\ + # N: __main__.B missing following __main__.SubP protocol members:\ + # N: meth reveal_type(fun(C())) # E: Revealed type is 'builtins.str' fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" @@ -108,7 +114,7 @@ class P1(Protocol): class P2(Protocol): def meth2(self) -> str: pass - +class B: pass class C1: def meth1(self) -> int: pass @@ -139,7 +145,12 @@ c2: C2 y: AnotherP x = c -x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P") +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ + # N: __main__.B missing following __main__.P protocol members:\ + # N: meth1, meth2 +x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P")\ + # N: __main__.C1 missing following __main__.P protocol members:\ + # N: meth2 x = c2 x = y y = x @@ -170,7 +181,9 @@ reveal_type(x.meth1()) # E: Revealed type is 'builtins.int' reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' x = C() # OK -x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2") +x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2")\ + # N: __main__.Cbad missing following __main__.P2 protocol members:\ + # N: meth2 [out] [case testCannotAssignNormalToProtocol] @@ -367,6 +380,16 @@ if isinstance(x, R): [builtins fixtures/isinstance.pyi] [out] +[case testRuntimeIterableProtocolCheck] +from typing import Iterable, List, Union + +x: Union[int, List[str]] + +if isinstance(x, Iterable): + reveal_type(x) # E: Revealed type is 'builtins.list[builtins.str]' +[builtins fixtures/isinstancelist.pyi] +[out] + -- Non-Instances and protocol types (Callable vs __call__ etc.) -- ------------------------------------------------------------ From db123a320b4deda9bdcf17c1052c9aeaa3cfd226 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Mar 2017 15:53:42 +0200 Subject: [PATCH 013/117] Add notes for missing members also for arguments and return types --- mypy/checker.py | 8 +++++--- mypy/checkexpr.py | 9 ++++++++- mypy/subtypes.py | 2 +- test-data/unit/check-namedtuple.test | 4 +++- test-data/unit/check-protocols.test | 19 ++++++++++++++++--- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index a411e68b8909..f5931bf12783 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2304,9 +2304,11 @@ def check_subtype(self, subtype: Type, supertype: Type, context: Context, self.fail(msg, context) if (isinstance(supertype, Instance) and supertype.type.is_protocol and isinstance(subtype, Instance)): - self.note('{} missing following {} protocol members:' - .format(subtype, supertype), context) - self.note(', '.join(get_missing_members(subtype, supertype)), context) + missing = get_missing_members(subtype, supertype) + if missing: + self.note('{} missing following {} protocol members:' + .format(subtype, supertype), context) + self.note(', '.join(missing), context) return False def contains_none(self, t: Type) -> bool: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 50194e259790..e3f497a8315a 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -34,7 +34,7 @@ from mypy import join from mypy.meet import meet_simple from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent +from mypy.subtypes import is_subtype, is_equivalent, get_missing_members from mypy import applytype from mypy import erasetype from mypy.checkmember import analyze_member_access, type_object_type, bind_self @@ -867,6 +867,13 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, return messages.incompatible_argument(n, m, callee, original_caller_type, caller_kind, context) + if (isinstance(original_caller_type, Instance) and + isinstance(callee_type, Instance) and callee_type.type.is_protocol): + missing = get_missing_members(original_caller_type, callee_type) + if missing: + messages.note('{} missing following {} protocol members:' + .format(original_caller_type, callee_type), context) + messages.note(', '.join(missing), context) def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], arg_names: List[str], diff --git a/mypy/subtypes.py b/mypy/subtypes.py index b9a162e28ad8..68a938d7a2c2 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -362,7 +362,7 @@ def get_missing_members(left: Instance, right: Instance) -> List[str]: for member in right.type.protocol_members: if not find_member(member, left): missing.append(member) - return missing + return sorted(missing) def is_callable_subtype(left: CallableType, right: CallableType, diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 71a058b24ac6..e8faaebfcd8e 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -286,7 +286,9 @@ from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) reveal_type(X._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' -X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any] +X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any]\ + # N: builtins.str missing following typing.Iterable[Any] protocol members:\ + # N: __iter__ -- # FIX: not a proper class method -- x = None # type: X diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index fbaf50ae5827..fcd5b16e876d 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -37,7 +37,16 @@ x = B() # E: Incompatible types in assignment (expression has type "B", variable # N: meth fun(C()) -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P"\ + # N: __main__.B missing following __main__.P protocol members:\ + # N: meth + +def fun2() -> P: + return C() +def fun3() -> P: + return B() # E: Incompatible return value type (got "B", expected "P")\ + # N: __main__.B missing following __main__.P protocol members:\ + # N: meth [out] [case testSimpleProtocolOneAbstractMethod] @@ -72,7 +81,9 @@ x = B() # E: Incompatible types in assignment (expression has type "B", variable # N: meth fun(C()) fun(D()) -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P"\ + # N: __main__.B missing following __main__.P protocol members:\ + # N: meth [out] [case testSimpleProtocolOneMethodOverride] @@ -102,7 +113,9 @@ x = B() # E: Incompatible types in assignment (expression has type "B", variable # N: meth reveal_type(fun(C())) # E: Revealed type is 'builtins.str' -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP"\ + # N: __main__.B missing following __main__.SubP protocol members:\ + # N: meth [out] [case testSimpleProtocolTwoMethodsMerge] From 31bf16c5ee870c9c76570c64f1b6750b04440765 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Mar 2017 16:29:17 +0200 Subject: [PATCH 014/117] Prohibit definition via self.x in protocol classes --- mypy/semanal.py | 25 ++++++++++++--------- test-data/unit/check-protocols.test | 35 ++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 4c7b56d3fd99..0725891f6d46 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -668,7 +668,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.analyze_base_classes(defn) self.analyze_metaclass(defn) - + defn.info.is_protocol = is_protocol runtime_protocol = False for decorator in defn.decorators: if self.analyze_class_decorator(defn, decorator): @@ -680,7 +680,6 @@ def visit_class_def(self, defn: ClassDef) -> None: defn.defs.accept(self) self.calculate_abstract_status(defn.info) - defn.info.is_protocol = is_protocol defn.info.runtime_protocol = runtime_protocol self.setup_type_promotion(defn) @@ -1698,15 +1697,19 @@ def analyze_member_lvalue(self, lval: MemberExpr) -> None: lval.accept(self) if (self.is_self_member_ref(lval) and self.type.get(lval.name) is None): - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.set_line(lval) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) + if self.type.is_protocol: + # Protocol members can't be defined via self + self.fail("Protocol members cannot be defined via assignment to self", lval) + else: + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.set_line(lval) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) self.check_lvalue_validity(lval.node, lval) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index fcd5b16e876d..3b01bf5f57b3 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -237,7 +237,10 @@ f1(x2) f2(x1) [out] -[case testSemanalErrorsInProtocols] +-- Semanal errors in protocol types +-- -------------------------------- + +[case testBasicSemanalErrorsInProtocols] from typing import Protocol, Generic, TypeVar, Iterable T = TypeVar('T') @@ -263,6 +266,36 @@ class P5(Iterable[S], Protocol[T]): # E: If Generic[...] or Protocol[...] is pre pass [out] +[case testProhibitSelfDefinitionInProtocols] +from typing import Protocol + +class P(Protocol): + def __init__(self, a: int) -> None: + self.a = a # E: Protocol members cannot be defined via assignment to self\ + # E: "P" has no attribute "a" + +class B: pass +class C: + def __init__(self, a: int) -> None: + pass + +x: P +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") +# The above is because of incompatible __init__, maybe we could allow this? +# (mypy allows incompatible __init__ in nominal subclasses.) +x = C(1) + +class P2(Protocol): + a: int + def __init__(self) -> None: + self.a = 1 + +class B2: + a: int + +x2: P2 = B2() # OK +[out] + -- Generic protocol types -- ---------------------- From 65a8546296c6cd6ea29822bbff3fd92999affe53 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Mar 2017 17:37:16 +0200 Subject: [PATCH 015/117] Make more things in fixtures structural subtypes --- mypy/join.py | 8 +++++--- mypy/meet.py | 9 +++++++-- mypy/subtypes.py | 7 +++++-- test-data/unit/fixtures/dict.pyi | 4 ++-- test-data/unit/fixtures/isinstancelist.pyi | 8 ++++---- test-data/unit/fixtures/list.pyi | 2 +- 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index 6e7a5a1ec467..b3514c830da7 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -10,7 +10,9 @@ UninhabitedType, TypeType, true_or_false ) from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent, is_subtype_ignoring_tvars +from mypy.subtypes import ( + is_subtype, is_equivalent, is_subtype_ignoring_tvars,is_protocol_implementation +) from mypy import experiments @@ -153,9 +155,9 @@ def visit_instance(self, t: Instance) -> Type: if isinstance(self.s, Instance): nominal = join_instances(t, self.s) structural = None # type: Instance - if t.type.is_protocol and is_subtype(self.s, t): + if t.type.is_protocol and is_protocol_implementation(self.s, t): structural = t - if self.s.type.is_protocol and is_subtype(t, self.s): + if self.s.type.is_protocol and is_protocol_implementation(t, self.s): structural = self.s if not structural or is_better(nominal, structural): return nominal diff --git a/mypy/meet.py b/mypy/meet.py index 22d0a709838b..e4296f4f0a04 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -7,7 +7,7 @@ Instance, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, PartialType, DeletedType, UninhabitedType, TypeType ) -from mypy.subtypes import is_equivalent, is_subtype +from mypy.subtypes import is_equivalent, is_subtype, is_protocol_implementation from mypy import experiments @@ -47,7 +47,8 @@ def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool Note that this effectively checks against erased types, since type variables are erased at runtime and the overlapping check is based - on runtime behavior. + on runtime behavior. The exception is protocol types, it is not safe, + but convenient and is an opt-in behavior. If use_promotions is True, also consider type promotions (int and float would only be overlapping if it's True). @@ -91,6 +92,10 @@ class C(A, B): ... return True if s.type._promote and is_overlapping_types(s.type._promote, t): return True + if t.type.is_protocol and is_protocol_implementation(s, t): + return True + if s.type.is_protocol and is_protocol_implementation(t, s): + return True return t.type in s.type.mro or s.type in t.type.mro if isinstance(t, UnionType): return any(is_overlapping_types(item, s) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 68a938d7a2c2..ee5dd34681fe 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -285,9 +285,10 @@ def visit_type_type(self, left: TypeType) -> bool: ASSUMING = [] # type: List[Tuple[Instance, Instance]] -def is_protocol_implementation(left: Instance, right: Instance) -> bool: +def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: global ASSUMING assert right.type.is_protocol + is_compat = is_subtype if allow_any else is_proper_subtype for (l, r) in ASSUMING: if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True @@ -295,7 +296,7 @@ def is_protocol_implementation(left: Instance, right: Instance) -> bool: for member in right.type.protocol_members: supertype = find_member(member, right) subtype = find_member(member, left) - if not subtype or not is_subtype(subtype, supertype): + if not subtype or not is_compat(subtype, supertype): ASSUMING.pop() return False return True @@ -610,6 +611,8 @@ def is_proper_subtype(t: Type, s: Type) -> bool: if isinstance(t, Instance): if isinstance(s, Instance): + if s.type.is_protocol: + return is_protocol_implementation(t, s, allow_any=False) if not t.type.has_base(s.type.fullname()): return False diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index 2c37400b62a2..b8e6be24ceb2 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -11,7 +11,7 @@ class object: class type: pass -class dict(Iterable[KT], Mapping[KT, VT], Generic[KT, VT]): +class dict(Generic[KT, VT]): @overload def __init__(self, **kwargs: VT) -> None: pass @overload @@ -31,7 +31,7 @@ class int: # for convenience class str: pass # for keyword argument key type class unicode: pass # needed for py2 docstrings -class list(Iterable[T], Generic[T]): # needed by some test cases +class list(Generic[T]): # needed by some test cases def __iter__(self) -> Iterator[T]: pass def __mul__(self, x: int) -> list[T]: pass diff --git a/test-data/unit/fixtures/isinstancelist.pyi b/test-data/unit/fixtures/isinstancelist.pyi index 9096961b06ab..04a6e78fe0d1 100644 --- a/test-data/unit/fixtures/isinstancelist.pyi +++ b/test-data/unit/fixtures/isinstancelist.pyi @@ -1,4 +1,4 @@ -from typing import builtinclass, Iterable, Iterator, TypeVar, List, Mapping, overload, Tuple, Set, Union +from typing import builtinclass, Iterable, Iterator, TypeVar, List, Mapping, overload, Tuple, Set, Union, Generic @builtinclass class object: @@ -27,14 +27,14 @@ T = TypeVar('T') KT = TypeVar('KT') VT = TypeVar('VT') -class list(Iterable[T]): +class list(Generic[T]): def __iter__(self) -> Iterator[T]: pass def __mul__(self, x: int) -> list[T]: pass def __setitem__(self, x: int, v: T) -> None: pass def __getitem__(self, x: int) -> T: pass def __add__(self, x: List[T]) -> T: pass -class dict(Iterable[KT], Mapping[KT, VT]): +class dict(Mapping[KT, VT]): @overload def __init__(self, **kwargs: VT) -> None: pass @overload @@ -43,7 +43,7 @@ class dict(Iterable[KT], Mapping[KT, VT]): def __iter__(self) -> Iterator[KT]: pass def update(self, a: Mapping[KT, VT]) -> None: pass -class set(Iterable[T]): +class set(Generic[T]): def __iter__(self) -> Iterator[T]: pass def add(self, x: T) -> None: pass def discard(self, x: T) -> None: pass diff --git a/test-data/unit/fixtures/list.pyi b/test-data/unit/fixtures/list.pyi index ce9978a38c4c..6214749d24c3 100644 --- a/test-data/unit/fixtures/list.pyi +++ b/test-data/unit/fixtures/list.pyi @@ -11,7 +11,7 @@ class object: class type: pass class ellipsis: pass -class list(Iterable[T], Generic[T]): +class list(Generic[T]): @overload def __init__(self) -> None: pass @overload From 2768d76497362c9b734a028cf8935ad55bb44953 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Mar 2017 21:40:56 +0200 Subject: [PATCH 016/117] Structural support for __call__; add more tests --- mypy/constraints.py | 4 + mypy/subtypes.py | 15 +- test-data/unit/check-protocols.test | 225 +++++++++++++++++++- test-data/unit/lib-stub/mypy_extensions.pyi | 1 + test-data/unit/lib-stub/typing.pyi | 3 + 5 files changed, 243 insertions(+), 5 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 06b16a377085..ae667dad5b50 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -404,6 +404,10 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: return self.infer_against_overloaded(self.actual, template) elif isinstance(self.actual, TypeType): return infer_constraints(template.ret_type, self.actual.item, self.direction) + elif isinstance(self.actual, Instance): + call = mypy.subtypes.find_member('__call__', self.actual) + if call: + return infer_constraints(template, call, self.direction) else: return [] diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ee5dd34681fe..ee2f6ff9382e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -156,6 +156,9 @@ def visit_instance(self, left: Instance) -> bool: return any(base.fullname() == 'builtins.type' for base in mro) else: return False + if isinstance(right, CallableType): + call = find_member('__call__', left) + return call and is_subtype(call, right) else: return False @@ -283,21 +286,22 @@ def visit_type_type(self, left: TypeType) -> bool: ASSUMING = [] # type: List[Tuple[Instance, Instance]] +ASSUMING_PROPER = [] # type: List[Tuple[Instance, Instance]] def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: - global ASSUMING assert right.type.is_protocol is_compat = is_subtype if allow_any else is_proper_subtype - for (l, r) in ASSUMING: + assuming = ASSUMING if allow_any else ASSUMING_PROPER + for (l, r) in reversed(assuming): if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True - ASSUMING.append((left, right)) + assuming.append((left, right)) for member in right.type.protocol_members: supertype = find_member(member, right) subtype = find_member(member, left) if not subtype or not is_compat(subtype, supertype): - ASSUMING.pop() + assuming.pop() return False return True @@ -629,6 +633,9 @@ def check_argument(left: Type, right: Type, variance: int) -> bool: return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in zip(t.args, s.args, s.type.defn.type_vars)) + if isinstance(s, CallableType): + call = find_member('__call__', t) + return call and is_proper_subtype(call, s) return False else: return sametypes.is_same_type(t, s) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 3b01bf5f57b3..ff35ee868734 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -296,12 +296,86 @@ class B2: x2: P2 = B2() # OK [out] +[case testProtocolsInMypyExtensions] +from mypy_extensions import Protocol, runtime + +@runtime +class P(Protocol): + def meth(self) -> int: + pass + +x: object +if isinstance(x, P): + reveal_type(x) # E: Revealed type is '__main__.P' + reveal_type(x.meth()) # E: Revealed type is 'builtins.int' + +class C: + def meth(self) -> int: + pass + +z: P = C() +[builtins fixtures/isinstancelist.pyi] +[out] + -- Generic protocol types -- ---------------------- +[case testBasicGenericProtocols] +from typing import Protocol, Sequence, TypeVar + +T = TypeVar('T') + +class Closeable(Protocol[T]): + def close(self) -> T: + pass + +class F: + def close(self) -> int: + return 0 + +def close(arg: Closeable[T]) -> T: + return arg.close() + +def close_all(args: Sequence[Closeable[T]]) -> T: + for arg in args: + return arg.close() + return args[0].close() + +arg: Closeable[int] + +reveal_type(close(F())) # E: Revealed type is 'builtins.int*' +reveal_type(close(arg)) # E: Revealed type is 'builtins.int*' +reveal_type(close_all([F()])) # E: Revealed type is 'builtins.int*' +reveal_type(close_all([arg])) # E: Revealed type is 'builtins.int*' +[builtins fixtures/isinstancelist.pyi] +[out] + -- Recursive protocol types -- ------------------------ +[case testBasicRecursiveProtocols] +from typing import Protocol, Sequence, List, Generic, TypeVar + +T = TypeVar('T') + +class Traversable(Protocol): + leaves: Sequence[Traversable] + +class C: pass + +class D(Generic[T]): + @property + def leaves(self) -> List[D[T]]: + pass + +t: Traversable +t = D[int]() # OK +t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable")\ + # N: __main__.C missing following __main__.Traversable protocol members:\ + # N: leaves +[builtins fixtures/list.pyi] +[out] + -- @property, @classmethod and @staticmethod in protocol types -- ----------------------------------------------------------- @@ -422,7 +496,7 @@ if isinstance(x, P): # E: Only @runtime protocols can be used with instance and if isinstance(x, R): reveal_type(x) # E: Revealed type is '__main__.R' - reveal_type(x.meth()) # E: Revealed type is 'builtins.int' # TODO: this should be Any + reveal_type(x.meth()) # E: Revealed type is 'builtins.int' [builtins fixtures/isinstance.pyi] [out] @@ -436,8 +510,157 @@ if isinstance(x, Iterable): [builtins fixtures/isinstancelist.pyi] [out] +[case testConcreteClassesInProtocolsIsInstance] +from typing import Protocol, runtime, TypeVar, Generic + +T = TypeVar('T') + +@runtime +class P1(Protocol): + def meth1(self) -> int: + pass +@runtime +class P2(Protocol): + def meth2(self) -> int: + pass +@runtime +class P(P1, P2, Protocol): + pass + +class C1(Generic[T]): + def meth1(self) -> T: + pass +class C2: + def meth2(self) -> int: + pass +class C(C1[int], C2): pass + +c = C() +if isinstance(c, P1): + reveal_type(c) # E: Revealed type is '__main__.C' +else: + reveal_type(c) # Unreachable +if isinstance(c, P): + reveal_type(c) # E: Revealed type is '__main__.C' +else: + reveal_type(c) # Unreachable + +c1i: C1[int] +if isinstance(c1i, P1): + reveal_type(c1i) # E: Revealed type is '__main__.C1[builtins.int]' +else: + reveal_type(c1i) # Unreachable +if isinstance(c1i, P): + reveal_type(c1i) # Unreachable +else: + reveal_type(c1i) # E: Revealed type is '__main__.C1[builtins.int]' + +c1s: C1[str] +if isinstance(c1s, P1): + reveal_type(c1s) # Unreachable +else: + reveal_type(c1s) # E: Revealed type is '__main__.C1[builtins.str]' + +c2: C2 +if isinstance(c2, P): + reveal_type(c2) # Unreachable +else: + reveal_type(c2) # E: Revealed type is '__main__.C2' + +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testConcreteClassesUnionInProtocolsIsInstance] +from typing import Protocol, runtime, TypeVar, Generic, Union + +T = TypeVar('T') + +@runtime +class P1(Protocol): + def meth1(self) -> int: + pass +@runtime +class P2(Protocol): + def meth2(self) -> int: + pass + +class C1(Generic[T]): + def meth1(self) -> T: + pass +class C2: + def meth2(self) -> int: + pass + +x: Union[C1[int], C2] +if isinstance(x, P1): + reveal_type(x) # E: Revealed type is '__main__.C1[builtins.int]' +else: + reveal_type(x) # E: Revealed type is '__main__.C2' + +if isinstance(x, P2): + reveal_type(x) # E: Revealed type is '__main__.C2' +else: + reveal_type(x) # E: Revealed type is '__main__.C1[builtins.int]' +[builtins fixtures/isinstancelist.pyi] +[out] + -- Non-Instances and protocol types (Callable vs __call__ etc.) -- ------------------------------------------------------------ +[case testBasicCallableStructuralSubtyping] +from typing import Callable, Generic, TypeVar + +def apply(f: Callable[[int], int], x: int) -> int: + return f(x) + +class Add5: + def __call__(self, x: int) -> int: + return x + 5 + +apply(Add5(), 5) + +T = TypeVar('T') +def apply_gen(f: Callable[[T], T]) -> T: + pass + +reveal_type(apply_gen(Add5())) # E: Revealed type is 'builtins.int*' +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testStructuralSupportForPartial] +from typing import Callable, TypeVar, Generic, Any + +T = TypeVar('T') + +class partial(Generic[T]): + def __init__(self, func: Callable[..., T], *args: Any) -> None: ... + def __call__(self, *args: Any) -> T: ... + +def inc(a: int, temp: str) -> int: + pass + +def foo(f: Callable[[int], T]) -> T: + return f(1) + +reveal_type(foo(partial(inc, 'temp'))) # E: Revealed type is 'builtins.int*' +[builtins fixtures/list.pyi] +[out] + -- Standard protocol types (SupportsInt, Reversible, etc.) -- ------------------------------------------------------- + +[case testBasicSupportsIntProtocol] +from typing import SupportsInt + +class Bar: + def __int__(self): + return 1 + +def foo(a: SupportsInt): + return int(a) + +foo(1) +foo(Bar()) + +[builtins fixtures/isinstancelist.pyi] +[out] diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index 57f064eba684..ee3dabc29d82 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -7,3 +7,4 @@ def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: pass class NoReturn: pass class Protocol: pass +def runtime(x: T) -> T: pass diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 35c40af45485..b52ed8502f3a 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -112,6 +112,9 @@ class Mapping(Protocol[T, U]): class MutableMapping(Mapping[T, U], Protocol): def __setitem__(self, k: T, v: U) -> None: pass +class SupportsInt(Protocol): + def __int__(self) -> int: pass + def NewType(name: str, tp: Type[T]) -> Callable[[T], T]: def new_type(x): return x From 93beb75e28fed08e8901f483f963f81723d33d98 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Mar 2017 22:12:53 +0200 Subject: [PATCH 017/117] Fix lint and tests --- mypy/constraints.py | 2 ++ mypy/join.py | 2 +- mypy/subtypes.py | 14 +++++++++++--- test-data/unit/check-protocols.test | 3 +-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index ae667dad5b50..ca3f57b2d342 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -408,6 +408,8 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: call = mypy.subtypes.find_member('__call__', self.actual) if call: return infer_constraints(template, call, self.direction) + else: + return [] else: return [] diff --git a/mypy/join.py b/mypy/join.py index b3514c830da7..057ad594965d 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -11,7 +11,7 @@ ) from mypy.maptype import map_instance_to_supertype from mypy.subtypes import ( - is_subtype, is_equivalent, is_subtype_ignoring_tvars,is_protocol_implementation + is_subtype, is_equivalent, is_subtype_ignoring_tvars, is_protocol_implementation ) from mypy import experiments diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ee2f6ff9382e..ea6968d3b859 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -158,7 +158,9 @@ def visit_instance(self, left: Instance) -> bool: return False if isinstance(right, CallableType): call = find_member('__call__', left) - return call and is_subtype(call, right) + if call: + return is_subtype(call, right) + return False else: return False @@ -291,7 +293,11 @@ def visit_type_type(self, left: TypeType) -> bool: def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: assert right.type.is_protocol - is_compat = is_subtype if allow_any else is_proper_subtype + is_compat = None # type: Callable[[Type, Type], bool] + if allow_any: + is_compat = is_subtype + else: + is_compat = is_proper_subtype assuming = ASSUMING if allow_any else ASSUMING_PROPER for (l, r) in reversed(assuming): if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): @@ -635,7 +641,9 @@ def check_argument(left: Type, right: Type, variance: int) -> bool: zip(t.args, s.args, s.type.defn.type_vars)) if isinstance(s, CallableType): call = find_member('__call__', t) - return call and is_proper_subtype(call, s) + if call: + return is_proper_subtype(call, s) + return False return False else: return sametypes.is_same_type(t, s) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index ff35ee868734..6f538de05de7 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -657,9 +657,8 @@ class Bar: return 1 def foo(a: SupportsInt): - return int(a) + pass -foo(1) foo(Bar()) [builtins fixtures/isinstancelist.pyi] From 5135fd957d9b33d4d0a2055c14f4450317b433c1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Mar 2017 22:42:30 +0200 Subject: [PATCH 018/117] Beautify error notes --- mypy/checker.py | 5 +++-- mypy/checkexpr.py | 5 +++-- test-data/unit/check-async-await.test | 30 +++++++++++++-------------- test-data/unit/check-namedtuple.test | 2 +- test-data/unit/check-protocols.test | 22 ++++++++++---------- 5 files changed, 33 insertions(+), 31 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f5931bf12783..798434594406 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2306,8 +2306,9 @@ def check_subtype(self, subtype: Type, supertype: Type, context: Context, isinstance(subtype, Instance)): missing = get_missing_members(subtype, supertype) if missing: - self.note('{} missing following {} protocol members:' - .format(subtype, supertype), context) + self.note("'{}' missing following '{}' protocol members:" + .format(subtype.type.fullname(), supertype.type.fullname()), + context) self.note(', '.join(missing), context) return False diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e3f497a8315a..7344c31c8ff0 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -871,8 +871,9 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, isinstance(callee_type, Instance) and callee_type.type.is_protocol): missing = get_missing_members(original_caller_type, callee_type) if missing: - messages.note('{} missing following {} protocol members:' - .format(original_caller_type, callee_type), context) + messages.note("'{}' missing following '{}' protocol members:" + .format(original_caller_type.type.fullname(), + callee_type.type.fullname()), context) messages.note(', '.join(missing), context) def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index be17c85cb154..3623007ebd70 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -85,7 +85,7 @@ async def f() -> int: return x [out] main:7: error: Incompatible types in await (actual type Generator[int, None, str], expected type Awaitable[Any]) -main:7: note: typing.Generator[builtins.int, builtins.None, builtins.str] missing following typing.Awaitable[Any] protocol members: +main:7: note: 'typing.Generator' missing following 'typing.Awaitable' protocol members: main:7: note: __await__ [case testAwaitIteratorError] @@ -98,7 +98,7 @@ async def f() -> int: return x [out] main:6: error: Incompatible types in await (actual type Iterator[Any], expected type Awaitable[Any]) -main:6: note: typing.Iterator[Any] missing following typing.Awaitable[Any] protocol members: +main:6: note: 'typing.Iterator' missing following 'typing.Awaitable' protocol members: main:6: note: __await__ [case testAwaitArgumentError] @@ -111,7 +111,7 @@ async def f() -> int: [builtins fixtures/async_await.pyi] [out] main:5: error: Incompatible types in await (actual type "int", expected type Awaitable[Any]) -main:5: note: builtins.int missing following typing.Awaitable[Any] protocol members: +main:5: note: 'builtins.int' missing following 'typing.Awaitable' protocol members: main:5: note: __await__ [case testAwaitResultError] @@ -156,7 +156,7 @@ async def f() -> None: [builtins fixtures/async_await.pyi] [out] main:4: error: AsyncIterable expected -main:4: note: builtins.list[builtins.int*] missing following typing.AsyncIterable[Any] protocol members: +main:4: note: 'builtins.list' missing following 'typing.AsyncIterable' protocol members: main:4: note: __aiter__ main:4: error: List[int] has no attribute "__aiter__" @@ -240,19 +240,19 @@ async def wrong_iterable(obj: Iterable[int]): [out] main:18: error: AsyncIterable expected -main:18: note: typing.Iterable[builtins.int] missing following typing.AsyncIterable[Any] protocol members: +main:18: note: 'typing.Iterable' missing following 'typing.AsyncIterable' protocol members: main:18: note: __aiter__ main:18: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:19: error: Iterable expected -main:19: note: __main__.asyncify[builtins.int*] missing following typing.Iterable[Any] protocol members: +main:19: note: '__main__.asyncify' missing following 'typing.Iterable' protocol members: main:19: note: __iter__ main:19: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? main:20: error: AsyncIterable expected -main:20: note: typing.Iterable[builtins.int] missing following typing.AsyncIterable[Any] protocol members: +main:20: note: 'typing.Iterable' missing following 'typing.AsyncIterable' protocol members: main:20: note: __aiter__ main:20: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:21: error: Iterable expected -main:21: note: __main__.asyncify[builtins.int*] missing following typing.Iterable[Any] protocol members: +main:21: note: '__main__.asyncify' missing following 'typing.Iterable' protocol members: main:21: note: __iter__ main:21: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? [builtins fixtures/async_await.pyi] @@ -288,7 +288,7 @@ class C: async def __aexit__(self, x, y, z) -> None: pass async def f() -> None: async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any])\ - # N: builtins.int missing following typing.Awaitable[Any] protocol members:\ + # N: 'builtins.int' missing following 'typing.Awaitable' protocol members:\ # N: __await__ pass [builtins fixtures/async_await.pyi] @@ -312,7 +312,7 @@ class C: def __aexit__(self, x, y, z) -> int: pass async def f() -> None: async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any])\ - # N: builtins.int missing following typing.Awaitable[Any] protocol members:\ + # N: 'builtins.int' missing following 'typing.Awaitable' protocol members:\ # N: __await__ pass [builtins fixtures/async_await.pyi] @@ -543,7 +543,7 @@ def h() -> None: [out] main:9: error: Iterable expected -main:9: note: typing.AsyncGenerator[builtins.int, builtins.None] missing following typing.Iterable[Any] protocol members: +main:9: note: 'typing.AsyncGenerator' missing following 'typing.Iterable' protocol members: main:9: note: __iter__ main:9: error: AsyncGenerator[int, None] has no attribute "__iter__"; maybe "__aiter__"? @@ -638,13 +638,13 @@ def plain_host_generator() -> Generator[str, None, None]: async def plain_host_coroutine() -> None: x = 0 x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any])\ - # N: typing.Generator[builtins.str, builtins.None, builtins.int] missing following typing.Awaitable[Any] protocol members:\ + # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members:\ # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any])\ - # N: __main__.It missing following typing.Awaitable[Any] protocol members:\ + # N: '__main__.It' missing following 'typing.Awaitable' protocol members:\ # N: __await__ x = await other_coroutine() @@ -663,13 +663,13 @@ def decorated_host_generator() -> Generator[str, None, None]: async def decorated_host_coroutine() -> None: x = 0 x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any])\ - # N: typing.Generator[builtins.str, builtins.None, builtins.int] missing following typing.Awaitable[Any] protocol members:\ + # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members:\ # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any])\ - # N: __main__.It missing following typing.Awaitable[Any] protocol members:\ + # N: '__main__.It' missing following 'typing.Awaitable' protocol members:\ # N: __await__ x = await other_coroutine() diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index e8faaebfcd8e..825feb53fa42 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -287,7 +287,7 @@ from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) reveal_type(X._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any]\ - # N: builtins.str missing following typing.Iterable[Any] protocol members:\ + # N: 'builtins.str' missing following 'typing.Iterable' protocol members:\ # N: __iter__ -- # FIX: not a proper class method diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 6f538de05de7..bf89ea879762 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -33,19 +33,19 @@ def fun(x: P) -> None: x = C() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ - # N: __main__.B missing following __main__.P protocol members:\ + # N: '__main__.B' missing following '__main__.P' protocol members:\ # N: meth fun(C()) fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P"\ - # N: __main__.B missing following __main__.P protocol members:\ + # N: '__main__.B' missing following '__main__.P' protocol members:\ # N: meth def fun2() -> P: return C() def fun3() -> P: return B() # E: Incompatible return value type (got "B", expected "P")\ - # N: __main__.B missing following __main__.P protocol members:\ + # N: '__main__.B' missing following '__main__.P' protocol members:\ # N: meth [out] @@ -77,12 +77,12 @@ def fun(x: P) -> None: x = C() x = D() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ - # N: __main__.B missing following __main__.P protocol members:\ + # N: '__main__.B' missing following '__main__.P' protocol members:\ # N: meth fun(C()) fun(D()) fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P"\ - # N: __main__.B missing following __main__.P protocol members:\ + # N: '__main__.B' missing following '__main__.P' protocol members:\ # N: meth [out] @@ -109,12 +109,12 @@ def fun(x: SubP) -> str: z = x x = C() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP")\ - # N: __main__.B missing following __main__.SubP protocol members:\ + # N: '__main__.B' missing following '__main__.SubP' protocol members:\ # N: meth reveal_type(fun(C())) # E: Revealed type is 'builtins.str' fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP"\ - # N: __main__.B missing following __main__.SubP protocol members:\ + # N: '__main__.B' missing following '__main__.SubP' protocol members:\ # N: meth [out] @@ -159,10 +159,10 @@ y: AnotherP x = c x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ - # N: __main__.B missing following __main__.P protocol members:\ + # N: '__main__.B' missing following '__main__.P' protocol members:\ # N: meth1, meth2 x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P")\ - # N: __main__.C1 missing following __main__.P protocol members:\ + # N: '__main__.C1' missing following '__main__.P' protocol members:\ # N: meth2 x = c2 x = y @@ -195,7 +195,7 @@ reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' x = C() # OK x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2")\ - # N: __main__.Cbad missing following __main__.P2 protocol members:\ + # N: '__main__.Cbad' missing following '__main__.P2' protocol members:\ # N: meth2 [out] @@ -371,7 +371,7 @@ class D(Generic[T]): t: Traversable t = D[int]() # OK t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable")\ - # N: __main__.C missing following __main__.Traversable protocol members:\ + # N: '__main__.C' missing following '__main__.Traversable' protocol members:\ # N: leaves [builtins fixtures/list.pyi] [out] From fdeb89b6008cdd1b2f77304b08b419a6143bcd44 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Mar 2017 10:47:45 +0200 Subject: [PATCH 019/117] Bind self to potential subtype; add tests for structural self-types --- mypy/constraints.py | 10 +++--- mypy/subtypes.py | 28 +++++++-------- test-data/unit/check-protocols.test | 55 ++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index ca3f57b2d342..f9a6a75f0244 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -323,8 +323,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: mypy.subtypes.is_subtype(instance, erase_typevars(template))): INFERRING.append(template.type) for member in template.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance) - temp = mypy.subtypes.find_member(member, template) + inst = mypy.subtypes.find_member(member, instance, instance) + temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) INFERRING.pop() return res @@ -333,8 +333,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: mypy.subtypes.is_subtype(erase_typevars(template), instance)): INFERRING.append(instance.type) for member in instance.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance) - temp = mypy.subtypes.find_member(member, template) + inst = mypy.subtypes.find_member(member, instance, instance) + temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) INFERRING.pop() return res @@ -405,7 +405,7 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: elif isinstance(self.actual, TypeType): return infer_constraints(template.ret_type, self.actual.item, self.direction) elif isinstance(self.actual, Instance): - call = mypy.subtypes.find_member('__call__', self.actual) + call = mypy.subtypes.find_member('__call__', self.actual, self.actual) if call: return infer_constraints(template, call, self.direction) else: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ea6968d3b859..5ab8c1ccc321 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -157,7 +157,7 @@ def visit_instance(self, left: Instance) -> bool: else: return False if isinstance(right, CallableType): - call = find_member('__call__', left) + call = find_member('__call__', left, left) if call: return is_subtype(call, right) return False @@ -304,15 +304,15 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool return True assuming.append((left, right)) for member in right.type.protocol_members: - supertype = find_member(member, right) - subtype = find_member(member, left) + supertype = find_member(member, right, left) + subtype = find_member(member, left, left) if not subtype or not is_compat(subtype, supertype): assuming.pop() return False return True -def find_member(name: str, itype: Instance) -> Optional[Type]: +def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type]: info = itype.type method = info.get_method(name) if method: @@ -320,8 +320,8 @@ def find_member(name: str, itype: Instance) -> Optional[Type]: assert isinstance(method, OverloadedFuncDef) dec = method.items[0] assert isinstance(dec, Decorator) - return find_var_type(dec.var, itype) - return map_method(method, itype) + return find_var_type(dec.var, itype, subtype) + return map_method(method, itype, subtype) else: node = info.get(name) if not node: @@ -331,12 +331,12 @@ def find_member(name: str, itype: Instance) -> Optional[Type]: if isinstance(v, Decorator): v = v.var if isinstance(v, Var): - return find_var_type(v, itype) + return find_var_type(v, itype, subtype) if not v and name not in ['__getattr__', '__setattr__', '__getattribute__']: for method_name in ('__getattribute__', '__getattr__'): method = info.get_method(method_name) if method and method.info.fullname() != 'builtins.object': - getattr_type = map_method(method, itype) + getattr_type = map_method(method, itype, subtype) if isinstance(getattr_type, CallableType): return getattr_type.ret_type if itype.type.fallback_to_any: @@ -344,7 +344,7 @@ def find_member(name: str, itype: Instance) -> Optional[Type]: return None -def find_var_type(var: Var, itype: Instance) -> Type: +def find_var_type(var: Var, itype: Instance, subtype: Instance) -> Type: from mypy.checkmember import bind_self itype = map_instance_to_supertype(itype, var.info) typ = var.type @@ -352,7 +352,7 @@ def find_var_type(var: Var, itype: Instance) -> Type: return AnyType() typ = expand_type_by_instance(typ, itype) if isinstance(typ, FunctionLike) and not var.is_staticmethod: - signature = bind_self(typ) + signature = bind_self(typ, subtype) assert isinstance(signature, CallableType) if var.is_property: return signature.ret_type @@ -360,10 +360,10 @@ def find_var_type(var: Var, itype: Instance) -> Type: return typ -def map_method(method: FuncBase, itype: Instance) -> Type: +def map_method(method: FuncBase, itype: Instance, subtype: Instance) -> Type: from mypy.checkmember import bind_self signature = function_type(method, Instance(itype.type.mro[-1], [])) - signature = bind_self(signature) + signature = bind_self(signature, subtype) itype = map_instance_to_supertype(itype, method.info) return expand_type_by_instance(signature, itype) @@ -371,7 +371,7 @@ def map_method(method: FuncBase, itype: Instance) -> Type: def get_missing_members(left: Instance, right: Instance) -> List[str]: missing = [] # type: List[str] for member in right.type.protocol_members: - if not find_member(member, left): + if not find_member(member, left, left): missing.append(member) return sorted(missing) @@ -640,7 +640,7 @@ def check_argument(left: Type, right: Type, variance: int) -> bool: return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in zip(t.args, s.args, s.type.defn.type_vars)) if isinstance(s, CallableType): - call = find_member('__call__', t) + call = find_member('__call__', t, t) if call: return is_proper_subtype(call, s) return False diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index bf89ea879762..5e624093b864 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -338,7 +338,7 @@ def close(arg: Closeable[T]) -> T: def close_all(args: Sequence[Closeable[T]]) -> T: for arg in args: - return arg.close() + arg.close() return args[0].close() arg: Closeable[int] @@ -350,6 +350,59 @@ reveal_type(close_all([arg])) # E: Revealed type is 'builtins.int*' [builtins fixtures/isinstancelist.pyi] [out] +[case testSelfTypesWithProtocols] +from typing import Protocol, TypeVar + +T = TypeVar('T', bound=Shape) + +class Shape(Protocol): + def combine(self: T, other: T) -> T: + pass + +class Triangle: + def combine(self, other: Shape) -> Shape: + pass + +class Rectangle: + def combine(self, other: T) -> T: + pass + +class Circle: + def combine(self: T, other: Shape) -> T: + pass + +class Bad: + def combine(self, other: int) -> str: + pass + +class AnotherShape(Protocol): + def combine(self: T, other: T) -> T: + pass + +class NonProtoShape: + def combine(self: T, other: T) -> T: + pass + +s: Shape +a: AnotherShape +t = Triangle() +r = Rectangle() +c = Circle() +n = NonProtoShape() + +# This compatibility matrix matches those for nominal subclasses; should be updated if the latter changes +s = t # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") +s = r +s = c +s3: Shape = c +s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") +s = a +s = n + +s2: Shape +n2: NonProtoShape = s2 # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") +[out] + -- Recursive protocol types -- ------------------------ From 1fbcb4a2e6353f67e24de00f01373242f7e0c3cd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Mar 2017 10:57:19 +0200 Subject: [PATCH 020/117] One more test --- test-data/unit/check-protocols.test | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 5e624093b864..2473bea2523a 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -359,6 +359,10 @@ class Shape(Protocol): def combine(self: T, other: T) -> T: pass +class Template(Protocol): + def combine(self, other: Template) -> Template: + pass + class Triangle: def combine(self, other: Shape) -> Shape: pass @@ -389,8 +393,11 @@ t = Triangle() r = Rectangle() c = Circle() n = NonProtoShape() +tm: Template + +# The compatibility matrix below matches behavior for *nominal* subclasses with self-types; +# should be updated if the latter changes -# This compatibility matrix matches those for nominal subclasses; should be updated if the latter changes s = t # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") s = r s = c @@ -398,7 +405,8 @@ s3: Shape = c s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") s = a s = n - +tm = n # E: Incompatible types in assignment (expression has type "NonProtoShape", variable has type "Template") +tm = r s2: Shape n2: NonProtoShape = s2 # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") [out] @@ -432,7 +440,7 @@ t = C() # E: Incompatible types in assignment (expression has type "C", variable -- @property, @classmethod and @staticmethod in protocol types -- ----------------------------------------------------------- --- Meet and join of protocol types +-- Join and meet of protocol types -- ------------------------------- -- Unions of protocol types From d012e387f9c7a341839250183d1afb612a3af17d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Mar 2017 12:23:55 +0200 Subject: [PATCH 021/117] Move global data to nodes.TypeInfo --- mypy/constraints.py | 17 +++++++---------- mypy/nodes.py | 12 +++++++++++- mypy/subtypes.py | 8 ++------ 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index f9a6a75f0244..9c67bda37c3b 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -261,9 +261,6 @@ def visit_uninhabited_type(self, t: UninhabitedType) -> bool: return False -INFERRING = [] # type: List[nodes.TypeInfo] - - class ConstraintBuilderVisitor(TypeVisitor[List[Constraint]]): """Visitor class for inferring type constraints.""" @@ -311,7 +308,7 @@ def visit_type_var(self, template: TypeVarType) -> List[Constraint]: # Non-leaf types def visit_instance(self, template: Instance) -> List[Constraint]: - global INFERRING + inferring = nodes.TypeInfo.inferring actual = self.actual res = [] # type: List[Constraint] if isinstance(actual, CallableType) and actual.fallback is not None: @@ -319,24 +316,24 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if isinstance(actual, Instance): instance = actual if (template.type.is_protocol and self.direction == SUPERTYPE_OF and - template.type not in INFERRING and + template.type not in inferring and mypy.subtypes.is_subtype(instance, erase_typevars(template))): - INFERRING.append(template.type) + inferring.append(template.type) for member in template.type.protocol_members: inst = mypy.subtypes.find_member(member, instance, instance) temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) - INFERRING.pop() + inferring.pop() return res elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and - instance.type not in INFERRING and + instance.type not in inferring and mypy.subtypes.is_subtype(erase_typevars(template), instance)): - INFERRING.append(instance.type) + inferring.append(instance.type) for member in instance.type.protocol_members: inst = mypy.subtypes.find_member(member, instance, instance) temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) - INFERRING.pop() + inferring.pop() return res if (self.direction == SUBTYPE_OF and template.type.has_base(instance.type.fullname())): diff --git a/mypy/nodes.py b/mypy/nodes.py index 1828468e1d3b..c6ce942c0e38 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4,7 +4,7 @@ from abc import abstractmethod from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, ClassVar ) import mypy.strconv @@ -1925,6 +1925,16 @@ class is generic then it will be a type constructor of higher kind. runtime_protocol = False # Does this protocol support isinstance checks? abstract_attributes = None # type: List[str] protocol_members = None # type: List[str] + + # These represent global structural subtype matrices. + # If concurrent/parallel type checking will be added in future, + # then there should be one matrix per thread/process to avoid false negatives + # during the type checking phase. + assuming = [] # type: ClassVar[List[Tuple[Instance, Instance]]] + assuming_proper = [] # type: ClassVar[List[Tuple[Instance, Instance]]] + # Ditto for temporary stack of recursive constraint inference. + inferring = [] # type: ClassVar[List[TypeInfo]] + # Classes inheriting from Enum shadow their true members with a __getattr__, so we # have to treat them as a special case. is_enum = False diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5ab8c1ccc321..8b9628df9822 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -13,7 +13,7 @@ from mypy import messages, sametypes from mypy.nodes import ( CONTRAVARIANT, COVARIANT, FuncBase, Var, Decorator, OverloadedFuncDef, - ARG_POS, ARG_OPT, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, + ARG_POS, ARG_OPT, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeInfo ) from mypy.maptype import map_instance_to_supertype from mypy.expandtype import expand_type_by_instance @@ -287,10 +287,6 @@ def visit_type_type(self, left: TypeType) -> bool: return False -ASSUMING = [] # type: List[Tuple[Instance, Instance]] -ASSUMING_PROPER = [] # type: List[Tuple[Instance, Instance]] - - def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: assert right.type.is_protocol is_compat = None # type: Callable[[Type, Type], bool] @@ -298,7 +294,7 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool is_compat = is_subtype else: is_compat = is_proper_subtype - assuming = ASSUMING if allow_any else ASSUMING_PROPER + assuming = TypeInfo.assuming if allow_any else TypeInfo.assuming_proper for (l, r) in reversed(assuming): if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True From 801770dd530d4c9450c769102dd5edfd4d8e580c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Mar 2017 15:47:33 +0200 Subject: [PATCH 022/117] Add comments and docstrings; recognize and treat abstract variables --- mypy/constraints.py | 6 ++++ mypy/fastparse.py | 2 +- mypy/join.py | 1 + mypy/nodes.py | 8 +++-- mypy/semanal.py | 13 ++++++-- mypy/subtypes.py | 31 ++++++++++++++++++- test-data/unit/check-protocols.test | 46 +++++++++++++++++++++++++++++ 7 files changed, 101 insertions(+), 6 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 9c67bda37c3b..af91250c1277 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -316,6 +316,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if isinstance(actual, Instance): instance = actual if (template.type.is_protocol and self.direction == SUPERTYPE_OF and + # We avoid infinite recursion for structural subtypes by checking + # whether this TypeInfo already appeared in the inference chain. template.type not in inferring and mypy.subtypes.is_subtype(instance, erase_typevars(template))): inferring.append(template.type) @@ -326,6 +328,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: inferring.pop() return res elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and + # We avoid infinite recursion for structural subtypes by checking + # whether this TypeInfo already appeared in the inference chain. instance.type not in inferring and mypy.subtypes.is_subtype(erase_typevars(template), instance)): inferring.append(instance.type) @@ -402,6 +406,8 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: elif isinstance(self.actual, TypeType): return infer_constraints(template.ret_type, self.actual.item, self.direction) elif isinstance(self.actual, Instance): + # Instances with __call__ method defined are considered structural + # subtypes of Callable with a compatible signature. call = mypy.subtypes.find_member('__call__', self.actual, self.actual) if call: return infer_constraints(template, call, self.direction) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 1699f351f4f0..1dc5f05ae965 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -517,7 +517,7 @@ def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt: @with_line def visit_AnnAssign(self, n: ast3.AnnAssign) -> AssignmentStmt: if n.value is None: # always allow 'x: int' - rvalue = TempNode(AnyType()) # type: Expression + rvalue = TempNode(AnyType(), no_rhs=True) # type: Expression else: rvalue = self.visit(n.value) typ = TypeConverter(self.errors, line=n.lineno).visit(n.annotation) diff --git a/mypy/join.py b/mypy/join.py index 057ad594965d..f03e82597453 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -159,6 +159,7 @@ def visit_instance(self, t: Instance) -> Type: structural = t if self.s.type.is_protocol and is_protocol_implementation(t, self.s): structural = self.s + # structural type for join is preferred if not structural or is_better(nominal, structural): return nominal return structural diff --git a/mypy/nodes.py b/mypy/nodes.py index c6ce942c0e38..97749fee2d3f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -666,6 +666,7 @@ class Var(SymbolNode): is_property = False is_settable_property = False is_classvar = False + is_abstract_var = False # Set to true when this variable refers to a module we were unable to # parse for some reason (eg a silenced module) is_suppressed_import = False @@ -673,7 +674,7 @@ class Var(SymbolNode): FLAGS = [ 'is_self', 'is_ready', 'is_initialized_in_class', 'is_staticmethod', 'is_classmethod', 'is_property', 'is_settable_property', 'is_suppressed_import', - 'is_classvar' + 'is_classvar', 'is_abstract_var' ] def __init__(self, name: str, type: 'mypy.types.Type' = None) -> None: @@ -1881,9 +1882,12 @@ class TempNode(Expression): """ type = None # type: mypy.types.Type + # Is it used to indicate no right hand side in assignment? + no_rhs = False # type: bool - def __init__(self, typ: 'mypy.types.Type') -> None: + def __init__(self, typ: 'mypy.types.Type', no_rhs: bool = False) -> None: self.type = typ + self.no_rhs = no_rhs def __repr__(self) -> str: return 'TempNode(%s)' % str(self.type) diff --git a/mypy/semanal.py b/mypy/semanal.py index adf812961005..e6e84c5c39dd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -749,6 +749,10 @@ def calculate_abstract_status(self, typ: TypeInfo) -> None: if fdef.is_abstract and name not in concrete: typ.is_abstract = True abstract.append(name) + elif isinstance(node, Var): + if node.is_abstract_var and name not in concrete: + typ.is_abstract = True + abstract.append(name) concrete.add(name) typ.abstract_attributes = sorted(abstract) @@ -1500,6 +1504,10 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: if s.type: allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) s.type = self.anal_type(s.type, allow_tuple_literal) + if (self.type and self.type.is_protocol and isinstance(lval, NameExpr) and + isinstance(s.rvalue, TempNode)) and s.rvalue.no_rhs: + if isinstance(lval.node, Var): + lval.node.is_abstract_var = True else: # For simple assignments, allow binding type aliases. # Also set the type if the rvalue is a simple literal. @@ -1695,9 +1703,10 @@ def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], def analyze_member_lvalue(self, lval: MemberExpr) -> None: lval.accept(self) + node = self.type.get(lval.name) if (self.is_self_member_ref(lval) and - self.type.get(lval.name) is None): - if self.type.is_protocol: + (node is None or isinstance(node.node, Var) and node.node.is_abstract_var)): + if self.type.is_protocol and node is None: # Protocol members can't be defined via self self.fail("Protocol members cannot be defined via assignment to self", lval) else: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 8b9628df9822..a8e31079ed83 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -151,12 +151,13 @@ def visit_instance(self, left: Instance) -> bool: if isinstance(item, Instance): return is_subtype(left, item.type.metaclass_type) elif isinstance(item, AnyType): - # Special case: all metaclasses are subtypes of Type[Any] + # Special case: all metaclasses are subtypes of Type[Any]. mro = left.type.mro or [] return any(base.fullname() == 'builtins.type' for base in mro) else: return False if isinstance(right, CallableType): + # Special case: Instance can by a subtype of Callable. call = find_member('__call__', left, left) if call: return is_subtype(call, right) @@ -288,6 +289,12 @@ def visit_type_type(self, left: TypeType) -> bool: def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: + """Check whether 'left' implements the protocol 'right'. If 'allow_any' is False, then + check for a proper subtype. Treat recursive protocols by using a global 'assuming' + structural subtype matrix (in sparse representation). If concurrent type checking + will be implemented, then every thread/process should use its own matrix + (see comment in nodes.TypeInfo). + """ assert right.type.is_protocol is_compat = None # type: Callable[[Type, Type], bool] if allow_any: @@ -309,6 +316,9 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type]: + """Find the type of member by 'name' in 'itype's TypeInfo. Apply type arguments + from 'itype', and bind 'self' to 'subtype'. Return None if member was not found. + """ info = itype.type method = info.get_method(name) if method: @@ -319,6 +329,7 @@ def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type] return find_var_type(dec.var, itype, subtype) return map_method(method, itype, subtype) else: + # don't have such method, maybe variable or decorator? node = info.get(name) if not node: v = None @@ -330,6 +341,10 @@ def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type] return find_var_type(v, itype, subtype) if not v and name not in ['__getattr__', '__setattr__', '__getattribute__']: for method_name in ('__getattribute__', '__getattr__'): + # Normally, mypy assumes that instances that define __getattr__ have all + # attributes with the corresponding return type. If this will produce + # many false negatives, then this could be prohibited for + # structural subtyping. method = info.get_method(method_name) if method and method.info.fullname() != 'builtins.object': getattr_type = map_method(method, itype, subtype) @@ -341,12 +356,16 @@ def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type] def find_var_type(var: Var, itype: Instance, subtype: Instance) -> Type: + """Find type of a variable 'var' (maybe also a decorated method). + Apply type arguments from 'itype', and bind 'self' to 'subtype'. + """ from mypy.checkmember import bind_self itype = map_instance_to_supertype(itype, var.info) typ = var.type if typ is None: return AnyType() typ = expand_type_by_instance(typ, itype) + # We don't need to bind 'self' for static methods, since there is no 'self'. if isinstance(typ, FunctionLike) and not var.is_staticmethod: signature = bind_self(typ, subtype) assert isinstance(signature, CallableType) @@ -357,6 +376,12 @@ def find_var_type(var: Var, itype: Instance, subtype: Instance) -> Type: def map_method(method: FuncBase, itype: Instance, subtype: Instance) -> Type: + """Map 'method' to the base where it was defined. Apply type arguments + from 'itype', and bind 'self' type to 'subtype'. + This function should be used only for non-decorated methods. Decorated + methods (including @staticmethod and @property) are treated + by 'find_var_type'. + """ from mypy.checkmember import bind_self signature = function_type(method, Instance(itype.type.mro[-1], [])) signature = bind_self(signature, subtype) @@ -365,6 +390,10 @@ def map_method(method: FuncBase, itype: Instance, subtype: Instance) -> Type: def get_missing_members(left: Instance, right: Instance) -> List[str]: + """Find all protocol members of 'right' that are not implemented + (i.e. completely missing) in 'left'. + """ + assert right.type.is_protocol missing = [] # type: List[str] for member in right.type.protocol_members: if not find_member(member, left, left): diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 2473bea2523a..b0c8a628a376 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -440,6 +440,52 @@ t = C() # E: Incompatible types in assignment (expression has type "C", variable -- @property, @classmethod and @staticmethod in protocol types -- ----------------------------------------------------------- +[case testCannotInstantiateAbstractMethodExplicitProtocolSubtypes] +from typing import Protocol +from abc import abstractmethod + +class P(Protocol): + @abstractmethod + def meth(self) -> int: + pass + +class A(P): + pass + +A() # E: Cannot instantiate abstract class 'A' with abstract attribute 'meth' + +class C(A): + def meth(self) -> int: + pass +class C2(P): + def meth(self) -> int: + pass + +C() +C2() +[out] + +[case testCannotInstantiateAbstractVariableExplicitProtocolSubtypes] +from typing import Protocol + +class P(Protocol): + attr: int + +class A(P): + pass + +A() # E: Cannot instantiate abstract class 'A' with abstract attribute 'attr' + +class C(A): + attr: int +class C2(P): + def __init__(self) -> None: + self.attr = 1 + +C() +C2() +[out] + -- Join and meet of protocol types -- ------------------------------- From a625f372d2b81a6a7729d8e3033c99896cfa88d6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Mar 2017 16:10:50 +0200 Subject: [PATCH 023/117] Fix tests and lint --- mypy/nodes.py | 4 ++-- mypy/semanal.py | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 97749fee2d3f..10b49dfd3454 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1934,8 +1934,8 @@ class is generic then it will be a type constructor of higher kind. # If concurrent/parallel type checking will be added in future, # then there should be one matrix per thread/process to avoid false negatives # during the type checking phase. - assuming = [] # type: ClassVar[List[Tuple[Instance, Instance]]] - assuming_proper = [] # type: ClassVar[List[Tuple[Instance, Instance]]] + assuming = [] # type: ClassVar[List[Tuple[mypy.types.Instance, mypy.types.Instance]]] + assuming_proper = [] # type: ClassVar[List[Tuple[mypy.types.Instance, mypy.types.Instance]]] # Ditto for temporary stack of recursive constraint inference. inferring = [] # type: ClassVar[List[TypeInfo]] diff --git a/mypy/semanal.py b/mypy/semanal.py index e6e84c5c39dd..df283fc76de7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1703,22 +1703,22 @@ def analyze_tuple_or_list_lvalue(self, lval: Union[ListExpr, TupleExpr], def analyze_member_lvalue(self, lval: MemberExpr) -> None: lval.accept(self) - node = self.type.get(lval.name) - if (self.is_self_member_ref(lval) and - (node is None or isinstance(node.node, Var) and node.node.is_abstract_var)): - if self.type.is_protocol and node is None: - # Protocol members can't be defined via self - self.fail("Protocol members cannot be defined via assignment to self", lval) - else: - # Implicit attribute definition in __init__. - lval.is_def = True - v = Var(lval.name) - v.set_line(lval) - v.info = self.type - v.is_ready = False - lval.def_var = v - lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) + if self.is_self_member_ref(lval): + node = self.type.get(lval.name) + if node is None or isinstance(node.node, Var) and node.node.is_abstract_var: + if self.type.is_protocol and node is None: + # Protocol members can't be defined via self + self.fail("Protocol members cannot be defined via assignment to self", lval) + else: + # Implicit attribute definition in __init__. + lval.is_def = True + v = Var(lval.name) + v.set_line(lval) + v.info = self.type + v.is_ready = False + lval.def_var = v + lval.node = v + self.type.names[lval.name] = SymbolTableNode(MDEF, v) self.check_lvalue_validity(lval.node, lval) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: From adb68ebffec20b6367c12b824ead7acba2f3fc5d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Mar 2017 16:51:28 +0200 Subject: [PATCH 024/117] Add two more tests for recursive --- test-data/unit/check-protocols.test | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index b0c8a628a376..ce7b47871f62 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -437,6 +437,55 @@ t = C() # E: Incompatible types in assignment (expression has type "C", variable [builtins fixtures/list.pyi] [out] +[case testComplexRecursiveProtocols] +from typing import Protocol, TypeVar + +T = TypeVar('T') + +class Linked(Protocol[T]): + val: T + next: Linked[T] + +class L: + val: int + next: L + +def last(seq: Linked[T]) -> T: + pass + +reveal_type(last(L())) # E: Revealed type is 'builtins.int*' + +[builtins fixtures/list.pyi] +[out] + +[case testMutuallyRecursiveProtocols] +from typing import Protocol, Sequence, List + +class P1(Protocol): + attr1: Sequence[P2] + +class P2(Protocol): + attr2: Sequence[P1] + +class C: pass + +class A: + attr1: List[B] + +class B: + attr2: List[A] + +t: P1 +t = A() # OK +t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1")\ + # N: '__main__.B' missing following '__main__.P1' protocol members:\ + # N: attr1 +t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1")\ + # N: '__main__.C' missing following '__main__.P1' protocol members:\ + # N: attr1 +[builtins fixtures/list.pyi] +[out] + -- @property, @classmethod and @staticmethod in protocol types -- ----------------------------------------------------------- From b9a0b2d3ecdd9425b0f8dc9f0fb68227e7bdbc11 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Mar 2017 17:08:01 +0200 Subject: [PATCH 025/117] Add test for Sized --- test-data/unit/check-protocols.test | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index ce7b47871f62..1a0fac7901a9 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -802,8 +802,29 @@ reveal_type(foo(partial(inc, 'temp'))) # E: Revealed type is 'builtins.int*' [builtins fixtures/list.pyi] [out] --- Standard protocol types (SupportsInt, Reversible, etc.) --- ------------------------------------------------------- +-- Standard protocol types (SupportsInt, Sized, etc.) +-- -------------------------------------------------- + +-- More tests could be added for types from typing converted to protocols + +[case testBasicSizedProtocol] +from typing import Sized + +class Foo: + def __len__(self) -> int: + return 42 + +def bar(a: Sized) -> int: + return a.__len__() + +bar(Foo()) +bar(1) + +[builtins fixtures/isinstancelist.pyi] +[out] +main:11: error: Argument 1 to "bar" has incompatible type "int"; expected "Sized" +main:11: note: 'builtins.int' missing following 'typing.Sized' protocol members: +main:11: note: __len__ [case testBasicSupportsIntProtocol] from typing import SupportsInt @@ -816,6 +837,10 @@ def foo(a: SupportsInt): pass foo(Bar()) +foo('no way') [builtins fixtures/isinstancelist.pyi] [out] +main:11: error: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" +main:11: note: 'builtins.str' missing following 'typing.SupportsInt' protocol members: +main:11: note: __int__ From 969f76f308369b7e92591124bade00248af891ed Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 1 Apr 2017 13:39:39 +0200 Subject: [PATCH 026/117] Fix crash in typeanal.py; ignore positional arg names for subtyping; allow fallback to nominal subtyping --- mypy/subtypes.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index a8e31079ed83..ac6f79f0ac04 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -60,8 +60,11 @@ def is_subtype(left: Type, right: Type, elif is_named_instance(right, 'builtins.type'): return is_subtype(left, TypeType(AnyType())) else: - return left.accept(SubtypeVisitor(right, type_parameter_checker, - ignore_pos_arg_names=ignore_pos_arg_names)) + result = left.accept(SubtypeVisitor(right, type_parameter_checker, + ignore_pos_arg_names=ignore_pos_arg_names)) + # Useful for debugging + # print(left, right, result) + return result def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: @@ -134,7 +137,10 @@ def visit_instance(self, left: Instance) -> bool: return True rname = right.type.fullname() if right.type.is_protocol: - return is_protocol_implementation(left, right) + if is_protocol_implementation(left, right): + return True + # always try a nominal check (even if structural returns False) + # there might be errors that a user wants to silence *once* if not left.type.has_base(rname) and rname != 'builtins.object': return False @@ -296,20 +302,30 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool (see comment in nodes.TypeInfo). """ assert right.type.is_protocol - is_compat = None # type: Callable[[Type, Type], bool] - if allow_any: - is_compat = is_subtype - else: - is_compat = is_proper_subtype assuming = TypeInfo.assuming if allow_any else TypeInfo.assuming_proper for (l, r) in reversed(assuming): if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True assuming.append((left, right)) + if right.type.protocol_members is None: + # This type has not been yet analyzed, probably a call from make_simplified_union + assuming.pop() + return False for member in right.type.protocol_members: supertype = find_member(member, right, left) subtype = find_member(member, left, left) - if not subtype or not is_compat(subtype, supertype): + # Useful for debugging: + # print(member, 'of', left, 'has type', subtype) + # print(member, 'of', right, 'has type', supertype) + if not subtype: + assuming.pop() + return False + if allow_any: + # nominal check currently ignore arg names + is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) + else: + is_compat = is_proper_subtype(subtype, supertype) + if not is_compat: assuming.pop() return False return True From 2e5de3e555e687f33349a13f9e59493edbccd736 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 1 Apr 2017 16:40:46 +0200 Subject: [PATCH 027/117] Change order of structural and nominal checks to boost speed; minor fixes --- mypy/constraints.py | 47 ++++++++++++++-------------- mypy/nodes.py | 2 ++ mypy/semanal.py | 3 ++ mypy/subtypes.py | 10 +++--- test-data/unit/lib-stub/builtins.pyi | 2 +- test-data/unit/lib-stub/typing.pyi | 9 +++--- 6 files changed, 39 insertions(+), 34 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index af91250c1277..054f6ad9dc0b 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -315,30 +315,6 @@ def visit_instance(self, template: Instance) -> List[Constraint]: actual = actual.fallback if isinstance(actual, Instance): instance = actual - if (template.type.is_protocol and self.direction == SUPERTYPE_OF and - # We avoid infinite recursion for structural subtypes by checking - # whether this TypeInfo already appeared in the inference chain. - template.type not in inferring and - mypy.subtypes.is_subtype(instance, erase_typevars(template))): - inferring.append(template.type) - for member in template.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance, instance) - temp = mypy.subtypes.find_member(member, template, template) - res.extend(infer_constraints(temp, inst, self.direction)) - inferring.pop() - return res - elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and - # We avoid infinite recursion for structural subtypes by checking - # whether this TypeInfo already appeared in the inference chain. - instance.type not in inferring and - mypy.subtypes.is_subtype(erase_typevars(template), instance)): - inferring.append(instance.type) - for member in instance.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance, instance) - temp = mypy.subtypes.find_member(member, template, template) - res.extend(infer_constraints(temp, inst, self.direction)) - inferring.pop() - return res if (self.direction == SUBTYPE_OF and template.type.has_base(instance.type.fullname())): mapped = map_instance_to_supertype(template, instance.type) @@ -362,6 +338,29 @@ def visit_instance(self, template: Instance) -> List[Constraint]: res.extend(infer_constraints( template.args[j], mapped.args[j], neg_op(self.direction))) return res + if (template.type.is_protocol and self.direction == SUPERTYPE_OF and + # We avoid infinite recursion for structural subtypes by checking + # whether this TypeInfo already appeared in the inference chain. + template.type not in inferring and + mypy.subtypes.is_subtype(instance, erase_typevars(template))): + inferring.append(template.type) + for member in template.type.protocol_members: + inst = mypy.subtypes.find_member(member, instance, instance) + temp = mypy.subtypes.find_member(member, template, template) + res.extend(infer_constraints(temp, inst, self.direction)) + inferring.pop() + return res + elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and + # We avoid infinite recursion for structural subtypes also here. + instance.type not in inferring and + mypy.subtypes.is_subtype(erase_typevars(template), instance)): + inferring.append(instance.type) + for member in instance.type.protocol_members: + inst = mypy.subtypes.find_member(member, instance, instance) + temp = mypy.subtypes.find_member(member, template, template) + res.extend(infer_constraints(temp, inst, self.direction)) + inferring.pop() + return res if isinstance(actual, AnyType): # IDEA: Include both ways, i.e. add negation as well? return self.infer_against_any(template.args) diff --git a/mypy/nodes.py b/mypy/nodes.py index f4111abd5347..ad27e251d0fa 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2036,6 +2036,8 @@ def is_generic(self) -> bool: return len(self.type_vars) > 0 def get(self, name: str) -> 'SymbolTableNode': + if self.mro is None: # Might be because of a previous error. + return None for cls in self.mro: n = cls.names.get(name) if n: diff --git a/mypy/semanal.py b/mypy/semanal.py index e538eec9d1df..f06a3645faf9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3450,6 +3450,9 @@ def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) - ('False', bool_type), ('__debug__', bool_type), ]) + else: + # We are running tests + literal_types.append(('True', AnyType())) for name, typ in literal_types: v = Var(name, typ) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ac6f79f0ac04..f648cec125c0 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -136,12 +136,12 @@ def visit_instance(self, left: Instance) -> bool: ignore_pos_arg_names=self.ignore_pos_arg_names): return True rname = right.type.fullname() - if right.type.is_protocol: - if is_protocol_implementation(left, right): - return True - # always try a nominal check (even if structural returns False) - # there might be errors that a user wants to silence *once* + # Always try a nominal check if possible, + # there might be errors that a user wants to silence *once*. + # Also, nominal checks are much faster. if not left.type.has_base(rname) and rname != 'builtins.object': + if right.type.is_protocol: + return is_protocol_implementation(left, right) return False # Map left type to corresponding right instances. diff --git a/test-data/unit/lib-stub/builtins.pyi b/test-data/unit/lib-stub/builtins.pyi index 5010235a53ab..44552ae7400f 100644 --- a/test-data/unit/lib-stub/builtins.pyi +++ b/test-data/unit/lib-stub/builtins.pyi @@ -17,7 +17,7 @@ class bytes: pass class tuple: pass class function: pass - +class bool: pass class ellipsis: pass # Definition of None is implicit diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index b52ed8502f3a..f07369e5cf79 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -26,6 +26,7 @@ Dict = 0 Set = 0 T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') @@ -42,14 +43,14 @@ class Sized(Protocol): def __len__(self) -> int: pass @runtime -class Iterable(Protocol[T]): +class Iterable(Protocol[T_co]): @abstractmethod - def __iter__(self) -> 'Iterator[T]': pass + def __iter__(self) -> 'Iterator[T_co]': pass @runtime -class Iterator(Iterable[T], Protocol): +class Iterator(Iterable[T_co], Protocol): @abstractmethod - def __next__(self) -> T: pass + def __next__(self) -> T_co: pass class Generator(Iterator[T], Generic[T, U, V]): @abstractmethod From 2b6d19833b201183de21ea74d9c3f2360af914aa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 1 Apr 2017 21:51:45 +0200 Subject: [PATCH 028/117] Add structural support for tuples and named tuples; add tests --- mypy/constraints.py | 5 + test-data/unit/check-protocols.test | 265 ++++++++++++++++++++- test-data/unit/fixtures/isinstancelist.pyi | 13 +- 3 files changed, 275 insertions(+), 8 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 054f6ad9dc0b..9ea5c32433eb 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -374,6 +374,11 @@ def visit_instance(self, template: Instance) -> List[Constraint]: cb = infer_constraints(template.args[0], item, SUPERTYPE_OF) res.extend(cb) return res + elif isinstance(actual, TupleType) and template.type.is_protocol: + if mypy.subtypes.is_subtype(actual.fallback, erase_typevars(template)): + res.extend(infer_constraints(template, actual.fallback, self.direction)) + return res + return [] else: return [] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 1a0fac7901a9..373dd37fe350 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -350,6 +350,87 @@ reveal_type(close_all([arg])) # E: Revealed type is 'builtins.int*' [builtins fixtures/isinstancelist.pyi] [out] +[case testProtocolBasicGenericInference] +from typing import Generic, TypeVar, Protocol +T = TypeVar('T') +S = TypeVar('S') + +class P(Protocol[T, S]): + x: T + y: S + +class C: + x: int + y: int + +def fun3(x: P[T, T]) -> T: + pass + +reveal_type(fun3(C())) # E: Revealed type is 'builtins.int*' +[out] + +[case testGenericSubProtocols] +from typing import TypeVar, Protocol, Tuple, Generic + +T = TypeVar('T') +S = TypeVar('S') + +class P1(Protocol[T]): + attr1: T +class P2(P1[T], Protocol[T, S]): + attr2: Tuple[T, S] + +class C: + def __init__(self, a1: int, a2: Tuple[int, int]) -> None: + self.attr1 = a1 + self.attr2 = a2 +c: C + +var: P2[int, int] = c +var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str]) + +class D(Generic[T]): + attr1: T + +def f(x: T) -> T: + y: P2[T, T] = D[T]() # E: Incompatible types in assignment (expression has type D[T], variable has type P2[T, T])\ + # N: '__main__.D' missing following '__main__.P2' protocol members:\ + # N: attr2 + return x +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testGenericSubProtocolsExtension] +from typing import TypeVar, Protocol, Union + +T = TypeVar('T') +S = TypeVar('S') + +class P1(Protocol[T]): + attr1: T +class P2(Protocol[T]): + attr2: T + +class P(P1[T], P2[S], Protocol): + pass + +class C: + attr1: int + attr2: str + +class A: + attr1: A +class B: + attr2: B +class D(A, B): pass + +x: P = D() + +var: P[Union[int, P], Union[P, str]] = C() # OK +var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) + +[out] + [case testSelfTypesWithProtocols] from typing import Protocol, TypeVar @@ -535,8 +616,129 @@ C() C2() [out] --- Join and meet of protocol types --- ------------------------------- +-- Join and meet with protocol types +-- --------------------------------- + +[case testJoinProtocolWithProtocol] +from typing import Protocol + +class P(Protocol): + attr: int +class P2(Protocol): + attr: int + attr2: str + +x: P +y: P2 + +l = [x, y] + +reveal_type(l) # E: Revealed type is 'builtins.list[__main__.P*]' + +[builtins fixtures/list.pyi] +[out] + +[case testJoinProtocolWithNormal] +from typing import Protocol + +class P(Protocol): + attr: int + +class C: + attr: int + +x: P +y: C + +l = [x, y] + +reveal_type(l) # E: Revealed type is 'builtins.list[__main__.P*]' + +[builtins fixtures/list.pyi] +[out] + +[case testMeetProtocolWithProtocol] +from typing import Protocol, Callable, TypeVar + +class P(Protocol): + attr: int +class P2(Protocol): + attr: int + attr2: str + +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: P, y: P2) -> None: pass +reveal_type(f(g)) # E: Revealed type is '__main__.P2*' +[out] + +[case testMeetProtocolWithNormal] +from typing import Protocol, Callable, TypeVar + +class P(Protocol): + attr: int +class C: + attr: int + +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: P, y: C) -> None: pass +reveal_type(f(g)) # E: Revealed type is '__main__.C*' +[out] + +[case testInferProtocolFromProtocol] +from typing import Protocol, Sequence, TypeVar, Union + +class Box(Protocol): + content: int + +T = TypeVar('T') +class Linked(Protocol[T]): + val: T + next: Linked[T] + +class L: + val: Box + next: L + +def last(seq: Linked[T]) -> T: + pass + +reveal_type(last(L())) # E: Revealed type is '__main__.Box*' +[out] + +[case testOverloadOnProtocol] +from typing import overload, Protocol, runtime + +@runtime +class P1(Protocol): + attr1: int +class P2(Protocol): + attr2: str + +class C1: + attr1: int +class C2: + attr2: str +class C: pass + +@overload +def f(x: P1) -> int: ... +@overload +def f(x: P2) -> str: ... +def f(x): + if isinstance(x, P1): + return P1.attr1 + if isinstance(x, P2): # E: Only @runtime protocols can be used with instance and class checks + return P1.attr2 + +reveal_type(f(C1())) # E: Revealed type is 'builtins.int' +reveal_type(f(C2())) # E: Revealed type is 'builtins.str' + +f(C()) # E: No overload variant of "f" matches argument types [__main__.C] + +[builtins fixtures/isinstance.pyi] +[out] -- Unions of protocol types -- ------------------------ @@ -763,6 +965,65 @@ else: -- Non-Instances and protocol types (Callable vs __call__ etc.) -- ------------------------------------------------------------ +[case testBasicTupleStructuralSubtyping] +from typing import Tuple, TypeVar, Protocol + +T = TypeVar('T') + +class MyProto(Protocol[T]): + def __len__(self) -> T: + pass + +t: Tuple[int, str] + +def f(x: MyProto[int]) -> None: + pass + +f(t) # OK +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testBasicNamedTupleStructuralSubtyping] +from typing import NamedTuple, TypeVar, Protocol + +T = TypeVar('T') +S = TypeVar('S') + +class P(Protocol[T, S]): + x: T + y: S + +class N(NamedTuple): + x: int + y: str +class N2(NamedTuple): + x: int +class N3(NamedTuple): + x: int + y: int + +z: N +z3: N3 + +def fun(x: P[int, str]) -> None: + pass +def fun2(x: P[int, int]) -> None: + pass +def fun3(x: P[T, T]) -> T: + return x.x + +fun(z) +fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int] + +fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str] + +reveal_type(fun3(z)) # E: Revealed type is 'builtins.object*' + +reveal_type(fun3(z3)) # E: Revealed type is 'builtins.int*' +[builtins fixtures/isinstancelist.pyi] +[out] + + [case testBasicCallableStructuralSubtyping] from typing import Callable, Generic, TypeVar diff --git a/test-data/unit/fixtures/isinstancelist.pyi b/test-data/unit/fixtures/isinstancelist.pyi index 04a6e78fe0d1..5f569258cc25 100644 --- a/test-data/unit/fixtures/isinstancelist.pyi +++ b/test-data/unit/fixtures/isinstancelist.pyi @@ -1,4 +1,4 @@ -from typing import builtinclass, Iterable, Iterator, TypeVar, List, Mapping, overload, Tuple, Set, Union, Generic +from typing import builtinclass, Iterable, Iterator, TypeVar, List, Mapping, overload, Tuple, Set, Union, Generic, Any @builtinclass class object: @@ -8,7 +8,12 @@ class object: class type: def __init__(self, x) -> None: pass -class tuple: pass +T = TypeVar('T') +KT = TypeVar('KT') +VT = TypeVar('VT') + +class tuple(Generic[T]): + def __len__(self) -> int: pass class function: pass def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass @@ -23,10 +28,6 @@ class str: def __add__(self, x: str) -> str: pass def __getitem__(self, x: int) -> str: pass -T = TypeVar('T') -KT = TypeVar('KT') -VT = TypeVar('VT') - class list(Generic[T]): def __iter__(self) -> Iterator[T]: pass def __mul__(self, x: int) -> list[T]: pass From 937c629f1d100bea12649f094549fea3eea29db7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 2 Apr 2017 01:35:05 +0200 Subject: [PATCH 029/117] Add support for staticmethod, classmethod, property, ClassVar + tests and refactoring --- mypy/subtypes.py | 96 ++++++++++--- test-data/unit/check-protocols.test | 210 +++++++++++++++++++++++++++- 2 files changed, 280 insertions(+), 26 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index f648cec125c0..b86847058266 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,4 +1,5 @@ -from typing import List, Optional, Dict, Callable, Tuple +from typing import List, Optional, Dict, Callable, Tuple, Iterator +from contextlib import contextmanager from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, FormalArgument, NoneTyp, function_type, @@ -21,6 +22,12 @@ from mypy import experiments +# Flags for detected protocol members +IS_SETTABLE = 1 +IS_CLASSVAR = 2 +IS_CLASS_OR_STATIC = 3 + + TypeParameterChecker = Callable[[Type, Type, int], bool] @@ -294,6 +301,12 @@ def visit_type_type(self, left: TypeType) -> bool: return False +@contextmanager +def pop_on_exit(lst: List[Tuple[Type, Type]]) -> Iterator[None]: + yield + lst.pop() + + def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: """Check whether 'left' implements the protocol 'right'. If 'allow_any' is False, then check for a proper subtype. Treat recursive protocols by using a global 'assuming' @@ -306,28 +319,37 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool for (l, r) in reversed(assuming): if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True - assuming.append((left, right)) - if right.type.protocol_members is None: - # This type has not been yet analyzed, probably a call from make_simplified_union - assuming.pop() - return False - for member in right.type.protocol_members: - supertype = find_member(member, right, left) - subtype = find_member(member, left, left) - # Useful for debugging: - # print(member, 'of', left, 'has type', subtype) - # print(member, 'of', right, 'has type', supertype) - if not subtype: - assuming.pop() - return False - if allow_any: - # nominal check currently ignore arg names - is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) - else: - is_compat = is_proper_subtype(subtype, supertype) - if not is_compat: - assuming.pop() + with pop_on_exit(assuming): + assuming.append((left, right)) + if right.type.protocol_members is None: + # This type has not been yet analyzed, probably a call from make_simplified_union return False + for member in right.type.protocol_members: + supertype = find_member(member, right, left) + subtype = find_member(member, left, left) + # Useful for debugging: + # print(member, 'of', left, 'has type', subtype) + # print(member, 'of', right, 'has type', supertype) + if not subtype: + return False + if allow_any: + # nominal check currently ignore arg names + is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) + else: + is_compat = is_proper_subtype(subtype, supertype) + if not is_compat: + return False + subflags = get_member_flags(member, left.type) + superflags = get_member_flags(member, right.type) + if (IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags or + IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags): + return False + if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: + return False + # this rule is copied from nominal check in checker.py + if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: + return False + assuming.append((left, right)) return True @@ -371,6 +393,36 @@ def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type] return None +def get_member_flags(name: str, info: TypeInfo) -> List[int]: + """Detect whether a member 'name' is settable and whether it is an + instance or class variable. + """ + method = info.get_method(name) + if method: + # this could be settable property + if method.is_property: + assert isinstance(method, OverloadedFuncDef) + dec = method.items[0] + assert isinstance(dec, Decorator) + if dec.var.is_settable_property: + return [IS_SETTABLE] + return [] + node = info.get(name) + if not node: + return [] + v = node.node + if isinstance(v, Decorator): + if v.var.is_staticmethod or v.var.is_classmethod: + return [IS_CLASS_OR_STATIC] + # just a variable + if isinstance(v, Var): + flags = [IS_SETTABLE] + if v.is_classvar: + flags.append(IS_CLASSVAR) + return flags + return [] + + def find_var_type(var: Var, itype: Instance, subtype: Instance) -> Type: """Find type of a variable 'var' (maybe also a decorated method). Apply type arguments from 'itype', and bind 'self' to 'subtype'. diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 373dd37fe350..bb23929356ae 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -317,6 +317,22 @@ z: P = C() [builtins fixtures/isinstancelist.pyi] [out] +[case testProtocolsCannotInheritFromNormal] +from typing import Protocol + +class C: pass +class D: pass + +class P(C, Protocol): # E: All bases of a protocol must be protocols + attr: int + +class P2(P, D, Protocol): # E: All bases of a protocol must be protocols + pass + +p: P2 +reveal_type(p.attr) # E: Revealed type is 'builtins.int' +[out] + -- Generic protocol types -- ---------------------- @@ -482,7 +498,6 @@ tm: Template s = t # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") s = r s = c -s3: Shape = c s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") s = a s = n @@ -506,9 +521,7 @@ class Traversable(Protocol): class C: pass class D(Generic[T]): - @property - def leaves(self) -> List[D[T]]: - pass + leaves: List[D[T]] t: Traversable t = D[int]() # OK @@ -616,6 +629,136 @@ C() C2() [out] +[case testClassVarsInProtocols] +from typing import Protocol, ClassVar + +class PInst(Protocol): + v: int + +class PClass(Protocol): + v: ClassVar[int] + +class CInst: + v: int + +class CClass: + v: ClassVar[int] + +x: PInst +y: PClass + +x = CInst() +x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst") +y = CClass() +y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass") +[out] + +[case testPropertyInProtocols] +from typing import Protocol + +class PP(Protocol): + @property + def attr(self) -> int: + pass + +class P(Protocol): + attr: int + +x: P +y: PP +y = x + +x2: P +y2: PP +x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P") +[builtins fixtures/property.pyi] +[out] + +[case testSettablePropertyInProtocols] +from typing import Protocol + +class PPS(Protocol): + @property + def attr(self) -> int: + pass + @attr.setter + def attr(self, x: int) -> None: + pass + +class PP(Protocol): + @property + def attr(self) -> int: + pass + +class P(Protocol): + attr: int + +x: P +z: PPS +z = x + +x2: P +z2: PPS +x2 = z2 + +y3: PP +z3: PPS +y3 = z3 + +y4: PP +z4: PPS +z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS") +[builtins fixtures/property.pyi] +[out] + +[case testStaticAndClassMethodsInProtocols] +from typing import Protocol, Type, TypeVar + +class P(Protocol): + def meth(self, x: int) -> str: + pass + +class PC(Protocol): + @classmethod + def meth(cls, x: int) -> str: + pass + +class B: + @staticmethod + def meth(x: int) -> str: + pass + +class C: + def meth(self, x: int) -> str: + pass + +x: P +x = C() +x = B() + +y: PC +y = B() +y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC") +[builtins fixtures/classmethod.pyi] +[out] + +[case testOverloadedMethodsInProtocols] +from typing import overload, Protocol, Union + +class P(Protocol): + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass + def f(self, x): pass + +class C: + def f(self, x: Union[int, str]) -> None: + pass + +x: P = C() +[out] + -- Join and meet with protocol types -- --------------------------------- @@ -743,6 +886,65 @@ f(C()) # E: No overload variant of "f" matches argument types [__main__.C] -- Unions of protocol types -- ------------------------ +[case testBasicUnionsOfProtocols] +from typing import Union, Protocol + +class P1(Protocol): + attr1: int +class P2(Protocol): + attr2: int + +class C1: + attr1: int +class C2: + attr2: int +class C(C1, C2): + pass + +class B: ... + +x: Union[P1, P2] + +x = C1() +x = C2() +x = C() +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "Union[P1, P2]") +[out] + +[case testUnionsOfNormalClassesWithProtocols] +from typing import Protocol, Union + +class P1(Protocol): + attr1: int +class P2(Protocol): + attr2: int + +class C1: + attr1: int +class C2: + attr2: int +class C(C1, C2): + pass + +class D1: + attr1: int + +def f1(x: P1) -> None: + pass +def f2(x: P2) -> None: + pass + +x: Union[C1, C2] +y: Union[C1, D1] +z: Union[C, D1] + +f1(x) # E: Argument 1 to "f1" has incompatible type "Union[C1, C2]"; expected "P1" +f1(y) +f1(z) +f2(x) # E: Argument 1 to "f2" has incompatible type "Union[C1, C2]"; expected "P2" +f2(z) # E: Argument 1 to "f2" has incompatible type "Union[C, D1]"; expected "P2" +[out] + -- Type[] with protocol types -- -------------------------- From e8e6661d9978c537b017ec2a4fe1559c5501a100 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 2 Apr 2017 12:37:14 +0200 Subject: [PATCH 030/117] Speed improvements; more carefull treatment of strucutral inference --- mypy/constraints.py | 15 ++++++++------- mypy/nodes.py | 14 +++++++++----- mypy/subtypes.py | 35 ++++++++++++++++++----------------- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 9ea5c32433eb..548b7d5c526c 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -308,13 +308,14 @@ def visit_type_var(self, template: TypeVarType) -> List[Constraint]: # Non-leaf types def visit_instance(self, template: Instance) -> List[Constraint]: - inferring = nodes.TypeInfo.inferring actual = self.actual res = [] # type: List[Constraint] if isinstance(actual, CallableType) and actual.fallback is not None: actual = actual.fallback if isinstance(actual, Instance): instance = actual + # We always try nominal inference if possible, + # it is much faster than the structural one. if (self.direction == SUBTYPE_OF and template.type.has_base(instance.type.fullname())): mapped = map_instance_to_supertype(template, instance.type) @@ -341,25 +342,25 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking # whether this TypeInfo already appeared in the inference chain. - template.type not in inferring and + not any(is_same_type(template, t) for t in template.type.inferring) and mypy.subtypes.is_subtype(instance, erase_typevars(template))): - inferring.append(template.type) + template.type.inferring.append(template) for member in template.type.protocol_members: inst = mypy.subtypes.find_member(member, instance, instance) temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) - inferring.pop() + template.type.inferring.pop() return res elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and # We avoid infinite recursion for structural subtypes also here. - instance.type not in inferring and + not any(is_same_type(instance, i) for i in instance.type.inferring) and mypy.subtypes.is_subtype(erase_typevars(template), instance)): - inferring.append(instance.type) + instance.type.inferring.append(instance) for member in instance.type.protocol_members: inst = mypy.subtypes.find_member(member, instance, instance) temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) - inferring.pop() + instance.type.inferring.pop() return res if isinstance(actual, AnyType): # IDEA: Include both ways, i.e. add negation as well? diff --git a/mypy/nodes.py b/mypy/nodes.py index ad27e251d0fa..d7d3deb22e81 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4,7 +4,7 @@ from abc import abstractmethod from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, ClassVar + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional ) import mypy.strconv @@ -1949,14 +1949,15 @@ class is generic then it will be a type constructor of higher kind. abstract_attributes = None # type: List[str] protocol_members = None # type: List[str] - # These represent global structural subtype matrices. + # These represent structural subtype matrices. Note that these are shared + # by all 'Instance's with given TypeInfo. # If concurrent/parallel type checking will be added in future, # then there should be one matrix per thread/process to avoid false negatives # during the type checking phase. - assuming = [] # type: ClassVar[List[Tuple[mypy.types.Instance, mypy.types.Instance]]] - assuming_proper = [] # type: ClassVar[List[Tuple[mypy.types.Instance, mypy.types.Instance]]] + assuming = None # type: List[Tuple[mypy.types.Instance, mypy.types.Instance]] + assuming_proper = None # type: List[Tuple[mypy.types.Instance, mypy.types.Instance]] # Ditto for temporary stack of recursive constraint inference. - inferring = [] # type: ClassVar[List[TypeInfo]] + inferring = None # type: List[mypy.types.Instance] # Classes inheriting from Enum shadow their true members with a __getattr__, so we # have to treat them as a special case. @@ -2020,6 +2021,9 @@ def __init__(self, names: 'SymbolTable', defn: ClassDef, module_name: str) -> No self._fullname = defn.fullname self.is_abstract = False self.abstract_attributes = [] + self.assuming = [] + self.assuming_proper = [] + self.inferring = [] if defn.type_vars: for vd in defn.type_vars: self.type_vars.append(vd.name) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index b86847058266..fe699c968c11 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Dict, Callable, Tuple, Iterator +from typing import List, Optional, Dict, Callable, Tuple, Iterator, Set from contextlib import contextmanager from mypy.types import ( @@ -13,8 +13,8 @@ # import mypy.solve from mypy import messages, sametypes from mypy.nodes import ( - CONTRAVARIANT, COVARIANT, FuncBase, Var, Decorator, OverloadedFuncDef, - ARG_POS, ARG_OPT, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeInfo + FuncBase, Var, Decorator, OverloadedFuncDef, TypeInfo, CONTRAVARIANT, COVARIANT, + ARG_POS, ARG_OPT, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2 ) from mypy.maptype import map_instance_to_supertype from mypy.expandtype import expand_type_by_instance @@ -145,7 +145,7 @@ def visit_instance(self, left: Instance) -> bool: rname = right.type.fullname() # Always try a nominal check if possible, # there might be errors that a user wants to silence *once*. - # Also, nominal checks are much faster. + # Also, nominal checks are faster. if not left.type.has_base(rname) and rname != 'builtins.object': if right.type.is_protocol: return is_protocol_implementation(left, right) @@ -302,9 +302,11 @@ def visit_type_type(self, left: TypeType) -> bool: @contextmanager -def pop_on_exit(lst: List[Tuple[Type, Type]]) -> Iterator[None]: +def pop_on_exit(stack: List[Tuple[Instance, Instance]], + left: Instance, right: Instance) -> Iterator[None]: + stack.append((left, right)) yield - lst.pop() + stack.pop() def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: @@ -315,12 +317,11 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool (see comment in nodes.TypeInfo). """ assert right.type.is_protocol - assuming = TypeInfo.assuming if allow_any else TypeInfo.assuming_proper + assuming = right.type.assuming if allow_any else right.type.assuming_proper for (l, r) in reversed(assuming): if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True - with pop_on_exit(assuming): - assuming.append((left, right)) + with pop_on_exit(assuming, left, right): if right.type.protocol_members is None: # This type has not been yet analyzed, probably a call from make_simplified_union return False @@ -393,7 +394,7 @@ def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type] return None -def get_member_flags(name: str, info: TypeInfo) -> List[int]: +def get_member_flags(name: str, info: TypeInfo) -> Set[int]: """Detect whether a member 'name' is settable and whether it is an instance or class variable. """ @@ -405,22 +406,22 @@ def get_member_flags(name: str, info: TypeInfo) -> List[int]: dec = method.items[0] assert isinstance(dec, Decorator) if dec.var.is_settable_property: - return [IS_SETTABLE] - return [] + return {IS_SETTABLE} + return set() node = info.get(name) if not node: - return [] + return set() v = node.node if isinstance(v, Decorator): if v.var.is_staticmethod or v.var.is_classmethod: - return [IS_CLASS_OR_STATIC] + return {IS_CLASS_OR_STATIC} # just a variable if isinstance(v, Var): - flags = [IS_SETTABLE] + flags = {IS_SETTABLE} if v.is_classvar: - flags.append(IS_CLASSVAR) + flags.add(IS_CLASSVAR) return flags - return [] + return set() def find_var_type(var: Var, itype: Instance, subtype: Instance) -> Type: From 8e6306f11d7ad4cf590e1399e2828d40f898e322 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 2 Apr 2017 18:59:56 +0200 Subject: [PATCH 031/117] Minor fixes and one more test --- mypy/meet.py | 4 +- mypy/semanal.py | 8 ++- mypy/subtypes.py | 75 +++++++++++------------ test-data/unit/check-namedtuple.test | 1 + test-data/unit/check-protocols.test | 92 +++++++++++++++++++--------- 5 files changed, 109 insertions(+), 71 deletions(-) diff --git a/mypy/meet.py b/mypy/meet.py index e4296f4f0a04..bbf9a5162c7d 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -92,11 +92,13 @@ class C(A, B): ... return True if s.type._promote and is_overlapping_types(s.type._promote, t): return True + if t.type in s.type.mro or s.type in t.type.mro: + return True if t.type.is_protocol and is_protocol_implementation(s, t): return True if s.type.is_protocol and is_protocol_implementation(t, s): return True - return t.type in s.type.mro or s.type in t.type.mro + return False if isinstance(t, UnionType): return any(is_overlapping_types(item, s) for item in t.items) diff --git a/mypy/semanal.py b/mypy/semanal.py index f06a3645faf9..a70c3efe270e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1505,7 +1505,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) s.type = self.anal_type(s.type, allow_tuple_literal) if (self.type and self.type.is_protocol and isinstance(lval, NameExpr) and - isinstance(s.rvalue, TempNode)) and s.rvalue.no_rhs: + isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs): if isinstance(lval.node, Var): lval.node.is_abstract_var = True else: @@ -1707,8 +1707,8 @@ def analyze_member_lvalue(self, lval: MemberExpr) -> None: if self.is_self_member_ref(lval): node = self.type.get(lval.name) if node is None or isinstance(node.node, Var) and node.node.is_abstract_var: + # Protocol members can't be defined via self if self.type.is_protocol and node is None: - # Protocol members can't be defined via self self.fail("Protocol members cannot be defined via assignment to self", lval) else: # Implicit attribute definition in __init__. @@ -3451,7 +3451,9 @@ def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) - ('__debug__', bool_type), ]) else: - # We are running tests + # We are running tests without 'bool' in builtins. + # TODO: Find a permanent solution to this problem. + # Maybe add 'bool' to all fixtures? literal_types.append(('True', AnyType())) for name, typ in literal_types: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index fe699c968c11..6168bd683f3d 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -69,7 +69,7 @@ def is_subtype(left: Type, right: Type, else: result = left.accept(SubtypeVisitor(right, type_parameter_checker, ignore_pos_arg_names=ignore_pos_arg_names)) - # Useful for debugging + # Useful for debugging: # print(left, right, result) return result @@ -145,18 +145,17 @@ def visit_instance(self, left: Instance) -> bool: rname = right.type.fullname() # Always try a nominal check if possible, # there might be errors that a user wants to silence *once*. - # Also, nominal checks are faster. - if not left.type.has_base(rname) and rname != 'builtins.object': - if right.type.is_protocol: - return is_protocol_implementation(left, right) - return False - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(left, right.type) - - return all(self.check_type_parameter(lefta, righta, tvar.variance) - for lefta, righta, tvar in - zip(t.args, right.args, right.type.defn.type_vars)) + if left.type.has_base(rname) or rname == 'builtins.object': + # Map left type to corresponding right instances. + t = map_instance_to_supertype(left, right.type) + nominal = all(self.check_type_parameter(lefta, righta, tvar.variance) + for lefta, righta, tvar in + zip(t.args, right.args, right.type.defn.type_vars)) + if nominal: + return True + if right.type.is_protocol and is_protocol_implementation(left, right): + return True + return False if isinstance(right, TypeType): item = right.item if isinstance(item, TupleType): @@ -170,7 +169,7 @@ def visit_instance(self, left: Instance) -> bool: else: return False if isinstance(right, CallableType): - # Special case: Instance can by a subtype of Callable. + # Special case: Instance can be a subtype of Callable. call = find_member('__call__', left, left) if call: return is_subtype(call, right) @@ -311,10 +310,8 @@ def pop_on_exit(stack: List[Tuple[Instance, Instance]], def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: """Check whether 'left' implements the protocol 'right'. If 'allow_any' is False, then - check for a proper subtype. Treat recursive protocols by using a global 'assuming' - structural subtype matrix (in sparse representation). If concurrent type checking - will be implemented, then every thread/process should use its own matrix - (see comment in nodes.TypeInfo). + check for a proper subtype. Treat recursive protocols by using the 'assuming' + structural subtype matrix (in sparse representation), see comment in nodes.TypeInfo. """ assert right.type.is_protocol assuming = right.type.assuming if allow_any else right.type.assuming_proper @@ -334,7 +331,7 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool if not subtype: return False if allow_any: - # nominal check currently ignore arg names + # nominal check currently ignores arg names is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) else: is_compat = is_proper_subtype(subtype, supertype) @@ -350,6 +347,9 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool # this rule is copied from nominal check in checker.py if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: return False + # This additional push serves as poor man's subtype cache. + # (This already gives a decent speed-up.) + # TODO: Implement a general fast subtype cache? assuming.append((left, right)) return True @@ -452,7 +452,7 @@ def map_method(method: FuncBase, itype: Instance, subtype: Instance) -> Type: by 'find_var_type'. """ from mypy.checkmember import bind_self - signature = function_type(method, Instance(itype.type.mro[-1], [])) + signature = function_type(method, fallback=Instance(itype.type.mro[-1], [])) signature = bind_self(signature, subtype) itype = map_instance_to_supertype(itype, method.info) return expand_type_by_instance(signature, itype) @@ -715,24 +715,23 @@ def is_proper_subtype(t: Type, s: Type) -> bool: if isinstance(t, Instance): if isinstance(s, Instance): - if s.type.is_protocol: - return is_protocol_implementation(t, s, allow_any=False) - if not t.type.has_base(s.type.fullname()): - return False - - def check_argument(left: Type, right: Type, variance: int) -> bool: - if variance == COVARIANT: - return is_proper_subtype(left, right) - elif variance == CONTRAVARIANT: - return is_proper_subtype(right, left) - else: - return sametypes.is_same_type(left, right) - - # Map left type to corresponding right instances. - t = map_instance_to_supertype(t, s.type) - - return all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in - zip(t.args, s.args, s.type.defn.type_vars)) + if t.type.has_base(s.type.fullname()): + def check_argument(left: Type, right: Type, variance: int) -> bool: + if variance == COVARIANT: + return is_proper_subtype(left, right) + elif variance == CONTRAVARIANT: + return is_proper_subtype(right, left) + else: + return sametypes.is_same_type(left, right) + # Map left type to corresponding right instances. + t = map_instance_to_supertype(t, s.type) + nominal = all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in + zip(t.args, s.args, s.type.defn.type_vars)) + if nominal: + return True + if s.type.is_protocol and is_protocol_implementation(t, s, allow_any=False): + return True + return False if isinstance(s, CallableType): call = find_member('__call__', t, t) if call: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 825feb53fa42..8d640caa04ee 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -289,6 +289,7 @@ reveal_type(X._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, built X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any]\ # N: 'builtins.str' missing following 'typing.Iterable' protocol members:\ # N: __iter__ +# Note that the above will not fail with real stubs, since 'str', is a subtype of 'Iterable[str]' -- # FIX: not a proper class method -- x = None # type: X diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index bb23929356ae..383b1f3c3a30 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -19,13 +19,11 @@ class P(Protocol): pass class B: pass - class C: def meth(self) -> None: pass x: P - def fun(x: P) -> None: x.meth() x.meth(x) # E: Too many arguments for "meth" of "P" @@ -59,11 +57,9 @@ class P(Protocol): pass class B: pass - class C: def meth(self) -> None: pass - class D(B): def meth(self) -> None: pass @@ -127,6 +123,8 @@ class P1(Protocol): class P2(Protocol): def meth2(self) -> str: pass +class P(P1, P2, Protocol): pass + class B: pass class C1: def meth1(self) -> int: @@ -146,8 +144,6 @@ class AnotherP(Protocol): def meth2(self) -> str: pass -class P(P1, P2, Protocol): pass - x: P reveal_type(x.meth1()) # E: Revealed type is 'builtins.int' reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' @@ -290,7 +286,7 @@ class P2(Protocol): def __init__(self) -> None: self.a = 1 -class B2: +class B2(P2): a: int x2: P2 = B2() # OK @@ -336,7 +332,41 @@ reveal_type(p.attr) # E: Revealed type is 'builtins.int' -- Generic protocol types -- ---------------------- -[case testBasicGenericProtocols] +[case testAutomaticProtocolVariance] +from typing import TypeVar, Protocol + +T = TypeVar('T') + +class Pco(Protocol[T]): + attr: T +class Pcontra(Protocol[T]): + def meth(self, x: T) -> None: + pass +class Pinv(Protocol[T]): + attr: T + def meth(self, x: T) -> None: + pass + +class A: pass +class B(A): pass + +x1: Pco[B] +y1: Pco[A] +x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B]) +y1 = x1 + +x2: Pcontra[B] +y2: Pcontra[A] +y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A]) +x2 = y2 + +x3: Pinv[B] +y3: Pinv[A] +y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A]) +x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B]) +[out] + +[case testGenericProtocolsInference1] from typing import Protocol, Sequence, TypeVar T = TypeVar('T') @@ -366,7 +396,7 @@ reveal_type(close_all([arg])) # E: Revealed type is 'builtins.int*' [builtins fixtures/isinstancelist.pyi] [out] -[case testProtocolBasicGenericInference] +[case testProtocolGenericInference2] from typing import Generic, TypeVar, Protocol T = TypeVar('T') S = TypeVar('S') @@ -381,8 +411,11 @@ class C: def fun3(x: P[T, T]) -> T: pass - reveal_type(fun3(C())) # E: Revealed type is 'builtins.int*' + +def fun4(x: T, y: P[T, T]) -> T: + pass +reveal_type(fun4('a', C())) # E: Revealed type is 'builtins.object*' [out] [case testGenericSubProtocols] @@ -400,15 +433,18 @@ class C: def __init__(self, a1: int, a2: Tuple[int, int]) -> None: self.attr1 = a1 self.attr2 = a2 -c: C +c: C var: P2[int, int] = c var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str]) class D(Generic[T]): attr1: T +class E(D[T]): + attr2: Tuple[T, T] def f(x: T) -> T: + z: P2[T, T] = E[T]() y: P2[T, T] = D[T]() # E: Incompatible types in assignment (expression has type D[T], variable has type P2[T, T])\ # N: '__main__.D' missing following '__main__.P2' protocol members:\ # N: attr2 @@ -426,7 +462,6 @@ class P1(Protocol[T]): attr1: T class P2(Protocol[T]): attr2: T - class P(P1[T], P2[S], Protocol): pass @@ -440,7 +475,7 @@ class B: attr2: B class D(A, B): pass -x: P = D() +x: P = D() # Same as P[Any, Any] var: P[Union[int, P], Union[P, str]] = C() # OK var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) @@ -493,8 +528,7 @@ n = NonProtoShape() tm: Template # The compatibility matrix below matches behavior for *nominal* subclasses with self-types; -# should be updated if the latter changes - +# these should be updated if the latter changes. s = t # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") s = r s = c @@ -510,7 +544,7 @@ n2: NonProtoShape = s2 # E: Incompatible types in assignment (expression has typ -- Recursive protocol types -- ------------------------ -[case testBasicRecursiveProtocols] +[case testRecursiveProtocols1] from typing import Protocol, Sequence, List, Generic, TypeVar T = TypeVar('T') @@ -531,11 +565,10 @@ t = C() # E: Incompatible types in assignment (expression has type "C", variable [builtins fixtures/list.pyi] [out] -[case testComplexRecursiveProtocols] +[case testRecursiveProtocols2] from typing import Protocol, TypeVar T = TypeVar('T') - class Linked(Protocol[T]): val: T next: Linked[T] @@ -548,7 +581,6 @@ def last(seq: Linked[T]) -> T: pass reveal_type(last(L())) # E: Revealed type is 'builtins.int*' - [builtins fixtures/list.pyi] [out] @@ -755,8 +787,12 @@ class P(Protocol): class C: def f(self, x: Union[int, str]) -> None: pass +class D: + def f(self, x: int) -> None: + pass x: P = C() +x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") [out] -- Join and meet with protocol types @@ -777,7 +813,6 @@ y: P2 l = [x, y] reveal_type(l) # E: Revealed type is 'builtins.list[__main__.P*]' - [builtins fixtures/list.pyi] [out] @@ -796,7 +831,6 @@ y: C l = [x, y] reveal_type(l) # E: Revealed type is 'builtins.list[__main__.P*]' - [builtins fixtures/list.pyi] [out] @@ -830,24 +864,24 @@ reveal_type(f(g)) # E: Revealed type is '__main__.C*' [out] [case testInferProtocolFromProtocol] -from typing import Protocol, Sequence, TypeVar, Union - -class Box(Protocol): - content: int +from typing import Protocol, Sequence, TypeVar, Generic T = TypeVar('T') +class Box(Protocol[T]): + content: T class Linked(Protocol[T]): val: T next: Linked[T] -class L: - val: Box - next: L +class L(Generic[T]): + val: Box[T] + next: L[T] def last(seq: Linked[T]) -> T: pass -reveal_type(last(L())) # E: Revealed type is '__main__.Box*' +reveal_type(last(L[int]())) # E: Revealed type is '__main__.Box*[builtins.int*]' +reveal_type(last(L[str]()).content) # E: Revealed type is 'builtins.str*' [out] [case testOverloadOnProtocol] From 3f2d7074e03cff1f0260d1f5685d85d7c110516b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 3 Apr 2017 00:26:19 +0200 Subject: [PATCH 032/117] Start adding some docs --- docs/source/class_basics.rst | 32 +++++++++++++++++++++++++++++--- docs/source/faq.rst | 33 ++++++++++++++++++--------------- docs/source/generics.rst | 2 ++ mypy/constraints.py | 2 +- mypy/join.py | 2 +- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index dc778d39424f..7996952573e7 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -151,7 +151,33 @@ concrete. As with normal overrides, a dynamically typed method can implement a statically typed abstract method defined in an abstract base class. -.. note:: +.. _protocol-types: + +Protocols and structural subtyping +********************************** + +Mypy provides support for structural subtyping and protocol classes. + +.. code-block:: python + + from typing import Iterator, Iterable + + class Bucket: + ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + + def collect(items: Iterable[int]) -> int: ... + result: int = collect(Bucket()) # Passes type check + +Users can also define protocols. + +.. code-block:: python + + from typing import Protocol + + class SupportsClose(Protocol): + def close(self) -> None: + ... - There are also plans to support more Python-style "duck typing" in - the type system. The details are still open. +Protocols can be generic. See :ref:`generic-classes`. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 9fd73b42b8a3..03a355f46487 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -101,35 +101,38 @@ Is mypy free? Yes. Mypy is free software, and it can also be used for commercial and proprietary projects. Mypy is available under the MIT license. -Why not use structural subtyping? -********************************* +Can I use structural subtyping? +******************************* -Mypy primarily uses `nominal subtyping -`_ instead of +Mypy provides support for both `nominal subtyping +`_ and `structural subtyping `_. Some argue that structural subtyping is better suited for languages with duck -typing such as Python. - -Here are some reasons why mypy uses nominal subtyping: +typing such as Python. Mypy however primarily uses nominal subtyping. +Here are some reasons why: 1. It is easy to generate short and informative error messages when using a nominal type system. This is especially important when using type inference. -2. Python supports basically nominal isinstance tests and they are - widely used in programs. It is not clear how to support isinstance - in a purely structural type system while remaining compatible with - Python idioms. +2. Python provides built-in support for nominal ``isinstance()`` tests and + they are widely used in programs. Only limited support for structural + ``isinstance()`` exists for ABCs in ``collections.abc`` and ``typing`` + standard library modules. 3. Many programmers are already familiar with nominal subtyping and it has been successfully used in languages such as Java, C++ and C#. Only few languages use structural subtyping. -However, structural subtyping can also be useful. Structural subtyping -is a likely feature to be added to mypy in the future, even though we -expect that most mypy programs will still primarily use nominal -subtyping. +However, structural subtyping can also be useful. For example, a "public API" +will be more flexible and convenient for users if it is typed with protocols. +Also, using protocol types removes the necessity to explicitly declare +implementations of ABCs. Finally, protocol types may feel more natural for +Python programmers. As a rule of thumb, we recommend using protocols for +function argument types and normal classes for return types. For more details +about protocol types and structural subtyping see :ref:`protocol-types` and +`PEP 544 `_. I like Python and I have no need for static typing ************************************************** diff --git a/docs/source/generics.rst b/docs/source/generics.rst index bd0e0549fd1a..17209cd7346b 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -1,6 +1,8 @@ Generics ======== +.. _generic-classes: + Defining generic classes ************************ diff --git a/mypy/constraints.py b/mypy/constraints.py index 548b7d5c526c..417c4ff6d9b7 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -341,7 +341,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: return res if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking - # whether this TypeInfo already appeared in the inference chain. + # whether this type already appeared in the inference chain. not any(is_same_type(template, t) for t in template.type.inferring) and mypy.subtypes.is_subtype(instance, erase_typevars(template))): template.type.inferring.append(template) diff --git a/mypy/join.py b/mypy/join.py index f03e82597453..90ca74789aa6 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -157,7 +157,7 @@ def visit_instance(self, t: Instance) -> Type: structural = None # type: Instance if t.type.is_protocol and is_protocol_implementation(self.s, t): structural = t - if self.s.type.is_protocol and is_protocol_implementation(t, self.s): + elif self.s.type.is_protocol and is_protocol_implementation(t, self.s): structural = self.s # structural type for join is preferred if not structural or is_better(nominal, structural): From 957c76dd36890deaa6d889cc4ed05e8921eedaea Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 3 Apr 2017 14:38:34 +0200 Subject: [PATCH 033/117] Add one more test --- test-data/unit/check-protocols.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 383b1f3c3a30..a3d4586cfffc 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -659,6 +659,12 @@ class C2(P): C() C2() + +class P2(Protocol): + attr: int = 1 + +class B(P2): pass +B() # OK, attr is not abstract [out] [case testClassVarsInProtocols] From 8cf08ec985e8569e44a8ecccdf6acd4103098dc3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 3 Apr 2017 22:27:11 +0200 Subject: [PATCH 034/117] Try to sync typeshed --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 48920fea7ee6..1ea3d2de5794 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 48920fea7ee6c1d413fba12fe709c652da065393 +Subproject commit 1ea3d2de5794c2224a1bc086aa471d0097699bf1 From c9629da00ec8f96cd51a5701b401881215921f16 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 4 Apr 2017 13:13:50 +0200 Subject: [PATCH 035/117] Simplify one test --- test-data/unit/check-protocols.test | 53 +++++++---------------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index a3d4586cfffc..816d69cc4143 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -482,63 +482,36 @@ var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignmen [out] -[case testSelfTypesWithProtocols] +[case testSelfTypesWithProtocolsBehaveAsWithNominal] from typing import Protocol, TypeVar T = TypeVar('T', bound=Shape) - class Shape(Protocol): def combine(self: T, other: T) -> T: pass -class Template(Protocol): - def combine(self, other: Template) -> Template: - pass - -class Triangle: - def combine(self, other: Shape) -> Shape: - pass - -class Rectangle: - def combine(self, other: T) -> T: +class NonProtoShape: + def combine(self: T, other: T) -> T: pass - class Circle: def combine(self: T, other: Shape) -> T: pass - +class Triangle: + def combine(self, other: Shape) -> Shape: + pass class Bad: def combine(self, other: int) -> str: pass -class AnotherShape(Protocol): - def combine(self: T, other: T) -> T: - pass - -class NonProtoShape: - def combine(self: T, other: T) -> T: - pass - +def f(s: Shape) -> None: pass s: Shape -a: AnotherShape -t = Triangle() -r = Rectangle() -c = Circle() -n = NonProtoShape() -tm: Template - -# The compatibility matrix below matches behavior for *nominal* subclasses with self-types; -# these should be updated if the latter changes. -s = t # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") -s = r -s = c + +f(NonProtoShape()) +f(Circle()) +s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") -s = a -s = n -tm = n # E: Incompatible types in assignment (expression has type "NonProtoShape", variable has type "Template") -tm = r -s2: Shape -n2: NonProtoShape = s2 # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") + +n2: NonProtoShape = s # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") [out] -- Recursive protocol types From affec0c14be03ed76c87ef020c75fe365582496a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 4 Apr 2017 15:45:26 +0200 Subject: [PATCH 036/117] Add more docs --- docs/source/class_basics.rst | 84 ++++++++++++++++++++++++++++++++---- docs/source/faq.rst | 6 +-- 2 files changed, 79 insertions(+), 11 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 7996952573e7..f8dcdb1293c8 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -157,6 +157,55 @@ Protocols and structural subtyping ********************************** Mypy provides support for structural subtyping and protocol classes. +To define a protocol class, one must inherit the special ``typing.Protocol`` +class: + +.. code-block:: python + + from typing import Protocol + + class SupportsClose(Protocol): + def close(self) -> None: + ... + + class UnrelatedClass: + # some methods + def close(self) -> None: + self.resource.release() + + def close_all(things: Sequence[SupportsClose]) -> None: + for thing in things: + thing.close() + + close_all([UnrelatedClass(), open('some/file')]) # This passes type check + +Subprotocols are also supported. Inheriting from an existing protocol does +not automatically turn a subclass into a protocol, it just creates a usual +ABC. The ``typing.Protocol`` base must always be explicitly present. +Generic and recursive protocols are also supported: + +.. code-block:: python + + from typing import Protocol, TypeVar + + T = TypeVar('T') + class Linked(Protocol[T]): + val: T + next: 'Linked[T]' + + class L: + val: int + next: 'L' + + def last(seq: Linked[T]) -> T: + ... + + result = last(L()) # The inferred type of 'result' is 'int' + +See :ref:`generic-classes` for more details on generic classes. +The standard ABCs in ``typing`` module are protocols, so that the following +class will be considered a subtype of ``typing.Sized`` and +``typing.Iterable[int]``: .. code-block:: python @@ -164,20 +213,39 @@ Mypy provides support for structural subtyping and protocol classes. class Bucket: ... - def __len__(self) -> int: ... - def __iter__(self) -> Iterator[int]: ... + def __len__(self) -> int: + return 22 + def __iter__(self) -> Iterator[int]: + yield 22 def collect(items: Iterable[int]) -> int: ... result: int = collect(Bucket()) # Passes type check -Users can also define protocols. +To use a protocol class with ``isinstance()``, one needs to decorate it with +a special ``typing.runtime`` decorator. It will add support for basic runtime +structural checks: .. code-block:: python - from typing import Protocol + from typing import Protocol, runtime - class SupportsClose(Protocol): - def close(self) -> None: - ... + @runtime + class Portable(Protocol): + handles: int + + class Mug: + def __init__(self) -> None: + self.handles = 1 + + mug = Mug() + if isinstance(mug, Portable): + use(mug.handles) # Works statically and at runtime. + +See `PEP 544 `_ for +specification of structural subtyping in Python. + +.. note:: -Protocols can be generic. See :ref:`generic-classes`. + The support for structural subtyping is still experimental. Some features + might be not yet implemented, mypy could pass unsafe code or reject + working code. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 03a355f46487..902dff25a20b 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -109,8 +109,8 @@ Mypy provides support for both `nominal subtyping `structural subtyping `_. Some argue that structural subtyping is better suited for languages with duck -typing such as Python. Mypy however primarily uses nominal subtyping. -Here are some reasons why: +typing such as Python. Mypy however primarily uses nominal subtyping, +leaving structural subtyping opt-in. Here are some reasons why: 1. It is easy to generate short and informative error messages when using a nominal type system. This is especially important when @@ -129,7 +129,7 @@ However, structural subtyping can also be useful. For example, a "public API" will be more flexible and convenient for users if it is typed with protocols. Also, using protocol types removes the necessity to explicitly declare implementations of ABCs. Finally, protocol types may feel more natural for -Python programmers. As a rule of thumb, we recommend using protocols for +Python programmers. As a rule of thumb, one could prefer protocols for function argument types and normal classes for return types. For more details about protocol types and structural subtyping see :ref:`protocol-types` and `PEP 544 `_. From 01948ddcb427f7c359214e6cfe248b7f423ff01c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 4 Apr 2017 17:49:00 +0200 Subject: [PATCH 037/117] Allow incompatible __init__ in structural subtypes --- mypy/subtypes.py | 5 ++++- test-data/unit/check-protocols.test | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 86fdfef8e8bd..0fa88d0986d1 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -327,9 +327,12 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool return True with pop_on_exit(assuming, left, right): if right.type.protocol_members is None: - # This type has not been yet analyzed, probably a call from make_simplified_union + # This type has not been yet analyzed, probably a call from make_simplified_union. return False for member in right.type.protocol_members: + # nominal subtyping currently ignores '__init__' and '__new__' signatures + if member in ('__init__', '__new__'): + continue supertype = find_member(member, right, left) subtype = find_member(member, left, left) # Useful for debugging: diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 816d69cc4143..76b4d2166226 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -276,9 +276,8 @@ class C: pass x: P -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") -# The above is because of incompatible __init__, maybe we could allow this? -# (mypy allows incompatible __init__ in nominal subclasses.) +x = B() +# The above has an incompatible __init__, but mypy ignores this for nominal subtypes? x = C(1) class P2(Protocol): From 0ae409a551ac3ccb98058007a06bbeac9ba5f3f0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 4 Apr 2017 19:07:56 +0200 Subject: [PATCH 038/117] Fix minor failure after merge --- test-data/unit/check-protocols.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 76b4d2166226..9e0c5349c899 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1167,7 +1167,8 @@ x: Union[C1[int], C2] if isinstance(x, P1): reveal_type(x) # E: Revealed type is '__main__.C1[builtins.int]' else: - reveal_type(x) # E: Revealed type is '__main__.C2' + # This happens because of rules in restrict_subtype_away, maybe we can relax rules for protocols? + reveal_type(x) # E: Revealed type is 'Union[__main__.C1[builtins.int], __main__.C2]' if isinstance(x, P2): reveal_type(x) # E: Revealed type is '__main__.C2' From cb27279feef4a82876b9a0d3a714da20f3d8ae31 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 4 Apr 2017 20:43:51 +0200 Subject: [PATCH 039/117] Re-run Travis From 798954ac83745b86e686970910344924cfbef4c3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 9 Apr 2017 23:54:47 +0200 Subject: [PATCH 040/117] Implement subtype cache --- mypy/nodes.py | 4 ++ mypy/subtypes.py | 16 +++-- mypy/types.py | 110 ++++++++++++++++++++++++++++++++ test-data/unit/check-lists.test | 2 +- typeshed | 2 +- 5 files changed, 128 insertions(+), 6 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index c997b534f460..f842c8c8ec5d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1964,6 +1964,8 @@ class is generic then it will be a type constructor of higher kind. assuming_proper = None # type: List[Tuple[mypy.types.Instance, mypy.types.Instance]] # Ditto for temporary stack of recursive constraint inference. inferring = None # type: List[mypy.types.Instance] + cache = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] + cache_proper = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] # Classes inheriting from Enum shadow their true members with a __getattr__, so we # have to treat them as a special case. @@ -2030,6 +2032,8 @@ def __init__(self, names: 'SymbolTable', defn: ClassDef, module_name: str) -> No self.assuming = [] self.assuming_proper = [] self.inferring = [] + self.cache = set() + self.cache_proper = set() self.add_type_vars() def add_type_vars(self) -> None: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index a610d8fdb6c4..f39ffec62808 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -147,10 +147,13 @@ def visit_instance(self, left: Instance) -> bool: if isinstance(right, TupleType) and right.fallback.type.is_enum: return is_subtype(left, right.fallback) if isinstance(right, Instance): + if (left, right) in right.type.cache: + return True for base in left.type.mro: if base._promote and is_subtype( base._promote, self.right, self.check_type_parameter, ignore_pos_arg_names=self.ignore_pos_arg_names): + right.type.cache.add((left, right)) return True rname = right.type.fullname() # Always try a nominal check if possible, @@ -162,6 +165,7 @@ def visit_instance(self, left: Instance) -> bool: for lefta, righta, tvar in zip(t.args, right.args, right.type.defn.type_vars)) if nominal: + right.type.cache.add((left, right)) return True if right.type.is_protocol and is_protocol_implementation(left, right): return True @@ -360,10 +364,10 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool # this rule is copied from nominal check in checker.py if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: return False - # This additional push serves as poor man's subtype cache. - # (This already gives a decent speed-up.) - # TODO: Implement a general fast subtype cache? - assuming.append((left, right)) + if allow_any: + right.type.cache.add((left, right)) + else: + right.type.cache_proper.add((left, right)) return True @@ -781,8 +785,11 @@ def visit_deleted_type(self, left: DeletedType) -> bool: def visit_instance(self, left: Instance) -> bool: right = self.right if isinstance(right, Instance): + if (left, right) in right.type.cache_proper: + return True for base in left.type.mro: if base._promote and is_proper_subtype(base._promote, right): + right.type.cache_proper.add((left, right)) return True if left.type.has_base(right.type.fullname()): @@ -799,6 +806,7 @@ def check_argument(leftarg: Type, rightarg: Type, variance: int) -> bool: nominal = all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in zip(left.args, right.args, right.type.defn.type_vars)) if nominal: + right.type.cache_proper.add((left, right)) return True if (right.type.is_protocol and is_protocol_implementation(left, right, allow_any=False)): diff --git a/mypy/types.py b/mypy/types.py index 1bba37724ced..e59a2c69ae77 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -56,6 +56,12 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: def __repr__(self) -> str: return self.accept(TypeStrVisitor()) + def __hash__(self) -> int: + return hash(type(self)) + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) + def serialize(self) -> Union[JsonDict, str]: raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) @@ -210,6 +216,15 @@ def __init__(self, def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_unbound_type(self) + def __hash__(self) -> int: + return hash((UnboundType, self.name, self.optional, tuple(self.args))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UnboundType): + return False + return (self.name == other.name and self.optional == other.optional and + self.args == other.args) + def serialize(self) -> JsonDict: return {'.class': 'UnboundType', 'name': self.name, @@ -245,6 +260,14 @@ def serialize(self) -> JsonDict: 'items': [t.serialize() for t in self.items], } + def __hash__(self) -> int: + return hash((TypeList, tuple(self.items))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypeList): + return False + return self.items == other.items + @classmethod def deserialize(cls, data: JsonDict) -> 'TypeList': assert data['.class'] == 'TypeList' @@ -388,6 +411,14 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: type_ref = None # type: str + def __hash__(self) -> int: + return hash((Instance, self.type, tuple(self.args))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Instance): + return False + return self.type == other.type and self.args == other.args + def serialize(self) -> Union[JsonDict, str]: assert self.type is not None type_ref = self.type.alt_fullname or self.type.fullname() @@ -450,6 +481,14 @@ def erase_to_union_or_bound(self) -> Type: else: return self.upper_bound + def __hash__(self) -> int: + return hash((TypeVarType, self.id)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypeVarType): + return False + return self.id == other.id + def serialize(self) -> JsonDict: assert not self.id.is_meta_var() return {'.class': 'TypeVarType', @@ -723,6 +762,23 @@ def type_var_ids(self) -> List[TypeVarId]: a.append(tv.id) return a + def __hash__(self) -> int: + return hash((CallableType, self.ret_type, self.is_type_obj(), + self.is_ellipsis_args, self.name) + + tuple(self.arg_types) + tuple(self.arg_names) + tuple(self.arg_kinds)) + + def __eq__(self, other: object) -> bool: + if isinstance(other, CallableType): + return (self.ret_type == other.ret_type and + self.arg_types == other.arg_types and + self.arg_names == other.arg_names and + self.arg_kinds == other.arg_kinds and + self.name == other.name and + self.is_type_obj() == other.is_type_obj() and + self.is_ellipsis_args == other.is_ellipsis_args) + else: + return False + def serialize(self) -> JsonDict: # TODO: As an optimization, leave out everything related to # generic functions for non-generic functions. @@ -803,6 +859,14 @@ def get_name(self) -> str: def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_overloaded(self) + def __hash__(self) -> int: + return hash((Overloaded, tuple(self.items()))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Overloaded): + return False + return self.items() == other.items() + def serialize(self) -> JsonDict: return {'.class': 'Overloaded', 'items': [t.serialize() for t in self.items()], @@ -844,6 +908,14 @@ def length(self) -> int: def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_tuple_type(self) + def __hash__(self) -> int: + return hash((TupleType, tuple(self.items), self.fallback)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TupleType): + return False + return self.items == other.items and self.fallback == other.fallback + def serialize(self) -> JsonDict: return {'.class': 'TupleType', 'items': [t.serialize() for t in self.items], @@ -894,6 +966,20 @@ def __init__(self, items: 'OrderedDict[str, Type]', fallback: Instance, def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_typeddict_type(self) + def __hash__(self) -> int: + return hash((TypedDictType, tuple(self.items.items()))) + + def __eq__(self, other: object) -> bool: + if isinstance(other, TypedDictType): + if self.items.keys() != other.items.keys(): + return False + for (_, left_item_type, right_item_type) in self.zip(other): + if not left_item_type == right_item_type: + return False + return True + else: + return False + def serialize(self) -> JsonDict: return {'.class': 'TypedDictType', 'items': [[n, t.serialize()] for (n, t) in self.items.items()], @@ -964,6 +1050,14 @@ def __init__(self, type: Type, line: int = -1, column: int = -1) -> None: self.type = type super().__init__(line, column) + def __hash__(self) -> int: + return hash((StarType, self.type)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, StarType): + return False + return self.type == other.type + def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_star_type(self) @@ -979,6 +1073,14 @@ def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None: self.can_be_false = any(item.can_be_false for item in items) super().__init__(line, column) + def __hash__(self) -> int: + return hash(frozenset(self.items)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UnionType): + return False + return frozenset(self.items) == frozenset(other.items) + @staticmethod def make_union(items: List[Type], line: int = -1, column: int = -1) -> Type: if len(items) > 1: @@ -1159,6 +1261,14 @@ def __init__(self, item: Type, *, line: int = -1, column: int = -1) -> None: def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_type_type(self) + def __hash__(self) -> int: + return hash((TypeType, self.item)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TypeType): + return False + return self.item == other.item + def serialize(self) -> JsonDict: return {'.class': 'TypeType', 'item': self.item.serialize()} diff --git a/test-data/unit/check-lists.test b/test-data/unit/check-lists.test index c9c67e80d4fb..c575f4b76da4 100644 --- a/test-data/unit/check-lists.test +++ b/test-data/unit/check-lists.test @@ -64,7 +64,7 @@ class C: pass [case testListWithStarExpr] (x, *a) = [1, 2, 3] a = [1, *[2, 3]] -reveal_type(a) # E: Revealed type is 'builtins.list[builtins.int]' +reveal_type(a) # E: Revealed type is 'builtins.list[builtins.int*]' b = [0, *a] reveal_type(b) # E: Revealed type is 'builtins.list[builtins.int*]' c = [*a, 0] diff --git a/typeshed b/typeshed index 7027c3e4e8f5..72c46c2b48f0 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 7027c3e4e8f5d00d5269fc11ccaf768012725abd +Subproject commit 72c46c2b48f02696ea3f1356d29a8df7b63f2515 From 7d2327a389710d699f09db0a37e598e4830d1d13 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 10 Apr 2017 00:16:10 +0200 Subject: [PATCH 041/117] Restore correct typeshed commit --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 72c46c2b48f0..761ee6afb3d9 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 72c46c2b48f02696ea3f1356d29a8df7b63f2515 +Subproject commit 761ee6afb3d97b8ee2531d8aae701606700f3b19 From 88d4fedcab3bf9049ce29fa2e12e458cf7f3edac Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 10 Apr 2017 00:39:11 +0200 Subject: [PATCH 042/117] Typeshed again --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 761ee6afb3d9..f7e4cb8c7921 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 761ee6afb3d97b8ee2531d8aae701606700f3b19 +Subproject commit f7e4cb8c7921cb7abd6d8293d27725854221b267 From 85082d58988e699d7d43cddb84fa4592d59801cc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Apr 2017 00:35:41 +0200 Subject: [PATCH 043/117] Switch to NotImplemented instead of False in __eq__ --- mypy/types.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 2b586fa36b8e..a892ac3279fc 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -221,7 +221,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, UnboundType): - return False + return NotImplemented return (self.name == other.name and self.optional == other.optional and self.args == other.args) @@ -265,7 +265,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TypeList): - return False + return NotImplemented return self.items == other.items @classmethod @@ -416,7 +416,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, Instance): - return False + return NotImplemented return self.type == other.type and self.args == other.args def serialize(self) -> Union[JsonDict, str]: @@ -486,7 +486,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TypeVarType): - return False + return NotImplemented return self.id == other.id def serialize(self) -> JsonDict: @@ -783,7 +783,7 @@ def __eq__(self, other: object) -> bool: self.is_type_obj() == other.is_type_obj() and self.is_ellipsis_args == other.is_ellipsis_args) else: - return False + return NotImplemented def serialize(self) -> JsonDict: # TODO: As an optimization, leave out everything related to @@ -874,7 +874,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, Overloaded): - return False + return NotImplemented return self.items() == other.items() def serialize(self) -> JsonDict: @@ -923,7 +923,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TupleType): - return False + return NotImplemented return self.items == other.items and self.fallback == other.fallback def serialize(self) -> JsonDict: @@ -988,7 +988,7 @@ def __eq__(self, other: object) -> bool: return False return True else: - return False + return NotImplemented def serialize(self) -> JsonDict: return {'.class': 'TypedDictType', @@ -1065,7 +1065,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, StarType): - return False + return NotImplemented return self.type == other.type def accept(self, visitor: 'TypeVisitor[T]') -> T: @@ -1088,7 +1088,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, UnionType): - return False + return NotImplemented return frozenset(self.items) == frozenset(other.items) @staticmethod @@ -1276,7 +1276,7 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TypeType): - return False + return NotImplemented return self.item == other.item def serialize(self) -> JsonDict: From 2733e1add508bc2956facfd8db57c54abd9db0de Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Apr 2017 16:18:10 +0200 Subject: [PATCH 044/117] Prohibit using NewType with protocols --- mypy/semanal.py | 2 ++ test-data/unit/check-newtype.test | 13 +++++++++++++ typeshed | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2eb576a0198c..102a6be21af0 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1850,6 +1850,8 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type.fallback) newtype_class_info.tuple_type = old_type elif isinstance(old_type, Instance): + if old_type.type.is_protocol: + self.fail("NewType cannot be used with protocol classes", s) newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type) else: message = "Argument 2 to NewType(...) must be subclassable (got {})" diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 25adf9885d0d..e3127e6daf2f 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -318,6 +318,19 @@ B = NewType('B', A) class C(B): pass # E: Cannot subclass NewType [out] +[case testCannotUseNewTypeWithProtocols] +from typing import Protocol, NewType + +class P(Protocol): + attr: int +class D: + attr: int + +C = NewType('C', P) # E: NewType cannot be used with protocol classes + +x: C = C(D()) # We still accept this, treating 'C' as non-protocol subclass. +[out] + [case testNewTypeAny] from typing import NewType Any = NewType('Any', int) diff --git a/typeshed b/typeshed index 359c8cc313d1..72c46c2b48f0 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 359c8cc313d18e309a80548c1d4643506302a525 +Subproject commit 72c46c2b48f02696ea3f1356d29a8df7b63f2515 From 232428f6d64e7bcc33e0e750923eba4db3c5bf64 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Apr 2017 23:17:14 +0200 Subject: [PATCH 045/117] Address CR --- docs/source/class_basics.rst | 173 +++++++++++++++++++++++++++++++---- docs/source/faq.rst | 5 +- 2 files changed, 157 insertions(+), 21 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index f8dcdb1293c8..f08501235042 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -156,38 +156,167 @@ base class. Protocols and structural subtyping ********************************** -Mypy provides support for structural subtyping and protocol classes. +.. note:: + + The support for structural subtyping is still experimental. Some features + might be not yet implemented, mypy could pass unsafe code or reject + working code. + +There are two main type systems with respect to subtyping: nominal subtyping +and structural subtyping. The *nominal* subtyping is based on class hierarchy, +so that if class ``D`` inherits from class ``C``, then it is a subtype +of ``C``. This type system is primarily used in mypy since it allows +to produce clear and concise error messages, and since Python provides native +``isinstance()`` checks based on class hierarchy. The *structural* subtyping +however has its own advantages. In this system class ``D`` is a subtype +of class ``C`` if the former has all attributes of the latter with +compatible types. For example: + +.. code-block:: python + + from typing import Sized + + def my_len(obj: Sized) -> int: + ... + + class MyCollection: + ... + def __len__(self) -> int: + return 42 + + my_len(MyCollection()) # OK, since 'MyCollection' is a subtype of 'Sized' + +This type system is a static equivalent of duck typing, well known by Python +programmers. Mypy provides an opt-in support for structural subtyping via +protocol classes described in this section. +See `PEP 544 `_ for +specification of protocols and structural subtyping in Python. + +User defined protocols +********************** + To define a protocol class, one must inherit the special ``typing.Protocol`` class: .. code-block:: python - from typing import Protocol + from typing import Protocol, Iterable class SupportsClose(Protocol): def close(self) -> None: ... - class UnrelatedClass: + class Resource: # Note, this class does not have 'SupportsClose' base. # some methods def close(self) -> None: self.resource.release() - def close_all(things: Sequence[SupportsClose]) -> None: + def close_all(things: Iterable[SupportsClose]) -> None: for thing in things: thing.close() - close_all([UnrelatedClass(), open('some/file')]) # This passes type check + close_all([Resource(), open('some/file')]) # This passes type check + +Defining subprotocols +********************* + +Subprotocols are also supported. Existing protocols can be extended +and merged using multiple inheritance. For example: + +.. code-block:: python + + # continuing from previous example + + class SizedLabeledResource(SupportsClose, Sized, Protocol): + label: str + + class AdvancedResource(Resource): + def __init__(self, label: str) -> None: + self.label = label + def __len__(self) -> int: + ... + + resource: SizedLabeledResource + + # some code + + resource = AdvancedResource('handle with care') # OK + +Note that inheriting from existing protocols does not automatically turn +a subclass into a protocol, it just creates a usual (non-protocol) ABC that +implements given protocols. The ``typing.Protocol`` base must always be +explicitly present: -Subprotocols are also supported. Inheriting from an existing protocol does -not automatically turn a subclass into a protocol, it just creates a usual -ABC. The ``typing.Protocol`` base must always be explicitly present. -Generic and recursive protocols are also supported: +.. code-block:: python + + class NewProtocol(Sized): # This is NOT a protocol + new_attr: int + + class Concrete: + new_attr: int + def __len__(self) -> int: + ... + + x: NewProtocol = Concrete() # Error, nominal subtyping is used by default + +Generic protocols +***************** + +Generic protocols are also supported, generic protocols mostly follow +the normal rules for generic classes, the main difference is that declaring +variance is not necessary for protocols, mypy can infer it. Examples: .. code-block:: python from typing import Protocol, TypeVar + T = TypeVar('T') + + class Box(Protocol[T]): + content: T + + def do_stuff(one: Box[str], other: Box[bytes]) -> None: + ... + + class StringWrapper: + def __init__(self, content: str) -> None: + self.content = content + + class BytesWrapper: + def __init__(self, content: bytes) -> None: + self.content = content + + do_stuff(StringWrapper('one'), BytesWrapper(b'other')) # OK + + x: Box[float] + y: Box[int] + x = y # This is also OK due to inferred covariance of the protocol 'Box'. + +See :ref:`generic-classes` for more details on generic classes and variance. + +Recursive protocols +******************* + +Protocols (both generic and non-generic) can be recursive and mutually +recursive. This could be useful for declaring abstract recursive collections +such as trees and linked lists: + +.. code-block:: python + + from typing import Protocol, TypeVar, Optional + + class TreeLike(Protocol): + value: int + left: Optional['TreeLike'] + right: Optional['TreeLike'] + + class SimpleTree: + def __init__(self, value: int) -> None: + self.value = value + self.left = self.right = None + + root: TreeLike = SimpleTree(0) # OK + T = TypeVar('T') class Linked(Protocol[T]): val: T @@ -202,10 +331,15 @@ Generic and recursive protocols are also supported: result = last(L()) # The inferred type of 'result' is 'int' -See :ref:`generic-classes` for more details on generic classes. -The standard ABCs in ``typing`` module are protocols, so that the following -class will be considered a subtype of ``typing.Sized`` and -``typing.Iterable[int]``: +Predefined protocols in ``typing`` module +***************************************** + +Most ABCs in ``typing`` module are protocol classes describing +common Python protocols such as ``Iterator``, ``Awaitable``, ``Mapping``, etc. +(see `Python Docs `_ +for an exhaustive list) +For example, the following class will be considered a subtype of +``typing.Sized`` and ``typing.Iterable[int]``: .. code-block:: python @@ -221,6 +355,9 @@ class will be considered a subtype of ``typing.Sized`` and def collect(items: Iterable[int]) -> int: ... result: int = collect(Bucket()) # Passes type check +Using ``isinstance()`` with protocols +************************************* + To use a protocol class with ``isinstance()``, one needs to decorate it with a special ``typing.runtime`` decorator. It will add support for basic runtime structural checks: @@ -241,11 +378,9 @@ structural checks: if isinstance(mug, Portable): use(mug.handles) # Works statically and at runtime. -See `PEP 544 `_ for -specification of structural subtyping in Python. - .. note:: + ``isinstance()`` is with protocols not completely safe at runtime. + For example, signatures of methods are not checked. The runtime + implementation only checks the presence of all protocol members + in object's MRO. - The support for structural subtyping is still experimental. Some features - might be not yet implemented, mypy could pass unsafe code or reject - working code. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 902dff25a20b..f6393d881f37 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -107,8 +107,9 @@ Can I use structural subtyping? Mypy provides support for both `nominal subtyping `_ and `structural subtyping -`_. Some argue -that structural subtyping is better suited for languages with duck +`_. +Support for structural subtyping is considered experimental. +Some argue that structural subtyping is better suited for languages with duck typing such as Python. Mypy however primarily uses nominal subtyping, leaving structural subtyping opt-in. Here are some reasons why: From 4c04e0b0e140bd3dd67ceff20576d5c2dfb50ed0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Apr 2017 23:18:24 +0200 Subject: [PATCH 046/117] Restore current typeshed commit --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 72c46c2b48f0..359c8cc313d1 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 72c46c2b48f02696ea3f1356d29a8df7b63f2515 +Subproject commit 359c8cc313d18e309a80548c1d4643506302a525 From e81a3f9fcc379b2dcdb427b43f36abdf8c3fc833 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Apr 2017 21:36:25 +0200 Subject: [PATCH 047/117] Add some support for __dunder__ = None idiom --- mypy/subtypes.py | 3 ++- test-data/unit/check-protocols.test | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index f39ffec62808..38039b886bf4 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -352,7 +352,8 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) else: is_compat = is_proper_subtype(subtype, supertype) - if not is_compat: + if not is_compat or isinstance(subtype, NoneTyp): + # We want __hash__ = None idiom to work even without --strict-optional return False subflags = get_member_flags(member, left.type) superflags = get_member_flags(member, right.type) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 9e0c5349c899..add8176b706d 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -233,6 +233,32 @@ f1(x2) f2(x1) [out] +[case testNoneDisablesProtocolImplementation] +from typing import Protocol + +class MyHashable(Protocol): + def __my_hash__(self) -> int: + return 0 + +class C: + __my_hash__ = None + +var: MyHashable = C() # E: Incompatible types in assignment (expression has type "C", variable has type "MyHashable") +[out] + +[case testNoneDisablesProtocolSubclassingWithStrictOptional] +# flags: --strict-optional +from typing import Protocol + +class MyHashable(Protocol): + def __my_hash__(self) -> int: + return 0 + +class C(MyHashable): + __my_hash__ = None # E: Incompatible types in assignment \ +(expression has type None, base class "MyHashable" defined the type as Callable[[MyHashable], int]) +[out] + -- Semanal errors in protocol types -- -------------------------------- From 79d8e30091d0ff015c521ced095d5200286ae2aa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 21 Apr 2017 15:09:05 +0200 Subject: [PATCH 048/117] Trigger AppVeyor re-build From 480b977aeb02453be4e19e3ff9641299438bcbfb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Apr 2017 13:28:55 +0200 Subject: [PATCH 049/117] Disable covariant subtyping for mutable members --- mypy/constraints.py | 7 ++++ mypy/subtypes.py | 4 ++ test-data/unit/check-protocols.test | 61 +++++++++++++++++++++++------ typeshed | 2 +- 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 39d025aad967..3e5dce650573 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -348,6 +348,10 @@ def visit_instance(self, template: Instance) -> List[Constraint]: inst = mypy.subtypes.find_member(member, instance, instance) temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) + if (mypy.subtypes.IS_SETTABLE in + mypy.subtypes.get_member_flags(member, template.type)): + # Settable members are invariant, add opposite constraints + res.extend(infer_constraints(temp, inst, neg_op(self.direction))) template.type.inferring.pop() return res elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and @@ -359,6 +363,9 @@ def visit_instance(self, template: Instance) -> List[Constraint]: inst = mypy.subtypes.find_member(member, instance, instance) temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) + if (mypy.subtypes.IS_SETTABLE in + mypy.subtypes.get_member_flags(member, instance.type)): + res.extend(infer_constraints(temp, inst, neg_op(self.direction))) instance.type.inferring.pop() return res if isinstance(actual, AnyType): diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 0e3212d8200b..23dd05960f4c 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -354,6 +354,10 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool return False subflags = get_member_flags(member, left.type) superflags = get_member_flags(member, right.type) + if IS_SETTABLE in superflags: + # Check opposite direction for settable attributes. + if not is_subtype(supertype, subtype): + return False if (IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags or IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags): return False diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index add8176b706d..148fe6d8ea44 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -363,14 +363,13 @@ from typing import TypeVar, Protocol T = TypeVar('T') class Pco(Protocol[T]): - attr: T + def meth(self) -> T: + pass class Pcontra(Protocol[T]): def meth(self, x: T) -> None: pass class Pinv(Protocol[T]): attr: T - def meth(self, x: T) -> None: - pass class A: pass class B(A): pass @@ -437,6 +436,20 @@ class C: def fun3(x: P[T, T]) -> T: pass reveal_type(fun3(C())) # E: Revealed type is 'builtins.int*' +[out] + +[case testProtocolGenericInferenceCovariant] +from typing import Generic, TypeVar, Protocol +T = TypeVar('T') +S = TypeVar('S') + +class P(Protocol[T, S]): + def x(self) -> T: pass + def y(self) -> S: pass + +class C: + def x(self) -> int: pass + def y(self) -> int: pass def fun4(x: T, y: P[T, T]) -> T: pass @@ -477,7 +490,7 @@ def f(x: T) -> T: [builtins fixtures/isinstancelist.pyi] [out] -[case testGenericSubProtocolsExtension] +[case testGenericSubProtocolsExtensionInvariant] from typing import TypeVar, Protocol, Union T = TypeVar('T') @@ -502,9 +515,28 @@ class D(A, B): pass x: P = D() # Same as P[Any, Any] -var: P[Union[int, P], Union[P, str]] = C() # OK -var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) +var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[int, P[Any, Any]], Union[P[Any, Any], str]]) +[out] +[case testGenericSubProtocolsExtensionCovariant] +from typing import TypeVar, Protocol, Union + +T = TypeVar('T') +S = TypeVar('S') + +class P1(Protocol[T]): + def attr1(self) -> T: pass +class P2(Protocol[T]): + def attr2(self) -> T: pass +class P(P1[T], P2[S], Protocol): + pass + +class C: + def attr1(self) -> int: pass + def attr2(self) -> str: pass + +var: P[Union[int, P], Union[P, str]] = C() # OK for covariant +var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) [out] [case testSelfTypesWithProtocolsBehaveAsWithNominal] @@ -548,7 +580,8 @@ from typing import Protocol, Sequence, List, Generic, TypeVar T = TypeVar('T') class Traversable(Protocol): - leaves: Sequence[Traversable] + @property + def leaves(self) -> Sequence[Traversable]: pass class C: pass @@ -569,11 +602,11 @@ from typing import Protocol, TypeVar T = TypeVar('T') class Linked(Protocol[T]): val: T - next: Linked[T] + def next(self) -> Linked[T]: pass class L: val: int - next: L + def next(self) -> L: pass def last(seq: Linked[T]) -> T: pass @@ -586,10 +619,12 @@ reveal_type(last(L())) # E: Revealed type is 'builtins.int*' from typing import Protocol, Sequence, List class P1(Protocol): - attr1: Sequence[P2] + @property + def attr1(self) -> Sequence[P2]: pass class P2(Protocol): - attr2: Sequence[P1] + @property + def attr2(self) -> Sequence[P1]: pass class C: pass @@ -875,11 +910,11 @@ class Box(Protocol[T]): content: T class Linked(Protocol[T]): val: T - next: Linked[T] + def next(self) -> Linked[T]: pass class L(Generic[T]): val: Box[T] - next: L[T] + def next(self) -> L[T]: pass def last(seq: Linked[T]) -> T: pass diff --git a/typeshed b/typeshed index 8b835f95001b..a934d57f3bb0 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 8b835f95001b61734c6b147d3aa6eb4fbe7bce03 +Subproject commit a934d57f3bb03ee62b882a055e655521f6608769 From 70c3ae0cbf7ae828258cd8b247749b4c8c25e191 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Apr 2017 13:57:11 +0200 Subject: [PATCH 050/117] Fix remaining test --- test-data/unit/check-protocols.test | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 148fe6d8ea44..c8613cc5f450 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1266,8 +1266,10 @@ T = TypeVar('T') S = TypeVar('S') class P(Protocol[T, S]): - x: T - y: S + @property + def x(self) -> T: pass + @property + def y(self) -> S: pass class N(NamedTuple): x: int @@ -1296,7 +1298,7 @@ fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, reveal_type(fun3(z)) # E: Revealed type is 'builtins.object*' reveal_type(fun3(z3)) # E: Revealed type is 'builtins.int*' -[builtins fixtures/isinstancelist.pyi] +[builtins fixtures/list.pyi] [out] From 509113a0d1ba37ae504f0450ccc87faefda78a2f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Apr 2017 16:46:22 +0200 Subject: [PATCH 051/117] Add much more detailed error messages --- mypy/checker.py | 44 ++++++++++++++++++---- mypy/checkexpr.py | 9 +---- mypy/semanal.py | 2 +- mypy/subtypes.py | 37 ++++++++++++++++++- test-data/unit/check-protocols.test | 57 +++++++++++++++++++++-------- 5 files changed, 117 insertions(+), 32 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 5f0c6bed20c1..6332da0a31b1 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -43,7 +43,8 @@ from mypy.subtypes import ( is_subtype, is_equivalent, is_proper_subtype, is_more_precise, restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_subtype, - unify_generic_callable, get_missing_members + unify_generic_callable, get_missing_members, get_conflict_types, get_all_flags, + IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC ) from mypy.maptype import map_instance_to_supertype from mypy.typevars import fill_typevars, has_no_typevars @@ -2316,14 +2317,43 @@ def check_subtype(self, subtype: Type, supertype: Type, context: Context, self.fail(msg, context) if (isinstance(supertype, Instance) and supertype.type.is_protocol and isinstance(subtype, Instance)): - missing = get_missing_members(subtype, supertype) - if missing: - self.note("'{}' missing following '{}' protocol members:" - .format(subtype.type.fullname(), supertype.type.fullname()), - context) - self.note(', '.join(missing), context) + self.report_protocol_problems(subtype, supertype, context) return False + def report_protocol_problems(self, subtype: Instance, supertype: Instance, + context: Context) -> None: + """Report possible protocol conflicts between 'subtype' and 'supertype'. + This includes missing members, incompatible types, and incompatible + attribute flags, such as settable vs read-only or class variable vs + instance variable. + """ + missing = get_missing_members(subtype, supertype) + if missing: + self.note("'{}' missing following '{}' protocol members:" + .format(subtype.type.fullname(), supertype.type.fullname()), + context) + self.note(', '.join(missing), context) + conflict_types = get_conflict_types(subtype, supertype) + if conflict_types: + self.note('Following members of {} have ' + 'conflicts:'.format(subtype), context) + for name, got, expected in conflict_types: + self.note('{}: expected {}, got {}'.format(name, expected, got), + context) + for name, subflags, superflags in get_all_flags(subtype, supertype): + if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: + self.note('Protocol member {}.{}: expected instance variable,' + ' got class variable'.format(supertype, name), context) + if IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: + self.note('Protocol member {}.{}: expected class variable,' + ' got instance variable'.format(supertype, name), context) + if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: + self.note('Protocol member {}.{}: expected settable variable,' + ' got read-only attribute'.format(supertype, name), context) + if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: + self.note('Protocol member {}.{}: expected class or static method' + .format(supertype, name), context) + def contains_none(self, t: Type) -> bool: return ( isinstance(t, NoneTyp) or diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index cc79e186f11c..5fba4b2d8ab5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -34,7 +34,7 @@ from mypy import join from mypy.meet import narrow_declared_type from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent, get_missing_members +from mypy.subtypes import is_subtype, is_equivalent from mypy import applytype from mypy import erasetype from mypy.checkmember import analyze_member_access, type_object_type, bind_self @@ -888,12 +888,7 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, caller_kind, context) if (isinstance(original_caller_type, Instance) and isinstance(callee_type, Instance) and callee_type.type.is_protocol): - missing = get_missing_members(original_caller_type, callee_type) - if missing: - messages.note("'{}' missing following '{}' protocol members:" - .format(original_caller_type.type.fullname(), - callee_type.type.fullname()), context) - messages.note(', '.join(missing), context) + self.chk.report_protocol_problems(original_caller_type, callee_type, context) def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], arg_names: List[str], diff --git a/mypy/semanal.py b/mypy/semanal.py index 12547b1b1cb7..84e0d77c87c3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3751,7 +3751,7 @@ def add_protocol_members(typ: TypeInfo) -> None: if base.is_protocol: for name in base.names: members.add(name) - typ.protocol_members = list(members) + typ.protocol_members = sorted(list(members)) def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 23dd05960f4c..68e47edf9a51 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -413,9 +413,24 @@ def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type] return None +def get_all_flags(left: Instance, right: Instance) -> List[Tuple[str, Set[int], Set[int]]]: + """Return all attribute flags for members that are present in both + 'left' and 'right'. + """ + assert right.type.is_protocol + all_flags = [] # type: List[Tuple[str, Set[int], Set[int]]] + for member in right.type.protocol_members: + if find_member(member, left, left): + item = (member, + get_member_flags(member, left.type), + get_member_flags(member, right.type)) + all_flags.append(item) + return all_flags + + def get_member_flags(name: str, info: TypeInfo) -> Set[int]: - """Detect whether a member 'name' is settable and whether it is an - instance or class variable. + """Detect whether a member 'name' is settable, whether it is an + instance or class variable, and whether it is class or static method. """ method = info.get_method(name) if method: @@ -489,6 +504,24 @@ def get_missing_members(left: Instance, right: Instance) -> List[str]: return sorted(missing) +def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, Type]]: + assert right.type.is_protocol + conflicts = [] # type: List[Tuple[str, Type, Type]] + for member in right.type.protocol_members: + if member in ('__init__', '__new__'): + continue + supertype = find_member(member, right, left) + subtype = find_member(member, left, left) + if not subtype: + continue + is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) + if IS_SETTABLE in get_member_flags(member, right.type): + is_compat = is_compat and is_subtype(supertype, subtype) + if not is_compat: + conflicts.append((member, subtype, supertype)) + return conflicts + + def is_callable_subtype(left: CallableType, right: CallableType, ignore_return: bool = False, ignore_pos_arg_names: bool = False, diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index c8613cc5f450..0acdfcbcc705 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -376,18 +376,26 @@ class B(A): pass x1: Pco[B] y1: Pco[A] -x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B]) +x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B])\ + # N: Following members of __main__.Pco[__main__.A] have conflicts:\ + # N: meth: expected def () -> __main__.B*, got def () -> __main__.A* y1 = x1 x2: Pcontra[B] y2: Pcontra[A] -y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A]) +y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A])\ + # N: Following members of __main__.Pcontra[__main__.B] have conflicts:\ + # N: meth: expected def (x: __main__.A*), got def (x: __main__.B*) x2 = y2 x3: Pinv[B] y3: Pinv[A] -y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A]) -x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B]) +y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A])\ + # N: Following members of __main__.Pinv[__main__.B] have conflicts:\ + # N: attr: expected __main__.A*, got __main__.B* +x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B])\ + # N: Following members of __main__.Pinv[__main__.A] have conflicts:\ + # N: attr: expected __main__.B*, got __main__.A* [out] [case testGenericProtocolsInference1] @@ -474,7 +482,9 @@ class C: c: C var: P2[int, int] = c -var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str]) +var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str])\ + # N: Following members of __main__.C have conflicts:\ + # N: attr2: expected Tuple[builtins.int*, builtins.str*], got Tuple[builtins.int, builtins.int] class D(Generic[T]): attr1: T @@ -515,7 +525,10 @@ class D(A, B): pass x: P = D() # Same as P[Any, Any] -var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[int, P[Any, Any]], Union[P[Any, Any], str]]) +var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[int, P[Any, Any]], Union[P[Any, Any], str]])\ + # N: Following members of __main__.C have conflicts:\ + # N: attr1: expected Union[builtins.int, __main__.P[Any, Any]], got builtins.int\ + # N: attr2: expected Union[__main__.P[Any, Any], builtins.str], got builtins.str [out] [case testGenericSubProtocolsExtensionCovariant] @@ -536,7 +549,10 @@ class C: def attr2(self) -> str: pass var: P[Union[int, P], Union[P, str]] = C() # OK for covariant -var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) +var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]])\ + # N: Following members of __main__.C have conflicts:\ + # N: attr1: expected def () -> Union[builtins.str, __main__.P[Any, Any]], got def () -> builtins.int\ + # N: attr2: expected def () -> Union[__main__.P[Any, Any], builtins.int], got def () -> builtins.str [out] [case testSelfTypesWithProtocolsBehaveAsWithNominal] @@ -565,8 +581,12 @@ s: Shape f(NonProtoShape()) f(Circle()) -s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") -s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") +s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape")\ + # N: Following members of __main__.Triangle have conflicts:\ + # N: combine: expected def (other: __main__.Triangle*) -> __main__.Triangle*, got def (other: __main__.Shape) -> __main__.Shape +s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape")\ + # N: Following members of __main__.Bad have conflicts:\ + # N: combine: expected def (other: __main__.Bad*) -> __main__.Bad*, got def (other: builtins.int) -> builtins.str n2: NonProtoShape = s # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") [out] @@ -719,9 +739,11 @@ x: PInst y: PClass x = CInst() -x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst") +x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst")\ + # N: Protocol member __main__.PInst.v: expected instance variable, got class variable y = CClass() -y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass") +y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass")\ + # N: Protocol member __main__.PClass.v: expected class variable, got instance variable [out] [case testPropertyInProtocols] @@ -741,7 +763,8 @@ y = x x2: P y2: PP -x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P") +x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P")\ + # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -778,7 +801,8 @@ y3 = z3 y4: PP z4: PPS -z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS") +z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS")\ + # N: Protocol member __main__.PPS.attr: expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -809,7 +833,8 @@ x = B() y: PC y = B() -y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC") +y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC")\ + # N: Protocol member __main__.PC.meth: expected class or static method [builtins fixtures/classmethod.pyi] [out] @@ -831,7 +856,9 @@ class D: pass x: P = C() -x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") +x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P")\ + # N: Following members of __main__.D have conflicts:\ + # N: f: expected Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str), got def (x: builtins.int) [out] -- Join and meet with protocol types From 0cb098508e8d26f90209807f0a31f8fb7848a8b5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Apr 2017 16:56:18 +0200 Subject: [PATCH 052/117] Fix one error message in one more test --- test-data/unit/check-expressions.test | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index bc3f00bfe2df..7386691b7b76 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1672,6 +1672,11 @@ a = {'a': 1} b = {'z': 26, **a} c = {**b} d = {**a, **b, 'c': 3} -e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str] -f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int] +# These errors changed because Mapping is a protocol +e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str]\ + # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts:\ + # N: __getitem__: expected def (builtins.int*) -> builtins.str*, got def (builtins.str*) -> builtins.int* +f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int]\ + # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts:\ + # N: __getitem__: expected def (builtins.int*) -> builtins.int*, got def (builtins.str*) -> builtins.int* [builtins fixtures/dict.pyi] From 9f554b621c8bb784b0f4fce3d80e319586b9d1f2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Apr 2017 12:39:07 +0200 Subject: [PATCH 053/117] Fix minor metaclass self-type bug, add various tests --- mypy/constraints.py | 8 +-- mypy/subtypes.py | 6 +-- test-data/unit/check-protocols.test | 75 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 3e5dce650573..7a32fe6aceb4 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -307,7 +307,7 @@ def visit_type_var(self, template: TypeVarType) -> List[Constraint]: # Non-leaf types def visit_instance(self, template: Instance) -> List[Constraint]: - actual = self.actual + original_actual = actual = self.actual res = [] # type: List[Constraint] if isinstance(actual, CallableType) and actual.fallback is not None: actual = actual.fallback @@ -345,8 +345,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: mypy.subtypes.is_subtype(instance, erase_typevars(template))): template.type.inferring.append(template) for member in template.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance, instance) - temp = mypy.subtypes.find_member(member, template, template) + inst = mypy.subtypes.find_member(member, instance, original_actual) + temp = mypy.subtypes.find_member(member, template, original_actual) res.extend(infer_constraints(temp, inst, self.direction)) if (mypy.subtypes.IS_SETTABLE in mypy.subtypes.get_member_flags(member, template.type)): @@ -360,7 +360,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: mypy.subtypes.is_subtype(erase_typevars(template), instance)): instance.type.inferring.append(instance) for member in instance.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance, instance) + inst = mypy.subtypes.find_member(member, instance, template) temp = mypy.subtypes.find_member(member, template, template) res.extend(infer_constraints(temp, inst, self.direction)) if (mypy.subtypes.IS_SETTABLE in diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 68e47edf9a51..5f7ab0930536 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -373,7 +373,7 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool return True -def find_member(name: str, itype: Instance, subtype: Instance) -> Optional[Type]: +def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: """Find the type of member by 'name' in 'itype's TypeInfo. Apply type arguments from 'itype', and bind 'self' to 'subtype'. Return None if member was not found. """ @@ -458,7 +458,7 @@ def get_member_flags(name: str, info: TypeInfo) -> Set[int]: return set() -def find_var_type(var: Var, itype: Instance, subtype: Instance) -> Type: +def find_var_type(var: Var, itype: Instance, subtype: Type) -> Type: """Find type of a variable 'var' (maybe also a decorated method). Apply type arguments from 'itype', and bind 'self' to 'subtype'. """ @@ -478,7 +478,7 @@ def find_var_type(var: Var, itype: Instance, subtype: Instance) -> Type: return typ -def map_method(method: FuncBase, itype: Instance, subtype: Instance) -> Type: +def map_method(method: FuncBase, itype: Instance, subtype: Type) -> Type: """Map 'method' to the base where it was defined. Apply type arguments from 'itype', and bind 'self' type to 'subtype'. This function should be used only for non-decorated methods. Decorated diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 0acdfcbcc705..6b866b5e988a 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1410,3 +1410,78 @@ foo('no way') main:11: error: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" main:11: note: 'builtins.str' missing following 'typing.SupportsInt' protocol members: main:11: note: __int__ + +-- Additional test and corner cases for protocols +-- ---------------------------------------------- + +[case testErrorsForProtocolsInDifferentPlaces] +from typing import Protocol + +class P(Protocol): + attr1: int + attr2: str + attr3: int + +class C: + attr1: str + @property + def attr2(self) -> int: pass + +x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P")\ + # N: '__main__.C' missing following '__main__.P' protocol members:\ + # N: attr3\ + # N: Following members of __main__.C have conflicts:\ + # N: attr1: expected builtins.int, got builtins.str\ + # N: attr2: expected builtins.str, got builtins.int\ + # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute + +def f(x: P) -> P: + return C() # E: Incompatible return value type (got "C", expected "P")\ + # N: '__main__.C' missing following '__main__.P' protocol members:\ + # N: attr3\ + # N: Following members of __main__.C have conflicts:\ + # N: attr1: expected builtins.int, got builtins.str\ + # N: attr2: expected builtins.str, got builtins.int\ + # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute + +f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P"\ + # N: '__main__.C' missing following '__main__.P' protocol members:\ + # N: attr3\ + # N: Following members of __main__.C have conflicts:\ + # N: attr1: expected builtins.int, got builtins.str\ + # N: attr2: expected builtins.str, got builtins.int\ + # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute +[builtins fixtures/list.pyi] +[out] + +[case testIterableProtocolOnClass] +from typing import TypeVar, Iterator +T = TypeVar('T', bound='A') + +class A: + def __iter__(self: T) -> Iterator[T]: pass + +class B(A): pass + +reveal_type(list(b for b in B())) # E: Revealed type is 'builtins.list[__main__.B*]' +reveal_type(list(B())) # E: Revealed type is 'builtins.list[__main__.B*]' +[builtins fixtures/list.pyi] +[out] + +[case testIterableProtocolOnMetaclass] +from typing import TypeVar, Iterator, Type +T = TypeVar('T', bound='E') + +class EMeta(type): + def __iter__(self: Type[T]) -> Iterator[T]: pass + +class E(metaclass=EMeta): + pass + +class C(E): + pass + +reveal_type(list(c for c in C)) # E: Revealed type is 'builtins.list[__main__.C*]' +reveal_type(list(C)) # E: Revealed type is 'builtins.list[__main__.C*]' +[builtins fixtures/list.pyi] +[out] From 70463a588c91e4d9a5b99e17dcf1735ac4998394 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Apr 2017 18:31:08 +0200 Subject: [PATCH 054/117] Add two more tests (including for Any compatibility in protocols) --- test-data/unit/check-protocols.test | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 6b866b5e988a..426044051556 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -591,6 +591,21 @@ s = Bad() # E: Incompatible types in assignment (expression has type "Bad", vari n2: NonProtoShape = s # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") [out] +[case testBadVarianceInProtocols] +from typing import Protocol, TypeVar + +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class Proto(Protocol[T_co, T_contra]): + def one(self, x: T_co) -> None: # E: Cannot use a covariant type variable as a parameter + pass + def other(self) -> T_contra: # E: Cannot use a contravariant type variable as return type + pass + +[builtins fixtures/list.pyi] +[out] + -- Recursive protocol types -- ------------------------ @@ -1414,6 +1429,42 @@ main:11: note: __int__ -- Additional test and corner cases for protocols -- ---------------------------------------------- +[case testAnyWithProtocols] +from typing import Protocol, Any, TypeVar + +T = TypeVar('T') + +class P1(Protocol): + attr1: int +class P2(Protocol[T]): + attr2: T +class P3(Protocol): + attr: P3 + +def f1(x: P1) -> None: pass +def f2(x: P2[str]) -> None: pass +def f3(x: P3) -> None: pass + +class C1: + attr1: Any +class C2: + attr2: Any +class C3: + attr: Any + +f1(C1()) +f2(C2()) +f3(C3()) + +f2(C3()) # E: Argument 1 to "f2" has incompatible type "C3"; expected P2[str]\ + # N: '__main__.C3' missing following '__main__.P2' protocol members:\ + # N: attr2 +a: Any +f1(a) +f2(a) +f3(a) +[out] + [case testErrorsForProtocolsInDifferentPlaces] from typing import Protocol From 8dfe8eafc1c2bc43ec56780c5ef3cd48a8e8f013 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 27 Apr 2017 16:10:43 +0200 Subject: [PATCH 055/117] Add two corner case tests: __getattr__; implicit types --- test-data/unit/check-protocols.test | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 426044051556..a1d245b5fbea 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1536,3 +1536,56 @@ reveal_type(list(c for c in C)) # E: Revealed type is 'builtins.list[__main__.C reveal_type(list(C)) # E: Revealed type is 'builtins.list[__main__.C*]' [builtins fixtures/list.pyi] [out] + +[case testClassesGetattrWithProtocols] +from typing import Protocol + +class P(Protocol): + attr: int + +class PP(Protocol): + @property + def attr(self) -> int: + pass + +class C: + def __getattr__(self, attr: str) -> int: + pass + +class D: + def __getattr__(self, attr: str) -> str: + pass + +def fun(x: P) -> None: + reveal_type(P.attr) # E: Revealed type is 'builtins.int' +def fun_p(x: PP) -> None: + reveal_type(P.attr) # E: Revealed type is 'builtins.int' + +fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P"\ + # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute +fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP"\ + # N: Following members of __main__.D have conflicts:\ + # N: attr: expected builtins.int, got builtins.str +fun_p(C()) # OK +[builtins fixtures/list.pyi] +[out] + +[case testIpmlicitTypesInProtocols] +from typing import Protocol + +class P(Protocol): + x = 1 # We could actually prohibit inferred types in protocol declarations + +class C: + x: int + +class D: + x: str + +x: P +x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P")\ + # N: Following members of __main__.D have conflicts:\ + # N: x: expected builtins.int, got builtins.str +x = C() # OK +[builtins fixtures/list.pyi] +[out] From 06a168046553af9dc1893d0275a00d1baf900eda Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 3 May 2017 00:00:07 +0200 Subject: [PATCH 056/117] Fix fixture after merge --- test-data/unit/check-protocols.test | 2 +- test-data/unit/fixtures/dict.pyi | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index a1d245b5fbea..a0690fb994a1 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -335,7 +335,7 @@ class C: pass z: P = C() -[builtins fixtures/isinstancelist.pyi] +[builtins fixtures/dict.pyi] [out] [case testProtocolsCannotInheritFromNormal] diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index 9f5089354df2..626ba602257e 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -41,4 +41,5 @@ class float: pass class bool: pass class ellipsis: pass +def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass class BaseException: pass From 985d1f78b9dc98c3ea68c2ecf2718c0234b9797c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 5 May 2017 21:01:23 +0200 Subject: [PATCH 057/117] Add variance checks and modify tests accordingly --- mypy/checker.py | 25 +++++++++++++++++++++++- test-data/unit/check-incremental.test | 2 +- test-data/unit/check-protocols.test | 28 ++++++++++++++------------- test-data/unit/lib-stub/typing.pyi | 18 ++++++++++------- 4 files changed, 51 insertions(+), 22 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e37620d30ff0..671b5fba7596 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -27,7 +27,7 @@ RefExpr, YieldExpr, BackquoteExpr, Import, ImportFrom, ImportAll, ImportBase, AwaitExpr, PromoteExpr, Node, EnumCallExpr, ARG_POS, MDEF, - CONTRAVARIANT, COVARIANT) + CONTRAVARIANT, COVARIANT, INVARIANT) from mypy import nodes from mypy.types import ( Type, AnyType, CallableType, FunctionLike, Overloaded, TupleType, TypedDictType, @@ -1073,6 +1073,8 @@ def erase_override(t: Type) -> Type: def visit_class_def(self, defn: ClassDef) -> None: """Type check a class definition.""" typ = defn.info + if typ.is_protocol and typ.defn.type_vars: + self.check_protocol_variance(defn) with self.errors.enter_type(defn.name), self.enter_partial_types(): old_binder = self.binder self.binder = ConditionalTypeBinder() @@ -1084,6 +1086,27 @@ def visit_class_def(self, defn: ClassDef) -> None: # Otherwise we've already found errors; more errors are not useful self.check_multiple_inheritance(typ) + def check_protocol_variance(self, defn: ClassDef) -> None: + info = defn.info + object_type = Instance(info.mro[-1], []) + tvars = info.defn.type_vars + for i, tvar in enumerate(tvars): + up_args = [object_type if i == j else AnyType() for j, _ in enumerate(tvars)] + down_args = [UninhabitedType() if i == j else AnyType() for j, _ in enumerate(tvars)] + up, down = Instance(info, up_args), Instance(info, down_args) + is_co = is_subtype(down, up) + is_contra = is_subtype(up, down) + if is_co: + expected = 'covariant' + elif is_contra: + expected = 'contravariant' + else: + expected = 'invariant' + # TODO: add more variance checks (actual covariant and actual invariant) + if tvar.variance == INVARIANT and expected != 'invariant': + self.fail("Invariant type variable '{}' used in protocol where" + " {} one is expected".format(tvar.name, expected), defn) + def check_multiple_inheritance(self, typ: TypeInfo) -> None: """Check for multiple inheritance related errors.""" if len(typ.bases) <= 1: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 85b836f5749e..94da7b88de39 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1485,7 +1485,7 @@ y = C() [file a.py] from typing import Protocol, TypeVar -T = TypeVar('T') +T = TypeVar('T', covariant=True) class P(Protocol[T]): def meth(self) -> T: pass diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index a0690fb994a1..3f5e6d0911a9 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -265,8 +265,8 @@ class C(MyHashable): [case testBasicSemanalErrorsInProtocols] from typing import Protocol, Generic, TypeVar, Iterable -T = TypeVar('T') -S = TypeVar('S') +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) class P1(Protocol[T, T]): # E: Duplicate type variables in Generic[...] or Protocol[...] def meth(self) -> T: @@ -362,10 +362,11 @@ from typing import TypeVar, Protocol T = TypeVar('T') -class Pco(Protocol[T]): +# In case of these errors we proceed with inferred variance. +class Pco(Protocol[T]): # E: Invariant type variable 'T' used in protocol where covariant one is expected def meth(self) -> T: pass -class Pcontra(Protocol[T]): +class Pcontra(Protocol[T]): # E: Invariant type variable 'T' used in protocol where contravariant one is expected def meth(self, x: T) -> None: pass class Pinv(Protocol[T]): @@ -401,7 +402,7 @@ x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], vari [case testGenericProtocolsInference1] from typing import Protocol, Sequence, TypeVar -T = TypeVar('T') +T = TypeVar('T', covariant=True) class Closeable(Protocol[T]): def close(self) -> T: @@ -448,8 +449,9 @@ reveal_type(fun3(C())) # E: Revealed type is 'builtins.int*' [case testProtocolGenericInferenceCovariant] from typing import Generic, TypeVar, Protocol -T = TypeVar('T') -S = TypeVar('S') +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) +U = TypeVar('U') class P(Protocol[T, S]): def x(self) -> T: pass @@ -459,7 +461,7 @@ class C: def x(self) -> int: pass def y(self) -> int: pass -def fun4(x: T, y: P[T, T]) -> T: +def fun4(x: U, y: P[U, U]) -> U: pass reveal_type(fun4('a', C())) # E: Revealed type is 'builtins.object*' [out] @@ -534,8 +536,8 @@ var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment [case testGenericSubProtocolsExtensionCovariant] from typing import TypeVar, Protocol, Union -T = TypeVar('T') -S = TypeVar('S') +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) class P1(Protocol[T]): def attr1(self) -> T: pass @@ -1286,7 +1288,7 @@ else: [case testBasicTupleStructuralSubtyping] from typing import Tuple, TypeVar, Protocol -T = TypeVar('T') +T = TypeVar('T', covariant=True) class MyProto(Protocol[T]): def __len__(self) -> T: @@ -1304,8 +1306,8 @@ f(t) # OK [case testBasicNamedTupleStructuralSubtyping] from typing import NamedTuple, TypeVar, Protocol -T = TypeVar('T') -S = TypeVar('S') +T = TypeVar('T', covariant=True) +S = TypeVar('S', covariant=True) class P(Protocol[T, S]): @property diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index e328513febad..cf1b1d2d21e9 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -28,12 +28,16 @@ Set = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') +# Note: definitions below are different from typeshed, variances are declared +# to silence the protocol variance checks. Maybe it is better to use type: ignore? + @runtime -class Container(Protocol[T]): +class Container(Protocol[T_contra]): @abstractmethod # Use int because bool isn't in the default test builtins def __contains__(self, arg: T) -> int: pass @@ -102,17 +106,17 @@ class AsyncIterator(AsyncIterable[T], Protocol): def __anext__(self) -> Awaitable[T]: pass @runtime -class Sequence(Iterable[T], Protocol): +class Sequence(Iterable[T_co], Protocol): @abstractmethod - def __getitem__(self, n: Any) -> T: pass + def __getitem__(self, n: Any) -> T_co: pass @runtime -class Mapping(Protocol[T, U]): - def __getitem__(self, key: T) -> U: pass +class Mapping(Protocol[T_contra, T_co]): + def __getitem__(self, key: T_contra) -> T_co: pass @runtime -class MutableMapping(Mapping[T, U], Protocol): - def __setitem__(self, k: T, v: U) -> None: pass +class MutableMapping(Mapping[T_contra, U], Protocol): + def __setitem__(self, k: T_contra, v: U) -> None: pass class SupportsInt(Protocol): def __int__(self) -> int: pass From 8d2e199a8e0679237209b620065a7003f0559dd1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 May 2017 16:46:20 +0200 Subject: [PATCH 058/117] Minor update to docs --- docs/source/class_basics.rst | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index f08501235042..2ed4bd9501c8 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -263,8 +263,9 @@ Generic protocols ***************** Generic protocols are also supported, generic protocols mostly follow -the normal rules for generic classes, the main difference is that declaring -variance is not necessary for protocols, mypy can infer it. Examples: +the normal rules for generic classes, the main difference is that mypy checks +that declared variance of type variables is compatible with +the class definition. Examples: .. code-block:: python @@ -290,7 +291,20 @@ variance is not necessary for protocols, mypy can infer it. Examples: x: Box[float] y: Box[int] - x = y # This is also OK due to inferred covariance of the protocol 'Box'. + x = y # Error, since the protocol 'Box' is invariant. + + class AnotherBox(Protocol[T]): # Error, covariant type variable expected + def content(self) -> T: + ... + + T_co = TypeVar('T_co', covariant=True) + class AnotherBox(Protocol[T_co]): # OK + def content(self) -> T_co: + ... + + ax: AnotherBox[float] + ay: AnotherBox[int] + ax = ay # OK for covariant protocols See :ref:`generic-classes` for more details on generic classes and variance. @@ -320,11 +334,11 @@ such as trees and linked lists: T = TypeVar('T') class Linked(Protocol[T]): val: T - next: 'Linked[T]' + def next(self) -> 'Linked[T]': ... class L: val: int - next: 'L' + def next(self) -> 'L': ... def last(seq: Linked[T]) -> T: ... From 4dfa3e13aa086cc17be0d8d8fac72351026ca1b7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 21 May 2017 22:45:04 +0200 Subject: [PATCH 059/117] Fix trailing whitespace --- mypy/semanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index df1f6be0d6c6..de5a43269506 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3726,7 +3726,7 @@ def visit_class_def(self, tdef: ClassDef) -> None: if tdef.info.is_protocol: if not isinstance(type, Instance) or not type.type.is_protocol: if type.type.fullname() != 'builtins.object': - self.fail('All bases of a protocol must be protocols', tdef) + self.fail('All bases of a protocol must be protocols', tdef) # Recompute MRO now that we have analyzed all modules, to pick # up superclasses of bases imported from other modules in an # import loop. (Only do so if we succeeded the first time.) From 6d050605f991aa45e2f3bd567cb5b0e202dd5a01 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 22 May 2017 16:00:21 +0200 Subject: [PATCH 060/117] Detailed error messages also for structural subtyping of tuples and callables --- mypy/checker.py | 14 +++++++++++--- mypy/checkexpr.py | 10 ++++++++-- test-data/unit/check-protocols.test | 12 ++++++++++-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 671b5fba7596..1cdd922f5d7d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -44,7 +44,7 @@ is_subtype, is_equivalent, is_proper_subtype, is_more_precise, restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_subtype, unify_generic_callable, get_missing_members, get_conflict_types, get_all_flags, - IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC + IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC, find_member ) from mypy.maptype import map_instance_to_supertype from mypy.typevars import fill_typevars, has_no_typevars @@ -2345,17 +2345,25 @@ def check_subtype(self, subtype: Type, supertype: Type, context: Context, msg += ' (' + ', '.join(extra_info) + ')' self.fail(msg, context) if (isinstance(supertype, Instance) and supertype.type.is_protocol and - isinstance(subtype, Instance)): + isinstance(subtype, (Instance, TupleType))): self.report_protocol_problems(subtype, supertype, context) + if isinstance(supertype, CallableType) and isinstance(subtype, Instance): + call = find_member('__call__', subtype, subtype) + if call: + self.note("{}.__call__ has type {}".format(subtype, call), context) return False - def report_protocol_problems(self, subtype: Instance, supertype: Instance, + def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertype: Instance, context: Context) -> None: """Report possible protocol conflicts between 'subtype' and 'supertype'. This includes missing members, incompatible types, and incompatible attribute flags, such as settable vs read-only or class variable vs instance variable. """ + if isinstance(subtype, TupleType): + if not isinstance(subtype.fallback, Instance): + return + subtype = subtype.fallback missing = get_missing_members(subtype, supertype) if missing: self.note("'{}' missing following '{}' protocol members:" diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e00419ad213d..446f34177515 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -34,7 +34,7 @@ from mypy import join from mypy.meet import narrow_declared_type from mypy.maptype import map_instance_to_supertype -from mypy.subtypes import is_subtype, is_equivalent +from mypy.subtypes import is_subtype, is_equivalent, find_member from mypy import applytype from mypy import erasetype from mypy.checkmember import analyze_member_access, type_object_type, bind_self @@ -888,9 +888,15 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, return messages.incompatible_argument(n, m, callee, original_caller_type, caller_kind, context) - if (isinstance(original_caller_type, Instance) and + if (isinstance(original_caller_type, (Instance, TupleType)) and isinstance(callee_type, Instance) and callee_type.type.is_protocol): self.chk.report_protocol_problems(original_caller_type, callee_type, context) + if (isinstance(callee_type, CallableType) and + isinstance(original_caller_type, Instance)): + call = find_member('__call__', original_caller_type, original_caller_type) + if call: + self.chk.note("{}.__call__ has type {}" + .format(original_caller_type, call), context) def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], arg_names: List[str], diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 3f5e6d0911a9..905ffc8b3231 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1335,9 +1335,13 @@ def fun3(x: P[T, T]) -> T: return x.x fun(z) -fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int] +fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int]\ + # N: Following members of __main__.N have conflicts:\ + # N: y: expected builtins.int*, got builtins.str -fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str] +fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str]\ + # N: '__main__.N2' missing following '__main__.P' protocol members:\ + # N: y reveal_type(fun3(z)) # E: Revealed type is 'builtins.object*' @@ -1363,6 +1367,10 @@ def apply_gen(f: Callable[[T], T]) -> T: pass reveal_type(apply_gen(Add5())) # E: Revealed type is 'builtins.int*' +def apply_str(f: Callable[[str], int], x: str) -> int: + return f(x) +apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected Callable[[str], int]\ + # N: __main__.Add5.__call__ has type def (x: builtins.int) -> builtins.int [builtins fixtures/isinstancelist.pyi] [out] From 759409f84dfd7a3afe4648b033af9db4d7877885 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 22 May 2017 16:45:05 +0200 Subject: [PATCH 061/117] Add more protocol variance checks --- mypy/checker.py | 18 ++++++++++++------ mypy/subtypes.py | 13 +++++++++---- test-data/unit/check-protocols.test | 26 +++++++++++++++++++++++++- test-data/unit/lib-stub/typing.pyi | 2 +- 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 1cdd922f5d7d..55432b0c8a55 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1094,18 +1094,24 @@ def check_protocol_variance(self, defn: ClassDef) -> None: up_args = [object_type if i == j else AnyType() for j, _ in enumerate(tvars)] down_args = [UninhabitedType() if i == j else AnyType() for j, _ in enumerate(tvars)] up, down = Instance(info, up_args), Instance(info, down_args) - is_co = is_subtype(down, up) - is_contra = is_subtype(up, down) + # TODO: add advanced variance checks for recursive protocols + is_co = is_subtype(down, up, ignore_declared_variance=True) + is_contra = is_subtype(up, down, ignore_declared_variance=True) if is_co: expected = 'covariant' elif is_contra: expected = 'contravariant' else: expected = 'invariant' - # TODO: add more variance checks (actual covariant and actual invariant) - if tvar.variance == INVARIANT and expected != 'invariant': - self.fail("Invariant type variable '{}' used in protocol where" - " {} one is expected".format(tvar.name, expected), defn) + if tvar.variance == COVARIANT: + actual = 'Covariant' + elif tvar.variance == CONTRAVARIANT: + actual = 'Contravariant' + else: + actual = 'Invariant' + if expected != actual.lower(): + self.fail("{} type variable '{}' used in protocol where" + " {} one is expected".format(actual, tvar.name, expected), defn) def check_multiple_inheritance(self, typ: TypeInfo) -> None: """Check for multiple inheritance related errors.""" diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 2598f3c9c7dc..14566e746e3b 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -44,7 +44,8 @@ def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: def is_subtype(left: Type, right: Type, type_parameter_checker: TypeParameterChecker = check_type_parameter, - *, ignore_pos_arg_names: bool = False) -> bool: + *, ignore_pos_arg_names: bool = False, + ignore_declared_variance: bool = False) -> bool: """Is 'left' subtype of 'right'? Also consider Any to be a subtype of any type, and vice versa. This @@ -83,7 +84,8 @@ def is_subtype(left: Type, right: Type, elif is_named_instance(right, 'builtins.type'): return is_subtype(left, TypeType(AnyType())) return left.accept(SubtypeVisitor(right, type_parameter_checker, - ignore_pos_arg_names=ignore_pos_arg_names)) + ignore_pos_arg_names=ignore_pos_arg_names, + ignore_declared_variance=ignore_declared_variance)) def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: @@ -107,10 +109,12 @@ class SubtypeVisitor(TypeVisitor[bool]): def __init__(self, right: Type, type_parameter_checker: TypeParameterChecker, - *, ignore_pos_arg_names: bool = False) -> None: + *, ignore_pos_arg_names: bool = False, + ignore_declared_variance: bool = False) -> None: self.right = right self.check_type_parameter = type_parameter_checker self.ignore_pos_arg_names = ignore_pos_arg_names + self.ignore_declared_variance = ignore_declared_variance # visit_x(left) means: is left (which is an instance of X) a subtype of # right? @@ -158,7 +162,8 @@ def visit_instance(self, left: Instance) -> bool: rname = right.type.fullname() # Always try a nominal check if possible, # there might be errors that a user wants to silence *once*. - if left.type.has_base(rname) or rname == 'builtins.object': + if ((left.type.has_base(rname) or rname == 'builtins.object') and + not self.ignore_declared_variance): # Map left type to corresponding right instances. t = map_instance_to_supertype(left, right.type) nominal = all(self.check_type_parameter(lefta, righta, tvar.variance) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 905ffc8b3231..262527c48454 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -599,12 +599,36 @@ from typing import Protocol, TypeVar T_co = TypeVar('T_co', covariant=True) T_contra = TypeVar('T_contra', contravariant=True) -class Proto(Protocol[T_co, T_contra]): +class Proto(Protocol[T_co, T_contra]): # type: ignore def one(self, x: T_co) -> None: # E: Cannot use a covariant type variable as a parameter pass def other(self) -> T_contra: # E: Cannot use a contravariant type variable as return type pass +# Check that we respect user overrides of variance after the errors are reported +x: Proto[int, float] +y: Proto[float, int] +y = x # OK +[builtins fixtures/list.pyi] +[out] + +[case testSubtleBadVarianceInProtocols] +from typing import Protocol, TypeVar, Iterable, Sequence + +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class Proto(Protocol[T_co, T_contra]): # E: Covariant type variable 'T_co' used in protocol where contravariant one is expected\ + # E: Contravariant type variable 'T_contra' used in protocol where covariant one is expected + def one(self, x: Iterable[T_co]) -> None: + pass + def other(self) -> Sequence[T_contra]: + pass + +# Check that we respect user overrides of variance after the errors are reported +x: Proto[int, float] +y: Proto[float, int] +y = x # OK [builtins fixtures/list.pyi] [out] diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 4a62fab130cd..7afc103bf3f6 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -42,7 +42,7 @@ S = TypeVar('S') class Container(Protocol[T_contra]): @abstractmethod # Use int because bool isn't in the default test builtins - def __contains__(self, arg: T) -> int: pass + def __contains__(self, arg: T_contra) -> int: pass @runtime class Sized(Protocol): From 166b6da79e160570a8c0deeda226f8b4999f6436 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 May 2017 12:04:29 +0200 Subject: [PATCH 062/117] Fix careless merge (#N: formatting) --- test-data/unit/check-async-await.test | 24 ++-- test-data/unit/check-expressions.test | 8 +- test-data/unit/check-namedtuple.test | 4 +- test-data/unit/check-protocols.test | 174 +++++++++++++------------- 4 files changed, 105 insertions(+), 105 deletions(-) diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index 3623007ebd70..e01cbb5ddfdb 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -287,8 +287,8 @@ class C: def __aenter__(self) -> int: pass async def __aexit__(self, x, y, z) -> None: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any])\ - # N: 'builtins.int' missing following 'typing.Awaitable' protocol members:\ + async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any]) \ + # N: 'builtins.int' missing following 'typing.Awaitable' protocol members: \ # N: __await__ pass [builtins fixtures/async_await.pyi] @@ -311,8 +311,8 @@ class C: async def __aenter__(self) -> int: pass def __aexit__(self, x, y, z) -> int: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any])\ - # N: 'builtins.int' missing following 'typing.Awaitable' protocol members:\ + async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any]) \ + # N: 'builtins.int' missing following 'typing.Awaitable' protocol members: \ # N: __await__ pass [builtins fixtures/async_await.pyi] @@ -637,14 +637,14 @@ def plain_host_generator() -> Generator[str, None, None]: async def plain_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any])\ - # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members:\ + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) \ + # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members: \ # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any])\ - # N: '__main__.It' missing following 'typing.Awaitable' protocol members:\ + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) \ + # N: '__main__.It' missing following 'typing.Awaitable' protocol members: \ # N: __await__ x = await other_coroutine() @@ -662,14 +662,14 @@ def decorated_host_generator() -> Generator[str, None, None]: @coroutine async def decorated_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any])\ - # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members:\ + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) \ + # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members: \ # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any])\ - # N: '__main__.It' missing following 'typing.Awaitable' protocol members:\ + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) \ + # N: '__main__.It' missing following 'typing.Awaitable' protocol members: \ # N: __await__ x = await other_coroutine() diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 17f8b6c1d91a..60440378d498 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1670,10 +1670,10 @@ b = {'z': 26, **a} c = {**b} d = {**a, **b, 'c': 3} # These errors changed because Mapping is a protocol -e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str]\ - # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts:\ +e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str] \ + # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ # N: __getitem__: expected def (builtins.int*) -> builtins.str*, got def (builtins.str*) -> builtins.int* -f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int]\ - # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts:\ +f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int] \ + # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ # N: __getitem__: expected def (builtins.int*) -> builtins.int*, got def (builtins.str*) -> builtins.int* [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 03488dadcf9d..9e92664cb7f8 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -286,8 +286,8 @@ from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) reveal_type(X._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' -X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any]\ - # N: 'builtins.str' missing following 'typing.Iterable' protocol members:\ +X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any] \ + # N: 'builtins.str' missing following 'typing.Iterable' protocol members: \ # N: __iter__ # Note that the above will not fail with real stubs, since 'str', is a subtype of 'Iterable[str]' diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 262527c48454..c6fb1296d9d9 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -30,20 +30,20 @@ def fun(x: P) -> None: x.bad # E: "P" has no attribute "bad" x = C() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ - # N: '__main__.B' missing following '__main__.P' protocol members:\ +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ + # N: '__main__.B' missing following '__main__.P' protocol members: \ # N: meth fun(C()) -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P"\ - # N: '__main__.B' missing following '__main__.P' protocol members:\ +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" \ + # N: '__main__.B' missing following '__main__.P' protocol members: \ # N: meth def fun2() -> P: return C() def fun3() -> P: - return B() # E: Incompatible return value type (got "B", expected "P")\ - # N: '__main__.B' missing following '__main__.P' protocol members:\ + return B() # E: Incompatible return value type (got "B", expected "P") \ + # N: '__main__.B' missing following '__main__.P' protocol members: \ # N: meth [out] @@ -72,13 +72,13 @@ def fun(x: P) -> None: x = C() x = D() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ - # N: '__main__.B' missing following '__main__.P' protocol members:\ +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ + # N: '__main__.B' missing following '__main__.P' protocol members: \ # N: meth fun(C()) fun(D()) -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P"\ - # N: '__main__.B' missing following '__main__.P' protocol members:\ +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" \ + # N: '__main__.B' missing following '__main__.P' protocol members: \ # N: meth [out] @@ -104,13 +104,13 @@ def fun(x: SubP) -> str: z = x x = C() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP")\ - # N: '__main__.B' missing following '__main__.SubP' protocol members:\ +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") \ + # N: '__main__.B' missing following '__main__.SubP' protocol members: \ # N: meth reveal_type(fun(C())) # E: Revealed type is 'builtins.str' -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP"\ - # N: '__main__.B' missing following '__main__.SubP' protocol members:\ +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" \ + # N: '__main__.B' missing following '__main__.SubP' protocol members: \ # N: meth [out] @@ -154,11 +154,11 @@ c2: C2 y: AnotherP x = c -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P")\ - # N: '__main__.B' missing following '__main__.P' protocol members:\ +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ + # N: '__main__.B' missing following '__main__.P' protocol members: \ # N: meth1, meth2 -x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P")\ - # N: '__main__.C1' missing following '__main__.P' protocol members:\ +x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P") \ + # N: '__main__.C1' missing following '__main__.P' protocol members: \ # N: meth2 x = c2 x = y @@ -190,8 +190,8 @@ reveal_type(x.meth1()) # E: Revealed type is 'builtins.int' reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' x = C() # OK -x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2")\ - # N: '__main__.Cbad' missing following '__main__.P2' protocol members:\ +x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2") \ + # N: '__main__.Cbad' missing following '__main__.P2' protocol members: \ # N: meth2 [out] @@ -293,7 +293,7 @@ from typing import Protocol class P(Protocol): def __init__(self, a: int) -> None: - self.a = a # E: Protocol members cannot be defined via assignment to self\ + self.a = a # E: Protocol members cannot be defined via assignment to self \ # E: "P" has no attribute "a" class B: pass @@ -377,25 +377,25 @@ class B(A): pass x1: Pco[B] y1: Pco[A] -x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B])\ - # N: Following members of __main__.Pco[__main__.A] have conflicts:\ +x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B]) \ + # N: Following members of __main__.Pco[__main__.A] have conflicts: \ # N: meth: expected def () -> __main__.B*, got def () -> __main__.A* y1 = x1 x2: Pcontra[B] y2: Pcontra[A] -y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A])\ - # N: Following members of __main__.Pcontra[__main__.B] have conflicts:\ +y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A]) \ + # N: Following members of __main__.Pcontra[__main__.B] have conflicts: \ # N: meth: expected def (x: __main__.A*), got def (x: __main__.B*) x2 = y2 x3: Pinv[B] y3: Pinv[A] -y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A])\ - # N: Following members of __main__.Pinv[__main__.B] have conflicts:\ +y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A]) \ + # N: Following members of __main__.Pinv[__main__.B] have conflicts: \ # N: attr: expected __main__.A*, got __main__.B* -x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B])\ - # N: Following members of __main__.Pinv[__main__.A] have conflicts:\ +x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B]) \ + # N: Following members of __main__.Pinv[__main__.A] have conflicts: \ # N: attr: expected __main__.B*, got __main__.A* [out] @@ -484,8 +484,8 @@ class C: c: C var: P2[int, int] = c -var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str])\ - # N: Following members of __main__.C have conflicts:\ +var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str]) \ + # N: Following members of __main__.C have conflicts: \ # N: attr2: expected Tuple[builtins.int*, builtins.str*], got Tuple[builtins.int, builtins.int] class D(Generic[T]): @@ -495,8 +495,8 @@ class E(D[T]): def f(x: T) -> T: z: P2[T, T] = E[T]() - y: P2[T, T] = D[T]() # E: Incompatible types in assignment (expression has type D[T], variable has type P2[T, T])\ - # N: '__main__.D' missing following '__main__.P2' protocol members:\ + y: P2[T, T] = D[T]() # E: Incompatible types in assignment (expression has type D[T], variable has type P2[T, T]) \ + # N: '__main__.D' missing following '__main__.P2' protocol members: \ # N: attr2 return x [builtins fixtures/isinstancelist.pyi] @@ -527,9 +527,9 @@ class D(A, B): pass x: P = D() # Same as P[Any, Any] -var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[int, P[Any, Any]], Union[P[Any, Any], str]])\ - # N: Following members of __main__.C have conflicts:\ - # N: attr1: expected Union[builtins.int, __main__.P[Any, Any]], got builtins.int\ +var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[int, P[Any, Any]], Union[P[Any, Any], str]]) \ + # N: Following members of __main__.C have conflicts: \ + # N: attr1: expected Union[builtins.int, __main__.P[Any, Any]], got builtins.int \ # N: attr2: expected Union[__main__.P[Any, Any], builtins.str], got builtins.str [out] @@ -551,9 +551,9 @@ class C: def attr2(self) -> str: pass var: P[Union[int, P], Union[P, str]] = C() # OK for covariant -var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]])\ - # N: Following members of __main__.C have conflicts:\ - # N: attr1: expected def () -> Union[builtins.str, __main__.P[Any, Any]], got def () -> builtins.int\ +var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) \ + # N: Following members of __main__.C have conflicts: \ + # N: attr1: expected def () -> Union[builtins.str, __main__.P[Any, Any]], got def () -> builtins.int \ # N: attr2: expected def () -> Union[__main__.P[Any, Any], builtins.int], got def () -> builtins.str [out] @@ -583,11 +583,11 @@ s: Shape f(NonProtoShape()) f(Circle()) -s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape")\ - # N: Following members of __main__.Triangle have conflicts:\ +s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") \ + # N: Following members of __main__.Triangle have conflicts: \ # N: combine: expected def (other: __main__.Triangle*) -> __main__.Triangle*, got def (other: __main__.Shape) -> __main__.Shape -s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape")\ - # N: Following members of __main__.Bad have conflicts:\ +s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") \ + # N: Following members of __main__.Bad have conflicts: \ # N: combine: expected def (other: __main__.Bad*) -> __main__.Bad*, got def (other: builtins.int) -> builtins.str n2: NonProtoShape = s # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") @@ -618,7 +618,7 @@ from typing import Protocol, TypeVar, Iterable, Sequence T_co = TypeVar('T_co', covariant=True) T_contra = TypeVar('T_contra', contravariant=True) -class Proto(Protocol[T_co, T_contra]): # E: Covariant type variable 'T_co' used in protocol where contravariant one is expected\ +class Proto(Protocol[T_co, T_contra]): # E: Covariant type variable 'T_co' used in protocol where contravariant one is expected \ # E: Contravariant type variable 'T_contra' used in protocol where covariant one is expected def one(self, x: Iterable[T_co]) -> None: pass @@ -651,8 +651,8 @@ class D(Generic[T]): t: Traversable t = D[int]() # OK -t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable")\ - # N: '__main__.C' missing following '__main__.Traversable' protocol members:\ +t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable") \ + # N: '__main__.C' missing following '__main__.Traversable' protocol members: \ # N: leaves [builtins fixtures/list.pyi] [out] @@ -697,11 +697,11 @@ class B: t: P1 t = A() # OK -t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1")\ - # N: '__main__.B' missing following '__main__.P1' protocol members:\ +t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1") \ + # N: '__main__.B' missing following '__main__.P1' protocol members: \ # N: attr1 -t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1")\ - # N: '__main__.C' missing following '__main__.P1' protocol members:\ +t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1") \ + # N: '__main__.C' missing following '__main__.P1' protocol members: \ # N: attr1 [builtins fixtures/list.pyi] [out] @@ -780,10 +780,10 @@ x: PInst y: PClass x = CInst() -x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst")\ +x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst") \ # N: Protocol member __main__.PInst.v: expected instance variable, got class variable y = CClass() -y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass")\ +y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass") \ # N: Protocol member __main__.PClass.v: expected class variable, got instance variable [out] @@ -804,7 +804,7 @@ y = x x2: P y2: PP -x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P")\ +x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P") \ # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -842,7 +842,7 @@ y3 = z3 y4: PP z4: PPS -z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS")\ +z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS") \ # N: Protocol member __main__.PPS.attr: expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -874,7 +874,7 @@ x = B() y: PC y = B() -y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC")\ +y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC") \ # N: Protocol member __main__.PC.meth: expected class or static method [builtins fixtures/classmethod.pyi] [out] @@ -897,8 +897,8 @@ class D: pass x: P = C() -x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P")\ - # N: Following members of __main__.D have conflicts:\ +x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ + # N: Following members of __main__.D have conflicts: \ # N: f: expected Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str), got def (x: builtins.int) [out] @@ -1359,12 +1359,12 @@ def fun3(x: P[T, T]) -> T: return x.x fun(z) -fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int]\ - # N: Following members of __main__.N have conflicts:\ +fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int] \ + # N: Following members of __main__.N have conflicts: \ # N: y: expected builtins.int*, got builtins.str -fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str]\ - # N: '__main__.N2' missing following '__main__.P' protocol members:\ +fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str] \ + # N: '__main__.N2' missing following '__main__.P' protocol members: \ # N: y reveal_type(fun3(z)) # E: Revealed type is 'builtins.object*' @@ -1393,7 +1393,7 @@ def apply_gen(f: Callable[[T], T]) -> T: reveal_type(apply_gen(Add5())) # E: Revealed type is 'builtins.int*' def apply_str(f: Callable[[str], int], x: str) -> int: return f(x) -apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected Callable[[str], int]\ +apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected Callable[[str], int] \ # N: __main__.Add5.__call__ has type def (x: builtins.int) -> builtins.int [builtins fixtures/isinstancelist.pyi] [out] @@ -1490,8 +1490,8 @@ f1(C1()) f2(C2()) f3(C3()) -f2(C3()) # E: Argument 1 to "f2" has incompatible type "C3"; expected P2[str]\ - # N: '__main__.C3' missing following '__main__.P2' protocol members:\ +f2(C3()) # E: Argument 1 to "f2" has incompatible type "C3"; expected P2[str] \ + # N: '__main__.C3' missing following '__main__.P2' protocol members: \ # N: attr2 a: Any f1(a) @@ -1512,29 +1512,29 @@ class C: @property def attr2(self) -> int: pass -x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P")\ - # N: '__main__.C' missing following '__main__.P' protocol members:\ - # N: attr3\ - # N: Following members of __main__.C have conflicts:\ - # N: attr1: expected builtins.int, got builtins.str\ - # N: attr2: expected builtins.str, got builtins.int\ +x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ + # N: '__main__.C' missing following '__main__.P' protocol members: \ + # N: attr3 \ + # N: Following members of __main__.C have conflicts: \ + # N: attr1: expected builtins.int, got builtins.str \ + # N: attr2: expected builtins.str, got builtins.int \ # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute def f(x: P) -> P: - return C() # E: Incompatible return value type (got "C", expected "P")\ - # N: '__main__.C' missing following '__main__.P' protocol members:\ - # N: attr3\ - # N: Following members of __main__.C have conflicts:\ - # N: attr1: expected builtins.int, got builtins.str\ - # N: attr2: expected builtins.str, got builtins.int\ + return C() # E: Incompatible return value type (got "C", expected "P") \ + # N: '__main__.C' missing following '__main__.P' protocol members: \ + # N: attr3 \ + # N: Following members of __main__.C have conflicts: \ + # N: attr1: expected builtins.int, got builtins.str \ + # N: attr2: expected builtins.str, got builtins.int \ # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute -f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P"\ - # N: '__main__.C' missing following '__main__.P' protocol members:\ - # N: attr3\ - # N: Following members of __main__.C have conflicts:\ - # N: attr1: expected builtins.int, got builtins.str\ - # N: attr2: expected builtins.str, got builtins.int\ +f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ + # N: '__main__.C' missing following '__main__.P' protocol members: \ + # N: attr3 \ + # N: Following members of __main__.C have conflicts: \ + # N: attr1: expected builtins.int, got builtins.str \ + # N: attr2: expected builtins.str, got builtins.int \ # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute [builtins fixtures/list.pyi] [out] @@ -1595,10 +1595,10 @@ def fun(x: P) -> None: def fun_p(x: PP) -> None: reveal_type(P.attr) # E: Revealed type is 'builtins.int' -fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P"\ +fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P" \ # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute -fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP"\ - # N: Following members of __main__.D have conflicts:\ +fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" \ + # N: Following members of __main__.D have conflicts: \ # N: attr: expected builtins.int, got builtins.str fun_p(C()) # OK [builtins fixtures/list.pyi] @@ -1617,8 +1617,8 @@ class D: x: str x: P -x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P")\ - # N: Following members of __main__.D have conflicts:\ +x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ + # N: Following members of __main__.D have conflicts: \ # N: x: expected builtins.int, got builtins.str x = C() # OK [builtins fixtures/list.pyi] From d652a698734379b3f07655559517635993d9fced Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 1 Jun 2017 10:18:38 +0200 Subject: [PATCH 063/117] Fix merge --- test-data/unit/check-protocols.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index c6fb1296d9d9..84dc7c1a8d1e 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1555,7 +1555,7 @@ reveal_type(list(B())) # E: Revealed type is 'builtins.list[__main__.B*]' [case testIterableProtocolOnMetaclass] from typing import TypeVar, Iterator, Type -T = TypeVar('T', bound='E') +T = TypeVar('T') class EMeta(type): def __iter__(self: Type[T]) -> Iterator[T]: pass From 803ce1eb486f19be73392a15918d8d5001d63afc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 4 Jun 2017 02:12:33 +0200 Subject: [PATCH 064/117] Fix merge --- test-data/unit/check-incomplete-fixture.test | 2 ++ test-data/unit/lib-stub/builtins.pyi | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-incomplete-fixture.test b/test-data/unit/check-incomplete-fixture.test index 68c7c6c9aa0f..759e274b0207 100644 --- a/test-data/unit/check-incomplete-fixture.test +++ b/test-data/unit/check-incomplete-fixture.test @@ -77,6 +77,8 @@ for y in x: main:2: error: "tuple" expects no type arguments, but 1 given main:3: error: Value of type "tuple" is not indexable main:4: error: Iterable expected +main:4: note: 'builtins.tuple' missing following 'typing.Iterable' protocol members: +main:4: note: __iter__ main:4: error: "tuple" has no attribute "__iter__" [case testClassmethodMissingFromStubs] diff --git a/test-data/unit/lib-stub/builtins.pyi b/test-data/unit/lib-stub/builtins.pyi index 0c8fc2052e35..87b50f532376 100644 --- a/test-data/unit/lib-stub/builtins.pyi +++ b/test-data/unit/lib-stub/builtins.pyi @@ -15,7 +15,6 @@ class bytes: pass class tuple: pass class function: pass -class bool: pass class ellipsis: pass # Definition of None is implicit From 3c0411cfea6057867bad7f5096d5a857854d210d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 11 Jun 2017 10:31:24 +0200 Subject: [PATCH 065/117] Improve notes formatting: add offsets --- mypy/checker.py | 15 +- mypy/errors.py | 4 +- mypy/messages.py | 8 +- test-data/unit/check-async-await.test | 60 +++---- test-data/unit/check-expressions.test | 8 +- test-data/unit/check-incomplete-fixture.test | 4 +- test-data/unit/check-namedtuple.test | 4 +- test-data/unit/check-protocols.test | 158 +++++++++---------- 8 files changed, 133 insertions(+), 128 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3985e19c3f0a..4a49d3236b1b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2415,23 +2415,26 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp attribute flags, such as settable vs read-only or class variable vs instance variable. """ + OFFSET = 4 # Four spaces, so that notes will look like this: + # note: 'Cls' is missing following 'Proto' member(s): + # note: method, attr if isinstance(subtype, TupleType): if not isinstance(subtype.fallback, Instance): return subtype = subtype.fallback missing = get_missing_members(subtype, supertype) if missing: - self.note("'{}' missing following '{}' protocol members:" + self.note("'{}' is missing following '{}' protocol member(s):" .format(subtype.type.fullname(), supertype.type.fullname()), context) - self.note(', '.join(missing), context) + self.note(', '.join(missing), context, offset=OFFSET) conflict_types = get_conflict_types(subtype, supertype) if conflict_types: - self.note('Following members of {} have ' + self.note('Following member(s) of {} have ' 'conflicts:'.format(subtype), context) for name, got, expected in conflict_types: self.note('{}: expected {}, got {}'.format(name, expected, got), - context) + context, offset=OFFSET) for name, subflags, superflags in get_all_flags(subtype, supertype): if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note('Protocol member {}.{}: expected instance variable,' @@ -2584,9 +2587,9 @@ def warn(self, msg: str, context: Context) -> None: """Produce a warning message.""" self.msg.warn(msg, context) - def note(self, msg: str, context: Context) -> None: + def note(self, msg: str, context: Context, offset: int = 0) -> None: """Produce a note.""" - self.msg.note(msg, context) + self.msg.note(msg, context, offset=offset) def iterable_item_type(self, instance: Instance) -> Type: iterable = map_instance_to_supertype( diff --git a/mypy/errors.py b/mypy/errors.py index 6648784be310..daee68676241 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -253,7 +253,7 @@ def set_import_context(self, ctx: List[Tuple[str, int]]) -> None: def report(self, line: int, column: int, message: str, blocker: bool = False, severity: str = 'error', file: str = None, only_once: bool = False, - origin_line: int = None) -> None: + origin_line: int = None, offset: int = 0) -> None: """Report message at the given line using the current error context. Args: @@ -270,6 +270,8 @@ def report(self, line: int, column: int, message: str, blocker: bool = False, type = None # Omit type context if nested function if file is None: file = self.file + if offset: + message = " " * offset + message info = ErrorInfo(self.import_context(), file, self.current_module(), type, self.function_or_member[-1], line, column, severity, message, blocker, only_once, diff --git a/mypy/messages.py b/mypy/messages.py index 305007592602..dd7847c8eceb 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -155,12 +155,12 @@ def is_errors(self) -> bool: return self.errors.is_errors() def report(self, msg: str, context: Context, severity: str, - file: str = None, origin: Context = None) -> None: + file: str = None, origin: Context = None, offset: int = 0) -> None: """Report an error or note (unless disabled).""" if self.disable_count <= 0: self.errors.report(context.get_line() if context else -1, context.get_column() if context else -1, - msg.strip(), severity=severity, file=file, + msg.strip(), severity=severity, file=file, offset=offset, origin_line=origin.get_line() if origin else None) def fail(self, msg: str, context: Context, file: str = None, @@ -169,9 +169,9 @@ def fail(self, msg: str, context: Context, file: str = None, self.report(msg, context, 'error', file=file, origin=origin) def note(self, msg: str, context: Context, file: str = None, - origin: Context = None) -> None: + origin: Context = None, offset: int = 0) -> None: """Report a note (unless disabled).""" - self.report(msg, context, 'note', file=file, origin=origin) + self.report(msg, context, 'note', file=file, origin=origin, offset=offset) def warn(self, msg: str, context: Context, file: str = None, origin: Context = None) -> None: diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index f2d4f8de86c8..fb5ae39fea4a 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -94,8 +94,8 @@ async def f() -> int: [typing fixtures/typing-full.pyi] [out] main:7: error: Incompatible types in await (actual type Generator[int, None, str], expected type Awaitable[Any]) -main:7: note: 'typing.Generator' missing following 'typing.Awaitable' protocol members: -main:7: note: __await__ +main:7: note: 'typing.Generator' is missing following 'typing.Awaitable' protocol member(s): +main:7: note: __await__ [case testAwaitIteratorError] @@ -108,8 +108,8 @@ async def f() -> int: [typing fixtures/typing-full.pyi] [out] main:6: error: Incompatible types in await (actual type Iterator[Any], expected type Awaitable[Any]) -main:6: note: 'typing.Iterator' missing following 'typing.Awaitable' protocol members: -main:6: note: __await__ +main:6: note: 'typing.Iterator' is missing following 'typing.Awaitable' protocol member(s): +main:6: note: __await__ [case testAwaitArgumentError] @@ -122,8 +122,8 @@ async def f() -> int: [typing fixtures/typing-full.pyi] [out] main:5: error: Incompatible types in await (actual type "int", expected type Awaitable[Any]) -main:5: note: 'builtins.int' missing following 'typing.Awaitable' protocol members: -main:5: note: __await__ +main:5: note: 'builtins.int' is missing following 'typing.Awaitable' protocol member(s): +main:5: note: __await__ [case testAwaitResultError] @@ -170,8 +170,8 @@ async def f() -> None: [typing fixtures/typing-full.pyi] [out] main:4: error: AsyncIterable expected -main:4: note: 'builtins.list' missing following 'typing.AsyncIterable' protocol members: -main:4: note: __aiter__ +main:4: note: 'builtins.list' is missing following 'typing.AsyncIterable' protocol member(s): +main:4: note: __aiter__ main:4: error: List[int] has no attribute "__aiter__" [case testAsyncForTypeComments] @@ -256,20 +256,20 @@ async def wrong_iterable(obj: Iterable[int]): [out] main:18: error: AsyncIterable expected -main:18: note: 'typing.Iterable' missing following 'typing.AsyncIterable' protocol members: -main:18: note: __aiter__ +main:18: note: 'typing.Iterable' is missing following 'typing.AsyncIterable' protocol member(s): +main:18: note: __aiter__ main:18: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:19: error: Iterable expected -main:19: note: '__main__.asyncify' missing following 'typing.Iterable' protocol members: -main:19: note: __iter__ +main:19: note: '__main__.asyncify' is missing following 'typing.Iterable' protocol member(s): +main:19: note: __iter__ main:19: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? main:20: error: AsyncIterable expected -main:20: note: 'typing.Iterable' missing following 'typing.AsyncIterable' protocol members: -main:20: note: __aiter__ +main:20: note: 'typing.Iterable' is missing following 'typing.AsyncIterable' protocol member(s): +main:20: note: __aiter__ main:20: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:21: error: Iterable expected -main:21: note: '__main__.asyncify' missing following 'typing.Iterable' protocol members: -main:21: note: __iter__ +main:21: note: '__main__.asyncify' is missing following 'typing.Iterable' protocol member(s): +main:21: note: __iter__ main:21: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? [builtins fixtures/async_await.pyi] [typing fixtures/typing-full.pyi] @@ -307,8 +307,8 @@ class C: async def __aexit__(self, x, y, z) -> None: pass async def f() -> None: async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any]) \ - # N: 'builtins.int' missing following 'typing.Awaitable' protocol members: \ - # N: __await__ + # N: 'builtins.int' is missing following 'typing.Awaitable' protocol member(s): \ + # N: __await__ pass [builtins fixtures/async_await.pyi] [typing fixtures/typing-full.pyi] @@ -331,8 +331,8 @@ class C: def __aexit__(self, x, y, z) -> int: pass async def f() -> None: async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any]) \ - # N: 'builtins.int' missing following 'typing.Awaitable' protocol members: \ - # N: __await__ + # N: 'builtins.int' is missing following 'typing.Awaitable' protocol member(s): \ + # N: __await__ pass [builtins fixtures/async_await.pyi] [typing fixtures/typing-full.pyi] @@ -570,8 +570,8 @@ def h() -> None: [out] main:9: error: Iterable expected -main:9: note: 'typing.AsyncGenerator' missing following 'typing.Iterable' protocol members: -main:9: note: __iter__ +main:9: note: 'typing.AsyncGenerator' is missing following 'typing.Iterable' protocol member(s): +main:9: note: __iter__ main:9: error: AsyncGenerator[int, None] has no attribute "__iter__"; maybe "__aiter__"? [case testAsyncGeneratorNoYieldFrom] @@ -667,14 +667,14 @@ def plain_host_generator() -> Generator[str, None, None]: async def plain_host_coroutine() -> None: x = 0 x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) \ - # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members: \ - # N: __await__ + # N: 'typing.Generator' is missing following 'typing.Awaitable' protocol member(s): \ + # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) \ - # N: '__main__.It' missing following 'typing.Awaitable' protocol members: \ - # N: __await__ + # N: '__main__.It' is missing following 'typing.Awaitable' protocol member(s): \ + # N: __await__ x = await other_coroutine() @coroutine @@ -692,14 +692,14 @@ def decorated_host_generator() -> Generator[str, None, None]: async def decorated_host_coroutine() -> None: x = 0 x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) \ - # N: 'typing.Generator' missing following 'typing.Awaitable' protocol members: \ - # N: __await__ + # N: 'typing.Generator' is missing following 'typing.Awaitable' protocol member(s): \ + # N: __await__ x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) \ - # N: '__main__.It' missing following 'typing.Awaitable' protocol members: \ - # N: __await__ + # N: '__main__.It' is missing following 'typing.Awaitable' protocol member(s): \ + # N: __await__ x = await other_coroutine() [builtins fixtures/async_await.pyi] diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 919d086fe908..7e959de4e4ac 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1681,11 +1681,11 @@ c = {**b} d = {**a, **b, 'c': 3} # These errors changed because Mapping is a protocol e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str] \ - # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ - # N: __getitem__: expected def (builtins.int*) -> builtins.str*, got def (builtins.str*) -> builtins.int* + # N: Following member(s) of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ + # N: __getitem__: expected def (builtins.int*) -> builtins.str*, got def (builtins.str*) -> builtins.int* f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int] \ - # N: Following members of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ - # N: __getitem__: expected def (builtins.int*) -> builtins.int*, got def (builtins.str*) -> builtins.int* + # N: Following member(s) of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ + # N: __getitem__: expected def (builtins.int*) -> builtins.int*, got def (builtins.str*) -> builtins.int* [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-incomplete-fixture.test b/test-data/unit/check-incomplete-fixture.test index 759e274b0207..2e291da585ba 100644 --- a/test-data/unit/check-incomplete-fixture.test +++ b/test-data/unit/check-incomplete-fixture.test @@ -77,8 +77,8 @@ for y in x: main:2: error: "tuple" expects no type arguments, but 1 given main:3: error: Value of type "tuple" is not indexable main:4: error: Iterable expected -main:4: note: 'builtins.tuple' missing following 'typing.Iterable' protocol members: -main:4: note: __iter__ +main:4: note: 'builtins.tuple' is missing following 'typing.Iterable' protocol member(s): +main:4: note: __iter__ main:4: error: "tuple" has no attribute "__iter__" [case testClassmethodMissingFromStubs] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 9e92664cb7f8..adf3fae9c3ef 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -287,8 +287,8 @@ from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) reveal_type(X._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any] \ - # N: 'builtins.str' missing following 'typing.Iterable' protocol members: \ - # N: __iter__ + # N: 'builtins.str' is missing following 'typing.Iterable' protocol member(s): \ + # N: __iter__ # Note that the above will not fail with real stubs, since 'str', is a subtype of 'Iterable[str]' -- # FIX: not a proper class method diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 84dc7c1a8d1e..a26747b56d63 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -31,20 +31,20 @@ def fun(x: P) -> None: x = C() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ - # N: '__main__.B' missing following '__main__.P' protocol members: \ - # N: meth + # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ + # N: meth fun(C()) fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" \ - # N: '__main__.B' missing following '__main__.P' protocol members: \ - # N: meth + # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ + # N: meth def fun2() -> P: return C() def fun3() -> P: return B() # E: Incompatible return value type (got "B", expected "P") \ - # N: '__main__.B' missing following '__main__.P' protocol members: \ - # N: meth + # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ + # N: meth [out] [case testSimpleProtocolOneAbstractMethod] @@ -73,13 +73,13 @@ def fun(x: P) -> None: x = C() x = D() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ - # N: '__main__.B' missing following '__main__.P' protocol members: \ - # N: meth + # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ + # N: meth fun(C()) fun(D()) fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" \ - # N: '__main__.B' missing following '__main__.P' protocol members: \ - # N: meth + # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ + # N: meth [out] [case testSimpleProtocolOneMethodOverride] @@ -105,13 +105,13 @@ def fun(x: SubP) -> str: z = x x = C() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") \ - # N: '__main__.B' missing following '__main__.SubP' protocol members: \ - # N: meth + # N: '__main__.B' is missing following '__main__.SubP' protocol member(s): \ + # N: meth reveal_type(fun(C())) # E: Revealed type is 'builtins.str' fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" \ - # N: '__main__.B' missing following '__main__.SubP' protocol members: \ - # N: meth + # N: '__main__.B' is missing following '__main__.SubP' protocol member(s): \ + # N: meth [out] [case testSimpleProtocolTwoMethodsMerge] @@ -155,11 +155,11 @@ y: AnotherP x = c x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ - # N: '__main__.B' missing following '__main__.P' protocol members: \ - # N: meth1, meth2 + # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ + # N: meth1, meth2 x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P") \ - # N: '__main__.C1' missing following '__main__.P' protocol members: \ - # N: meth2 + # N: '__main__.C1' is missing following '__main__.P' protocol member(s): \ + # N: meth2 x = c2 x = y y = x @@ -191,8 +191,8 @@ reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' x = C() # OK x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2") \ - # N: '__main__.Cbad' missing following '__main__.P2' protocol members: \ - # N: meth2 + # N: '__main__.Cbad' is missing following '__main__.P2' protocol member(s): \ + # N: meth2 [out] [case testCannotAssignNormalToProtocol] @@ -378,25 +378,25 @@ class B(A): pass x1: Pco[B] y1: Pco[A] x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B]) \ - # N: Following members of __main__.Pco[__main__.A] have conflicts: \ - # N: meth: expected def () -> __main__.B*, got def () -> __main__.A* + # N: Following member(s) of __main__.Pco[__main__.A] have conflicts: \ + # N: meth: expected def () -> __main__.B*, got def () -> __main__.A* y1 = x1 x2: Pcontra[B] y2: Pcontra[A] y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A]) \ - # N: Following members of __main__.Pcontra[__main__.B] have conflicts: \ - # N: meth: expected def (x: __main__.A*), got def (x: __main__.B*) + # N: Following member(s) of __main__.Pcontra[__main__.B] have conflicts: \ + # N: meth: expected def (x: __main__.A*), got def (x: __main__.B*) x2 = y2 x3: Pinv[B] y3: Pinv[A] y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A]) \ - # N: Following members of __main__.Pinv[__main__.B] have conflicts: \ - # N: attr: expected __main__.A*, got __main__.B* + # N: Following member(s) of __main__.Pinv[__main__.B] have conflicts: \ + # N: attr: expected __main__.A*, got __main__.B* x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B]) \ - # N: Following members of __main__.Pinv[__main__.A] have conflicts: \ - # N: attr: expected __main__.B*, got __main__.A* + # N: Following member(s) of __main__.Pinv[__main__.A] have conflicts: \ + # N: attr: expected __main__.B*, got __main__.A* [out] [case testGenericProtocolsInference1] @@ -485,8 +485,8 @@ class C: c: C var: P2[int, int] = c var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str]) \ - # N: Following members of __main__.C have conflicts: \ - # N: attr2: expected Tuple[builtins.int*, builtins.str*], got Tuple[builtins.int, builtins.int] + # N: Following member(s) of __main__.C have conflicts: \ + # N: attr2: expected Tuple[builtins.int*, builtins.str*], got Tuple[builtins.int, builtins.int] class D(Generic[T]): attr1: T @@ -496,8 +496,8 @@ class E(D[T]): def f(x: T) -> T: z: P2[T, T] = E[T]() y: P2[T, T] = D[T]() # E: Incompatible types in assignment (expression has type D[T], variable has type P2[T, T]) \ - # N: '__main__.D' missing following '__main__.P2' protocol members: \ - # N: attr2 + # N: '__main__.D' is missing following '__main__.P2' protocol member(s): \ + # N: attr2 return x [builtins fixtures/isinstancelist.pyi] [out] @@ -528,9 +528,9 @@ class D(A, B): pass x: P = D() # Same as P[Any, Any] var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[int, P[Any, Any]], Union[P[Any, Any], str]]) \ - # N: Following members of __main__.C have conflicts: \ - # N: attr1: expected Union[builtins.int, __main__.P[Any, Any]], got builtins.int \ - # N: attr2: expected Union[__main__.P[Any, Any], builtins.str], got builtins.str + # N: Following member(s) of __main__.C have conflicts: \ + # N: attr1: expected Union[builtins.int, __main__.P[Any, Any]], got builtins.int \ + # N: attr2: expected Union[__main__.P[Any, Any], builtins.str], got builtins.str [out] [case testGenericSubProtocolsExtensionCovariant] @@ -552,9 +552,9 @@ class C: var: P[Union[int, P], Union[P, str]] = C() # OK for covariant var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) \ - # N: Following members of __main__.C have conflicts: \ - # N: attr1: expected def () -> Union[builtins.str, __main__.P[Any, Any]], got def () -> builtins.int \ - # N: attr2: expected def () -> Union[__main__.P[Any, Any], builtins.int], got def () -> builtins.str + # N: Following member(s) of __main__.C have conflicts: \ + # N: attr1: expected def () -> Union[builtins.str, __main__.P[Any, Any]], got def () -> builtins.int \ + # N: attr2: expected def () -> Union[__main__.P[Any, Any], builtins.int], got def () -> builtins.str [out] [case testSelfTypesWithProtocolsBehaveAsWithNominal] @@ -584,11 +584,11 @@ s: Shape f(NonProtoShape()) f(Circle()) s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") \ - # N: Following members of __main__.Triangle have conflicts: \ - # N: combine: expected def (other: __main__.Triangle*) -> __main__.Triangle*, got def (other: __main__.Shape) -> __main__.Shape + # N: Following member(s) of __main__.Triangle have conflicts: \ + # N: combine: expected def (other: __main__.Triangle*) -> __main__.Triangle*, got def (other: __main__.Shape) -> __main__.Shape s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") \ - # N: Following members of __main__.Bad have conflicts: \ - # N: combine: expected def (other: __main__.Bad*) -> __main__.Bad*, got def (other: builtins.int) -> builtins.str + # N: Following member(s) of __main__.Bad have conflicts: \ + # N: combine: expected def (other: __main__.Bad*) -> __main__.Bad*, got def (other: builtins.int) -> builtins.str n2: NonProtoShape = s # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") [out] @@ -652,8 +652,8 @@ class D(Generic[T]): t: Traversable t = D[int]() # OK t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable") \ - # N: '__main__.C' missing following '__main__.Traversable' protocol members: \ - # N: leaves + # N: '__main__.C' is missing following '__main__.Traversable' protocol member(s): \ + # N: leaves [builtins fixtures/list.pyi] [out] @@ -698,11 +698,11 @@ class B: t: P1 t = A() # OK t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1") \ - # N: '__main__.B' missing following '__main__.P1' protocol members: \ - # N: attr1 + # N: '__main__.B' is missing following '__main__.P1' protocol member(s): \ + # N: attr1 t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1") \ - # N: '__main__.C' missing following '__main__.P1' protocol members: \ - # N: attr1 + # N: '__main__.C' is missing following '__main__.P1' protocol member(s): \ + # N: attr1 [builtins fixtures/list.pyi] [out] @@ -898,8 +898,8 @@ class D: x: P = C() x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ - # N: Following members of __main__.D have conflicts: \ - # N: f: expected Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str), got def (x: builtins.int) + # N: Following member(s) of __main__.D have conflicts: \ + # N: f: expected Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str), got def (x: builtins.int) [out] -- Join and meet with protocol types @@ -1360,12 +1360,12 @@ def fun3(x: P[T, T]) -> T: fun(z) fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int] \ - # N: Following members of __main__.N have conflicts: \ - # N: y: expected builtins.int*, got builtins.str + # N: Following member(s) of __main__.N have conflicts: \ + # N: y: expected builtins.int*, got builtins.str fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str] \ - # N: '__main__.N2' missing following '__main__.P' protocol members: \ - # N: y + # N: '__main__.N2' is missing following '__main__.P' protocol member(s): \ + # N: y reveal_type(fun3(z)) # E: Revealed type is 'builtins.object*' @@ -1438,8 +1438,8 @@ bar(1) [builtins fixtures/isinstancelist.pyi] [out] main:11: error: Argument 1 to "bar" has incompatible type "int"; expected "Sized" -main:11: note: 'builtins.int' missing following 'typing.Sized' protocol members: -main:11: note: __len__ +main:11: note: 'builtins.int' is missing following 'typing.Sized' protocol member(s): +main:11: note: __len__ [case testBasicSupportsIntProtocol] from typing import SupportsInt @@ -1457,8 +1457,8 @@ foo('no way') [builtins fixtures/isinstancelist.pyi] [out] main:11: error: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" -main:11: note: 'builtins.str' missing following 'typing.SupportsInt' protocol members: -main:11: note: __int__ +main:11: note: 'builtins.str' is missing following 'typing.SupportsInt' protocol member(s): +main:11: note: __int__ -- Additional test and corner cases for protocols -- ---------------------------------------------- @@ -1491,8 +1491,8 @@ f2(C2()) f3(C3()) f2(C3()) # E: Argument 1 to "f2" has incompatible type "C3"; expected P2[str] \ - # N: '__main__.C3' missing following '__main__.P2' protocol members: \ - # N: attr2 + # N: '__main__.C3' is missing following '__main__.P2' protocol member(s): \ + # N: attr2 a: Any f1(a) f2(a) @@ -1513,28 +1513,28 @@ class C: def attr2(self) -> int: pass x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ - # N: '__main__.C' missing following '__main__.P' protocol members: \ - # N: attr3 \ - # N: Following members of __main__.C have conflicts: \ - # N: attr1: expected builtins.int, got builtins.str \ - # N: attr2: expected builtins.str, got builtins.int \ + # N: '__main__.C' is missing following '__main__.P' protocol member(s): \ + # N: attr3 \ + # N: Following member(s) of __main__.C have conflicts: \ + # N: attr1: expected builtins.int, got builtins.str \ + # N: attr2: expected builtins.str, got builtins.int \ # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute def f(x: P) -> P: return C() # E: Incompatible return value type (got "C", expected "P") \ - # N: '__main__.C' missing following '__main__.P' protocol members: \ - # N: attr3 \ - # N: Following members of __main__.C have conflicts: \ - # N: attr1: expected builtins.int, got builtins.str \ - # N: attr2: expected builtins.str, got builtins.int \ + # N: '__main__.C' is missing following '__main__.P' protocol member(s): \ + # N: attr3 \ + # N: Following member(s) of __main__.C have conflicts: \ + # N: attr1: expected builtins.int, got builtins.str \ + # N: attr2: expected builtins.str, got builtins.int \ # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ - # N: '__main__.C' missing following '__main__.P' protocol members: \ - # N: attr3 \ - # N: Following members of __main__.C have conflicts: \ - # N: attr1: expected builtins.int, got builtins.str \ - # N: attr2: expected builtins.str, got builtins.int \ + # N: '__main__.C' is missing following '__main__.P' protocol member(s): \ + # N: attr3 \ + # N: Following member(s) of __main__.C have conflicts: \ + # N: attr1: expected builtins.int, got builtins.str \ + # N: attr2: expected builtins.str, got builtins.int \ # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute [builtins fixtures/list.pyi] [out] @@ -1598,8 +1598,8 @@ def fun_p(x: PP) -> None: fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P" \ # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" \ - # N: Following members of __main__.D have conflicts: \ - # N: attr: expected builtins.int, got builtins.str + # N: Following member(s) of __main__.D have conflicts: \ + # N: attr: expected builtins.int, got builtins.str fun_p(C()) # OK [builtins fixtures/list.pyi] [out] @@ -1618,8 +1618,8 @@ class D: x: P x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ - # N: Following members of __main__.D have conflicts: \ - # N: x: expected builtins.int, got builtins.str + # N: Following member(s) of __main__.D have conflicts: \ + # N: x: expected builtins.int, got builtins.str x = C() # OK [builtins fixtures/list.pyi] [out] From 491b31f94b4c45d276362f5a9d06bc5d2abe7c86 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 11 Jun 2017 11:30:31 +0200 Subject: [PATCH 066/117] Add more incremental tests --- test-data/unit/check-incremental.test | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 3d66d40af424..c332b8ae7e05 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1499,6 +1499,64 @@ class P(Protocol[T]): def meth(self) -> T: pass +[case testIncrementalSwitchFromNominalToStructural] +import a +[file a.py] +from b import B, fun +class C(B): + def x(self) -> int: pass + def y(self) -> int: pass +fun(C()) + +[file b.py] +class B: + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[file b.py.2] +from typing import Protocol +class B(Protocol): + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[file a.py.3] +from b import fun +class C: + def x(self) -> int: pass + def y(self) -> int: pass +fun(C()) +[out1] +[out2] +[out3] + +[case testIncrementalSwitchFromStructuralToNominal] +import a +[file a.py] +from b import fun +class C: + def x(self) -> int: pass + def y(self) -> int: pass +fun(C()) + +[file b.py] +from typing import Protocol +class B(Protocol): + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[file b.py.2] +class B: + def x(self) -> float: pass +def fun(arg: B) -> None: + arg.x() + +[out1] +[out2] +tmp/a.py:5: error: Argument 1 to "fun" has incompatible type "C"; expected "B" + [case testIncrementalWorksWithNamedTuple] import foo From 98f01801fc7b90cfa9f140e83b5c47b95514bc4c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 11 Jun 2017 12:26:24 +0200 Subject: [PATCH 067/117] Add support for __setattr__ (and few docstrings) --- mypy/checker.py | 7 +++++++ mypy/subtypes.py | 8 +++++++- test-data/unit/check-protocols.test | 4 ++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4a49d3236b1b..e7643161ff1d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1112,6 +1112,13 @@ def visit_class_def(self, defn: ClassDef) -> None: self.check_multiple_inheritance(typ) def check_protocol_variance(self, defn: ClassDef) -> None: + """Check that protocol definition is compatible with declared + variances of type variables. + + Note that we also prohibit declaring protocol classes as invariant + if they are actually covariant/contravariant, since this may break + transitivity of subtyping, see PEP 544. + """ info = defn.info object_type = Instance(info.mro[-1], []) tvars = info.defn.type_vars diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 41d2f645ebf4..04773c23343e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -442,17 +442,20 @@ def get_member_flags(name: str, info: TypeInfo) -> Set[int]: instance or class variable, and whether it is class or static method. """ method = info.get_method(name) + setattr_meth = info.get_method('__setattr__') if method: # this could be settable property if method.is_property: assert isinstance(method, OverloadedFuncDef) dec = method.items[0] assert isinstance(dec, Decorator) - if dec.var.is_settable_property: + if dec.var.is_settable_property or setattr_meth: return {IS_SETTABLE} return set() node = info.get(name) if not node: + if setattr_meth: + return {IS_SETTABLE} return set() v = node.node if isinstance(v, Decorator): @@ -514,6 +517,9 @@ def get_missing_members(left: Instance, right: Instance) -> List[str]: def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, Type]]: + """Find members that are defined in 'left' but have incompatible types. + Return them as a list of ('member', 'got', 'expected'). + """ assert right.type.is_protocol conflicts = [] # type: List[Tuple[str, Type, Type]] for member in right.type.protocol_members: diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index a26747b56d63..9addcd4d443d 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1585,6 +1585,9 @@ class PP(Protocol): class C: def __getattr__(self, attr: str) -> int: pass +class C2(C): + def __setattr__(self, attr: str, val: int) -> None: + pass class D: def __getattr__(self, attr: str) -> str: @@ -1597,6 +1600,7 @@ def fun_p(x: PP) -> None: fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P" \ # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute +fun(C2()) fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" \ # N: Following member(s) of __main__.D have conflicts: \ # N: attr: expected builtins.int, got builtins.str From 0f55718881860293e217ba44193baa264d844c61 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 11 Jun 2017 13:19:01 +0200 Subject: [PATCH 068/117] Simplify hashes --- mypy/types.py | 24 ++++++++++++------------ typeshed | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index a1e688af35bf..10e0a4dcbc13 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -217,7 +217,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_unbound_type(self) def __hash__(self) -> int: - return hash((UnboundType, self.name, self.optional, tuple(self.args))) + return hash((self.name, self.optional, tuple(self.args))) def __eq__(self, other: object) -> bool: if not isinstance(other, UnboundType): @@ -281,7 +281,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_type_list(self) def __hash__(self) -> int: - return hash((TypeList, tuple(self.items))) + return hash(tuple(self.items)) def __eq__(self, other: object) -> bool: if not isinstance(other, TypeList): @@ -443,7 +443,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: type_ref = None # type: str def __hash__(self) -> int: - return hash((Instance, self.type, tuple(self.args))) + return hash((self.type, tuple(self.args))) def __eq__(self, other: object) -> bool: if not isinstance(other, Instance): @@ -513,7 +513,7 @@ def erase_to_union_or_bound(self) -> Type: return self.upper_bound def __hash__(self) -> int: - return hash((TypeVarType, self.id)) + return hash(self.id) def __eq__(self, other: object) -> bool: if not isinstance(other, TypeVarType): @@ -802,9 +802,9 @@ def type_var_ids(self) -> List[TypeVarId]: return a def __hash__(self) -> int: - return hash((CallableType, self.ret_type, self.is_type_obj(), - self.is_ellipsis_args, self.name) + - tuple(self.arg_types) + tuple(self.arg_names) + tuple(self.arg_kinds)) + return hash((self.ret_type, self.is_type_obj(), + self.is_ellipsis_args, self.name, + tuple(self.arg_types), tuple(self.arg_names), tuple(self.arg_kinds))) def __eq__(self, other: object) -> bool: if isinstance(other, CallableType): @@ -901,7 +901,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_overloaded(self) def __hash__(self) -> int: - return hash((Overloaded, tuple(self.items()))) + return hash(tuple(self.items())) def __eq__(self, other: object) -> bool: if not isinstance(other, Overloaded): @@ -950,7 +950,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_tuple_type(self) def __hash__(self) -> int: - return hash((TupleType, tuple(self.items), self.fallback)) + return hash((tuple(self.items), self.fallback)) def __eq__(self, other: object) -> bool: if not isinstance(other, TupleType): @@ -1008,7 +1008,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_typeddict_type(self) def __hash__(self) -> int: - return hash((TypedDictType, tuple(self.items.items()))) + return hash(tuple(self.items.items())) def __eq__(self, other: object) -> bool: if isinstance(other, TypedDictType): @@ -1092,7 +1092,7 @@ def __init__(self, type: Type, line: int = -1, column: int = -1) -> None: super().__init__(line, column) def __hash__(self) -> int: - return hash((StarType, self.type)) + return hash(self.type) def __eq__(self, other: object) -> bool: if not isinstance(other, StarType): @@ -1303,7 +1303,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_type_type(self) def __hash__(self) -> int: - return hash((TypeType, self.item)) + return hash(self.item) def __eq__(self, other: object) -> bool: if not isinstance(other, TypeType): diff --git a/typeshed b/typeshed index be80c368161e..766f9d3b1097 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit be80c368161eeace47d93eb9908ddda1aacf3b86 +Subproject commit 766f9d3b1097890da630afc1048ee53d6206adeb From 36f3d6d3b67e21435b47f2a7a9555322930372ac Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 11 Jun 2017 17:00:50 +0200 Subject: [PATCH 069/117] Restore typeshed commit --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 766f9d3b1097..be80c368161e 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 766f9d3b1097890da630afc1048ee53d6206adeb +Subproject commit be80c368161eeace47d93eb9908ddda1aacf3b86 From 05b70ab64550675397778b1197aceecc28ecc5c7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 11 Jun 2017 17:41:26 +0200 Subject: [PATCH 070/117] Add explanation of covariant mutable overriding to common issues docs --- docs/source/common_issues.rst | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index 0c8b500d8f06..fdb5c002e0a0 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -226,6 +226,46 @@ Possible strategies in such situations are: return x[0] f_good(new_lst) # OK +Covariant subtyping of mutable protocol members is rejected +----------------------------------------------------------- + +Mypy rejects this because this is potentially unsafe. +Consider this example: + +.. code-block:: python + + from typing import Protocol + class P(Protocol): + x: float + + def fun(arg: P) -> None: + arg.x = 3.14 + + class C: + x: int + c = C() + fun(c) # This is not safe + c.x << 5 # Since this will fail! + +To work around this problem consider whether "mutating" is actually part +of a protocol. If not, then one can use a ``@property`` in +the protocol definition: + +.. code-block:: python + + from typing import Protocol + class P(Protocol): + @property + def x(self) -> float: + pass + + def fun(arg: P) -> None: + ... + + class C: + x: int + fun(C()) # OK + Declaring a supertype as variable type -------------------------------------- From 228f62167cd576d64beeabe06d58bbec8f546fe5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 28 Jun 2017 10:04:36 +0200 Subject: [PATCH 071/117] Fix brocken merge (now Mapping needs to be Iterable in typing-full fixture because of new tests) --- test-data/unit/fixtures/typing-full.pyi | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 60afd21fbfe7..acd10c0dea53 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -118,16 +118,17 @@ class Sequence(Iterable[T_co], Protocol): def __getitem__(self, n: Any) -> T_co: pass @runtime -class Mapping(Protocol[T_contra, T_co]): - def __getitem__(self, key: T_contra) -> T_co: pass +class Mapping(Iterable[T], Protocol[T, T_co]): + def __getitem__(self, key: T) -> T_co: pass @overload - def get(self, k: T_contra) -> Optional[T_co]: pass + def get(self, k: T) -> Optional[T_co]: pass @overload - def get(self, k: T_contra, default: Union[T_co, V]) -> Union[T_co, V]: pass + def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass + @runtime -class MutableMapping(Mapping[T_contra, U], Protocol): - def __setitem__(self, k: T_contra, v: U) -> None: pass +class MutableMapping(Mapping[T, U], Protocol): + def __setitem__(self, k: T, v: U) -> None: pass class SupportsInt(Protocol): def __int__(self) -> int: pass @@ -136,7 +137,7 @@ def runtime(cls: T) -> T: return cls class ContextManager(Generic[T]): - def __enter__(self) -> T: ... - def __exit__(self, exc_type, exc_value, traceback): ... + def __enter__(self) -> T: pass + def __exit__(self, exc_type, exc_value, traceback): pass TYPE_CHECKING = 1 From eb06c55beb52b81fdf718ff98f61c6e107a93843 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 4 Jul 2017 21:52:12 +0200 Subject: [PATCH 072/117] Fix strict-optional --- mypy/constraints.py | 2 ++ mypy/join.py | 2 +- mypy/subtypes.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 46f09695449a..2161220b3a2b 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -347,6 +347,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: for member in template.type.protocol_members: inst = mypy.subtypes.find_member(member, instance, original_actual) temp = mypy.subtypes.find_member(member, template, original_actual) + assert inst is not None and temp is not None res.extend(infer_constraints(temp, inst, self.direction)) if (mypy.subtypes.IS_SETTABLE in mypy.subtypes.get_member_flags(member, template.type)): @@ -362,6 +363,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: for member in instance.type.protocol_members: inst = mypy.subtypes.find_member(member, instance, template) temp = mypy.subtypes.find_member(member, template, template) + assert inst is not None and temp is not None res.extend(infer_constraints(temp, inst, self.direction)) if (mypy.subtypes.IS_SETTABLE in mypy.subtypes.get_member_flags(member, instance.type)): diff --git a/mypy/join.py b/mypy/join.py index cdc285f32ef4..811e2abf1cf4 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -142,7 +142,7 @@ def visit_type_var(self, t: TypeVarType) -> Type: def visit_instance(self, t: Instance) -> Type: if isinstance(self.s, Instance): nominal = join_instances(t, self.s) - structural = None # type: Instance + structural = None # type: Optional[Instance] if t.type.is_protocol and is_protocol_implementation(self.s, t): structural = t elif self.s.type.is_protocol and is_protocol_implementation(t, self.s): diff --git a/mypy/subtypes.py b/mypy/subtypes.py index d4fcc9a0d300..2cc0371b5c54 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -358,6 +358,7 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool if member in ('__init__', '__new__'): continue supertype = find_member(member, right, left) + assert supertype is not None subtype = find_member(member, left, left) # Useful for debugging: # print(member, 'of', left, 'has type', subtype) @@ -537,6 +538,7 @@ def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, if member in ('__init__', '__new__'): continue supertype = find_member(member, right, left) + assert supertype is not None subtype = find_member(member, left, left) if not subtype: continue From 59aeed24fa04ec5e2cabc1f2af12a86d696fd755 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 4 Jul 2017 22:27:30 +0200 Subject: [PATCH 073/117] Fix other merge problems --- mypy/checkexpr.py | 4 ++-- test-data/unit/check-isinstance.test | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 417dc1a9c2bc..764674c4b6cd 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -245,8 +245,8 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: callee_type = self.apply_method_signature_hook( e, callee_type, object_type, signature_hook) ret_type = self.check_call_expr_with_callee_type(callee_type, e, fullname, object_type) - if isinstance(e.callee, RefExpr) and e.callee.fullname in ('builtins.isinstance', - 'builtins.issubclass'): + if (isinstance(e.callee, RefExpr) and len(e.args) == 2 and + e.callee.fullname in ('builtins.isinstance', 'builtins.issubclass')): for expr in mypy.checker.flatten(e.args[1]): tp = self.chk.type_map[expr] if (isinstance(tp, CallableType) and tp.is_type_obj() and diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 795bcb1d3447..85dc8ea4d2a6 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1747,7 +1747,7 @@ if isinstance(x): # E: Too few arguments for "isinstance" [case testIsInstanceTooManyArgs] isinstance(1, 1, 1) # E: Too many arguments for "isinstance" \ - # E: Argument 2 to "isinstance" has incompatible type "int"; expected "Union[type, tuple]" + # E: Argument 2 to "isinstance" has incompatible type "int"; expected "Union[type, Tuple[Any, ...]]" x: object if isinstance(x, str, 1): # E: Too many arguments for "isinstance" reveal_type(x) # E: Revealed type is 'builtins.object' From ee18ddeb780ba9911a00e4c6cc2ed26cb9da366d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jul 2017 13:45:25 +0200 Subject: [PATCH 074/117] Fix fixtures/dict.pyi fixture --- test-data/unit/fixtures/dict.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index 78712885243e..49cf02976d21 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -24,6 +24,7 @@ class dict(Generic[KT, VT]): def get(self, k: KT) -> Optional[VT]: pass @overload def get(self, k: KT, default: Union[KT, T]) -> Union[VT, T]: pass + def __len__(self) -> int: ... class int: # for convenience def __add__(self, x: int) -> int: pass From 34d1cd1788e5cf085fce2719ac00caff0872f3fb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jul 2017 14:19:33 +0200 Subject: [PATCH 075/117] Fix TypedDict joins with protocols via fallback --- mypy/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/join.py b/mypy/join.py index 811e2abf1cf4..d16cd58ac286 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -250,7 +250,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: required_keys = set(items.keys()) & t.required_keys & self.s.required_keys return TypedDictType(items, required_keys, fallback) elif isinstance(self.s, Instance): - return join_instances(self.s, t.fallback) + return join_types(self.s, t.fallback) else: return self.default(self.s) From fbbd1698da98906abf5342f31cfdbdcab19fc4fe Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jul 2017 17:07:49 +0200 Subject: [PATCH 076/117] Use typing_extensions instead of typing in the docs --- docs/source/class_basics.rst | 26 ++++++++++++++++++-------- docs/source/common_issues.rst | 6 ++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 2ed4bd9501c8..471d9e76bdfa 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -195,12 +195,13 @@ specification of protocols and structural subtyping in Python. User defined protocols ********************** -To define a protocol class, one must inherit the special ``typing.Protocol`` -class: +To define a protocol class, one must inherit the special +``typing_extensions.Protocol`` class: .. code-block:: python - from typing import Protocol, Iterable + from typing import Iterable + from typing_extensions import Protocol class SupportsClose(Protocol): def close(self) -> None: @@ -217,6 +218,13 @@ class: close_all([Resource(), open('some/file')]) # This passes type check +.. note:: + + The ``Protocol`` base class is currently provided in ``typing_extensions`` + package. Stub files are however allowed to use + ``from typing import Protocol``. When structural subtyping is mature and + PEP 544 is accepted, ``Protocol`` will be included in the ``typing`` module. + Defining subprotocols ********************* @@ -244,8 +252,8 @@ and merged using multiple inheritance. For example: Note that inheriting from existing protocols does not automatically turn a subclass into a protocol, it just creates a usual (non-protocol) ABC that -implements given protocols. The ``typing.Protocol`` base must always be -explicitly present: +implements given protocols. The ``typing_extensions.Protocol`` base must always +be explicitly present: .. code-block:: python @@ -269,7 +277,8 @@ the class definition. Examples: .. code-block:: python - from typing import Protocol, TypeVar + from typing import TypeVar + from typing_extensions import Protocol T = TypeVar('T') @@ -317,7 +326,8 @@ such as trees and linked lists: .. code-block:: python - from typing import Protocol, TypeVar, Optional + from typing import TypeVar, Optional + from typing_extensions import Protocol class TreeLike(Protocol): value: int @@ -378,7 +388,7 @@ structural checks: .. code-block:: python - from typing import Protocol, runtime + from typing_extensions import Protocol, runtime @runtime class Portable(Protocol): diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index fdb5c002e0a0..857b2fe3d3a1 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -234,7 +234,8 @@ Consider this example: .. code-block:: python - from typing import Protocol + from typing_extensions import Protocol + class P(Protocol): x: float @@ -253,7 +254,8 @@ the protocol definition: .. code-block:: python - from typing import Protocol + from typing_extensions import Protocol + class P(Protocol): @property def x(self) -> float: From 3acd19eabb64b53ff24d193d6a9b1114877a0a49 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jul 2017 17:29:28 +0200 Subject: [PATCH 077/117] Use pre-PEP-526 syntax in docs where possible --- docs/source/class_basics.rst | 27 ++++++++++++++++++--------- docs/source/common_issues.rst | 4 ++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 471d9e76bdfa..3de3fada5b87 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -223,7 +223,8 @@ To define a protocol class, one must inherit the special The ``Protocol`` base class is currently provided in ``typing_extensions`` package. Stub files are however allowed to use ``from typing import Protocol``. When structural subtyping is mature and - PEP 544 is accepted, ``Protocol`` will be included in the ``typing`` module. + `PEP 544 `_ is accepted, + ``Protocol`` will be included in the ``typing`` module. Defining subprotocols ********************* @@ -244,7 +245,7 @@ and merged using multiple inheritance. For example: def __len__(self) -> int: ... - resource: SizedLabeledResource + resource = None # type: SizedLabeledResource # some code @@ -261,11 +262,19 @@ be explicitly present: new_attr: int class Concrete: - new_attr: int + new_attr = None # type: int def __len__(self) -> int: ... + # Below is an error, since nominal subtyping is used by default + x = Concrete() # type: NewProtocol # Error! - x: NewProtocol = Concrete() # Error, nominal subtyping is used by default +.. note:: + + The `PEP 526 `_ variable + annotations can be used to declare protocol attributes. However, protocols + are also supported on Python 2.7 and Python 3.3+ with the help of type + comments and properties, see + `backwards compatibility in PEP 544 `_. Generic protocols ***************** @@ -298,8 +307,8 @@ the class definition. Examples: do_stuff(StringWrapper('one'), BytesWrapper(b'other')) # OK - x: Box[float] - y: Box[int] + x = None # type: Box[float] + y = None # type: Box[int] x = y # Error, since the protocol 'Box' is invariant. class AnotherBox(Protocol[T]): # Error, covariant type variable expected @@ -311,8 +320,8 @@ the class definition. Examples: def content(self) -> T_co: ... - ax: AnotherBox[float] - ay: AnotherBox[int] + ax = None # type: AnotherBox[float] + ay = None # type: AnotherBox[int] ax = ay # OK for covariant protocols See :ref:`generic-classes` for more details on generic classes and variance. @@ -339,7 +348,7 @@ such as trees and linked lists: self.value = value self.left = self.right = None - root: TreeLike = SimpleTree(0) # OK + root = SimpleTree(0) # type: TreeLike # OK T = TypeVar('T') class Linked(Protocol[T]): diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index 857b2fe3d3a1..e31d9a7bc4e6 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -243,7 +243,7 @@ Consider this example: arg.x = 3.14 class C: - x: int + x = 42 c = C() fun(c) # This is not safe c.x << 5 # Since this will fail! @@ -265,7 +265,7 @@ the protocol definition: ... class C: - x: int + x = 42 fun(C()) # OK Declaring a supertype as variable type From 22ad7716d9fcaab66e4d9c3d6275fc62d056b370 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jul 2017 18:45:05 +0200 Subject: [PATCH 078/117] Move generic protocols after nominal generics docs --- docs/source/class_basics.rst | 81 ++++-------------------------------- docs/source/generics.rst | 67 +++++++++++++++++++++++++++++ mypy/subtypes.py | 4 +- typeshed | 2 +- 4 files changed, 80 insertions(+), 74 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 3de3fada5b87..3325c6c82bcb 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -276,62 +276,11 @@ be explicitly present: comments and properties, see `backwards compatibility in PEP 544 `_. -Generic protocols -***************** - -Generic protocols are also supported, generic protocols mostly follow -the normal rules for generic classes, the main difference is that mypy checks -that declared variance of type variables is compatible with -the class definition. Examples: - -.. code-block:: python - - from typing import TypeVar - from typing_extensions import Protocol - - T = TypeVar('T') - - class Box(Protocol[T]): - content: T - - def do_stuff(one: Box[str], other: Box[bytes]) -> None: - ... - - class StringWrapper: - def __init__(self, content: str) -> None: - self.content = content - - class BytesWrapper: - def __init__(self, content: bytes) -> None: - self.content = content - - do_stuff(StringWrapper('one'), BytesWrapper(b'other')) # OK - - x = None # type: Box[float] - y = None # type: Box[int] - x = y # Error, since the protocol 'Box' is invariant. - - class AnotherBox(Protocol[T]): # Error, covariant type variable expected - def content(self) -> T: - ... - - T_co = TypeVar('T_co', covariant=True) - class AnotherBox(Protocol[T_co]): # OK - def content(self) -> T_co: - ... - - ax = None # type: AnotherBox[float] - ay = None # type: AnotherBox[int] - ax = ay # OK for covariant protocols - -See :ref:`generic-classes` for more details on generic classes and variance. - Recursive protocols ******************* -Protocols (both generic and non-generic) can be recursive and mutually -recursive. This could be useful for declaring abstract recursive collections -such as trees and linked lists: +Protocols can be recursive and mutually recursive. This could be useful for +declaring abstract recursive collections such as trees and linked lists: .. code-block:: python @@ -339,31 +288,19 @@ such as trees and linked lists: from typing_extensions import Protocol class TreeLike(Protocol): - value: int - left: Optional['TreeLike'] - right: Optional['TreeLike'] + value: int + @property + def left(self) -> Optional['TreeLike']: ... + @property + def right(self) -> Optional['TreeLike']: ... class SimpleTree: def __init__(self, value: int) -> None: - self.value = value - self.left = self.right = None + self.value = value + self.left = self.right = None root = SimpleTree(0) # type: TreeLike # OK - T = TypeVar('T') - class Linked(Protocol[T]): - val: T - def next(self) -> 'Linked[T]': ... - - class L: - val: int - def next(self) -> 'L': ... - - def last(seq: Linked[T]) -> T: - ... - - result = last(L()) # The inferred type of 'result' is 'int' - Predefined protocols in ``typing`` module ***************************************** diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 17209cd7346b..8f8f5355b93e 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -491,6 +491,73 @@ restrict the valid values for the type parameter in the same way. A type variable may not have both a value restriction (see :ref:`type-variable-value-restriction`) and an upper bound. +Generic protocols +***************** + +Generic protocols (see :ref:`protocol-types`) are also supported, generic +protocols mostly follow the normal rules for generic classes, the main +difference is that mypy checks that declared variance of type variables is +compatible with the class definition. Examples: + +.. code-block:: python + + from typing import TypeVar + from typing_extensions import Protocol + + T = TypeVar('T') + + class Box(Protocol[T]): + content: T + + def do_stuff(one: Box[str], other: Box[bytes]) -> None: + ... + + class StringWrapper: + def __init__(self, content: str) -> None: + self.content = content + + class BytesWrapper: + def __init__(self, content: bytes) -> None: + self.content = content + + do_stuff(StringWrapper('one'), BytesWrapper(b'other')) # OK + + x = None # type: Box[float] + y = None # type: Box[int] + x = y # Error, since the protocol 'Box' is invariant. + + class AnotherBox(Protocol[T]): # Error, covariant type variable expected + def content(self) -> T: + ... + + T_co = TypeVar('T_co', covariant=True) + class AnotherBox(Protocol[T_co]): # OK + def content(self) -> T_co: + ... + + ax = None # type: AnotherBox[float] + ay = None # type: AnotherBox[int] + ax = ay # OK for covariant protocols + +See :ref:`variance-of-generics` above for more details on variance. +Generic protocols can be recursive, for example: + +.. code-block:: python + + T = TypeVar('T') + class Linked(Protocol[T]): + val: T + def next(self) -> 'Linked[T]': ... + + class L: + val: int + def next(self) -> 'L': ... + + def last(seq: Linked[T]) -> T: + ... + + result = last(L()) # The inferred type of 'result' is 'int' + .. _declaring-decorators: Declaring decorators diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 2cc0371b5c54..9d46a8f3b980 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -370,7 +370,9 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) else: is_compat = is_proper_subtype(subtype, supertype) - if not is_compat or isinstance(subtype, NoneTyp): + if not is_compat: + return False + if isinstance(subtype, NoneTyp) and member.startswith('__') and member.endswith('__'): # We want __hash__ = None idiom to work even without --strict-optional return False subflags = get_member_flags(member, left.type) diff --git a/typeshed b/typeshed index 5a95e19322cb..2e36717aa899 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 5a95e19322cbd68cadab1c50da0ef01d201cfd99 +Subproject commit 2e36717aa89989e0c5050035f4f572dfbea389f9 From 4f2391e781d351da3a7d0022145633ba0dea7cd4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jul 2017 18:53:20 +0200 Subject: [PATCH 079/117] Change recommendations about using protocols --- docs/source/faq.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index f6393d881f37..b131e3f7e2a2 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -127,12 +127,11 @@ leaving structural subtyping opt-in. Here are some reasons why: C#. Only few languages use structural subtyping. However, structural subtyping can also be useful. For example, a "public API" -will be more flexible and convenient for users if it is typed with protocols. -Also, using protocol types removes the necessity to explicitly declare -implementations of ABCs. Finally, protocol types may feel more natural for -Python programmers. As a rule of thumb, one could prefer protocols for -function argument types and normal classes for return types. For more details -about protocol types and structural subtyping see :ref:`protocol-types` and +may be more flexible if it is typed with protocols. Also, using protocol types +removes the necessity to explicitly declare implementations of ABCs. +As a rule of thumb, we recommend using nominal classes where possible, and +protocols where necessary. For more details about protocol types and structural +subtyping see :ref:`protocol-types` and `PEP 544 `_. I like Python and I have no need for static typing From 359a43bbc88b0790a997e555bba18b575bc3493f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jul 2017 18:58:51 +0200 Subject: [PATCH 080/117] Restore correct typeshed commit --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 2e36717aa899..5a95e19322cb 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 2e36717aa89989e0c5050035f4f572dfbea389f9 +Subproject commit 5a95e19322cbd68cadab1c50da0ef01d201cfd99 From eacff5f216a4bef3db9390183194ebc18deeebc5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 09:27:19 +0200 Subject: [PATCH 081/117] Switch from mypy_extensions to typing_extensions in semanal and in tests --- mypy/semanal.py | 8 ++++---- test-data/unit/check-protocols.test | 4 ++-- test-data/unit/lib-stub/mypy_extensions.pyi | 2 -- test-data/unit/lib-stub/typing_extensions.pyi | 6 ++++++ 4 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 test-data/unit/lib-stub/typing_extensions.pyi diff --git a/mypy/semanal.py b/mypy/semanal.py index 47412283448c..43df8556f1f6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -747,7 +747,7 @@ def leave_class(self) -> None: def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> bool: decorator.accept(self) return (isinstance(decorator, RefExpr) and - decorator.fullname in ('typing.runtime', 'mypy_extensions.runtime')) + decorator.fullname in ('typing.runtime', 'typing_extensions.runtime')) def calculate_abstract_status(self, typ: TypeInfo) -> None: """Calculate abstract status of a class. @@ -810,7 +810,7 @@ def detect_protocol_base(self, defn: ClassDef) -> bool: sym = self.lookup_qualified(base.name, base) if sym is None or sym.node is None: continue - if sym.node.fullname() in ('typing.Protocol', 'mypy_extensions.Protocol'): + if sym.node.fullname() in ('typing.Protocol', 'typing_extensions.Protocol'): return True return False @@ -847,7 +847,7 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: if isinstance(base, UnboundType): sym = self.lookup_qualified(base.name, base) if sym is not None and sym.node is not None: - if (sym.node.fullname() in ('typing.Protocol', 'mypy_extensions.Protocol') and + if (sym.node.fullname() in ('typing.Protocol', 'typing_extensions.Protocol') and i not in removed): # also remove bare 'Protocol' bases removed.append(i) @@ -883,7 +883,7 @@ def analyze_typevar_declaration(self, t: Type) -> Optional[TypeVarList]: return None if (sym.node.fullname() == 'typing.Generic' or sym.node.fullname() == 'typing.Protocol' and t.args or - sym.node.fullname() == 'mypy_extensions.Protocol' and t.args): + sym.node.fullname() == 'typing_extensions.Protocol' and t.args): tvars = [] # type: TypeVarList for arg in unbound.args: tvar = self.analyze_unbound_tvar(arg) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 9addcd4d443d..3509a375973a 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -317,8 +317,8 @@ class B2(P2): x2: P2 = B2() # OK [out] -[case testProtocolsInMypyExtensions] -from mypy_extensions import Protocol, runtime +[case testProtocolsInTypingExtensions] +from typing_extensions import Protocol, runtime @runtime class P(Protocol): diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index e944a1f783d4..a604c9684eeb 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -19,5 +19,3 @@ def KwArg(type: _T = ...) -> _T: ... def TypedDict(typename: str, fields: Dict[str, Type[_T]], *, total: Any = ...) -> Type[dict]: ... class NoReturn: pass -class Protocol: pass -def runtime(x: T) -> T: pass diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi new file mode 100644 index 000000000000..8c5be8f3637f --- /dev/null +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -0,0 +1,6 @@ +from typing import TypeVar + +_T = TypeVar('_T') + +class Protocol: pass +def runtime(x: _T) -> _T: pass From afe129134b6819fa1bdd2995fdab3c77eac00995 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 10:07:58 +0200 Subject: [PATCH 082/117] Fix lint --- mypy/semanal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 43df8556f1f6..cf94b5fda3f0 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -847,7 +847,8 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: if isinstance(base, UnboundType): sym = self.lookup_qualified(base.name, base) if sym is not None and sym.node is not None: - if (sym.node.fullname() in ('typing.Protocol', 'typing_extensions.Protocol') and + if (sym.node.fullname() in ('typing.Protocol', + 'typing_extensions.Protocol') and i not in removed): # also remove bare 'Protocol' bases removed.append(i) From 1858ed986a05327e3439302b0732e72036bda3e0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 13:08:45 +0200 Subject: [PATCH 083/117] Move most protocol erro formatting to messages.py --- mypy/checker.py | 74 +++++----------------------- mypy/checkexpr.py | 8 ++- mypy/messages.py | 76 ++++++++++++++++++++++++++++- test-data/unit/check-protocols.test | 10 ++-- 4 files changed, 93 insertions(+), 75 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f8c65d87dd80..5e18e6def1bd 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -44,8 +44,7 @@ from mypy.subtypes import ( is_subtype, is_equivalent, is_proper_subtype, is_more_precise, restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_subtype, - unify_generic_callable, get_missing_members, get_conflict_types, get_all_flags, - IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC, find_member + unify_generic_callable, find_member ) from mypy.maptype import map_instance_to_supertype from mypy.typevars import fill_typevars, has_no_typevars @@ -1131,23 +1130,14 @@ def check_protocol_variance(self, defn: ClassDef) -> None: down_args = [UninhabitedType() if i == j else AnyType() for j, _ in enumerate(tvars)] up, down = Instance(info, up_args), Instance(info, down_args) # TODO: add advanced variance checks for recursive protocols - is_co = is_subtype(down, up, ignore_declared_variance=True) - is_contra = is_subtype(up, down, ignore_declared_variance=True) - if is_co: - expected = 'covariant' - elif is_contra: - expected = 'contravariant' + if is_subtype(down, up, ignore_declared_variance=True): + expected = COVARIANT + elif is_subtype(up, down, ignore_declared_variance=True): + expected = CONTRAVARIANT else: - expected = 'invariant' - if tvar.variance == COVARIANT: - actual = 'Covariant' - elif tvar.variance == CONTRAVARIANT: - actual = 'Contravariant' - else: - actual = 'Invariant' - if expected != actual.lower(): - self.fail("{} type variable '{}' used in protocol where" - " {} one is expected".format(actual, tvar.name, expected), defn) + expected = INVARIANT + if expected != tvar.variance: + self.msg.bad_proto_variance(tvar.variance, tvar.name, expected, defn) def check_multiple_inheritance(self, typ: TypeInfo) -> None: """Check for multiple inheritance related errors.""" @@ -1327,8 +1317,7 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type isinstance(lvalue_type.item, Instance) and (lvalue_type.item.type.is_abstract or lvalue_type.item.type.is_protocol)): - self.fail("Can only assign concrete classes" - " to a variable of type '{}'".format(lvalue_type), rvalue) + self.msg.concrete_only_assign(lvalue_type, rvalue) return if rvalue_type and infer_lvalue_type: self.binder.assign_type(lvalue, rvalue_type, lvalue_type, False) @@ -2438,54 +2427,13 @@ def check_subtype(self, subtype: Type, supertype: Type, context: Context, self.note(note_msg, context) if (isinstance(supertype, Instance) and supertype.type.is_protocol and isinstance(subtype, (Instance, TupleType))): - self.report_protocol_problems(subtype, supertype, context) + self.msg.report_protocol_problems(subtype, supertype, context) if isinstance(supertype, CallableType) and isinstance(subtype, Instance): call = find_member('__call__', subtype, subtype) if call: - self.note("{}.__call__ has type {}".format(subtype, call), context) + self.msg.note_call(subtype, call, context) return False - def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertype: Instance, - context: Context) -> None: - """Report possible protocol conflicts between 'subtype' and 'supertype'. - This includes missing members, incompatible types, and incompatible - attribute flags, such as settable vs read-only or class variable vs - instance variable. - """ - OFFSET = 4 # Four spaces, so that notes will look like this: - # note: 'Cls' is missing following 'Proto' member(s): - # note: method, attr - if isinstance(subtype, TupleType): - if not isinstance(subtype.fallback, Instance): - return - subtype = subtype.fallback - missing = get_missing_members(subtype, supertype) - if missing: - self.note("'{}' is missing following '{}' protocol member(s):" - .format(subtype.type.fullname(), supertype.type.fullname()), - context) - self.note(', '.join(missing), context, offset=OFFSET) - conflict_types = get_conflict_types(subtype, supertype) - if conflict_types: - self.note('Following member(s) of {} have ' - 'conflicts:'.format(subtype), context) - for name, got, expected in conflict_types: - self.note('{}: expected {}, got {}'.format(name, expected, got), - context, offset=OFFSET) - for name, subflags, superflags in get_all_flags(subtype, supertype): - if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: - self.note('Protocol member {}.{}: expected instance variable,' - ' got class variable'.format(supertype, name), context) - if IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: - self.note('Protocol member {}.{}: expected class variable,' - ' got instance variable'.format(supertype, name), context) - if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: - self.note('Protocol member {}.{}: expected settable variable,' - ' got read-only attribute'.format(supertype, name), context) - if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: - self.note('Protocol member {}.{}: expected class or static method' - .format(supertype, name), context) - def contains_none(self, t: Type) -> bool: return ( isinstance(t, NoneTyp) or diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 764674c4b6cd..99cbc3ab5feb 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1019,8 +1019,7 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, (callee_type.item.type.is_abstract or callee_type.item.type.is_protocol) and # ...except for classmethod first argument not caller_type.is_classmethod_class): - messages.fail("Only concrete class can be given where '{}' is expected" - .format(callee_type), context) + self.msg.concrete_only_call(callee_type, context) elif not is_subtype(caller_type, callee_type): if self.chk.should_suppress_optional_error([caller_type, callee_type]): return @@ -1028,13 +1027,12 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, caller_kind, context) if (isinstance(original_caller_type, (Instance, TupleType)) and isinstance(callee_type, Instance) and callee_type.type.is_protocol): - self.chk.report_protocol_problems(original_caller_type, callee_type, context) + self.msg.report_protocol_problems(original_caller_type, callee_type, context) if (isinstance(callee_type, CallableType) and isinstance(original_caller_type, Instance)): call = find_member('__call__', original_caller_type, original_caller_type) if call: - self.chk.note("{}.__call__ has type {}" - .format(original_caller_type, call), context) + self.msg.note_call(original_caller_type, call, context) def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], arg_names: List[str], diff --git a/mypy/messages.py b/mypy/messages.py index 9c0aaac7c4c7..4a68c4cc58c9 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -6,7 +6,7 @@ import re import difflib -from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple, Optional +from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple, Optional, Union from mypy.erasetype import erase_type from mypy.errors import Errors @@ -18,7 +18,7 @@ from mypy.nodes import ( TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, ARG_POS, ARG_OPT, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, - ReturnStmt, NameExpr, Var + ReturnStmt, NameExpr, Var, CONTRAVARIANT, COVARIANT ) @@ -959,6 +959,78 @@ def untyped_decorated_function(self, typ: Type, context: Context) -> None: self.fail('Type of decorated function contains type "Any" ({})'.format( self.format(typ)), context) + def bad_proto_variance(self, actual: int, tvar_name: str, expected: int, + context: Context) -> None: + msg = capitalize("{} type variable '{}' used in protocol where" + " {} one is expected".format(variance_string(actual), + tvar_name, + variance_string(expected))) + self.fail(msg, context) + + def concrete_only_assign(self, typ: Type, context: Context) -> None: + self.fail("Can only assign concrete classes to a variable of type '{}'" + .format(self.format(typ)), context) + + def concrete_only_call(self, typ: Type, context: Context) -> None: + self.fail("Only concrete class can be given where '{}' is expected" + .format(self.format(typ)), context) + + def note_call(self, subtype: Type, call: Type, context: Context) -> None: + self.note("'{}.__call__' has type '{}'".format(self.format(subtype).replace('"', ''), + self.format(call)), context) + + def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertype: Instance, + context: Context) -> None: + """Report possible protocol conflicts between 'subtype' and 'supertype'. + This includes missing members, incompatible types, and incompatible + attribute flags, such as settable vs read-only or class variable vs + instance variable. + """ + from mypy.subtypes import (get_missing_members, get_conflict_types, get_all_flags, + is_subtype, IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC) + OFFSET = 4 # Four spaces, so that notes will look like this: + # note: 'Cls' is missing following 'Proto' member(s): + # note: method, attr + if isinstance(subtype, TupleType): + if not isinstance(subtype.fallback, Instance): + return + subtype = subtype.fallback + missing = get_missing_members(subtype, supertype) + if missing: + self.note("'{}' is missing following '{}' protocol member(s):" + .format(subtype.type.fullname(), supertype.type.fullname()), + context) + self.note(', '.join(missing), context, offset=OFFSET) + conflict_types = get_conflict_types(subtype, supertype) + if conflict_types: + self.note('Following member(s) of {} have ' + 'conflicts:'.format(subtype), context) + for name, got, expected in conflict_types: + self.note('{}: expected {}, got {}'.format(name, expected, got), + context, offset=OFFSET) + for name, subflags, superflags in get_all_flags(subtype, supertype): + if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: + self.note('Protocol member {}.{}: expected instance variable,' + ' got class variable'.format(supertype, name), context) + if IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: + self.note('Protocol member {}.{}: expected class variable,' + ' got instance variable'.format(supertype, name), context) + if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: + self.note('Protocol member {}.{}: expected settable variable,' + ' got read-only attribute'.format(supertype, name), context) + if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: + self.note('Protocol member {}.{}: expected class or static method' + .format(supertype, name), context) + + +def variance_string(variance: int) -> str: + if variance == COVARIANT: + return 'covariant' + elif variance == CONTRAVARIANT: + return 'contravariant' + else: + return 'invariant' + def capitalize(s: str) -> str: """Capitalize the first character of a string.""" diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 3509a375973a..2675d81444bc 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1108,7 +1108,7 @@ def f(cls: Type[P]) -> P: def g() -> P: return P() # E: Cannot instantiate protocol class "__main__.P" -f(P) # E: Only concrete class can be given where 'Type[__main__.P]' is expected +f(P) # E: Only concrete class can be given where 'Type[P]' is expected f(B) # OK f(C) # OK x: Type[P1] @@ -1133,7 +1133,7 @@ Alias = P GoodAlias = C Alias() # E: Cannot instantiate protocol class "__main__.P" GoodAlias() -f(Alias) # E: Only concrete class can be given where 'Type[__main__.P]' is expected +f(Alias) # E: Only concrete class can be given where 'Type[P]' is expected f(GoodAlias) [out] @@ -1149,13 +1149,13 @@ class C: var: Type[P] var() -var = P # E: Can only assign concrete classes to a variable of type 'Type[__main__.P]' +var = P # E: Can only assign concrete classes to a variable of type 'Type[P]' var = B # OK var = C # OK var_old = None # type: Type[P] # Old syntax for variable annotations var_old() -var_old = P # E: Can only assign concrete classes to a variable of type 'Type[__main__.P]' +var_old = P # E: Can only assign concrete classes to a variable of type 'Type[P]' var_old = B # OK var_old = C # OK [out] @@ -1394,7 +1394,7 @@ reveal_type(apply_gen(Add5())) # E: Revealed type is 'builtins.int*' def apply_str(f: Callable[[str], int], x: str) -> int: return f(x) apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected Callable[[str], int] \ - # N: __main__.Add5.__call__ has type def (x: builtins.int) -> builtins.int + # N: 'Add5.__call__' has type 'Callable[[int], int]' [builtins fixtures/isinstancelist.pyi] [out] From 057a871e562880aaa3e0e97e4a88bc63dfc634ec Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 13:12:26 +0200 Subject: [PATCH 084/117] Fix few error messages in tests --- test-data/unit/check-abstract.test | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test-data/unit/check-abstract.test b/test-data/unit/check-abstract.test index 54ccaec4a9f3..cd8cbe43b595 100644 --- a/test-data/unit/check-abstract.test +++ b/test-data/unit/check-abstract.test @@ -174,8 +174,8 @@ def f(cls: Type[A]) -> A: def g() -> A: return A() # E: Cannot instantiate abstract class 'A' with abstract attribute 'm' -f(A) # E: Only concrete class can be given where 'Type[__main__.A]' is expected -f(B) # E: Only concrete class can be given where 'Type[__main__.A]' is expected +f(A) # E: Only concrete class can be given where 'Type[A]' is expected +f(B) # E: Only concrete class can be given where 'Type[A]' is expected f(C) # OK x: Type[B] f(x) # OK @@ -200,7 +200,7 @@ Alias = A GoodAlias = C Alias() # E: Cannot instantiate abstract class 'A' with abstract attribute 'm' GoodAlias() -f(Alias) # E: Only concrete class can be given where 'Type[__main__.A]' is expected +f(Alias) # E: Only concrete class can be given where 'Type[A]' is expected f(GoodAlias) [out] @@ -218,14 +218,14 @@ class C(B): var: Type[A] var() -var = A # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' -var = B # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' +var = A # E: Can only assign concrete classes to a variable of type 'Type[A]' +var = B # E: Can only assign concrete classes to a variable of type 'Type[A]' var = C # OK var_old = None # type: Type[A] # Old syntax for variable annotations var_old() -var_old = A # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' -var_old = B # E: Can only assign concrete classes to a variable of type 'Type[__main__.A]' +var_old = A # E: Can only assign concrete classes to a variable of type 'Type[A]' +var_old = B # E: Can only assign concrete classes to a variable of type 'Type[A]' var_old = C # OK [out] From 28c1b9dfcb87e3840fc6d93725cae6e4c8e1151c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 13:40:16 +0200 Subject: [PATCH 085/117] Limit display of missing protocol members --- mypy/messages.py | 7 ++- test-data/unit/check-async-await.test | 42 ++----------- test-data/unit/check-incomplete-fixture.test | 2 - test-data/unit/check-namedtuple.test | 5 +- test-data/unit/check-protocols.test | 66 ++++++-------------- 5 files changed, 30 insertions(+), 92 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 4a68c4cc58c9..a68a51099d57 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -996,9 +996,10 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp return subtype = subtype.fallback missing = get_missing_members(subtype, supertype) - if missing: - self.note("'{}' is missing following '{}' protocol member(s):" - .format(subtype.type.fullname(), supertype.type.fullname()), + if missing and len(missing) < len(supertype.type.protocol_members) and len(missing) < 3: + self.note("'{}' is missing following '{}' protocol member{}:" + .format(subtype.type.name(), supertype.type.name(), + 's' if len(missing) > 1 else ''), context) self.note(', '.join(missing), context, offset=OFFSET) conflict_types = get_conflict_types(subtype, supertype) diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index fb5ae39fea4a..f8ac01d8c830 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -94,8 +94,6 @@ async def f() -> int: [typing fixtures/typing-full.pyi] [out] main:7: error: Incompatible types in await (actual type Generator[int, None, str], expected type Awaitable[Any]) -main:7: note: 'typing.Generator' is missing following 'typing.Awaitable' protocol member(s): -main:7: note: __await__ [case testAwaitIteratorError] @@ -108,8 +106,6 @@ async def f() -> int: [typing fixtures/typing-full.pyi] [out] main:6: error: Incompatible types in await (actual type Iterator[Any], expected type Awaitable[Any]) -main:6: note: 'typing.Iterator' is missing following 'typing.Awaitable' protocol member(s): -main:6: note: __await__ [case testAwaitArgumentError] @@ -122,8 +118,6 @@ async def f() -> int: [typing fixtures/typing-full.pyi] [out] main:5: error: Incompatible types in await (actual type "int", expected type Awaitable[Any]) -main:5: note: 'builtins.int' is missing following 'typing.Awaitable' protocol member(s): -main:5: note: __await__ [case testAwaitResultError] @@ -170,8 +164,6 @@ async def f() -> None: [typing fixtures/typing-full.pyi] [out] main:4: error: AsyncIterable expected -main:4: note: 'builtins.list' is missing following 'typing.AsyncIterable' protocol member(s): -main:4: note: __aiter__ main:4: error: List[int] has no attribute "__aiter__" [case testAsyncForTypeComments] @@ -256,20 +248,12 @@ async def wrong_iterable(obj: Iterable[int]): [out] main:18: error: AsyncIterable expected -main:18: note: 'typing.Iterable' is missing following 'typing.AsyncIterable' protocol member(s): -main:18: note: __aiter__ main:18: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:19: error: Iterable expected -main:19: note: '__main__.asyncify' is missing following 'typing.Iterable' protocol member(s): -main:19: note: __iter__ main:19: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? main:20: error: AsyncIterable expected -main:20: note: 'typing.Iterable' is missing following 'typing.AsyncIterable' protocol member(s): -main:20: note: __aiter__ main:20: error: Iterable[int] has no attribute "__aiter__"; maybe "__iter__"? main:21: error: Iterable expected -main:21: note: '__main__.asyncify' is missing following 'typing.Iterable' protocol member(s): -main:21: note: __iter__ main:21: error: asyncify[int] has no attribute "__iter__"; maybe "__aiter__"? [builtins fixtures/async_await.pyi] [typing fixtures/typing-full.pyi] @@ -306,9 +290,7 @@ class C: def __aenter__(self) -> int: pass async def __aexit__(self, x, y, z) -> None: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any]) \ - # N: 'builtins.int' is missing following 'typing.Awaitable' protocol member(s): \ - # N: __await__ + async with C() as x: # E: Incompatible types in "async with" for __aenter__ (actual type "int", expected type Awaitable[Any]) pass [builtins fixtures/async_await.pyi] [typing fixtures/typing-full.pyi] @@ -330,9 +312,7 @@ class C: async def __aenter__(self) -> int: pass def __aexit__(self, x, y, z) -> int: pass async def f() -> None: - async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any]) \ - # N: 'builtins.int' is missing following 'typing.Awaitable' protocol member(s): \ - # N: __await__ + async with C() as x: # E: Incompatible types in "async with" for __aexit__ (actual type "int", expected type Awaitable[Any]) pass [builtins fixtures/async_await.pyi] [typing fixtures/typing-full.pyi] @@ -570,8 +550,6 @@ def h() -> None: [out] main:9: error: Iterable expected -main:9: note: 'typing.AsyncGenerator' is missing following 'typing.Iterable' protocol member(s): -main:9: note: __iter__ main:9: error: AsyncGenerator[int, None] has no attribute "__iter__"; maybe "__aiter__"? [case testAsyncGeneratorNoYieldFrom] @@ -666,15 +644,11 @@ def plain_host_generator() -> Generator[str, None, None]: async def plain_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) \ - # N: 'typing.Generator' is missing following 'typing.Awaitable' protocol member(s): \ - # N: __await__ + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) \ - # N: '__main__.It' is missing following 'typing.Awaitable' protocol member(s): \ - # N: __await__ + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) x = await other_coroutine() @coroutine @@ -691,15 +665,11 @@ def decorated_host_generator() -> Generator[str, None, None]: @coroutine async def decorated_host_coroutine() -> None: x = 0 - x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) \ - # N: 'typing.Generator' is missing following 'typing.Awaitable' protocol member(s): \ - # N: __await__ + x = await plain_generator() # E: Incompatible types in await (actual type Generator[str, None, int], expected type Awaitable[Any]) x = await plain_coroutine() x = await decorated_generator() x = await decorated_coroutine() - x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) \ - # N: '__main__.It' is missing following 'typing.Awaitable' protocol member(s): \ - # N: __await__ + x = await other_iterator() # E: Incompatible types in await (actual type "It", expected type Awaitable[Any]) x = await other_coroutine() [builtins fixtures/async_await.pyi] diff --git a/test-data/unit/check-incomplete-fixture.test b/test-data/unit/check-incomplete-fixture.test index 2e291da585ba..68c7c6c9aa0f 100644 --- a/test-data/unit/check-incomplete-fixture.test +++ b/test-data/unit/check-incomplete-fixture.test @@ -77,8 +77,6 @@ for y in x: main:2: error: "tuple" expects no type arguments, but 1 given main:3: error: Value of type "tuple" is not indexable main:4: error: Iterable expected -main:4: note: 'builtins.tuple' is missing following 'typing.Iterable' protocol member(s): -main:4: note: __iter__ main:4: error: "tuple" has no attribute "__iter__" [case testClassmethodMissingFromStubs] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index adf3fae9c3ef..81aa3f6957e6 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -286,10 +286,7 @@ from typing import NamedTuple X = NamedTuple('X', [('x', int), ('y', str)]) reveal_type(X._make([5, 'a'])) # E: Revealed type is 'Tuple[builtins.int, builtins.str, fallback=__main__.X]' -X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any] \ - # N: 'builtins.str' is missing following 'typing.Iterable' protocol member(s): \ - # N: __iter__ -# Note that the above will not fail with real stubs, since 'str', is a subtype of 'Iterable[str]' +X._make('a b') # E: Argument 1 to X._make has incompatible type "str"; expected Iterable[Any] -- # FIX: not a proper class method -- x = None # type: X diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 2675d81444bc..ec23ddbbde94 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -30,21 +30,15 @@ def fun(x: P) -> None: x.bad # E: "P" has no attribute "bad" x = C() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ - # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ - # N: meth +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") fun(C()) -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" \ - # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ - # N: meth +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" def fun2() -> P: return C() def fun3() -> P: - return B() # E: Incompatible return value type (got "B", expected "P") \ - # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ - # N: meth + return B() # E: Incompatible return value type (got "B", expected "P") [out] [case testSimpleProtocolOneAbstractMethod] @@ -72,14 +66,10 @@ def fun(x: P) -> None: x = C() x = D() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ - # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ - # N: meth +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") fun(C()) fun(D()) -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" \ - # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ - # N: meth +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" [out] [case testSimpleProtocolOneMethodOverride] @@ -104,14 +94,10 @@ def fun(x: SubP) -> str: z = x x = C() -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") \ - # N: '__main__.B' is missing following '__main__.SubP' protocol member(s): \ - # N: meth +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "SubP") reveal_type(fun(C())) # E: Revealed type is 'builtins.str' -fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" \ - # N: '__main__.B' is missing following '__main__.SubP' protocol member(s): \ - # N: meth +fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" [out] [case testSimpleProtocolTwoMethodsMerge] @@ -154,11 +140,9 @@ c2: C2 y: AnotherP x = c -x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") \ - # N: '__main__.B' is missing following '__main__.P' protocol member(s): \ - # N: meth1, meth2 +x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P") x = c1 # E: Incompatible types in assignment (expression has type "C1", variable has type "P") \ - # N: '__main__.C1' is missing following '__main__.P' protocol member(s): \ + # N: 'C1' is missing following 'P' protocol member: \ # N: meth2 x = c2 x = y @@ -191,7 +175,7 @@ reveal_type(x.meth2()) # E: Revealed type is 'builtins.str' x = C() # OK x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2") \ - # N: '__main__.Cbad' is missing following '__main__.P2' protocol member(s): \ + # N: 'Cbad' is missing following 'P2' protocol member: \ # N: meth2 [out] @@ -496,7 +480,7 @@ class E(D[T]): def f(x: T) -> T: z: P2[T, T] = E[T]() y: P2[T, T] = D[T]() # E: Incompatible types in assignment (expression has type D[T], variable has type P2[T, T]) \ - # N: '__main__.D' is missing following '__main__.P2' protocol member(s): \ + # N: 'D' is missing following 'P2' protocol member: \ # N: attr2 return x [builtins fixtures/isinstancelist.pyi] @@ -651,9 +635,7 @@ class D(Generic[T]): t: Traversable t = D[int]() # OK -t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable") \ - # N: '__main__.C' is missing following '__main__.Traversable' protocol member(s): \ - # N: leaves +t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable") [builtins fixtures/list.pyi] [out] @@ -697,12 +679,8 @@ class B: t: P1 t = A() # OK -t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1") \ - # N: '__main__.B' is missing following '__main__.P1' protocol member(s): \ - # N: attr1 -t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1") \ - # N: '__main__.C' is missing following '__main__.P1' protocol member(s): \ - # N: attr1 +t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1") +t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1") [builtins fixtures/list.pyi] [out] @@ -1364,7 +1342,7 @@ fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int # N: y: expected builtins.int*, got builtins.str fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str] \ - # N: '__main__.N2' is missing following '__main__.P' protocol member(s): \ + # N: 'N2' is missing following 'P' protocol member: \ # N: y reveal_type(fun3(z)) # E: Revealed type is 'builtins.object*' @@ -1438,8 +1416,6 @@ bar(1) [builtins fixtures/isinstancelist.pyi] [out] main:11: error: Argument 1 to "bar" has incompatible type "int"; expected "Sized" -main:11: note: 'builtins.int' is missing following 'typing.Sized' protocol member(s): -main:11: note: __len__ [case testBasicSupportsIntProtocol] from typing import SupportsInt @@ -1457,8 +1433,6 @@ foo('no way') [builtins fixtures/isinstancelist.pyi] [out] main:11: error: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" -main:11: note: 'builtins.str' is missing following 'typing.SupportsInt' protocol member(s): -main:11: note: __int__ -- Additional test and corner cases for protocols -- ---------------------------------------------- @@ -1490,9 +1464,7 @@ f1(C1()) f2(C2()) f3(C3()) -f2(C3()) # E: Argument 1 to "f2" has incompatible type "C3"; expected P2[str] \ - # N: '__main__.C3' is missing following '__main__.P2' protocol member(s): \ - # N: attr2 +f2(C3()) # E: Argument 1 to "f2" has incompatible type "C3"; expected P2[str] a: Any f1(a) f2(a) @@ -1513,7 +1485,7 @@ class C: def attr2(self) -> int: pass x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ - # N: '__main__.C' is missing following '__main__.P' protocol member(s): \ + # N: 'C' is missing following 'P' protocol member: \ # N: attr3 \ # N: Following member(s) of __main__.C have conflicts: \ # N: attr1: expected builtins.int, got builtins.str \ @@ -1522,7 +1494,7 @@ x: P = C() # E: Incompatible types in assignment (expression has type "C", varia def f(x: P) -> P: return C() # E: Incompatible return value type (got "C", expected "P") \ - # N: '__main__.C' is missing following '__main__.P' protocol member(s): \ + # N: 'C' is missing following 'P' protocol member: \ # N: attr3 \ # N: Following member(s) of __main__.C have conflicts: \ # N: attr1: expected builtins.int, got builtins.str \ @@ -1530,7 +1502,7 @@ def f(x: P) -> P: # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ - # N: '__main__.C' is missing following '__main__.P' protocol member(s): \ + # N: 'C' is missing following 'P' protocol member: \ # N: attr3 \ # N: Following member(s) of __main__.C have conflicts: \ # N: attr1: expected builtins.int, got builtins.str \ From ad2bcaa75b0f49690c13981738ee97db529cd0da Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 14:24:32 +0200 Subject: [PATCH 086/117] Limit (and simplify) display of conflicting protocol member types --- mypy/checkexpr.py | 2 +- mypy/messages.py | 19 +++--- test-data/unit/check-expressions.test | 8 +-- test-data/unit/check-protocols.test | 98 ++++++++++++--------------- 4 files changed, 59 insertions(+), 68 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 99cbc3ab5feb..dcb855c9b83e 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -515,7 +515,7 @@ def check_call(self, callee: Type, args: List[Expression], # Exceptions for Type[...] and classmethod first argument and not callee.from_type_type and not callee.is_classmethod_class): self.chk.fail('Cannot instantiate protocol class "{}"' - .format(callee.type_object().fullname()), context) + .format(callee.type_object().name()), context) formal_to_actual = map_actuals_to_formals( arg_kinds, arg_names, diff --git a/mypy/messages.py b/mypy/messages.py index a68a51099d57..286f90df8a7e 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1003,25 +1003,28 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp context) self.note(', '.join(missing), context, offset=OFFSET) conflict_types = get_conflict_types(subtype, supertype) - if conflict_types: + if conflict_types and (not is_subtype(subtype, erase_type(supertype)) or + not subtype.type.defn.type_vars or + not supertype.type.defn.type_vars): self.note('Following member(s) of {} have ' - 'conflicts:'.format(subtype), context) - for name, got, expected in conflict_types: - self.note('{}: expected {}, got {}'.format(name, expected, got), + 'conflicts:'.format(self.format(subtype)), context) + for name, got, exp in conflict_types: + self.note('{}: expected {}, got {}'.format(name, + *self.format_distinctly(exp, got)), context, offset=OFFSET) for name, subflags, superflags in get_all_flags(subtype, supertype): if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note('Protocol member {}.{}: expected instance variable,' - ' got class variable'.format(supertype, name), context) + ' got class variable'.format(supertype.type.name(), name), context) if IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: self.note('Protocol member {}.{}: expected class variable,' - ' got instance variable'.format(supertype, name), context) + ' got instance variable'.format(supertype.type.name(), name), context) if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: self.note('Protocol member {}.{}: expected settable variable,' - ' got read-only attribute'.format(supertype, name), context) + ' got read-only attribute'.format(supertype.type.name(), name), context) if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: self.note('Protocol member {}.{}: expected class or static method' - .format(supertype, name), context) + .format(supertype.type.name(), name), context) def variance_string(variance: int) -> str: diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index a335d33e462f..8d026e3dd50b 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1701,12 +1701,8 @@ b = {'z': 26, **a} c = {**b} d = {**a, **b, 'c': 3} # These errors changed because Mapping is a protocol -e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str] \ - # N: Following member(s) of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ - # N: __getitem__: expected def (builtins.int*) -> builtins.str*, got def (builtins.str*) -> builtins.int* -f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int] \ - # N: Following member(s) of builtins.dict[builtins.str*, builtins.int*] have conflicts: \ - # N: __getitem__: expected def (builtins.int*) -> builtins.int*, got def (builtins.str*) -> builtins.int* +e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str] +f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int] [builtins fixtures/dict.pyi] [case testDictIncompatibleTypeErrorMessage] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index ec23ddbbde94..814bb075c834 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -8,7 +8,7 @@ class P(Protocol): def meth(self) -> None: pass -P() # E: Cannot instantiate protocol class "__main__.P" +P() # E: Cannot instantiate protocol class "P" [out] [case testSimpleProtocolOneMethod] @@ -361,26 +361,18 @@ class B(A): pass x1: Pco[B] y1: Pco[A] -x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B]) \ - # N: Following member(s) of __main__.Pco[__main__.A] have conflicts: \ - # N: meth: expected def () -> __main__.B*, got def () -> __main__.A* +x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B]) y1 = x1 x2: Pcontra[B] y2: Pcontra[A] -y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A]) \ - # N: Following member(s) of __main__.Pcontra[__main__.B] have conflicts: \ - # N: meth: expected def (x: __main__.A*), got def (x: __main__.B*) +y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A]) x2 = y2 x3: Pinv[B] y3: Pinv[A] -y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A]) \ - # N: Following member(s) of __main__.Pinv[__main__.B] have conflicts: \ - # N: attr: expected __main__.A*, got __main__.B* -x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B]) \ - # N: Following member(s) of __main__.Pinv[__main__.A] have conflicts: \ - # N: attr: expected __main__.B*, got __main__.A* +y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A]) +x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B]) [out] [case testGenericProtocolsInference1] @@ -469,8 +461,8 @@ class C: c: C var: P2[int, int] = c var2: P2[int, str] = c # E: Incompatible types in assignment (expression has type "C", variable has type P2[int, str]) \ - # N: Following member(s) of __main__.C have conflicts: \ - # N: attr2: expected Tuple[builtins.int*, builtins.str*], got Tuple[builtins.int, builtins.int] + # N: Following member(s) of "C" have conflicts: \ + # N: attr2: expected "Tuple[int, str]", got "Tuple[int, int]" class D(Generic[T]): attr1: T @@ -512,9 +504,9 @@ class D(A, B): pass x: P = D() # Same as P[Any, Any] var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[int, P[Any, Any]], Union[P[Any, Any], str]]) \ - # N: Following member(s) of __main__.C have conflicts: \ - # N: attr1: expected Union[builtins.int, __main__.P[Any, Any]], got builtins.int \ - # N: attr2: expected Union[__main__.P[Any, Any], builtins.str], got builtins.str + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "Union[int, P[Any, Any]]", got "int" \ + # N: attr2: expected "Union[P[Any, Any], str]", got "str" [out] [case testGenericSubProtocolsExtensionCovariant] @@ -536,9 +528,9 @@ class C: var: P[Union[int, P], Union[P, str]] = C() # OK for covariant var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) \ - # N: Following member(s) of __main__.C have conflicts: \ - # N: attr1: expected def () -> Union[builtins.str, __main__.P[Any, Any]], got def () -> builtins.int \ - # N: attr2: expected def () -> Union[__main__.P[Any, Any], builtins.int], got def () -> builtins.str + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected Callable[[], Union[str, P[Any, Any]]], got Callable[[], int] \ + # N: attr2: expected Callable[[], Union[P[Any, Any], int]], got Callable[[], str] [out] [case testSelfTypesWithProtocolsBehaveAsWithNominal] @@ -568,11 +560,11 @@ s: Shape f(NonProtoShape()) f(Circle()) s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") \ - # N: Following member(s) of __main__.Triangle have conflicts: \ - # N: combine: expected def (other: __main__.Triangle*) -> __main__.Triangle*, got def (other: __main__.Shape) -> __main__.Shape + # N: Following member(s) of "Triangle" have conflicts: \ + # N: combine: expected Callable[[Triangle], Triangle], got Callable[[Shape], Shape] s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") \ - # N: Following member(s) of __main__.Bad have conflicts: \ - # N: combine: expected def (other: __main__.Bad*) -> __main__.Bad*, got def (other: builtins.int) -> builtins.str + # N: Following member(s) of "Bad" have conflicts: \ + # N: combine: expected Callable[[Bad], Bad], got Callable[[int], str] n2: NonProtoShape = s # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") [out] @@ -759,10 +751,10 @@ y: PClass x = CInst() x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst") \ - # N: Protocol member __main__.PInst.v: expected instance variable, got class variable + # N: Protocol member PInst.v: expected instance variable, got class variable y = CClass() y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass") \ - # N: Protocol member __main__.PClass.v: expected class variable, got instance variable + # N: Protocol member PClass.v: expected class variable, got instance variable [out] [case testPropertyInProtocols] @@ -783,7 +775,7 @@ y = x x2: P y2: PP x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P") \ - # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute + # N: Protocol member P.attr: expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -821,7 +813,7 @@ y3 = z3 y4: PP z4: PPS z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS") \ - # N: Protocol member __main__.PPS.attr: expected settable variable, got read-only attribute + # N: Protocol member PPS.attr: expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -853,7 +845,7 @@ x = B() y: PC y = B() y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC") \ - # N: Protocol member __main__.PC.meth: expected class or static method + # N: Protocol member PC.meth: expected class or static method [builtins fixtures/classmethod.pyi] [out] @@ -876,8 +868,8 @@ class D: x: P = C() x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ - # N: Following member(s) of __main__.D have conflicts: \ - # N: f: expected Overload(def (x: builtins.int) -> builtins.int, def (x: builtins.str) -> builtins.str), got def (x: builtins.int) + # N: Following member(s) of "D" have conflicts: \ + # N: f: expected overloaded function, got Callable[[int], None] [out] -- Join and meet with protocol types @@ -1084,7 +1076,7 @@ class C: def f(cls: Type[P]) -> P: return cls() # OK def g() -> P: - return P() # E: Cannot instantiate protocol class "__main__.P" + return P() # E: Cannot instantiate protocol class "P" f(P) # E: Only concrete class can be given where 'Type[P]' is expected f(B) # OK @@ -1109,7 +1101,7 @@ def f(cls: Type[P]) -> P: Alias = P GoodAlias = C -Alias() # E: Cannot instantiate protocol class "__main__.P" +Alias() # E: Cannot instantiate protocol class "P" GoodAlias() f(Alias) # E: Only concrete class can be given where 'Type[P]' is expected f(GoodAlias) @@ -1338,8 +1330,8 @@ def fun3(x: P[T, T]) -> T: fun(z) fun2(z) # E: Argument 1 to "fun2" has incompatible type "N"; expected P[int, int] \ - # N: Following member(s) of __main__.N have conflicts: \ - # N: y: expected builtins.int*, got builtins.str + # N: Following member(s) of "N" have conflicts: \ + # N: y: expected "int", got "str" fun(N2(1)) # E: Argument 1 to "fun" has incompatible type "N2"; expected P[int, str] \ # N: 'N2' is missing following 'P' protocol member: \ @@ -1487,27 +1479,27 @@ class C: x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ # N: 'C' is missing following 'P' protocol member: \ # N: attr3 \ - # N: Following member(s) of __main__.C have conflicts: \ - # N: attr1: expected builtins.int, got builtins.str \ - # N: attr2: expected builtins.str, got builtins.int \ - # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "int", got "str" \ + # N: attr2: expected "str", got "int" \ + # N: Protocol member P.attr2: expected settable variable, got read-only attribute def f(x: P) -> P: return C() # E: Incompatible return value type (got "C", expected "P") \ # N: 'C' is missing following 'P' protocol member: \ # N: attr3 \ - # N: Following member(s) of __main__.C have conflicts: \ - # N: attr1: expected builtins.int, got builtins.str \ - # N: attr2: expected builtins.str, got builtins.int \ - # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "int", got "str" \ + # N: attr2: expected "str", got "int" \ + # N: Protocol member P.attr2: expected settable variable, got read-only attribute f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ # N: 'C' is missing following 'P' protocol member: \ # N: attr3 \ - # N: Following member(s) of __main__.C have conflicts: \ - # N: attr1: expected builtins.int, got builtins.str \ - # N: attr2: expected builtins.str, got builtins.int \ - # N: Protocol member __main__.P.attr2: expected settable variable, got read-only attribute + # N: Following member(s) of "C" have conflicts: \ + # N: attr1: expected "int", got "str" \ + # N: attr2: expected "str", got "int" \ + # N: Protocol member P.attr2: expected settable variable, got read-only attribute [builtins fixtures/list.pyi] [out] @@ -1571,11 +1563,11 @@ def fun_p(x: PP) -> None: reveal_type(P.attr) # E: Revealed type is 'builtins.int' fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P" \ - # N: Protocol member __main__.P.attr: expected settable variable, got read-only attribute + # N: Protocol member P.attr: expected settable variable, got read-only attribute fun(C2()) fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" \ - # N: Following member(s) of __main__.D have conflicts: \ - # N: attr: expected builtins.int, got builtins.str + # N: Following member(s) of "D" have conflicts: \ + # N: attr: expected "int", got "str" fun_p(C()) # OK [builtins fixtures/list.pyi] [out] @@ -1594,8 +1586,8 @@ class D: x: P x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ - # N: Following member(s) of __main__.D have conflicts: \ - # N: x: expected builtins.int, got builtins.str + # N: Following member(s) of "D" have conflicts: \ + # N: x: expected "int", got "str" x = C() # OK [builtins fixtures/list.pyi] [out] From 8af72487d8a2ba48b0d4c608ef5d6963e099fa50 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 15:51:19 +0200 Subject: [PATCH 087/117] Use pretty callable formatting. Credits to @markkohdev --- mypy/errors.py | 4 ++ mypy/messages.py | 75 ++++++++++++++++++++++++--- test-data/unit/check-expressions.test | 1 - test-data/unit/check-protocols.test | 71 ++++++++++++++++--------- 4 files changed, 118 insertions(+), 33 deletions(-) diff --git a/mypy/errors.py b/mypy/errors.py index 60c8efed10d8..a642196eeb3f 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -11,6 +11,7 @@ T = TypeVar('T') +allowed_duplicates = ['@overload', 'Got:', 'Expected:'] class ErrorInfo: @@ -473,6 +474,9 @@ def remove_duplicates(self, errors: List[Tuple[Optional[str], int, int, str, str while (j >= 0 and errors[j][0] == errors[i][0] and errors[j][1] == errors[i][1]): if (errors[j][3] == errors[i][3] and + # Allow duplicate notes in overload conficts reporting + not (errors[i][3] == 'note' and + errors[i][4].strip() in allowed_duplicates) and errors[j][4] == errors[i][4]): # ignore column dup = True break diff --git a/mypy/messages.py b/mypy/messages.py index 286f90df8a7e..3eb4045e5a1b 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -989,7 +989,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp from mypy.subtypes import (get_missing_members, get_conflict_types, get_all_flags, is_subtype, IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC) OFFSET = 4 # Four spaces, so that notes will look like this: - # note: 'Cls' is missing following 'Proto' member(s): + # note: 'Cls' is missing following 'Proto' members: # note: method, attr if isinstance(subtype, TupleType): if not isinstance(subtype.fallback, Instance): @@ -1009,23 +1009,82 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp self.note('Following member(s) of {} have ' 'conflicts:'.format(self.format(subtype)), context) for name, got, exp in conflict_types: - self.note('{}: expected {}, got {}'.format(name, - *self.format_distinctly(exp, got)), - context, offset=OFFSET) + if (not isinstance(exp, (CallableType, Overloaded)) or + not isinstance(exp, (CallableType, Overloaded))): + self.note('{}: expected {}, got {}'.format(name, + *self.format_distinctly(exp, got)), + context, offset=OFFSET) + else: + self.note('Expected:', context, offset=OFFSET) + if isinstance(exp, CallableType): + self.note(self.pretty_callable(exp), context, offset=2 * OFFSET) + else: + assert isinstance(exp, Overloaded) + for item in exp.items(): + self.note('@overload', context, offset=2 * OFFSET) + self.note(self.pretty_callable(item), context, offset=2 * OFFSET) + self.note('Got:', context, offset=OFFSET) + if isinstance(got, CallableType): + self.note(self.pretty_callable(got), context, offset=2 * OFFSET) + else: + assert isinstance(got, Overloaded) + for item in got.items(): + self.note('@overload', context, offset=2 * OFFSET) + self.note(self.pretty_callable(item), context, offset=2 * OFFSET) for name, subflags, superflags in get_all_flags(subtype, supertype): if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: - self.note('Protocol member {}.{}: expected instance variable,' + self.note('Protocol member {}.{} expected instance variable,' ' got class variable'.format(supertype.type.name(), name), context) if IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: - self.note('Protocol member {}.{}: expected class variable,' + self.note('Protocol member {}.{} expected class variable,' ' got instance variable'.format(supertype.type.name(), name), context) if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: - self.note('Protocol member {}.{}: expected settable variable,' + self.note('Protocol member {}.{} expected settable variable,' ' got read-only attribute'.format(supertype.type.name(), name), context) if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: - self.note('Protocol member {}.{}: expected class or static method' + self.note('Protocol member {}.{} expected class or static method' .format(supertype.type.name(), name), context) + def pretty_callable(self, tp: CallableType) -> str: + s = '' + bare_asterisk = False + for i in range(len(tp.arg_types)): + if len(tp.arg_types) > 5: + s += '\n' + if s != '': + s += ', ' + if tp.arg_kinds[i] in (ARG_NAMED, ARG_NAMED_OPT) and not bare_asterisk: + s += '*, ' + bare_asterisk = True + if tp.arg_kinds[i] == ARG_STAR: + s += '*' + if tp.arg_kinds[i] == ARG_STAR2: + s += '**' + name = tp.arg_names[i] + if name: + s += name + ': ' + s += self.format(tp.arg_types[i]).replace('"', '') + if tp.arg_kinds[i] in (ARG_OPT, ARG_NAMED_OPT): + s += ' =' + + # If we got a "special arg" (i.e: self, cls, etc...), prepend it to the arg list + if tp.definition is not None and tp.definition.name() is not None: + definition_args = getattr(tp.definition, 'arg_names') + if definition_args and tp.arg_names != definition_args \ + and len(definition_args) > 0: + special_arg = definition_args[0] + if s != '': + s = ', ' + s + s = special_arg + s + s = '{}({})'.format(tp.definition.name(), s) + else: + s = '({})'.format(s) + + s += ' -> {}'.format(self.format(tp.ret_type).replace('"', '')) + if tp.variables: + s = '{} {}'.format(tp.variables, s) + return 'def {}'.format(s) + def variance_string(variance: int) -> str: if variance == COVARIANT: diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 8d026e3dd50b..03bf5bd9e9f5 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1700,7 +1700,6 @@ a = {'a': 1} b = {'z': 26, **a} c = {**b} d = {**a, **b, 'c': 3} -# These errors changed because Mapping is a protocol e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type Dict[str, int]; expected Mapping[int, str] f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type Dict[str, int] [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 814bb075c834..6d5bdfb186fb 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -527,11 +527,18 @@ class C: def attr2(self) -> str: pass var: P[Union[int, P], Union[P, str]] = C() # OK for covariant -var2: P[Union[str, P], Union[P, int]] = C() # E: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) \ - # N: Following member(s) of "C" have conflicts: \ - # N: attr1: expected Callable[[], Union[str, P[Any, Any]]], got Callable[[], int] \ - # N: attr2: expected Callable[[], Union[P[Any, Any], int]], got Callable[[], str] +var2: P[Union[str, P], Union[P, int]] = C() [out] +main:18: error: Incompatible types in assignment (expression has type "C", variable has type P[Union[str, P[Any, Any]], Union[P[Any, Any], int]]) +main:18: note: Following member(s) of "C" have conflicts: +main:18: note: Expected: +main:18: note: def attr1(self) -> Union[str, P[Any, Any]] +main:18: note: Got: +main:18: note: def attr1(self) -> int +main:18: note: Expected: +main:18: note: def attr2(self) -> Union[P[Any, Any], int] +main:18: note: Got: +main:18: note: def attr2(self) -> str [case testSelfTypesWithProtocolsBehaveAsWithNominal] from typing import Protocol, TypeVar @@ -559,15 +566,24 @@ s: Shape f(NonProtoShape()) f(Circle()) -s = Triangle() # E: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") \ - # N: Following member(s) of "Triangle" have conflicts: \ - # N: combine: expected Callable[[Triangle], Triangle], got Callable[[Shape], Shape] -s = Bad() # E: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") \ - # N: Following member(s) of "Bad" have conflicts: \ - # N: combine: expected Callable[[Bad], Bad], got Callable[[int], str] - -n2: NonProtoShape = s # E: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") +s = Triangle() +s = Bad() + +n2: NonProtoShape = s [out] +main:26: error: Incompatible types in assignment (expression has type "Triangle", variable has type "Shape") +main:26: note: Following member(s) of "Triangle" have conflicts: +main:26: note: Expected: +main:26: note: def combine(self, other: Triangle) -> Triangle +main:26: note: Got: +main:26: note: def combine(self, other: Shape) -> Shape +main:27: error: Incompatible types in assignment (expression has type "Bad", variable has type "Shape") +main:27: note: Following member(s) of "Bad" have conflicts: +main:27: note: Expected: +main:27: note: def combine(self, other: Bad) -> Bad +main:27: note: Got: +main:27: note: def combine(self, other: int) -> str +main:29: error: Incompatible types in assignment (expression has type "Shape", variable has type "NonProtoShape") [case testBadVarianceInProtocols] from typing import Protocol, TypeVar @@ -751,10 +767,10 @@ y: PClass x = CInst() x = CClass() # E: Incompatible types in assignment (expression has type "CClass", variable has type "PInst") \ - # N: Protocol member PInst.v: expected instance variable, got class variable + # N: Protocol member PInst.v expected instance variable, got class variable y = CClass() y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass") \ - # N: Protocol member PClass.v: expected class variable, got instance variable + # N: Protocol member PClass.v expected class variable, got instance variable [out] [case testPropertyInProtocols] @@ -775,7 +791,7 @@ y = x x2: P y2: PP x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P") \ - # N: Protocol member P.attr: expected settable variable, got read-only attribute + # N: Protocol member P.attr expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -813,7 +829,7 @@ y3 = z3 y4: PP z4: PPS z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS") \ - # N: Protocol member PPS.attr: expected settable variable, got read-only attribute + # N: Protocol member PPS.attr expected settable variable, got read-only attribute [builtins fixtures/property.pyi] [out] @@ -845,7 +861,7 @@ x = B() y: PC y = B() y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC") \ - # N: Protocol member PC.meth: expected class or static method + # N: Protocol member PC.meth expected class or static method [builtins fixtures/classmethod.pyi] [out] @@ -867,10 +883,17 @@ class D: pass x: P = C() -x = D() # E: Incompatible types in assignment (expression has type "D", variable has type "P") \ - # N: Following member(s) of "D" have conflicts: \ - # N: f: expected overloaded function, got Callable[[int], None] +x = D() [out] +main:18: error: Incompatible types in assignment (expression has type "D", variable has type "P") +main:18: note: Following member(s) of "D" have conflicts: +main:18: note: Expected: +main:18: note: @overload +main:18: note: def f(self, x: int) -> int +main:18: note: @overload +main:18: note: def f(self, x: str) -> str +main:18: note: Got: +main:18: note: def f(self, x: int) -> None -- Join and meet with protocol types -- --------------------------------- @@ -1482,7 +1505,7 @@ x: P = C() # E: Incompatible types in assignment (expression has type "C", varia # N: Following member(s) of "C" have conflicts: \ # N: attr1: expected "int", got "str" \ # N: attr2: expected "str", got "int" \ - # N: Protocol member P.attr2: expected settable variable, got read-only attribute + # N: Protocol member P.attr2 expected settable variable, got read-only attribute def f(x: P) -> P: return C() # E: Incompatible return value type (got "C", expected "P") \ @@ -1491,7 +1514,7 @@ def f(x: P) -> P: # N: Following member(s) of "C" have conflicts: \ # N: attr1: expected "int", got "str" \ # N: attr2: expected "str", got "int" \ - # N: Protocol member P.attr2: expected settable variable, got read-only attribute + # N: Protocol member P.attr2 expected settable variable, got read-only attribute f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ # N: 'C' is missing following 'P' protocol member: \ @@ -1499,7 +1522,7 @@ f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ # N: Following member(s) of "C" have conflicts: \ # N: attr1: expected "int", got "str" \ # N: attr2: expected "str", got "int" \ - # N: Protocol member P.attr2: expected settable variable, got read-only attribute + # N: Protocol member P.attr2 expected settable variable, got read-only attribute [builtins fixtures/list.pyi] [out] @@ -1563,7 +1586,7 @@ def fun_p(x: PP) -> None: reveal_type(P.attr) # E: Revealed type is 'builtins.int' fun(C()) # E: Argument 1 to "fun" has incompatible type "C"; expected "P" \ - # N: Protocol member P.attr: expected settable variable, got read-only attribute + # N: Protocol member P.attr expected settable variable, got read-only attribute fun(C2()) fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" \ # N: Following member(s) of "D" have conflicts: \ From 9ca98b2fe36524cf0f2f15ff357e0fe663d62129 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 16:50:12 +0200 Subject: [PATCH 088/117] Truncate output (max 2 conflicts, max 2 overloads), better formatting for typevars --- mypy/messages.py | 37 +++++++-- mypy/semanal.py | 3 +- test-data/unit/check-protocols.test | 119 ++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 8 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 3eb4045e5a1b..6be53f668b26 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -991,12 +991,14 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp OFFSET = 4 # Four spaces, so that notes will look like this: # note: 'Cls' is missing following 'Proto' members: # note: method, attr + MAX_ITEMS = 2 # Maximum number of conflicts, missing members, and overloads shown if isinstance(subtype, TupleType): if not isinstance(subtype.fallback, Instance): return subtype = subtype.fallback missing = get_missing_members(subtype, supertype) - if missing and len(missing) < len(supertype.type.protocol_members) and len(missing) < 3: + if (missing and len(missing) < len(supertype.type.protocol_members) and + len(missing) <= MAX_ITEMS): self.note("'{}' is missing following '{}' protocol member{}:" .format(subtype.type.name(), supertype.type.name(), 's' if len(missing) > 1 else ''), @@ -1008,7 +1010,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp not supertype.type.defn.type_vars): self.note('Following member(s) of {} have ' 'conflicts:'.format(self.format(subtype)), context) - for name, got, exp in conflict_types: + for name, got, exp in conflict_types[:MAX_ITEMS]: if (not isinstance(exp, (CallableType, Overloaded)) or not isinstance(exp, (CallableType, Overloaded))): self.note('{}: expected {}, got {}'.format(name, @@ -1020,17 +1022,28 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp self.note(self.pretty_callable(exp), context, offset=2 * OFFSET) else: assert isinstance(exp, Overloaded) - for item in exp.items(): + for item in exp.items()[:MAX_ITEMS]: self.note('@overload', context, offset=2 * OFFSET) self.note(self.pretty_callable(item), context, offset=2 * OFFSET) + if len(exp.items()) > MAX_ITEMS: + self.note('<{} more overload(s) not shown>' + .format(len(exp.items()) - MAX_ITEMS), + context, offset=2 * OFFSET) self.note('Got:', context, offset=OFFSET) if isinstance(got, CallableType): self.note(self.pretty_callable(got), context, offset=2 * OFFSET) else: assert isinstance(got, Overloaded) - for item in got.items(): + for item in got.items()[:MAX_ITEMS]: self.note('@overload', context, offset=2 * OFFSET) self.note(self.pretty_callable(item), context, offset=2 * OFFSET) + if len(got.items()) > MAX_ITEMS: + self.note('<{} more overload(s) not shown>' + .format(len(got.items()) - MAX_ITEMS), + context, offset=2 * OFFSET) + if len(conflict_types) > MAX_ITEMS: + self.note('<{} more conflict(s) not shown>'.format(len(conflict_types) - MAX_ITEMS), + context, offset=OFFSET) for name, subflags, superflags in get_all_flags(subtype, supertype): if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note('Protocol member {}.{} expected instance variable,' @@ -1049,8 +1062,6 @@ def pretty_callable(self, tp: CallableType) -> str: s = '' bare_asterisk = False for i in range(len(tp.arg_types)): - if len(tp.arg_types) > 5: - s += '\n' if s != '': s += ', ' if tp.arg_kinds[i] in (ARG_NAMED, ARG_NAMED_OPT) and not bare_asterisk: @@ -1082,7 +1093,19 @@ def pretty_callable(self, tp: CallableType) -> str: s += ' -> {}'.format(self.format(tp.ret_type).replace('"', '')) if tp.variables: - s = '{} {}'.format(tp.variables, s) + tvars = [] + for tvar in tp.variables: + if (tvar.upper_bound and isinstance(tvar.upper_bound, Instance) and + tvar.upper_bound.type.fullname() != 'builtins.object'): + tvars.append('{} <: {}'.format(tvar.name, + self.format(tvar.upper_bound).replace('"', ''))) + elif tvar.values: + tvars.append('{} in ({})' + .format(tvar.name, ', '.join([self.format(tp).replace('"', '') + for tp in tvar.values]))) + else: + tvars.append(tvar.name) + s = '[{}] {}'.format(', '.join(tvars), s) return 'def {}'.format(s) diff --git a/mypy/semanal.py b/mypy/semanal.py index cf94b5fda3f0..422a786301fe 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -545,7 +545,8 @@ def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: assert defn.impl is defn.items[-1] defn.items = defn.items[:-1] - elif not self.is_stub_file and not non_overload_indexes: + elif (not self.is_stub_file and not non_overload_indexes and + not (self.type and self.type.is_protocol)): self.fail( "An overloaded function outside a stub file must have an implementation", defn) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 6d5bdfb186fb..5d91c23e9b58 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1614,3 +1614,122 @@ x = D() # E: Incompatible types in assignment (expression has type "D", variable x = C() # OK [builtins fixtures/list.pyi] [out] + +[case testProtocolIncompatibilityWithGenericMethod] +from typing import Protocol, TypeVar + +T = TypeVar('T') +S = TypeVar('S') + +class A(Protocol): + def f(self, x: T) -> None: pass +class B: + def f(self, x: S, y: T) -> None: pass + +x: A = B() +[out] +main:11: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:11: note: Following member(s) of "B" have conflicts: +main:11: note: Expected: +main:11: note: def [T] f(self, x: T) -> None +main:11: note: Got: +main:11: note: def [S, T] f(self, x: S, y: T) -> None + +[case testProtocolIncompatibilityWithGenericMethodBounded] +from typing import Protocol, TypeVar + +T = TypeVar('T') +S = TypeVar('S', bound=int) + +class A(Protocol): + def f(self, x: T) -> None: pass +class B: + def f(self, x: S, y: T) -> None: pass + +x: A = B() +[out] +main:11: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:11: note: Following member(s) of "B" have conflicts: +main:11: note: Expected: +main:11: note: def [T] f(self, x: T) -> None +main:11: note: Got: +main:11: note: def [S <: int, T] f(self, x: S, y: T) -> None + +[case testProtocolIncompatibilityWithGenericRestricted] +from typing import Protocol, TypeVar + +T = TypeVar('T') +S = TypeVar('S', int, str) + +class A(Protocol): + def f(self, x: T) -> None: pass +class B: + def f(self, x: S, y: T) -> None: pass + +x: A = B() +[out] +main:11: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:11: note: Following member(s) of "B" have conflicts: +main:11: note: Expected: +main:11: note: def [T] f(self, x: T) -> None +main:11: note: Got: +main:11: note: def [S in (int, str), T] f(self, x: S, y: T) -> None + +[case testProtocolIncompatibilityWithManyOverloads] +from typing import Protocol, overload + +class C1: pass +class C2: pass +class A(Protocol): + @overload + def f(self, x: int) -> int: pass + @overload + def f(self, x: str) -> str: pass + @overload + def f(self, x: C1) -> C2: pass + @overload + def f(self, x: C2) -> C1: pass + +class B: + def f(self) -> None: pass + +x: A = B() +[out] +main:18: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:18: note: Following member(s) of "B" have conflicts: +main:18: note: Expected: +main:18: note: @overload +main:18: note: def f(self, x: int) -> int +main:18: note: @overload +main:18: note: def f(self, x: str) -> str +main:18: note: <2 more overload(s) not shown> +main:18: note: Got: +main:18: note: def f(self) -> None + +[case testProtocolIncompatibilityWithManyConflicts] +from typing import Protocol + +class A(Protocol): + def f(self, x: int) -> None: pass + def g(self, x: int) -> None: pass + def h(self, x: int) -> None: pass + def i(self, x: int) -> None: pass +class B: + def f(self, x: str) -> None: pass + def g(self, x: str) -> None: pass + def h(self, x: str) -> None: pass + def i(self, x: str) -> None: pass + +x: A = B() +[out] +main:14: error: Incompatible types in assignment (expression has type "B", variable has type "A") +main:14: note: Following member(s) of "B" have conflicts: +main:14: note: Expected: +main:14: note: def f(self, x: int) -> None +main:14: note: Got: +main:14: note: def f(self, x: str) -> None +main:14: note: Expected: +main:14: note: def g(self, x: int) -> None +main:14: note: Got: +main:14: note: def g(self, x: str) -> None +main:14: note: <2 more conflict(s) not shown> From 0cb13b7681c49e4ea6affbfc0805ebe0f7768af9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 16:53:37 +0200 Subject: [PATCH 089/117] Fix lint --- mypy/messages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/messages.py b/mypy/messages.py index 6be53f668b26..a2c7b101fe23 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1042,7 +1042,8 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp .format(len(got.items()) - MAX_ITEMS), context, offset=2 * OFFSET) if len(conflict_types) > MAX_ITEMS: - self.note('<{} more conflict(s) not shown>'.format(len(conflict_types) - MAX_ITEMS), + self.note('<{} more conflict(s) not shown>' + .format(len(conflict_types) - MAX_ITEMS), context, offset=OFFSET) for name, subflags, superflags in get_all_flags(subtype, supertype): if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: From fd0640820ef7745ad760ebe3c596a97b8b3b6775 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 17:12:29 +0200 Subject: [PATCH 090/117] Factor out some common code --- mypy/messages.py | 52 +++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index a2c7b101fe23..a29534f00616 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -976,7 +976,7 @@ def concrete_only_call(self, typ: Type, context: Context) -> None: .format(self.format(typ)), context) def note_call(self, subtype: Type, call: Type, context: Context) -> None: - self.note("'{}.__call__' has type '{}'".format(self.format(subtype).replace('"', ''), + self.note("'{}.__call__' has type '{}'".format(strip_quotes(self.format(subtype)), self.format(call)), context) def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertype: Instance, @@ -992,10 +992,21 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp # note: 'Cls' is missing following 'Proto' members: # note: method, attr MAX_ITEMS = 2 # Maximum number of conflicts, missing members, and overloads shown + + def pretty_overload(tp: Overloaded) -> None: + for item in tp.items()[:MAX_ITEMS]: + self.note('@overload', context, offset=2 * OFFSET) + self.note(self.pretty_callable(item), context, offset=2 * OFFSET) + if len(tp.items()) > MAX_ITEMS: + self.note('<{} more overload(s) not shown>'.format(len(exp.items()) - MAX_ITEMS), + context, offset=2 * OFFSET) + if isinstance(subtype, TupleType): if not isinstance(subtype.fallback, Instance): return subtype = subtype.fallback + + # Report missing members missing = get_missing_members(subtype, supertype) if (missing and len(missing) < len(supertype.type.protocol_members) and len(missing) <= MAX_ITEMS): @@ -1004,6 +1015,8 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp 's' if len(missing) > 1 else ''), context) self.note(', '.join(missing), context, offset=OFFSET) + + # Report member type conflicts conflict_types = get_conflict_types(subtype, supertype) if conflict_types and (not is_subtype(subtype, erase_type(supertype)) or not subtype.type.defn.type_vars or @@ -1022,29 +1035,19 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp self.note(self.pretty_callable(exp), context, offset=2 * OFFSET) else: assert isinstance(exp, Overloaded) - for item in exp.items()[:MAX_ITEMS]: - self.note('@overload', context, offset=2 * OFFSET) - self.note(self.pretty_callable(item), context, offset=2 * OFFSET) - if len(exp.items()) > MAX_ITEMS: - self.note('<{} more overload(s) not shown>' - .format(len(exp.items()) - MAX_ITEMS), - context, offset=2 * OFFSET) + pretty_overload(exp) self.note('Got:', context, offset=OFFSET) if isinstance(got, CallableType): self.note(self.pretty_callable(got), context, offset=2 * OFFSET) else: assert isinstance(got, Overloaded) - for item in got.items()[:MAX_ITEMS]: - self.note('@overload', context, offset=2 * OFFSET) - self.note(self.pretty_callable(item), context, offset=2 * OFFSET) - if len(got.items()) > MAX_ITEMS: - self.note('<{} more overload(s) not shown>' - .format(len(got.items()) - MAX_ITEMS), - context, offset=2 * OFFSET) + pretty_overload(got) if len(conflict_types) > MAX_ITEMS: self.note('<{} more conflict(s) not shown>' .format(len(conflict_types) - MAX_ITEMS), context, offset=OFFSET) + + # Report flag conflicts (i.e. settable vs read-only etc.) for name, subflags, superflags in get_all_flags(subtype, supertype): if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note('Protocol member {}.{} expected instance variable,' @@ -1060,10 +1063,14 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertyp .format(supertype.type.name(), name), context) def pretty_callable(self, tp: CallableType) -> str: + """Return a nice easily-readable representation of a callable type. + For example: + def [T <: int] f(self, x: int, y: T) -> None + """ s = '' bare_asterisk = False for i in range(len(tp.arg_types)): - if s != '': + if s: s += ', ' if tp.arg_kinds[i] in (ARG_NAMED, ARG_NAMED_OPT) and not bare_asterisk: s += '*, ' @@ -1075,7 +1082,7 @@ def pretty_callable(self, tp: CallableType) -> str: name = tp.arg_names[i] if name: s += name + ': ' - s += self.format(tp.arg_types[i]).replace('"', '') + s += strip_quotes(self.format(tp.arg_types[i])) if tp.arg_kinds[i] in (ARG_OPT, ARG_NAMED_OPT): s += ' =' @@ -1084,25 +1091,24 @@ def pretty_callable(self, tp: CallableType) -> str: definition_args = getattr(tp.definition, 'arg_names') if definition_args and tp.arg_names != definition_args \ and len(definition_args) > 0: - special_arg = definition_args[0] - if s != '': + if s: s = ', ' + s - s = special_arg + s + s = definition_args[0] + s s = '{}({})'.format(tp.definition.name(), s) else: s = '({})'.format(s) - s += ' -> {}'.format(self.format(tp.ret_type).replace('"', '')) + s += ' -> ' + strip_quotes(self.format(tp.ret_type)) if tp.variables: tvars = [] for tvar in tp.variables: if (tvar.upper_bound and isinstance(tvar.upper_bound, Instance) and tvar.upper_bound.type.fullname() != 'builtins.object'): tvars.append('{} <: {}'.format(tvar.name, - self.format(tvar.upper_bound).replace('"', ''))) + strip_quotes(self.format(tvar.upper_bound)))) elif tvar.values: tvars.append('{} in ({})' - .format(tvar.name, ', '.join([self.format(tp).replace('"', '') + .format(tvar.name, ', '.join([strip_quotes(self.format(tp)) for tp in tvar.values]))) else: tvars.append(tvar.name) From 969a64b6822caaff9284d4e2f763b78213afe653 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Jul 2017 17:27:54 +0200 Subject: [PATCH 091/117] Remove some redundant code --- mypy/checkexpr.py | 5 ++--- mypy/semanal.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index dcb855c9b83e..2a51030845c7 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2357,9 +2357,8 @@ def check_awaitable_expr(self, t: Type, ctx: Context, msg: str) -> Type: Also used by `async for` and `async with`. """ - if not self.chk.check_subtype(t, self.chk.named_generic_type('typing.Awaitable', - [AnyType()]), - ctx, msg, 'actual type', 'expected type'): + if not self.chk.check_subtype(t, self.named_type('typing.Awaitable'), ctx, + msg, 'actual type', 'expected type'): return AnyType() else: method = self.analyze_external_member_access('__await__', t, ctx) diff --git a/mypy/semanal.py b/mypy/semanal.py index 422a786301fe..4e3d7b20ebd2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1812,7 +1812,6 @@ def analyze_member_lvalue(self, lval: MemberExpr) -> None: if self.is_self_member_ref(lval): node = self.type.get(lval.name) if node is None or isinstance(node.node, Var) and node.node.is_abstract_var: - # Protocol members can't be defined via self if self.type.is_protocol and node is None: self.fail("Protocol members cannot be defined via assignment to self", lval) else: From e93f8396f050b58860c98996787c7a1dc4916db4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Jul 2017 18:58:28 -0700 Subject: [PATCH 092/117] Fix broken merge --- mypy/messages.py | 2 +- mypy/semanal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index df7ef336cad0..73ac6220b194 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -173,7 +173,7 @@ def fail(self, msg: str, context: Context, file: Optional[str] = None, self.report(msg, context, 'error', file=file, origin=origin) def note(self, msg: str, context: Context, file: Optional[str] = None, - origin: Optional[Context] = None offset: int = 0) -> None: + origin: Optional[Context] = None, offset: int = 0) -> None: """Report a note (unless disabled).""" self.report(msg, context, 'note', file=file, origin=origin, offset=offset) diff --git a/mypy/semanal.py b/mypy/semanal.py index 45bc90140f11..2b3d6f4702d5 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1858,7 +1858,7 @@ def analyze_member_lvalue(self, lval: MemberExpr) -> None: v.is_ready = False lval.def_var = v lval.node = v - self.type.names[lval.name] = SymbolTableNode(MDEF, v) + self.type.names[lval.name] = SymbolTableNode(MDEF, v, implicit=True) self.check_lvalue_validity(lval.node, lval) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: From 9292971a0f61a7263033484a7b6cd280392a3eb2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 16 Jul 2017 20:54:21 +0200 Subject: [PATCH 093/117] First part of CR --- mypy/nodes.py | 6 ++++++ mypy/subtypes.py | 28 +++++++++++++++++++--------- test-data/unit/check-protocols.test | 4 ++-- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 852270f39266..bce801876cff 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1988,9 +1988,15 @@ class is generic then it will be a type constructor of higher kind. assuming = None # type: List[Tuple[mypy.types.Instance, mypy.types.Instance]] assuming_proper = None # type: List[Tuple[mypy.types.Instance, mypy.types.Instance]] # Ditto for temporary stack of recursive constraint inference. + # We make 'assuming' and 'inferring' attributes here instead of passing they as kwargs, + # since this would require to pass them in many dozens of calls. In particular, + # there is a dependency infer_constraint -> is_subtype -> is_callable_subtype -> + # -> infer_constraints. inferring = None # type: List[mypy.types.Instance] cache = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] cache_proper = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] + # 'inferring' and 'assumig' can't be also made sets, since we need to use + # is_same_type to correctly treat unions. # Classes inheriting from Enum shadow their true members with a __getattr__, so we # have to treat them as a special case. diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 9d46a8f3b980..bf23cd9a5b67 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -166,7 +166,7 @@ def visit_instance(self, left: Instance) -> bool: zip(t.args, right.args, right.type.defn.type_vars)) if nominal: right.type.cache.add((left, right)) - return True + return nominal if right.type.is_protocol and is_protocol_implementation(left, right): return True return False @@ -342,7 +342,17 @@ def pop_on_exit(stack: List[Tuple[Instance, Instance]], def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: """Check whether 'left' implements the protocol 'right'. If 'allow_any' is False, then check for a proper subtype. Treat recursive protocols by using the 'assuming' - structural subtype matrix (in sparse representation), see comment in nodes.TypeInfo. + structural subtype matrix (in sparse representation, i.e. as a list of pairs + (subtype, supertype)), see also comment in nodes.TypeInfo. When we enter a check for classes + (A, P), defined as following:: + + class P(Protocol): + def f(self) -> P: ... + class A: + def f(self) -> A: ... + + this results in A being a subtype of P without infinite recursion. On every false result, + we pop the assumption, thus avoiding an infinite recursion as well. """ assert right.type.is_protocol assuming = right.type.assuming if allow_any else right.type.assuming_proper @@ -350,13 +360,12 @@ def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True with pop_on_exit(assuming, left, right): - if right.type.protocol_members is None: - # This type has not been yet analyzed, probably a call from make_simplified_union. - return False for member in right.type.protocol_members: # nominal subtyping currently ignores '__init__' and '__new__' signatures if member in ('__init__', '__new__'): continue + # The third argiment below indicates to what self type is bound. + # We always bind self to the subtype. (Similarly to nominal types). supertype = find_member(member, right, left) assert supertype is not None subtype = find_member(member, left, left) @@ -489,18 +498,19 @@ def find_var_type(var: Var, itype: Instance, subtype: Type) -> Type: Apply type arguments from 'itype', and bind 'self' to 'subtype'. """ from mypy.checkmember import bind_self - itype = map_instance_to_supertype(itype, var.info) typ = var.type if typ is None: return AnyType() - typ = expand_type_by_instance(typ, itype) # We don't need to bind 'self' for static methods, since there is no 'self'. if isinstance(typ, FunctionLike) and not var.is_staticmethod: signature = bind_self(typ, subtype) assert isinstance(signature, CallableType) if var.is_property: - return signature.ret_type - return signature + typ = signature.ret_type + else: + typ = signature + itype = map_instance_to_supertype(itype, var.info) + typ = expand_type_by_instance(typ, itype) return typ diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 5d91c23e9b58..f36dfd241cf9 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -362,12 +362,12 @@ class B(A): pass x1: Pco[B] y1: Pco[A] x1 = y1 # E: Incompatible types in assignment (expression has type Pco[A], variable has type Pco[B]) -y1 = x1 +y1 = x1 # E: Incompatible types in assignment (expression has type Pco[B], variable has type Pco[A]) x2: Pcontra[B] y2: Pcontra[A] y2 = x2 # E: Incompatible types in assignment (expression has type Pcontra[B], variable has type Pcontra[A]) -x2 = y2 +x2 = y2 # E: Incompatible types in assignment (expression has type Pcontra[A], variable has type Pcontra[B]) x3: Pinv[B] y3: Pinv[A] From 4c3f4e25aa8ee3786410b05a89a17e03a9fc7bd3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 16 Jul 2017 22:54:54 +0200 Subject: [PATCH 094/117] Second part of CR (few more comments left) --- mypy/semanal.py | 14 ++++----- mypy/subtypes.py | 78 ++++++++++++++++++++++-------------------------- 2 files changed, 41 insertions(+), 51 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2b3d6f4702d5..3c53a929a6a2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -712,15 +712,12 @@ def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]: self.analyze_base_classes(defn) self.analyze_metaclass(defn) defn.info.is_protocol = is_protocol - runtime_protocol = False + defn.info.runtime_protocol = False for decorator in defn.decorators: - if self.analyze_class_decorator(defn, decorator): - runtime_protocol = True - + self.analyze_class_decorator(defn, decorator) self.enter_class(defn.info) yield True self.calculate_abstract_status(defn.info) - defn.info.runtime_protocol = runtime_protocol self.setup_type_promotion(defn) self.leave_class() @@ -744,10 +741,11 @@ def leave_class(self) -> None: self.locals.pop() self.type = self.type_stack.pop() - def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> bool: + def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None: decorator.accept(self) - return (isinstance(decorator, RefExpr) and - decorator.fullname in ('typing.runtime', 'typing_extensions.runtime')) + if (isinstance(decorator, RefExpr) and + decorator.fullname in ('typing.runtime', 'typing_extensions.runtime')): + defn.info.runtime_protocol = True def calculate_abstract_status(self, typ: TypeInfo) -> None: """Calculate abstract status of a class. diff --git a/mypy/subtypes.py b/mypy/subtypes.py index bf23cd9a5b67..e32554b0e7ad 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Dict, Callable, Tuple, Iterator, Set, cast +from typing import List, Optional, Dict, Callable, Tuple, Iterator, Set, Union, cast from contextlib import contextmanager from mypy.types import ( @@ -416,8 +416,8 @@ def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: assert isinstance(method, OverloadedFuncDef) dec = method.items[0] assert isinstance(dec, Decorator) - return find_var_type(dec.var, itype, subtype) - return map_method(method, itype, subtype) + return find_node_type(dec.var, itype, subtype) + return find_node_type(method, itype, subtype) else: # don't have such method, maybe variable or decorator? node = info.get(name) @@ -428,7 +428,7 @@ def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: if isinstance(v, Decorator): v = v.var if isinstance(v, Var): - return find_var_type(v, itype, subtype) + return find_node_type(v, itype, subtype) if not v and name not in ['__getattr__', '__setattr__', '__getattribute__']: for method_name in ('__getattribute__', '__getattr__'): # Normally, mypy assumes that instances that define __getattr__ have all @@ -437,7 +437,7 @@ def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: # structural subtyping. method = info.get_method(method_name) if method and method.info.fullname() != 'builtins.object': - getattr_type = map_method(method, itype, subtype) + getattr_type = find_node_type(method, itype, subtype) if isinstance(getattr_type, CallableType): return getattr_type.ret_type if itype.type.fallback_to_any: @@ -445,21 +445,6 @@ def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: return None -def get_all_flags(left: Instance, right: Instance) -> List[Tuple[str, Set[int], Set[int]]]: - """Return all attribute flags for members that are present in both - 'left' and 'right'. - """ - assert right.type.is_protocol - all_flags = [] # type: List[Tuple[str, Set[int], Set[int]]] - for member in right.type.protocol_members: - if find_member(member, left, left): - item = (member, - get_member_flags(member, left.type), - get_member_flags(member, right.type)) - all_flags.append(item) - return all_flags - - def get_member_flags(name: str, info: TypeInfo) -> Set[int]: """Detect whether a member 'name' is settable, whether it is an instance or class variable, and whether it is class or static method. @@ -493,44 +478,35 @@ def get_member_flags(name: str, info: TypeInfo) -> Set[int]: return set() -def find_var_type(var: Var, itype: Instance, subtype: Type) -> Type: - """Find type of a variable 'var' (maybe also a decorated method). +def find_node_type(node: Union[Var, FuncBase], itype: Instance, subtype: Type) -> Type: + """Find type of a variable or method 'node' (maybe also a decorated method). Apply type arguments from 'itype', and bind 'self' to 'subtype'. """ from mypy.checkmember import bind_self - typ = var.type + if isinstance(node, FuncBase): + typ = function_type(node, fallback=Instance(itype.type.mro[-1], [])) # type: Type + else: + typ = node.type if typ is None: return AnyType() # We don't need to bind 'self' for static methods, since there is no 'self'. - if isinstance(typ, FunctionLike) and not var.is_staticmethod: + if isinstance(node, FuncBase) or isinstance(typ, FunctionLike) and not node.is_staticmethod: + assert isinstance(typ, FunctionLike) signature = bind_self(typ, subtype) - assert isinstance(signature, CallableType) - if var.is_property: + if node.is_property: + assert isinstance(signature, CallableType) typ = signature.ret_type else: typ = signature - itype = map_instance_to_supertype(itype, var.info) + itype = map_instance_to_supertype(itype, node.info) typ = expand_type_by_instance(typ, itype) return typ -def map_method(method: FuncBase, itype: Instance, subtype: Type) -> Type: - """Map 'method' to the base where it was defined. Apply type arguments - from 'itype', and bind 'self' type to 'subtype'. - This function should be used only for non-decorated methods. Decorated - methods (including @staticmethod and @property) are treated - by 'find_var_type'. - """ - from mypy.checkmember import bind_self - signature = function_type(method, fallback=Instance(itype.type.mro[-1], [])) - signature = bind_self(signature, subtype) - itype = map_instance_to_supertype(itype, method.info) - return expand_type_by_instance(signature, itype) - - def get_missing_members(left: Instance, right: Instance) -> List[str]: """Find all protocol members of 'right' that are not implemented - (i.e. completely missing) in 'left'. + (i.e. completely missing) in 'left'. This is a helper to collect information + for better error reporting. """ assert right.type.is_protocol missing = [] # type: List[str] @@ -542,7 +518,8 @@ def get_missing_members(left: Instance, right: Instance) -> List[str]: def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, Type]]: """Find members that are defined in 'left' but have incompatible types. - Return them as a list of ('member', 'got', 'expected'). + Return them as a list of ('member', 'got', 'expected'). This is a helper + to collect information for better error reporting. """ assert right.type.is_protocol conflicts = [] # type: List[Tuple[str, Type, Type]] @@ -562,6 +539,21 @@ def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, return conflicts +def get_all_flags(left: Instance, right: Instance) -> List[Tuple[str, Set[int], Set[int]]]: + """Return all attribute flags for members that are present in both + 'left' and 'right'. This is a helper to collect information for better error reporting. + """ + assert right.type.is_protocol + all_flags = [] # type: List[Tuple[str, Set[int], Set[int]]] + for member in right.type.protocol_members: + if find_member(member, left, left): + item = (member, + get_member_flags(member, left.type), + get_member_flags(member, right.type)) + all_flags.append(item) + return all_flags + + def is_callable_subtype(left: CallableType, right: CallableType, ignore_return: bool = False, ignore_pos_arg_names: bool = False, From e71dff09b276fff7b105d9cf71c8acd1c13d7e81 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Jul 2017 18:42:14 +0200 Subject: [PATCH 095/117] Fix strict-optional check --- mypy/subtypes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index e32554b0e7ad..8a6f4f8fa9a2 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -484,7 +484,8 @@ def find_node_type(node: Union[Var, FuncBase], itype: Instance, subtype: Type) - """ from mypy.checkmember import bind_self if isinstance(node, FuncBase): - typ = function_type(node, fallback=Instance(itype.type.mro[-1], [])) # type: Type + typ = function_type(node, + fallback=Instance(itype.type.mro[-1], [])) # type: Optional[Type] else: typ = node.type if typ is None: From d19596443c6e15a6f69257083bdc04349384004c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Jul 2017 20:39:33 +0200 Subject: [PATCH 096/117] Don't hash most types (including syntetic) --- mypy/types.py | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 4c551f785b7f..a624beb1ed4b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -43,12 +43,6 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: def __repr__(self) -> str: return self.accept(TypeStrVisitor()) - def __hash__(self) -> int: - return hash(type(self)) - - def __eq__(self, other: object) -> bool: - return isinstance(other, type(self)) - def serialize(self) -> Union[JsonDict, str]: raise NotImplementedError('Cannot serialize {} instance'.format(self.__class__.__name__)) @@ -258,14 +252,6 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_type_list(self) - def __hash__(self) -> int: - return hash(tuple(self.items)) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, TypeList): - return NotImplemented - return self.items == other.items - def serialize(self) -> JsonDict: assert False, "Sythetic types don't serialize" @@ -325,6 +311,12 @@ def copy_modified(self, special_form=special_form, line=self.line, column=self.column) + def __hash__(self) -> int: + return hash(AnyType) + + def __eq__(self, other: object) -> bool: + return isinstance(other, AnyType) + def serialize(self) -> JsonDict: return {'.class': 'AnyType'} @@ -359,6 +351,12 @@ def __init__(self, is_noreturn: bool = False, line: int = -1, column: int = -1) def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_uninhabited_type(self) + def __hash__(self) -> int: + return hash(UninhabitedType) + + def __eq__(self, other: object) -> bool: + return isinstance(other, UninhabitedType) + def serialize(self) -> JsonDict: return {'.class': 'UninhabitedType', 'is_noreturn': self.is_noreturn} @@ -380,6 +378,12 @@ class NoneTyp(Type): def __init__(self, line: int = -1, column: int = -1) -> None: super().__init__(line, column) + def __hash__(self) -> int: + return hash(NoneTyp) + + def __eq__(self, other: object) -> bool: + return isinstance(other, NoneTyp) + def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_none_type(self) @@ -1024,16 +1028,17 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_typeddict_type(self) def __hash__(self) -> int: - return hash(tuple(self.items.items())) + return hash((frozenset(self.items.items()), self.fallback, + frozenset(self.required_keys))) def __eq__(self, other: object) -> bool: if isinstance(other, TypedDictType): - if self.items.keys() != other.items.keys(): + if frozenset(self.items.keys()) != frozenset(other.items.keys()): return False for (_, left_item_type, right_item_type) in self.zip(other): if not left_item_type == right_item_type: return False - return True + return self.fallback == other.fallback and self.required_keys == other.required_keys else: return NotImplemented @@ -1115,14 +1120,6 @@ def __init__(self, type: Type, line: int = -1, column: int = -1) -> None: self.type = type super().__init__(line, column) - def __hash__(self) -> int: - return hash(self.type) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, StarType): - return NotImplemented - return self.type == other.type - def accept(self, visitor: 'TypeVisitor[T]') -> T: assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_star_type(self) From 318bb70ec727426bad17ed09a318384f3948c2c3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Jul 2017 20:46:45 +0200 Subject: [PATCH 097/117] Correct comment in test --- test-data/unit/check-protocols.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index f36dfd241cf9..9ec3d4e5b8a7 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -346,7 +346,7 @@ from typing import TypeVar, Protocol T = TypeVar('T') -# In case of these errors we proceed with inferred variance. +# In case of these errors we proceed with declared variance. class Pco(Protocol[T]): # E: Invariant type variable 'T' used in protocol where covariant one is expected def meth(self) -> T: pass From 3d8782f2ee74aa4f3cbee9c366769e23f41969f4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Jul 2017 22:10:25 +0200 Subject: [PATCH 098/117] Improve structural support for TypedDict and minor things --- mypy/checker.py | 2 +- mypy/checkexpr.py | 2 +- mypy/constraints.py | 3 +- mypy/messages.py | 26 ++++++++++------- mypy/subtypes.py | 4 +-- test-data/unit/check-typeddict.test | 45 +++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 16 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 5e18e6def1bd..4233d6e7356c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2426,7 +2426,7 @@ def check_subtype(self, subtype: Type, supertype: Type, context: Context, if note_msg: self.note(note_msg, context) if (isinstance(supertype, Instance) and supertype.type.is_protocol and - isinstance(subtype, (Instance, TupleType))): + isinstance(subtype, (Instance, TupleType, TypedDictType))): self.msg.report_protocol_problems(subtype, supertype, context) if isinstance(supertype, CallableType) and isinstance(subtype, Instance): call = find_member('__call__', subtype, subtype) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 06b3e0467884..d478c215567c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1025,7 +1025,7 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, return messages.incompatible_argument(n, m, callee, original_caller_type, caller_kind, context) - if (isinstance(original_caller_type, (Instance, TupleType)) and + if (isinstance(original_caller_type, (Instance, TupleType, TypedDictType)) and isinstance(callee_type, Instance) and callee_type.type.is_protocol): self.msg.report_protocol_problems(original_caller_type, callee_type, context) if (isinstance(callee_type, CallableType) and diff --git a/mypy/constraints.py b/mypy/constraints.py index 2161220b3a2b..7ea54b9ca7a8 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -383,7 +383,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: cb = infer_constraints(template.args[0], item, SUPERTYPE_OF) res.extend(cb) return res - elif isinstance(actual, TupleType) and template.type.is_protocol: + elif (isinstance(actual, TupleType) and template.type.is_protocol and + self.direction == SUPERTYPE_OF): if mypy.subtypes.is_subtype(actual.fallback, erase_typevars(template)): res.extend(infer_constraints(template, actual.fallback, self.direction)) return res diff --git a/mypy/messages.py b/mypy/messages.py index 73ac6220b194..9893635d9c44 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1012,8 +1012,8 @@ def note_call(self, subtype: Type, call: Type, context: Context) -> None: self.note("'{}.__call__' has type '{}'".format(strip_quotes(self.format(subtype)), self.format(call)), context) - def report_protocol_problems(self, subtype: Union[Instance, TupleType], supertype: Instance, - context: Context) -> None: + def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDictType], + supertype: Instance, context: Context) -> None: """Report possible protocol conflicts between 'subtype' and 'supertype'. This includes missing members, incompatible types, and incompatible attribute flags, such as settable vs read-only or class variable vs @@ -1031,10 +1031,16 @@ def pretty_overload(tp: Overloaded) -> None: self.note('@overload', context, offset=2 * OFFSET) self.note(self.pretty_callable(item), context, offset=2 * OFFSET) if len(tp.items()) > MAX_ITEMS: - self.note('<{} more overload(s) not shown>'.format(len(exp.items()) - MAX_ITEMS), + self.note('<{} more overload(s) not shown>'.format(len(tp.items()) - MAX_ITEMS), context, offset=2 * OFFSET) - if isinstance(subtype, TupleType): + def print_more(conflicts: Sequence[Any]) -> None: + if len(conflicts) > MAX_ITEMS: + self.note('<{} more conflict(s) not shown>' + .format(len(conflicts) - MAX_ITEMS), + context, offset=OFFSET) + + if isinstance(subtype, (TupleType, TypedDictType)): if not isinstance(subtype.fallback, Instance): return subtype = subtype.fallback @@ -1044,8 +1050,7 @@ def pretty_overload(tp: Overloaded) -> None: if (missing and len(missing) < len(supertype.type.protocol_members) and len(missing) <= MAX_ITEMS): self.note("'{}' is missing following '{}' protocol member{}:" - .format(subtype.type.name(), supertype.type.name(), - 's' if len(missing) > 1 else ''), + .format(subtype.type.name(), supertype.type.name(), plural_s(missing)), context) self.note(', '.join(missing), context, offset=OFFSET) @@ -1075,13 +1080,11 @@ def pretty_overload(tp: Overloaded) -> None: else: assert isinstance(got, Overloaded) pretty_overload(got) - if len(conflict_types) > MAX_ITEMS: - self.note('<{} more conflict(s) not shown>' - .format(len(conflict_types) - MAX_ITEMS), - context, offset=OFFSET) + print_more(conflict_types) # Report flag conflicts (i.e. settable vs read-only etc.) - for name, subflags, superflags in get_all_flags(subtype, supertype): + conflict_flags = get_all_flags(subtype, supertype) + for name, subflags, superflags in conflict_flags[:MAX_ITEMS]: if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note('Protocol member {}.{} expected instance variable,' ' got class variable'.format(supertype.type.name(), name), context) @@ -1094,6 +1097,7 @@ def pretty_overload(tp: Overloaded) -> None: if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: self.note('Protocol member {}.{} expected class or static method' .format(supertype.type.name(), name), context) + print_more(conflict_flags) def pretty_callable(self, tp: CallableType) -> str: """Return a nice easily-readable representation of a callable type. diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 8a6f4f8fa9a2..ff9efcac57c2 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -514,7 +514,7 @@ def get_missing_members(left: Instance, right: Instance) -> List[str]: for member in right.type.protocol_members: if not find_member(member, left, left): missing.append(member) - return sorted(missing) + return missing def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, Type]]: @@ -876,7 +876,7 @@ def check_argument(leftarg: Type, rightarg: Type, variance: int) -> bool: zip(left.args, right.args, right.type.defn.type_vars)) if nominal: right.type.cache_proper.add((left, right)) - return True + return nominal if (right.type.is_protocol and is_protocol_implementation(left, right, allow_any=False)): return True diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 193c7468d442..3eae4187a55a 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -372,6 +372,51 @@ ll = [b, c] f(ll) # E: Argument 1 to "f" has incompatible type List[TypedDict({'x': int, 'z': str})]; expected "A" [builtins fixtures/dict.pyi] +[case testTypedDictWithSimpleProtocol] +from typing_extensions import Protocol +from mypy_extensions import TypedDict + +class StrIntMap(Protocol): + def __getitem__(self, key: str) -> int: ... + +A = TypedDict('A', {'x': int, 'y': int}) +B = TypedDict('B', {'x': int, 'y': str}) + +def fun(arg: StrIntMap) -> None: ... +a: A +b: B +fun(a) +fun(b) # Error +[builtins fixtures/dict.pyi] +[out] +main:14: error: Argument 1 to "fun" has incompatible type "B"; expected "StrIntMap" +main:14: note: Following member(s) of "B" have conflicts: +main:14: note: Expected: +main:14: note: def __getitem__(self, str) -> int +main:14: note: Got: +main:14: note: def __getitem__(self, str) -> object + +[case testTypedDictWithSimpleProtocolInference] +from typing_extensions import Protocol, TypeVar +from mypy_extensions import TypedDict + +T_co = TypeVar('T_co', covariant=True) +T = TypeVar('T') + +class StrMap(Protocol[T_co]): + def __getitem__(self, key: str) -> T_co: ... + +A = TypedDict('A', {'x': int, 'y': int}) +B = TypedDict('B', {'x': int, 'y': str}) + +def fun(arg: StrMap[T]) -> T: + return arg['whatever'] +a: A +b: B +reveal_type(fun(a)) # E: Revealed type is 'builtins.int*' +reveal_type(fun(b)) # E: Revealed type is 'builtins.object*' +[builtins fixtures/dict.pyi] +[out] -- Join From 6e1100c181b1bd4e464227270647f4b02b5c593e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Jul 2017 23:12:43 +0200 Subject: [PATCH 099/117] Don't show detailed erorrs for TypedDicts in simple cases --- mypy/messages.py | 4 ++++ test-data/unit/check-typeddict.test | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 9893635d9c44..121ac9635042 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1025,6 +1025,10 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict # note: 'Cls' is missing following 'Proto' members: # note: method, attr MAX_ITEMS = 2 # Maximum number of conflicts, missing members, and overloads shown + # List of special situations where we don't want to report additional problems + exclusions = {TypedDictType: ['typing.Mapping', 'typing.Dict']} + if supertype.type.fullname() in exclusions[type(subtype)]: + return def pretty_overload(tp: Overloaded) -> None: for item in tp.items()[:MAX_ITEMS]: diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 3eae4187a55a..b6e418f17d63 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -343,7 +343,9 @@ Point = TypedDict('Point', {'x': int, 'y': int}) def as_dict(p: Point) -> Dict[str, int]: return p # E: Incompatible return value type (got "Point", expected Dict[str, int]) def as_mutable_mapping(p: Point) -> MutableMapping[str, int]: - return p # E: Incompatible return value type (got "Point", expected MutableMapping[str, int]) + return p # E: Incompatible return value type (got "Point", expected MutableMapping[str, int]) \ + # N: 'Point' is missing following 'MutableMapping' protocol member: \ + # N: __setitem__ [builtins fixtures/dict.pyi] [case testCanConvertTypedDictToAny] @@ -1177,9 +1179,16 @@ def f(x: int) -> None: ... def f(x): pass a: A -f(a) # E: Argument 1 to "f" has incompatible type "A"; expected Iterable[int] +f(a) [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] +[out] +main:13: error: Argument 1 to "f" has incompatible type "A"; expected Iterable[int] +main:13: note: Following member(s) of "A" have conflicts: +main:13: note: Expected: +main:13: note: def __iter__(self) -> Iterator[int] +main:13: note: Got: +main:13: note: def __iter__(self) -> Iterator[str] [case testTypedDictOverloading3] from typing import overload From 5bfa3b28d0b3347048f80521843aec01e7097a97 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 17 Jul 2017 23:51:06 +0200 Subject: [PATCH 100/117] Add also an exception for NamedTuples --- mypy/messages.py | 4 +++- test-data/unit/check-protocols.test | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mypy/messages.py b/mypy/messages.py index 121ac9635042..c884593da664 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1026,7 +1026,9 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict # note: method, attr MAX_ITEMS = 2 # Maximum number of conflicts, missing members, and overloads shown # List of special situations where we don't want to report additional problems - exclusions = {TypedDictType: ['typing.Mapping', 'typing.Dict']} + exclusions = {TypedDictType: ['typing.Mapping', 'typing.Dict'], + TupleType: ['typing.Iterable', 'typing.Sequence'], + Instance: []} # type: Dict[type, List[str]] if supertype.type.fullname() in exclusions[type(subtype)]: return diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 9ec3d4e5b8a7..16a09941d86f 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1733,3 +1733,18 @@ main:14: note: def g(self, x: int) -> None main:14: note: Got: main:14: note: def g(self, x: str) -> None main:14: note: <2 more conflict(s) not shown> + +[case testDontShowNotesForTupleAndIterableProtocol] +from typing import Iterable, Sequence, Protocol, NamedTuple + +class N(NamedTuple): + x: int + +def f1(x: Iterable[str]) -> None: pass +def f2(x: Sequence[str]) -> None: pass + +# The errors below should be short +f1(N(1)) # E: Argument 1 to "f1" has incompatible type "N"; expected Iterable[str] +f2(N(2)) # E: Argument 1 to "f2" has incompatible type "N"; expected Sequence[str] +[builtins fixtures/tuple.pyi] +[out] From 82f01b70ca6b2d1fed70b66714048b4a36c3374d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Jul 2017 01:52:55 +0200 Subject: [PATCH 101/117] Add one more test --- test-data/unit/check-protocols.test | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 16a09941d86f..217282ae9394 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1748,3 +1748,30 @@ f1(N(1)) # E: Argument 1 to "f1" has incompatible type "N"; expected Iterable[s f2(N(2)) # E: Argument 1 to "f2" has incompatible type "N"; expected Sequence[str] [builtins fixtures/tuple.pyi] [out] + +[case testNotManyFlagConflitsShownInProtocols] +from typing import Protocol + +class AllSettable(Protocol): + a: int + b: int + c: int + d: int + +class AllReadOnly: + @property + def a(self) -> int: pass + @property + def b(self) -> int: pass + @property + def c(self) -> int: pass + @property + def d(self) -> int: pass + +x: AllSettable = AllReadOnly() +[builtins fixtures/property.pyi] +[out] +main:19: error: Incompatible types in assignment (expression has type "AllReadOnly", variable has type "AllSettable") +main:19: note: Protocol member AllSettable.a expected settable variable, got read-only attribute +main:19: note: Protocol member AllSettable.b expected settable variable, got read-only attribute +main:19: note: <2 more conflict(s) not shown> From c7304bdf525b5529b7a9533e003b03ea187f54f6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Jul 2017 01:58:02 +0200 Subject: [PATCH 102/117] Dict is not a protocol --- mypy/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/messages.py b/mypy/messages.py index c884593da664..b3f75b7f2c12 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1026,7 +1026,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict # note: method, attr MAX_ITEMS = 2 # Maximum number of conflicts, missing members, and overloads shown # List of special situations where we don't want to report additional problems - exclusions = {TypedDictType: ['typing.Mapping', 'typing.Dict'], + exclusions = {TypedDictType: ['typing.Mapping'], TupleType: ['typing.Iterable', 'typing.Sequence'], Instance: []} # type: Dict[type, List[str]] if supertype.type.fullname() in exclusions[type(subtype)]: From 58e6b360ba0047136f93bb89c94698b9fa09fac8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Jul 2017 11:02:56 +0200 Subject: [PATCH 103/117] Two minor tweaks: more precise isinstance, require explicit types for protocol members --- mypy/semanal.py | 8 +++++--- mypy/subtypes.py | 3 ++- test-data/unit/check-protocols.test | 7 +++---- test-data/unit/pythoneval.test | 5 +++++ typeshed | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 3c53a929a6a2..1a5c348c8b56 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1604,9 +1604,11 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: if isinstance(lval.node, Var): lval.node.is_abstract_var = True else: - # Set the type if the rvalue is a simple literal. - if (s.type is None and len(s.lvalues) == 1 and - isinstance(s.lvalues[0], NameExpr)): + if (any(isinstance(lv, NameExpr) and lv.is_def for lv in s.lvalues) and + self.type and self.type.is_protocol and not self.is_func_scope()): + self.fail('All protocol members must have explicitly declared types', s) + # Set the type if the rvalue is a simple literal (even if the above error occurred). + if len(s.lvalues) == 1 and isinstance(s.lvalues[0], NameExpr): if s.lvalues[0].is_def: s.type = self.analyze_simple_literal_type(s.rvalue) if s.type: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ff9efcac57c2..ada3c2ac6b25 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -802,7 +802,8 @@ def restrict_subtype_away(t: Type, s: Type) -> Type: # Since runtime type checks will ignore type arguments, erase the types. erased_s = erase_type(s) new_items = [item for item in t.relevant_items() - if (not is_proper_subtype(erase_type(item), erased_s) + if (not (is_proper_subtype(erase_type(item), erased_s) or + is_proper_subtype(item, erased_s)) or isinstance(item, AnyType))] return UnionType.make_union(new_items) else: diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 217282ae9394..772d572b65b7 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1289,8 +1289,7 @@ x: Union[C1[int], C2] if isinstance(x, P1): reveal_type(x) # E: Revealed type is '__main__.C1[builtins.int]' else: - # This happens because of rules in restrict_subtype_away, maybe we can relax rules for protocols? - reveal_type(x) # E: Revealed type is 'Union[__main__.C1[builtins.int], __main__.C2]' + reveal_type(x) # E: Revealed type is '__main__.C2' if isinstance(x, P2): reveal_type(x) # E: Revealed type is '__main__.C2' @@ -1449,7 +1448,7 @@ foo('no way') [out] main:11: error: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" --- Additional test and corner cases for protocols +-- Additional tests and corner cases for protocols -- ---------------------------------------------- [case testAnyWithProtocols] @@ -1599,7 +1598,7 @@ fun_p(C()) # OK from typing import Protocol class P(Protocol): - x = 1 # We could actually prohibit inferred types in protocol declarations + x = 1 # E: All protocol members must have explicitly declared types class C: x: int diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 9ec9997f0169..095e8f3612da 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1394,3 +1394,8 @@ o: object = p it2: Iterable[int] = p [out] _testCanConvertTypedDictToAnySuperclassOfMapping.py:11: error: Incompatible types in assignment (expression has type "Point", variable has type Iterable[int]) +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Following member(s) of "Point" have conflicts: +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Expected: +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: def __iter__(self) -> Iterator[int] +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Got: +_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: def __iter__(self) -> Iterator[str] diff --git a/typeshed b/typeshed index 47b3979a839f..2e36717aa899 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 47b3979a839f5b30ee3462cb4ffaa6bc4e788dcb +Subproject commit 2e36717aa89989e0c5050035f4f572dfbea389f9 From eefe88169153a025766d04fbc91097b173244694 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Jul 2017 11:09:47 +0200 Subject: [PATCH 104/117] Restore typeshed commit --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 2e36717aa899..47b3979a839f 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 2e36717aa89989e0c5050035f4f572dfbea389f9 +Subproject commit 47b3979a839f5b30ee3462cb4ffaa6bc4e788dcb From 96a04ae14ba699ad393c852790435c15d25f9164 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Jul 2017 11:30:52 +0200 Subject: [PATCH 105/117] pythoneval tests should wait until typeshed PR is merged --- test-data/unit/pythoneval.test | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 095e8f3612da..9ec9997f0169 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1394,8 +1394,3 @@ o: object = p it2: Iterable[int] = p [out] _testCanConvertTypedDictToAnySuperclassOfMapping.py:11: error: Incompatible types in assignment (expression has type "Point", variable has type Iterable[int]) -_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Following member(s) of "Point" have conflicts: -_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Expected: -_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: def __iter__(self) -> Iterator[int] -_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: Got: -_testCanConvertTypedDictToAnySuperclassOfMapping.py:11: note: def __iter__(self) -> Iterator[str] From 8b6006c753b84cd619fa7d2f9ec1943a5331f7bc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 28 Jul 2017 21:14:25 +0200 Subject: [PATCH 106/117] Fix a silly mistake in merge --- test-data/unit/fixtures/isinstancelist.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/fixtures/isinstancelist.pyi b/test-data/unit/fixtures/isinstancelist.pyi index 7732beed8077..99aca1befe39 100644 --- a/test-data/unit/fixtures/isinstancelist.pyi +++ b/test-data/unit/fixtures/isinstancelist.pyi @@ -23,7 +23,7 @@ T = TypeVar('T') KT = TypeVar('KT') VT = TypeVar('VT') -class tuple(Generic[T]): pass +class tuple(Generic[T]): def __len__(self) -> int: pass class list(Generic[T]): From f76349b5b4c8bd4d17d503790aa6e0cac80141fa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 28 Jul 2017 22:00:52 +0200 Subject: [PATCH 107/117] Replace one last mention of typing with typing_extensions --- docs/source/class_basics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 3325c6c82bcb..011d6af4942a 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -329,8 +329,8 @@ Using ``isinstance()`` with protocols ************************************* To use a protocol class with ``isinstance()``, one needs to decorate it with -a special ``typing.runtime`` decorator. It will add support for basic runtime -structural checks: +a special ``typing_extensions.runtime`` decorator. It will add support for +basic runtime structural checks: .. code-block:: python From 91fe9fd60f197a03e00cf745c715f563767462ae Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 10 Aug 2017 18:34:03 +0200 Subject: [PATCH 108/117] Add TypeOfAny --- mypy/checker.py | 6 ++++-- mypy/semanal.py | 2 +- mypy/subtypes.py | 4 ++-- test-data/unit/check-typeddict.test | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 7080dcb2241d..16329871b0bc 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1172,8 +1172,10 @@ def check_protocol_variance(self, defn: ClassDef) -> None: object_type = Instance(info.mro[-1], []) tvars = info.defn.type_vars for i, tvar in enumerate(tvars): - up_args = [object_type if i == j else AnyType() for j, _ in enumerate(tvars)] - down_args = [UninhabitedType() if i == j else AnyType() for j, _ in enumerate(tvars)] + up_args = [object_type if i == j else AnyType(TypeOfAny.special_form) + for j, _ in enumerate(tvars)] + down_args = [UninhabitedType() if i == j else AnyType(TypeOfAny.special_form) + for j, _ in enumerate(tvars)] up, down = Instance(info, up_args), Instance(info, down_args) # TODO: add advanced variance checks for recursive protocols if is_subtype(down, up, ignore_declared_variance=True): diff --git a/mypy/semanal.py b/mypy/semanal.py index e3f5e2bb9448..fb63b96850a9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3828,7 +3828,7 @@ def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) - # We are running tests without 'bool' in builtins. # TODO: Find a permanent solution to this problem. # Maybe add 'bool' to all fixtures? - literal_types.append(('True', AnyType())) + literal_types.append(('True', AnyType(TypeOfAny.special_form))) for name, typ in literal_types: v = Var(name, typ) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 648c6cef6034..d6abfd81ef2a 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -441,7 +441,7 @@ def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: if isinstance(getattr_type, CallableType): return getattr_type.ret_type if itype.type.fallback_to_any: - return AnyType() + return AnyType(TypeOfAny.special_form) return None @@ -489,7 +489,7 @@ def find_node_type(node: Union[Var, FuncBase], itype: Instance, subtype: Type) - else: typ = node.type if typ is None: - return AnyType() + return AnyType(TypeOfAny.from_error) # We don't need to bind 'self' for static methods, since there is no 'self'. if isinstance(node, FuncBase) or isinstance(typ, FunctionLike) and not node.is_staticmethod: assert isinstance(typ, FunctionLike) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index b6e418f17d63..a13d1e704f41 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -399,8 +399,9 @@ main:14: note: Got: main:14: note: def __getitem__(self, str) -> object [case testTypedDictWithSimpleProtocolInference] -from typing_extensions import Protocol, TypeVar +from typing_extensions import Protocol from mypy_extensions import TypedDict +from typing import TypeVar T_co = TypeVar('T_co', covariant=True) T = TypeVar('T') From aaea344ac7b7bdeb620d4882fe11982a4581a9f8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 10 Aug 2017 18:42:04 +0200 Subject: [PATCH 109/117] Remove some ocassionally added files --- tstc.py | 2 -- tstd.py | 2 -- tste.py | 4 ---- 3 files changed, 8 deletions(-) delete mode 100644 tstc.py delete mode 100644 tstd.py delete mode 100644 tste.py diff --git a/tstc.py b/tstc.py deleted file mode 100644 index bf413be60b1a..000000000000 --- a/tstc.py +++ /dev/null @@ -1,2 +0,0 @@ -from typing import List -Alias = int diff --git a/tstd.py b/tstd.py deleted file mode 100644 index 6bee95a8b4f6..000000000000 --- a/tstd.py +++ /dev/null @@ -1,2 +0,0 @@ -from tstc import Alias -x: Alias diff --git a/tste.py b/tste.py deleted file mode 100644 index 52dfd822ccc0..000000000000 --- a/tste.py +++ /dev/null @@ -1,4 +0,0 @@ -from tstd import Alias -x: Alias - -1+1 From 27c3e36d7908b4dce97490c3a772f819be2a61a4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 10 Aug 2017 20:20:45 +0200 Subject: [PATCH 110/117] Limit unnecessary conflicts display and fix conflict count (with test) --- mypy/messages.py | 7 +++++-- mypy/subtypes.py | 13 ++++++++++--- test-data/unit/check-protocols.test | 22 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index b24d0d890a72..5d5cfc63f86f 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1026,7 +1026,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict attribute flags, such as settable vs read-only or class variable vs instance variable. """ - from mypy.subtypes import (get_missing_members, get_conflict_types, get_all_flags, + from mypy.subtypes import (get_missing_members, get_conflict_types, get_bad_flags, is_subtype, IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC) OFFSET = 4 # Four spaces, so that notes will look like this: # note: 'Cls' is missing following 'Proto' members: @@ -1066,6 +1066,9 @@ def print_more(conflicts: Sequence[Any]) -> None: .format(subtype.type.name(), supertype.type.name(), plural_s(missing)), context) self.note(', '.join(missing), context, offset=OFFSET) + elif len(missing) > MAX_ITEMS or len(missing) == len(supertype.type.protocol_members): + # this is an obviously wrong type: too many missing members + return # Report member type conflicts conflict_types = get_conflict_types(subtype, supertype) @@ -1096,7 +1099,7 @@ def print_more(conflicts: Sequence[Any]) -> None: print_more(conflict_types) # Report flag conflicts (i.e. settable vs read-only etc.) - conflict_flags = get_all_flags(subtype, supertype) + conflict_flags = get_bad_flags(subtype, supertype) for name, subflags, superflags in conflict_flags[:MAX_ITEMS]: if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note('Protocol member {}.{} expected instance variable,' diff --git a/mypy/subtypes.py b/mypy/subtypes.py index d6abfd81ef2a..10fa4cf1c711 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -540,8 +540,8 @@ def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, return conflicts -def get_all_flags(left: Instance, right: Instance) -> List[Tuple[str, Set[int], Set[int]]]: - """Return all attribute flags for members that are present in both +def get_bad_flags(left: Instance, right: Instance) -> List[Tuple[str, Set[int], Set[int]]]: + """Return all incompatible attribute flags for members that are present in both 'left' and 'right'. This is a helper to collect information for better error reporting. """ assert right.type.is_protocol @@ -552,7 +552,14 @@ def get_all_flags(left: Instance, right: Instance) -> List[Tuple[str, Set[int], get_member_flags(member, left.type), get_member_flags(member, right.type)) all_flags.append(item) - return all_flags + bad_flags = [] + for name, subflags, superflags in all_flags: + if (IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags or + IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags or + IS_SETTABLE in superflags and IS_SETTABLE not in subflags or + IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags): + bad_flags.append((name, subflags, superflags)) + return bad_flags def is_callable_subtype(left: CallableType, right: CallableType, diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 772d572b65b7..376eb08e6f09 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1774,3 +1774,25 @@ main:19: error: Incompatible types in assignment (expression has type "AllReadOn main:19: note: Protocol member AllSettable.a expected settable variable, got read-only attribute main:19: note: Protocol member AllSettable.b expected settable variable, got read-only attribute main:19: note: <2 more conflict(s) not shown> + +[case testProtocolsMoreConflictsNotShown] +from typing_extensions import Protocol +from typing import Generic, TypeVar + +T = TypeVar('T') + +class MockMapping(Protocol[T]): + def a(self, x: T) -> int: pass + def b(self, x: T) -> int: pass + def c(self, x: T) -> int: pass + d: T + e: T + f: T + +class MockDict(MockMapping[T]): + more: int + +def f(x: MockMapping[int]) -> None: pass +x: MockDict[str] +f(x) # E: Argument 1 to "f" has incompatible type MockDict[str]; expected MockMapping[int] +[out] From cfa539dbdea98e8300d9c1b6d06c3c8e27e411bc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Aug 2017 15:24:31 +0200 Subject: [PATCH 111/117] Remove some strange file --- et qq | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 et qq diff --git a/et qq b/et qq deleted file mode 100644 index 938b92d04373..000000000000 --- a/et qq +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/mypy/checker.py b/mypy/checker.py -index c65f956..8600902 100644 ---- a/mypy/checker.py -+++ b/mypy/checker.py -@@ -2350,7 +2350,7 @@ class TypeChecker(NodeVisitor[None]): - the name refers to a compatible generic type. - """ - info = self.lookup_typeinfo(name) -- # TODO: assert len(args) == len(info.defn.type_vars) -+ assert len(args) == len(info.defn.type_vars) - return Instance(info, args) -  - def lookup_typeinfo(self, fullname: str) -> TypeInfo: -diff --git a/mypy/semanal.py b/mypy/semanal.py -index e7db2c2..ecfda19 100644 ---- a/mypy/semanal.py -+++ b/mypy/semanal.py -@@ -1066,7 +1066,7 @@ class SemanticAnalyzer(NodeVisitor): - node = sym.node - assert isinstance(node, TypeInfo) - if args: -- # TODO: assert len(args) == len(node.defn.type_vars) -+ assert len(args) == len(node.defn.type_vars) - return Instance(node, args) - return Instance(node, [AnyType()] * len(node.defn.type_vars)) -  -@@ -1077,7 +1077,7 @@ class SemanticAnalyzer(NodeVisitor): - node = sym.node - assert isinstance(node, TypeInfo) - if args: -- # TODO: assert len(args) == len(node.defn.type_vars) -+ assert len(args) == len(node.defn.type_vars) - return Instance(node, args) - return Instance(node, [AnyType()] * len(node.defn.type_vars)) -  -@@ -3685,7 +3685,7 @@ class ThirdPass(TraverserVisitor): - node = sym.node - assert isinstance(node, TypeInfo) - if args: -- # TODO: assert len(args) == len(node.defn.type_vars) -+ assert len(args) == len(node.defn.type_vars) - return Instance(node, args) - return Instance(node, [AnyType()] * len(node.defn.type_vars)) -  From 713db0ca5dd51913eda10785049d8b840259f24a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Aug 2017 15:56:45 +0200 Subject: [PATCH 112/117] Implememt first part of recent comments --- docs/source/class_basics.rst | 64 ++++++++---------------------- test-data/unit/lib-stub/typing.pyi | 2 +- 2 files changed, 17 insertions(+), 49 deletions(-) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 011d6af4942a..e03c299e977c 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -170,21 +170,7 @@ to produce clear and concise error messages, and since Python provides native ``isinstance()`` checks based on class hierarchy. The *structural* subtyping however has its own advantages. In this system class ``D`` is a subtype of class ``C`` if the former has all attributes of the latter with -compatible types. For example: - -.. code-block:: python - - from typing import Sized - - def my_len(obj: Sized) -> int: - ... - - class MyCollection: - ... - def __len__(self) -> int: - return 42 - - my_len(MyCollection()) # OK, since 'MyCollection' is a subtype of 'Sized' +compatible types. This type system is a static equivalent of duck typing, well known by Python programmers. Mypy provides an opt-in support for structural subtyping via @@ -221,10 +207,11 @@ To define a protocol class, one must inherit the special .. note:: The ``Protocol`` base class is currently provided in ``typing_extensions`` - package. Stub files are however allowed to use - ``from typing import Protocol``. When structural subtyping is mature and + package. When structural subtyping is mature and `PEP 544 `_ is accepted, - ``Protocol`` will be included in the ``typing`` module. + ``Protocol`` will be included in the ``typing`` module. As well, several + types such as ``typing.Sized``, ``typing.Iterable`` etc. will be made + protocols. Defining subprotocols ********************* @@ -236,16 +223,20 @@ and merged using multiple inheritance. For example: # continuing from previous example - class SizedLabeledResource(SupportsClose, Sized, Protocol): + class SupportsRead(Protocol): + def read(self, amount: int) -> bytes: ... + + class TaggedReadableResource(SupportsClose, SupportsRead, Protocol): label: str class AdvancedResource(Resource): def __init__(self, label: str) -> None: self.label = label - def __len__(self) -> int: + def read(self, amount: int) -> bytes: + # some implementation ... - resource = None # type: SizedLabeledResource + resource = None # type: TaggedReadableResource # some code @@ -258,12 +249,12 @@ be explicitly present: .. code-block:: python - class NewProtocol(Sized): # This is NOT a protocol + class NewProtocol(SupportsClose): # This is NOT a protocol new_attr: int class Concrete: new_attr = None # type: int - def __len__(self) -> int: + def close(self) -> None: ... # Below is an error, since nominal subtyping is used by default x = Concrete() # type: NewProtocol # Error! @@ -297,34 +288,11 @@ declaring abstract recursive collections such as trees and linked lists: class SimpleTree: def __init__(self, value: int) -> None: self.value = value - self.left = self.right = None + self.left: Optional['SimpleTree'] = None + self.right: Optional['SimpleTree'] = None root = SimpleTree(0) # type: TreeLike # OK -Predefined protocols in ``typing`` module -***************************************** - -Most ABCs in ``typing`` module are protocol classes describing -common Python protocols such as ``Iterator``, ``Awaitable``, ``Mapping``, etc. -(see `Python Docs `_ -for an exhaustive list) -For example, the following class will be considered a subtype of -``typing.Sized`` and ``typing.Iterable[int]``: - -.. code-block:: python - - from typing import Iterator, Iterable - - class Bucket: - ... - def __len__(self) -> int: - return 22 - def __iter__(self) -> Iterator[int]: - yield 22 - - def collect(items: Iterable[int]) -> int: ... - result: int = collect(Bucket()) # Passes type check - Using ``isinstance()`` with protocols ************************************* diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index e97d75fc1c62..8be4abca389a 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -12,7 +12,7 @@ Union = 0 Optional = 0 TypeVar = 0 Generic = 0 -Protocol = 0 +Protocol = 0 # This is not yet defined in typeshed, see PR typeshed/#1220 Tuple = 0 Callable = 0 _promote = 0 From 40d635f19e1fcb6661396ba72c2265248a9833ea Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Aug 2017 17:59:23 +0200 Subject: [PATCH 113/117] Address second part of recent comments --- mypy/semanal.py | 5 +- test-data/unit/check-incremental.test | 19 +++- test-data/unit/check-newtype.test | 3 + test-data/unit/check-protocols.test | 151 +++++++++++++++----------- 4 files changed, 114 insertions(+), 64 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index fb63b96850a9..08cb3453fa4d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -746,7 +746,10 @@ def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None decorator.accept(self) if (isinstance(decorator, RefExpr) and decorator.fullname in ('typing.runtime', 'typing_extensions.runtime')): - defn.info.runtime_protocol = True + if defn.info.is_protocol: + defn.info.runtime_protocol = True + else: + self.fail('@runtime should be only used with protocol classes', defn) def calculate_abstract_status(self, typ: TypeInfo) -> None: """Calculate abstract status of a class. diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index f1b8cc484152..916a17752c7d 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1502,7 +1502,9 @@ class MyClass: [stale] [case testIncrementalWorksWithBasicProtocols] -from a import P +import a +[file a.py] +from b import P x: int y: P[int] @@ -1511,9 +1513,20 @@ x = y.meth() class C: def meth(self) -> int: pass +y = C() + +[file a.py.2] +from b import P + +x: str +y: P[str] +x = y.meth() +class C: + def meth(self) -> str: + pass y = C() -[file a.py] +[file b.py] from typing import Protocol, TypeVar T = TypeVar('T', covariant=True) @@ -1531,6 +1544,7 @@ class C(B): fun(C()) [file b.py] +from typing import Protocol class B: def x(self) -> float: pass def fun(arg: B) -> None: @@ -1570,6 +1584,7 @@ def fun(arg: B) -> None: arg.x() [file b.py.2] +from typing import Protocol class B: def x(self) -> float: pass def fun(arg: B) -> None: diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 30058e480904..c9e01edf3ef4 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -343,6 +343,9 @@ class D: C = NewType('C', P) # E: NewType cannot be used with protocol classes x: C = C(D()) # We still accept this, treating 'C' as non-protocol subclass. +reveal_type(x.attr) # E: Revealed type is 'builtins.int' +x.bad_attr # E: "C" has no attribute "bad_attr" +C(1) # E: Argument 1 to "C" has incompatible type "int"; expected "P" [out] [case testNewTypeAny] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 376eb08e6f09..655f32a4c317 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -9,7 +9,6 @@ class P(Protocol): pass P() # E: Cannot instantiate protocol class "P" -[out] [case testSimpleProtocolOneMethod] from typing import Protocol @@ -39,7 +38,6 @@ def fun2() -> P: return C() def fun3() -> P: return B() # E: Incompatible return value type (got "B", expected "P") -[out] [case testSimpleProtocolOneAbstractMethod] from typing import Protocol @@ -70,7 +68,20 @@ x = B() # E: Incompatible types in assignment (expression has type "B", variable fun(C()) fun(D()) fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "P" -[out] +fun(x) + +[case testProtocolMethodBodies] +from typing import Protocol, List + +class P(Protocol): + def meth(self) -> int: + return 'no way' # E: Incompatible return value type (got "str", expected "int") + +# explicit ellipsis is OK in protocol methods +class P2(Protocol): + def meth2(self) -> List[int]: + ... +[builtins fixtures/list.pyi] [case testSimpleProtocolOneMethodOverride] from typing import Protocol, Union @@ -98,7 +109,6 @@ x = B() # E: Incompatible types in assignment (expression has type "B", variable reveal_type(fun(C())) # E: Revealed type is 'builtins.str' fun(B()) # E: Argument 1 to "fun" has incompatible type "B"; expected "SubP" -[out] [case testSimpleProtocolTwoMethodsMerge] from typing import Protocol @@ -147,7 +157,6 @@ x = c1 # E: Incompatible types in assignment (expression has type "C1", variable x = c2 x = y y = x -[out] [case testSimpleProtocolTwoMethodsExtend] from typing import Protocol @@ -177,7 +186,6 @@ x = C() # OK x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", variable has type "P2") \ # N: 'Cbad' is missing following 'P2' protocol member: \ # N: meth2 -[out] [case testCannotAssignNormalToProtocol] from typing import Protocol @@ -192,7 +200,6 @@ class C: x: C y: P x = y # E: Incompatible types in assignment (expression has type "P", variable has type "C") -[out] [case testIndependentProtocolSubtyping] from typing import Protocol @@ -215,7 +222,6 @@ def f2(x: P2) -> None: pass f1(x2) f2(x1) -[out] [case testNoneDisablesProtocolImplementation] from typing import Protocol @@ -228,7 +234,6 @@ class C: __my_hash__ = None var: MyHashable = C() # E: Incompatible types in assignment (expression has type "C", variable has type "MyHashable") -[out] [case testNoneDisablesProtocolSubclassingWithStrictOptional] # flags: --strict-optional @@ -241,7 +246,26 @@ class MyHashable(Protocol): class C(MyHashable): __my_hash__ = None # E: Incompatible types in assignment \ (expression has type None, base class "MyHashable" defined the type as Callable[[MyHashable], int]) + +[case testProtocolsWithNoneAndStrictOptional] +# flags: --strict-optional +from typing import Protocol +class P(Protocol): + x = 0 # type: int + +class C: + x = None + +x: P = C() # Error! +def f(x: P) -> None: pass +f(C()) # Error! [out] +main:9: error: Incompatible types in assignment (expression has type "C", variable has type "P") +main:9: note: Following member(s) of "C" have conflicts: +main:9: note: x: expected "int", got None +main:11: error: Argument 1 to "f" has incompatible type "C"; expected "P" +main:11: note: Following member(s) of "C" have conflicts: +main:11: note: x: expected "int", got None -- Semanal errors in protocol types -- -------------------------------- @@ -270,7 +294,6 @@ class P4(Protocol[T]): class P5(Iterable[S], Protocol[T]): # E: If Generic[...] or Protocol[...] is present it should list all type variables def meth(self) -> T: pass -[out] [case testProhibitSelfDefinitionInProtocols] from typing import Protocol @@ -299,9 +322,8 @@ class B2(P2): a: int x2: P2 = B2() # OK -[out] -[case testProtocolsInTypingExtensions] +[case testProtocolAndRuntimeAreDefinedAlsoInTypingExtensions] from typing_extensions import Protocol, runtime @runtime @@ -320,7 +342,6 @@ class C: z: P = C() [builtins fixtures/dict.pyi] -[out] [case testProtocolsCannotInheritFromNormal] from typing import Protocol @@ -334,9 +355,9 @@ class P(C, Protocol): # E: All bases of a protocol must be protocols class P2(P, D, Protocol): # E: All bases of a protocol must be protocols pass +P2() # E: Cannot instantiate abstract class 'P2' with abstract attribute 'attr' p: P2 reveal_type(p.attr) # E: Revealed type is 'builtins.int' -[out] -- Generic protocol types -- ---------------------- @@ -373,7 +394,27 @@ x3: Pinv[B] y3: Pinv[A] y3 = x3 # E: Incompatible types in assignment (expression has type Pinv[B], variable has type Pinv[A]) x3 = y3 # E: Incompatible types in assignment (expression has type Pinv[A], variable has type Pinv[B]) -[out] + +[case testProtocolVarianceWithCallableAndList] +from typing import Protocol, TypeVar, Callable, List +T = TypeVar('T') +S = TypeVar('S') +T_co = TypeVar('T_co', covariant=True) + +class P(Protocol[T, S]): # E: Invariant type variable 'T' used in protocol where covariant one is expected \ + # E: Invariant type variable 'S' used in protocol where contravariant one is expected + def fun(self, callback: Callable[[T], S]) -> None: pass + +class P2(Protocol[T_co]): # E: Covariant type variable 'T_co' used in protocol where invariant one is expected + lst: List[T_co] +[builtins fixtures/list.pyi] + +[case testProtocolVarianceWithUnusedVariable] +from typing import Protocol, TypeVar +T = TypeVar('T') + +class P(Protocol[T]): # E: Invariant type variable 'T' used in protocol where covariant one is expected + attr: int [case testGenericProtocolsInference1] from typing import Protocol, Sequence, TypeVar @@ -393,7 +434,7 @@ def close(arg: Closeable[T]) -> T: def close_all(args: Sequence[Closeable[T]]) -> T: for arg in args: - arg.close() + arg.close() return args[0].close() arg: Closeable[int] @@ -403,7 +444,6 @@ reveal_type(close(arg)) # E: Revealed type is 'builtins.int*' reveal_type(close_all([F()])) # E: Revealed type is 'builtins.int*' reveal_type(close_all([arg])) # E: Revealed type is 'builtins.int*' [builtins fixtures/isinstancelist.pyi] -[out] [case testProtocolGenericInference2] from typing import Generic, TypeVar, Protocol @@ -421,7 +461,6 @@ class C: def fun3(x: P[T, T]) -> T: pass reveal_type(fun3(C())) # E: Revealed type is 'builtins.int*' -[out] [case testProtocolGenericInferenceCovariant] from typing import Generic, TypeVar, Protocol @@ -440,7 +479,34 @@ class C: def fun4(x: U, y: P[U, U]) -> U: pass reveal_type(fun4('a', C())) # E: Revealed type is 'builtins.object*' -[out] + +[case testUnrealtedGenericProtolsEquivalent] +from typing import TypeVar, Protocol +T = TypeVar('T') + +class PA(Protocol[T]): + attr: int + def meth(self) -> T: pass + def other(self, arg: T) -> None: pass +class PB(Protocol[T]): # exactly the same as above + attr: int + def meth(self) -> T: pass + def other(self, arg: T) -> None: pass + +def fun(x: PA[T]) -> PA[T]: + y: PB[T] = x + z: PB[T] + return z + +x: PA +y: PB +x = y +y = x + +xi: PA[int] +yi: PB[int] +xi = yi +yi = xi [case testGenericSubProtocols] from typing import TypeVar, Protocol, Tuple, Generic @@ -476,7 +542,6 @@ def f(x: T) -> T: # N: attr2 return x [builtins fixtures/isinstancelist.pyi] -[out] [case testGenericSubProtocolsExtensionInvariant] from typing import TypeVar, Protocol, Union @@ -507,7 +572,6 @@ var: P[Union[int, P], Union[P, str]] = C() # E: Incompatible types in assignment # N: Following member(s) of "C" have conflicts: \ # N: attr1: expected "Union[int, P[Any, Any]]", got "int" \ # N: attr2: expected "Union[P[Any, Any], str]", got "str" -[out] [case testGenericSubProtocolsExtensionCovariant] from typing import TypeVar, Protocol, Union @@ -602,7 +666,6 @@ x: Proto[int, float] y: Proto[float, int] y = x # OK [builtins fixtures/list.pyi] -[out] [case testSubtleBadVarianceInProtocols] from typing import Protocol, TypeVar, Iterable, Sequence @@ -622,7 +685,6 @@ x: Proto[int, float] y: Proto[float, int] y = x # OK [builtins fixtures/list.pyi] -[out] -- Recursive protocol types -- ------------------------ @@ -645,7 +707,6 @@ t: Traversable t = D[int]() # OK t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "Traversable") [builtins fixtures/list.pyi] -[out] [case testRecursiveProtocols2] from typing import Protocol, TypeVar @@ -664,7 +725,6 @@ def last(seq: Linked[T]) -> T: reveal_type(last(L())) # E: Revealed type is 'builtins.int*' [builtins fixtures/list.pyi] -[out] [case testMutuallyRecursiveProtocols] from typing import Protocol, Sequence, List @@ -690,7 +750,6 @@ t = A() # OK t = B() # E: Incompatible types in assignment (expression has type "B", variable has type "P1") t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1") [builtins fixtures/list.pyi] -[out] -- @property, @classmethod and @staticmethod in protocol types -- ----------------------------------------------------------- @@ -718,7 +777,6 @@ class C2(P): C() C2() -[out] [case testCannotInstantiateAbstractVariableExplicitProtocolSubtypes] from typing import Protocol @@ -745,7 +803,6 @@ class P2(Protocol): class B(P2): pass B() # OK, attr is not abstract -[out] [case testClassVarsInProtocols] from typing import Protocol, ClassVar @@ -771,7 +828,6 @@ x = CClass() # E: Incompatible types in assignment (expression has type "CClass" y = CClass() y = CInst() # E: Incompatible types in assignment (expression has type "CInst", variable has type "PClass") \ # N: Protocol member PClass.v expected class variable, got instance variable -[out] [case testPropertyInProtocols] from typing import Protocol @@ -793,7 +849,6 @@ y2: PP x2 = y2 # E: Incompatible types in assignment (expression has type "PP", variable has type "P") \ # N: Protocol member P.attr expected settable variable, got read-only attribute [builtins fixtures/property.pyi] -[out] [case testSettablePropertyInProtocols] from typing import Protocol @@ -831,7 +886,6 @@ z4: PPS z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variable has type "PPS") \ # N: Protocol member PPS.attr expected settable variable, got read-only attribute [builtins fixtures/property.pyi] -[out] [case testStaticAndClassMethodsInProtocols] from typing import Protocol, Type, TypeVar @@ -863,7 +917,6 @@ y = B() y = C() # E: Incompatible types in assignment (expression has type "C", variable has type "PC") \ # N: Protocol member PC.meth expected class or static method [builtins fixtures/classmethod.pyi] -[out] [case testOverloadedMethodsInProtocols] from typing import overload, Protocol, Union @@ -914,7 +967,6 @@ l = [x, y] reveal_type(l) # E: Revealed type is 'builtins.list[__main__.P*]' [builtins fixtures/list.pyi] -[out] [case testJoinProtocolWithNormal] from typing import Protocol @@ -932,7 +984,6 @@ l = [x, y] reveal_type(l) # E: Revealed type is 'builtins.list[__main__.P*]' [builtins fixtures/list.pyi] -[out] [case testMeetProtocolWithProtocol] from typing import Protocol, Callable, TypeVar @@ -947,7 +998,6 @@ T = TypeVar('T') def f(x: Callable[[T, T], None]) -> T: pass def g(x: P, y: P2) -> None: pass reveal_type(f(g)) # E: Revealed type is '__main__.P2*' -[out] [case testMeetProtocolWithNormal] from typing import Protocol, Callable, TypeVar @@ -961,7 +1011,6 @@ T = TypeVar('T') def f(x: Callable[[T, T], None]) -> T: pass def g(x: P, y: C) -> None: pass reveal_type(f(g)) # E: Revealed type is '__main__.C*' -[out] [case testInferProtocolFromProtocol] from typing import Protocol, Sequence, TypeVar, Generic @@ -982,7 +1031,6 @@ def last(seq: Linked[T]) -> T: reveal_type(last(L[int]())) # E: Revealed type is '__main__.Box*[builtins.int*]' reveal_type(last(L[str]()).content) # E: Revealed type is 'builtins.str*' -[out] [case testOverloadOnProtocol] from typing import overload, Protocol, runtime @@ -1015,7 +1063,6 @@ reveal_type(f(C2())) # E: Revealed type is 'builtins.str' f(C()) # E: No overload variant of "f" matches argument types [__main__.C] [builtins fixtures/isinstance.pyi] -[out] -- Unions of protocol types -- ------------------------ @@ -1043,7 +1090,6 @@ x = C1() x = C2() x = C() x = B() # E: Incompatible types in assignment (expression has type "B", variable has type "Union[P1, P2]") -[out] [case testUnionsOfNormalClassesWithProtocols] from typing import Protocol, Union @@ -1077,7 +1123,6 @@ f1(y) f1(z) f2(x) # E: Argument 1 to "f2" has incompatible type "Union[C1, C2]"; expected "P2" f2(z) # E: Argument 1 to "f2" has incompatible type "Union[C, D1]"; expected "P2" -[out] -- Type[] with protocol types -- -------------------------- @@ -1108,7 +1153,6 @@ x: Type[P1] xbad: Type[Pbad] f(x) # OK f(xbad) # E: Argument 1 to "f" has incompatible type Type[Pbad]; expected Type[P] -[out] [case testInstantiationProtocolInTypeForAliases] from typing import Type, Protocol @@ -1128,7 +1172,6 @@ Alias() # E: Cannot instantiate protocol class "P" GoodAlias() f(Alias) # E: Only concrete class can be given where 'Type[P]' is expected f(GoodAlias) -[out] [case testInstantiationProtocolInTypeForVariables] from typing import Type, Protocol @@ -1151,7 +1194,6 @@ var_old() var_old = P # E: Can only assign concrete classes to a variable of type 'Type[P]' var_old = B # OK var_old = C # OK -[out] [case testInstantiationProtocolInTypeForClassMethods] from typing import Type, Protocol @@ -1166,7 +1208,6 @@ class C(Protocol): cls() #OK for classmethods Logger.log(cls) #OK for classmethods [builtins fixtures/classmethod.pyi] -[out] -- isinsatnce() with @runtime protocols -- ------------------------------------ @@ -1174,6 +1215,10 @@ class C(Protocol): [case testSimpleRuntimeProtocolCheck] from typing import Protocol, runtime +@runtime # E: @runtime should be only used with protocol classes +class C: + pass + class P(Protocol): def meth(self) -> None: pass @@ -1192,7 +1237,6 @@ if isinstance(x, R): reveal_type(x) # E: Revealed type is '__main__.R' reveal_type(x.meth()) # E: Revealed type is 'builtins.int' [builtins fixtures/isinstance.pyi] -[out] [case testRuntimeIterableProtocolCheck] from typing import Iterable, List, Union @@ -1202,7 +1246,6 @@ x: Union[int, List[str]] if isinstance(x, Iterable): reveal_type(x) # E: Revealed type is 'builtins.list[builtins.str]' [builtins fixtures/isinstancelist.pyi] -[out] [case testConcreteClassesInProtocolsIsInstance] from typing import Protocol, runtime, TypeVar, Generic @@ -1262,7 +1305,6 @@ else: reveal_type(c2) # E: Revealed type is '__main__.C2' [builtins fixtures/isinstancelist.pyi] -[out] [case testConcreteClassesUnionInProtocolsIsInstance] from typing import Protocol, runtime, TypeVar, Generic, Union @@ -1296,7 +1338,6 @@ if isinstance(x, P2): else: reveal_type(x) # E: Revealed type is '__main__.C1[builtins.int]' [builtins fixtures/isinstancelist.pyi] -[out] -- Non-Instances and protocol types (Callable vs __call__ etc.) -- ------------------------------------------------------------ @@ -1317,7 +1358,6 @@ def f(x: MyProto[int]) -> None: f(t) # OK [builtins fixtures/isinstancelist.pyi] -[out] [case testBasicNamedTupleStructuralSubtyping] from typing import NamedTuple, TypeVar, Protocol @@ -1363,8 +1403,6 @@ reveal_type(fun3(z)) # E: Revealed type is 'builtins.object*' reveal_type(fun3(z3)) # E: Revealed type is 'builtins.int*' [builtins fixtures/list.pyi] -[out] - [case testBasicCallableStructuralSubtyping] from typing import Callable, Generic, TypeVar @@ -1388,7 +1426,6 @@ def apply_str(f: Callable[[str], int], x: str) -> int: apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected Callable[[str], int] \ # N: 'Add5.__call__' has type 'Callable[[int], int]' [builtins fixtures/isinstancelist.pyi] -[out] [case testStructuralSupportForPartial] from typing import Callable, TypeVar, Generic, Any @@ -1407,7 +1444,6 @@ def foo(f: Callable[[int], T]) -> T: reveal_type(foo(partial(inc, 'temp'))) # E: Revealed type is 'builtins.int*' [builtins fixtures/list.pyi] -[out] -- Standard protocol types (SupportsInt, Sized, etc.) -- -------------------------------------------------- @@ -1483,7 +1519,6 @@ a: Any f1(a) f2(a) f3(a) -[out] [case testErrorsForProtocolsInDifferentPlaces] from typing import Protocol @@ -1523,7 +1558,6 @@ f(C()) # E: Argument 1 to "f" has incompatible type "C"; expected "P" \ # N: attr2: expected "str", got "int" \ # N: Protocol member P.attr2 expected settable variable, got read-only attribute [builtins fixtures/list.pyi] -[out] [case testIterableProtocolOnClass] from typing import TypeVar, Iterator @@ -1537,7 +1571,6 @@ class B(A): pass reveal_type(list(b for b in B())) # E: Revealed type is 'builtins.list[__main__.B*]' reveal_type(list(B())) # E: Revealed type is 'builtins.list[__main__.B*]' [builtins fixtures/list.pyi] -[out] [case testIterableProtocolOnMetaclass] from typing import TypeVar, Iterator, Type @@ -1555,7 +1588,6 @@ class C(E): reveal_type(list(c for c in C)) # E: Revealed type is 'builtins.list[__main__.C*]' reveal_type(list(C)) # E: Revealed type is 'builtins.list[__main__.C*]' [builtins fixtures/list.pyi] -[out] [case testClassesGetattrWithProtocols] from typing import Protocol @@ -1592,7 +1624,6 @@ fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" # N: attr: expected "int", got "str" fun_p(C()) # OK [builtins fixtures/list.pyi] -[out] [case testIpmlicitTypesInProtocols] from typing import Protocol @@ -1612,7 +1643,6 @@ x = D() # E: Incompatible types in assignment (expression has type "D", variable # N: x: expected "int", got "str" x = C() # OK [builtins fixtures/list.pyi] -[out] [case testProtocolIncompatibilityWithGenericMethod] from typing import Protocol, TypeVar @@ -1746,7 +1776,6 @@ def f2(x: Sequence[str]) -> None: pass f1(N(1)) # E: Argument 1 to "f1" has incompatible type "N"; expected Iterable[str] f2(N(2)) # E: Argument 1 to "f2" has incompatible type "N"; expected Sequence[str] [builtins fixtures/tuple.pyi] -[out] [case testNotManyFlagConflitsShownInProtocols] from typing import Protocol @@ -1795,4 +1824,4 @@ class MockDict(MockMapping[T]): def f(x: MockMapping[int]) -> None: pass x: MockDict[str] f(x) # E: Argument 1 to "f" has incompatible type MockDict[str]; expected MockMapping[int] -[out] + From 19073e5d7bd0c8ad9f563e0d7ba606e10452d52e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 14 Aug 2017 23:45:58 +0200 Subject: [PATCH 114/117] First part of todays comments --- mypy/errors.py | 3 +- mypy/messages.py | 8 +- mypy/semanal.py | 19 +- test-data/unit/check-protocols.test | 296 +++++++++++++++++++++++++--- 4 files changed, 288 insertions(+), 38 deletions(-) diff --git a/mypy/errors.py b/mypy/errors.py index 82ed001d653f..eb6a17efaf3c 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -485,7 +485,8 @@ def remove_duplicates(self, errors: List[Tuple[Optional[str], int, int, str, str if (errors[j][3] == errors[i][3] and # Allow duplicate notes in overload conficts reporting not (errors[i][3] == 'note' and - errors[i][4].strip() in allowed_duplicates) and + errors[i][4].strip() in allowed_duplicates + or errors[i][4].strip().startswith('def ')) and errors[j][4] == errors[i][4]): # ignore column dup = True break diff --git a/mypy/messages.py b/mypy/messages.py index 5d5cfc63f86f..7210bda994b3 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1017,7 +1017,7 @@ def concrete_only_call(self, typ: Type, context: Context) -> None: def note_call(self, subtype: Type, call: Type, context: Context) -> None: self.note("'{}.__call__' has type '{}'".format(strip_quotes(self.format(subtype)), - self.format(call)), context) + self.format(call, verbosity=1)), context) def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDictType], supertype: Instance, context: Context) -> None: @@ -1038,6 +1038,10 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict Instance: []} # type: Dict[type, List[str]] if supertype.type.fullname() in exclusions[type(subtype)]: return + if any(isinstance(tp, UninhabitedType) for tp in supertype.args): + # We don't want to add notes for failed inference (e.g. Iterable[]). + # This will be only confusing a user even more. + return def pretty_overload(tp: Overloaded) -> None: for item in tp.items()[:MAX_ITEMS]: @@ -1079,7 +1083,7 @@ def print_more(conflicts: Sequence[Any]) -> None: 'conflicts:'.format(self.format(subtype)), context) for name, got, exp in conflict_types[:MAX_ITEMS]: if (not isinstance(exp, (CallableType, Overloaded)) or - not isinstance(exp, (CallableType, Overloaded))): + not isinstance(got, (CallableType, Overloaded))): self.note('{}: expected {}, got {}'.format(name, *self.format_distinctly(exp, got)), context, offset=OFFSET) diff --git a/mypy/semanal.py b/mypy/semanal.py index 08cb3453fa4d..e413c7fd5882 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -543,12 +543,17 @@ def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: if defn.impl is not None: assert defn.impl is defn.items[-1] defn.items = defn.items[:-1] - - elif (not self.is_stub_file and not non_overload_indexes and - not (self.type and self.type.is_protocol)): - self.fail( - "An overloaded function outside a stub file must have an implementation", - defn) + elif not self.is_stub_file and not non_overload_indexes: + if not (self.is_class_scope() and self.type.is_protocol): + self.fail( + "An overloaded function outside a stub file must have an implementation", + defn) + else: + for item in defn.items: + if isinstance(item, Decorator): + item.func.is_abstract = True + else: + item.is_abstract = True if types: defn.type = Overloaded(types) @@ -749,7 +754,7 @@ def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None if defn.info.is_protocol: defn.info.runtime_protocol = True else: - self.fail('@runtime should be only used with protocol classes', defn) + self.fail('@runtime can only be used with protocol classes', defn) def calculate_abstract_status(self, typ: TypeInfo) -> None: """Calculate abstract status of a class. diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 655f32a4c317..70e41f3a8c9e 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -187,6 +187,33 @@ x = Cbad() # E: Incompatible types in assignment (expression has type "Cbad", va # N: 'Cbad' is missing following 'P2' protocol member: \ # N: meth2 +[case testProtocolMethodVsAttributeErrors] +from typing import Protocol + +class P(Protocol): + def meth(self) -> int: + pass +class C: + meth: int +x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ + # N: Following member(s) of "C" have conflicts: \ + # N: meth: expected Callable[[], int], got "int" + +[case testProtocolMethodVsAttributeErrors2] +from typing import Protocol + +class P(Protocol): + @property + def meth(self) -> int: + pass +class C: + def meth(self) -> int: + pass +x: P = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P") \ + # N: Following member(s) of "C" have conflicts: \ + # N: meth: expected "int", got Callable[[], int] +[builtins fixtures/property.pyi] + [case testCannotAssignNormalToProtocol] from typing import Protocol @@ -362,6 +389,39 @@ reveal_type(p.attr) # E: Revealed type is 'builtins.int' -- Generic protocol types -- ---------------------- +[case testGenericMethodWithProtocol] +from typing import Protocol, TypeVar +T = TypeVar('T') + +class P(Protocol): + def meth(self, x: int) -> int: + return x +class C: + def meth(self, x: T) -> T: + return x + +x: P = C() + +[case testGenericMethodWithProtocol2] +from typing import Protocol, TypeVar +T = TypeVar('T') + +class P(Protocol): + def meth(self, x: T) -> T: + return x +class C: + def meth(self, x: int) -> int: + return x + +x: P = C() +[out] +main:11: error: Incompatible types in assignment (expression has type "C", variable has type "P") +main:11: note: Following member(s) of "C" have conflicts: +main:11: note: Expected: +main:11: note: def [T] meth(self, x: T) -> T +main:11: note: Got: +main:11: note: def meth(self, x: int) -> int + [case testAutomaticProtocolVariance] from typing import TypeVar, Protocol @@ -726,22 +786,34 @@ def last(seq: Linked[T]) -> T: reveal_type(last(L())) # E: Revealed type is 'builtins.int*' [builtins fixtures/list.pyi] +[case testRecursiveProtocolSubtleMismatch] +from typing import Protocol, TypeVar + +T = TypeVar('T') +class Linked(Protocol[T]): + val: T + def next(self) -> Linked[T]: pass +class L: + val: int + def next(self) -> int: pass + +def last(seq: Linked[T]) -> T: + pass +last(L()) # E: Argument 1 to "last" has incompatible type "L"; expected Linked[] + [case testMutuallyRecursiveProtocols] from typing import Protocol, Sequence, List class P1(Protocol): @property def attr1(self) -> Sequence[P2]: pass - class P2(Protocol): @property def attr2(self) -> Sequence[P1]: pass class C: pass - class A: attr1: List[B] - class B: attr2: List[A] @@ -751,6 +823,47 @@ t = B() # E: Incompatible types in assignment (expression has type "B", variable t = C() # E: Incompatible types in assignment (expression has type "C", variable has type "P1") [builtins fixtures/list.pyi] +[case testMutuallyRecursiveProtocolsTypesWithSubteMismatch] +from typing import Protocol, Sequence, List + +class P1(Protocol): + @property + def attr1(self) -> Sequence[P2]: pass +class P2(Protocol): + @property + def attr2(self) -> Sequence[P1]: pass + +class C: pass +class A: + attr1: List[B] +class B: + attr2: List[C] + +t: P1 +t = A() # E: Incompatible types in assignment (expression has type "A", variable has type "P1") \ + # N: Following member(s) of "A" have conflicts: \ + # N: attr1: expected Sequence[P2], got List[B] +[builtins fixtures/list.pyi] + +[case testMutuallyRecursiveProtocolsTypesWithSubteMismatchWriteable] +from typing import Protocol + +class P1(Protocol): + @property + def attr1(self) -> P2: pass +class P2(Protocol): + attr2: P1 + +class A: + attr1: B +class B: + attr2: A + +x: P1 = A() # E: Incompatible types in assignment (expression has type "A", variable has type "P1") \ + # N: Following member(s) of "A" have conflicts: \ + # N: attr1: expected "P2", got "B" +[builtins fixtures/property.pyi] + -- @property, @classmethod and @staticmethod in protocol types -- ----------------------------------------------------------- @@ -926,7 +1039,6 @@ class P(Protocol): def f(self, x: int) -> int: pass @overload def f(self, x: str) -> str: pass - def f(self, x): pass class C: def f(self, x: Union[int, str]) -> None: @@ -938,15 +1050,96 @@ class D: x: P = C() x = D() [out] -main:18: error: Incompatible types in assignment (expression has type "D", variable has type "P") -main:18: note: Following member(s) of "D" have conflicts: -main:18: note: Expected: -main:18: note: @overload -main:18: note: def f(self, x: int) -> int -main:18: note: @overload -main:18: note: def f(self, x: str) -> str -main:18: note: Got: -main:18: note: def f(self, x: int) -> None +main:17: error: Incompatible types in assignment (expression has type "D", variable has type "P") +main:17: note: Following member(s) of "D" have conflicts: +main:17: note: Expected: +main:17: note: @overload +main:17: note: def f(self, x: int) -> int +main:17: note: @overload +main:17: note: def f(self, x: str) -> str +main:17: note: Got: +main:17: note: def f(self, x: int) -> None + +[case testCannotInstantiateProtocolWithOverloadedUnimplementedMethod] +from typing import overload, Protocol + +class P(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass +class C(P): + pass +C() # E: Cannot instantiate abstract class 'C' with abstract attribute 'meth' + +[case testCanUseOverloadedImplementationsInProtocols] +from typing import overload, Protocol, Union +class P(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bool: pass + def meth(self, x: Union[int, str]): + if isinstance(x, int): + return x + return True + +class C(P): + pass +x = C() +reveal_type(x.meth('hi')) # E: Revealed type is 'builtins.bool' +[builtins fixtures/isinstance.pyi] + +[case testProtocolsWithIdenticalOverloads] +from typing import overload, Protocol + +class PA(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass +class PB(Protocol): # identical to above + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass + +x: PA +y: PB +x = y +def fun(arg: PB) -> None: pass +fun(x) + +[case testProtocolsWithIncompatibleOverloads] +from typing import overload, Protocol + +class PA(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: str) -> bytes: pass +class PB(Protocol): + @overload + def meth(self, x: int) -> int: pass + @overload + def meth(self, x: bytes) -> str: pass + +x: PA +y: PB +x = y +[out] +main:16: error: Incompatible types in assignment (expression has type "PB", variable has type "PA") +main:16: note: Following member(s) of "PB" have conflicts: +main:16: note: Expected: +main:16: note: @overload +main:16: note: def meth(self, x: int) -> int +main:16: note: @overload +main:16: note: def meth(self, x: str) -> bytes +main:16: note: Got: +main:16: note: @overload +main:16: note: def meth(self, x: int) -> int +main:16: note: @overload +main:16: note: def meth(self, x: bytes) -> str -- Join and meet with protocol types -- --------------------------------- @@ -963,11 +1156,27 @@ class P2(Protocol): x: P y: P2 +l0 = [x, x] +l1 = [y, y] l = [x, y] - +reveal_type(l0) # E: Revealed type is 'builtins.list[__main__.P*]' +reveal_type(l1) # E: Revealed type is 'builtins.list[__main__.P2*]' reveal_type(l) # E: Revealed type is 'builtins.list[__main__.P*]' [builtins fixtures/list.pyi] +[case testJoinOfIncompatibleProtocols] +from typing import Protocol + +class P(Protocol): + attr: int +class P2(Protocol): + attr2: str + +x: P +y: P2 +reveal_type([x, y]) # E: Revealed type is 'builtins.list[builtins.object*]' +[builtins fixtures/list.pyi] + [case testJoinProtocolWithNormal] from typing import Protocol @@ -999,6 +1208,19 @@ def f(x: Callable[[T, T], None]) -> T: pass def g(x: P, y: P2) -> None: pass reveal_type(f(g)) # E: Revealed type is '__main__.P2*' +[case testMeetOfIncompatibleProtocols] +from typing import Protocol, Callable, TypeVar + +class P(Protocol): + attr: int +class P2(Protocol): + attr2: str + +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: P, y: P2) -> None: pass +x = f(g) # E: "f" does not return a value + [case testMeetProtocolWithNormal] from typing import Protocol, Callable, TypeVar @@ -1059,9 +1281,10 @@ def f(x): reveal_type(f(C1())) # E: Revealed type is 'builtins.int' reveal_type(f(C2())) # E: Revealed type is 'builtins.str' - +class D(C1, C2): pass # Compatible with both P1 and P2 +# FIXME: the below is not right, see #1322 +reveal_type(f(D())) # E: Revealed type is 'Any' f(C()) # E: No overload variant of "f" matches argument types [__main__.C] - [builtins fixtures/isinstance.pyi] -- Unions of protocol types @@ -1209,13 +1432,13 @@ class C(Protocol): Logger.log(cls) #OK for classmethods [builtins fixtures/classmethod.pyi] --- isinsatnce() with @runtime protocols +-- isinstance() with @runtime protocols -- ------------------------------------ [case testSimpleRuntimeProtocolCheck] from typing import Protocol, runtime -@runtime # E: @runtime should be only used with protocol classes +@runtime # E: @runtime can only be used with protocol classes class C: pass @@ -1352,11 +1575,12 @@ class MyProto(Protocol[T]): pass t: Tuple[int, str] - def f(x: MyProto[int]) -> None: pass - f(t) # OK + +y: MyProto[str] +y = t # E: Incompatible types in assignment (expression has type "Tuple[int, str]", variable has type MyProto[str]) [builtins fixtures/isinstancelist.pyi] [case testBasicNamedTupleStructuralSubtyping] @@ -1424,7 +1648,26 @@ reveal_type(apply_gen(Add5())) # E: Revealed type is 'builtins.int*' def apply_str(f: Callable[[str], int], x: str) -> int: return f(x) apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected Callable[[str], int] \ - # N: 'Add5.__call__' has type 'Callable[[int], int]' + # N: 'Add5.__call__' has type 'Callable[[Arg(int, 'x')], int]' +[builtins fixtures/isinstancelist.pyi] + +[case testMoreComplexCallableStructuralSubtyping] +from mypy_extensions import Arg, VarArg +from typing import Protocol, Callable + +def call_soon(cb: Callable[[Arg(int, 'x'), VarArg(str)], int]): pass + +class Good: + def __call__(self, x: int, *rest: str) -> int: pass +class Bad1: + def __call__(self, x: int, *rest: int) -> int: pass +class Bad2: + def __call__(self, y: int, *rest: str) -> int: pass +call_soon(Good()) +call_soon(Bad1()) # E: Argument 1 to "call_soon" has incompatible type "Bad1"; expected Callable[[int, VarArg(str)], int] \ + # N: 'Bad1.__call__' has type 'Callable[[Arg(int, 'x'), VarArg(int)], int]' +call_soon(Bad2()) # E: Argument 1 to "call_soon" has incompatible type "Bad2"; expected Callable[[int, VarArg(str)], int] \ + # N: 'Bad2.__call__' has type 'Callable[[Arg(int, 'y'), VarArg(str)], int]' [builtins fixtures/isinstancelist.pyi] [case testStructuralSupportForPartial] @@ -1461,11 +1704,10 @@ def bar(a: Sized) -> int: return a.__len__() bar(Foo()) -bar(1) +bar((1, 2)) +bar(1) # E: Argument 1 to "bar" has incompatible type "int"; expected "Sized" [builtins fixtures/isinstancelist.pyi] -[out] -main:11: error: Argument 1 to "bar" has incompatible type "int"; expected "Sized" [case testBasicSupportsIntProtocol] from typing import SupportsInt @@ -1478,11 +1720,9 @@ def foo(a: SupportsInt): pass foo(Bar()) -foo('no way') +foo('no way') # E: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" [builtins fixtures/isinstancelist.pyi] -[out] -main:11: error: Argument 1 to "foo" has incompatible type "str"; expected "SupportsInt" -- Additional tests and corner cases for protocols -- ---------------------------------------------- @@ -1625,7 +1865,7 @@ fun_p(D()) # E: Argument 1 to "fun_p" has incompatible type "D"; expected "PP" fun_p(C()) # OK [builtins fixtures/list.pyi] -[case testIpmlicitTypesInProtocols] +[case testImplicitTypesInProtocols] from typing import Protocol class P(Protocol): From 50d98c0d8a668f4aa7b64fd5980331e2a9235fd9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Aug 2017 18:12:50 +0200 Subject: [PATCH 115/117] Second part of yesterdays comments --- mypy/constraints.py | 45 +++++++++++++++--------- mypy/join.py | 4 ++- mypy/messages.py | 50 ++++++++++++++------------- mypy/nodes.py | 35 ++++++++++++++++--- test-data/unit/check-protocols.test | 53 +++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 46 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index e3d1f6991404..7180dd00f25f 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -341,18 +341,14 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking # whether this type already appeared in the inference chain. + # This is a conservative way break the inference cycles. + # It never produces any "false" constraints but gives up soon + # on purely structural inference cycles, see #3829. not any(is_same_type(template, t) for t in template.type.inferring) and mypy.subtypes.is_subtype(instance, erase_typevars(template))): template.type.inferring.append(template) - for member in template.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance, original_actual) - temp = mypy.subtypes.find_member(member, template, original_actual) - assert inst is not None and temp is not None - res.extend(infer_constraints(temp, inst, self.direction)) - if (mypy.subtypes.IS_SETTABLE in - mypy.subtypes.get_member_flags(member, template.type)): - # Settable members are invariant, add opposite constraints - res.extend(infer_constraints(temp, inst, neg_op(self.direction))) + self.infer_constraints_from_protocol_members(res, instance, template, + original_actual, template) template.type.inferring.pop() return res elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and @@ -360,14 +356,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: not any(is_same_type(instance, i) for i in instance.type.inferring) and mypy.subtypes.is_subtype(erase_typevars(template), instance)): instance.type.inferring.append(instance) - for member in instance.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance, template) - temp = mypy.subtypes.find_member(member, template, template) - assert inst is not None and temp is not None - res.extend(infer_constraints(temp, inst, self.direction)) - if (mypy.subtypes.IS_SETTABLE in - mypy.subtypes.get_member_flags(member, instance.type)): - res.extend(infer_constraints(temp, inst, neg_op(self.direction))) + self.infer_constraints_from_protocol_members(res, instance, template, + template, instance) instance.type.inferring.pop() return res if isinstance(actual, AnyType): @@ -392,6 +382,27 @@ def visit_instance(self, template: Instance) -> List[Constraint]: else: return [] + def infer_constraints_from_protocol_members(self, res: List[Constraint], + instance: Instance, template: Instance, + subtype: Type, protocol: Instance) -> None: + """Infer constraints for situations where either 'template' or 'instance' is a protocol. + + The 'protocol' is the one of two that is an instance of protocol type, 'subtype' + is the type used to bind self during inference. Currently, we just infer constrains for + every protocol member type (both ways for settable members). + """ + for member in protocol.type.protocol_members: + inst = mypy.subtypes.find_member(member, instance, subtype) + temp = mypy.subtypes.find_member(member, template, subtype) + assert inst is not None and temp is not None + # The above is safe since at this point we know that 'instance' is a subtype + # of (erased) 'template', therefore it defines all protocol members + res.extend(infer_constraints(temp, inst, self.direction)) + if (mypy.subtypes.IS_SETTABLE in + mypy.subtypes.get_member_flags(member, protocol.type)): + # Settable members are invariant, add opposite constraints + res.extend(infer_constraints(temp, inst, neg_op(self.direction))) + def visit_callable_type(self, template: CallableType) -> List[Constraint]: if isinstance(self.actual, CallableType): cactual = self.actual diff --git a/mypy/join.py b/mypy/join.py index bd13b7f4b31b..e306a3f35895 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -146,7 +146,9 @@ def visit_instance(self, t: Instance) -> Type: structural = t elif self.s.type.is_protocol and is_protocol_implementation(t, self.s): structural = self.s - # structural type for join is preferred + # Structural join is preferred in the case where we have found both + # structural and nominal and they have same MRO length (see two comments + # in join_instances_via_supertype). Otherwise, just return the nominal join. if not structural or is_better(nominal, structural): return nominal return structural diff --git a/mypy/messages.py b/mypy/messages.py index 7210bda994b3..ad2ee0bfc18d 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1022,6 +1022,7 @@ def note_call(self, subtype: Type, call: Type, context: Context) -> None: def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDictType], supertype: Instance, context: Context) -> None: """Report possible protocol conflicts between 'subtype' and 'supertype'. + This includes missing members, incompatible types, and incompatible attribute flags, such as settable vs read-only or class variable vs instance variable. @@ -1043,20 +1044,6 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict # This will be only confusing a user even more. return - def pretty_overload(tp: Overloaded) -> None: - for item in tp.items()[:MAX_ITEMS]: - self.note('@overload', context, offset=2 * OFFSET) - self.note(self.pretty_callable(item), context, offset=2 * OFFSET) - if len(tp.items()) > MAX_ITEMS: - self.note('<{} more overload(s) not shown>'.format(len(tp.items()) - MAX_ITEMS), - context, offset=2 * OFFSET) - - def print_more(conflicts: Sequence[Any]) -> None: - if len(conflicts) > MAX_ITEMS: - self.note('<{} more conflict(s) not shown>' - .format(len(conflicts) - MAX_ITEMS), - context, offset=OFFSET) - if isinstance(subtype, (TupleType, TypedDictType)): if not isinstance(subtype.fallback, Instance): return @@ -1071,7 +1058,7 @@ def print_more(conflicts: Sequence[Any]) -> None: context) self.note(', '.join(missing), context, offset=OFFSET) elif len(missing) > MAX_ITEMS or len(missing) == len(supertype.type.protocol_members): - # this is an obviously wrong type: too many missing members + # This is an obviously wrong type: too many missing members return # Report member type conflicts @@ -1093,14 +1080,14 @@ def print_more(conflicts: Sequence[Any]) -> None: self.note(self.pretty_callable(exp), context, offset=2 * OFFSET) else: assert isinstance(exp, Overloaded) - pretty_overload(exp) + self.pretty_overload(exp, context, OFFSET, MAX_ITEMS) self.note('Got:', context, offset=OFFSET) if isinstance(got, CallableType): self.note(self.pretty_callable(got), context, offset=2 * OFFSET) else: assert isinstance(got, Overloaded) - pretty_overload(got) - print_more(conflict_types) + self.pretty_overload(got, context, OFFSET, MAX_ITEMS) + self.print_more(conflict_types, context, OFFSET, MAX_ITEMS) # Report flag conflicts (i.e. settable vs read-only etc.) conflict_flags = get_bad_flags(subtype, supertype) @@ -1117,7 +1104,23 @@ def print_more(conflicts: Sequence[Any]) -> None: if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: self.note('Protocol member {}.{} expected class or static method' .format(supertype.type.name(), name), context) - print_more(conflict_flags) + self.print_more(conflict_flags, context, OFFSET, MAX_ITEMS) + + def pretty_overload(self, tp: Overloaded, context: Context, + offset: int, max_items: int) -> None: + for item in tp.items()[:max_items]: + self.note('@overload', context, offset=2 * offset) + self.note(self.pretty_callable(item), context, offset=2 * offset) + if len(tp.items()) > max_items: + self.note('<{} more overload(s) not shown>'.format(len(tp.items()) - max_items), + context, offset=2 * offset) + + def print_more(self, conflicts: Sequence[Any], context: Context, + offset: int, max_items: int) -> None: + if len(conflicts) > max_items: + self.note('<{} more conflict(s) not shown>' + .format(len(conflicts) - max_items), + context, offset=offset) def pretty_callable(self, tp: CallableType) -> str: """Return a nice easily-readable representation of a callable type. @@ -1125,15 +1128,16 @@ def pretty_callable(self, tp: CallableType) -> str: def [T <: int] f(self, x: int, y: T) -> None """ s = '' - bare_asterisk = False + asterisk = False for i in range(len(tp.arg_types)): if s: s += ', ' - if tp.arg_kinds[i] in (ARG_NAMED, ARG_NAMED_OPT) and not bare_asterisk: + if tp.arg_kinds[i] in (ARG_NAMED, ARG_NAMED_OPT) and not asterisk: s += '*, ' - bare_asterisk = True + asterisk = True if tp.arg_kinds[i] == ARG_STAR: s += '*' + asterisk = True if tp.arg_kinds[i] == ARG_STAR2: s += '**' name = tp.arg_names[i] @@ -1141,7 +1145,7 @@ def [T <: int] f(self, x: int, y: T) -> None s += name + ': ' s += strip_quotes(self.format(tp.arg_types[i])) if tp.arg_kinds[i] in (ARG_OPT, ARG_NAMED_OPT): - s += ' =' + s += ' = ...' # If we got a "special arg" (i.e: self, cls, etc...), prepend it to the arg list if tp.definition is not None and tp.definition.name() is not None: diff --git a/mypy/nodes.py b/mypy/nodes.py index 83631145c7a1..d3067901f531 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1933,7 +1933,8 @@ class TempNode(Expression): """ type = None # type: mypy.types.Type - # Is it used to indicate no right hand side in assignment? + # Is this TempNode used to indicate absence of a right hand side in an annotated assignment? + # (e.g. for 'x: int' the rvalue is TempNode(AnyType(TypeOfAny.special_form), no_rhs=True)) no_rhs = False # type: bool def __init__(self, typ: 'mypy.types.Type', no_rhs: bool = False) -> None: @@ -1979,24 +1980,48 @@ class is generic then it will be a type constructor of higher kind. is_protocol = False # Is this a protocol class? runtime_protocol = False # Does this protocol support isinstance checks? abstract_attributes = None # type: List[str] + # Protocol members are names of all attributes/methods defined in a protocol + # and in all its supertypes (except for 'object'). protocol_members = None # type: List[str] - # These represent structural subtype matrices. Note that these are shared - # by all 'Instance's with given TypeInfo. + # The attributes 'assuming' and 'assuming_proper' represent structural subtype matrices. + # + # In languages with structural subtyping, one can keep a global subtype matrix like this: + # . A B C . + # A 1 0 0 + # B 1 1 1 + # C 1 0 1 + # . + # where 1 indicates that the type in corresponding row is a subtype of the type + # in corresponding column. This matrix typically starts filled with all 1's and + # a typechecker tries to "disprove" every subtyping relation using atomic (or nominal) types. + # However, we don't want to keep this huge global state. Instead, we keep the subtype + # information in the form of list of pairs (subtype, supertype) shared by all 'Instance's + # with given supertype's TypeInfo. When we enter a subtype check we push a pair in this list + # thus assuming that we started with 1 in corresponding matrix element. Such algorithm allows + # to treat recursive and mutually recursive protocols and other kinds of complex situations. + # # If concurrent/parallel type checking will be added in future, # then there should be one matrix per thread/process to avoid false negatives # during the type checking phase. assuming = None # type: List[Tuple[mypy.types.Instance, mypy.types.Instance]] assuming_proper = None # type: List[Tuple[mypy.types.Instance, mypy.types.Instance]] - # Ditto for temporary stack of recursive constraint inference. + # Ditto for temporary 'inferring' stack of recursive constraint inference. + # It contains Instance's of protocol types that appeared as an argument to + # constraints.infer_constraints(). We need 'inferring' to avoid infinite recursion for + # recursive and mutually recursive protocols. + # # We make 'assuming' and 'inferring' attributes here instead of passing they as kwargs, # since this would require to pass them in many dozens of calls. In particular, # there is a dependency infer_constraint -> is_subtype -> is_callable_subtype -> # -> infer_constraints. inferring = None # type: List[mypy.types.Instance] + # 'cache' and 'cache_proper' are subtype caches, implemented as sets of pairs + # of (subtype, supertype), where supertypes are instances of given TypeInfo. + # We need the caches, since subtype checks for structural types are very slow. cache = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] cache_proper = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] - # 'inferring' and 'assumig' can't be also made sets, since we need to use + # 'inferring' and 'assuming' can't be also made sets, since we need to use # is_same_type to correctly treat unions. # Classes inheriting from Enum shadow their true members with a __getattr__, so we diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 70e41f3a8c9e..ba190988be61 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -864,6 +864,22 @@ x: P1 = A() # E: Incompatible types in assignment (expression has type "A", vari # N: attr1: expected "P2", got "B" [builtins fixtures/property.pyi] +-- FIXME: things like this should work +[case testWeirdRecursiveInferenceForProtocols-skip] +from typing import Protocol, TypeVar, Generic +T_co = TypeVar('T_co', covariant=True) +T = TypeVar('T') + +class P(Protocol[T_co]): + def meth(self) -> P[T_co]: pass + +class C(Generic[T]): + def meth(self) -> C[T]: pass + +x: C[int] +def f(arg: P[T]) -> T: pass +reveal_type(f(x)) #E: Revealed type is 'builtins.int*' + -- @property, @classmethod and @staticmethod in protocol types -- ----------------------------------------------------------- @@ -1688,6 +1704,19 @@ def foo(f: Callable[[int], T]) -> T: reveal_type(foo(partial(inc, 'temp'))) # E: Revealed type is 'builtins.int*' [builtins fixtures/list.pyi] +[case testStructuralInferenceForCallable] +from typing import Callable, TypeVar, Tuple + +T = TypeVar('T') +S = TypeVar('S') + +class Actual: + def __call__(self, arg: int) -> str: pass + +def fun(cb: Callable[[T], S]) -> Tuple[T, S]: pass +reveal_type(fun(Actual())) # E: Revealed type is 'Tuple[builtins.int*, builtins.str*]' +[builtins fixtures/tuple.pyi] + -- Standard protocol types (SupportsInt, Sized, etc.) -- -------------------------------------------------- @@ -2065,3 +2094,27 @@ def f(x: MockMapping[int]) -> None: pass x: MockDict[str] f(x) # E: Argument 1 to "f" has incompatible type MockDict[str]; expected MockMapping[int] +[case testProtocolNotesForComplexSignatures] +from typing import Protocol, Optional + +class P(Protocol): + def meth(self, x: int, *args: str) -> None: pass + def other(self, *args, hint: Optional[str] = None, **kwargs: str) -> None: pass +class C: + def meth(self) -> int: pass + def other(self) -> int: pass + +x: P = C() +[builtins fixtures/dict.pyi] +[out] +main:10: error: Incompatible types in assignment (expression has type "C", variable has type "P") +main:10: note: Following member(s) of "C" have conflicts: +main:10: note: Expected: +main:10: note: def meth(self, x: int, *args: str) -> None +main:10: note: Got: +main:10: note: def meth(self) -> int +main:10: note: Expected: +main:10: note: def other(self, *args: Any, hint: Optional[str] = ..., **kwargs: str) -> None +main:10: note: Got: +main:10: note: def other(self) -> int + From 9411255dd9922d8a768f709fbe6794ff655e3af6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Aug 2017 20:09:45 +0200 Subject: [PATCH 116/117] Last part of review comments --- mypy/checkmember.py | 2 + mypy/messages.py | 72 +++++++++++++++++++++++--- mypy/nodes.py | 25 +++++++-- mypy/subtypes.py | 123 ++++++++++++++------------------------------ 4 files changed, 128 insertions(+), 94 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 93e210523c46..ceb9a4ff00ce 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -53,6 +53,8 @@ def analyze_member_access(name: str, the fallback type, for example. original_type is always the type used in the initial call. """ + # TODO: this and following functions share some logic with subtypes.find_member, + # consider refactoring. if isinstance(typ, Instance): if name == '__init__' and not is_super: # Accessing __init__ in statically typed code would compromise diff --git a/mypy/messages.py b/mypy/messages.py index ad2ee0bfc18d..21bf2bf85334 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -6,7 +6,7 @@ import re import difflib -from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple, Optional, Union +from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple, Set, Optional, Union from mypy.erasetype import erase_type from mypy.errors import Errors @@ -1027,8 +1027,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict attribute flags, such as settable vs read-only or class variable vs instance variable. """ - from mypy.subtypes import (get_missing_members, get_conflict_types, get_bad_flags, - is_subtype, IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC) + from mypy.subtypes import is_subtype, IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC OFFSET = 4 # Four spaces, so that notes will look like this: # note: 'Cls' is missing following 'Proto' members: # note: method, attr @@ -1050,7 +1049,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict subtype = subtype.fallback # Report missing members - missing = get_missing_members(subtype, supertype) + missing = get_missing_protocol_members(subtype, supertype) if (missing and len(missing) < len(supertype.type.protocol_members) and len(missing) <= MAX_ITEMS): self.note("'{}' is missing following '{}' protocol member{}:" @@ -1062,7 +1061,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict return # Report member type conflicts - conflict_types = get_conflict_types(subtype, supertype) + conflict_types = get_conflict_protocol_types(subtype, supertype) if conflict_types and (not is_subtype(subtype, erase_type(supertype)) or not subtype.type.defn.type_vars or not supertype.type.defn.type_vars): @@ -1090,7 +1089,7 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict self.print_more(conflict_types, context, OFFSET, MAX_ITEMS) # Report flag conflicts (i.e. settable vs read-only etc.) - conflict_flags = get_bad_flags(subtype, supertype) + conflict_flags = get_bad_protocol_flags(subtype, supertype) for name, subflags, superflags in conflict_flags[:MAX_ITEMS]: if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note('Protocol member {}.{} expected instance variable,' @@ -1186,6 +1185,67 @@ def variance_string(variance: int) -> str: return 'invariant' +def get_missing_protocol_members(left: Instance, right: Instance) -> List[str]: + """Find all protocol members of 'right' that are not implemented + (i.e. completely missing) in 'left'. + """ + from mypy.subtypes import find_member + assert right.type.is_protocol + missing = [] # type: List[str] + for member in right.type.protocol_members: + if not find_member(member, left, left): + missing.append(member) + return missing + + +def get_conflict_protocol_types(left: Instance, right: Instance) -> List[Tuple[str, Type, Type]]: + """Find members that are defined in 'left' but have incompatible types. + Return them as a list of ('member', 'got', 'expected'). + """ + from mypy.subtypes import find_member, is_subtype, get_member_flags, IS_SETTABLE + assert right.type.is_protocol + conflicts = [] # type: List[Tuple[str, Type, Type]] + for member in right.type.protocol_members: + if member in ('__init__', '__new__'): + continue + supertype = find_member(member, right, left) + assert supertype is not None + subtype = find_member(member, left, left) + if not subtype: + continue + is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) + if IS_SETTABLE in get_member_flags(member, right.type): + is_compat = is_compat and is_subtype(supertype, subtype) + if not is_compat: + conflicts.append((member, subtype, supertype)) + return conflicts + + +def get_bad_protocol_flags(left: Instance, right: Instance + ) -> List[Tuple[str, Set[int], Set[int]]]: + """Return all incompatible attribute flags for members that are present in both + 'left' and 'right'. + """ + from mypy.subtypes import (find_member, get_member_flags, + IS_SETTABLE, IS_CLASSVAR, IS_CLASS_OR_STATIC) + assert right.type.is_protocol + all_flags = [] # type: List[Tuple[str, Set[int], Set[int]]] + for member in right.type.protocol_members: + if find_member(member, left, left): + item = (member, + get_member_flags(member, left.type), + get_member_flags(member, right.type)) + all_flags.append(item) + bad_flags = [] + for name, subflags, superflags in all_flags: + if (IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags or + IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags or + IS_SETTABLE in superflags and IS_SETTABLE not in subflags or + IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags): + bad_flags.append((name, subflags, superflags)) + return bad_flags + + def capitalize(s: str) -> str: """Capitalize the first character of a string.""" if s == '': diff --git a/mypy/nodes.py b/mypy/nodes.py index d3067901f531..400339c083c1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2016,11 +2016,11 @@ class is generic then it will be a type constructor of higher kind. # there is a dependency infer_constraint -> is_subtype -> is_callable_subtype -> # -> infer_constraints. inferring = None # type: List[mypy.types.Instance] - # 'cache' and 'cache_proper' are subtype caches, implemented as sets of pairs + # '_cache' and '_cache_proper' are subtype caches, implemented as sets of pairs # of (subtype, supertype), where supertypes are instances of given TypeInfo. # We need the caches, since subtype checks for structural types are very slow. - cache = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] - cache_proper = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] + _cache = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] + _cache_proper = None # type: Set[Tuple[mypy.types.Type, mypy.types.Type]] # 'inferring' and 'assuming' can't be also made sets, since we need to use # is_same_type to correctly treat unions. @@ -2086,8 +2086,8 @@ def __init__(self, names: 'SymbolTable', defn: ClassDef, module_name: str) -> No self.assuming = [] self.assuming_proper = [] self.inferring = [] - self.cache = set() - self.cache_proper = set() + self._cache = set() + self._cache_proper = set() self.add_type_vars() def add_type_vars(self) -> None: @@ -2122,6 +2122,21 @@ def get_containing_type_info(self, name: str) -> 'Optional[TypeInfo]': return cls return None + def record_subtype_cache_entry(self, left: 'mypy.types.Instance', + right: 'mypy.types.Instance', + proper_subtype: bool = False) -> None: + if proper_subtype: + self._cache_proper.add((left, right)) + else: + self._cache.add((left, right)) + + def is_cached_subtype_check(self, left: 'mypy.types.Instance', + right: 'mypy.types.Instance', + proper_subtype: bool = False) -> bool: + if not proper_subtype: + return (left, right) in self._cache + return (left, right) in self._cache_proper + def __getitem__(self, name: str) -> 'SymbolTableNode': n = self.get(name) if n: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 10fa4cf1c711..f37d106c7250 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -143,16 +143,17 @@ def visit_instance(self, left: Instance) -> bool: if isinstance(right, TupleType) and right.fallback.type.is_enum: return is_subtype(left, right.fallback) if isinstance(right, Instance): - if (left, right) in right.type.cache: + if right.type.is_cached_subtype_check(left, right): return True # NOTE: left.type.mro may be None in quick mode if there # was an error somewhere. if left.type.mro is not None: for base in left.type.mro: + # TODO: Also pass recursively ignore_declared_variance if base._promote and is_subtype( base._promote, self.right, self.check_type_parameter, ignore_pos_arg_names=self.ignore_pos_arg_names): - right.type.cache.add((left, right)) + right.type.record_subtype_cache_entry(left, right) return True rname = right.type.fullname() # Always try a nominal check if possible, @@ -165,7 +166,7 @@ def visit_instance(self, left: Instance) -> bool: for lefta, righta, tvar in zip(t.args, right.args, right.type.defn.type_vars)) if nominal: - right.type.cache.add((left, right)) + right.type.record_subtype_cache_entry(left, right) return nominal if right.type.is_protocol and is_protocol_implementation(left, right): return True @@ -339,11 +340,14 @@ def pop_on_exit(stack: List[Tuple[Instance, Instance]], stack.pop() -def is_protocol_implementation(left: Instance, right: Instance, allow_any: bool = True) -> bool: - """Check whether 'left' implements the protocol 'right'. If 'allow_any' is False, then - check for a proper subtype. Treat recursive protocols by using the 'assuming' - structural subtype matrix (in sparse representation, i.e. as a list of pairs - (subtype, supertype)), see also comment in nodes.TypeInfo. When we enter a check for classes +def is_protocol_implementation(left: Instance, right: Instance, + proper_subtype: bool = False) -> bool: + """Check whether 'left' implements the protocol 'right'. + + If 'proper_subtype' is True, then check for a proper subtype. + Treat recursive protocols by using the 'assuming' structural subtype matrix + (in sparse representation, i.e. as a list of pairs (subtype, supertype)), + see also comment in nodes.TypeInfo. When we enter a check for classes (A, P), defined as following:: class P(Protocol): @@ -351,11 +355,12 @@ def f(self) -> P: ... class A: def f(self) -> A: ... - this results in A being a subtype of P without infinite recursion. On every false result, - we pop the assumption, thus avoiding an infinite recursion as well. + this results in A being a subtype of P without infinite recursion. + On every false result, we pop the assumption, thus avoiding an infinite recursion + as well. """ assert right.type.is_protocol - assuming = right.type.assuming if allow_any else right.type.assuming_proper + assuming = right.type.assuming_proper if proper_subtype else right.type.assuming for (l, r) in reversed(assuming): if sametypes.is_same_type(l, left) and sametypes.is_same_type(r, right): return True @@ -364,7 +369,7 @@ def f(self) -> A: ... # nominal subtyping currently ignores '__init__' and '__new__' signatures if member in ('__init__', '__new__'): continue - # The third argiment below indicates to what self type is bound. + # The third argument below indicates to what self type is bound. # We always bind self to the subtype. (Similarly to nominal types). supertype = find_member(member, right, left) assert supertype is not None @@ -374,8 +379,8 @@ def f(self) -> A: ... # print(member, 'of', right, 'has type', supertype) if not subtype: return False - if allow_any: - # nominal check currently ignores arg names + if not proper_subtype: + # Nominal check currently ignores arg names is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) else: is_compat = is_proper_subtype(subtype, supertype) @@ -395,20 +400,21 @@ def f(self) -> A: ... return False if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: return False - # this rule is copied from nominal check in checker.py + # This rule is copied from nominal check in checker.py if IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags: return False - if allow_any: - right.type.cache.add((left, right)) - else: - right.type.cache_proper.add((left, right)) + right.type.record_subtype_cache_entry(left, right, proper_subtype) return True def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: - """Find the type of member by 'name' in 'itype's TypeInfo. Apply type arguments - from 'itype', and bind 'self' to 'subtype'. Return None if member was not found. + """Find the type of member by 'name' in 'itype's TypeInfo. + + Fin the member type after applying type arguments from 'itype', and binding + 'self' to 'subtype'. Return None if member was not found. """ + # TODO: this code shares some logic with checkmember.analyze_member_access, + # consider refactoring. info = itype.type method = info.get_method(name) if method: @@ -448,6 +454,13 @@ def find_member(name: str, itype: Instance, subtype: Type) -> Optional[Type]: def get_member_flags(name: str, info: TypeInfo) -> Set[int]: """Detect whether a member 'name' is settable, whether it is an instance or class variable, and whether it is class or static method. + + The flags are defined as following: + * IS_SETTABLE: whether this attribute can be set, not set for methods and + non-settable properties; + * IS_CLASSVAR: set if the variable is annotated as 'x: ClassVar[t]'; + * IS_CLASS_OR_STATIC: set for methods decorated with @classmethod or + with @staticmethod. """ method = info.get_method(name) setattr_meth = info.get_method('__setattr__') @@ -504,64 +517,6 @@ def find_node_type(node: Union[Var, FuncBase], itype: Instance, subtype: Type) - return typ -def get_missing_members(left: Instance, right: Instance) -> List[str]: - """Find all protocol members of 'right' that are not implemented - (i.e. completely missing) in 'left'. This is a helper to collect information - for better error reporting. - """ - assert right.type.is_protocol - missing = [] # type: List[str] - for member in right.type.protocol_members: - if not find_member(member, left, left): - missing.append(member) - return missing - - -def get_conflict_types(left: Instance, right: Instance) -> List[Tuple[str, Type, Type]]: - """Find members that are defined in 'left' but have incompatible types. - Return them as a list of ('member', 'got', 'expected'). This is a helper - to collect information for better error reporting. - """ - assert right.type.is_protocol - conflicts = [] # type: List[Tuple[str, Type, Type]] - for member in right.type.protocol_members: - if member in ('__init__', '__new__'): - continue - supertype = find_member(member, right, left) - assert supertype is not None - subtype = find_member(member, left, left) - if not subtype: - continue - is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) - if IS_SETTABLE in get_member_flags(member, right.type): - is_compat = is_compat and is_subtype(supertype, subtype) - if not is_compat: - conflicts.append((member, subtype, supertype)) - return conflicts - - -def get_bad_flags(left: Instance, right: Instance) -> List[Tuple[str, Set[int], Set[int]]]: - """Return all incompatible attribute flags for members that are present in both - 'left' and 'right'. This is a helper to collect information for better error reporting. - """ - assert right.type.is_protocol - all_flags = [] # type: List[Tuple[str, Set[int], Set[int]]] - for member in right.type.protocol_members: - if find_member(member, left, left): - item = (member, - get_member_flags(member, left.type), - get_member_flags(member, right.type)) - all_flags.append(item) - bad_flags = [] - for name, subflags, superflags in all_flags: - if (IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags or - IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags or - IS_SETTABLE in superflags and IS_SETTABLE not in subflags or - IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags): - bad_flags.append((name, subflags, superflags)) - return bad_flags - - def is_callable_subtype(left: CallableType, right: CallableType, ignore_return: bool = False, ignore_pos_arg_names: bool = False, @@ -808,6 +763,8 @@ def restrict_subtype_away(t: Type, s: Type) -> Type: if isinstance(t, UnionType): # Since runtime type checks will ignore type arguments, erase the types. erased_s = erase_type(s) + # TODO: Implement more robust support for runtime isinstance() checks, + # see issue #3827 new_items = [item for item in t.relevant_items() if (not (is_proper_subtype(erase_type(item), erased_s) or is_proper_subtype(item, erased_s)) @@ -862,11 +819,11 @@ def visit_deleted_type(self, left: DeletedType) -> bool: def visit_instance(self, left: Instance) -> bool: right = self.right if isinstance(right, Instance): - if (left, right) in right.type.cache_proper: + if right.type.is_cached_subtype_check(left, right, proper_subtype=True): return True for base in left.type.mro: if base._promote and is_proper_subtype(base._promote, right): - right.type.cache_proper.add((left, right)) + right.type.record_subtype_cache_entry(left, right, proper_subtype=True) return True if left.type.has_base(right.type.fullname()): @@ -883,10 +840,10 @@ def check_argument(leftarg: Type, rightarg: Type, variance: int) -> bool: nominal = all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in zip(left.args, right.args, right.type.defn.type_vars)) if nominal: - right.type.cache_proper.add((left, right)) + right.type.record_subtype_cache_entry(left, right, proper_subtype=True) return nominal if (right.type.is_protocol and - is_protocol_implementation(left, right, allow_any=False)): + is_protocol_implementation(left, right, proper_subtype=True)): return True return False if isinstance(right, CallableType): From f1c915e80f215a3d7fc8b4f8ed93c58fc0e1f558 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 15 Aug 2017 20:24:20 +0200 Subject: [PATCH 117/117] One last missing comment --- mypy/subtypes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index f37d106c7250..5dfbc7c405bc 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -395,8 +395,7 @@ def f(self) -> A: ... # Check opposite direction for settable attributes. if not is_subtype(supertype, subtype): return False - if (IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags or - IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags): + if (IS_CLASSVAR in subflags) != (IS_CLASSVAR in superflags): return False if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: return False