From 7afe1309023b3b547f0abe0f58aff71a16ad1df3 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 21 Sep 2021 20:26:22 +0300 Subject: [PATCH 1/6] Adds support for `NamedTuple` subtyping, refs #11160 --- mypy/subtypes.py | 6 +- test-data/unit/check-namedtuple.test | 55 +++++++++++++++++++ test-data/unit/fixtures/typing-namedtuple.pyi | 17 ++++++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 test-data/unit/fixtures/typing-namedtuple.pyi diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 63cebc8aa483..3358803f2c80 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -263,7 +263,11 @@ def visit_instance(self, left: Instance) -> bool: rname = right.type.fullname # Always try a nominal check if possible, # there might be errors that a user wants to silence *once*. - if ((left.type.has_base(rname) or rname == 'builtins.object') and + # NamedTuples are a special case, because `NamedTuple` is not listed + # in `TypeInfo.mro`, so when `(a: NamedTuple) -> None` is used, + # we need to check for `is_named_tuple` property + if ((left.type.has_base(rname) or rname == 'builtins.object' + or (rname == 'typing.NamedTuple' and left.type.is_named_tuple)) and not self.ignore_declared_variance): # Map left type to corresponding right instances. t = map_instance_to_supertype(left, right.type) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index d47b069ea45e..55bb96ef1146 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -973,3 +973,58 @@ B = namedtuple('X', ['a']) # E: First argument to namedtuple() should be C = NamedTuple('X', [('a', 'Y')]) # E: First argument to namedtuple() should be "C", not "X" class Y: ... [builtins fixtures/tuple.pyi] + +[case testNamedTupleTypeIsASuperTypeOfOtherNamedTuples] +from typing import Tuple, NamedTuple + +class Bar(NamedTuple): + name: str = "Bar" + +class Baz(NamedTuple): + a: str + b: str + +def print_namedtuple(obj: NamedTuple) -> None: + pass + +b1: Bar +b2: Baz +print_namedtuple(b1) # ok +print_namedtuple(b2) # ok + +print_namedtuple(1) # E: Argument 1 to "print_namedtuple" has incompatible type "int"; expected "NamedTuple" +print_namedtuple(('bar',)) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[str]"; expected "NamedTuple" +print_namedtuple((1, 2)) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" +print_namedtuple((b1,)) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[Bar]"; expected "NamedTuple" +t: Tuple[str, ...] +print_namedtuple(t) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[str, ...]"; expected "NamedTuple" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleTypeIsASuperTypeOfOtherNamedTuplesReturns] +from typing import Tuple, NamedTuple + +class Bar(NamedTuple): + name: int + +class Baz(NamedTuple): + a: str + b: str + +def good1() -> NamedTuple: + b: Bar + return b +def good2() -> NamedTuple: + b: Baz + return b + +def bad1() -> NamedTuple: + return 1 # E: Incompatible return value type (got "int", expected "NamedTuple") +def bad2() -> NamedTuple: + return () # E: Incompatible return value type (got "Tuple[]", expected "NamedTuple") +def bad3() -> NamedTuple: + return (1, 2) # E: Incompatible return value type (got "Tuple[int, int]", expected "NamedTuple") + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi new file mode 100644 index 000000000000..ec0f888ddbf5 --- /dev/null +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -0,0 +1,17 @@ +TypeVar = 0 +Generic = 0 +Any = 0 +overload = 0 +Type = 0 + +T_co = TypeVar('T_co', covariant=True) + +class Iterable(Generic[T_co]): pass +class Iterator(Iterable[T_co]): pass +class Sequence(Iterable[T_co]): pass + +class Tuple(Sequence): pass +class NamedTuple(Tuple): pass + +class str: pass +class int: pass From c0976b4ef52c88747c30dd43dbb0bdb668516901 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 22 Sep 2021 13:57:15 +0300 Subject: [PATCH 2/6] Adds `._source` attribute to `NamedTuple` type --- test-data/unit/check-namedtuple.test | 2 +- test-data/unit/fixtures/typing-namedtuple.pyi | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 55bb96ef1146..8865c3cfc25c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -985,7 +985,7 @@ class Baz(NamedTuple): b: str def print_namedtuple(obj: NamedTuple) -> None: - pass + reveal_type(obj._source) # N: Revealed type is "builtins.str" b1: Bar b2: Baz diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index ec0f888ddbf5..438a439c62f8 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -11,7 +11,5 @@ class Iterator(Iterable[T_co]): pass class Sequence(Iterable[T_co]): pass class Tuple(Sequence): pass -class NamedTuple(Tuple): pass - -class str: pass -class int: pass +class NamedTuple(Tuple): + _source: str From 5fb7a881f048fd752433f1feacdeac8eb6d714b1 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 24 Sep 2021 17:09:15 +0300 Subject: [PATCH 3/6] Adds more tests and docs --- docs/source/kinds_of_types.rst | 20 ++++++++++++++++++++ test-data/unit/pythoneval.test | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 1efc2b30c328..1f2c1247652a 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -566,6 +566,26 @@ Python 3.6 introduced an alternative, class-based syntax for named tuples with t p = Point(x=1, y='x') # Argument has incompatible type "str"; expected "int" +.. note:: + + You can use raw ``NamedTuple`` pseudo-class to annotate type + where any ``NamedTuple`` is expected. + + For example, it can be useful for deserialization: + + .. code-block:: python + + def deserialize_named_tuple(arg: NamedTuple) -> Dict[str, Any]: + return arg._asdict() + + Point = namedtuple('Point', ['x', 'y']) + Person = NamedTuple('Person', [('name', str), ('age', int)]) + + deserialize_named_tuple(Point(x=1, y=2)) # ok + deserialize_named_tuple(Person(name='Nikita', age=18)) # ok + + deserialize_named_tuple((1, 2)) # Argument 1 to "deserialize_named_tuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" + .. _type-of-class: The type of class objects diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 8a7b39756867..5473b5e54b34 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1390,6 +1390,26 @@ x = X(a=1, b='s') [out] _testNamedTupleNew.py:12: note: Revealed type is "Tuple[builtins.int, fallback=_testNamedTupleNew.Child]" +[case testNamedTupleTypeInheritanceSpecialCase] +from typing import NamedTuple, Tuple +from collections import namedtuple + +A = NamedTuple('A', [('param', int)]) +B = namedtuple('B', ['param']) + +def accepts_named_tuple(arg: NamedTuple): + reveal_type(arg._asdict()) # N: Revealed type is "builtins.dict[builtins.str, Any]" + reveal_type(arg._fields) # N: Revealed type is "builtins.tuple[builtins.str]" + reveal_type(arg._field_defaults) # N: Revealed type is "builtins.dict[builtins.str, Any]" + +a = A(1) +b = B(1) + +accepts_named_tuple(a) +accepts_named_tuple(b) +accepts_named_tuple(1) # E: Argument 1 to "accepts_named_tuple" has incompatible type "int"; expected "NamedTuple" +accepts_named_tuple((1, 2)) # E: Argument 1 to "accepts_named_tuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" + [case testNewAnalyzerBasicTypeshed_newsemanal] from typing import Dict, List, Tuple From 11bd1aee48804d620ffac21c43b6359b7fd98838 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 24 Sep 2021 17:42:38 +0300 Subject: [PATCH 4/6] Fixes tests output location --- test-data/unit/pythoneval.test | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 5473b5e54b34..c15c898db46e 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1398,17 +1398,23 @@ A = NamedTuple('A', [('param', int)]) B = namedtuple('B', ['param']) def accepts_named_tuple(arg: NamedTuple): - reveal_type(arg._asdict()) # N: Revealed type is "builtins.dict[builtins.str, Any]" - reveal_type(arg._fields) # N: Revealed type is "builtins.tuple[builtins.str]" - reveal_type(arg._field_defaults) # N: Revealed type is "builtins.dict[builtins.str, Any]" + reveal_type(arg._asdict()) + reveal_type(arg._fields) + reveal_type(arg._field_defaults) a = A(1) b = B(1) accepts_named_tuple(a) accepts_named_tuple(b) -accepts_named_tuple(1) # E: Argument 1 to "accepts_named_tuple" has incompatible type "int"; expected "NamedTuple" -accepts_named_tuple((1, 2)) # E: Argument 1 to "accepts_named_tuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" +accepts_named_tuple(1) +accepts_named_tuple((1, 2)) +[out] +_testNamedTupleTypeInheritanceSpecialCase.py:8: note: Revealed type is "collections.OrderedDict[builtins.str, Any]" +_testNamedTupleTypeInheritanceSpecialCase.py:9: note: Revealed type is "builtins.tuple[builtins.str]" +_testNamedTupleTypeInheritanceSpecialCase.py:10: note: Revealed type is "builtins.dict[builtins.str, Any]" +_testNamedTupleTypeInheritanceSpecialCase.py:17: error: Argument 1 to "accepts_named_tuple" has incompatible type "int"; expected "NamedTuple" +_testNamedTupleTypeInheritanceSpecialCase.py:18: error: Argument 1 to "accepts_named_tuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" [case testNewAnalyzerBasicTypeshed_newsemanal] from typing import Dict, List, Tuple From 8c20b7cfdb4630e1d39e6e4890aab10fcf4f5fd3 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 4 Nov 2021 18:03:00 +0300 Subject: [PATCH 5/6] Improves tests and docs --- docs/source/kinds_of_types.rst | 3 ++ mypy/subtypes.py | 5 +-- test-data/unit/check-namedtuple.test | 36 +++++++++++++++++-- test-data/unit/fixtures/typing-namedtuple.pyi | 2 +- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 1f2c1247652a..f691bc899339 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -586,6 +586,9 @@ Python 3.6 introduced an alternative, class-based syntax for named tuples with t deserialize_named_tuple((1, 2)) # Argument 1 to "deserialize_named_tuple" has incompatible type "Tuple[int, int]"; expected "NamedTuple" + Note, that behavior is highly experimental, non-standard, + and can be not supported by other type checkers. + .. _type-of-class: The type of class objects diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 3358803f2c80..e10a7bba4e5f 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -267,8 +267,9 @@ def visit_instance(self, left: Instance) -> bool: # in `TypeInfo.mro`, so when `(a: NamedTuple) -> None` is used, # we need to check for `is_named_tuple` property if ((left.type.has_base(rname) or rname == 'builtins.object' - or (rname == 'typing.NamedTuple' and left.type.is_named_tuple)) and - not self.ignore_declared_variance): + or (rname == 'typing.NamedTuple' + and any(l.is_named_tuple for l in left.type.mro))) + and not self.ignore_declared_variance): # Map left type to corresponding right instances. t = map_instance_to_supertype(left, right.type) nominal = all(self.check_type_parameter(lefta, righta, tvar.variance) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 8865c3cfc25c..fdb0f841cff6 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -984,13 +984,27 @@ class Baz(NamedTuple): a: str b: str +class Biz(Baz): ... +class Other: ... +class Both1(Bar, Other): ... +class Both2(Other, Bar): ... +class Both3(Biz, Other): ... + def print_namedtuple(obj: NamedTuple) -> None: - reveal_type(obj._source) # N: Revealed type is "builtins.str" + reveal_type(obj.name) # N: Revealed type is "builtins.str" b1: Bar b2: Baz +b3: Biz +b4: Both1 +b5: Both2 +b6: Both3 print_namedtuple(b1) # ok print_namedtuple(b2) # ok +print_namedtuple(b3) # ok +print_namedtuple(b4) # ok +print_namedtuple(b5) # ok +print_namedtuple(b6) # ok print_namedtuple(1) # E: Argument 1 to "print_namedtuple" has incompatible type "int"; expected "NamedTuple" print_namedtuple(('bar',)) # E: Argument 1 to "print_namedtuple" has incompatible type "Tuple[str]"; expected "NamedTuple" @@ -1006,18 +1020,36 @@ print_namedtuple(t) # E: Argument 1 to "print_namedtuple" has incompatible type from typing import Tuple, NamedTuple class Bar(NamedTuple): - name: int + n: int class Baz(NamedTuple): a: str b: str +class Biz(Bar): ... +class Other: ... +class Both1(Bar, Other): ... +class Both2(Other, Bar): ... +class Both3(Biz, Other): ... + def good1() -> NamedTuple: b: Bar return b def good2() -> NamedTuple: b: Baz return b +def good3() -> NamedTuple: + b: Biz + return b +def good4() -> NamedTuple: + b: Both1 + return b +def good5() -> NamedTuple: + b: Both2 + return b +def good6() -> NamedTuple: + b: Both3 + return b def bad1() -> NamedTuple: return 1 # E: Incompatible return value type (got "int", expected "NamedTuple") diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index 438a439c62f8..a9052f661cb3 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -12,4 +12,4 @@ class Sequence(Iterable[T_co]): pass class Tuple(Sequence): pass class NamedTuple(Tuple): - _source: str + name: str From 137c28be56d303b10c364656b3b721d3e24b0c99 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 4 Nov 2021 18:31:35 +0300 Subject: [PATCH 6/6] Fixes tests, add `Mapping` to new fixture --- test-data/unit/fixtures/typing-namedtuple.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index a9052f661cb3..13b6c4cf9441 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -5,10 +5,12 @@ overload = 0 Type = 0 T_co = TypeVar('T_co', covariant=True) +KT = TypeVar('KT') class Iterable(Generic[T_co]): pass class Iterator(Iterable[T_co]): pass class Sequence(Iterable[T_co]): pass +class Mapping(Iterable[KT], Generic[KT, T_co]): pass class Tuple(Sequence): pass class NamedTuple(Tuple):